MCKL
Monte Carlo Kernel Library
avx2.hpp
Go to the documentation of this file.
1 //============================================================================
2 // MCKL/include/mckl/internal/avx2.hpp
3 //----------------------------------------------------------------------------
4 // MCKL: Monte Carlo Kernel Library
5 //----------------------------------------------------------------------------
6 // Copyright (c) 2013-2018, Yan Zhou
7 // All rights reserved.
8 //
9 // Redistribution and use in source and binary forms, with or without
10 // modification, are permitted provided that the following conditions are met:
11 //
12 // Redistributions of source code must retain the above copyright notice,
13 // this list of conditions and the following disclaimer.
14 //
15 // Redistributions in binary form must reproduce the above copyright notice,
16 // this list of conditions and the following disclaimer in the documentation
17 // and/or other materials provided with the distribution.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS AS IS
20 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
23 // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 // POSSIBILITY OF SUCH DAMAGE.
30 //============================================================================
31 
32 #ifndef MCKL_INTERNAL_AVX2_HPP
33 #define MCKL_INTERNAL_AVX2_HPP
34 
35 #include <mckl/internal/config.h>
36 #include <mckl/internal/avx2_op.hpp>
37 #include <mckl/internal/const_math.hpp>
38 #include <array>
39 
40 #if MCKL_HAS_FMA
41 #include <mckl/internal/fma.hpp>
42 #endif
43 
44 MCKL_PUSH_GCC_WARNING("-Wignored-attributes")
45 
46 namespace mckl {
47 
48 namespace internal {
49 
50 MCKL_INLINE inline void set_m128i(
51  const std::array<__m128i, 8> &s, std::array<__m256i, 4> &t)
52 {
53  std::get<0>(t) = _mm256_insertf128_si256(
54  _mm256_castsi128_si256(std::get<0>(s)), std::get<1>(s), 1);
55  std::get<1>(t) = _mm256_insertf128_si256(
56  _mm256_castsi128_si256(std::get<2>(s)), std::get<3>(s), 1);
57  std::get<2>(t) = _mm256_insertf128_si256(
58  _mm256_castsi128_si256(std::get<4>(s)), std::get<5>(s), 1);
59  std::get<3>(t) = _mm256_insertf128_si256(
60  _mm256_castsi128_si256(std::get<6>(s)), std::get<7>(s), 1);
61 }
62 
63 MCKL_INLINE inline void set_m128i(
64  const std::array<__m128i, 16> &s, std::array<__m256i, 8> &t)
65 {
66  std::get<0>(t) = _mm256_insertf128_si256(
67  _mm256_castsi128_si256(std::get<0x0>(s)), std::get<0x1>(s), 1);
68  std::get<1>(t) = _mm256_insertf128_si256(
69  _mm256_castsi128_si256(std::get<0x2>(s)), std::get<0x3>(s), 1);
70  std::get<2>(t) = _mm256_insertf128_si256(
71  _mm256_castsi128_si256(std::get<0x4>(s)), std::get<0x5>(s), 1);
72  std::get<3>(t) = _mm256_insertf128_si256(
73  _mm256_castsi128_si256(std::get<0x6>(s)), std::get<0x7>(s), 1);
74  std::get<4>(t) = _mm256_insertf128_si256(
75  _mm256_castsi128_si256(std::get<0x8>(s)), std::get<0x9>(s), 1);
76  std::get<5>(t) = _mm256_insertf128_si256(
77  _mm256_castsi128_si256(std::get<0xA>(s)), std::get<0xB>(s), 1);
78  std::get<6>(t) = _mm256_insertf128_si256(
79  _mm256_castsi128_si256(std::get<0xC>(s)), std::get<0xD>(s), 1);
80  std::get<7>(t) = _mm256_insertf128_si256(
81  _mm256_castsi128_si256(std::get<0xE>(s)), std::get<0xF>(s), 1);
82 }
83 
84 MCKL_INLINE inline void cvtepu32_epi64(
85  const std::array<__m128i, 4> &s, std::array<__m256i, 4> &t)
86 {
87  std::get<0>(t) = _mm256_cvtepu32_epi64(std::get<0>(s));
88  std::get<1>(t) = _mm256_cvtepu32_epi64(std::get<1>(s));
89  std::get<2>(t) = _mm256_cvtepu32_epi64(std::get<2>(s));
90  std::get<3>(t) = _mm256_cvtepu32_epi64(std::get<3>(s));
91 }
92 
93 MCKL_INLINE inline void cvtepu32_epi64(
94  const std::array<__m128i, 8> &s, std::array<__m256i, 8> &t)
95 {
96  std::get<0>(t) = _mm256_cvtepu32_epi64(std::get<0>(s));
97  std::get<1>(t) = _mm256_cvtepu32_epi64(std::get<1>(s));
98  std::get<2>(t) = _mm256_cvtepu32_epi64(std::get<2>(s));
99  std::get<3>(t) = _mm256_cvtepu32_epi64(std::get<3>(s));
100  std::get<4>(t) = _mm256_cvtepu32_epi64(std::get<4>(s));
101  std::get<5>(t) = _mm256_cvtepu32_epi64(std::get<5>(s));
102  std::get<6>(t) = _mm256_cvtepu32_epi64(std::get<6>(s));
103  std::get<7>(t) = _mm256_cvtepu32_epi64(std::get<7>(s));
104 }
105 
106 MCKL_INLINE inline void cvtepu32_epi64(
107  const std::array<__m128i, 16> &s, std::array<__m256i, 16> &t)
108 {
109  std::get<0x0>(t) = _mm256_cvtepu32_epi64(std::get<0x0>(s));
110  std::get<0x1>(t) = _mm256_cvtepu32_epi64(std::get<0x1>(s));
111  std::get<0x2>(t) = _mm256_cvtepu32_epi64(std::get<0x2>(s));
112  std::get<0x3>(t) = _mm256_cvtepu32_epi64(std::get<0x3>(s));
113  std::get<0x4>(t) = _mm256_cvtepu32_epi64(std::get<0x4>(s));
114  std::get<0x5>(t) = _mm256_cvtepu32_epi64(std::get<0x5>(s));
115  std::get<0x6>(t) = _mm256_cvtepu32_epi64(std::get<0x6>(s));
116  std::get<0x7>(t) = _mm256_cvtepu32_epi64(std::get<0x7>(s));
117  std::get<0x8>(t) = _mm256_cvtepu32_epi64(std::get<0x8>(s));
118  std::get<0x9>(t) = _mm256_cvtepu32_epi64(std::get<0x9>(s));
119  std::get<0xA>(t) = _mm256_cvtepu32_epi64(std::get<0xA>(s));
120  std::get<0xB>(t) = _mm256_cvtepu32_epi64(std::get<0xB>(s));
121  std::get<0xC>(t) = _mm256_cvtepu32_epi64(std::get<0xC>(s));
122  std::get<0xD>(t) = _mm256_cvtepu32_epi64(std::get<0xD>(s));
123  std::get<0xE>(t) = _mm256_cvtepu32_epi64(std::get<0xE>(s));
124  std::get<0xF>(t) = _mm256_cvtepu32_epi64(std::get<0xF>(s));
125 }
126 
127 MCKL_INLINE inline void cvtepu32_epi64(
128  const std::array<__m256i, 2> &s, std::array<__m256i, 4> &t)
129 {
130  const __m256i p = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
131  const __m256i m = _mm256_set1_epi64x(static_cast<MCKL_INT64>(0xFFFFFFFF));
132 
133  std::get<0>(t) = _mm256_permutevar8x32_epi32(std::get<0>(s), p);
134  std::get<2>(t) = _mm256_permutevar8x32_epi32(std::get<1>(s), p);
135 
136  std::get<1>(t) = _mm256_srli_epi64(std::get<0>(t), 32);
137  std::get<3>(t) = _mm256_srli_epi64(std::get<2>(t), 32);
138 
139  std::get<0>(t) = _mm256_and_si256(std::get<0>(t), m);
140  std::get<2>(t) = _mm256_and_si256(std::get<2>(t), m);
141 }
142 
143 MCKL_INLINE inline void cvtepu32_epi64(
144  const std::array<__m256i, 4> &s, std::array<__m256i, 8> &t)
145 {
146  const __m256i p = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
147  const __m256i m = _mm256_set1_epi64x(static_cast<MCKL_INT64>(0xFFFFFFFF));
148 
149  std::get<0>(t) = _mm256_permutevar8x32_epi32(std::get<0>(s), p);
150  std::get<2>(t) = _mm256_permutevar8x32_epi32(std::get<1>(s), p);
151  std::get<4>(t) = _mm256_permutevar8x32_epi32(std::get<2>(s), p);
152  std::get<6>(t) = _mm256_permutevar8x32_epi32(std::get<3>(s), p);
153 
154  std::get<1>(t) = _mm256_srli_epi64(std::get<0>(t), 32);
155  std::get<3>(t) = _mm256_srli_epi64(std::get<2>(t), 32);
156  std::get<5>(t) = _mm256_srli_epi64(std::get<4>(t), 32);
157  std::get<7>(t) = _mm256_srli_epi64(std::get<6>(t), 32);
158 
159  std::get<0>(t) = _mm256_and_si256(std::get<0>(t), m);
160  std::get<2>(t) = _mm256_and_si256(std::get<2>(t), m);
161  std::get<4>(t) = _mm256_and_si256(std::get<4>(t), m);
162  std::get<6>(t) = _mm256_and_si256(std::get<6>(t), m);
163 }
164 
165 MCKL_INLINE inline void cvtepu32_epi64(
166  const std::array<__m256i, 8> &s, std::array<__m256i, 16> &t)
167 {
168  const __m256i p = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
169  const __m256i m = _mm256_set1_epi64x(static_cast<MCKL_INT64>(0xFFFFFFFF));
170 
171  std::get<0x0>(t) = _mm256_permutevar8x32_epi32(std::get<0>(s), p);
172  std::get<0x2>(t) = _mm256_permutevar8x32_epi32(std::get<1>(s), p);
173  std::get<0x4>(t) = _mm256_permutevar8x32_epi32(std::get<2>(s), p);
174  std::get<0x6>(t) = _mm256_permutevar8x32_epi32(std::get<3>(s), p);
175  std::get<0x8>(t) = _mm256_permutevar8x32_epi32(std::get<4>(s), p);
176  std::get<0xA>(t) = _mm256_permutevar8x32_epi32(std::get<5>(s), p);
177  std::get<0xC>(t) = _mm256_permutevar8x32_epi32(std::get<6>(s), p);
178  std::get<0xE>(t) = _mm256_permutevar8x32_epi32(std::get<7>(s), p);
179 
180  std::get<0x1>(t) = _mm256_srli_epi64(std::get<0x0>(t), 32);
181  std::get<0x3>(t) = _mm256_srli_epi64(std::get<0x2>(t), 32);
182  std::get<0x5>(t) = _mm256_srli_epi64(std::get<0x4>(t), 32);
183  std::get<0x7>(t) = _mm256_srli_epi64(std::get<0x6>(t), 32);
184  std::get<0x9>(t) = _mm256_srli_epi64(std::get<0x8>(t), 32);
185  std::get<0xB>(t) = _mm256_srli_epi64(std::get<0xA>(t), 32);
186  std::get<0xD>(t) = _mm256_srli_epi64(std::get<0xC>(t), 32);
187  std::get<0xF>(t) = _mm256_srli_epi64(std::get<0xE>(t), 32);
188 
189  std::get<0x0>(t) = _mm256_and_si256(std::get<0x0>(t), m);
190  std::get<0x2>(t) = _mm256_and_si256(std::get<0x2>(t), m);
191  std::get<0x4>(t) = _mm256_and_si256(std::get<0x4>(t), m);
192  std::get<0x6>(t) = _mm256_and_si256(std::get<0x6>(t), m);
193  std::get<0x8>(t) = _mm256_and_si256(std::get<0x8>(t), m);
194  std::get<0xA>(t) = _mm256_and_si256(std::get<0xA>(t), m);
195  std::get<0xC>(t) = _mm256_and_si256(std::get<0xC>(t), m);
196  std::get<0xE>(t) = _mm256_and_si256(std::get<0xE>(t), m);
197 }
198 
199 MCKL_INLINE inline void cvtepi32_ps(std::array<__m256i, 4> &s)
200 {
201  std::get<0>(s) = _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<0>(s)));
202  std::get<1>(s) = _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<1>(s)));
203  std::get<2>(s) = _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<2>(s)));
204  std::get<3>(s) = _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<3>(s)));
205 }
206 
207 MCKL_INLINE inline void cvtepi32_ps(std::array<__m256i, 8> &s)
208 {
209  std::get<0>(s) = _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<0>(s)));
210  std::get<1>(s) = _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<1>(s)));
211  std::get<2>(s) = _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<2>(s)));
212  std::get<3>(s) = _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<3>(s)));
213  std::get<4>(s) = _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<4>(s)));
214  std::get<5>(s) = _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<5>(s)));
215  std::get<6>(s) = _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<6>(s)));
216  std::get<7>(s) = _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<7>(s)));
217 }
218 
219 MCKL_INLINE inline void cvtepi32_ps(std::array<__m256i, 16> &s)
220 {
221  std::get<0x0>(s) =
222  _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<0x0>(s)));
223  std::get<0x1>(s) =
224  _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<0x1>(s)));
225  std::get<0x2>(s) =
226  _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<0x2>(s)));
227  std::get<0x3>(s) =
228  _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<0x3>(s)));
229  std::get<0x4>(s) =
230  _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<0x4>(s)));
231  std::get<0x5>(s) =
232  _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<0x5>(s)));
233  std::get<0x6>(s) =
234  _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<0x6>(s)));
235  std::get<0x7>(s) =
236  _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<0x7>(s)));
237  std::get<0x8>(s) =
238  _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<0x8>(s)));
239  std::get<0x9>(s) =
240  _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<0x9>(s)));
241  std::get<0xA>(s) =
242  _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<0xA>(s)));
243  std::get<0xB>(s) =
244  _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<0xB>(s)));
245  std::get<0xC>(s) =
246  _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<0xC>(s)));
247  std::get<0xD>(s) =
248  _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<0xD>(s)));
249  std::get<0xE>(s) =
250  _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<0xE>(s)));
251  std::get<0xF>(s) =
252  _mm256_castps_si256(_mm256_cvtepi32_ps(std::get<0xF>(s)));
253 }
254 
255 template <std::size_t S>
256 MCKL_INLINE inline void cvtepi64_pd(std::array<__m256i, S> &s)
257 {
258  const __m256i m32 =
259  _mm256_castpd_si256(_mm256_set1_pd(Pow2<double, 32>::value));
260  const __m256i m52 =
261  _mm256_castpd_si256(_mm256_set1_pd(Pow2<double, 52>::value));
262  const __m256i mask =
263  _mm256_set1_epi64x(static_cast<MCKL_INT64>(0xFFFFFFFF));
264 
265  std::array<__m256i, S> t;
266  and_si256(s, mask, t);
267  srli_epi64<32>(s);
268  add_epi64(s, m52);
269  add_epi64(t, m52);
270  sub_pd(s, m52);
271  sub_pd(t, m52);
272 #if MCKL_USE_FMA
273  fmadd_pd(s, m32, t);
274 #else
275  mul_pd(s, m32);
276  add_pd(s, t);
277 #endif
278 }
279 
280 template <std::size_t I0, std::size_t I1, std::size_t I2, std::size_t I3,
281  std::size_t I4, std::size_t I5, std::size_t I6, std::size_t I7,
282  std::size_t N>
283 MCKL_INLINE inline void transpose8x32_si256(std::array<__m256i, N> &s)
284 {
285  __m256i t0 = _mm256_unpacklo_epi32(std::get<I0>(s), std::get<I1>(s));
286  __m256i t1 = _mm256_unpacklo_epi32(std::get<I2>(s), std::get<I3>(s));
287  __m256i t2 = _mm256_unpacklo_epi32(std::get<I4>(s), std::get<I5>(s));
288  __m256i t3 = _mm256_unpacklo_epi32(std::get<I6>(s), std::get<I7>(s));
289  __m256i t4 = _mm256_unpackhi_epi32(std::get<I0>(s), std::get<I1>(s));
290  __m256i t5 = _mm256_unpackhi_epi32(std::get<I2>(s), std::get<I3>(s));
291  __m256i t6 = _mm256_unpackhi_epi32(std::get<I4>(s), std::get<I5>(s));
292  __m256i t7 = _mm256_unpackhi_epi32(std::get<I6>(s), std::get<I7>(s));
293  __m256i t8 = _mm256_unpacklo_epi64(t0, t1);
294  __m256i t9 = _mm256_unpacklo_epi64(t2, t3);
295  __m256i tA = _mm256_unpacklo_epi64(t4, t5);
296  __m256i tB = _mm256_unpacklo_epi64(t6, t7);
297  __m256i tC = _mm256_unpackhi_epi64(t0, t1);
298  __m256i tD = _mm256_unpackhi_epi64(t2, t3);
299  __m256i tE = _mm256_unpackhi_epi64(t4, t5);
300  __m256i tF = _mm256_unpackhi_epi64(t6, t7);
301  std::get<I0>(s) = _mm256_permute2x128_si256(t8, t9, 0x20);
302  std::get<I1>(s) = _mm256_permute2x128_si256(tC, tD, 0x20);
303  std::get<I2>(s) = _mm256_permute2x128_si256(tA, tB, 0x20);
304  std::get<I3>(s) = _mm256_permute2x128_si256(tE, tF, 0x20);
305  std::get<I4>(s) = _mm256_permute2x128_si256(t8, t9, 0x31);
306  std::get<I5>(s) = _mm256_permute2x128_si256(tC, tD, 0x31);
307  std::get<I6>(s) = _mm256_permute2x128_si256(tA, tB, 0x31);
308  std::get<I7>(s) = _mm256_permute2x128_si256(tE, tF, 0x31);
309 }
310 
311 MCKL_INLINE inline void transpose8x32_load_si256(std::array<__m256i, 8> &s)
312 {
313  transpose8x32_si256<0, 1, 2, 3, 4, 5, 6, 7>(s);
314 }
315 
316 MCKL_INLINE inline void transpose8x32_store_si256(std::array<__m256i, 8> &s)
317 {
318  transpose8x32_si256<0, 1, 2, 3, 4, 5, 6, 7>(s);
319 }
320 
321 MCKL_INLINE inline void transpose8x32_load_si256(std::array<__m256i, 16> &s)
322 {
323  __m256i s1 = std::get<0x1>(s);
324  std::get<0x1>(s) = std::get<0x2>(s);
325  std::get<0x2>(s) = std::get<0x4>(s);
326  std::get<0x4>(s) = std::get<0x8>(s);
327  std::get<0x8>(s) = s1;
328 
329  __m256i s3 = std::get<0x3>(s);
330  std::get<0x3>(s) = std::get<0x6>(s);
331  std::get<0x6>(s) = std::get<0xC>(s);
332  std::get<0xC>(s) = std::get<0x9>(s);
333  std::get<0x9>(s) = s3;
334 
335  __m256i s7 = std::get<0x7>(s);
336  std::get<0x7>(s) = std::get<0xE>(s);
337  std::get<0xE>(s) = std::get<0xD>(s);
338  std::get<0xD>(s) = std::get<0xB>(s);
339  std::get<0xB>(s) = s7;
340 
341  __m256i s5 = std::get<0x5>(s);
342  std::get<0x5>(s) = std::get<0xA>(s);
343  std::get<0xA>(s) = s5;
344 
345  transpose8x32_si256<0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7>(s);
346  transpose8x32_si256<0x8, 0x9, 0xA, 0xB, 0xC, 0xD, 0xE, 0xF>(s);
347 }
348 
349 MCKL_INLINE inline void transpose8x32_store_si256(std::array<__m256i, 16> &s)
350 {
351  transpose8x32_si256<0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7>(s);
352  transpose8x32_si256<0x8, 0x9, 0xA, 0xB, 0xC, 0xD, 0xE, 0xF>(s);
353 
354  __m256i s2 = std::get<0x2>(s);
355  std::get<0x2>(s) = std::get<0x1>(s);
356  std::get<0x1>(s) = std::get<0x8>(s);
357  std::get<0x8>(s) = std::get<0x4>(s);
358  std::get<0x4>(s) = s2;
359 
360  __m256i s6 = std::get<0x6>(s);
361  std::get<0x6>(s) = std::get<0x3>(s);
362  std::get<0x3>(s) = std::get<0x9>(s);
363  std::get<0x9>(s) = std::get<0xC>(s);
364  std::get<0xC>(s) = s6;
365 
366  __m256i sE = std::get<0xE>(s);
367  std::get<0xE>(s) = std::get<0x7>(s);
368  std::get<0x7>(s) = std::get<0xB>(s);
369  std::get<0xB>(s) = std::get<0xD>(s);
370  std::get<0xD>(s) = sE;
371 
372  __m256i s5 = std::get<0x5>(s);
373  std::get<0x5>(s) = std::get<0xA>(s);
374  std::get<0xA>(s) = s5;
375 }
376 
377 template <std::size_t I0, std::size_t I1, std::size_t I2, std::size_t I3,
378  std::size_t N>
379 MCKL_INLINE inline void transpose4x64_si256(std::array<__m256i, N> &s)
380 {
381  __m256i t0 = _mm256_unpacklo_epi64(std::get<I0>(s), std::get<I1>(s));
382  __m256i t1 = _mm256_unpacklo_epi64(std::get<I2>(s), std::get<I3>(s));
383  __m256i t2 = _mm256_unpackhi_epi64(std::get<I0>(s), std::get<I1>(s));
384  __m256i t3 = _mm256_unpackhi_epi64(std::get<I2>(s), std::get<I3>(s));
385  std::get<I0>(s) = _mm256_permute2x128_si256(t0, t1, 0x20);
386  std::get<I1>(s) = _mm256_permute2x128_si256(t2, t3, 0x20);
387  std::get<I2>(s) = _mm256_permute2x128_si256(t0, t1, 0x31);
388  std::get<I3>(s) = _mm256_permute2x128_si256(t2, t3, 0x31);
389 }
390 
391 MCKL_INLINE inline void transpose4x64_load_si256(std::array<__m256i, 4> &s)
392 {
393  transpose4x64_si256<0, 1, 2, 3>(s);
394 }
395 
396 MCKL_INLINE inline void transpose4x64_store_si256(std::array<__m256i, 4> &s)
397 {
398  transpose4x64_si256<0, 1, 2, 3>(s);
399 }
400 
401 MCKL_INLINE inline void transpose4x64_load_si256(std::array<__m256i, 8> &s)
402 {
403  __m256i s1 = std::get<1>(s);
404  std::get<1>(s) = std::get<2>(s);
405  std::get<2>(s) = std::get<4>(s);
406  std::get<4>(s) = s1;
407 
408  __m256i s3 = std::get<3>(s);
409  std::get<3>(s) = std::get<6>(s);
410  std::get<6>(s) = std::get<5>(s);
411  std::get<5>(s) = s3;
412 
413  transpose4x64_si256<0, 1, 2, 3>(s);
414  transpose4x64_si256<4, 5, 6, 7>(s);
415 }
416 
417 MCKL_INLINE inline void transpose4x64_store_si256(std::array<__m256i, 8> &s)
418 {
419  transpose4x64_si256<0, 1, 2, 3>(s);
420  transpose4x64_si256<4, 5, 6, 7>(s);
421 
422  __m256i s2 = std::get<2>(s);
423  std::get<2>(s) = std::get<1>(s);
424  std::get<1>(s) = std::get<4>(s);
425  std::get<4>(s) = s2;
426 
427  __m256i s6 = std::get<6>(s);
428  std::get<6>(s) = std::get<3>(s);
429  std::get<3>(s) = std::get<5>(s);
430  std::get<5>(s) = s6;
431 }
432 
433 MCKL_INLINE inline void transpose4x64_load_si256(std::array<__m256i, 16> &s)
434 {
435  __m256i s1 = std::get<0x1>(s);
436  __m256i s2 = std::get<0x2>(s);
437  __m256i s3 = std::get<0x3>(s);
438  __m256i s6 = std::get<0x6>(s);
439  __m256i s7 = std::get<0x7>(s);
440  __m256i sB = std::get<0xB>(s);
441  std::get<0x1>(s) = std::get<0x4>(s);
442  std::get<0x2>(s) = std::get<0x8>(s);
443  std::get<0x3>(s) = std::get<0xC>(s);
444  std::get<0x6>(s) = std::get<0x9>(s);
445  std::get<0x7>(s) = std::get<0xD>(s);
446  std::get<0xB>(s) = std::get<0xE>(s);
447  std::get<0x4>(s) = s1;
448  std::get<0x8>(s) = s2;
449  std::get<0xC>(s) = s3;
450  std::get<0x9>(s) = s6;
451  std::get<0xD>(s) = s7;
452  std::get<0xE>(s) = sB;
453 
454  transpose4x64_si256<0x0, 0x1, 0x2, 0x3>(s);
455  transpose4x64_si256<0x4, 0x5, 0x6, 0x7>(s);
456  transpose4x64_si256<0x8, 0x9, 0xA, 0xB>(s);
457  transpose4x64_si256<0xC, 0xD, 0xE, 0xF>(s);
458 }
459 
460 MCKL_INLINE inline void transpose4x64_store_si256(std::array<__m256i, 16> &s)
461 {
462  transpose4x64_si256<0x0, 0x1, 0x2, 0x3>(s);
463  transpose4x64_si256<0x4, 0x5, 0x6, 0x7>(s);
464  transpose4x64_si256<0x8, 0x9, 0xA, 0xB>(s);
465  transpose4x64_si256<0xC, 0xD, 0xE, 0xF>(s);
466 
467  __m256i s1 = std::get<0x1>(s);
468  __m256i s2 = std::get<0x2>(s);
469  __m256i s3 = std::get<0x3>(s);
470  __m256i s6 = std::get<0x6>(s);
471  __m256i s7 = std::get<0x7>(s);
472  __m256i sB = std::get<0xB>(s);
473  std::get<0x1>(s) = std::get<0x4>(s);
474  std::get<0x2>(s) = std::get<0x8>(s);
475  std::get<0x3>(s) = std::get<0xC>(s);
476  std::get<0x6>(s) = std::get<0x9>(s);
477  std::get<0x7>(s) = std::get<0xD>(s);
478  std::get<0xB>(s) = std::get<0xE>(s);
479  std::get<0x4>(s) = s1;
480  std::get<0x8>(s) = s2;
481  std::get<0xC>(s) = s3;
482  std::get<0x9>(s) = s6;
483  std::get<0xD>(s) = s7;
484  std::get<0xE>(s) = sB;
485 }
486 
487 } // namespace internal
488 
489 } // namespace mckl
490 
491 MCKL_POP_GCC_WARNING
492 
493 #endif // MCKL_INTERNAL_AVX2_HPP
MCKL_PUSH_GCC_WARNING
#define MCKL_PUSH_GCC_WARNING(warning)
Definition: compiler.h:78
const_math.hpp
avx2_op.hpp
mckl
Definition: mcmc.hpp:40
MCKL_INLINE
#define MCKL_INLINE
Definition: clang.h:147
MCKL_POP_GCC_WARNING
#define MCKL_POP_GCC_WARNING
Definition: compiler.h:79
Generated by   doxygen 1.8.13