MCKL
Monte Carlo Kernel Library
threefry_avx2_8x64.hpp
Go to the documentation of this file.
1 //============================================================================
2 // MCKL/include/mckl/random/internal/threefry_avx2_8x64.hpp
3 //----------------------------------------------------------------------------
4 // MCKL: Monte Carlo Kernel Library
5 //----------------------------------------------------------------------------
6 // Copyright (c) 2013-2018, Yan Zhou
7 // All rights reserved.
8 //
9 // Redistribution and use in source and binary forms, with or without
10 // modification, are permitted provided that the following conditions are met:
11 //
12 // Redistributions of source code must retain the above copyright notice,
13 // this list of conditions and the following disclaimer.
14 //
15 // Redistributions in binary form must reproduce the above copyright notice,
16 // this list of conditions and the following disclaimer in the documentation
17 // and/or other materials provided with the distribution.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS AS IS
20 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
23 // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 // POSSIBILITY OF SUCH DAMAGE.
30 //============================================================================
31 
32 #ifndef MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_HPP
33 #define MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_HPP
34 
35 #include <mckl/random/internal/common.hpp>
36 #include <mckl/random/internal/threefry_common.hpp>
37 #include <mckl/random/internal/threefry_constants.hpp>
38 #include <mckl/random/internal/threefry_generic_8x64.hpp>
39 #include <mckl/random/increment.hpp>
40 
41 MCKL_PUSH_GCC_WARNING("-Wignored-attributes")
42 
43 #define MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_KBOX(N) \
44  ymmt0 = _mm256_set1_epi64x(static_cast<MCKL_INT64>( \
45  ThreefryKBox<T, K, N>::template key<0>(par))); \
46  ymmt1 = _mm256_set1_epi64x(static_cast<MCKL_INT64>( \
47  ThreefryKBox<T, K, N>::template key<1>(par))); \
48  ymmt2 = _mm256_set1_epi64x(static_cast<MCKL_INT64>( \
49  ThreefryKBox<T, K, N>::template key<2>(par))); \
50  ymmt3 = _mm256_set1_epi64x(static_cast<MCKL_INT64>( \
51  ThreefryKBox<T, K, N>::template key<3>(par))); \
52  ymmt4 = _mm256_set1_epi64x(static_cast<MCKL_INT64>( \
53  ThreefryKBox<T, K, N>::template key<4>(par))); \
54  ymmt5 = _mm256_set1_epi64x(static_cast<MCKL_INT64>( \
55  ThreefryKBox<T, K, N>::template key<5>(par))); \
56  ymmt6 = _mm256_set1_epi64x(static_cast<MCKL_INT64>( \
57  ThreefryKBox<T, K, N>::template key<6>(par))); \
58  ymmt7 = _mm256_set1_epi64x(static_cast<MCKL_INT64>( \
59  ThreefryKBox<T, K, N>::template key<7>(par))); \
60  ymms0 = _mm256_add_epi64(ymms0, ymmt0); \
61  ymms1 = _mm256_add_epi64(ymms1, ymmt1); \
62  ymms2 = _mm256_add_epi64(ymms2, ymmt2); \
63  ymms3 = _mm256_add_epi64(ymms3, ymmt3); \
64  ymms4 = _mm256_add_epi64(ymms4, ymmt4); \
65  ymms5 = _mm256_add_epi64(ymms5, ymmt5); \
66  ymms6 = _mm256_add_epi64(ymms6, ymmt6); \
67  ymms7 = _mm256_add_epi64(ymms7, ymmt7);
68 
69 #define MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_RBOX(N) \
70  { \
71  constexpr int L0 = Constants::rotate::value[0][(N - 1) % 8]; \
72  constexpr int L1 = Constants::rotate::value[1][(N - 1) % 8]; \
73  constexpr int L2 = Constants::rotate::value[2][(N - 1) % 8]; \
74  constexpr int L3 = Constants::rotate::value[3][(N - 1) % 8]; \
75  constexpr int R0 = 64 - L0; \
76  constexpr int R1 = 64 - L1; \
77  constexpr int R2 = 64 - L2; \
78  constexpr int R3 = 64 - L3; \
79  ymmt0 = _mm256_add_epi64(ymms0, ymms1); \
80  ymmt2 = _mm256_add_epi64(ymms2, ymms3); \
81  ymmt4 = _mm256_add_epi64(ymms4, ymms5); \
82  ymmt6 = _mm256_add_epi64(ymms6, ymms7); \
83  ymmt1 = _mm256_slli_epi64(ymms1, L0); \
84  ymmt3 = _mm256_slli_epi64(ymms3, L1); \
85  ymmt5 = _mm256_slli_epi64(ymms5, L2); \
86  ymmt7 = _mm256_slli_epi64(ymms7, L3); \
87  ymms1 = _mm256_srli_epi64(ymms1, R0); \
88  ymms3 = _mm256_srli_epi64(ymms3, R1); \
89  ymms5 = _mm256_srli_epi64(ymms5, R2); \
90  ymms7 = _mm256_srli_epi64(ymms7, R3); \
91  ymmt1 = _mm256_or_si256(ymms1, ymmt1); \
92  ymmt3 = _mm256_or_si256(ymms3, ymmt3); \
93  ymmt5 = _mm256_or_si256(ymms5, ymmt5); \
94  ymmt7 = _mm256_or_si256(ymms7, ymmt7); \
95  ymms0 = ymmt2; \
96  ymms2 = ymmt4; \
97  ymms4 = ymmt6; \
98  ymms6 = ymmt0; \
99  ymms1 = _mm256_xor_si256(ymmt0, ymmt1); \
100  ymms3 = _mm256_xor_si256(ymmt6, ymmt7); \
101  ymms5 = _mm256_xor_si256(ymmt4, ymmt5); \
102  ymms7 = _mm256_xor_si256(ymmt2, ymmt3); \
103  }
104 
105 #define MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_CYCLE_4(N) \
106  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_RBOX(N * 8 + 1); \
107  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_RBOX(N * 8 + 2); \
108  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_RBOX(N * 8 + 3); \
109  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_RBOX(N * 8 + 4); \
110  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_KBOX(N * 8 + 4);
111 
112 #define MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_CYCLE_8(N) \
113  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_RBOX(N * 8 + 1); \
114  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_RBOX(N * 8 + 2); \
115  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_RBOX(N * 8 + 3); \
116  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_RBOX(N * 8 + 4); \
117  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_KBOX(N * 8 + 4); \
118  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_RBOX(N * 8 + 5); \
119  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_RBOX(N * 8 + 6); \
120  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_RBOX(N * 8 + 7); \
121  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_RBOX(N * 8 + 8); \
122  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_KBOX(N * 8 + 8);
123 
124 namespace mckl {
125 
126 namespace internal {
127 
128 template <typename T>
129 class Threefry8x64GeneratorAVX2Impl
130 {
131  static_assert(std::numeric_limits<T>::digits == 64,
132  "**Threefry8x64GeneratorAVX2Impl** used with T other than a 64-bit "
133  "unsigned integers");
134 
135  static constexpr std::size_t K = 8;
136  static constexpr std::size_t Rounds = 20;
137  using Constants = ThreefryConstants<T, 8>;
138 
139  public:
140  static void eval(
141  const void *plain, void *cipher, const std::array<T, K + 4> &par)
142  {
143  Threefry8x64GeneratorGenericImpl<T>::eval(plain, cipher, par);
144  }
145 
146  template <typename ResultType>
147  static void eval(std::array<std::uint64_t, 8> &ctr, ResultType *r,
148  const std::array<T, K + 4> &par)
149  {
150  Threefry8x64GeneratorGenericImpl<T>::eval(ctr, r, par);
151  }
152 
153  template <typename ResultType>
154  static void eval(std::array<std::uint64_t, 8> &ctr, std::size_t n,
155  ResultType *r, const std::array<T, K + 4> &par)
156  {
157  constexpr std::size_t R = sizeof(T) * K / sizeof(ResultType);
158 
159  const std::size_t n0 =
160  static_cast<std::size_t>(std::min(static_cast<std::uint64_t>(n),
161  std::numeric_limits<std::uint64_t>::max() - ctr.front()));
162 
163  eval_kernel(ctr, n0, r, par);
164  n -= n0;
165  r += n0 * R;
166 
167  if (n != 0) {
168  eval(ctr, r, par);
169  n -= 1;
170  r += R;
171  }
172 
173  eval_kernel(ctr, n, r, par);
174  }
175 
176  private:
177  template <typename ResultType>
178  static void eval_kernel(std::array<std::uint64_t, 8> &ctr, std::size_t n,
179  ResultType *r, const std::array<T, K + 4> &par)
180  {
181  constexpr std::size_t S = 8;
182  constexpr std::size_t N = sizeof(__m256i) * S / (sizeof(T) * K);
183 
184  __m256i ymmc0 =
185  _mm256_set_epi64x(static_cast<MCKL_INT64>(std::get<3>(ctr)),
186  static_cast<MCKL_INT64>(std::get<2>(ctr)),
187  static_cast<MCKL_INT64>(std::get<1>(ctr)),
188  static_cast<MCKL_INT64>(std::get<0>(ctr)));
189  __m256i ymmc1 =
190  _mm256_set_epi64x(static_cast<MCKL_INT64>(std::get<7>(ctr)),
191  static_cast<MCKL_INT64>(std::get<6>(ctr)),
192  static_cast<MCKL_INT64>(std::get<5>(ctr)),
193  static_cast<MCKL_INT64>(std::get<4>(ctr)));
194  ctr.front() += n;
195 
196  __m256i *rptr = reinterpret_cast<__m256i *>(r);
197  while (n != 0) {
198  __m256i ymms0 =
199  _mm256_add_epi64(ymmc0, _mm256_set_epi64x(0, 0, 0, 1));
200  __m256i ymms2 =
201  _mm256_add_epi64(ymmc0, _mm256_set_epi64x(0, 0, 0, 2));
202  __m256i ymms4 =
203  _mm256_add_epi64(ymmc0, _mm256_set_epi64x(0, 0, 0, 3));
204  __m256i ymms6 =
205  _mm256_add_epi64(ymmc0, _mm256_set_epi64x(0, 0, 0, 4));
206  ymmc0 = ymms6;
207 
208  __m256i ymms1 = ymmc1;
209  __m256i ymms3 = ymmc1;
210  __m256i ymms5 = ymmc1;
211  __m256i ymms7 = ymmc1;
212 
213  __m256i ymmt0;
214  __m256i ymmt1;
215  __m256i ymmt2;
216  __m256i ymmt3;
217  __m256i ymmt4;
218  __m256i ymmt5;
219  __m256i ymmt6;
220  __m256i ymmt7;
221 
222  ymmt1 = ymms1;
223  ymms1 = ymms2;
224  ymms2 = ymms4;
225  ymms4 = ymmt1;
226 
227  ymmt3 = ymms3;
228  ymms3 = ymms6;
229  ymms6 = ymms5;
230  ymms5 = ymmt3;
231 
232  ymmt0 = _mm256_unpacklo_epi64(ymms0, ymms1);
233  ymmt1 = _mm256_unpacklo_epi64(ymms2, ymms3);
234  ymmt2 = _mm256_unpackhi_epi64(ymms0, ymms1);
235  ymmt3 = _mm256_unpackhi_epi64(ymms2, ymms3);
236  ymmt4 = _mm256_unpacklo_epi64(ymms4, ymms5);
237  ymmt5 = _mm256_unpacklo_epi64(ymms6, ymms7);
238  ymmt6 = _mm256_unpackhi_epi64(ymms4, ymms5);
239  ymmt7 = _mm256_unpackhi_epi64(ymms6, ymms7);
240  ymms0 = _mm256_permute2x128_si256(ymmt0, ymmt1, 0x20);
241  ymms1 = _mm256_permute2x128_si256(ymmt2, ymmt3, 0x20);
242  ymms2 = _mm256_permute2x128_si256(ymmt0, ymmt1, 0x31);
243  ymms3 = _mm256_permute2x128_si256(ymmt2, ymmt3, 0x31);
244  ymms4 = _mm256_permute2x128_si256(ymmt4, ymmt5, 0x20);
245  ymms5 = _mm256_permute2x128_si256(ymmt6, ymmt7, 0x20);
246  ymms6 = _mm256_permute2x128_si256(ymmt4, ymmt5, 0x31);
247  ymms7 = _mm256_permute2x128_si256(ymmt6, ymmt7, 0x31);
248 
249  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_KBOX(0);
250  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_CYCLE_8(0);
251  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_CYCLE_8(1);
252  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_CYCLE_4(2);
253 
254  ymmt0 = _mm256_unpacklo_epi64(ymms0, ymms1);
255  ymmt1 = _mm256_unpacklo_epi64(ymms2, ymms3);
256  ymmt2 = _mm256_unpackhi_epi64(ymms0, ymms1);
257  ymmt3 = _mm256_unpackhi_epi64(ymms2, ymms3);
258  ymmt4 = _mm256_unpacklo_epi64(ymms4, ymms5);
259  ymmt5 = _mm256_unpacklo_epi64(ymms6, ymms7);
260  ymmt6 = _mm256_unpackhi_epi64(ymms4, ymms5);
261  ymmt7 = _mm256_unpackhi_epi64(ymms6, ymms7);
262  ymms0 = _mm256_permute2x128_si256(ymmt0, ymmt1, 0x20);
263  ymms1 = _mm256_permute2x128_si256(ymmt2, ymmt3, 0x20);
264  ymms2 = _mm256_permute2x128_si256(ymmt0, ymmt1, 0x31);
265  ymms3 = _mm256_permute2x128_si256(ymmt2, ymmt3, 0x31);
266  ymms4 = _mm256_permute2x128_si256(ymmt4, ymmt5, 0x20);
267  ymms5 = _mm256_permute2x128_si256(ymmt6, ymmt7, 0x20);
268  ymms6 = _mm256_permute2x128_si256(ymmt4, ymmt5, 0x31);
269  ymms7 = _mm256_permute2x128_si256(ymmt6, ymmt7, 0x31);
270 
271  ymmt2 = ymms2;
272  ymms2 = ymms1;
273  ymms1 = ymms4;
274  ymms4 = ymmt2;
275 
276  ymmt6 = ymms6;
277  ymms6 = ymms3;
278  ymms3 = ymms5;
279  ymms5 = ymmt6;
280 
281  if (n >= N) {
282  n -= N;
283  _mm256_storeu_si256(rptr++, ymms0);
284  _mm256_storeu_si256(rptr++, ymms1);
285  _mm256_storeu_si256(rptr++, ymms2);
286  _mm256_storeu_si256(rptr++, ymms3);
287  _mm256_storeu_si256(rptr++, ymms4);
288  _mm256_storeu_si256(rptr++, ymms5);
289  _mm256_storeu_si256(rptr++, ymms6);
290  _mm256_storeu_si256(rptr++, ymms7);
291  } else {
292  std::array<__m256i, S> s;
293  std::get<0>(s) = ymms0;
294  std::get<1>(s) = ymms1;
295  std::get<2>(s) = ymms2;
296  std::get<3>(s) = ymms3;
297  std::get<4>(s) = ymms4;
298  std::get<5>(s) = ymms5;
299  std::get<6>(s) = ymms6;
300  std::get<7>(s) = ymms7;
301  std::memcpy(rptr, s.data(), n * sizeof(T) * K);
302  break;
303  }
304  }
305  }
306 }; // class Threefry8x64GeneratorAVX2Impl
307 
308 template <typename T>
309 class Threefish512GeneratorAVX2Impl
310 {
311  static_assert(std::numeric_limits<T>::digits == 64,
312  "**Threefish512GeneratorAVX2Impl** used with T other than a 64-bit "
313  "unsigned integers");
314 
315  static constexpr std::size_t K = 8;
316  static constexpr std::size_t Rounds = 20;
317  using Constants = ThreefryConstants<T, 8>;
318 
319  public:
320  static void eval(
321  const void *plain, void *cipher, const std::array<T, K + 4> &par)
322  {
323  Threefish512GeneratorGenericImpl<T>::eval(plain, cipher, par);
324  }
325 
326  template <typename ResultType>
327  static void eval(std::array<std::uint64_t, 8> &ctr, ResultType *r,
328  const std::array<T, K + 4> &par)
329  {
330  Threefish512GeneratorGenericImpl<T>::eval(ctr, r, par);
331  }
332 
333  template <typename ResultType>
334  static void eval(std::array<std::uint64_t, 8> &ctr, std::size_t n,
335  ResultType *r, const std::array<T, K + 4> &par)
336  {
337  constexpr std::size_t R = sizeof(T) * K / sizeof(ResultType);
338 
339  const std::size_t n0 =
340  static_cast<std::size_t>(std::min(static_cast<std::uint64_t>(n),
341  std::numeric_limits<std::uint64_t>::max() - ctr.front()));
342 
343  eval_kernel(ctr, n0, r, par);
344  n -= n0;
345  r += n0 * R;
346 
347  if (n != 0) {
348  eval(ctr, r, par);
349  n -= 1;
350  r += R;
351  }
352 
353  eval_kernel(ctr, n, r, par);
354  }
355 
356  private:
357  template <typename ResultType>
358  static void eval_kernel(std::array<std::uint64_t, 8> &ctr, std::size_t n,
359  ResultType *r, const std::array<T, K + 4> &par)
360  {
361  constexpr std::size_t S = 8;
362  constexpr std::size_t N = sizeof(__m256i) * S / (sizeof(T) * K);
363 
364  __m256i ymmc0 =
365  _mm256_set_epi64x(static_cast<MCKL_INT64>(std::get<3>(ctr)),
366  static_cast<MCKL_INT64>(std::get<2>(ctr)),
367  static_cast<MCKL_INT64>(std::get<1>(ctr)),
368  static_cast<MCKL_INT64>(std::get<0>(ctr)));
369  __m256i ymmc1 =
370  _mm256_set_epi64x(static_cast<MCKL_INT64>(std::get<7>(ctr)),
371  static_cast<MCKL_INT64>(std::get<6>(ctr)),
372  static_cast<MCKL_INT64>(std::get<5>(ctr)),
373  static_cast<MCKL_INT64>(std::get<4>(ctr)));
374  ctr.front() += n;
375 
376  __m256i *rptr = reinterpret_cast<__m256i *>(r);
377  while (n != 0) {
378  __m256i ymms0 =
379  _mm256_add_epi64(ymmc0, _mm256_set_epi64x(0, 0, 0, 1));
380  __m256i ymms2 =
381  _mm256_add_epi64(ymmc0, _mm256_set_epi64x(0, 0, 0, 2));
382  __m256i ymms4 =
383  _mm256_add_epi64(ymmc0, _mm256_set_epi64x(0, 0, 0, 3));
384  __m256i ymms6 =
385  _mm256_add_epi64(ymmc0, _mm256_set_epi64x(0, 0, 0, 4));
386  ymmc0 = ymms6;
387 
388  __m256i ymms1 = ymmc1;
389  __m256i ymms3 = ymmc1;
390  __m256i ymms5 = ymmc1;
391  __m256i ymms7 = ymmc1;
392 
393  __m256i ymmt0;
394  __m256i ymmt1;
395  __m256i ymmt2;
396  __m256i ymmt3;
397  __m256i ymmt4;
398  __m256i ymmt5;
399  __m256i ymmt6;
400  __m256i ymmt7;
401 
402  ymmt1 = ymms1;
403  ymms1 = ymms2;
404  ymms2 = ymms4;
405  ymms4 = ymmt1;
406 
407  ymmt3 = ymms3;
408  ymms3 = ymms6;
409  ymms6 = ymms5;
410  ymms5 = ymmt3;
411 
412  ymmt0 = _mm256_unpacklo_epi64(ymms0, ymms1);
413  ymmt1 = _mm256_unpacklo_epi64(ymms2, ymms3);
414  ymmt2 = _mm256_unpackhi_epi64(ymms0, ymms1);
415  ymmt3 = _mm256_unpackhi_epi64(ymms2, ymms3);
416  ymmt4 = _mm256_unpacklo_epi64(ymms4, ymms5);
417  ymmt5 = _mm256_unpacklo_epi64(ymms6, ymms7);
418  ymmt6 = _mm256_unpackhi_epi64(ymms4, ymms5);
419  ymmt7 = _mm256_unpackhi_epi64(ymms6, ymms7);
420  ymms0 = _mm256_permute2x128_si256(ymmt0, ymmt1, 0x20);
421  ymms1 = _mm256_permute2x128_si256(ymmt2, ymmt3, 0x20);
422  ymms2 = _mm256_permute2x128_si256(ymmt0, ymmt1, 0x31);
423  ymms3 = _mm256_permute2x128_si256(ymmt2, ymmt3, 0x31);
424  ymms4 = _mm256_permute2x128_si256(ymmt4, ymmt5, 0x20);
425  ymms5 = _mm256_permute2x128_si256(ymmt6, ymmt7, 0x20);
426  ymms6 = _mm256_permute2x128_si256(ymmt4, ymmt5, 0x31);
427  ymms7 = _mm256_permute2x128_si256(ymmt6, ymmt7, 0x31);
428 
429  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_KBOX(0);
430  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_CYCLE_8(0);
431  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_CYCLE_8(1);
432  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_CYCLE_8(2);
433  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_CYCLE_8(3);
434  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_CYCLE_8(4);
435  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_CYCLE_8(5);
436  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_CYCLE_8(6);
437  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_CYCLE_8(7);
438  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_CYCLE_8(8);
439 
440  ymmt0 = _mm256_unpacklo_epi64(ymms0, ymms1);
441  ymmt1 = _mm256_unpacklo_epi64(ymms2, ymms3);
442  ymmt2 = _mm256_unpackhi_epi64(ymms0, ymms1);
443  ymmt3 = _mm256_unpackhi_epi64(ymms2, ymms3);
444  ymmt4 = _mm256_unpacklo_epi64(ymms4, ymms5);
445  ymmt5 = _mm256_unpacklo_epi64(ymms6, ymms7);
446  ymmt6 = _mm256_unpackhi_epi64(ymms4, ymms5);
447  ymmt7 = _mm256_unpackhi_epi64(ymms6, ymms7);
448  ymms0 = _mm256_permute2x128_si256(ymmt0, ymmt1, 0x20);
449  ymms1 = _mm256_permute2x128_si256(ymmt2, ymmt3, 0x20);
450  ymms2 = _mm256_permute2x128_si256(ymmt0, ymmt1, 0x31);
451  ymms3 = _mm256_permute2x128_si256(ymmt2, ymmt3, 0x31);
452  ymms4 = _mm256_permute2x128_si256(ymmt4, ymmt5, 0x20);
453  ymms5 = _mm256_permute2x128_si256(ymmt6, ymmt7, 0x20);
454  ymms6 = _mm256_permute2x128_si256(ymmt4, ymmt5, 0x31);
455  ymms7 = _mm256_permute2x128_si256(ymmt6, ymmt7, 0x31);
456 
457  ymmt2 = ymms2;
458  ymms2 = ymms1;
459  ymms1 = ymms4;
460  ymms4 = ymmt2;
461 
462  ymmt6 = ymms6;
463  ymms6 = ymms3;
464  ymms3 = ymms5;
465  ymms5 = ymmt6;
466 
467  if (n >= N) {
468  n -= N;
469  _mm256_storeu_si256(rptr++, ymms0);
470  _mm256_storeu_si256(rptr++, ymms1);
471  _mm256_storeu_si256(rptr++, ymms2);
472  _mm256_storeu_si256(rptr++, ymms3);
473  _mm256_storeu_si256(rptr++, ymms4);
474  _mm256_storeu_si256(rptr++, ymms5);
475  _mm256_storeu_si256(rptr++, ymms6);
476  _mm256_storeu_si256(rptr++, ymms7);
477  } else {
478  std::array<__m256i, S> s;
479  std::get<0>(s) = ymms0;
480  std::get<1>(s) = ymms1;
481  std::get<2>(s) = ymms2;
482  std::get<3>(s) = ymms3;
483  std::get<4>(s) = ymms4;
484  std::get<5>(s) = ymms5;
485  std::get<6>(s) = ymms6;
486  std::get<7>(s) = ymms7;
487  std::memcpy(rptr, s.data(), n * sizeof(T) * K);
488  break;
489  }
490  }
491  }
492 }; // class Threefish512GeneratorAVX2Impl
493 
494 } // namespace internal
495 
496 } // namespace mckl
497 
498 MCKL_POP_GCC_WARNING
499 
500 #endif // MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_HPP
mckl::internal::Threefry8x64GeneratorAVX2Impl::eval
static void eval(std::array< std::uint64_t, 8 > &ctr, ResultType *r, const std::array< T, K+4 > &par)
Definition: threefry_avx2_8x64.hpp:147
MCKL_PUSH_GCC_WARNING
#define MCKL_PUSH_GCC_WARNING(warning)
Definition: compiler.h:78
MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_CYCLE_4
#define MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_CYCLE_4(N)
Definition: threefry_avx2_8x64.hpp:105
threefry_generic_8x64.hpp
mckl::internal::Threefish512GeneratorGenericImpl::eval
static void eval(const void *plain, void *cipher, const std::array< T, K+4 > &par)
Definition: threefry_generic_8x64.hpp:219
mckl::internal::Threefry8x64GeneratorAVX2Impl::eval
static void eval(const void *plain, void *cipher, const std::array< T, K+4 > &par)
Definition: threefry_avx2_8x64.hpp:140
mckl::internal::Threefish512GeneratorAVX2Impl::eval
static void eval(const void *plain, void *cipher, const std::array< T, K+4 > &par)
Definition: threefry_avx2_8x64.hpp:320
mckl::internal::Threefry8x64GeneratorGenericImpl::eval
static void eval(const void *plain, void *cipher, const std::array< T, K+4 > &par)
Definition: threefry_generic_8x64.hpp:158
MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_CYCLE_8
#define MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_CYCLE_8(N)
Definition: threefry_avx2_8x64.hpp:112
mckl::ThreefryConstants
Default Threefry constants.
Definition: threefry_constants.hpp:147
mckl::internal::Threefish512GeneratorAVX2Impl::eval
static void eval(std::array< std::uint64_t, 8 > &ctr, std::size_t n, ResultType *r, const std::array< T, K+4 > &par)
Definition: threefry_avx2_8x64.hpp:334
threefry_constants.hpp
mckl
Definition: mcmc.hpp:40
threefry_common.hpp
mckl::internal::Threefish512GeneratorAVX2Impl::eval
static void eval(std::array< std::uint64_t, 8 > &ctr, ResultType *r, const std::array< T, K+4 > &par)
Definition: threefry_avx2_8x64.hpp:327
MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_KBOX
#define MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_8X64_KBOX(N)
Definition: threefry_avx2_8x64.hpp:43
mckl::internal::Threefry8x64GeneratorAVX2Impl::eval
static void eval(std::array< std::uint64_t, 8 > &ctr, std::size_t n, ResultType *r, const std::array< T, K+4 > &par)
Definition: threefry_avx2_8x64.hpp:154
MCKL_POP_GCC_WARNING
#define MCKL_POP_GCC_WARNING
Definition: compiler.h:79
Generated by   doxygen 1.8.13