MCKL
Monte Carlo Kernel Library
threefry_avx2_4x64.hpp
Go to the documentation of this file.
1 //============================================================================
2 // MCKL/include/mckl/random/internal/threefry_avx2_4x64.hpp
3 //----------------------------------------------------------------------------
4 // MCKL: Monte Carlo Kernel Library
5 //----------------------------------------------------------------------------
6 // Copyright (c) 2013-2018, Yan Zhou
7 // All rights reserved.
8 //
9 // Redistribution and use in source and binary forms, with or without
10 // modification, are permitted provided that the following conditions are met:
11 //
12 // Redistributions of source code must retain the above copyright notice,
13 // this list of conditions and the following disclaimer.
14 //
15 // Redistributions in binary form must reproduce the above copyright notice,
16 // this list of conditions and the following disclaimer in the documentation
17 // and/or other materials provided with the distribution.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS AS IS
20 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
23 // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 // POSSIBILITY OF SUCH DAMAGE.
30 //============================================================================
31 
32 #ifndef MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_HPP
33 #define MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_HPP
34 
35 #include <mckl/random/internal/common.hpp>
36 #include <mckl/random/internal/threefry_common.hpp>
37 #include <mckl/random/internal/threefry_constants.hpp>
38 #include <mckl/random/internal/threefry_generic_4x64.hpp>
39 #include <mckl/random/increment.hpp>
40 
41 MCKL_PUSH_GCC_WARNING("-Wignored-attributes")
42 
43 #define MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_KBOX(N) \
44  ymmt0 = _mm256_set1_epi64x(static_cast<MCKL_INT64>( \
45  ThreefryKBox<T, K, N>::template key<0>(par))); \
46  ymmt1 = _mm256_set1_epi64x(static_cast<MCKL_INT64>( \
47  ThreefryKBox<T, K, N>::template key<1>(par))); \
48  ymmt2 = _mm256_set1_epi64x(static_cast<MCKL_INT64>( \
49  ThreefryKBox<T, K, N>::template key<2>(par))); \
50  ymmt3 = _mm256_set1_epi64x(static_cast<MCKL_INT64>( \
51  ThreefryKBox<T, K, N>::template key<3>(par))); \
52  ymms0 = _mm256_add_epi64(ymms0, ymmt0); \
53  ymms1 = _mm256_add_epi64(ymms1, ymmt1); \
54  ymms2 = _mm256_add_epi64(ymms2, ymmt2); \
55  ymms3 = _mm256_add_epi64(ymms3, ymmt3); \
56  ymms4 = _mm256_add_epi64(ymms4, ymmt0); \
57  ymms5 = _mm256_add_epi64(ymms5, ymmt1); \
58  ymms6 = _mm256_add_epi64(ymms6, ymmt2); \
59  ymms7 = _mm256_add_epi64(ymms7, ymmt3);
60 
61 #define MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_RBOX(N) \
62  { \
63  constexpr int L0 = Constants::rotate::value[0][(N - 1) % 8]; \
64  constexpr int L1 = Constants::rotate::value[1][(N - 1) % 8]; \
65  constexpr int R0 = 64 - L0; \
66  constexpr int R1 = 64 - L1; \
67  ymms0 = _mm256_add_epi64(ymms0, ymms1); \
68  ymms2 = _mm256_add_epi64(ymms2, ymms3); \
69  ymms4 = _mm256_add_epi64(ymms4, ymms5); \
70  ymms6 = _mm256_add_epi64(ymms6, ymms7); \
71  ymmt1 = _mm256_slli_epi64(ymms1, L0); \
72  ymmt3 = _mm256_slli_epi64(ymms3, L1); \
73  ymmt5 = _mm256_slli_epi64(ymms5, L0); \
74  ymmt7 = _mm256_slli_epi64(ymms7, L1); \
75  ymms1 = _mm256_srli_epi64(ymms1, R0); \
76  ymms3 = _mm256_srli_epi64(ymms3, R1); \
77  ymms5 = _mm256_srli_epi64(ymms5, R0); \
78  ymms7 = _mm256_srli_epi64(ymms7, R1); \
79  ymmt1 = _mm256_or_si256(ymms1, ymmt1); \
80  ymmt3 = _mm256_or_si256(ymms3, ymmt3); \
81  ymmt5 = _mm256_or_si256(ymms5, ymmt5); \
82  ymmt7 = _mm256_or_si256(ymms7, ymmt7); \
83  ymms1 = _mm256_xor_si256(ymms2, ymmt3); \
84  ymms3 = _mm256_xor_si256(ymms0, ymmt1); \
85  ymms5 = _mm256_xor_si256(ymms6, ymmt7); \
86  ymms7 = _mm256_xor_si256(ymms4, ymmt5); \
87  }
88 
89 #define MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_CYCLE_4(N) \
90  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_RBOX(N * 8 + 1); \
91  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_RBOX(N * 8 + 2); \
92  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_RBOX(N * 8 + 3); \
93  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_RBOX(N * 8 + 4); \
94  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_KBOX(N * 8 + 4);
95 
96 #define MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_CYCLE_8(N) \
97  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_RBOX(N * 8 + 1); \
98  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_RBOX(N * 8 + 2); \
99  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_RBOX(N * 8 + 3); \
100  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_RBOX(N * 8 + 4); \
101  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_KBOX(N * 8 + 4); \
102  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_RBOX(N * 8 + 5); \
103  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_RBOX(N * 8 + 6); \
104  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_RBOX(N * 8 + 7); \
105  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_RBOX(N * 8 + 8); \
106  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_KBOX(N * 8 + 8);
107 
108 namespace mckl {
109 
110 namespace internal {
111 
112 template <typename T>
113 class Threefry4x64GeneratorAVX2Impl
114 {
115  static_assert(std::numeric_limits<T>::digits == 64,
116  "**Threefry4x64GeneratorAVX2Impl** used with T other than a 64-bit "
117  "unsigned integers");
118 
119  static constexpr std::size_t K = 4;
120  static constexpr std::size_t Rounds = 20;
121  using Constants = ThreefryConstants<T, 4>;
122 
123  public:
124  static void eval(
125  const void *plain, void *cipher, const std::array<T, K + 4> &par)
126  {
127  Threefry4x64GeneratorGenericImpl<T>::eval(plain, cipher, par);
128  }
129 
130  template <typename ResultType>
131  static void eval(std::array<std::uint64_t, 4> &ctr, ResultType *r,
132  const std::array<T, K + 4> &par)
133  {
134  Threefry4x64GeneratorGenericImpl<T>::eval(ctr, r, par);
135  }
136 
137  template <typename ResultType>
138  static void eval(std::array<std::uint64_t, 4> &ctr, std::size_t n,
139  ResultType *r, const std::array<T, K + 4> &par)
140  {
141  constexpr std::size_t R = sizeof(T) * K / sizeof(ResultType);
142 
143  const std::size_t n0 =
144  static_cast<std::size_t>(std::min(static_cast<std::uint64_t>(n),
145  std::numeric_limits<std::uint64_t>::max() - ctr.front()));
146 
147  eval_kernel(ctr, n0, r, par);
148  n -= n0;
149  r += n0 * R;
150 
151  if (n != 0) {
152  eval(ctr, r, par);
153  n -= 1;
154  r += R;
155  }
156 
157  eval_kernel(ctr, n, r, par);
158  }
159 
160  template <typename ResultType>
161  static void eval_kernel(std::array<std::uint64_t, 4> &ctr, std::size_t n,
162  ResultType *r, const std::array<T, K + 4> &par)
163  {
164  constexpr std::size_t S = 8;
165  constexpr std::size_t N = sizeof(__m256i) * S / (sizeof(T) * K);
166 
167  __m256i ymmc =
168  _mm256_set_epi64x(static_cast<MCKL_INT64>(std::get<3>(ctr)),
169  static_cast<MCKL_INT64>(std::get<2>(ctr)),
170  static_cast<MCKL_INT64>(std::get<1>(ctr)),
171  static_cast<MCKL_INT64>(std::get<0>(ctr)));
172  ctr.front() += n;
173 
174  __m256i *rptr = reinterpret_cast<__m256i *>(r);
175  while (n != 0) {
176  __m256i ymms0 =
177  _mm256_add_epi64(ymmc, _mm256_set_epi64x(0, 0, 0, 1));
178  __m256i ymms1 =
179  _mm256_add_epi64(ymmc, _mm256_set_epi64x(0, 0, 0, 2));
180  __m256i ymms2 =
181  _mm256_add_epi64(ymmc, _mm256_set_epi64x(0, 0, 0, 3));
182  __m256i ymms3 =
183  _mm256_add_epi64(ymmc, _mm256_set_epi64x(0, 0, 0, 4));
184  __m256i ymms4 =
185  _mm256_add_epi64(ymmc, _mm256_set_epi64x(0, 0, 0, 5));
186  __m256i ymms5 =
187  _mm256_add_epi64(ymmc, _mm256_set_epi64x(0, 0, 0, 6));
188  __m256i ymms6 =
189  _mm256_add_epi64(ymmc, _mm256_set_epi64x(0, 0, 0, 7));
190  __m256i ymms7 =
191  _mm256_add_epi64(ymmc, _mm256_set_epi64x(0, 0, 0, 8));
192  ymmc = ymms7;
193 
194  __m256i ymmt0;
195  __m256i ymmt1;
196  __m256i ymmt2;
197  __m256i ymmt3;
198  __m256i ymmt4;
199  __m256i ymmt5;
200  __m256i ymmt6;
201  __m256i ymmt7;
202 
203  ymmt1 = ymms1;
204  ymms1 = ymms2;
205  ymms2 = ymms4;
206  ymms4 = ymmt1;
207 
208  ymmt3 = ymms3;
209  ymms3 = ymms6;
210  ymms6 = ymms5;
211  ymms5 = ymmt3;
212 
213  ymmt0 = _mm256_unpacklo_epi64(ymms0, ymms1);
214  ymmt1 = _mm256_unpacklo_epi64(ymms2, ymms3);
215  ymmt2 = _mm256_unpackhi_epi64(ymms0, ymms1);
216  ymmt3 = _mm256_unpackhi_epi64(ymms2, ymms3);
217  ymmt4 = _mm256_unpacklo_epi64(ymms4, ymms5);
218  ymmt5 = _mm256_unpacklo_epi64(ymms6, ymms7);
219  ymmt6 = _mm256_unpackhi_epi64(ymms4, ymms5);
220  ymmt7 = _mm256_unpackhi_epi64(ymms6, ymms7);
221  ymms0 = _mm256_permute2x128_si256(ymmt0, ymmt1, 0x20);
222  ymms1 = _mm256_permute2x128_si256(ymmt2, ymmt3, 0x20);
223  ymms2 = _mm256_permute2x128_si256(ymmt0, ymmt1, 0x31);
224  ymms3 = _mm256_permute2x128_si256(ymmt2, ymmt3, 0x31);
225  ymms4 = _mm256_permute2x128_si256(ymmt4, ymmt5, 0x20);
226  ymms5 = _mm256_permute2x128_si256(ymmt6, ymmt7, 0x20);
227  ymms6 = _mm256_permute2x128_si256(ymmt4, ymmt5, 0x31);
228  ymms7 = _mm256_permute2x128_si256(ymmt6, ymmt7, 0x31);
229 
230  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_KBOX(0);
231  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_CYCLE_8(0);
232  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_CYCLE_8(1);
233  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_CYCLE_4(2);
234 
235  ymmt0 = _mm256_unpacklo_epi64(ymms0, ymms1);
236  ymmt1 = _mm256_unpacklo_epi64(ymms2, ymms3);
237  ymmt2 = _mm256_unpackhi_epi64(ymms0, ymms1);
238  ymmt3 = _mm256_unpackhi_epi64(ymms2, ymms3);
239  ymmt4 = _mm256_unpacklo_epi64(ymms4, ymms5);
240  ymmt5 = _mm256_unpacklo_epi64(ymms6, ymms7);
241  ymmt6 = _mm256_unpackhi_epi64(ymms4, ymms5);
242  ymmt7 = _mm256_unpackhi_epi64(ymms6, ymms7);
243  ymms0 = _mm256_permute2x128_si256(ymmt0, ymmt1, 0x20);
244  ymms1 = _mm256_permute2x128_si256(ymmt2, ymmt3, 0x20);
245  ymms2 = _mm256_permute2x128_si256(ymmt0, ymmt1, 0x31);
246  ymms3 = _mm256_permute2x128_si256(ymmt2, ymmt3, 0x31);
247  ymms4 = _mm256_permute2x128_si256(ymmt4, ymmt5, 0x20);
248  ymms5 = _mm256_permute2x128_si256(ymmt6, ymmt7, 0x20);
249  ymms6 = _mm256_permute2x128_si256(ymmt4, ymmt5, 0x31);
250  ymms7 = _mm256_permute2x128_si256(ymmt6, ymmt7, 0x31);
251 
252  ymmt2 = ymms2;
253  ymms2 = ymms1;
254  ymms1 = ymms4;
255  ymms4 = ymmt2;
256 
257  ymmt6 = ymms6;
258  ymms6 = ymms3;
259  ymms3 = ymms5;
260  ymms5 = ymmt6;
261 
262  if (n >= N) {
263  n -= N;
264  _mm256_storeu_si256(rptr++, ymms0);
265  _mm256_storeu_si256(rptr++, ymms1);
266  _mm256_storeu_si256(rptr++, ymms2);
267  _mm256_storeu_si256(rptr++, ymms3);
268  _mm256_storeu_si256(rptr++, ymms4);
269  _mm256_storeu_si256(rptr++, ymms5);
270  _mm256_storeu_si256(rptr++, ymms6);
271  _mm256_storeu_si256(rptr++, ymms7);
272  } else {
273  std::array<__m256i, S> s;
274  std::get<0>(s) = ymms0;
275  std::get<1>(s) = ymms1;
276  std::get<2>(s) = ymms2;
277  std::get<3>(s) = ymms3;
278  std::get<4>(s) = ymms4;
279  std::get<5>(s) = ymms5;
280  std::get<6>(s) = ymms6;
281  std::get<7>(s) = ymms7;
282  std::memcpy(rptr, s.data(), n * sizeof(T) * K);
283  break;
284  }
285  }
286  }
287 }; // class Threefry4x64GeneratorAVX2Impl
288 
289 template <typename T>
290 class Threefish256GeneratorAVX2Impl
291 {
292  static_assert(std::numeric_limits<T>::digits == 64,
293  "**Threefish256GeneratorAVX2Impl** used with T other than a 64-bit "
294  "unsigned integers");
295 
296  static constexpr std::size_t K = 4;
297  static constexpr std::size_t Rounds = 72;
298  using Constants = ThreefryConstants<T, 4>;
299 
300  public:
301  static void eval(
302  const void *plain, void *cipher, const std::array<T, K + 4> &par)
303  {
304  Threefish256GeneratorGenericImpl<T>::eval(plain, cipher, par);
305  }
306 
307  template <typename ResultType>
308  static void eval(std::array<std::uint64_t, 4> &ctr, ResultType *r,
309  const std::array<T, K + 4> &par)
310  {
311  Threefish256GeneratorGenericImpl<T>::eval(ctr, r, par);
312  }
313 
314  template <typename ResultType>
315  static void eval(std::array<std::uint64_t, 4> &ctr, std::size_t n,
316  ResultType *r, const std::array<T, K + 4> &par)
317  {
318  constexpr std::size_t R = sizeof(T) * K / sizeof(ResultType);
319 
320  const std::size_t n0 =
321  static_cast<std::size_t>(std::min(static_cast<std::uint64_t>(n),
322  std::numeric_limits<std::uint64_t>::max() - ctr.front()));
323 
324  eval_kernel(ctr, n0, r, par);
325  n -= n0;
326  r += n0 * R;
327 
328  if (n != 0) {
329  eval(ctr, r, par);
330  n -= 1;
331  r += R;
332  }
333 
334  eval_kernel(ctr, n, r, par);
335  }
336 
337  private:
338  template <typename ResultType>
339  static void eval_kernel(std::array<std::uint64_t, 4> &ctr, std::size_t n,
340  ResultType *r, const std::array<T, K + 4> &par)
341  {
342  constexpr std::size_t S = 8;
343  constexpr std::size_t N = sizeof(__m256i) * S / (sizeof(T) * K);
344 
345  __m256i ymmc =
346  _mm256_set_epi64x(static_cast<MCKL_INT64>(std::get<3>(ctr)),
347  static_cast<MCKL_INT64>(std::get<2>(ctr)),
348  static_cast<MCKL_INT64>(std::get<1>(ctr)),
349  static_cast<MCKL_INT64>(std::get<0>(ctr)));
350  ctr.front() += n;
351 
352  __m256i *rptr = reinterpret_cast<__m256i *>(r);
353  while (n != 0) {
354  __m256i ymms0 =
355  _mm256_add_epi64(ymmc, _mm256_set_epi64x(0, 0, 0, 1));
356  __m256i ymms1 =
357  _mm256_add_epi64(ymmc, _mm256_set_epi64x(0, 0, 0, 2));
358  __m256i ymms2 =
359  _mm256_add_epi64(ymmc, _mm256_set_epi64x(0, 0, 0, 3));
360  __m256i ymms3 =
361  _mm256_add_epi64(ymmc, _mm256_set_epi64x(0, 0, 0, 4));
362  __m256i ymms4 =
363  _mm256_add_epi64(ymmc, _mm256_set_epi64x(0, 0, 0, 5));
364  __m256i ymms5 =
365  _mm256_add_epi64(ymmc, _mm256_set_epi64x(0, 0, 0, 6));
366  __m256i ymms6 =
367  _mm256_add_epi64(ymmc, _mm256_set_epi64x(0, 0, 0, 7));
368  __m256i ymms7 =
369  _mm256_add_epi64(ymmc, _mm256_set_epi64x(0, 0, 0, 8));
370  ymmc = ymms7;
371 
372  __m256i ymmt0;
373  __m256i ymmt1;
374  __m256i ymmt2;
375  __m256i ymmt3;
376  __m256i ymmt4;
377  __m256i ymmt5;
378  __m256i ymmt6;
379  __m256i ymmt7;
380 
381  ymmt1 = ymms1;
382  ymms1 = ymms2;
383  ymms2 = ymms4;
384  ymms4 = ymmt1;
385 
386  ymmt3 = ymms3;
387  ymms3 = ymms6;
388  ymms6 = ymms5;
389  ymms5 = ymmt3;
390 
391  ymmt0 = _mm256_unpacklo_epi64(ymms0, ymms1);
392  ymmt1 = _mm256_unpacklo_epi64(ymms2, ymms3);
393  ymmt2 = _mm256_unpackhi_epi64(ymms0, ymms1);
394  ymmt3 = _mm256_unpackhi_epi64(ymms2, ymms3);
395  ymmt4 = _mm256_unpacklo_epi64(ymms4, ymms5);
396  ymmt5 = _mm256_unpacklo_epi64(ymms6, ymms7);
397  ymmt6 = _mm256_unpackhi_epi64(ymms4, ymms5);
398  ymmt7 = _mm256_unpackhi_epi64(ymms6, ymms7);
399  ymms0 = _mm256_permute2x128_si256(ymmt0, ymmt1, 0x20);
400  ymms1 = _mm256_permute2x128_si256(ymmt2, ymmt3, 0x20);
401  ymms2 = _mm256_permute2x128_si256(ymmt0, ymmt1, 0x31);
402  ymms3 = _mm256_permute2x128_si256(ymmt2, ymmt3, 0x31);
403  ymms4 = _mm256_permute2x128_si256(ymmt4, ymmt5, 0x20);
404  ymms5 = _mm256_permute2x128_si256(ymmt6, ymmt7, 0x20);
405  ymms6 = _mm256_permute2x128_si256(ymmt4, ymmt5, 0x31);
406  ymms7 = _mm256_permute2x128_si256(ymmt6, ymmt7, 0x31);
407 
408  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_KBOX(0);
409  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_CYCLE_8(0);
410  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_CYCLE_8(1);
411  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_CYCLE_8(2);
412  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_CYCLE_8(3);
413  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_CYCLE_8(4);
414  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_CYCLE_8(5);
415  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_CYCLE_8(6);
416  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_CYCLE_8(7);
417  MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_CYCLE_8(8);
418 
419  ymmt0 = _mm256_unpacklo_epi64(ymms0, ymms1);
420  ymmt1 = _mm256_unpacklo_epi64(ymms2, ymms3);
421  ymmt2 = _mm256_unpackhi_epi64(ymms0, ymms1);
422  ymmt3 = _mm256_unpackhi_epi64(ymms2, ymms3);
423  ymmt4 = _mm256_unpacklo_epi64(ymms4, ymms5);
424  ymmt5 = _mm256_unpacklo_epi64(ymms6, ymms7);
425  ymmt6 = _mm256_unpackhi_epi64(ymms4, ymms5);
426  ymmt7 = _mm256_unpackhi_epi64(ymms6, ymms7);
427  ymms0 = _mm256_permute2x128_si256(ymmt0, ymmt1, 0x20);
428  ymms1 = _mm256_permute2x128_si256(ymmt2, ymmt3, 0x20);
429  ymms2 = _mm256_permute2x128_si256(ymmt0, ymmt1, 0x31);
430  ymms3 = _mm256_permute2x128_si256(ymmt2, ymmt3, 0x31);
431  ymms4 = _mm256_permute2x128_si256(ymmt4, ymmt5, 0x20);
432  ymms5 = _mm256_permute2x128_si256(ymmt6, ymmt7, 0x20);
433  ymms6 = _mm256_permute2x128_si256(ymmt4, ymmt5, 0x31);
434  ymms7 = _mm256_permute2x128_si256(ymmt6, ymmt7, 0x31);
435 
436  ymmt2 = ymms2;
437  ymms2 = ymms1;
438  ymms1 = ymms4;
439  ymms4 = ymmt2;
440 
441  ymmt6 = ymms6;
442  ymms6 = ymms3;
443  ymms3 = ymms5;
444  ymms5 = ymmt6;
445 
446  if (n >= N) {
447  n -= N;
448  _mm256_storeu_si256(rptr++, ymms0);
449  _mm256_storeu_si256(rptr++, ymms1);
450  _mm256_storeu_si256(rptr++, ymms2);
451  _mm256_storeu_si256(rptr++, ymms3);
452  _mm256_storeu_si256(rptr++, ymms4);
453  _mm256_storeu_si256(rptr++, ymms5);
454  _mm256_storeu_si256(rptr++, ymms6);
455  _mm256_storeu_si256(rptr++, ymms7);
456  } else {
457  std::array<__m256i, S> s;
458  std::get<0>(s) = ymms0;
459  std::get<1>(s) = ymms1;
460  std::get<2>(s) = ymms2;
461  std::get<3>(s) = ymms3;
462  std::get<4>(s) = ymms4;
463  std::get<5>(s) = ymms5;
464  std::get<6>(s) = ymms6;
465  std::get<7>(s) = ymms7;
466  std::memcpy(rptr, s.data(), n * sizeof(T) * K);
467  break;
468  }
469  }
470  }
471 }; // class Threefish256GeneratorAVX2Impl
472 
473 } // namespace internal
474 
475 } // namespace mckl
476 
477 MCKL_POP_GCC_WARNING
478 
479 #endif // MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_HPP
MCKL_PUSH_GCC_WARNING
#define MCKL_PUSH_GCC_WARNING(warning)
Definition: compiler.h:78
mckl::internal::Threefry4x64GeneratorAVX2Impl::eval
static void eval(std::array< std::uint64_t, 4 > &ctr, ResultType *r, const std::array< T, K+4 > &par)
Definition: threefry_avx2_4x64.hpp:131
threefry_generic_4x64.hpp
MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_CYCLE_4
#define MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_CYCLE_4(N)
Definition: threefry_avx2_4x64.hpp:89
mckl::internal::Threefry4x64GeneratorGenericImpl::eval
static void eval(const void *plain, void *cipher, const std::array< T, K+4 > &par)
Definition: threefry_generic_4x64.hpp:126
mckl::internal::Threefry4x64GeneratorAVX2Impl::eval
static void eval(const void *plain, void *cipher, const std::array< T, K+4 > &par)
Definition: threefry_avx2_4x64.hpp:124
MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_CYCLE_8
#define MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_CYCLE_8(N)
Definition: threefry_avx2_4x64.hpp:96
mckl::ThreefryConstants
Default Threefry constants.
Definition: threefry_constants.hpp:147
mckl::internal::Threefish256GeneratorAVX2Impl::eval
static void eval(const void *plain, void *cipher, const std::array< T, K+4 > &par)
Definition: threefry_avx2_4x64.hpp:301
threefry_constants.hpp
mckl
Definition: mcmc.hpp:40
mckl::internal::Threefry4x64GeneratorAVX2Impl::eval
static void eval(std::array< std::uint64_t, 4 > &ctr, std::size_t n, ResultType *r, const std::array< T, K+4 > &par)
Definition: threefry_avx2_4x64.hpp:138
mckl::internal::Threefish256GeneratorAVX2Impl::eval
static void eval(std::array< std::uint64_t, 4 > &ctr, ResultType *r, const std::array< T, K+4 > &par)
Definition: threefry_avx2_4x64.hpp:308
threefry_common.hpp
mckl::internal::Threefish256GeneratorGenericImpl::eval
static void eval(const void *plain, void *cipher, const std::array< T, K+4 > &par)
Definition: threefry_generic_4x64.hpp:187
mckl::internal::Threefry4x64GeneratorAVX2Impl::eval_kernel
static void eval_kernel(std::array< std::uint64_t, 4 > &ctr, std::size_t n, ResultType *r, const std::array< T, K+4 > &par)
Definition: threefry_avx2_4x64.hpp:161
MCKL_POP_GCC_WARNING
#define MCKL_POP_GCC_WARNING
Definition: compiler.h:79
MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_KBOX
#define MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_4X64_KBOX(N)
Definition: threefry_avx2_4x64.hpp:43
mckl::internal::Threefish256GeneratorAVX2Impl::eval
static void eval(std::array< std::uint64_t, 4 > &ctr, std::size_t n, ResultType *r, const std::array< T, K+4 > &par)
Definition: threefry_avx2_4x64.hpp:315
Generated by   doxygen 1.8.13