MCKL
Monte Carlo Kernel Library
threefry_avx2_32.hpp
Go to the documentation of this file.
1 //============================================================================
2 // MCKL/include/mckl/random/internal/threefry_avx2_32.hpp
3 //----------------------------------------------------------------------------
4 // MCKL: Monte Carlo Kernel Library
5 //----------------------------------------------------------------------------
6 // Copyright (c) 2013-2018, Yan Zhou
7 // All rights reserved.
8 //
9 // Redistribution and use in source and binary forms, with or without
10 // modification, are permitted provided that the following conditions are met:
11 //
12 // Redistributions of source code must retain the above copyright notice,
13 // this list of conditions and the following disclaimer.
14 //
15 // Redistributions in binary form must reproduce the above copyright notice,
16 // this list of conditions and the following disclaimer in the documentation
17 // and/or other materials provided with the distribution.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS AS IS
20 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
23 // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 // POSSIBILITY OF SUCH DAMAGE.
30 //============================================================================
31 
32 #ifndef MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_32_HPP
33 #define MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_32_HPP
34 
43 
44 MCKL_PUSH_GCC_WARNING("-Wignored-attributes")
45 
46 namespace mckl {
47 
48 namespace internal {
49 
50 template <typename T, std::size_t K, std::size_t Rounds, typename Constants>
52 {
53  public:
54  static void eval(
55  const void *plain, void *cipher, const std::array<T, K + 4> &par)
56  {
58  plain, cipher, par);
59  }
60 
61  template <typename ResultType>
62  static void eval(
63  Counter<T, K> &ctr, ResultType *r, const std::array<T, K + 4> &par)
64  {
66  ctr, r, par);
67  }
68 
69  template <typename ResultType>
70  static void eval(Counter<T, K> &ctr, std::size_t n, ResultType *r,
71  const std::array<T, K + 4> &par)
72  {
73  constexpr std::size_t S = K <= 8 ? 8 : K;
74  constexpr std::size_t N = sizeof(__m256i) * S / (sizeof(T) * K);
75  constexpr std::size_t R = sizeof(T) * K / sizeof(ResultType);
76 
77  while (n >= N) {
78  std::array<__m256i, S> s;
80  MCKL_INLINE_CALL transpose8x32_load_si256(s);
82  MCKL_INLINE_CALL transpose8x32_store_si256(s);
83  std::memcpy(r, s.data(), sizeof(T) * K * N);
84  n -= N;
85  r += N * R;
86  }
87 
88  alignas(MCKL_ALIGNMENT) std::array<ResultType, N * R> t;
90  ctr, n, t.data(), par);
91  std::memcpy(r, t.data(), sizeof(T) * K * n);
92  }
93 
94  private:
95  template <std::size_t, std::size_t S>
96  static void round(std::array<__m256i, S> &, const std::array<T, K + 4> &,
97  std::false_type)
98  {
99  }
100 
101  template <std::size_t N, std::size_t S>
102  static void round(std::array<__m256i, S> &s,
103  const std::array<T, K + 4> &par, std::true_type)
104  {
106  }
107 
108  template <std::size_t N, std::size_t S>
109  MCKL_INLINE static void kbox(
110  std::array<__m256i, S> &s, const std::array<T, K + 4> &par)
111  {
112  kbox<N>(s, par,
113  std::integral_constant<bool, (N % 4 == 0 && N <= Rounds)>());
114  }
115 
116  template <std::size_t, std::size_t S>
117  static void kbox(std::array<__m256i, S> &, const std::array<T, K + 4> &,
118  std::false_type)
119  {
120  }
121 
122  template <std::size_t N>
123  static void kbox(std::array<__m256i, 8> &s,
124  const std::array<T, K + 4> &par, std::true_type)
125  {
126  std::array<__m256i, K> k;
127  set_key<N>(k, par);
128 
129  std::get<0>(s) = _mm256_add_epi32(std::get<0>(s), std::get<0 % K>(k));
130  std::get<1>(s) = _mm256_add_epi32(std::get<1>(s), std::get<1 % K>(k));
131  std::get<2>(s) = _mm256_add_epi32(std::get<2>(s), std::get<2 % K>(k));
132  std::get<3>(s) = _mm256_add_epi32(std::get<3>(s), std::get<3 % K>(k));
133  std::get<4>(s) = _mm256_add_epi32(std::get<4>(s), std::get<4 % K>(k));
134  std::get<5>(s) = _mm256_add_epi32(std::get<5>(s), std::get<5 % K>(k));
135  std::get<6>(s) = _mm256_add_epi32(std::get<6>(s), std::get<6 % K>(k));
136  std::get<7>(s) = _mm256_add_epi32(std::get<7>(s), std::get<7 % K>(k));
137  }
138 
139  template <std::size_t N>
140  static void kbox(std::array<__m256i, 16> &s,
141  const std::array<T, K + 4> &par, std::true_type)
142  {
143  std::array<__m256i, K> k;
144  set_key<N>(k, par);
145 
146  std::get<0x0>(s) =
147  _mm256_add_epi32(std::get<0x0>(s), std::get<0x0 % K>(k));
148  std::get<0x1>(s) =
149  _mm256_add_epi32(std::get<0x1>(s), std::get<0x1 % K>(k));
150  std::get<0x2>(s) =
151  _mm256_add_epi32(std::get<0x2>(s), std::get<0x2 % K>(k));
152  std::get<0x3>(s) =
153  _mm256_add_epi32(std::get<0x3>(s), std::get<0x3 % K>(k));
154  std::get<0x4>(s) =
155  _mm256_add_epi32(std::get<0x4>(s), std::get<0x4 % K>(k));
156  std::get<0x5>(s) =
157  _mm256_add_epi32(std::get<0x5>(s), std::get<0x5 % K>(k));
158  std::get<0x6>(s) =
159  _mm256_add_epi32(std::get<0x6>(s), std::get<0x6 % K>(k));
160  std::get<0x7>(s) =
161  _mm256_add_epi32(std::get<0x7>(s), std::get<0x7 % K>(k));
162  std::get<0x8>(s) =
163  _mm256_add_epi32(std::get<0x8>(s), std::get<0x8 % K>(k));
164  std::get<0x9>(s) =
165  _mm256_add_epi32(std::get<0x9>(s), std::get<0x9 % K>(k));
166  std::get<0xA>(s) =
167  _mm256_add_epi32(std::get<0xA>(s), std::get<0xA % K>(k));
168  std::get<0xB>(s) =
169  _mm256_add_epi32(std::get<0xB>(s), std::get<0xB % K>(k));
170  std::get<0xC>(s) =
171  _mm256_add_epi32(std::get<0xC>(s), std::get<0xC % K>(k));
172  std::get<0xD>(s) =
173  _mm256_add_epi32(std::get<0xD>(s), std::get<0xD % K>(k));
174  std::get<0xE>(s) =
175  _mm256_add_epi32(std::get<0xE>(s), std::get<0xE % K>(k));
176  std::get<0xF>(s) =
177  _mm256_add_epi32(std::get<0xF>(s), std::get<0xF % K>(k));
178  }
179 
180  template <std::size_t N, std::size_t S>
181  MCKL_INLINE static void rbox(std::array<__m256i, S> &s)
182  {
183  rbox<N>(s, std::integral_constant<bool, (N > 0 && N <= Rounds)>());
184  }
185 
186  template <std::size_t, std::size_t S>
187  static void rbox(std::array<__m256i, S> &, std::false_type)
188  {
189  }
190 
191  template <std::size_t N>
192  static void rbox(std::array<__m256i, 8> &s, std::true_type)
193  {
194  constexpr int L0 = Constants::rotate::value[0 % (K / 2)][(N - 1) % 8];
195  constexpr int L1 = Constants::rotate::value[1 % (K / 2)][(N - 1) % 8];
196  constexpr int L2 = Constants::rotate::value[2 % (K / 2)][(N - 1) % 8];
197  constexpr int L3 = Constants::rotate::value[3 % (K / 2)][(N - 1) % 8];
198  constexpr int R0 = 32 - L0;
199  constexpr int R1 = 32 - L1;
200  constexpr int R2 = 32 - L2;
201  constexpr int R3 = 32 - L3;
202 
203  std::get<0>(s) = _mm256_add_epi32(std::get<0>(s), std::get<1>(s));
204  std::get<2>(s) = _mm256_add_epi32(std::get<2>(s), std::get<3>(s));
205  std::get<4>(s) = _mm256_add_epi32(std::get<4>(s), std::get<5>(s));
206  std::get<6>(s) = _mm256_add_epi32(std::get<6>(s), std::get<7>(s));
207 
208  __m256i l0 = _mm256_slli_epi32(std::get<1>(s), L0);
209  __m256i l1 = _mm256_slli_epi32(std::get<3>(s), L1);
210  __m256i l2 = _mm256_slli_epi32(std::get<5>(s), L2);
211  __m256i l3 = _mm256_slli_epi32(std::get<7>(s), L3);
212 
213  __m256i r0 = _mm256_srli_epi32(std::get<1>(s), R0);
214  __m256i r1 = _mm256_srli_epi32(std::get<3>(s), R1);
215  __m256i r2 = _mm256_srli_epi32(std::get<5>(s), R2);
216  __m256i r3 = _mm256_srli_epi32(std::get<7>(s), R3);
217 
218  std::get<1>(s) = _mm256_or_si256(l0, r0);
219  std::get<3>(s) = _mm256_or_si256(l1, r1);
220  std::get<5>(s) = _mm256_or_si256(l2, r2);
221  std::get<7>(s) = _mm256_or_si256(l3, r3);
222 
223  std::get<1>(s) = _mm256_xor_si256(std::get<0>(s), std::get<1>(s));
224  std::get<3>(s) = _mm256_xor_si256(std::get<2>(s), std::get<3>(s));
225  std::get<5>(s) = _mm256_xor_si256(std::get<4>(s), std::get<5>(s));
226  std::get<7>(s) = _mm256_xor_si256(std::get<6>(s), std::get<7>(s));
227 
228  permute(s);
229  }
230 
231  template <std::size_t N>
232  static void rbox(std::array<__m256i, 16> &s, std::true_type)
233  {
234  constexpr int L0 = Constants::rotate::value[0 % (K / 2)][(N - 1) % 8];
235  constexpr int L1 = Constants::rotate::value[1 % (K / 2)][(N - 1) % 8];
236  constexpr int L2 = Constants::rotate::value[2 % (K / 2)][(N - 1) % 8];
237  constexpr int L3 = Constants::rotate::value[3 % (K / 2)][(N - 1) % 8];
238  constexpr int L4 = Constants::rotate::value[4 % (K / 2)][(N - 1) % 8];
239  constexpr int L5 = Constants::rotate::value[5 % (K / 2)][(N - 1) % 8];
240  constexpr int L6 = Constants::rotate::value[6 % (K / 2)][(N - 1) % 8];
241  constexpr int L7 = Constants::rotate::value[7 % (K / 2)][(N - 1) % 8];
242  constexpr int R0 = 32 - L0;
243  constexpr int R1 = 32 - L1;
244  constexpr int R2 = 32 - L2;
245  constexpr int R3 = 32 - L3;
246  constexpr int R4 = 32 - L4;
247  constexpr int R5 = 32 - L5;
248  constexpr int R6 = 32 - L6;
249  constexpr int R7 = 32 - L7;
250 
251  std::get<0x0>(s) =
252  _mm256_add_epi32(std::get<0x0>(s), std::get<0x1>(s));
253  std::get<0x2>(s) =
254  _mm256_add_epi32(std::get<0x2>(s), std::get<0x3>(s));
255  std::get<0x4>(s) =
256  _mm256_add_epi32(std::get<0x4>(s), std::get<0x5>(s));
257  std::get<0x6>(s) =
258  _mm256_add_epi32(std::get<0x6>(s), std::get<0x7>(s));
259  std::get<0x8>(s) =
260  _mm256_add_epi32(std::get<0x8>(s), std::get<0x9>(s));
261  std::get<0xA>(s) =
262  _mm256_add_epi32(std::get<0xA>(s), std::get<0xB>(s));
263  std::get<0xC>(s) =
264  _mm256_add_epi32(std::get<0xC>(s), std::get<0xD>(s));
265  std::get<0xE>(s) =
266  _mm256_add_epi32(std::get<0xE>(s), std::get<0xF>(s));
267 
268  __m256i l0 = _mm256_slli_epi32(std::get<0x1>(s), L0);
269  __m256i l1 = _mm256_slli_epi32(std::get<0x3>(s), L1);
270  __m256i l2 = _mm256_slli_epi32(std::get<0x5>(s), L2);
271  __m256i l3 = _mm256_slli_epi32(std::get<0x7>(s), L3);
272  __m256i l4 = _mm256_slli_epi32(std::get<0x9>(s), L4);
273  __m256i l5 = _mm256_slli_epi32(std::get<0xB>(s), L5);
274  __m256i l6 = _mm256_slli_epi32(std::get<0xD>(s), L6);
275  __m256i l7 = _mm256_slli_epi32(std::get<0xF>(s), L7);
276 
277  std::get<0x1>(s) = _mm256_srli_epi32(std::get<0x1>(s), R0);
278  std::get<0x3>(s) = _mm256_srli_epi32(std::get<0x3>(s), R1);
279  std::get<0x5>(s) = _mm256_srli_epi32(std::get<0x5>(s), R2);
280  std::get<0x7>(s) = _mm256_srli_epi32(std::get<0x7>(s), R3);
281  std::get<0x9>(s) = _mm256_srli_epi32(std::get<0x9>(s), R4);
282  std::get<0xB>(s) = _mm256_srli_epi32(std::get<0xB>(s), R5);
283  std::get<0xD>(s) = _mm256_srli_epi32(std::get<0xD>(s), R6);
284  std::get<0xF>(s) = _mm256_srli_epi32(std::get<0xF>(s), R7);
285 
286  std::get<0x1>(s) = _mm256_or_si256(l0, std::get<0x1>(s));
287  std::get<0x3>(s) = _mm256_or_si256(l1, std::get<0x3>(s));
288  std::get<0x5>(s) = _mm256_or_si256(l2, std::get<0x5>(s));
289  std::get<0x7>(s) = _mm256_or_si256(l3, std::get<0x7>(s));
290  std::get<0x9>(s) = _mm256_or_si256(l4, std::get<0x9>(s));
291  std::get<0xB>(s) = _mm256_or_si256(l5, std::get<0xB>(s));
292  std::get<0xD>(s) = _mm256_or_si256(l6, std::get<0xD>(s));
293  std::get<0xF>(s) = _mm256_or_si256(l7, std::get<0xF>(s));
294 
295  std::get<0x1>(s) =
296  _mm256_xor_si256(std::get<0x0>(s), std::get<0x1>(s));
297  std::get<0x3>(s) =
298  _mm256_xor_si256(std::get<0x2>(s), std::get<0x3>(s));
299  std::get<0x5>(s) =
300  _mm256_xor_si256(std::get<0x4>(s), std::get<0x5>(s));
301  std::get<0x7>(s) =
302  _mm256_xor_si256(std::get<0x6>(s), std::get<0x7>(s));
303  std::get<0x9>(s) =
304  _mm256_xor_si256(std::get<0x8>(s), std::get<0x9>(s));
305  std::get<0xB>(s) =
306  _mm256_xor_si256(std::get<0xA>(s), std::get<0xB>(s));
307  std::get<0xD>(s) =
308  _mm256_xor_si256(std::get<0xC>(s), std::get<0xD>(s));
309  std::get<0xF>(s) =
310  _mm256_xor_si256(std::get<0xE>(s), std::get<0xF>(s));
311 
312  permute(s);
313  }
314 
315  template <std::size_t N>
316  static void set_key(std::array<__m256i, 2> &k, const std::array<T, 6> &par)
317  {
318  std::get<0>(k) = _mm256_set1_epi32(
319  static_cast<int>(ThreefryKBox<T, 2, N>::template key<0>(par)));
320  std::get<1>(k) = _mm256_set1_epi32(
321  static_cast<int>(ThreefryKBox<T, 2, N>::template key<1>(par)));
322  }
323 
324  template <std::size_t N>
325  static void set_key(std::array<__m256i, 4> &k, const std::array<T, 8> &par)
326  {
327  std::get<0>(k) = _mm256_set1_epi32(
328  static_cast<int>(ThreefryKBox<T, 4, N>::template key<0>(par)));
329  std::get<1>(k) = _mm256_set1_epi32(
330  static_cast<int>(ThreefryKBox<T, 4, N>::template key<1>(par)));
331  std::get<2>(k) = _mm256_set1_epi32(
332  static_cast<int>(ThreefryKBox<T, 4, N>::template key<2>(par)));
333  std::get<3>(k) = _mm256_set1_epi32(
334  static_cast<int>(ThreefryKBox<T, 4, N>::template key<3>(par)));
335  }
336 
337  template <std::size_t N>
338  static void set_key(
339  std::array<__m256i, 8> &k, const std::array<T, 12> &par)
340  {
341  std::get<0>(k) = _mm256_set1_epi32(
342  static_cast<int>(ThreefryKBox<T, 8, N>::template key<0>(par)));
343  std::get<1>(k) = _mm256_set1_epi32(
344  static_cast<int>(ThreefryKBox<T, 8, N>::template key<1>(par)));
345  std::get<2>(k) = _mm256_set1_epi32(
346  static_cast<int>(ThreefryKBox<T, 8, N>::template key<2>(par)));
347  std::get<3>(k) = _mm256_set1_epi32(
348  static_cast<int>(ThreefryKBox<T, 8, N>::template key<3>(par)));
349  std::get<4>(k) = _mm256_set1_epi32(
350  static_cast<int>(ThreefryKBox<T, 8, N>::template key<4>(par)));
351  std::get<5>(k) = _mm256_set1_epi32(
352  static_cast<int>(ThreefryKBox<T, 8, N>::template key<5>(par)));
353  std::get<6>(k) = _mm256_set1_epi32(
354  static_cast<int>(ThreefryKBox<T, 8, N>::template key<6>(par)));
355  std::get<7>(k) = _mm256_set1_epi32(
356  static_cast<int>(ThreefryKBox<T, 8, N>::template key<7>(par)));
357  }
358 
359  template <std::size_t N>
360  static void set_key(
361  std::array<__m256i, 16> &k, const std::array<T, 20> &par)
362  {
363  std::get<0x0>(k) = _mm256_set1_epi32(
364  static_cast<int>(ThreefryKBox<T, 16, N>::template key<0x0>(par)));
365  std::get<0x1>(k) = _mm256_set1_epi32(
366  static_cast<int>(ThreefryKBox<T, 16, N>::template key<0x1>(par)));
367  std::get<0x2>(k) = _mm256_set1_epi32(
368  static_cast<int>(ThreefryKBox<T, 16, N>::template key<0x2>(par)));
369  std::get<0x3>(k) = _mm256_set1_epi32(
370  static_cast<int>(ThreefryKBox<T, 16, N>::template key<0x3>(par)));
371  std::get<0x4>(k) = _mm256_set1_epi32(
372  static_cast<int>(ThreefryKBox<T, 16, N>::template key<0x4>(par)));
373  std::get<0x5>(k) = _mm256_set1_epi32(
374  static_cast<int>(ThreefryKBox<T, 16, N>::template key<0x5>(par)));
375  std::get<0x6>(k) = _mm256_set1_epi32(
376  static_cast<int>(ThreefryKBox<T, 16, N>::template key<0x6>(par)));
377  std::get<0x7>(k) = _mm256_set1_epi32(
378  static_cast<int>(ThreefryKBox<T, 16, N>::template key<0x7>(par)));
379  std::get<0x8>(k) = _mm256_set1_epi32(
380  static_cast<int>(ThreefryKBox<T, 16, N>::template key<0x8>(par)));
381  std::get<0x9>(k) = _mm256_set1_epi32(
382  static_cast<int>(ThreefryKBox<T, 16, N>::template key<0x9>(par)));
383  std::get<0xA>(k) = _mm256_set1_epi32(
384  static_cast<int>(ThreefryKBox<T, 16, N>::template key<0xA>(par)));
385  std::get<0xB>(k) = _mm256_set1_epi32(
386  static_cast<int>(ThreefryKBox<T, 16, N>::template key<0xB>(par)));
387  std::get<0xC>(k) = _mm256_set1_epi32(
388  static_cast<int>(ThreefryKBox<T, 16, N>::template key<0xC>(par)));
389  std::get<0xD>(k) = _mm256_set1_epi32(
390  static_cast<int>(ThreefryKBox<T, 16, N>::template key<0xD>(par)));
391  std::get<0xE>(k) = _mm256_set1_epi32(
392  static_cast<int>(ThreefryKBox<T, 16, N>::template key<0xE>(par)));
393  std::get<0xF>(k) = _mm256_set1_epi32(
394  static_cast<int>(ThreefryKBox<T, 16, N>::template key<0xF>(par)));
395  }
396 
397  template <std::size_t S>
398  static void permute(std::array<__m256i, S> &s)
399  {
400  permute<0>(s, std::integral_constant<bool, 0 < S / K>());
401  }
402 
403  template <std::size_t, std::size_t S>
404  static void permute(std::array<__m256i, S> &, std::false_type)
405  {
406  }
407 
408  template <std::size_t I, std::size_t S>
409  static void permute(std::array<__m256i, S> &s, std::true_type)
410  {
412  permute<I + 1>(s, std::integral_constant<bool, I + 1 < S / K>());
413  }
414 }; // class ThreefryGeneratorAVX2Impl32
415 
416 template <typename T>
419 {
420 }; // class ThreefryGeneratorAVX2Impl32
421 
422 template <typename T>
425 {
426 }; // class ThreefryGeneratorAVX2Impl32
427 
428 } // namespace internal
429 
430 } // namespace mckl
431 
433 
434 #endif // MCKL_RANDOM_INTERNAL_THREEFRY_AVX2_32_HPP
#define MCKL_PUSH_GCC_WARNING(warning)
Definition: compiler.h:78
void round(std::size_t n, const float *a, float *y)
Definition: vmf.hpp:310
#define MCKL_INLINE_CALL
Definition: intel.h:142
typename internal::CounterImpl< T, K >::type Counter
A counter type with the same width as std::array<T, K> but with possibly fewer elements.
Definition: increment.hpp:104
static void eval(const void *plain, void *cipher, const std::array< T, K+4 > &par)
#define MCKL_RANDOM_INTERNAL_THREEFRY_UNROLL_ROUND(N, s, par)
Default Threefry constants.
static void eval(Counter< T, K > &ctr, std::size_t n, ResultType *r, const std::array< T, K+4 > &par)
Definition: mcmc.hpp:40
static void eval(Counter< T, K > &ctr, ResultType *r, const std::array< T, K+4 > &par)
#define MCKL_ALIGNMENT
The default alignment for scalar type.
Definition: config.h:187
#define MCKL_INLINE
Definition: clang.h:147
#define MCKL_POP_GCC_WARNING
Definition: compiler.h:79
void increment_si256(std::array< T, K > &ctr, std::array< __m256i, S > &s, std::false_type)