MCKL
Monte Carlo Kernel Library
u01_avx512.hpp
Go to the documentation of this file.
1 //============================================================================
2 // MCKL/include/mckl/random/internal/u01_avx512.hpp
3 //----------------------------------------------------------------------------
4 // MCKL: Monte Carlo Kernel Library
5 //----------------------------------------------------------------------------
6 // Copyright (c) 2013-2018, Yan Zhou
7 // All rights reserved.
8 //
9 // Redistribution and use in source and binary forms, with or without
10 // modification, are permitted provided that the following conditions are met:
11 //
12 // Redistributions of source code must retain the above copyright notice,
13 // this list of conditions and the following disclaimer.
14 //
15 // Redistributions in binary form must reproduce the above copyright notice,
16 // this list of conditions and the following disclaimer in the documentation
17 // and/or other materials provided with the distribution.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS AS IS
20 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
23 // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 // POSSIBILITY OF SUCH DAMAGE.
30 //============================================================================
31 
32 #ifndef MCKL_RANDOM_INTERNAL_U01_AVX512_HPP
33 #define MCKL_RANDOM_INTERNAL_U01_AVX512_HPP
34 
35 #include <mckl/random/internal/common.hpp>
36 #include <mckl/random/internal/u01_generic.hpp>
37 
38 MCKL_PUSH_GCC_WARNING("-Wignored-attributes")
39 
40 #define MCKL_RANDOM_INTERNAL_U01_AVX512_LOADU_SI512(u) \
41  const __m512i *uptr = reinterpret_cast<const __m512i *>(u); \
42  __m512i zmm0 = _mm512_loadu_si512(uptr++); \
43  __m512i zmm1 = _mm512_loadu_si512(uptr++); \
44  __m512i zmm2 = _mm512_loadu_si512(uptr++); \
45  __m512i zmm3 = _mm512_loadu_si512(uptr++);
46 
47 #define MCKL_RANDOM_INTERNAL_U01_AVX512_LOADU_SI512_CVTEPU32_EPI64(u) \
48  const __m256i *uptr = reinterpret_cast<const __m256i *>(u); \
49  __m512i zmm0 = _mm512_cvtepu32_epi64(_mm256_loadu_si256(uptr++)); \
50  __m512i zmm1 = _mm512_cvtepu32_epi64(_mm256_loadu_si256(uptr++)); \
51  __m512i zmm2 = _mm512_cvtepu32_epi64(_mm256_loadu_si256(uptr++)); \
52  __m512i zmm3 = _mm512_cvtepu32_epi64(_mm256_loadu_si256(uptr++));
53 
54 #define MCKL_RANDOM_INTERNAL_U01_AVX512_STOREU_PS(r) \
55  _mm512_storeu_ps(r + 0x00, _mm512_castsi512_ps(zmm0)); \
56  _mm512_storeu_ps(r + 0x10, _mm512_castsi512_ps(zmm1)); \
57  _mm512_storeu_ps(r + 0x20, _mm512_castsi512_ps(zmm2)); \
58  _mm512_storeu_ps(r + 0x30, _mm512_castsi512_ps(zmm3));
59 
60 #define MCKL_RANDOM_INTERNAL_U01_AVX512_STOREU_PD(r) \
61  _mm512_storeu_pd(r + 0x00, _mm512_castsi512_pd(zmm0)); \
62  _mm512_storeu_pd(r + 0x08, _mm512_castsi512_pd(zmm1)); \
63  _mm512_storeu_pd(r + 0x10, _mm512_castsi512_pd(zmm2)); \
64  _mm512_storeu_pd(r + 0x18, _mm512_castsi512_pd(zmm3));
65 
66 #define MCKL_RANDOM_INTERNAL_U01_AVX512_AND1ADD_EPI32 \
67  const __m512i mask = _mm512_set1_epi32(1); \
68  zmm0 = _mm512_add_epi32(zmm0, _mm512_and_si512(zmm0, mask)); \
69  zmm1 = _mm512_add_epi32(zmm1, _mm512_and_si512(zmm1, mask)); \
70  zmm2 = _mm512_add_epi32(zmm2, _mm512_and_si512(zmm2, mask)); \
71  zmm3 = _mm512_add_epi32(zmm3, _mm512_and_si512(zmm3, mask));
72 
73 #define MCKL_RANDOM_INTERNAL_U01_AVX512_AND1ADD_EPI64 \
74  const __m512i mask = _mm512_set1_epi64(1); \
75  zmm0 = _mm512_add_epi64(zmm0, _mm512_and_si512(zmm0, mask)); \
76  zmm1 = _mm512_add_epi64(zmm1, _mm512_and_si512(zmm1, mask)); \
77  zmm2 = _mm512_add_epi64(zmm2, _mm512_and_si512(zmm2, mask)); \
78  zmm3 = _mm512_add_epi64(zmm3, _mm512_and_si512(zmm3, mask));
79 
80 #define MCKL_RANDOM_INTERNAL_U01_AVX512_SLLI_EPI32(imm8) \
81  zmm0 = _mm512_slli_epi32(zmm0, imm8); \
82  zmm1 = _mm512_slli_epi32(zmm1, imm8); \
83  zmm2 = _mm512_slli_epi32(zmm2, imm8); \
84  zmm3 = _mm512_slli_epi32(zmm3, imm8);
85 
86 #define MCKL_RANDOM_INTERNAL_U01_AVX512_SRLI_EPI32(imm8) \
87  zmm0 = _mm512_srli_epi32(zmm0, imm8); \
88  zmm1 = _mm512_srli_epi32(zmm1, imm8); \
89  zmm2 = _mm512_srli_epi32(zmm2, imm8); \
90  zmm3 = _mm512_srli_epi32(zmm3, imm8);
91 
92 #define MCKL_RANDOM_INTERNAL_U01_AVX512_SLLI_EPI64(imm8) \
93  zmm0 = _mm512_slli_epi64(zmm0, imm8); \
94  zmm1 = _mm512_slli_epi64(zmm1, imm8); \
95  zmm2 = _mm512_slli_epi64(zmm2, imm8); \
96  zmm3 = _mm512_slli_epi64(zmm3, imm8);
97 
98 #define MCKL_RANDOM_INTERNAL_U01_AVX512_SRLI_EPI64(imm8) \
99  zmm0 = _mm512_srli_epi64(zmm0, imm8); \
100  zmm1 = _mm512_srli_epi64(zmm1, imm8); \
101  zmm2 = _mm512_srli_epi64(zmm2, imm8); \
102  zmm3 = _mm512_srli_epi64(zmm3, imm8);
103 
104 #define MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU32_PS_31 \
105  zmm0 = _mm512_castps_si512(_mm512_cvtepi32_ps(zmm0)); \
106  zmm1 = _mm512_castps_si512(_mm512_cvtepi32_ps(zmm1)); \
107  zmm2 = _mm512_castps_si512(_mm512_cvtepi32_ps(zmm2)); \
108  zmm3 = _mm512_castps_si512(_mm512_cvtepi32_ps(zmm3));
109 
110 #define MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU32_PS \
111  const __m512 c = _mm512_set1_ps(Pow2<float, 23>::value); \
112  const __m512 d = _mm512_set1_ps(Pow2<float, 16>::value); \
113  const __m512i l = _mm512_set1_epi32(static_cast<int>(0xFFFF)); \
114  __m512i xmm0 = _mm512_and_si512(zmm0, l); \
115  __m512i xmm1 = _mm512_and_si512(zmm1, l); \
116  __m512i xmm2 = _mm512_and_si512(zmm2, l); \
117  __m512i xmm3 = _mm512_and_si512(zmm3, l); \
118  MCKL_RANDOM_INTERNAL_U01_AVX512_SRLI_EPI32(16); \
119  zmm0 = _mm512_add_epi32(zmm0, _mm512_castps_si512(c)); \
120  zmm1 = _mm512_add_epi32(zmm1, _mm512_castps_si512(c)); \
121  zmm2 = _mm512_add_epi32(zmm2, _mm512_castps_si512(c)); \
122  zmm3 = _mm512_add_epi32(zmm3, _mm512_castps_si512(c)); \
123  xmm0 = _mm512_add_epi32(xmm0, _mm512_castps_si512(c)); \
124  xmm1 = _mm512_add_epi32(xmm1, _mm512_castps_si512(c)); \
125  xmm2 = _mm512_add_epi32(xmm2, _mm512_castps_si512(c)); \
126  xmm3 = _mm512_add_epi32(xmm3, _mm512_castps_si512(c)); \
127  zmm0 = _mm512_castps_si512(_mm512_sub_ps(_mm512_castsi512_ps(zmm0), c)); \
128  zmm1 = _mm512_castps_si512(_mm512_sub_ps(_mm512_castsi512_ps(zmm1), c)); \
129  zmm2 = _mm512_castps_si512(_mm512_sub_ps(_mm512_castsi512_ps(zmm2), c)); \
130  zmm3 = _mm512_castps_si512(_mm512_sub_ps(_mm512_castsi512_ps(zmm3), c)); \
131  xmm0 = _mm512_castps_si512(_mm512_sub_ps(_mm512_castsi512_ps(xmm0), c)); \
132  xmm1 = _mm512_castps_si512(_mm512_sub_ps(_mm512_castsi512_ps(xmm1), c)); \
133  xmm2 = _mm512_castps_si512(_mm512_sub_ps(_mm512_castsi512_ps(xmm2), c)); \
134  xmm3 = _mm512_castps_si512(_mm512_sub_ps(_mm512_castsi512_ps(xmm3), c)); \
135  zmm0 = _mm512_castps_si512(_mm512_fmadd_ps( \
136  _mm512_castsi512_ps(zmm0), d, _mm512_castsi512_ps(xmm0))); \
137  zmm1 = _mm512_castps_si512(_mm512_fmadd_ps( \
138  _mm512_castsi512_ps(zmm1), d, _mm512_castsi512_ps(xmm1))); \
139  zmm2 = _mm512_castps_si512(_mm512_fmadd_ps( \
140  _mm512_castsi512_ps(zmm2), d, _mm512_castsi512_ps(xmm2))); \
141  zmm3 = _mm512_castps_si512(_mm512_fmadd_ps( \
142  _mm512_castsi512_ps(zmm3), d, _mm512_castsi512_ps(xmm3)));
143 
144 #define MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU64_PD_52 \
145  const __m512d c = _mm512_set1_pd(Pow2<double, 52>::value); \
146  zmm0 = _mm512_add_epi64(zmm0, _mm512_castpd_si512(c)); \
147  zmm1 = _mm512_add_epi64(zmm1, _mm512_castpd_si512(c)); \
148  zmm2 = _mm512_add_epi64(zmm2, _mm512_castpd_si512(c)); \
149  zmm3 = _mm512_add_epi64(zmm3, _mm512_castpd_si512(c)); \
150  zmm0 = _mm512_castpd_si512(_mm512_sub_pd(_mm512_castsi512_pd(zmm0), c)); \
151  zmm1 = _mm512_castpd_si512(_mm512_sub_pd(_mm512_castsi512_pd(zmm1), c)); \
152  zmm2 = _mm512_castpd_si512(_mm512_sub_pd(_mm512_castsi512_pd(zmm2), c)); \
153  zmm3 = _mm512_castpd_si512(_mm512_sub_pd(_mm512_castsi512_pd(zmm3), c));
154 
155 #define MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU64_PD \
156  const __m512d c = _mm512_set1_pd(Pow2<double, 52>::value); \
157  const __m512d d = _mm512_set1_pd(Pow2<double, 32>::value); \
158  const __m512i l = _mm512_set1_epi64(static_cast<MCKL_INT64>(0xFFFFFFFF)); \
159  __m512i xmm0 = _mm512_and_si512(zmm0, l); \
160  __m512i xmm1 = _mm512_and_si512(zmm1, l); \
161  __m512i xmm2 = _mm512_and_si512(zmm2, l); \
162  __m512i xmm3 = _mm512_and_si512(zmm3, l); \
163  MCKL_RANDOM_INTERNAL_U01_AVX512_SRLI_EPI64(32); \
164  zmm0 = _mm512_add_epi64(zmm0, _mm512_castpd_si512(c)); \
165  zmm1 = _mm512_add_epi64(zmm1, _mm512_castpd_si512(c)); \
166  zmm2 = _mm512_add_epi64(zmm2, _mm512_castpd_si512(c)); \
167  zmm3 = _mm512_add_epi64(zmm3, _mm512_castpd_si512(c)); \
168  xmm0 = _mm512_add_epi64(xmm0, _mm512_castpd_si512(c)); \
169  xmm1 = _mm512_add_epi64(xmm1, _mm512_castpd_si512(c)); \
170  xmm2 = _mm512_add_epi64(xmm2, _mm512_castpd_si512(c)); \
171  xmm3 = _mm512_add_epi64(xmm3, _mm512_castpd_si512(c)); \
172  zmm0 = _mm512_castpd_si512(_mm512_sub_pd(_mm512_castsi512_pd(zmm0), c)); \
173  zmm1 = _mm512_castpd_si512(_mm512_sub_pd(_mm512_castsi512_pd(zmm1), c)); \
174  zmm2 = _mm512_castpd_si512(_mm512_sub_pd(_mm512_castsi512_pd(zmm2), c)); \
175  zmm3 = _mm512_castpd_si512(_mm512_sub_pd(_mm512_castsi512_pd(zmm3), c)); \
176  xmm0 = _mm512_castpd_si512(_mm512_sub_pd(_mm512_castsi512_pd(xmm0), c)); \
177  xmm1 = _mm512_castpd_si512(_mm512_sub_pd(_mm512_castsi512_pd(xmm1), c)); \
178  xmm2 = _mm512_castpd_si512(_mm512_sub_pd(_mm512_castsi512_pd(xmm2), c)); \
179  xmm3 = _mm512_castpd_si512(_mm512_sub_pd(_mm512_castsi512_pd(xmm3), c)); \
180  zmm0 = _mm512_castpd_si512(_mm512_fmadd_pd( \
181  _mm512_castsi512_pd(zmm0), d, _mm512_castsi512_pd(xmm0))); \
182  zmm1 = _mm512_castpd_si512(_mm512_fmadd_pd( \
183  _mm512_castsi512_pd(zmm1), d, _mm512_castsi512_pd(xmm1))); \
184  zmm2 = _mm512_castpd_si512(_mm512_fmadd_pd( \
185  _mm512_castsi512_pd(zmm2), d, _mm512_castsi512_pd(xmm2))); \
186  zmm3 = _mm512_castpd_si512(_mm512_fmadd_pd( \
187  _mm512_castsi512_pd(zmm3), d, _mm512_castsi512_pd(xmm3)));
188 
189 #define MCKL_RANDOM_INTERNAL_U01_AVX512_MUL_PS(a) \
190  zmm0 = _mm512_castps_si512(_mm512_mul_ps(_mm512_castsi512_ps(zmm0), a)); \
191  zmm1 = _mm512_castps_si512(_mm512_mul_ps(_mm512_castsi512_ps(zmm1), a)); \
192  zmm2 = _mm512_castps_si512(_mm512_mul_ps(_mm512_castsi512_ps(zmm2), a)); \
193  zmm3 = _mm512_castps_si512(_mm512_mul_ps(_mm512_castsi512_ps(zmm3), a));
194 
195 #define MCKL_RANDOM_INTERNAL_U01_AVX512_MUL_PD(a) \
196  zmm0 = _mm512_castpd_si512(_mm512_mul_pd(_mm512_castsi512_pd(zmm0), a)); \
197  zmm1 = _mm512_castpd_si512(_mm512_mul_pd(_mm512_castsi512_pd(zmm1), a)); \
198  zmm2 = _mm512_castpd_si512(_mm512_mul_pd(_mm512_castsi512_pd(zmm2), a)); \
199  zmm3 = _mm512_castpd_si512(_mm512_mul_pd(_mm512_castsi512_pd(zmm3), a));
200 
201 #define MCKL_RANDOM_INTERNAL_U01_AVX512_FMADD_PS(a, b) \
202  zmm0 = _mm512_castps_si512( \
203  _mm512_fmadd_ps(_mm512_castsi512_ps(zmm0), a, b)); \
204  zmm1 = _mm512_castps_si512( \
205  _mm512_fmadd_ps(_mm512_castsi512_ps(zmm1), a, b)); \
206  zmm2 = _mm512_castps_si512( \
207  _mm512_fmadd_ps(_mm512_castsi512_ps(zmm2), a, b)); \
208  zmm3 = _mm512_castps_si512( \
209  _mm512_fmadd_ps(_mm512_castsi512_ps(zmm3), a, b));
210 
211 #define MCKL_RANDOM_INTERNAL_U01_AVX512_FMADD_PD(a, b) \
212  zmm0 = _mm512_castpd_si512( \
213  _mm512_fmadd_pd(_mm512_castsi512_pd(zmm0), a, b)); \
214  zmm1 = _mm512_castpd_si512( \
215  _mm512_fmadd_pd(_mm512_castsi512_pd(zmm1), a, b)); \
216  zmm2 = _mm512_castpd_si512( \
217  _mm512_fmadd_pd(_mm512_castsi512_pd(zmm2), a, b)); \
218  zmm3 = _mm512_castpd_si512( \
219  _mm512_fmadd_pd(_mm512_castsi512_pd(zmm3), a, b));
220 
221 namespace mckl {
222 
223 namespace internal {
224 
225 template <typename UIntType, typename RealType, typename Lower, typename Upper,
226  int = std::numeric_limits<UIntType>::digits>
227 class U01AVX512Impl : public U01GenericImpl<UIntType, RealType, Lower, Upper>
228 {
229 }; // class U01AVX512Impl
230 
231 template <typename UIntType, typename RealType, typename Lower, typename Upper>
232 class U01AVX512ImplBase
233 {
234  public:
235  MCKL_INLINE static RealType eval(UIntType u)
236  {
237  return U01GenericImpl<UIntType, RealType, Lower, Upper>::eval(u);
238  }
239 
240  MCKL_INLINE static void eval(std::size_t n, const UIntType *u, RealType *r)
241  {
242  constexpr std::size_t S = 4;
243  constexpr std::size_t N = sizeof(__m512i) * S / sizeof(RealType);
244 
245  while (n >= N) {
246  U01AVX512Impl<UIntType, RealType, Lower, Upper>::eval(u, r);
247  n -= N;
248  u += N;
249  r += N;
250  }
251 
252  for (std::size_t i = 0; i != n; ++i) {
253  r[i] = eval(u[i]);
254  }
255  }
256 }; // class U01AVX512ImplBase
257 
258 template <typename UIntType, typename RealType, int Q,
259  int = std::numeric_limits<UIntType>::digits>
260 class U01CanonicalAVX512Impl
261  : public U01CanonicalGenericImpl<UIntType, RealType, Q>
262 {
263 }; // class U01CanonicalAVX512Impl
264 
265 template <typename UIntType, typename RealType, int Q>
266 class U01CanonicalAVX512ImplBase
267 {
268  public:
269  MCKL_INLINE static RealType eval(const UIntType *u)
270  {
271  return U01CanonicalGenericImpl<UIntType, RealType, Q>::eval(u);
272  }
273 
274  MCKL_INLINE static void eval(std::size_t n, const UIntType *u, RealType *r)
275  {
276  constexpr std::size_t S = 4;
277  constexpr std::size_t N = sizeof(__m512i) * S / sizeof(RealType);
278 
279  while (n >= N) {
280  U01CanonicalAVX512Impl<UIntType, RealType, Q>::eval(u, r);
281  n -= N;
282  u += N * Q;
283  r += N;
284  }
285 
286  for (std::size_t i = 0; i != n; ++i, u += Q) {
287  r[i] = eval(u);
288  }
289  }
290 }; // class U01CanonicalCanonicalAVX512ImplBase
291 
292 template <typename UIntType>
293 class U01AVX512Impl<UIntType, float, Closed, Closed, 32>
294  : public U01AVX512ImplBase<UIntType, float, Closed, Closed>
295 {
296  public:
297  using U01AVX512ImplBase<UIntType, float, Closed, Closed>::eval;
298 
299  private:
300  friend U01AVX512ImplBase<UIntType, float, Closed, Closed>;
301 
302  MCKL_INLINE static void eval(const UIntType *u, float *r)
303  {
304  const __m512 d25 = _mm512_set1_ps(Pow2<float, -25>::value);
305 
306  MCKL_RANDOM_INTERNAL_U01_AVX512_LOADU_SI512(u)
307  MCKL_RANDOM_INTERNAL_U01_AVX512_SLLI_EPI32(1)
308  MCKL_RANDOM_INTERNAL_U01_AVX512_SRLI_EPI32(7)
309  MCKL_RANDOM_INTERNAL_U01_AVX512_AND1ADD_EPI32
310  MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU32_PS_31
311  MCKL_RANDOM_INTERNAL_U01_AVX512_MUL_PS(d25)
312  MCKL_RANDOM_INTERNAL_U01_AVX512_STOREU_PS(r)
313  }
314 }; // class U01AVX512Impl
315 
316 template <typename UIntType>
317 class U01AVX512Impl<UIntType, float, Closed, Open, 32>
318  : public U01AVX512ImplBase<UIntType, float, Closed, Open>
319 {
320  public:
321  using U01AVX512ImplBase<UIntType, float, Closed, Open>::eval;
322 
323  private:
324  friend U01AVX512ImplBase<UIntType, float, Closed, Open>;
325 
326  MCKL_INLINE static void eval(const UIntType *u, float *r)
327  {
328  const __m512 d24 = _mm512_set1_ps(Pow2<float, -24>::value);
329 
330  MCKL_RANDOM_INTERNAL_U01_AVX512_LOADU_SI512(u)
331  MCKL_RANDOM_INTERNAL_U01_AVX512_SRLI_EPI32(8)
332  MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU32_PS_31
333  MCKL_RANDOM_INTERNAL_U01_AVX512_MUL_PS(d24)
334  MCKL_RANDOM_INTERNAL_U01_AVX512_STOREU_PS(r)
335  }
336 }; // class U01AVX512Impl
337 
338 template <typename UIntType>
339 class U01AVX512Impl<UIntType, float, Open, Closed, 32>
340  : public U01AVX512ImplBase<UIntType, float, Open, Closed>
341 {
342  public:
343  using U01AVX512ImplBase<UIntType, float, Open, Closed>::eval;
344 
345  private:
346  friend U01AVX512ImplBase<UIntType, float, Open, Closed>;
347 
348  MCKL_INLINE static void eval(const UIntType *u, float *r)
349  {
350  const __m512 d24 = _mm512_set1_ps(Pow2<float, -24>::value);
351 
352  MCKL_RANDOM_INTERNAL_U01_AVX512_LOADU_SI512(u)
353  MCKL_RANDOM_INTERNAL_U01_AVX512_SRLI_EPI32(8)
354  MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU32_PS_31
355  MCKL_RANDOM_INTERNAL_U01_AVX512_FMADD_PS(d24, d24)
356  MCKL_RANDOM_INTERNAL_U01_AVX512_STOREU_PS(r)
357  }
358 }; // class U01AVX512Impl
359 
360 template <typename UIntType>
361 class U01AVX512Impl<UIntType, float, Open, Open, 32>
362  : public U01AVX512ImplBase<UIntType, float, Open, Open>
363 {
364  public:
365  using U01AVX512ImplBase<UIntType, float, Open, Open>::eval;
366 
367  private:
368  friend U01AVX512ImplBase<UIntType, float, Open, Open>;
369 
370  MCKL_INLINE static void eval(const UIntType *u, float *r)
371  {
372  const __m512 d23 = _mm512_set1_ps(Pow2<float, -23>::value);
373  const __m512 d24 = _mm512_set1_ps(Pow2<float, -24>::value);
374 
375  MCKL_RANDOM_INTERNAL_U01_AVX512_LOADU_SI512(u)
376  MCKL_RANDOM_INTERNAL_U01_AVX512_SRLI_EPI32(9)
377  MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU32_PS_31
378  MCKL_RANDOM_INTERNAL_U01_AVX512_FMADD_PS(d23, d24)
379  MCKL_RANDOM_INTERNAL_U01_AVX512_STOREU_PS(r)
380  }
381 }; // class U01AVX512Impl
382 
383 template <typename UIntType>
384 class U01AVX512Impl<UIntType, double, Closed, Closed, 32>
385  : public U01AVX512ImplBase<UIntType, double, Closed, Closed>
386 {
387  public:
388  using U01AVX512ImplBase<UIntType, double, Closed, Closed>::eval;
389 
390  private:
391  friend U01AVX512ImplBase<UIntType, double, Closed, Closed>;
392 
393  MCKL_INLINE static void eval(const UIntType *u, double *r)
394  {
395  const __m512d d32 = _mm512_set1_pd(Pow2<double, -32>::value);
396 
397  MCKL_RANDOM_INTERNAL_U01_AVX512_LOADU_SI512_CVTEPU32_EPI64(u)
398  MCKL_RANDOM_INTERNAL_U01_AVX512_AND1ADD_EPI64
399  MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU64_PD_52
400  MCKL_RANDOM_INTERNAL_U01_AVX512_MUL_PD(d32)
401  MCKL_RANDOM_INTERNAL_U01_AVX512_STOREU_PD(r)
402  }
403 }; // class U01AVX512Impl
404 
405 template <typename UIntType>
406 class U01AVX512Impl<UIntType, double, Closed, Open, 32>
407  : public U01AVX512ImplBase<UIntType, double, Closed, Open>
408 {
409  public:
410  using U01AVX512ImplBase<UIntType, double, Closed, Open>::eval;
411 
412  private:
413  friend U01AVX512ImplBase<UIntType, double, Closed, Open>;
414 
415  MCKL_INLINE static void eval(const UIntType *u, double *r)
416  {
417  const __m512d d32 = _mm512_set1_pd(Pow2<double, -32>::value);
418 
419  MCKL_RANDOM_INTERNAL_U01_AVX512_LOADU_SI512_CVTEPU32_EPI64(u)
420  MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU64_PD_52
421  MCKL_RANDOM_INTERNAL_U01_AVX512_MUL_PD(d32)
422  MCKL_RANDOM_INTERNAL_U01_AVX512_STOREU_PD(r)
423  }
424 }; // class U01AVX512Impl
425 
426 template <typename UIntType>
427 class U01AVX512Impl<UIntType, double, Open, Closed, 32>
428  : public U01AVX512ImplBase<UIntType, double, Open, Closed>
429 {
430  public:
431  using U01AVX512ImplBase<UIntType, double, Open, Closed>::eval;
432 
433  private:
434  friend U01AVX512ImplBase<UIntType, double, Open, Closed>;
435 
436  MCKL_INLINE static void eval(const UIntType *u, double *r)
437  {
438  const __m512d d32 = _mm512_set1_pd(Pow2<double, -32>::value);
439 
440  MCKL_RANDOM_INTERNAL_U01_AVX512_LOADU_SI512_CVTEPU32_EPI64(u)
441  MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU64_PD_52
442  MCKL_RANDOM_INTERNAL_U01_AVX512_FMADD_PD(d32, d32)
443  MCKL_RANDOM_INTERNAL_U01_AVX512_STOREU_PD(r)
444  }
445 }; // class U01AVX512Impl
446 
447 template <typename UIntType>
448 class U01AVX512Impl<UIntType, double, Open, Open, 32>
449  : public U01AVX512ImplBase<UIntType, double, Open, Open>
450 {
451  public:
452  using U01AVX512ImplBase<UIntType, double, Open, Open>::eval;
453 
454  private:
455  friend U01AVX512ImplBase<UIntType, double, Open, Open>;
456 
457  MCKL_INLINE static void eval(const UIntType *u, double *r)
458  {
459  const __m512d d32 = _mm512_set1_pd(Pow2<double, -32>::value);
460  const __m512d d33 = _mm512_set1_pd(Pow2<double, -33>::value);
461 
462  MCKL_RANDOM_INTERNAL_U01_AVX512_LOADU_SI512_CVTEPU32_EPI64(u)
463  MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU64_PD_52
464  MCKL_RANDOM_INTERNAL_U01_AVX512_FMADD_PD(d32, d33)
465  MCKL_RANDOM_INTERNAL_U01_AVX512_STOREU_PD(r)
466  }
467 }; // class U01AVX512Impl
468 
469 template <typename UIntType>
470 class U01AVX512Impl<UIntType, double, Closed, Closed, 64>
471  : public U01AVX512ImplBase<UIntType, double, Closed, Closed>
472 {
473  public:
474  using U01AVX512ImplBase<UIntType, double, Closed, Closed>::eval;
475 
476  private:
477  friend U01AVX512ImplBase<UIntType, double, Closed, Closed>;
478 
479  MCKL_INLINE static void eval(const UIntType *u, double *r)
480  {
481  const __m512d d54 = _mm512_set1_pd(Pow2<double, -54>::value);
482 
483  MCKL_RANDOM_INTERNAL_U01_AVX512_LOADU_SI512(u)
484  MCKL_RANDOM_INTERNAL_U01_AVX512_SLLI_EPI64(1)
485  MCKL_RANDOM_INTERNAL_U01_AVX512_SRLI_EPI64(10)
486  MCKL_RANDOM_INTERNAL_U01_AVX512_AND1ADD_EPI64
487  MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU64_PD
488  MCKL_RANDOM_INTERNAL_U01_AVX512_MUL_PD(d54)
489  MCKL_RANDOM_INTERNAL_U01_AVX512_STOREU_PD(r)
490  }
491 }; // class U01AVX512Impl
492 
493 template <typename UIntType>
494 class U01AVX512Impl<UIntType, double, Closed, Open, 64>
495  : public U01AVX512ImplBase<UIntType, double, Closed, Open>
496 {
497  public:
498  using U01AVX512ImplBase<UIntType, double, Closed, Open>::eval;
499 
500  private:
501  friend U01AVX512ImplBase<UIntType, double, Closed, Open>;
502 
503  MCKL_INLINE static void eval(const UIntType *u, double *r)
504  {
505  const __m512d d53 = _mm512_set1_pd(Pow2<double, -53>::value);
506 
507  MCKL_RANDOM_INTERNAL_U01_AVX512_LOADU_SI512(u)
508  MCKL_RANDOM_INTERNAL_U01_AVX512_SRLI_EPI64(11)
509  MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU64_PD
510  MCKL_RANDOM_INTERNAL_U01_AVX512_MUL_PD(d53)
511  MCKL_RANDOM_INTERNAL_U01_AVX512_STOREU_PD(r)
512  }
513 }; // class U01AVX512Impl
514 
515 template <typename UIntType>
516 class U01AVX512Impl<UIntType, double, Open, Closed, 64>
517  : public U01AVX512ImplBase<UIntType, double, Open, Closed>
518 {
519  public:
520  using U01AVX512ImplBase<UIntType, double, Open, Closed>::eval;
521 
522  private:
523  friend U01AVX512ImplBase<UIntType, double, Open, Closed>;
524 
525  MCKL_INLINE static void eval(const UIntType *u, double *r)
526  {
527  const __m512d d53 = _mm512_set1_pd(Pow2<double, -53>::value);
528 
529  MCKL_RANDOM_INTERNAL_U01_AVX512_LOADU_SI512(u)
530  MCKL_RANDOM_INTERNAL_U01_AVX512_SRLI_EPI64(11)
531  MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU64_PD
532  MCKL_RANDOM_INTERNAL_U01_AVX512_FMADD_PD(d53, d53)
533  MCKL_RANDOM_INTERNAL_U01_AVX512_STOREU_PD(r)
534  }
535 }; // class U01AVX512Impl
536 
537 template <typename UIntType>
538 class U01AVX512Impl<UIntType, double, Open, Open, 64>
539  : public U01AVX512ImplBase<UIntType, double, Open, Open>
540 {
541  public:
542  using U01AVX512ImplBase<UIntType, double, Open, Open>::eval;
543 
544  private:
545  friend U01AVX512ImplBase<UIntType, double, Open, Open>;
546 
547  MCKL_INLINE static void eval(const UIntType *u, double *r)
548  {
549  const __m512d d52 = _mm512_set1_pd(Pow2<double, -52>::value);
550  const __m512d d53 = _mm512_set1_pd(Pow2<double, -53>::value);
551 
552  MCKL_RANDOM_INTERNAL_U01_AVX512_LOADU_SI512(u)
553  MCKL_RANDOM_INTERNAL_U01_AVX512_SRLI_EPI64(12)
554  MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU64_PD_52
555  MCKL_RANDOM_INTERNAL_U01_AVX512_FMADD_PD(d52, d53)
556  MCKL_RANDOM_INTERNAL_U01_AVX512_STOREU_PD(r)
557  }
558 }; // class U01AVX512Impl
559 
560 template <typename UIntType>
561 class U01CanonicalAVX512Impl<UIntType, float, 1, 32>
562  : public U01CanonicalAVX512ImplBase<UIntType, float, 1>
563 {
564  public:
565  using U01CanonicalAVX512ImplBase<UIntType, float, 1>::eval;
566 
567  private:
568  friend U01CanonicalAVX512ImplBase<UIntType, float, 1>;
569 
570  MCKL_INLINE static void eval(const UIntType *u, float *r)
571  {
572  const __m512 d32 = _mm512_set1_ps(Pow2<float, -32>::value);
573 
574  MCKL_RANDOM_INTERNAL_U01_AVX512_LOADU_SI512(u);
575  MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU32_PS
576  MCKL_RANDOM_INTERNAL_U01_AVX512_MUL_PS(d32)
577  MCKL_RANDOM_INTERNAL_U01_AVX512_STOREU_PS(r)
578  }
579 }; // class U01CanonicalAVX512Impl
580 
581 template <typename UIntType>
582 class U01CanonicalAVX512Impl<UIntType, double, 2, 32>
583  : public U01CanonicalAVX512ImplBase<UIntType, double, 2>
584 {
585  public:
586  using U01CanonicalAVX512ImplBase<UIntType, double, 2>::eval;
587 
588  private:
589  friend U01CanonicalAVX512ImplBase<UIntType, double, 2>;
590 
591  MCKL_INLINE static void eval(const UIntType *u, double *r)
592  {
593  const __m512d d32 = _mm512_set1_pd(Pow2<double, -32>::value);
594  const __m512d d64 = _mm512_set1_pd(Pow2<double, -64>::value);
595  const __m512d c = _mm512_set1_pd(Pow2<double, 52>::value);
596  const __m512i l =
597  _mm512_set1_epi64(static_cast<MCKL_INT64>(0xFFFFFFFF));
598 
599  MCKL_RANDOM_INTERNAL_U01_AVX512_LOADU_SI512(u)
600 
601  __m512i xmm0 = _mm512_and_si512(zmm0, l);
602  __m512i xmm1 = _mm512_and_si512(zmm1, l);
603  __m512i xmm2 = _mm512_and_si512(zmm2, l);
604  __m512i xmm3 = _mm512_and_si512(zmm3, l);
605 
606  MCKL_RANDOM_INTERNAL_U01_AVX512_SRLI_EPI64(32);
607 
608  zmm0 = _mm512_add_epi64(zmm0, _mm512_castpd_si512(c));
609  zmm1 = _mm512_add_epi64(zmm1, _mm512_castpd_si512(c));
610  zmm2 = _mm512_add_epi64(zmm2, _mm512_castpd_si512(c));
611  zmm3 = _mm512_add_epi64(zmm3, _mm512_castpd_si512(c));
612 
613  xmm0 = _mm512_add_epi64(xmm0, _mm512_castpd_si512(c));
614  xmm1 = _mm512_add_epi64(xmm1, _mm512_castpd_si512(c));
615  xmm2 = _mm512_add_epi64(xmm2, _mm512_castpd_si512(c));
616  xmm3 = _mm512_add_epi64(xmm3, _mm512_castpd_si512(c));
617 
618  zmm0 =
619  _mm512_castpd_si512(_mm512_sub_pd(_mm512_castsi512_pd(zmm0), c));
620  zmm1 =
621  _mm512_castpd_si512(_mm512_sub_pd(_mm512_castsi512_pd(zmm1), c));
622  zmm2 =
623  _mm512_castpd_si512(_mm512_sub_pd(_mm512_castsi512_pd(zmm2), c));
624  zmm3 =
625  _mm512_castpd_si512(_mm512_sub_pd(_mm512_castsi512_pd(zmm3), c));
626 
627  xmm0 =
628  _mm512_castpd_si512(_mm512_sub_pd(_mm512_castsi512_pd(xmm0), c));
629  xmm1 =
630  _mm512_castpd_si512(_mm512_sub_pd(_mm512_castsi512_pd(xmm1), c));
631  xmm2 =
632  _mm512_castpd_si512(_mm512_sub_pd(_mm512_castsi512_pd(xmm2), c));
633  xmm3 =
634  _mm512_castpd_si512(_mm512_sub_pd(_mm512_castsi512_pd(xmm3), c));
635 
636  MCKL_RANDOM_INTERNAL_U01_AVX512_MUL_PD(d32);
637 
638  zmm0 = _mm512_castpd_si512(_mm512_fmadd_pd(
639  _mm512_castsi512_pd(xmm0), d64, _mm512_castsi512_pd(zmm0)));
640  zmm1 = _mm512_castpd_si512(_mm512_fmadd_pd(
641  _mm512_castsi512_pd(xmm1), d64, _mm512_castsi512_pd(zmm1)));
642  zmm2 = _mm512_castpd_si512(_mm512_fmadd_pd(
643  _mm512_castsi512_pd(xmm2), d64, _mm512_castsi512_pd(zmm2)));
644  zmm3 = _mm512_castpd_si512(_mm512_fmadd_pd(
645  _mm512_castsi512_pd(xmm3), d64, _mm512_castsi512_pd(zmm3)));
646 
647  MCKL_RANDOM_INTERNAL_U01_AVX512_STOREU_PD(r)
648  }
649 }; // class U01CanonicalAVX512Impl
650 
651 template <typename UIntType>
652 class U01CanonicalAVX512Impl<UIntType, double, 1, 64>
653  : public U01CanonicalAVX512ImplBase<UIntType, double, 1>
654 {
655  public:
656  using U01CanonicalAVX512ImplBase<UIntType, double, 1>::eval;
657 
658  private:
659  friend U01CanonicalAVX512ImplBase<UIntType, double, 1>;
660 
661  MCKL_INLINE static void eval(const UIntType *u, double *r)
662  {
663  const __m512d d64 = _mm512_set1_pd(Pow2<double, -64>::value);
664 
665  MCKL_RANDOM_INTERNAL_U01_AVX512_LOADU_SI512(u)
666  MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU64_PD
667  MCKL_RANDOM_INTERNAL_U01_AVX512_MUL_PD(d64)
668  MCKL_RANDOM_INTERNAL_U01_AVX512_STOREU_PD(r)
669  }
670 }; // class U01CanonicalAVX512Impl
671 
672 } // namespace internal
673 
674 } // namespace mckl
675 
676 MCKL_POP_GCC_WARNING
677 
678 #endif // MCKL_RANDOM_INTERNAL_U01_AVX512_HPP
MCKL_RANDOM_INTERNAL_U01_AVX512_AND1ADD_EPI32
#define MCKL_RANDOM_INTERNAL_U01_AVX512_AND1ADD_EPI32
Definition: u01_avx512.hpp:66
MCKL_RANDOM_INTERNAL_U01_AVX512_FMADD_PD
#define MCKL_RANDOM_INTERNAL_U01_AVX512_FMADD_PD(a, b)
Definition: u01_avx512.hpp:211
MCKL_RANDOM_INTERNAL_U01_AVX512_SRLI_EPI64
#define MCKL_RANDOM_INTERNAL_U01_AVX512_SRLI_EPI64(imm8)
Definition: u01_avx512.hpp:98
MCKL_RANDOM_INTERNAL_U01_AVX512_FMADD_PS
#define MCKL_RANDOM_INTERNAL_U01_AVX512_FMADD_PS(a, b)
Definition: u01_avx512.hpp:201
MCKL_PUSH_GCC_WARNING
#define MCKL_PUSH_GCC_WARNING(warning)
Definition: compiler.h:78
u01_generic.hpp
MCKL_RANDOM_INTERNAL_U01_AVX512_SLLI_EPI32
#define MCKL_RANDOM_INTERNAL_U01_AVX512_SLLI_EPI32(imm8)
Definition: u01_avx512.hpp:80
MCKL_RANDOM_INTERNAL_U01_AVX512_SLLI_EPI64
#define MCKL_RANDOM_INTERNAL_U01_AVX512_SLLI_EPI64(imm8)
Definition: u01_avx512.hpp:92
MCKL_RANDOM_INTERNAL_U01_AVX512_LOADU_SI512_CVTEPU32_EPI64
#define MCKL_RANDOM_INTERNAL_U01_AVX512_LOADU_SI512_CVTEPU32_EPI64(u)
Definition: u01_avx512.hpp:47
MCKL_RANDOM_INTERNAL_U01_AVX512_AND1ADD_EPI64
#define MCKL_RANDOM_INTERNAL_U01_AVX512_AND1ADD_EPI64
Definition: u01_avx512.hpp:73
mckl::internal::U01CanonicalGenericImpl::eval
static RealType eval(const UIntType *u)
Definition: u01_generic.hpp:170
MCKL_RANDOM_INTERNAL_U01_AVX512_MUL_PS
#define MCKL_RANDOM_INTERNAL_U01_AVX512_MUL_PS(a)
Definition: u01_avx512.hpp:189
MCKL_RANDOM_INTERNAL_U01_AVX512_MUL_PD
#define MCKL_RANDOM_INTERNAL_U01_AVX512_MUL_PD(a)
Definition: u01_avx512.hpp:195
MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU64_PD_52
#define MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU64_PD_52
Definition: u01_avx512.hpp:144
MCKL_RANDOM_INTERNAL_U01_AVX512_STOREU_PS
#define MCKL_RANDOM_INTERNAL_U01_AVX512_STOREU_PS(r)
Definition: u01_avx512.hpp:54
MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU64_PD
#define MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU64_PD
Definition: u01_avx512.hpp:155
MCKL_RANDOM_INTERNAL_U01_AVX512_SRLI_EPI32
#define MCKL_RANDOM_INTERNAL_U01_AVX512_SRLI_EPI32(imm8)
Definition: u01_avx512.hpp:86
mckl
Definition: mcmc.hpp:40
MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU32_PS
#define MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU32_PS
Definition: u01_avx512.hpp:110
MCKL_RANDOM_INTERNAL_U01_AVX512_STOREU_PD
#define MCKL_RANDOM_INTERNAL_U01_AVX512_STOREU_PD(r)
Definition: u01_avx512.hpp:60
mckl::internal::Pow2
Pow2Impl< RealType, P > Pow2
Definition: const_math.hpp:171
MCKL_INLINE
#define MCKL_INLINE
Definition: clang.h:147
MCKL_POP_GCC_WARNING
#define MCKL_POP_GCC_WARNING
Definition: compiler.h:79
MCKL_RANDOM_INTERNAL_U01_AVX512_LOADU_SI512
#define MCKL_RANDOM_INTERNAL_U01_AVX512_LOADU_SI512(u)
Definition: u01_avx512.hpp:40
MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU32_PS_31
#define MCKL_RANDOM_INTERNAL_U01_AVX512_CVTEPU32_PS_31
Definition: u01_avx512.hpp:104
Generated by   doxygen 1.8.13