MCKL
Monte Carlo Kernel Library
sse2.hpp
Go to the documentation of this file.
1 //============================================================================
2 // MCKL/include/mckl/internal/sse2.hpp
3 //----------------------------------------------------------------------------
4 // MCKL: Monte Carlo Kernel Library
5 //----------------------------------------------------------------------------
6 // Copyright (c) 2013-2018, Yan Zhou
7 // All rights reserved.
8 //
9 // Redistribution and use in source and binary forms, with or without
10 // modification, are permitted provided that the following conditions are met:
11 //
12 // Redistributions of source code must retain the above copyright notice,
13 // this list of conditions and the following disclaimer.
14 //
15 // Redistributions in binary form must reproduce the above copyright notice,
16 // this list of conditions and the following disclaimer in the documentation
17 // and/or other materials provided with the distribution.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS AS IS
20 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
23 // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 // POSSIBILITY OF SUCH DAMAGE.
30 //============================================================================
31 
32 #ifndef MCKL_INTERNAL_SSE2_HPP
33 #define MCKL_INTERNAL_SSE2_HPP
34 
35 #include <mckl/internal/config.h>
36 #include <mckl/internal/const_math.hpp>
37 #include <mckl/internal/sse2_op.hpp>
38 #include <array>
39 
40 MCKL_PUSH_GCC_WARNING("-Wignored-attributes")
41 
42 namespace mckl {
43 
44 namespace internal {
45 
46 MCKL_INLINE inline void cvtepu32_epi64(
47  const std::array<__m128i, 2> &s, std::array<__m128i, 4> &t)
48 {
49  const __m128i m = _mm_set1_epi64x(static_cast<MCKL_INT64>(0xFFFFFFFF));
50 
51  std::get<0>(t) = _mm_shuffle_epi32(std::get<0>(s), 0xD8);
52  std::get<2>(t) = _mm_shuffle_epi32(std::get<1>(s), 0xD8);
53 
54  std::get<1>(t) = _mm_srli_epi64(std::get<0>(t), 32);
55  std::get<3>(t) = _mm_srli_epi64(std::get<2>(t), 32);
56 
57  std::get<0>(t) = _mm_and_si128(std::get<0>(t), m);
58  std::get<2>(t) = _mm_and_si128(std::get<2>(t), m);
59 }
60 
61 MCKL_INLINE inline void cvtepu32_epi64(
62  const std::array<__m128i, 4> &s, std::array<__m128i, 8> &t)
63 {
64  const __m128i m = _mm_set1_epi64x(static_cast<MCKL_INT64>(0xFFFFFFFF));
65 
66  std::get<0>(t) = _mm_shuffle_epi32(std::get<0>(s), 0xD8);
67  std::get<2>(t) = _mm_shuffle_epi32(std::get<1>(s), 0xD8);
68  std::get<4>(t) = _mm_shuffle_epi32(std::get<2>(s), 0xD8);
69  std::get<6>(t) = _mm_shuffle_epi32(std::get<3>(s), 0xD8);
70 
71  std::get<1>(t) = _mm_srli_epi64(std::get<0>(t), 32);
72  std::get<3>(t) = _mm_srli_epi64(std::get<2>(t), 32);
73  std::get<5>(t) = _mm_srli_epi64(std::get<4>(t), 32);
74  std::get<7>(t) = _mm_srli_epi64(std::get<6>(t), 32);
75 
76  std::get<0>(t) = _mm_and_si128(std::get<0>(t), m);
77  std::get<2>(t) = _mm_and_si128(std::get<2>(t), m);
78  std::get<4>(t) = _mm_and_si128(std::get<4>(t), m);
79  std::get<6>(t) = _mm_and_si128(std::get<6>(t), m);
80 }
81 
82 MCKL_INLINE inline void cvtepu32_epi64(
83  const std::array<__m128i, 8> &s, std::array<__m128i, 16> &t)
84 {
85  const __m128i m = _mm_set1_epi64x(static_cast<MCKL_INT64>(0xFFFFFFFF));
86 
87  std::get<0x0>(t) = _mm_shuffle_epi32(std::get<0>(s), 0xD8);
88  std::get<0x2>(t) = _mm_shuffle_epi32(std::get<1>(s), 0xD8);
89  std::get<0x4>(t) = _mm_shuffle_epi32(std::get<2>(s), 0xD8);
90  std::get<0x6>(t) = _mm_shuffle_epi32(std::get<3>(s), 0xD8);
91  std::get<0x8>(t) = _mm_shuffle_epi32(std::get<4>(s), 0xD8);
92  std::get<0xA>(t) = _mm_shuffle_epi32(std::get<5>(s), 0xD8);
93  std::get<0xC>(t) = _mm_shuffle_epi32(std::get<6>(s), 0xD8);
94  std::get<0xE>(t) = _mm_shuffle_epi32(std::get<7>(s), 0xD8);
95 
96  std::get<0x1>(t) = _mm_srli_epi64(std::get<0x0>(t), 32);
97  std::get<0x3>(t) = _mm_srli_epi64(std::get<0x2>(t), 32);
98  std::get<0x5>(t) = _mm_srli_epi64(std::get<0x4>(t), 32);
99  std::get<0x7>(t) = _mm_srli_epi64(std::get<0x6>(t), 32);
100  std::get<0x9>(t) = _mm_srli_epi64(std::get<0x8>(t), 32);
101  std::get<0xB>(t) = _mm_srli_epi64(std::get<0xA>(t), 32);
102  std::get<0xD>(t) = _mm_srli_epi64(std::get<0xC>(t), 32);
103  std::get<0xF>(t) = _mm_srli_epi64(std::get<0xE>(t), 32);
104 
105  std::get<0x0>(t) = _mm_and_si128(std::get<0x0>(t), m);
106  std::get<0x2>(t) = _mm_and_si128(std::get<0x2>(t), m);
107  std::get<0x4>(t) = _mm_and_si128(std::get<0x4>(t), m);
108  std::get<0x6>(t) = _mm_and_si128(std::get<0x6>(t), m);
109  std::get<0x8>(t) = _mm_and_si128(std::get<0x8>(t), m);
110  std::get<0xA>(t) = _mm_and_si128(std::get<0xA>(t), m);
111  std::get<0xC>(t) = _mm_and_si128(std::get<0xC>(t), m);
112  std::get<0xE>(t) = _mm_and_si128(std::get<0xE>(t), m);
113 }
114 
115 MCKL_INLINE inline void cvtepi32_ps(std::array<__m128i, 4> &s)
116 {
117  std::get<0>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<0>(s)));
118  std::get<1>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<1>(s)));
119  std::get<2>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<2>(s)));
120  std::get<3>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<3>(s)));
121 }
122 
123 MCKL_INLINE inline void cvtepi32_ps(std::array<__m128i, 8> &s)
124 {
125  std::get<0>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<0>(s)));
126  std::get<1>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<1>(s)));
127  std::get<2>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<2>(s)));
128  std::get<3>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<3>(s)));
129  std::get<4>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<4>(s)));
130  std::get<5>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<5>(s)));
131  std::get<6>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<6>(s)));
132  std::get<7>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<7>(s)));
133 }
134 
135 MCKL_INLINE inline void cvtepi32_ps(std::array<__m128i, 16> &s)
136 {
137  std::get<0x0>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<0x0>(s)));
138  std::get<0x1>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<0x1>(s)));
139  std::get<0x2>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<0x2>(s)));
140  std::get<0x3>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<0x3>(s)));
141  std::get<0x4>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<0x4>(s)));
142  std::get<0x5>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<0x5>(s)));
143  std::get<0x6>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<0x6>(s)));
144  std::get<0x7>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<0x7>(s)));
145  std::get<0x8>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<0x8>(s)));
146  std::get<0x9>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<0x9>(s)));
147  std::get<0xA>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<0xA>(s)));
148  std::get<0xB>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<0xB>(s)));
149  std::get<0xC>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<0xC>(s)));
150  std::get<0xD>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<0xD>(s)));
151  std::get<0xE>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<0xE>(s)));
152  std::get<0xF>(s) = _mm_castps_si128(_mm_cvtepi32_ps(std::get<0xF>(s)));
153 }
154 
155 template <std::size_t S>
156 MCKL_INLINE inline void cvtepi64_pd(std::array<__m128i, S> &s)
157 {
158  const __m128i m32 = _mm_castpd_si128(_mm_set1_pd(Pow2<double, 32>::value));
159  const __m128i m52 = _mm_castpd_si128(_mm_set1_pd(Pow2<double, 52>::value));
160  const __m128i mask = _mm_set1_epi64x(static_cast<MCKL_INT64>(0xFFFFFFFF));
161 
162  std::array<__m128i, S> t;
163  and_si128(s, mask, t);
164  srli_epi64<32>(s);
165  add_epi64(s, m52);
166  add_epi64(t, m52);
167  sub_pd(s, m52);
168  sub_pd(t, m52);
169  mul_pd(s, m32);
170  add_pd(s, t);
171 }
172 
173 template <std::size_t I0, std::size_t I1, std::size_t I2, std::size_t I3,
174  std::size_t N>
175 MCKL_INLINE inline void transpose4x32_si128(std::array<__m128i, N> &s)
176 {
177  __m128i t0 = _mm_unpacklo_epi32(std::get<I0>(s), std::get<I1>(s));
178  __m128i t1 = _mm_unpacklo_epi32(std::get<I2>(s), std::get<I3>(s));
179  __m128i t2 = _mm_unpackhi_epi32(std::get<I0>(s), std::get<I1>(s));
180  __m128i t3 = _mm_unpackhi_epi32(std::get<I2>(s), std::get<I3>(s));
181  std::get<I0>(s) = _mm_unpacklo_epi64(t0, t1);
182  std::get<I1>(s) = _mm_unpackhi_epi64(t0, t1);
183  std::get<I2>(s) = _mm_unpacklo_epi64(t2, t3);
184  std::get<I3>(s) = _mm_unpackhi_epi64(t2, t3);
185 }
186 
187 MCKL_INLINE inline void transpose4x32_load_si128(std::array<__m128i, 4> &s)
188 {
189  transpose4x32_si128<0, 1, 2, 3>(s);
190 }
191 
192 MCKL_INLINE inline void transpose4x32_store_si128(std::array<__m128i, 4> &s)
193 {
194  transpose4x32_si128<0, 1, 2, 3>(s);
195 }
196 
197 MCKL_INLINE inline void transpose4x32_load_si128(std::array<__m128i, 8> &s)
198 {
199  __m128i s2 = std::get<2>(s);
200  std::get<2>(s) = std::get<1>(s);
201  std::get<1>(s) = std::get<4>(s);
202  std::get<4>(s) = s2;
203 
204  __m128i s6 = std::get<6>(s);
205  std::get<6>(s) = std::get<3>(s);
206  std::get<3>(s) = std::get<5>(s);
207  std::get<5>(s) = s6;
208 
209  transpose4x32_si128<0, 1, 2, 3>(s);
210  transpose4x32_si128<4, 5, 6, 7>(s);
211 }
212 
213 MCKL_INLINE inline void transpose4x32_store_si128(std::array<__m128i, 8> &s)
214 {
215  transpose4x32_si128<0, 1, 2, 3>(s);
216  transpose4x32_si128<4, 5, 6, 7>(s);
217 
218  __m128i s1 = std::get<1>(s);
219  std::get<1>(s) = std::get<2>(s);
220  std::get<2>(s) = std::get<4>(s);
221  std::get<4>(s) = s1;
222 
223  __m128i s3 = std::get<3>(s);
224  std::get<3>(s) = std::get<6>(s);
225  std::get<6>(s) = std::get<5>(s);
226  std::get<5>(s) = s3;
227 }
228 
229 MCKL_INLINE inline void transpose4x32_load_si128(std::array<__m128i, 16> &s)
230 {
231  __m128i s1 = std::get<0x1>(s);
232  __m128i s2 = std::get<0x2>(s);
233  __m128i s3 = std::get<0x3>(s);
234  __m128i s6 = std::get<0x6>(s);
235  __m128i s7 = std::get<0x7>(s);
236  __m128i sB = std::get<0xB>(s);
237  std::get<0x1>(s) = std::get<0x4>(s);
238  std::get<0x2>(s) = std::get<0x8>(s);
239  std::get<0x3>(s) = std::get<0xC>(s);
240  std::get<0x6>(s) = std::get<0x9>(s);
241  std::get<0x7>(s) = std::get<0xD>(s);
242  std::get<0xB>(s) = std::get<0xE>(s);
243  std::get<0x4>(s) = s1;
244  std::get<0x8>(s) = s2;
245  std::get<0xC>(s) = s3;
246  std::get<0x9>(s) = s6;
247  std::get<0xD>(s) = s7;
248  std::get<0xE>(s) = sB;
249 
250  transpose4x32_si128<0x0, 0x1, 0x2, 0x3>(s);
251  transpose4x32_si128<0x4, 0x5, 0x6, 0x7>(s);
252  transpose4x32_si128<0x8, 0x9, 0xA, 0xB>(s);
253  transpose4x32_si128<0xC, 0xD, 0xE, 0xF>(s);
254 }
255 
256 MCKL_INLINE inline void transpose4x32_store_si128(std::array<__m128i, 16> &s)
257 {
258  transpose4x32_si128<0x0, 0x1, 0x2, 0x3>(s);
259  transpose4x32_si128<0x4, 0x5, 0x6, 0x7>(s);
260  transpose4x32_si128<0x8, 0x9, 0xA, 0xB>(s);
261  transpose4x32_si128<0xC, 0xD, 0xE, 0xF>(s);
262 
263  __m128i s1 = std::get<0x1>(s);
264  __m128i s2 = std::get<0x2>(s);
265  __m128i s3 = std::get<0x3>(s);
266  __m128i s6 = std::get<0x6>(s);
267  __m128i s7 = std::get<0x7>(s);
268  __m128i sB = std::get<0xB>(s);
269  std::get<0x1>(s) = std::get<0x4>(s);
270  std::get<0x2>(s) = std::get<0x8>(s);
271  std::get<0x3>(s) = std::get<0xC>(s);
272  std::get<0x6>(s) = std::get<0x9>(s);
273  std::get<0x7>(s) = std::get<0xD>(s);
274  std::get<0xB>(s) = std::get<0xE>(s);
275  std::get<0x4>(s) = s1;
276  std::get<0x8>(s) = s2;
277  std::get<0xC>(s) = s3;
278  std::get<0x9>(s) = s6;
279  std::get<0xD>(s) = s7;
280  std::get<0xE>(s) = sB;
281 }
282 
283 template <std::size_t I0, std::size_t I1, std::size_t N>
284 MCKL_INLINE inline void transpose2x64_si128(std::array<__m128i, N> &s)
285 {
286  __m128i t0 = _mm_unpacklo_epi64(std::get<I0>(s), std::get<I1>(s));
287  __m128i t1 = _mm_unpackhi_epi64(std::get<I0>(s), std::get<I1>(s));
288  std::get<I0>(s) = t0;
289  std::get<I1>(s) = t1;
290 }
291 
292 MCKL_INLINE inline void transpose2x64_load_si128(std::array<__m128i, 2> &s)
293 {
294  transpose2x64_si128<0, 1>(s);
295 }
296 
297 MCKL_INLINE inline void transpose2x64_store_si128(std::array<__m128i, 2> &s)
298 {
299  transpose2x64_si128<0, 1>(s);
300 }
301 
302 MCKL_INLINE inline void transpose2x64_load_si128(std::array<__m128i, 4> &s)
303 {
304  __m128i s1 = std::get<1>(s);
305  std::get<1>(s) = std::get<2>(s);
306  std::get<2>(s) = s1;
307 
308  transpose2x64_si128<0, 1>(s);
309  transpose2x64_si128<2, 3>(s);
310 }
311 
312 MCKL_INLINE inline void transpose2x64_store_si128(std::array<__m128i, 4> &s)
313 {
314  transpose2x64_si128<0, 1>(s);
315  transpose2x64_si128<2, 3>(s);
316 
317  __m128i s1 = std::get<1>(s);
318  std::get<1>(s) = std::get<2>(s);
319  std::get<2>(s) = s1;
320 }
321 
322 MCKL_INLINE inline void transpose2x64_load_si128(std::array<__m128i, 8> &s)
323 {
324  __m128i s2 = std::get<2>(s);
325  std::get<2>(s) = std::get<1>(s);
326  std::get<1>(s) = std::get<4>(s);
327  std::get<4>(s) = s2;
328 
329  __m128i s6 = std::get<6>(s);
330  std::get<6>(s) = std::get<3>(s);
331  std::get<3>(s) = std::get<5>(s);
332  std::get<5>(s) = s6;
333 
334  transpose2x64_si128<0, 1>(s);
335  transpose2x64_si128<2, 3>(s);
336  transpose2x64_si128<4, 5>(s);
337  transpose2x64_si128<6, 7>(s);
338 }
339 
340 MCKL_INLINE inline void transpose2x64_store_si128(std::array<__m128i, 8> &s)
341 {
342  transpose2x64_si128<0, 1>(s);
343  transpose2x64_si128<2, 3>(s);
344  transpose2x64_si128<4, 5>(s);
345  transpose2x64_si128<6, 7>(s);
346 
347  __m128i s1 = std::get<1>(s);
348  std::get<1>(s) = std::get<2>(s);
349  std::get<2>(s) = std::get<4>(s);
350  std::get<4>(s) = s1;
351 
352  __m128i s3 = std::get<3>(s);
353  std::get<3>(s) = std::get<6>(s);
354  std::get<6>(s) = std::get<5>(s);
355  std::get<5>(s) = s3;
356 }
357 
358 MCKL_INLINE inline void transpose2x64_load_si128(std::array<__m128i, 16> &s)
359 {
360  __m128i s2 = std::get<0x2>(s);
361  std::get<0x2>(s) = std::get<0x1>(s);
362  std::get<0x1>(s) = std::get<0x8>(s);
363  std::get<0x8>(s) = std::get<0x4>(s);
364  std::get<0x4>(s) = s2;
365 
366  __m128i s6 = std::get<0x6>(s);
367  std::get<0x6>(s) = std::get<0x3>(s);
368  std::get<0x3>(s) = std::get<0x9>(s);
369  std::get<0x9>(s) = std::get<0xC>(s);
370  std::get<0xC>(s) = s6;
371 
372  __m128i sE = std::get<0xE>(s);
373  std::get<0xE>(s) = std::get<0x7>(s);
374  std::get<0x7>(s) = std::get<0xB>(s);
375  std::get<0xB>(s) = std::get<0xD>(s);
376  std::get<0xD>(s) = sE;
377 
378  __m128i s5 = std::get<0x5>(s);
379  std::get<0x5>(s) = std::get<0xA>(s);
380  std::get<0xA>(s) = s5;
381 
382  transpose2x64_si128<0x0, 0x1>(s);
383  transpose2x64_si128<0x2, 0x3>(s);
384  transpose2x64_si128<0x4, 0x5>(s);
385  transpose2x64_si128<0x6, 0x7>(s);
386  transpose2x64_si128<0x8, 0x9>(s);
387  transpose2x64_si128<0xA, 0xB>(s);
388  transpose2x64_si128<0xC, 0xD>(s);
389  transpose2x64_si128<0xE, 0xF>(s);
390 }
391 
392 MCKL_INLINE inline void transpose2x64_store_si128(std::array<__m128i, 16> &s)
393 {
394  transpose2x64_si128<0x0, 0x1>(s);
395  transpose2x64_si128<0x2, 0x3>(s);
396  transpose2x64_si128<0x4, 0x5>(s);
397  transpose2x64_si128<0x6, 0x7>(s);
398  transpose2x64_si128<0x8, 0x9>(s);
399  transpose2x64_si128<0xA, 0xB>(s);
400  transpose2x64_si128<0xC, 0xD>(s);
401  transpose2x64_si128<0xE, 0xF>(s);
402 
403  __m128i s1 = std::get<0x1>(s);
404  std::get<0x1>(s) = std::get<0x2>(s);
405  std::get<0x2>(s) = std::get<0x4>(s);
406  std::get<0x4>(s) = std::get<0x8>(s);
407  std::get<0x8>(s) = s1;
408 
409  __m128i s3 = std::get<0x3>(s);
410  std::get<0x3>(s) = std::get<0x6>(s);
411  std::get<0x6>(s) = std::get<0xC>(s);
412  std::get<0xC>(s) = std::get<0x9>(s);
413  std::get<0x9>(s) = s3;
414 
415  __m128i s7 = std::get<0x7>(s);
416  std::get<0x7>(s) = std::get<0xE>(s);
417  std::get<0xE>(s) = std::get<0xD>(s);
418  std::get<0xD>(s) = std::get<0xB>(s);
419  std::get<0xB>(s) = s7;
420 
421  __m128i s5 = std::get<0x5>(s);
422  std::get<0x5>(s) = std::get<0xA>(s);
423  std::get<0xA>(s) = s5;
424 }
425 
426 } // namespace internal
427 
428 } // namespace mckl
429 
430 MCKL_POP_GCC_WARNING
431 
432 #endif // MCKL_INTERNAL_SSE2_HPP
MCKL_PUSH_GCC_WARNING
#define MCKL_PUSH_GCC_WARNING(warning)
Definition: compiler.h:78
const_math.hpp
mckl
Definition: mcmc.hpp:40
sse2_op.hpp
MCKL_INLINE
#define MCKL_INLINE
Definition: clang.h:147
MCKL_POP_GCC_WARNING
#define MCKL_POP_GCC_WARNING
Definition: compiler.h:79
Generated by   doxygen 1.8.13