Vector Optimized Library of Kernels  2.4
Architecture-tuned implementations of math kernels
volk_32fc_x2_multiply_32fc.h
Go to the documentation of this file.
1 /* -*- c++ -*- */
2 /*
3  * Copyright 2012, 2014 Free Software Foundation, Inc.
4  *
5  * This file is part of GNU Radio
6  *
7  * GNU Radio is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation; either version 3, or (at your option)
10  * any later version.
11  *
12  * GNU Radio is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with GNU Radio; see the file COPYING. If not, write to
19  * the Free Software Foundation, Inc., 51 Franklin Street,
20  * Boston, MA 02110-1301, USA.
21  */
22 
70 #ifndef INCLUDED_volk_32fc_x2_multiply_32fc_u_H
71 #define INCLUDED_volk_32fc_x2_multiply_32fc_u_H
72 
73 #include <float.h>
74 #include <inttypes.h>
75 #include <stdio.h>
76 #include <volk/volk_complex.h>
77 
78 #if LV_HAVE_AVX2 && LV_HAVE_FMA
79 #include <immintrin.h>
87 static inline void volk_32fc_x2_multiply_32fc_u_avx2_fma(lv_32fc_t* cVector,
88  const lv_32fc_t* aVector,
89  const lv_32fc_t* bVector,
90  unsigned int num_points)
91 {
92  unsigned int number = 0;
93  const unsigned int quarterPoints = num_points / 4;
94 
95  lv_32fc_t* c = cVector;
96  const lv_32fc_t* a = aVector;
97  const lv_32fc_t* b = bVector;
98 
99  for (; number < quarterPoints; number++) {
100 
101  const __m256 x =
102  _mm256_loadu_ps((float*)a); // Load the ar + ai, br + bi as ar,ai,br,bi
103  const __m256 y =
104  _mm256_loadu_ps((float*)b); // Load the cr + ci, dr + di as cr,ci,dr,di
105 
106  const __m256 yl = _mm256_moveldup_ps(y); // Load yl with cr,cr,dr,dr
107  const __m256 yh = _mm256_movehdup_ps(y); // Load yh with ci,ci,di,di
108 
109  const __m256 tmp2x = _mm256_permute_ps(x, 0xB1); // Re-arrange x to be ai,ar,bi,br
110 
111  const __m256 tmp2 = _mm256_mul_ps(tmp2x, yh); // tmp2 = ai*ci,ar*ci,bi*di,br*di
112 
113  const __m256 z = _mm256_fmaddsub_ps(
114  x, yl, tmp2); // ar*cr-ai*ci, ai*cr+ar*ci, br*dr-bi*di, bi*dr+br*di
115 
116  _mm256_storeu_ps((float*)c, z); // Store the results back into the C container
117 
118  a += 4;
119  b += 4;
120  c += 4;
121  }
122 
123  _mm256_zeroupper();
124 
125  number = quarterPoints * 4;
126  for (; number < num_points; number++) {
127  *c++ = (*a++) * (*b++);
128  }
129 }
130 #endif /* LV_HAVE_AVX2 && LV_HAVE_FMA */
131 
132 
133 #ifdef LV_HAVE_AVX
134 #include <immintrin.h>
136 
137 static inline void volk_32fc_x2_multiply_32fc_u_avx(lv_32fc_t* cVector,
138  const lv_32fc_t* aVector,
139  const lv_32fc_t* bVector,
140  unsigned int num_points)
141 {
142  unsigned int number = 0;
143  const unsigned int quarterPoints = num_points / 4;
144 
145  __m256 x, y, z;
146  lv_32fc_t* c = cVector;
147  const lv_32fc_t* a = aVector;
148  const lv_32fc_t* b = bVector;
149 
150  for (; number < quarterPoints; number++) {
151  x = _mm256_loadu_ps(
152  (float*)a); // Load the ar + ai, br + bi ... as ar,ai,br,bi ...
153  y = _mm256_loadu_ps(
154  (float*)b); // Load the cr + ci, dr + di ... as cr,ci,dr,di ...
155  z = _mm256_complexmul_ps(x, y);
156  _mm256_storeu_ps((float*)c, z); // Store the results back into the C container
157 
158  a += 4;
159  b += 4;
160  c += 4;
161  }
162 
163  number = quarterPoints * 4;
164 
165  for (; number < num_points; number++) {
166  *c++ = (*a++) * (*b++);
167  }
168 }
169 #endif /* LV_HAVE_AVX */
170 
171 
172 #ifdef LV_HAVE_SSE3
173 #include <pmmintrin.h>
175 
176 static inline void volk_32fc_x2_multiply_32fc_u_sse3(lv_32fc_t* cVector,
177  const lv_32fc_t* aVector,
178  const lv_32fc_t* bVector,
179  unsigned int num_points)
180 {
181  unsigned int number = 0;
182  const unsigned int halfPoints = num_points / 2;
183 
184  __m128 x, y, z;
185  lv_32fc_t* c = cVector;
186  const lv_32fc_t* a = aVector;
187  const lv_32fc_t* b = bVector;
188 
189  for (; number < halfPoints; number++) {
190  x = _mm_loadu_ps((float*)a); // Load the ar + ai, br + bi as ar,ai,br,bi
191  y = _mm_loadu_ps((float*)b); // Load the cr + ci, dr + di as cr,ci,dr,di
192  z = _mm_complexmul_ps(x, y);
193  _mm_storeu_ps((float*)c, z); // Store the results back into the C container
194 
195  a += 2;
196  b += 2;
197  c += 2;
198  }
199 
200  if ((num_points % 2) != 0) {
201  *c = (*a) * (*b);
202  }
203 }
204 #endif /* LV_HAVE_SSE */
205 
206 
207 #ifdef LV_HAVE_GENERIC
208 
209 static inline void volk_32fc_x2_multiply_32fc_generic(lv_32fc_t* cVector,
210  const lv_32fc_t* aVector,
211  const lv_32fc_t* bVector,
212  unsigned int num_points)
213 {
214  lv_32fc_t* cPtr = cVector;
215  const lv_32fc_t* aPtr = aVector;
216  const lv_32fc_t* bPtr = bVector;
217  unsigned int number = 0;
218 
219  for (number = 0; number < num_points; number++) {
220  *cPtr++ = (*aPtr++) * (*bPtr++);
221  }
222 }
223 #endif /* LV_HAVE_GENERIC */
224 
225 
226 #endif /* INCLUDED_volk_32fc_x2_multiply_32fc_u_H */
227 #ifndef INCLUDED_volk_32fc_x2_multiply_32fc_a_H
228 #define INCLUDED_volk_32fc_x2_multiply_32fc_a_H
229 
230 #include <float.h>
231 #include <inttypes.h>
232 #include <stdio.h>
233 #include <volk/volk_complex.h>
234 
235 #if LV_HAVE_AVX2 && LV_HAVE_FMA
236 #include <immintrin.h>
244 static inline void volk_32fc_x2_multiply_32fc_a_avx2_fma(lv_32fc_t* cVector,
245  const lv_32fc_t* aVector,
246  const lv_32fc_t* bVector,
247  unsigned int num_points)
248 {
249  unsigned int number = 0;
250  const unsigned int quarterPoints = num_points / 4;
251 
252  lv_32fc_t* c = cVector;
253  const lv_32fc_t* a = aVector;
254  const lv_32fc_t* b = bVector;
255 
256  for (; number < quarterPoints; number++) {
257 
258  const __m256 x =
259  _mm256_load_ps((float*)a); // Load the ar + ai, br + bi as ar,ai,br,bi
260  const __m256 y =
261  _mm256_load_ps((float*)b); // Load the cr + ci, dr + di as cr,ci,dr,di
262 
263  const __m256 yl = _mm256_moveldup_ps(y); // Load yl with cr,cr,dr,dr
264  const __m256 yh = _mm256_movehdup_ps(y); // Load yh with ci,ci,di,di
265 
266  const __m256 tmp2x = _mm256_permute_ps(x, 0xB1); // Re-arrange x to be ai,ar,bi,br
267 
268  const __m256 tmp2 = _mm256_mul_ps(tmp2x, yh); // tmp2 = ai*ci,ar*ci,bi*di,br*di
269 
270  const __m256 z = _mm256_fmaddsub_ps(
271  x, yl, tmp2); // ar*cr-ai*ci, ai*cr+ar*ci, br*dr-bi*di, bi*dr+br*di
272 
273  _mm256_store_ps((float*)c, z); // Store the results back into the C container
274 
275  a += 4;
276  b += 4;
277  c += 4;
278  }
279 
280  _mm256_zeroupper();
281 
282  number = quarterPoints * 4;
283  for (; number < num_points; number++) {
284  *c++ = (*a++) * (*b++);
285  }
286 }
287 #endif /* LV_HAVE_AVX2 && LV_HAVE_FMA */
288 
289 
290 #ifdef LV_HAVE_AVX
291 #include <immintrin.h>
293 
294 static inline void volk_32fc_x2_multiply_32fc_a_avx(lv_32fc_t* cVector,
295  const lv_32fc_t* aVector,
296  const lv_32fc_t* bVector,
297  unsigned int num_points)
298 {
299  unsigned int number = 0;
300  const unsigned int quarterPoints = num_points / 4;
301 
302  __m256 x, y, z;
303  lv_32fc_t* c = cVector;
304  const lv_32fc_t* a = aVector;
305  const lv_32fc_t* b = bVector;
306 
307  for (; number < quarterPoints; number++) {
308  x = _mm256_load_ps((float*)a); // Load the ar + ai, br + bi ... as ar,ai,br,bi ...
309  y = _mm256_load_ps((float*)b); // Load the cr + ci, dr + di ... as cr,ci,dr,di ...
310  z = _mm256_complexmul_ps(x, y);
311  _mm256_store_ps((float*)c, z); // Store the results back into the C container
312 
313  a += 4;
314  b += 4;
315  c += 4;
316  }
317 
318  number = quarterPoints * 4;
319 
320  for (; number < num_points; number++) {
321  *c++ = (*a++) * (*b++);
322  }
323 }
324 #endif /* LV_HAVE_AVX */
325 
326 #ifdef LV_HAVE_SSE3
327 #include <pmmintrin.h>
329 
330 static inline void volk_32fc_x2_multiply_32fc_a_sse3(lv_32fc_t* cVector,
331  const lv_32fc_t* aVector,
332  const lv_32fc_t* bVector,
333  unsigned int num_points)
334 {
335  unsigned int number = 0;
336  const unsigned int halfPoints = num_points / 2;
337 
338  __m128 x, y, z;
339  lv_32fc_t* c = cVector;
340  const lv_32fc_t* a = aVector;
341  const lv_32fc_t* b = bVector;
342 
343  for (; number < halfPoints; number++) {
344  x = _mm_load_ps((float*)a); // Load the ar + ai, br + bi as ar,ai,br,bi
345  y = _mm_load_ps((float*)b); // Load the cr + ci, dr + di as cr,ci,dr,di
346  z = _mm_complexmul_ps(x, y);
347  _mm_store_ps((float*)c, z); // Store the results back into the C container
348 
349  a += 2;
350  b += 2;
351  c += 2;
352  }
353 
354  if ((num_points % 2) != 0) {
355  *c = (*a) * (*b);
356  }
357 }
358 #endif /* LV_HAVE_SSE */
359 
360 
361 #ifdef LV_HAVE_GENERIC
362 
364  const lv_32fc_t* aVector,
365  const lv_32fc_t* bVector,
366  unsigned int num_points)
367 {
368  lv_32fc_t* cPtr = cVector;
369  const lv_32fc_t* aPtr = aVector;
370  const lv_32fc_t* bPtr = bVector;
371  unsigned int number = 0;
372 
373  for (number = 0; number < num_points; number++) {
374  *cPtr++ = (*aPtr++) * (*bPtr++);
375  }
376 }
377 #endif /* LV_HAVE_GENERIC */
378 
379 
380 #ifdef LV_HAVE_NEON
381 #include <arm_neon.h>
382 
383 static inline void volk_32fc_x2_multiply_32fc_neon(lv_32fc_t* cVector,
384  const lv_32fc_t* aVector,
385  const lv_32fc_t* bVector,
386  unsigned int num_points)
387 {
388  lv_32fc_t* a_ptr = (lv_32fc_t*)aVector;
389  lv_32fc_t* b_ptr = (lv_32fc_t*)bVector;
390  unsigned int quarter_points = num_points / 4;
391  float32x4x2_t a_val, b_val, c_val;
392  float32x4x2_t tmp_real, tmp_imag;
393  unsigned int number = 0;
394 
395  for (number = 0; number < quarter_points; ++number) {
396  a_val = vld2q_f32((float*)a_ptr); // a0r|a1r|a2r|a3r || a0i|a1i|a2i|a3i
397  b_val = vld2q_f32((float*)b_ptr); // b0r|b1r|b2r|b3r || b0i|b1i|b2i|b3i
398  __VOLK_PREFETCH(a_ptr + 4);
399  __VOLK_PREFETCH(b_ptr + 4);
400 
401  // multiply the real*real and imag*imag to get real result
402  // a0r*b0r|a1r*b1r|a2r*b2r|a3r*b3r
403  tmp_real.val[0] = vmulq_f32(a_val.val[0], b_val.val[0]);
404  // a0i*b0i|a1i*b1i|a2i*b2i|a3i*b3i
405  tmp_real.val[1] = vmulq_f32(a_val.val[1], b_val.val[1]);
406 
407  // Multiply cross terms to get the imaginary result
408  // a0r*b0i|a1r*b1i|a2r*b2i|a3r*b3i
409  tmp_imag.val[0] = vmulq_f32(a_val.val[0], b_val.val[1]);
410  // a0i*b0r|a1i*b1r|a2i*b2r|a3i*b3r
411  tmp_imag.val[1] = vmulq_f32(a_val.val[1], b_val.val[0]);
412 
413  // store the results
414  c_val.val[0] = vsubq_f32(tmp_real.val[0], tmp_real.val[1]);
415  c_val.val[1] = vaddq_f32(tmp_imag.val[0], tmp_imag.val[1]);
416  vst2q_f32((float*)cVector, c_val);
417 
418  a_ptr += 4;
419  b_ptr += 4;
420  cVector += 4;
421  }
422 
423  for (number = quarter_points * 4; number < num_points; number++) {
424  *cVector++ = (*a_ptr++) * (*b_ptr++);
425  }
426 }
427 #endif /* LV_HAVE_NEON */
428 
429 
430 #ifdef LV_HAVE_NEON
431 
433  const lv_32fc_t* aVector,
434  const lv_32fc_t* bVector,
435  unsigned int num_points)
436 {
437  lv_32fc_t* a_ptr = (lv_32fc_t*)aVector;
438  lv_32fc_t* b_ptr = (lv_32fc_t*)bVector;
439  unsigned int quarter_points = num_points / 4;
440  float32x4x2_t a_val, b_val;
441  float32x4x2_t tmp_imag;
442  unsigned int number = 0;
443 
444  for (number = 0; number < quarter_points; ++number) {
445  a_val = vld2q_f32((float*)a_ptr); // a0r|a1r|a2r|a3r || a0i|a1i|a2i|a3i
446  b_val = vld2q_f32((float*)b_ptr); // b0r|b1r|b2r|b3r || b0i|b1i|b2i|b3i
447  __VOLK_PREFETCH(a_ptr + 4);
448  __VOLK_PREFETCH(b_ptr + 4);
449 
450  // do the first multiply
451  tmp_imag.val[1] = vmulq_f32(a_val.val[1], b_val.val[0]);
452  tmp_imag.val[0] = vmulq_f32(a_val.val[0], b_val.val[0]);
453 
454  // use multiply accumulate/subtract to get result
455  tmp_imag.val[1] = vmlaq_f32(tmp_imag.val[1], a_val.val[0], b_val.val[1]);
456  tmp_imag.val[0] = vmlsq_f32(tmp_imag.val[0], a_val.val[1], b_val.val[1]);
457 
458  // store
459  vst2q_f32((float*)cVector, tmp_imag);
460  // increment pointers
461  a_ptr += 4;
462  b_ptr += 4;
463  cVector += 4;
464  }
465 
466  for (number = quarter_points * 4; number < num_points; number++) {
467  *cVector++ = (*a_ptr++) * (*b_ptr++);
468  }
469 }
470 #endif /* LV_HAVE_NEON */
471 
472 
473 #ifdef LV_HAVE_NEONV7
474 
475 extern void volk_32fc_x2_multiply_32fc_a_neonasm(lv_32fc_t* cVector,
476  const lv_32fc_t* aVector,
477  const lv_32fc_t* bVector,
478  unsigned int num_points);
479 #endif /* LV_HAVE_NEONV7 */
480 
481 
482 #ifdef LV_HAVE_ORC
483 
484 extern void volk_32fc_x2_multiply_32fc_a_orc_impl(lv_32fc_t* cVector,
485  const lv_32fc_t* aVector,
486  const lv_32fc_t* bVector,
487  unsigned int num_points);
488 
489 static inline void volk_32fc_x2_multiply_32fc_u_orc(lv_32fc_t* cVector,
490  const lv_32fc_t* aVector,
491  const lv_32fc_t* bVector,
492  unsigned int num_points)
493 {
494  volk_32fc_x2_multiply_32fc_a_orc_impl(cVector, aVector, bVector, num_points);
495 }
496 
497 #endif /* LV_HAVE_ORC */
498 
499 #endif /* INCLUDED_volk_32fc_x2_multiply_32fc_a_H */
static void volk_32fc_x2_multiply_32fc_a_avx(lv_32fc_t *cVector, const lv_32fc_t *aVector, const lv_32fc_t *bVector, unsigned int num_points)
Definition: volk_32fc_x2_multiply_32fc.h:294
static __m256 _mm256_complexmul_ps(__m256 x, __m256 y)
Definition: volk_avx_intrinsics.h:32
static void volk_32fc_x2_multiply_32fc_a_generic(lv_32fc_t *cVector, const lv_32fc_t *aVector, const lv_32fc_t *bVector, unsigned int num_points)
Definition: volk_32fc_x2_multiply_32fc.h:363
static void volk_32fc_x2_multiply_32fc_generic(lv_32fc_t *cVector, const lv_32fc_t *aVector, const lv_32fc_t *bVector, unsigned int num_points)
Definition: volk_32fc_x2_multiply_32fc.h:209
static __m128 _mm_complexmul_ps(__m128 x, __m128 y)
Definition: volk_sse3_intrinsics.h:32
static void volk_32fc_x2_multiply_32fc_neon(lv_32fc_t *cVector, const lv_32fc_t *aVector, const lv_32fc_t *bVector, unsigned int num_points)
Definition: volk_32fc_x2_multiply_32fc.h:383
static void volk_32fc_x2_multiply_32fc_neon_opttests(lv_32fc_t *cVector, const lv_32fc_t *aVector, const lv_32fc_t *bVector, unsigned int num_points)
Definition: volk_32fc_x2_multiply_32fc.h:432
#define __VOLK_PREFETCH(addr)
Definition: volk_common.h:62
static void volk_32fc_x2_multiply_32fc_u_avx(lv_32fc_t *cVector, const lv_32fc_t *aVector, const lv_32fc_t *bVector, unsigned int num_points)
Definition: volk_32fc_x2_multiply_32fc.h:137
float complex lv_32fc_t
Definition: volk_complex.h:70
static void volk_32fc_x2_multiply_32fc_u_sse3(lv_32fc_t *cVector, const lv_32fc_t *aVector, const lv_32fc_t *bVector, unsigned int num_points)
Definition: volk_32fc_x2_multiply_32fc.h:176
static void volk_32fc_x2_multiply_32fc_a_sse3(lv_32fc_t *cVector, const lv_32fc_t *aVector, const lv_32fc_t *bVector, unsigned int num_points)
Definition: volk_32fc_x2_multiply_32fc.h:330