volk_32fc_x2_multiply_32fc.h

Go to the documentation of this file.

/* -*- c++ -*- */
/*
 * Copyright 2012, 2014 Free Software Foundation, Inc.
 *
 * This file is part of VOLK
 *
 * SPDX-License-Identifier: LGPL-3.0-or-later
 */
 
#ifndef INCLUDED_volk_32fc_x2_multiply_32fc_u_H
#define INCLUDED_volk_32fc_x2_multiply_32fc_u_H
 
#include <float.h>
#include <inttypes.h>
#include <stdio.h>
#include <volk/volk_complex.h>
 
#if LV_HAVE_AVX2 && LV_HAVE_FMA
#include <immintrin.h>
static inline void volk_32fc_x2_multiply_32fc_u_avx2_fma(lv_32fc_t* cVector,
                                                         const lv_32fc_t* aVector,
                                                         const lv_32fc_t* bVector,
                                                         unsigned int num_points)
{
    unsigned int number = 0;
    const unsigned int quarterPoints = num_points / 4;
 
    lv_32fc_t* c = cVector;
    const lv_32fc_t* a = aVector;
    const lv_32fc_t* b = bVector;
 
    for (; number < quarterPoints; number++) {
 
        const __m256 x =
            _mm256_loadu_ps((float*)a); // Load the ar + ai, br + bi as ar,ai,br,bi
        const __m256 y =
            _mm256_loadu_ps((float*)b); // Load the cr + ci, dr + di as cr,ci,dr,di
 
        const __m256 yl = _mm256_moveldup_ps(y); // Load yl with cr,cr,dr,dr
        const __m256 yh = _mm256_movehdup_ps(y); // Load yh with ci,ci,di,di
 
        const __m256 tmp2x = _mm256_permute_ps(x, 0xB1); // Re-arrange x to be ai,ar,bi,br
 
        const __m256 tmp2 = _mm256_mul_ps(tmp2x, yh); // tmp2 = ai*ci,ar*ci,bi*di,br*di
 
        const __m256 z = _mm256_fmaddsub_ps(
            x, yl, tmp2); // ar*cr-ai*ci, ai*cr+ar*ci, br*dr-bi*di, bi*dr+br*di
 
        _mm256_storeu_ps((float*)c, z); // Store the results back into the C container
 
        a += 4;
        b += 4;
        c += 4;
    }
 
    number = quarterPoints * 4;
    for (; number < num_points; number++) {
        *c++ = (*a++) * (*b++);
    }
}
#endif /* LV_HAVE_AVX2 && LV_HAVE_FMA */
 
 
#ifdef LV_HAVE_AVX
#include <immintrin.h>
#include <volk/volk_avx_intrinsics.h>
 
static inline void volk_32fc_x2_multiply_32fc_u_avx(lv_32fc_t* cVector,
                                                    const lv_32fc_t* aVector,
                                                    const lv_32fc_t* bVector,
                                                    unsigned int num_points)
{
    unsigned int number = 0;
    const unsigned int quarterPoints = num_points / 4;
 
    __m256 x, y, z;
    lv_32fc_t* c = cVector;
    const lv_32fc_t* a = aVector;
    const lv_32fc_t* b = bVector;
 
    for (; number < quarterPoints; number++) {
        x = _mm256_loadu_ps(
            (float*)a); // Load the ar + ai, br + bi ... as ar,ai,br,bi ...
        y = _mm256_loadu_ps(
            (float*)b); // Load the cr + ci, dr + di ... as cr,ci,dr,di ...
        z = _mm256_complexmul_ps(x, y);
        _mm256_storeu_ps((float*)c, z); // Store the results back into the C container
 
        a += 4;
        b += 4;
        c += 4;
    }
 
    number = quarterPoints * 4;
 
    for (; number < num_points; number++) {
        *c++ = (*a++) * (*b++);
    }
}
#endif /* LV_HAVE_AVX */
 
 
#ifdef LV_HAVE_SSE3
#include <pmmintrin.h>
#include <volk/volk_sse3_intrinsics.h>
 
static inline void volk_32fc_x2_multiply_32fc_u_sse3(lv_32fc_t* cVector,
                                                     const lv_32fc_t* aVector,
                                                     const lv_32fc_t* bVector,
                                                     unsigned int num_points)
{
    unsigned int number = 0;
    const unsigned int halfPoints = num_points / 2;
 
    __m128 x, y, z;
    lv_32fc_t* c = cVector;
    const lv_32fc_t* a = aVector;
    const lv_32fc_t* b = bVector;
 
    for (; number < halfPoints; number++) {
        x = _mm_loadu_ps((float*)a); // Load the ar + ai, br + bi as ar,ai,br,bi
        y = _mm_loadu_ps((float*)b); // Load the cr + ci, dr + di as cr,ci,dr,di
        z = _mm_complexmul_ps(x, y);
        _mm_storeu_ps((float*)c, z); // Store the results back into the C container
 
        a += 2;
        b += 2;
        c += 2;
    }
 
    if ((num_points % 2) != 0) {
        *c = (*a) * (*b);
    }
}
#endif /* LV_HAVE_SSE */
 
 
#ifdef LV_HAVE_GENERIC
 
static inline void volk_32fc_x2_multiply_32fc_generic(lv_32fc_t* cVector,
                                                      const lv_32fc_t* aVector,
                                                      const lv_32fc_t* bVector,
                                                      unsigned int num_points)
{
    lv_32fc_t* cPtr = cVector;
    const lv_32fc_t* aPtr = aVector;
    const lv_32fc_t* bPtr = bVector;
    unsigned int number = 0;
 
    for (number = 0; number < num_points; number++) {
        *cPtr++ = (*aPtr++) * (*bPtr++);
    }
}
#endif /* LV_HAVE_GENERIC */
 
 
#endif /* INCLUDED_volk_32fc_x2_multiply_32fc_u_H */
#ifndef INCLUDED_volk_32fc_x2_multiply_32fc_a_H
#define INCLUDED_volk_32fc_x2_multiply_32fc_a_H
 
#include <float.h>
#include <inttypes.h>
#include <stdio.h>
#include <volk/volk_complex.h>
 
#if LV_HAVE_AVX2 && LV_HAVE_FMA
#include <immintrin.h>
static inline void volk_32fc_x2_multiply_32fc_a_avx2_fma(lv_32fc_t* cVector,
                                                         const lv_32fc_t* aVector,
                                                         const lv_32fc_t* bVector,
                                                         unsigned int num_points)
{
    unsigned int number = 0;
    const unsigned int quarterPoints = num_points / 4;
 
    lv_32fc_t* c = cVector;
    const lv_32fc_t* a = aVector;
    const lv_32fc_t* b = bVector;
 
    for (; number < quarterPoints; number++) {
 
        const __m256 x =
            _mm256_load_ps((float*)a); // Load the ar + ai, br + bi as ar,ai,br,bi
        const __m256 y =
            _mm256_load_ps((float*)b); // Load the cr + ci, dr + di as cr,ci,dr,di
 
        const __m256 yl = _mm256_moveldup_ps(y); // Load yl with cr,cr,dr,dr
        const __m256 yh = _mm256_movehdup_ps(y); // Load yh with ci,ci,di,di
 
        const __m256 tmp2x = _mm256_permute_ps(x, 0xB1); // Re-arrange x to be ai,ar,bi,br
 
        const __m256 tmp2 = _mm256_mul_ps(tmp2x, yh); // tmp2 = ai*ci,ar*ci,bi*di,br*di
 
        const __m256 z = _mm256_fmaddsub_ps(
            x, yl, tmp2); // ar*cr-ai*ci, ai*cr+ar*ci, br*dr-bi*di, bi*dr+br*di
 
        _mm256_store_ps((float*)c, z); // Store the results back into the C container
 
        a += 4;
        b += 4;
        c += 4;
    }
 
    number = quarterPoints * 4;
    for (; number < num_points; number++) {
        *c++ = (*a++) * (*b++);
    }
}
#endif /* LV_HAVE_AVX2 && LV_HAVE_FMA */
 
 
#ifdef LV_HAVE_AVX
#include <immintrin.h>
#include <volk/volk_avx_intrinsics.h>
 
static inline void volk_32fc_x2_multiply_32fc_a_avx(lv_32fc_t* cVector,
                                                    const lv_32fc_t* aVector,
                                                    const lv_32fc_t* bVector,
                                                    unsigned int num_points)
{
    unsigned int number = 0;
    const unsigned int quarterPoints = num_points / 4;
 
    __m256 x, y, z;
    lv_32fc_t* c = cVector;
    const lv_32fc_t* a = aVector;
    const lv_32fc_t* b = bVector;
 
    for (; number < quarterPoints; number++) {
        x = _mm256_load_ps((float*)a); // Load the ar + ai, br + bi ... as ar,ai,br,bi ...
        y = _mm256_load_ps((float*)b); // Load the cr + ci, dr + di ... as cr,ci,dr,di ...
        z = _mm256_complexmul_ps(x, y);
        _mm256_store_ps((float*)c, z); // Store the results back into the C container
 
        a += 4;
        b += 4;
        c += 4;
    }
 
    number = quarterPoints * 4;
 
    for (; number < num_points; number++) {
        *c++ = (*a++) * (*b++);
    }
}
#endif /* LV_HAVE_AVX */
 
#ifdef LV_HAVE_SSE3
#include <pmmintrin.h>
#include <volk/volk_sse3_intrinsics.h>
 
static inline void volk_32fc_x2_multiply_32fc_a_sse3(lv_32fc_t* cVector,
                                                     const lv_32fc_t* aVector,
                                                     const lv_32fc_t* bVector,
                                                     unsigned int num_points)
{
    unsigned int number = 0;
    const unsigned int halfPoints = num_points / 2;
 
    __m128 x, y, z;
    lv_32fc_t* c = cVector;
    const lv_32fc_t* a = aVector;
    const lv_32fc_t* b = bVector;
 
    for (; number < halfPoints; number++) {
        x = _mm_load_ps((float*)a); // Load the ar + ai, br + bi as ar,ai,br,bi
        y = _mm_load_ps((float*)b); // Load the cr + ci, dr + di as cr,ci,dr,di
        z = _mm_complexmul_ps(x, y);
        _mm_store_ps((float*)c, z); // Store the results back into the C container
 
        a += 2;
        b += 2;
        c += 2;
    }
 
    if ((num_points % 2) != 0) {
        *c = (*a) * (*b);
    }
}
#endif /* LV_HAVE_SSE */
 
 
#ifdef LV_HAVE_GENERIC
 
static inline void volk_32fc_x2_multiply_32fc_a_generic(lv_32fc_t* cVector,
                                                        const lv_32fc_t* aVector,
                                                        const lv_32fc_t* bVector,
                                                        unsigned int num_points)
{
    lv_32fc_t* cPtr = cVector;
    const lv_32fc_t* aPtr = aVector;
    const lv_32fc_t* bPtr = bVector;
    unsigned int number = 0;
 
    for (number = 0; number < num_points; number++) {
        *cPtr++ = (*aPtr++) * (*bPtr++);
    }
}
#endif /* LV_HAVE_GENERIC */
 
 
#ifdef LV_HAVE_NEON
#include <arm_neon.h>
 
static inline void volk_32fc_x2_multiply_32fc_neon(lv_32fc_t* cVector,
                                                   const lv_32fc_t* aVector,
                                                   const lv_32fc_t* bVector,
                                                   unsigned int num_points)
{
    lv_32fc_t* a_ptr = (lv_32fc_t*)aVector;
    lv_32fc_t* b_ptr = (lv_32fc_t*)bVector;
    unsigned int quarter_points = num_points / 4;
    float32x4x2_t a_val, b_val, c_val;
    float32x4x2_t tmp_real, tmp_imag;
    unsigned int number = 0;
 
    for (number = 0; number < quarter_points; ++number) {
        a_val = vld2q_f32((float*)a_ptr); // a0r|a1r|a2r|a3r || a0i|a1i|a2i|a3i
        b_val = vld2q_f32((float*)b_ptr); // b0r|b1r|b2r|b3r || b0i|b1i|b2i|b3i
        __VOLK_PREFETCH(a_ptr + 4);
        __VOLK_PREFETCH(b_ptr + 4);
 
        // multiply the real*real and imag*imag to get real result
        // a0r*b0r|a1r*b1r|a2r*b2r|a3r*b3r
        tmp_real.val[0] = vmulq_f32(a_val.val[0], b_val.val[0]);
        // a0i*b0i|a1i*b1i|a2i*b2i|a3i*b3i
        tmp_real.val[1] = vmulq_f32(a_val.val[1], b_val.val[1]);
 
        // Multiply cross terms to get the imaginary result
        // a0r*b0i|a1r*b1i|a2r*b2i|a3r*b3i
        tmp_imag.val[0] = vmulq_f32(a_val.val[0], b_val.val[1]);
        // a0i*b0r|a1i*b1r|a2i*b2r|a3i*b3r
        tmp_imag.val[1] = vmulq_f32(a_val.val[1], b_val.val[0]);
 
        // store the results
        c_val.val[0] = vsubq_f32(tmp_real.val[0], tmp_real.val[1]);
        c_val.val[1] = vaddq_f32(tmp_imag.val[0], tmp_imag.val[1]);
        vst2q_f32((float*)cVector, c_val);
 
        a_ptr += 4;
        b_ptr += 4;
        cVector += 4;
    }
 
    for (number = quarter_points * 4; number < num_points; number++) {
        *cVector++ = (*a_ptr++) * (*b_ptr++);
    }
}
#endif /* LV_HAVE_NEON */
 
 
#ifdef LV_HAVE_NEON
 
static inline void volk_32fc_x2_multiply_32fc_neon_opttests(lv_32fc_t* cVector,
                                                            const lv_32fc_t* aVector,
                                                            const lv_32fc_t* bVector,
                                                            unsigned int num_points)
{
    lv_32fc_t* a_ptr = (lv_32fc_t*)aVector;
    lv_32fc_t* b_ptr = (lv_32fc_t*)bVector;
    unsigned int quarter_points = num_points / 4;
    float32x4x2_t a_val, b_val;
    float32x4x2_t tmp_imag;
    unsigned int number = 0;
 
    for (number = 0; number < quarter_points; ++number) {
        a_val = vld2q_f32((float*)a_ptr); // a0r|a1r|a2r|a3r || a0i|a1i|a2i|a3i
        b_val = vld2q_f32((float*)b_ptr); // b0r|b1r|b2r|b3r || b0i|b1i|b2i|b3i
        __VOLK_PREFETCH(a_ptr + 4);
        __VOLK_PREFETCH(b_ptr + 4);
 
        // do the first multiply
        tmp_imag.val[1] = vmulq_f32(a_val.val[1], b_val.val[0]);
        tmp_imag.val[0] = vmulq_f32(a_val.val[0], b_val.val[0]);
 
        // use multiply accumulate/subtract to get result
        tmp_imag.val[1] = vmlaq_f32(tmp_imag.val[1], a_val.val[0], b_val.val[1]);
        tmp_imag.val[0] = vmlsq_f32(tmp_imag.val[0], a_val.val[1], b_val.val[1]);
 
        // store
        vst2q_f32((float*)cVector, tmp_imag);
        // increment pointers
        a_ptr += 4;
        b_ptr += 4;
        cVector += 4;
    }
 
    for (number = quarter_points * 4; number < num_points; number++) {
        *cVector++ = (*a_ptr++) * (*b_ptr++);
    }
}
#endif /* LV_HAVE_NEON */
 
 
#ifdef LV_HAVE_NEONV7
 
extern void volk_32fc_x2_multiply_32fc_a_neonasm(lv_32fc_t* cVector,
                                                 const lv_32fc_t* aVector,
                                                 const lv_32fc_t* bVector,
                                                 unsigned int num_points);
#endif /* LV_HAVE_NEONV7 */
 
 
#ifdef LV_HAVE_ORC
 
extern void volk_32fc_x2_multiply_32fc_a_orc_impl(lv_32fc_t* cVector,
                                                  const lv_32fc_t* aVector,
                                                  const lv_32fc_t* bVector,
                                                  unsigned int num_points);
 
static inline void volk_32fc_x2_multiply_32fc_u_orc(lv_32fc_t* cVector,
                                                    const lv_32fc_t* aVector,
                                                    const lv_32fc_t* bVector,
                                                    unsigned int num_points)
{
    volk_32fc_x2_multiply_32fc_a_orc_impl(cVector, aVector, bVector, num_points);
}
 
#endif /* LV_HAVE_ORC */
 
#endif /* INCLUDED_volk_32fc_x2_multiply_32fc_a_H */