349 lines
9.5 KiB
C
349 lines
9.5 KiB
C
|
/* ----------------------------------------------------------------------
|
||
|
|
* Project: CMSIS DSP Library
|
||
|
|
* Title: arm_helium_utils.h
|
||
|
|
* Description: Utility functions for Helium development
|
||
|
|
*
|
||
|
|
* $Date: 09. September 2019
|
||
|
|
* $Revision: V.1.5.1
|
||
|
|
*
|
||
|
|
* Target Processor: Cortex-M cores
|
||
|
|
* -------------------------------------------------------------------- */
|
||
|
|
/*
|
||
|
|
* Copyright (C) 2010-2019 ARM Limited or its affiliates. All rights reserved.
|
||
|
|
*
|
||
|
|
* SPDX-License-Identifier: Apache-2.0
|
||
|
|
*
|
||
|
|
* Licensed under the Apache License, Version 2.0 (the License); you may
|
||
|
|
* not use this file except in compliance with the License.
|
||
|
|
* You may obtain a copy of the License at
|
||
|
|
*
|
||
|
|
* www.apache.org/licenses/LICENSE-2.0
|
||
|
|
*
|
||
|
|
* Unless required by applicable law or agreed to in writing, software
|
||
|
|
* distributed under the License is distributed on an AS IS BASIS, WITHOUT
|
||
|
|
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
|
|
* See the License for the specific language governing permissions and
|
||
|
|
* limitations under the License.
|
||
|
|
*/
|
||
|
|
|
||
|
|
#ifndef _ARM_UTILS_HELIUM_H_
|
||
|
|
#define _ARM_UTILS_HELIUM_H_
|
||
|
|
|
||
|
|
/***************************************
|
||
|
|
|
||
|
|
Definitions available for MVEF and MVEI
|
||
|
|
|
||
|
|
***************************************/
|
||
|
|
#if defined (ARM_MATH_HELIUM) || defined(ARM_MATH_MVEF) || defined(ARM_MATH_MVEI)
|
||
|
|
|
||
|
|
#define INACTIVELANE 0 /* inactive lane content */
|
||
|
|
|
||
|
|
|
||
|
|
#endif /* defined (ARM_MATH_HELIUM) || defined(ARM_MATH_MVEF) || defined(ARM_MATH_MVEI) */
|
||
|
|
|
||
|
|
/***************************************
|
||
|
|
|
||
|
|
Definitions available for MVEF only
|
||
|
|
|
||
|
|
***************************************/
|
||
|
|
#if defined (ARM_MATH_HELIUM) || defined(ARM_MATH_MVEF)
|
||
|
|
|
||
|
|
__STATIC_FORCEINLINE float32_t vecAddAcrossF32Mve(float32x4_t in)
|
||
|
|
{
|
||
|
|
float32_t acc;
|
||
|
|
|
||
|
|
acc = vgetq_lane(in, 0) + vgetq_lane(in, 1) +
|
||
|
|
vgetq_lane(in, 2) + vgetq_lane(in, 3);
|
||
|
|
|
||
|
|
return acc;
|
||
|
|
}
|
||
|
|
|
||
|
|
/* newton initial guess */
|
||
|
|
#define INVSQRT_MAGIC_F32 0x5f3759df
|
||
|
|
|
||
|
|
#define INVSQRT_NEWTON_MVE_F32(invSqrt, xHalf, xStart)\
|
||
|
|
{ \
|
||
|
|
float32x4_t tmp; \
|
||
|
|
\
|
||
|
|
/* tmp = xhalf * x * x */ \
|
||
|
|
tmp = vmulq(xStart, xStart); \
|
||
|
|
tmp = vmulq(tmp, xHalf); \
|
||
|
|
/* (1.5f - xhalf * x * x) */ \
|
||
|
|
tmp = vsubq(vdupq_n_f32(1.5f), tmp); \
|
||
|
|
/* x = x*(1.5f-xhalf*x*x); */ \
|
||
|
|
invSqrt = vmulq(tmp, xStart); \
|
||
|
|
}
|
||
|
|
#endif /* defined (ARM_MATH_HELIUM) || defined(ARM_MATH_MVEF) */
|
||
|
|
|
||
|
|
/***************************************
|
||
|
|
|
||
|
|
Definitions available for MVEI only
|
||
|
|
|
||
|
|
***************************************/
|
||
|
|
#if defined (ARM_MATH_HELIUM) || defined(ARM_MATH_MVEI)
|
||
|
|
|
||
|
|
|
||
|
|
#include "arm_common_tables.h"
|
||
|
|
|
||
|
|
/* Following functions are used to transpose matrix in f32 and q31 cases */
|
||
|
|
__STATIC_INLINE arm_status arm_mat_trans_32bit_2x2_mve(
|
||
|
|
uint32_t * pDataSrc,
|
||
|
|
uint32_t * pDataDest)
|
||
|
|
{
|
||
|
|
static const uint32x4_t vecOffs = { 0, 2, 1, 3 };
|
||
|
|
/*
|
||
|
|
*
|
||
|
|
* | 0 1 | => | 0 2 |
|
||
|
|
* | 2 3 | | 1 3 |
|
||
|
|
*
|
||
|
|
*/
|
||
|
|
uint32x4_t vecIn = vldrwq_u32((uint32_t const *)pDataSrc);
|
||
|
|
vstrwq_scatter_shifted_offset_u32(pDataDest, vecOffs, vecIn);
|
||
|
|
|
||
|
|
return (ARM_MATH_SUCCESS);
|
||
|
|
}
|
||
|
|
|
||
|
|
__STATIC_INLINE arm_status arm_mat_trans_32bit_3x3_mve(
|
||
|
|
uint32_t * pDataSrc,
|
||
|
|
uint32_t * pDataDest)
|
||
|
|
{
|
||
|
|
const uint32x4_t vecOffs1 = { 0, 3, 6, 1};
|
||
|
|
const uint32x4_t vecOffs2 = { 4, 7, 2, 5};
|
||
|
|
/*
|
||
|
|
*
|
||
|
|
* | 0 1 2 | | 0 3 6 | 4 x 32 flattened version | 0 3 6 1 |
|
||
|
|
* | 3 4 5 | => | 1 4 7 | => | 4 7 2 5 |
|
||
|
|
* | 6 7 8 | | 2 5 8 | (row major) | 8 . . . |
|
||
|
|
*
|
||
|
|
*/
|
||
|
|
uint32x4_t vecIn1 = vldrwq_u32((uint32_t const *) pDataSrc);
|
||
|
|
uint32x4_t vecIn2 = vldrwq_u32((uint32_t const *) &pDataSrc[4]);
|
||
|
|
|
||
|
|
vstrwq_scatter_shifted_offset_u32(pDataDest, vecOffs1, vecIn1);
|
||
|
|
vstrwq_scatter_shifted_offset_u32(pDataDest, vecOffs2, vecIn2);
|
||
|
|
|
||
|
|
pDataDest[8] = pDataSrc[8];
|
||
|
|
|
||
|
|
return (ARM_MATH_SUCCESS);
|
||
|
|
}
|
||
|
|
|
||
|
|
__STATIC_INLINE arm_status arm_mat_trans_32bit_4x4_mve(uint32_t * pDataSrc, uint32_t * pDataDest)
|
||
|
|
{
|
||
|
|
/*
|
||
|
|
* 4x4 Matrix transposition
|
||
|
|
* is 4 x de-interleave operation
|
||
|
|
*
|
||
|
|
* 0 1 2 3 0 4 8 12
|
||
|
|
* 4 5 6 7 1 5 9 13
|
||
|
|
* 8 9 10 11 2 6 10 14
|
||
|
|
* 12 13 14 15 3 7 11 15
|
||
|
|
*/
|
||
|
|
|
||
|
|
uint32x4x4_t vecIn;
|
||
|
|
|
||
|
|
vecIn = vld4q((uint32_t const *) pDataSrc);
|
||
|
|
vstrwq(pDataDest, vecIn.val[0]);
|
||
|
|
pDataDest += 4;
|
||
|
|
vstrwq(pDataDest, vecIn.val[1]);
|
||
|
|
pDataDest += 4;
|
||
|
|
vstrwq(pDataDest, vecIn.val[2]);
|
||
|
|
pDataDest += 4;
|
||
|
|
vstrwq(pDataDest, vecIn.val[3]);
|
||
|
|
|
||
|
|
return (ARM_MATH_SUCCESS);
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
__STATIC_INLINE arm_status arm_mat_trans_32bit_generic_mve(
|
||
|
|
uint16_t srcRows,
|
||
|
|
uint16_t srcCols,
|
||
|
|
uint32_t * pDataSrc,
|
||
|
|
uint32_t * pDataDest)
|
||
|
|
{
|
||
|
|
uint32x4_t vecOffs;
|
||
|
|
uint32_t i;
|
||
|
|
uint32_t blkCnt;
|
||
|
|
uint32_t const *pDataC;
|
||
|
|
uint32_t *pDataDestR;
|
||
|
|
uint32x4_t vecIn;
|
||
|
|
|
||
|
|
vecOffs = vidupq_u32((uint32_t)0, 1);
|
||
|
|
vecOffs = vecOffs * srcCols;
|
||
|
|
|
||
|
|
i = srcCols;
|
||
|
|
do
|
||
|
|
{
|
||
|
|
pDataC = (uint32_t const *) pDataSrc;
|
||
|
|
pDataDestR = pDataDest;
|
||
|
|
|
||
|
|
blkCnt = srcRows >> 2;
|
||
|
|
while (blkCnt > 0U)
|
||
|
|
{
|
||
|
|
vecIn = vldrwq_gather_shifted_offset_u32(pDataC, vecOffs);
|
||
|
|
vstrwq(pDataDestR, vecIn);
|
||
|
|
pDataDestR += 4;
|
||
|
|
pDataC = pDataC + srcCols * 4;
|
||
|
|
/*
|
||
|
|
* Decrement the blockSize loop counter
|
||
|
|
*/
|
||
|
|
blkCnt--;
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* tail
|
||
|
|
*/
|
||
|
|
blkCnt = srcRows & 3;
|
||
|
|
if (blkCnt > 0U)
|
||
|
|
{
|
||
|
|
mve_pred16_t p0 = vctp32q(blkCnt);
|
||
|
|
vecIn = vldrwq_gather_shifted_offset_u32(pDataC, vecOffs);
|
||
|
|
vstrwq_p(pDataDestR, vecIn, p0);
|
||
|
|
}
|
||
|
|
|
||
|
|
pDataSrc += 1;
|
||
|
|
pDataDest += srcRows;
|
||
|
|
}
|
||
|
|
while (--i);
|
||
|
|
|
||
|
|
return (ARM_MATH_SUCCESS);
|
||
|
|
}
|
||
|
|
|
||
|
|
#if !defined(ARM_DSP_CONFIG_TABLES) || defined(ARM_ALL_FAST_TABLES) || defined(ARM_TABLE_FAST_SQRT_Q31_MVE)
|
||
|
|
__STATIC_INLINE q31x4_t FAST_VSQRT_Q31(q31x4_t vecIn)
|
||
|
|
{
|
||
|
|
q63x2_t vecTmpLL;
|
||
|
|
q31x4_t vecTmp0, vecTmp1;
|
||
|
|
q31_t scale;
|
||
|
|
q63_t tmp64;
|
||
|
|
q31x4_t vecNrm, vecDst, vecIdx, vecSignBits;
|
||
|
|
|
||
|
|
|
||
|
|
vecSignBits = vclsq(vecIn);
|
||
|
|
vecSignBits = vbicq(vecSignBits, 1);
|
||
|
|
/*
|
||
|
|
* in = in << no_of_sign_bits;
|
||
|
|
*/
|
||
|
|
vecNrm = vshlq(vecIn, vecSignBits);
|
||
|
|
/*
|
||
|
|
* index = in >> 24;
|
||
|
|
*/
|
||
|
|
vecIdx = vecNrm >> 24;
|
||
|
|
vecIdx = vecIdx << 1;
|
||
|
|
|
||
|
|
vecTmp0 = vldrwq_gather_shifted_offset_s32(sqrtTable_Q31, vecIdx);
|
||
|
|
|
||
|
|
vecIdx = vecIdx + 1;
|
||
|
|
|
||
|
|
vecTmp1 = vldrwq_gather_shifted_offset_s32(sqrtTable_Q31, vecIdx);
|
||
|
|
|
||
|
|
vecTmp1 = vqrdmulhq(vecTmp1, vecNrm);
|
||
|
|
vecTmp0 = vecTmp0 - vecTmp1;
|
||
|
|
vecTmp1 = vqrdmulhq(vecTmp0, vecTmp0);
|
||
|
|
vecTmp1 = vqrdmulhq(vecNrm, vecTmp1);
|
||
|
|
vecTmp1 = vdupq_n_s32(0x18000000) - vecTmp1;
|
||
|
|
vecTmp0 = vqrdmulhq(vecTmp0, vecTmp1);
|
||
|
|
vecTmpLL = vmullbq_int(vecNrm, vecTmp0);
|
||
|
|
|
||
|
|
/*
|
||
|
|
* scale elements 0, 2
|
||
|
|
*/
|
||
|
|
scale = 26 + (vecSignBits[0] >> 1);
|
||
|
|
tmp64 = asrl(vecTmpLL[0], scale);
|
||
|
|
vecDst[0] = (q31_t) tmp64;
|
||
|
|
|
||
|
|
scale = 26 + (vecSignBits[2] >> 1);
|
||
|
|
tmp64 = asrl(vecTmpLL[1], scale);
|
||
|
|
vecDst[2] = (q31_t) tmp64;
|
||
|
|
|
||
|
|
vecTmpLL = vmulltq_int(vecNrm, vecTmp0);
|
||
|
|
|
||
|
|
/*
|
||
|
|
* scale elements 1, 3
|
||
|
|
*/
|
||
|
|
scale = 26 + (vecSignBits[1] >> 1);
|
||
|
|
tmp64 = asrl(vecTmpLL[0], scale);
|
||
|
|
vecDst[1] = (q31_t) tmp64;
|
||
|
|
|
||
|
|
scale = 26 + (vecSignBits[3] >> 1);
|
||
|
|
tmp64 = asrl(vecTmpLL[1], scale);
|
||
|
|
vecDst[3] = (q31_t) tmp64;
|
||
|
|
/*
|
||
|
|
* set negative values to 0
|
||
|
|
*/
|
||
|
|
vecDst = vdupq_m(vecDst, 0, vcmpltq_n_s32(vecIn, 0));
|
||
|
|
|
||
|
|
return vecDst;
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
|
||
|
|
#if !defined(ARM_DSP_CONFIG_TABLES) || defined(ARM_ALL_FAST_TABLES) || defined(ARM_TABLE_FAST_SQRT_Q15_MVE)
|
||
|
|
__STATIC_INLINE q15x8_t FAST_VSQRT_Q15(q15x8_t vecIn)
|
||
|
|
{
|
||
|
|
q31x4_t vecTmpLev, vecTmpLodd, vecSignL;
|
||
|
|
q15x8_t vecTmp0, vecTmp1;
|
||
|
|
q15x8_t vecNrm, vecDst, vecIdx, vecSignBits;
|
||
|
|
|
||
|
|
vecDst = vuninitializedq_s16();
|
||
|
|
|
||
|
|
vecSignBits = vclsq(vecIn);
|
||
|
|
vecSignBits = vbicq(vecSignBits, 1);
|
||
|
|
/*
|
||
|
|
* in = in << no_of_sign_bits;
|
||
|
|
*/
|
||
|
|
vecNrm = vshlq(vecIn, vecSignBits);
|
||
|
|
|
||
|
|
vecIdx = vecNrm >> 8;
|
||
|
|
vecIdx = vecIdx << 1;
|
||
|
|
|
||
|
|
vecTmp0 = vldrhq_gather_shifted_offset_s16(sqrtTable_Q15, vecIdx);
|
||
|
|
|
||
|
|
vecIdx = vecIdx + 1;
|
||
|
|
|
||
|
|
vecTmp1 = vldrhq_gather_shifted_offset_s16(sqrtTable_Q15, vecIdx);
|
||
|
|
|
||
|
|
vecTmp1 = vqrdmulhq(vecTmp1, vecNrm);
|
||
|
|
vecTmp0 = vecTmp0 - vecTmp1;
|
||
|
|
vecTmp1 = vqrdmulhq(vecTmp0, vecTmp0);
|
||
|
|
vecTmp1 = vqrdmulhq(vecNrm, vecTmp1);
|
||
|
|
vecTmp1 = vdupq_n_s16(0x1800) - vecTmp1;
|
||
|
|
vecTmp0 = vqrdmulhq(vecTmp0, vecTmp1);
|
||
|
|
|
||
|
|
vecSignBits = vecSignBits >> 1;
|
||
|
|
|
||
|
|
vecTmpLev = vmullbq_int(vecNrm, vecTmp0);
|
||
|
|
vecTmpLodd = vmulltq_int(vecNrm, vecTmp0);
|
||
|
|
|
||
|
|
vecTmp0 = vecSignBits + 10;
|
||
|
|
/*
|
||
|
|
* negate sign to apply register based vshl
|
||
|
|
*/
|
||
|
|
vecTmp0 = -vecTmp0;
|
||
|
|
|
||
|
|
/*
|
||
|
|
* shift even elements
|
||
|
|
*/
|
||
|
|
vecSignL = vmovlbq(vecTmp0);
|
||
|
|
vecTmpLev = vshlq(vecTmpLev, vecSignL);
|
||
|
|
/*
|
||
|
|
* shift odd elements
|
||
|
|
*/
|
||
|
|
vecSignL = vmovltq(vecTmp0);
|
||
|
|
vecTmpLodd = vshlq(vecTmpLodd, vecSignL);
|
||
|
|
/*
|
||
|
|
* merge and narrow odd and even parts
|
||
|
|
*/
|
||
|
|
vecDst = vmovnbq_s32(vecDst, vecTmpLev);
|
||
|
|
vecDst = vmovntq_s32(vecDst, vecTmpLodd);
|
||
|
|
/*
|
||
|
|
* set negative values to 0
|
||
|
|
*/
|
||
|
|
vecDst = vdupq_m(vecDst, 0, vcmpltq_n_s16(vecIn, 0));
|
||
|
|
|
||
|
|
return vecDst;
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
|
||
|
|
#endif /* defined (ARM_MATH_HELIUM) || defined(ARM_MATH_MVEI) */
|
||
|
|
|
||
|
|
#endif
|