1215 lines
51 KiB
C
1215 lines
51 KiB
C
|
/*
|
||
|
|
* Copyright (C) 2013 The Android Open Source Project
|
||
|
|
*
|
||
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||
|
|
* you may not use this file except in compliance with the License.
|
||
|
|
* You may obtain a copy of the License at
|
||
|
|
*
|
||
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||
|
|
*
|
||
|
|
* Unless required by applicable law or agreed to in writing, software
|
||
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
|
|
* See the License for the specific language governing permissions and
|
||
|
|
* limitations under the License.
|
||
|
|
*/
|
||
|
|
|
||
|
|
#ifndef ANDROID_AUDIO_RESAMPLER_FIR_PROCESS_NEON_H
|
||
|
|
#define ANDROID_AUDIO_RESAMPLER_FIR_PROCESS_NEON_H
|
||
|
|
|
||
|
|
namespace android {
|
||
|
|
|
||
|
|
// depends on AudioResamplerFirOps.h, AudioResamplerFirProcess.h
|
||
|
|
|
||
|
|
#if USE_NEON
|
||
|
|
|
||
|
|
// use intrinsics if inline arm32 assembly is not possible
|
||
|
|
#if !USE_INLINE_ASSEMBLY
|
||
|
|
#define USE_INTRINSIC
|
||
|
|
#endif
|
||
|
|
|
||
|
|
// following intrinsics available only on ARM 64 bit ACLE
|
||
|
|
#ifndef __aarch64__
|
||
|
|
#undef vld1q_f32_x2
|
||
|
|
#undef vld1q_s32_x2
|
||
|
|
#endif
|
||
|
|
|
||
|
|
#define TO_STRING2(x) #x
|
||
|
|
#define TO_STRING(x) TO_STRING2(x)
|
||
|
|
// uncomment to print GCC version, may be relevant for intrinsic optimizations
|
||
|
|
/* #pragma message ("GCC version: " TO_STRING(__GNUC__) \
|
||
|
|
"." TO_STRING(__GNUC_MINOR__) \
|
||
|
|
"." TO_STRING(__GNUC_PATCHLEVEL__)) */
|
||
|
|
|
||
|
|
//
|
||
|
|
// NEON specializations are enabled for Process() and ProcessL() in AudioResamplerFirProcess.h
|
||
|
|
//
|
||
|
|
// Two variants are presented here:
|
||
|
|
// ARM NEON inline assembly which appears up to 10-15% faster than intrinsics (gcc 4.9) for arm32.
|
||
|
|
// ARM NEON intrinsics which can also be used by arm64 and x86/64 with NEON header.
|
||
|
|
//
|
||
|
|
|
||
|
|
// Macros to save a mono/stereo accumulator sample in q0 (and q4) as stereo out.
|
||
|
|
// These are only used for inline assembly.
|
||
|
|
#define ASSEMBLY_ACCUMULATE_MONO \
|
||
|
|
"vld1.s32 {d2}, [%[vLR]:64] \n"/* (1) load volumes */\
|
||
|
|
"vld1.s32 {d3}, %[out] \n"/* (2) unaligned load the output */\
|
||
|
|
"vpadd.s32 d0, d0, d1 \n"/* (1) add all 4 partial sums */\
|
||
|
|
"vpadd.s32 d0, d0, d0 \n"/* (1+4d) and replicate L/R */\
|
||
|
|
"vqrdmulh.s32 d0, d0, d2 \n"/* (2+3d) apply volume */\
|
||
|
|
"vqadd.s32 d3, d3, d0 \n"/* (1+4d) accumulate result (saturating) */\
|
||
|
|
"vst1.s32 {d3}, %[out] \n"/* (2+2d) store result */
|
||
|
|
|
||
|
|
#define ASSEMBLY_ACCUMULATE_STEREO \
|
||
|
|
"vld1.s32 {d2}, [%[vLR]:64] \n"/* (1) load volumes*/\
|
||
|
|
"vld1.s32 {d3}, %[out] \n"/* (2) unaligned load the output*/\
|
||
|
|
"vpadd.s32 d0, d0, d1 \n"/* (1) add all 4 partial sums from q0*/\
|
||
|
|
"vpadd.s32 d8, d8, d9 \n"/* (1) add all 4 partial sums from q4*/\
|
||
|
|
"vpadd.s32 d0, d0, d8 \n"/* (1+4d) combine into L/R*/\
|
||
|
|
"vqrdmulh.s32 d0, d0, d2 \n"/* (2+3d) apply volume*/\
|
||
|
|
"vqadd.s32 d3, d3, d0 \n"/* (1+4d) accumulate result (saturating)*/\
|
||
|
|
"vst1.s32 {d3}, %[out] \n"/* (2+2d)store result*/
|
||
|
|
|
||
|
|
template <int CHANNELS, int STRIDE, bool FIXED>
|
||
|
|
static inline void ProcessNeonIntrinsic(int32_t* out,
|
||
|
|
int count,
|
||
|
|
const int16_t* coefsP,
|
||
|
|
const int16_t* coefsN,
|
||
|
|
const int16_t* sP,
|
||
|
|
const int16_t* sN,
|
||
|
|
const int32_t* volumeLR,
|
||
|
|
uint32_t lerpP,
|
||
|
|
const int16_t* coefsP1,
|
||
|
|
const int16_t* coefsN1)
|
||
|
|
{
|
||
|
|
ALOG_ASSERT(count > 0 && (count & 7) == 0); // multiple of 8
|
||
|
|
static_assert(CHANNELS == 1 || CHANNELS == 2, "CHANNELS must be 1 or 2");
|
||
|
|
|
||
|
|
sP -= CHANNELS*((STRIDE>>1)-1);
|
||
|
|
coefsP = (const int16_t*)__builtin_assume_aligned(coefsP, 16);
|
||
|
|
coefsN = (const int16_t*)__builtin_assume_aligned(coefsN, 16);
|
||
|
|
|
||
|
|
int16x4_t interp;
|
||
|
|
if (!FIXED) {
|
||
|
|
interp = vdup_n_s16(lerpP);
|
||
|
|
//interp = (int16x4_t)vset_lane_s32 ((int32x2_t)lerpP, interp, 0);
|
||
|
|
coefsP1 = (const int16_t*)__builtin_assume_aligned(coefsP1, 16);
|
||
|
|
coefsN1 = (const int16_t*)__builtin_assume_aligned(coefsN1, 16);
|
||
|
|
}
|
||
|
|
int32x4_t accum, accum2;
|
||
|
|
// warning uninitialized if we use veorq_s32
|
||
|
|
// (alternative to below) accum = veorq_s32(accum, accum);
|
||
|
|
accum = vdupq_n_s32(0);
|
||
|
|
if (CHANNELS == 2) {
|
||
|
|
// (alternative to below) accum2 = veorq_s32(accum2, accum2);
|
||
|
|
accum2 = vdupq_n_s32(0);
|
||
|
|
}
|
||
|
|
do {
|
||
|
|
int16x8_t posCoef = vld1q_s16(coefsP);
|
||
|
|
coefsP += 8;
|
||
|
|
int16x8_t negCoef = vld1q_s16(coefsN);
|
||
|
|
coefsN += 8;
|
||
|
|
if (!FIXED) { // interpolate
|
||
|
|
int16x8_t posCoef1 = vld1q_s16(coefsP1);
|
||
|
|
coefsP1 += 8;
|
||
|
|
int16x8_t negCoef1 = vld1q_s16(coefsN1);
|
||
|
|
coefsN1 += 8;
|
||
|
|
|
||
|
|
posCoef1 = vsubq_s16(posCoef1, posCoef);
|
||
|
|
negCoef = vsubq_s16(negCoef, negCoef1);
|
||
|
|
|
||
|
|
posCoef1 = vqrdmulhq_lane_s16(posCoef1, interp, 0);
|
||
|
|
negCoef = vqrdmulhq_lane_s16(negCoef, interp, 0);
|
||
|
|
|
||
|
|
posCoef = vaddq_s16(posCoef, posCoef1);
|
||
|
|
negCoef = vaddq_s16(negCoef, negCoef1);
|
||
|
|
}
|
||
|
|
switch (CHANNELS) {
|
||
|
|
case 1: {
|
||
|
|
int16x8_t posSamp = vld1q_s16(sP);
|
||
|
|
int16x8_t negSamp = vld1q_s16(sN);
|
||
|
|
sN += 8;
|
||
|
|
posSamp = vrev64q_s16(posSamp);
|
||
|
|
|
||
|
|
// dot product
|
||
|
|
accum = vmlal_s16(accum, vget_low_s16(posSamp), vget_high_s16(posCoef)); // reversed
|
||
|
|
accum = vmlal_s16(accum, vget_high_s16(posSamp), vget_low_s16(posCoef)); // reversed
|
||
|
|
accum = vmlal_s16(accum, vget_low_s16(negSamp), vget_low_s16(negCoef));
|
||
|
|
accum = vmlal_s16(accum, vget_high_s16(negSamp), vget_high_s16(negCoef));
|
||
|
|
sP -= 8;
|
||
|
|
} break;
|
||
|
|
case 2: {
|
||
|
|
int16x8x2_t posSamp = vld2q_s16(sP);
|
||
|
|
int16x8x2_t negSamp = vld2q_s16(sN);
|
||
|
|
sN += 16;
|
||
|
|
posSamp.val[0] = vrev64q_s16(posSamp.val[0]);
|
||
|
|
posSamp.val[1] = vrev64q_s16(posSamp.val[1]);
|
||
|
|
|
||
|
|
// dot product
|
||
|
|
accum = vmlal_s16(accum, vget_low_s16(posSamp.val[0]), vget_high_s16(posCoef)); // r
|
||
|
|
accum = vmlal_s16(accum, vget_high_s16(posSamp.val[0]), vget_low_s16(posCoef)); // r
|
||
|
|
accum2 = vmlal_s16(accum2, vget_low_s16(posSamp.val[1]), vget_high_s16(posCoef)); // r
|
||
|
|
accum2 = vmlal_s16(accum2, vget_high_s16(posSamp.val[1]), vget_low_s16(posCoef)); // r
|
||
|
|
accum = vmlal_s16(accum, vget_low_s16(negSamp.val[0]), vget_low_s16(negCoef));
|
||
|
|
accum = vmlal_s16(accum, vget_high_s16(negSamp.val[0]), vget_high_s16(negCoef));
|
||
|
|
accum2 = vmlal_s16(accum2, vget_low_s16(negSamp.val[1]), vget_low_s16(negCoef));
|
||
|
|
accum2 = vmlal_s16(accum2, vget_high_s16(negSamp.val[1]), vget_high_s16(negCoef));
|
||
|
|
sP -= 16;
|
||
|
|
} break;
|
||
|
|
}
|
||
|
|
} while (count -= 8);
|
||
|
|
|
||
|
|
// multiply by volume and save
|
||
|
|
volumeLR = (const int32_t*)__builtin_assume_aligned(volumeLR, 8);
|
||
|
|
int32x2_t vLR = vld1_s32(volumeLR);
|
||
|
|
int32x2_t outSamp = vld1_s32(out);
|
||
|
|
// combine and funnel down accumulator
|
||
|
|
int32x2_t outAccum = vpadd_s32(vget_low_s32(accum), vget_high_s32(accum));
|
||
|
|
if (CHANNELS == 1) {
|
||
|
|
// duplicate accum to both L and R
|
||
|
|
outAccum = vpadd_s32(outAccum, outAccum);
|
||
|
|
} else if (CHANNELS == 2) {
|
||
|
|
// accum2 contains R, fold in
|
||
|
|
int32x2_t outAccum2 = vpadd_s32(vget_low_s32(accum2), vget_high_s32(accum2));
|
||
|
|
outAccum = vpadd_s32(outAccum, outAccum2);
|
||
|
|
}
|
||
|
|
outAccum = vqrdmulh_s32(outAccum, vLR);
|
||
|
|
outSamp = vqadd_s32(outSamp, outAccum);
|
||
|
|
vst1_s32(out, outSamp);
|
||
|
|
}
|
||
|
|
|
||
|
|
template <int CHANNELS, int STRIDE, bool FIXED>
|
||
|
|
static inline void ProcessNeonIntrinsic(int32_t* out,
|
||
|
|
int count,
|
||
|
|
const int32_t* coefsP,
|
||
|
|
const int32_t* coefsN,
|
||
|
|
const int16_t* sP,
|
||
|
|
const int16_t* sN,
|
||
|
|
const int32_t* volumeLR,
|
||
|
|
uint32_t lerpP,
|
||
|
|
const int32_t* coefsP1,
|
||
|
|
const int32_t* coefsN1)
|
||
|
|
{
|
||
|
|
ALOG_ASSERT(count > 0 && (count & 7) == 0); // multiple of 8
|
||
|
|
static_assert(CHANNELS == 1 || CHANNELS == 2, "CHANNELS must be 1 or 2");
|
||
|
|
|
||
|
|
sP -= CHANNELS*((STRIDE>>1)-1);
|
||
|
|
coefsP = (const int32_t*)__builtin_assume_aligned(coefsP, 16);
|
||
|
|
coefsN = (const int32_t*)__builtin_assume_aligned(coefsN, 16);
|
||
|
|
|
||
|
|
int32x2_t interp;
|
||
|
|
if (!FIXED) {
|
||
|
|
interp = vdup_n_s32(lerpP);
|
||
|
|
coefsP1 = (const int32_t*)__builtin_assume_aligned(coefsP1, 16);
|
||
|
|
coefsN1 = (const int32_t*)__builtin_assume_aligned(coefsN1, 16);
|
||
|
|
}
|
||
|
|
int32x4_t accum, accum2;
|
||
|
|
// warning uninitialized if we use veorq_s32
|
||
|
|
// (alternative to below) accum = veorq_s32(accum, accum);
|
||
|
|
accum = vdupq_n_s32(0);
|
||
|
|
if (CHANNELS == 2) {
|
||
|
|
// (alternative to below) accum2 = veorq_s32(accum2, accum2);
|
||
|
|
accum2 = vdupq_n_s32(0);
|
||
|
|
}
|
||
|
|
do {
|
||
|
|
#ifdef vld1q_s32_x2
|
||
|
|
int32x4x2_t posCoef = vld1q_s32_x2(coefsP);
|
||
|
|
coefsP += 8;
|
||
|
|
int32x4x2_t negCoef = vld1q_s32_x2(coefsN);
|
||
|
|
coefsN += 8;
|
||
|
|
#else
|
||
|
|
int32x4x2_t posCoef;
|
||
|
|
posCoef.val[0] = vld1q_s32(coefsP);
|
||
|
|
coefsP += 4;
|
||
|
|
posCoef.val[1] = vld1q_s32(coefsP);
|
||
|
|
coefsP += 4;
|
||
|
|
int32x4x2_t negCoef;
|
||
|
|
negCoef.val[0] = vld1q_s32(coefsN);
|
||
|
|
coefsN += 4;
|
||
|
|
negCoef.val[1] = vld1q_s32(coefsN);
|
||
|
|
coefsN += 4;
|
||
|
|
#endif
|
||
|
|
if (!FIXED) { // interpolate
|
||
|
|
#ifdef vld1q_s32_x2
|
||
|
|
int32x4x2_t posCoef1 = vld1q_s32_x2(coefsP1);
|
||
|
|
coefsP1 += 8;
|
||
|
|
int32x4x2_t negCoef1 = vld1q_s32_x2(coefsN1);
|
||
|
|
coefsN1 += 8;
|
||
|
|
#else
|
||
|
|
int32x4x2_t posCoef1;
|
||
|
|
posCoef1.val[0] = vld1q_s32(coefsP1);
|
||
|
|
coefsP1 += 4;
|
||
|
|
posCoef1.val[1] = vld1q_s32(coefsP1);
|
||
|
|
coefsP1 += 4;
|
||
|
|
int32x4x2_t negCoef1;
|
||
|
|
negCoef1.val[0] = vld1q_s32(coefsN1);
|
||
|
|
coefsN1 += 4;
|
||
|
|
negCoef1.val[1] = vld1q_s32(coefsN1);
|
||
|
|
coefsN1 += 4;
|
||
|
|
#endif
|
||
|
|
|
||
|
|
posCoef1.val[0] = vsubq_s32(posCoef1.val[0], posCoef.val[0]);
|
||
|
|
posCoef1.val[1] = vsubq_s32(posCoef1.val[1], posCoef.val[1]);
|
||
|
|
negCoef.val[0] = vsubq_s32(negCoef.val[0], negCoef1.val[0]);
|
||
|
|
negCoef.val[1] = vsubq_s32(negCoef.val[1], negCoef1.val[1]);
|
||
|
|
|
||
|
|
posCoef1.val[0] = vqrdmulhq_lane_s32(posCoef1.val[0], interp, 0);
|
||
|
|
posCoef1.val[1] = vqrdmulhq_lane_s32(posCoef1.val[1], interp, 0);
|
||
|
|
negCoef.val[0] = vqrdmulhq_lane_s32(negCoef.val[0], interp, 0);
|
||
|
|
negCoef.val[1] = vqrdmulhq_lane_s32(negCoef.val[1], interp, 0);
|
||
|
|
|
||
|
|
posCoef.val[0] = vaddq_s32(posCoef.val[0], posCoef1.val[0]);
|
||
|
|
posCoef.val[1] = vaddq_s32(posCoef.val[1], posCoef1.val[1]);
|
||
|
|
negCoef.val[0] = vaddq_s32(negCoef.val[0], negCoef1.val[0]);
|
||
|
|
negCoef.val[1] = vaddq_s32(negCoef.val[1], negCoef1.val[1]);
|
||
|
|
}
|
||
|
|
switch (CHANNELS) {
|
||
|
|
case 1: {
|
||
|
|
int16x8_t posSamp = vld1q_s16(sP);
|
||
|
|
int16x8_t negSamp = vld1q_s16(sN);
|
||
|
|
sN += 8;
|
||
|
|
posSamp = vrev64q_s16(posSamp);
|
||
|
|
|
||
|
|
int32x4_t posSamp0 = vshll_n_s16(vget_low_s16(posSamp), 15);
|
||
|
|
int32x4_t posSamp1 = vshll_n_s16(vget_high_s16(posSamp), 15);
|
||
|
|
int32x4_t negSamp0 = vshll_n_s16(vget_low_s16(negSamp), 15);
|
||
|
|
int32x4_t negSamp1 = vshll_n_s16(vget_high_s16(negSamp), 15);
|
||
|
|
|
||
|
|
// dot product
|
||
|
|
posSamp0 = vqrdmulhq_s32(posSamp0, posCoef.val[1]); // reversed
|
||
|
|
posSamp1 = vqrdmulhq_s32(posSamp1, posCoef.val[0]); // reversed
|
||
|
|
negSamp0 = vqrdmulhq_s32(negSamp0, negCoef.val[0]);
|
||
|
|
negSamp1 = vqrdmulhq_s32(negSamp1, negCoef.val[1]);
|
||
|
|
|
||
|
|
accum = vaddq_s32(accum, posSamp0);
|
||
|
|
negSamp0 = vaddq_s32(negSamp0, negSamp1);
|
||
|
|
accum = vaddq_s32(accum, posSamp1);
|
||
|
|
accum = vaddq_s32(accum, negSamp0);
|
||
|
|
|
||
|
|
sP -= 8;
|
||
|
|
} break;
|
||
|
|
case 2: {
|
||
|
|
int16x8x2_t posSamp = vld2q_s16(sP);
|
||
|
|
int16x8x2_t negSamp = vld2q_s16(sN);
|
||
|
|
sN += 16;
|
||
|
|
posSamp.val[0] = vrev64q_s16(posSamp.val[0]);
|
||
|
|
posSamp.val[1] = vrev64q_s16(posSamp.val[1]);
|
||
|
|
|
||
|
|
// left
|
||
|
|
int32x4_t posSamp0 = vshll_n_s16(vget_low_s16(posSamp.val[0]), 15);
|
||
|
|
int32x4_t posSamp1 = vshll_n_s16(vget_high_s16(posSamp.val[0]), 15);
|
||
|
|
int32x4_t negSamp0 = vshll_n_s16(vget_low_s16(negSamp.val[0]), 15);
|
||
|
|
int32x4_t negSamp1 = vshll_n_s16(vget_high_s16(negSamp.val[0]), 15);
|
||
|
|
|
||
|
|
// dot product
|
||
|
|
posSamp0 = vqrdmulhq_s32(posSamp0, posCoef.val[1]); // reversed
|
||
|
|
posSamp1 = vqrdmulhq_s32(posSamp1, posCoef.val[0]); // reversed
|
||
|
|
negSamp0 = vqrdmulhq_s32(negSamp0, negCoef.val[0]);
|
||
|
|
negSamp1 = vqrdmulhq_s32(negSamp1, negCoef.val[1]);
|
||
|
|
|
||
|
|
accum = vaddq_s32(accum, posSamp0);
|
||
|
|
negSamp0 = vaddq_s32(negSamp0, negSamp1);
|
||
|
|
accum = vaddq_s32(accum, posSamp1);
|
||
|
|
accum = vaddq_s32(accum, negSamp0);
|
||
|
|
|
||
|
|
// right
|
||
|
|
posSamp0 = vshll_n_s16(vget_low_s16(posSamp.val[1]), 15);
|
||
|
|
posSamp1 = vshll_n_s16(vget_high_s16(posSamp.val[1]), 15);
|
||
|
|
negSamp0 = vshll_n_s16(vget_low_s16(negSamp.val[1]), 15);
|
||
|
|
negSamp1 = vshll_n_s16(vget_high_s16(negSamp.val[1]), 15);
|
||
|
|
|
||
|
|
// dot product
|
||
|
|
posSamp0 = vqrdmulhq_s32(posSamp0, posCoef.val[1]); // reversed
|
||
|
|
posSamp1 = vqrdmulhq_s32(posSamp1, posCoef.val[0]); // reversed
|
||
|
|
negSamp0 = vqrdmulhq_s32(negSamp0, negCoef.val[0]);
|
||
|
|
negSamp1 = vqrdmulhq_s32(negSamp1, negCoef.val[1]);
|
||
|
|
|
||
|
|
accum2 = vaddq_s32(accum2, posSamp0);
|
||
|
|
negSamp0 = vaddq_s32(negSamp0, negSamp1);
|
||
|
|
accum2 = vaddq_s32(accum2, posSamp1);
|
||
|
|
accum2 = vaddq_s32(accum2, negSamp0);
|
||
|
|
|
||
|
|
sP -= 16;
|
||
|
|
} break;
|
||
|
|
}
|
||
|
|
} while (count -= 8);
|
||
|
|
|
||
|
|
// multiply by volume and save
|
||
|
|
volumeLR = (const int32_t*)__builtin_assume_aligned(volumeLR, 8);
|
||
|
|
int32x2_t vLR = vld1_s32(volumeLR);
|
||
|
|
int32x2_t outSamp = vld1_s32(out);
|
||
|
|
// combine and funnel down accumulator
|
||
|
|
int32x2_t outAccum = vpadd_s32(vget_low_s32(accum), vget_high_s32(accum));
|
||
|
|
if (CHANNELS == 1) {
|
||
|
|
// duplicate accum to both L and R
|
||
|
|
outAccum = vpadd_s32(outAccum, outAccum);
|
||
|
|
} else if (CHANNELS == 2) {
|
||
|
|
// accum2 contains R, fold in
|
||
|
|
int32x2_t outAccum2 = vpadd_s32(vget_low_s32(accum2), vget_high_s32(accum2));
|
||
|
|
outAccum = vpadd_s32(outAccum, outAccum2);
|
||
|
|
}
|
||
|
|
outAccum = vqrdmulh_s32(outAccum, vLR);
|
||
|
|
outSamp = vqadd_s32(outSamp, outAccum);
|
||
|
|
vst1_s32(out, outSamp);
|
||
|
|
}
|
||
|
|
|
||
|
|
template <int CHANNELS, int STRIDE, bool FIXED>
|
||
|
|
static inline void ProcessNeonIntrinsic(float* out,
|
||
|
|
int count,
|
||
|
|
const float* coefsP,
|
||
|
|
const float* coefsN,
|
||
|
|
const float* sP,
|
||
|
|
const float* sN,
|
||
|
|
const float* volumeLR,
|
||
|
|
float lerpP,
|
||
|
|
const float* coefsP1,
|
||
|
|
const float* coefsN1)
|
||
|
|
{
|
||
|
|
ALOG_ASSERT(count > 0 && (count & 7) == 0); // multiple of 8
|
||
|
|
static_assert(CHANNELS == 1 || CHANNELS == 2, "CHANNELS must be 1 or 2");
|
||
|
|
|
||
|
|
sP -= CHANNELS*((STRIDE>>1)-1);
|
||
|
|
coefsP = (const float*)__builtin_assume_aligned(coefsP, 16);
|
||
|
|
coefsN = (const float*)__builtin_assume_aligned(coefsN, 16);
|
||
|
|
|
||
|
|
float32x2_t interp;
|
||
|
|
if (!FIXED) {
|
||
|
|
interp = vdup_n_f32(lerpP);
|
||
|
|
coefsP1 = (const float*)__builtin_assume_aligned(coefsP1, 16);
|
||
|
|
coefsN1 = (const float*)__builtin_assume_aligned(coefsN1, 16);
|
||
|
|
}
|
||
|
|
float32x4_t accum, accum2;
|
||
|
|
// warning uninitialized if we use veorq_s32
|
||
|
|
// (alternative to below) accum = veorq_s32(accum, accum);
|
||
|
|
accum = vdupq_n_f32(0);
|
||
|
|
if (CHANNELS == 2) {
|
||
|
|
// (alternative to below) accum2 = veorq_s32(accum2, accum2);
|
||
|
|
accum2 = vdupq_n_f32(0);
|
||
|
|
}
|
||
|
|
do {
|
||
|
|
#ifdef vld1q_f32_x2
|
||
|
|
float32x4x2_t posCoef = vld1q_f32_x2(coefsP);
|
||
|
|
coefsP += 8;
|
||
|
|
float32x4x2_t negCoef = vld1q_f32_x2(coefsN);
|
||
|
|
coefsN += 8;
|
||
|
|
#else
|
||
|
|
float32x4x2_t posCoef;
|
||
|
|
posCoef.val[0] = vld1q_f32(coefsP);
|
||
|
|
coefsP += 4;
|
||
|
|
posCoef.val[1] = vld1q_f32(coefsP);
|
||
|
|
coefsP += 4;
|
||
|
|
float32x4x2_t negCoef;
|
||
|
|
negCoef.val[0] = vld1q_f32(coefsN);
|
||
|
|
coefsN += 4;
|
||
|
|
negCoef.val[1] = vld1q_f32(coefsN);
|
||
|
|
coefsN += 4;
|
||
|
|
#endif
|
||
|
|
if (!FIXED) { // interpolate
|
||
|
|
#ifdef vld1q_f32_x2
|
||
|
|
float32x4x2_t posCoef1 = vld1q_f32_x2(coefsP1);
|
||
|
|
coefsP1 += 8;
|
||
|
|
float32x4x2_t negCoef1 = vld1q_f32_x2(coefsN1);
|
||
|
|
coefsN1 += 8;
|
||
|
|
#else
|
||
|
|
float32x4x2_t posCoef1;
|
||
|
|
posCoef1.val[0] = vld1q_f32(coefsP1);
|
||
|
|
coefsP1 += 4;
|
||
|
|
posCoef1.val[1] = vld1q_f32(coefsP1);
|
||
|
|
coefsP1 += 4;
|
||
|
|
float32x4x2_t negCoef1;
|
||
|
|
negCoef1.val[0] = vld1q_f32(coefsN1);
|
||
|
|
coefsN1 += 4;
|
||
|
|
negCoef1.val[1] = vld1q_f32(coefsN1);
|
||
|
|
coefsN1 += 4;
|
||
|
|
#endif
|
||
|
|
posCoef1.val[0] = vsubq_f32(posCoef1.val[0], posCoef.val[0]);
|
||
|
|
posCoef1.val[1] = vsubq_f32(posCoef1.val[1], posCoef.val[1]);
|
||
|
|
negCoef.val[0] = vsubq_f32(negCoef.val[0], negCoef1.val[0]);
|
||
|
|
negCoef.val[1] = vsubq_f32(negCoef.val[1], negCoef1.val[1]);
|
||
|
|
|
||
|
|
posCoef.val[0] = vmlaq_lane_f32(posCoef.val[0], posCoef1.val[0], interp, 0);
|
||
|
|
posCoef.val[1] = vmlaq_lane_f32(posCoef.val[1], posCoef1.val[1], interp, 0);
|
||
|
|
negCoef.val[0] = vmlaq_lane_f32(negCoef1.val[0], negCoef.val[0], interp, 0); // rev
|
||
|
|
negCoef.val[1] = vmlaq_lane_f32(negCoef1.val[1], negCoef.val[1], interp, 0); // rev
|
||
|
|
}
|
||
|
|
switch (CHANNELS) {
|
||
|
|
case 1: {
|
||
|
|
#ifdef vld1q_f32_x2
|
||
|
|
float32x4x2_t posSamp = vld1q_f32_x2(sP);
|
||
|
|
float32x4x2_t negSamp = vld1q_f32_x2(sN);
|
||
|
|
sN += 8;
|
||
|
|
sP -= 8;
|
||
|
|
#else
|
||
|
|
float32x4x2_t posSamp;
|
||
|
|
posSamp.val[0] = vld1q_f32(sP);
|
||
|
|
sP += 4;
|
||
|
|
posSamp.val[1] = vld1q_f32(sP);
|
||
|
|
sP -= 12;
|
||
|
|
float32x4x2_t negSamp;
|
||
|
|
negSamp.val[0] = vld1q_f32(sN);
|
||
|
|
sN += 4;
|
||
|
|
negSamp.val[1] = vld1q_f32(sN);
|
||
|
|
sN += 4;
|
||
|
|
#endif
|
||
|
|
// effectively we want a vrev128q_f32()
|
||
|
|
posSamp.val[0] = vrev64q_f32(posSamp.val[0]);
|
||
|
|
posSamp.val[1] = vrev64q_f32(posSamp.val[1]);
|
||
|
|
posSamp.val[0] = vcombine_f32(
|
||
|
|
vget_high_f32(posSamp.val[0]), vget_low_f32(posSamp.val[0]));
|
||
|
|
posSamp.val[1] = vcombine_f32(
|
||
|
|
vget_high_f32(posSamp.val[1]), vget_low_f32(posSamp.val[1]));
|
||
|
|
|
||
|
|
accum = vmlaq_f32(accum, posSamp.val[0], posCoef.val[1]);
|
||
|
|
accum = vmlaq_f32(accum, posSamp.val[1], posCoef.val[0]);
|
||
|
|
accum = vmlaq_f32(accum, negSamp.val[0], negCoef.val[0]);
|
||
|
|
accum = vmlaq_f32(accum, negSamp.val[1], negCoef.val[1]);
|
||
|
|
} break;
|
||
|
|
case 2: {
|
||
|
|
float32x4x2_t posSamp0 = vld2q_f32(sP);
|
||
|
|
sP += 8;
|
||
|
|
float32x4x2_t negSamp0 = vld2q_f32(sN);
|
||
|
|
sN += 8;
|
||
|
|
posSamp0.val[0] = vrev64q_f32(posSamp0.val[0]);
|
||
|
|
posSamp0.val[1] = vrev64q_f32(posSamp0.val[1]);
|
||
|
|
posSamp0.val[0] = vcombine_f32(
|
||
|
|
vget_high_f32(posSamp0.val[0]), vget_low_f32(posSamp0.val[0]));
|
||
|
|
posSamp0.val[1] = vcombine_f32(
|
||
|
|
vget_high_f32(posSamp0.val[1]), vget_low_f32(posSamp0.val[1]));
|
||
|
|
|
||
|
|
float32x4x2_t posSamp1 = vld2q_f32(sP);
|
||
|
|
sP -= 24;
|
||
|
|
float32x4x2_t negSamp1 = vld2q_f32(sN);
|
||
|
|
sN += 8;
|
||
|
|
posSamp1.val[0] = vrev64q_f32(posSamp1.val[0]);
|
||
|
|
posSamp1.val[1] = vrev64q_f32(posSamp1.val[1]);
|
||
|
|
posSamp1.val[0] = vcombine_f32(
|
||
|
|
vget_high_f32(posSamp1.val[0]), vget_low_f32(posSamp1.val[0]));
|
||
|
|
posSamp1.val[1] = vcombine_f32(
|
||
|
|
vget_high_f32(posSamp1.val[1]), vget_low_f32(posSamp1.val[1]));
|
||
|
|
|
||
|
|
// Note: speed is affected by accumulation order.
|
||
|
|
// Also, speed appears slower using vmul/vadd instead of vmla for
|
||
|
|
// stereo case, comparable for mono.
|
||
|
|
|
||
|
|
accum = vmlaq_f32(accum, negSamp0.val[0], negCoef.val[0]);
|
||
|
|
accum = vmlaq_f32(accum, negSamp1.val[0], negCoef.val[1]);
|
||
|
|
accum2 = vmlaq_f32(accum2, negSamp0.val[1], negCoef.val[0]);
|
||
|
|
accum2 = vmlaq_f32(accum2, negSamp1.val[1], negCoef.val[1]);
|
||
|
|
|
||
|
|
accum = vmlaq_f32(accum, posSamp0.val[0], posCoef.val[1]); // reversed
|
||
|
|
accum = vmlaq_f32(accum, posSamp1.val[0], posCoef.val[0]); // reversed
|
||
|
|
accum2 = vmlaq_f32(accum2, posSamp0.val[1], posCoef.val[1]); // reversed
|
||
|
|
accum2 = vmlaq_f32(accum2, posSamp1.val[1], posCoef.val[0]); // reversed
|
||
|
|
} break;
|
||
|
|
}
|
||
|
|
} while (count -= 8);
|
||
|
|
|
||
|
|
// multiply by volume and save
|
||
|
|
volumeLR = (const float*)__builtin_assume_aligned(volumeLR, 8);
|
||
|
|
float32x2_t vLR = vld1_f32(volumeLR);
|
||
|
|
float32x2_t outSamp = vld1_f32(out);
|
||
|
|
// combine and funnel down accumulator
|
||
|
|
float32x2_t outAccum = vpadd_f32(vget_low_f32(accum), vget_high_f32(accum));
|
||
|
|
if (CHANNELS == 1) {
|
||
|
|
// duplicate accum to both L and R
|
||
|
|
outAccum = vpadd_f32(outAccum, outAccum);
|
||
|
|
} else if (CHANNELS == 2) {
|
||
|
|
// accum2 contains R, fold in
|
||
|
|
float32x2_t outAccum2 = vpadd_f32(vget_low_f32(accum2), vget_high_f32(accum2));
|
||
|
|
outAccum = vpadd_f32(outAccum, outAccum2);
|
||
|
|
}
|
||
|
|
outSamp = vmla_f32(outSamp, outAccum, vLR);
|
||
|
|
vst1_f32(out, outSamp);
|
||
|
|
}
|
||
|
|
|
||
|
|
template <>
|
||
|
|
inline void ProcessL<1, 16>(int32_t* const out,
|
||
|
|
int count,
|
||
|
|
const int16_t* coefsP,
|
||
|
|
const int16_t* coefsN,
|
||
|
|
const int16_t* sP,
|
||
|
|
const int16_t* sN,
|
||
|
|
const int32_t* const volumeLR)
|
||
|
|
{
|
||
|
|
#ifdef USE_INTRINSIC
|
||
|
|
ProcessNeonIntrinsic<1, 16, true>(out, count, coefsP, coefsN, sP, sN, volumeLR,
|
||
|
|
0 /*lerpP*/, NULL /*coefsP1*/, NULL /*coefsN1*/);
|
||
|
|
#else
|
||
|
|
const int CHANNELS = 1; // template specialization does not preserve params
|
||
|
|
const int STRIDE = 16;
|
||
|
|
sP -= CHANNELS*((STRIDE>>1)-1);
|
||
|
|
asm (
|
||
|
|
"veor q0, q0, q0 \n"// (0 - combines+) accumulator = 0
|
||
|
|
|
||
|
|
"1: \n"
|
||
|
|
|
||
|
|
"vld1.16 {q2}, [%[sP]] \n"// (2+0d) load 8 16-bits mono samples
|
||
|
|
"vld1.16 {q3}, [%[sN]]! \n"// (2) load 8 16-bits mono samples
|
||
|
|
"vld1.16 {q8}, [%[coefsP0]:128]! \n"// (1) load 8 16-bits coefs
|
||
|
|
"vld1.16 {q10}, [%[coefsN0]:128]! \n"// (1) load 8 16-bits coefs
|
||
|
|
|
||
|
|
"vrev64.16 q2, q2 \n"// (1) reverse s3, s2, s1, s0, s7, s6, s5, s4
|
||
|
|
|
||
|
|
// reordering the vmal to do d6, d7 before d4, d5 is slower(?)
|
||
|
|
"vmlal.s16 q0, d4, d17 \n"// (1+0d) multiply (reversed)samples by coef
|
||
|
|
"vmlal.s16 q0, d5, d16 \n"// (1) multiply (reversed)samples by coef
|
||
|
|
"vmlal.s16 q0, d6, d20 \n"// (1) multiply neg samples
|
||
|
|
"vmlal.s16 q0, d7, d21 \n"// (1) multiply neg samples
|
||
|
|
|
||
|
|
// moving these ARM instructions before neon above seems to be slower
|
||
|
|
"subs %[count], %[count], #8 \n"// (1) update loop counter
|
||
|
|
"sub %[sP], %[sP], #16 \n"// (0) move pointer to next set of samples
|
||
|
|
|
||
|
|
// sP used after branch (warning)
|
||
|
|
"bne 1b \n"// loop
|
||
|
|
|
||
|
|
ASSEMBLY_ACCUMULATE_MONO
|
||
|
|
|
||
|
|
: [out] "=Uv" (out[0]),
|
||
|
|
[count] "+r" (count),
|
||
|
|
[coefsP0] "+r" (coefsP),
|
||
|
|
[coefsN0] "+r" (coefsN),
|
||
|
|
[sP] "+r" (sP),
|
||
|
|
[sN] "+r" (sN)
|
||
|
|
: [vLR] "r" (volumeLR)
|
||
|
|
: "cc", "memory",
|
||
|
|
"q0", "q1", "q2", "q3",
|
||
|
|
"q8", "q10"
|
||
|
|
);
|
||
|
|
#endif
|
||
|
|
}
|
||
|
|
|
||
|
|
template <>
|
||
|
|
inline void ProcessL<2, 16>(int32_t* const out,
|
||
|
|
int count,
|
||
|
|
const int16_t* coefsP,
|
||
|
|
const int16_t* coefsN,
|
||
|
|
const int16_t* sP,
|
||
|
|
const int16_t* sN,
|
||
|
|
const int32_t* const volumeLR)
|
||
|
|
{
|
||
|
|
#ifdef USE_INTRINSIC
|
||
|
|
ProcessNeonIntrinsic<2, 16, true>(out, count, coefsP, coefsN, sP, sN, volumeLR,
|
||
|
|
0 /*lerpP*/, NULL /*coefsP1*/, NULL /*coefsN1*/);
|
||
|
|
#else
|
||
|
|
const int CHANNELS = 2; // template specialization does not preserve params
|
||
|
|
const int STRIDE = 16;
|
||
|
|
sP -= CHANNELS*((STRIDE>>1)-1);
|
||
|
|
asm (
|
||
|
|
"veor q0, q0, q0 \n"// (1) acc_L = 0
|
||
|
|
"veor q4, q4, q4 \n"// (0 combines+) acc_R = 0
|
||
|
|
|
||
|
|
"1: \n"
|
||
|
|
|
||
|
|
"vld2.16 {q2, q3}, [%[sP]] \n"// (3+0d) load 8 16-bits stereo frames
|
||
|
|
"vld2.16 {q5, q6}, [%[sN]]! \n"// (3) load 8 16-bits stereo frames
|
||
|
|
"vld1.16 {q8}, [%[coefsP0]:128]! \n"// (1) load 8 16-bits coefs
|
||
|
|
"vld1.16 {q10}, [%[coefsN0]:128]! \n"// (1) load 8 16-bits coefs
|
||
|
|
|
||
|
|
"vrev64.16 q2, q2 \n"// (1) reverse 8 samples of positive left
|
||
|
|
"vrev64.16 q3, q3 \n"// (0 combines+) reverse positive right
|
||
|
|
|
||
|
|
"vmlal.s16 q0, d4, d17 \n"// (1) multiply (reversed) samples left
|
||
|
|
"vmlal.s16 q0, d5, d16 \n"// (1) multiply (reversed) samples left
|
||
|
|
"vmlal.s16 q4, d6, d17 \n"// (1) multiply (reversed) samples right
|
||
|
|
"vmlal.s16 q4, d7, d16 \n"// (1) multiply (reversed) samples right
|
||
|
|
"vmlal.s16 q0, d10, d20 \n"// (1) multiply samples left
|
||
|
|
"vmlal.s16 q0, d11, d21 \n"// (1) multiply samples left
|
||
|
|
"vmlal.s16 q4, d12, d20 \n"// (1) multiply samples right
|
||
|
|
"vmlal.s16 q4, d13, d21 \n"// (1) multiply samples right
|
||
|
|
|
||
|
|
// moving these ARM before neon seems to be slower
|
||
|
|
"subs %[count], %[count], #8 \n"// (1) update loop counter
|
||
|
|
"sub %[sP], %[sP], #32 \n"// (0) move pointer to next set of samples
|
||
|
|
|
||
|
|
// sP used after branch (warning)
|
||
|
|
"bne 1b \n"// loop
|
||
|
|
|
||
|
|
ASSEMBLY_ACCUMULATE_STEREO
|
||
|
|
|
||
|
|
: [out] "=Uv" (out[0]),
|
||
|
|
[count] "+r" (count),
|
||
|
|
[coefsP0] "+r" (coefsP),
|
||
|
|
[coefsN0] "+r" (coefsN),
|
||
|
|
[sP] "+r" (sP),
|
||
|
|
[sN] "+r" (sN)
|
||
|
|
: [vLR] "r" (volumeLR)
|
||
|
|
: "cc", "memory",
|
||
|
|
"q0", "q1", "q2", "q3",
|
||
|
|
"q4", "q5", "q6",
|
||
|
|
"q8", "q10"
|
||
|
|
);
|
||
|
|
#endif
|
||
|
|
}
|
||
|
|
|
||
|
|
template <>
|
||
|
|
inline void Process<1, 16>(int32_t* const out,
|
||
|
|
int count,
|
||
|
|
const int16_t* coefsP,
|
||
|
|
const int16_t* coefsN,
|
||
|
|
const int16_t* coefsP1,
|
||
|
|
const int16_t* coefsN1,
|
||
|
|
const int16_t* sP,
|
||
|
|
const int16_t* sN,
|
||
|
|
uint32_t lerpP,
|
||
|
|
const int32_t* const volumeLR)
|
||
|
|
{
|
||
|
|
#ifdef USE_INTRINSIC
|
||
|
|
ProcessNeonIntrinsic<1, 16, false>(out, count, coefsP, coefsN, sP, sN, volumeLR,
|
||
|
|
lerpP, coefsP1, coefsN1);
|
||
|
|
#else
|
||
|
|
|
||
|
|
const int CHANNELS = 1; // template specialization does not preserve params
|
||
|
|
const int STRIDE = 16;
|
||
|
|
sP -= CHANNELS*((STRIDE>>1)-1);
|
||
|
|
asm (
|
||
|
|
"vmov.32 d2[0], %[lerpP] \n"// load the positive phase S32 Q15
|
||
|
|
"veor q0, q0, q0 \n"// (0 - combines+) accumulator = 0
|
||
|
|
|
||
|
|
"1: \n"
|
||
|
|
|
||
|
|
"vld1.16 {q2}, [%[sP]] \n"// (2+0d) load 8 16-bits mono samples
|
||
|
|
"vld1.16 {q3}, [%[sN]]! \n"// (2) load 8 16-bits mono samples
|
||
|
|
"vld1.16 {q8}, [%[coefsP0]:128]! \n"// (1) load 8 16-bits coefs
|
||
|
|
"vld1.16 {q9}, [%[coefsP1]:128]! \n"// (1) load 8 16-bits coefs for interpolation
|
||
|
|
"vld1.16 {q10}, [%[coefsN1]:128]! \n"// (1) load 8 16-bits coefs
|
||
|
|
"vld1.16 {q11}, [%[coefsN0]:128]! \n"// (1) load 8 16-bits coefs for interpolation
|
||
|
|
|
||
|
|
"vsub.s16 q9, q9, q8 \n"// (1) interpolate (step1) 1st set of coefs
|
||
|
|
"vsub.s16 q11, q11, q10 \n"// (1) interpolate (step1) 2nd set of coets
|
||
|
|
|
||
|
|
"vqrdmulh.s16 q9, q9, d2[0] \n"// (2) interpolate (step2) 1st set of coefs
|
||
|
|
"vqrdmulh.s16 q11, q11, d2[0] \n"// (2) interpolate (step2) 2nd set of coefs
|
||
|
|
|
||
|
|
"vrev64.16 q2, q2 \n"// (1) reverse s3, s2, s1, s0, s7, s6, s5, s4
|
||
|
|
|
||
|
|
"vadd.s16 q8, q8, q9 \n"// (1+2d) interpolate (step3) 1st set
|
||
|
|
"vadd.s16 q10, q10, q11 \n"// (1+1d) interpolate (step3) 2nd set
|
||
|
|
|
||
|
|
// reordering the vmal to do d6, d7 before d4, d5 is slower(?)
|
||
|
|
"vmlal.s16 q0, d4, d17 \n"// (1+0d) multiply reversed samples by coef
|
||
|
|
"vmlal.s16 q0, d5, d16 \n"// (1) multiply reversed samples by coef
|
||
|
|
"vmlal.s16 q0, d6, d20 \n"// (1) multiply neg samples
|
||
|
|
"vmlal.s16 q0, d7, d21 \n"// (1) multiply neg samples
|
||
|
|
|
||
|
|
// moving these ARM instructions before neon above seems to be slower
|
||
|
|
"subs %[count], %[count], #8 \n"// (1) update loop counter
|
||
|
|
"sub %[sP], %[sP], #16 \n"// (0) move pointer to next set of samples
|
||
|
|
|
||
|
|
// sP used after branch (warning)
|
||
|
|
"bne 1b \n"// loop
|
||
|
|
|
||
|
|
ASSEMBLY_ACCUMULATE_MONO
|
||
|
|
|
||
|
|
: [out] "=Uv" (out[0]),
|
||
|
|
[count] "+r" (count),
|
||
|
|
[coefsP0] "+r" (coefsP),
|
||
|
|
[coefsN0] "+r" (coefsN),
|
||
|
|
[coefsP1] "+r" (coefsP1),
|
||
|
|
[coefsN1] "+r" (coefsN1),
|
||
|
|
[sP] "+r" (sP),
|
||
|
|
[sN] "+r" (sN)
|
||
|
|
: [lerpP] "r" (lerpP),
|
||
|
|
[vLR] "r" (volumeLR)
|
||
|
|
: "cc", "memory",
|
||
|
|
"q0", "q1", "q2", "q3",
|
||
|
|
"q8", "q9", "q10", "q11"
|
||
|
|
);
|
||
|
|
#endif
|
||
|
|
}
|
||
|
|
|
||
|
|
template <>
|
||
|
|
inline void Process<2, 16>(int32_t* const out,
|
||
|
|
int count,
|
||
|
|
const int16_t* coefsP,
|
||
|
|
const int16_t* coefsN,
|
||
|
|
const int16_t* coefsP1,
|
||
|
|
const int16_t* coefsN1,
|
||
|
|
const int16_t* sP,
|
||
|
|
const int16_t* sN,
|
||
|
|
uint32_t lerpP,
|
||
|
|
const int32_t* const volumeLR)
|
||
|
|
{
|
||
|
|
#ifdef USE_INTRINSIC
|
||
|
|
ProcessNeonIntrinsic<2, 16, false>(out, count, coefsP, coefsN, sP, sN, volumeLR,
|
||
|
|
lerpP, coefsP1, coefsN1);
|
||
|
|
#else
|
||
|
|
const int CHANNELS = 2; // template specialization does not preserve params
|
||
|
|
const int STRIDE = 16;
|
||
|
|
sP -= CHANNELS*((STRIDE>>1)-1);
|
||
|
|
asm (
|
||
|
|
"vmov.32 d2[0], %[lerpP] \n"// load the positive phase
|
||
|
|
"veor q0, q0, q0 \n"// (1) acc_L = 0
|
||
|
|
"veor q4, q4, q4 \n"// (0 combines+) acc_R = 0
|
||
|
|
|
||
|
|
"1: \n"
|
||
|
|
|
||
|
|
"vld2.16 {q2, q3}, [%[sP]] \n"// (3+0d) load 8 16-bits stereo frames
|
||
|
|
"vld2.16 {q5, q6}, [%[sN]]! \n"// (3) load 8 16-bits stereo frames
|
||
|
|
"vld1.16 {q8}, [%[coefsP0]:128]! \n"// (1) load 8 16-bits coefs
|
||
|
|
"vld1.16 {q9}, [%[coefsP1]:128]! \n"// (1) load 8 16-bits coefs for interpolation
|
||
|
|
"vld1.16 {q10}, [%[coefsN1]:128]! \n"// (1) load 8 16-bits coefs
|
||
|
|
"vld1.16 {q11}, [%[coefsN0]:128]! \n"// (1) load 8 16-bits coefs for interpolation
|
||
|
|
|
||
|
|
"vsub.s16 q9, q9, q8 \n"// (1) interpolate (step1) 1st set of coefs
|
||
|
|
"vsub.s16 q11, q11, q10 \n"// (1) interpolate (step1) 2nd set of coets
|
||
|
|
|
||
|
|
"vqrdmulh.s16 q9, q9, d2[0] \n"// (2) interpolate (step2) 1st set of coefs
|
||
|
|
"vqrdmulh.s16 q11, q11, d2[0] \n"// (2) interpolate (step2) 2nd set of coefs
|
||
|
|
|
||
|
|
"vrev64.16 q2, q2 \n"// (1) reverse 8 samples of positive left
|
||
|
|
"vrev64.16 q3, q3 \n"// (1) reverse 8 samples of positive right
|
||
|
|
|
||
|
|
"vadd.s16 q8, q8, q9 \n"// (1+1d) interpolate (step3) 1st set
|
||
|
|
"vadd.s16 q10, q10, q11 \n"// (1+1d) interpolate (step3) 2nd set
|
||
|
|
|
||
|
|
"vmlal.s16 q0, d4, d17 \n"// (1) multiply reversed samples left
|
||
|
|
"vmlal.s16 q0, d5, d16 \n"// (1) multiply reversed samples left
|
||
|
|
"vmlal.s16 q4, d6, d17 \n"// (1) multiply reversed samples right
|
||
|
|
"vmlal.s16 q4, d7, d16 \n"// (1) multiply reversed samples right
|
||
|
|
"vmlal.s16 q0, d10, d20 \n"// (1) multiply samples left
|
||
|
|
"vmlal.s16 q0, d11, d21 \n"// (1) multiply samples left
|
||
|
|
"vmlal.s16 q4, d12, d20 \n"// (1) multiply samples right
|
||
|
|
"vmlal.s16 q4, d13, d21 \n"// (1) multiply samples right
|
||
|
|
|
||
|
|
// moving these ARM before neon seems to be slower
|
||
|
|
"subs %[count], %[count], #8 \n"// (1) update loop counter
|
||
|
|
"sub %[sP], %[sP], #32 \n"// (0) move pointer to next set of samples
|
||
|
|
|
||
|
|
// sP used after branch (warning)
|
||
|
|
"bne 1b \n"// loop
|
||
|
|
|
||
|
|
ASSEMBLY_ACCUMULATE_STEREO
|
||
|
|
|
||
|
|
: [out] "=Uv" (out[0]),
|
||
|
|
[count] "+r" (count),
|
||
|
|
[coefsP0] "+r" (coefsP),
|
||
|
|
[coefsN0] "+r" (coefsN),
|
||
|
|
[coefsP1] "+r" (coefsP1),
|
||
|
|
[coefsN1] "+r" (coefsN1),
|
||
|
|
[sP] "+r" (sP),
|
||
|
|
[sN] "+r" (sN)
|
||
|
|
: [lerpP] "r" (lerpP),
|
||
|
|
[vLR] "r" (volumeLR)
|
||
|
|
: "cc", "memory",
|
||
|
|
"q0", "q1", "q2", "q3",
|
||
|
|
"q4", "q5", "q6",
|
||
|
|
"q8", "q9", "q10", "q11"
|
||
|
|
);
|
||
|
|
#endif
|
||
|
|
}
|
||
|
|
|
||
|
|
template <>
|
||
|
|
inline void ProcessL<1, 16>(int32_t* const out,
|
||
|
|
int count,
|
||
|
|
const int32_t* coefsP,
|
||
|
|
const int32_t* coefsN,
|
||
|
|
const int16_t* sP,
|
||
|
|
const int16_t* sN,
|
||
|
|
const int32_t* const volumeLR)
|
||
|
|
{
|
||
|
|
#ifdef USE_INTRINSIC
|
||
|
|
ProcessNeonIntrinsic<1, 16, true>(out, count, coefsP, coefsN, sP, sN, volumeLR,
|
||
|
|
0 /*lerpP*/, NULL /*coefsP1*/, NULL /*coefsN1*/);
|
||
|
|
#else
|
||
|
|
const int CHANNELS = 1; // template specialization does not preserve params
|
||
|
|
const int STRIDE = 16;
|
||
|
|
sP -= CHANNELS*((STRIDE>>1)-1);
|
||
|
|
asm (
|
||
|
|
"veor q0, q0, q0 \n"// result, initialize to 0
|
||
|
|
|
||
|
|
"1: \n"
|
||
|
|
|
||
|
|
"vld1.16 {q2}, [%[sP]] \n"// load 8 16-bits mono samples
|
||
|
|
"vld1.16 {q3}, [%[sN]]! \n"// load 8 16-bits mono samples
|
||
|
|
"vld1.32 {q8, q9}, [%[coefsP0]:128]! \n"// load 8 32-bits coefs
|
||
|
|
"vld1.32 {q10, q11}, [%[coefsN0]:128]! \n"// load 8 32-bits coefs
|
||
|
|
|
||
|
|
"vrev64.16 q2, q2 \n"// reverse 8 samples of the positive side
|
||
|
|
|
||
|
|
"vshll.s16 q12, d4, #15 \n"// extend samples to 31 bits
|
||
|
|
"vshll.s16 q13, d5, #15 \n"// extend samples to 31 bits
|
||
|
|
|
||
|
|
"vshll.s16 q14, d6, #15 \n"// extend samples to 31 bits
|
||
|
|
"vshll.s16 q15, d7, #15 \n"// extend samples to 31 bits
|
||
|
|
|
||
|
|
"vqrdmulh.s32 q12, q12, q9 \n"// multiply samples
|
||
|
|
"vqrdmulh.s32 q13, q13, q8 \n"// multiply samples
|
||
|
|
"vqrdmulh.s32 q14, q14, q10 \n"// multiply samples
|
||
|
|
"vqrdmulh.s32 q15, q15, q11 \n"// multiply samples
|
||
|
|
|
||
|
|
"vadd.s32 q0, q0, q12 \n"// accumulate result
|
||
|
|
"vadd.s32 q13, q13, q14 \n"// accumulate result
|
||
|
|
"vadd.s32 q0, q0, q15 \n"// accumulate result
|
||
|
|
"vadd.s32 q0, q0, q13 \n"// accumulate result
|
||
|
|
|
||
|
|
"sub %[sP], %[sP], #16 \n"// move pointer to next set of samples
|
||
|
|
"subs %[count], %[count], #8 \n"// update loop counter
|
||
|
|
|
||
|
|
"bne 1b \n"// loop
|
||
|
|
|
||
|
|
ASSEMBLY_ACCUMULATE_MONO
|
||
|
|
|
||
|
|
: [out] "=Uv" (out[0]),
|
||
|
|
[count] "+r" (count),
|
||
|
|
[coefsP0] "+r" (coefsP),
|
||
|
|
[coefsN0] "+r" (coefsN),
|
||
|
|
[sP] "+r" (sP),
|
||
|
|
[sN] "+r" (sN)
|
||
|
|
: [vLR] "r" (volumeLR)
|
||
|
|
: "cc", "memory",
|
||
|
|
"q0", "q1", "q2", "q3",
|
||
|
|
"q8", "q9", "q10", "q11",
|
||
|
|
"q12", "q13", "q14", "q15"
|
||
|
|
);
|
||
|
|
#endif
|
||
|
|
}
|
||
|
|
|
||
|
|
template <>
|
||
|
|
inline void ProcessL<2, 16>(int32_t* const out,
|
||
|
|
int count,
|
||
|
|
const int32_t* coefsP,
|
||
|
|
const int32_t* coefsN,
|
||
|
|
const int16_t* sP,
|
||
|
|
const int16_t* sN,
|
||
|
|
const int32_t* const volumeLR)
|
||
|
|
{
|
||
|
|
#ifdef USE_INTRINSIC
|
||
|
|
ProcessNeonIntrinsic<2, 16, true>(out, count, coefsP, coefsN, sP, sN, volumeLR,
|
||
|
|
0 /*lerpP*/, NULL /*coefsP1*/, NULL /*coefsN1*/);
|
||
|
|
#else
|
||
|
|
const int CHANNELS = 2; // template specialization does not preserve params
|
||
|
|
const int STRIDE = 16;
|
||
|
|
sP -= CHANNELS*((STRIDE>>1)-1);
|
||
|
|
asm (
|
||
|
|
"veor q0, q0, q0 \n"// result, initialize to 0
|
||
|
|
"veor q4, q4, q4 \n"// result, initialize to 0
|
||
|
|
|
||
|
|
"1: \n"
|
||
|
|
|
||
|
|
"vld2.16 {q2, q3}, [%[sP]] \n"// load 8 16-bits stereo frames
|
||
|
|
"vld2.16 {q5, q6}, [%[sN]]! \n"// load 8 16-bits stereo frames
|
||
|
|
"vld1.32 {q8, q9}, [%[coefsP0]:128]! \n"// load 8 32-bits coefs
|
||
|
|
"vld1.32 {q10, q11}, [%[coefsN0]:128]! \n"// load 8 32-bits coefs
|
||
|
|
|
||
|
|
"vrev64.16 q2, q2 \n"// reverse 8 samples of positive left
|
||
|
|
"vrev64.16 q3, q3 \n"// reverse 8 samples of positive right
|
||
|
|
|
||
|
|
"vshll.s16 q12, d4, #15 \n"// extend samples to 31 bits
|
||
|
|
"vshll.s16 q13, d5, #15 \n"// extend samples to 31 bits
|
||
|
|
|
||
|
|
"vshll.s16 q14, d10, #15 \n"// extend samples to 31 bits
|
||
|
|
"vshll.s16 q15, d11, #15 \n"// extend samples to 31 bits
|
||
|
|
|
||
|
|
"vqrdmulh.s32 q12, q12, q9 \n"// multiply samples by coef
|
||
|
|
"vqrdmulh.s32 q13, q13, q8 \n"// multiply samples by coef
|
||
|
|
"vqrdmulh.s32 q14, q14, q10 \n"// multiply samples by coef
|
||
|
|
"vqrdmulh.s32 q15, q15, q11 \n"// multiply samples by coef
|
||
|
|
|
||
|
|
"vadd.s32 q0, q0, q12 \n"// accumulate result
|
||
|
|
"vadd.s32 q13, q13, q14 \n"// accumulate result
|
||
|
|
"vadd.s32 q0, q0, q15 \n"// accumulate result
|
||
|
|
"vadd.s32 q0, q0, q13 \n"// accumulate result
|
||
|
|
|
||
|
|
"vshll.s16 q12, d6, #15 \n"// extend samples to 31 bits
|
||
|
|
"vshll.s16 q13, d7, #15 \n"// extend samples to 31 bits
|
||
|
|
|
||
|
|
"vshll.s16 q14, d12, #15 \n"// extend samples to 31 bits
|
||
|
|
"vshll.s16 q15, d13, #15 \n"// extend samples to 31 bits
|
||
|
|
|
||
|
|
"vqrdmulh.s32 q12, q12, q9 \n"// multiply samples by coef
|
||
|
|
"vqrdmulh.s32 q13, q13, q8 \n"// multiply samples by coef
|
||
|
|
"vqrdmulh.s32 q14, q14, q10 \n"// multiply samples by coef
|
||
|
|
"vqrdmulh.s32 q15, q15, q11 \n"// multiply samples by coef
|
||
|
|
|
||
|
|
"vadd.s32 q4, q4, q12 \n"// accumulate result
|
||
|
|
"vadd.s32 q13, q13, q14 \n"// accumulate result
|
||
|
|
"vadd.s32 q4, q4, q15 \n"// accumulate result
|
||
|
|
"vadd.s32 q4, q4, q13 \n"// accumulate result
|
||
|
|
|
||
|
|
"subs %[count], %[count], #8 \n"// update loop counter
|
||
|
|
"sub %[sP], %[sP], #32 \n"// move pointer to next set of samples
|
||
|
|
|
||
|
|
"bne 1b \n"// loop
|
||
|
|
|
||
|
|
ASSEMBLY_ACCUMULATE_STEREO
|
||
|
|
|
||
|
|
: [out] "=Uv" (out[0]),
|
||
|
|
[count] "+r" (count),
|
||
|
|
[coefsP0] "+r" (coefsP),
|
||
|
|
[coefsN0] "+r" (coefsN),
|
||
|
|
[sP] "+r" (sP),
|
||
|
|
[sN] "+r" (sN)
|
||
|
|
: [vLR] "r" (volumeLR)
|
||
|
|
: "cc", "memory",
|
||
|
|
"q0", "q1", "q2", "q3",
|
||
|
|
"q4", "q5", "q6",
|
||
|
|
"q8", "q9", "q10", "q11",
|
||
|
|
"q12", "q13", "q14", "q15"
|
||
|
|
);
|
||
|
|
#endif
|
||
|
|
}
|
||
|
|
|
||
|
|
template <>
|
||
|
|
inline void Process<1, 16>(int32_t* const out,
|
||
|
|
int count,
|
||
|
|
const int32_t* coefsP,
|
||
|
|
const int32_t* coefsN,
|
||
|
|
const int32_t* coefsP1,
|
||
|
|
const int32_t* coefsN1,
|
||
|
|
const int16_t* sP,
|
||
|
|
const int16_t* sN,
|
||
|
|
uint32_t lerpP,
|
||
|
|
const int32_t* const volumeLR)
|
||
|
|
{
|
||
|
|
#ifdef USE_INTRINSIC
|
||
|
|
ProcessNeonIntrinsic<1, 16, false>(out, count, coefsP, coefsN, sP, sN, volumeLR,
|
||
|
|
lerpP, coefsP1, coefsN1);
|
||
|
|
#else
|
||
|
|
const int CHANNELS = 1; // template specialization does not preserve params
|
||
|
|
const int STRIDE = 16;
|
||
|
|
sP -= CHANNELS*((STRIDE>>1)-1);
|
||
|
|
asm (
|
||
|
|
"vmov.32 d2[0], %[lerpP] \n"// load the positive phase
|
||
|
|
"veor q0, q0, q0 \n"// result, initialize to 0
|
||
|
|
|
||
|
|
"1: \n"
|
||
|
|
|
||
|
|
"vld1.16 {q2}, [%[sP]] \n"// load 8 16-bits mono samples
|
||
|
|
"vld1.16 {q3}, [%[sN]]! \n"// load 8 16-bits mono samples
|
||
|
|
"vld1.32 {q8, q9}, [%[coefsP0]:128]! \n"// load 8 32-bits coefs
|
||
|
|
"vld1.32 {q12, q13}, [%[coefsP1]:128]! \n"// load 8 32-bits coefs
|
||
|
|
"vld1.32 {q10, q11}, [%[coefsN1]:128]! \n"// load 8 32-bits coefs
|
||
|
|
"vld1.32 {q14, q15}, [%[coefsN0]:128]! \n"// load 8 32-bits coefs
|
||
|
|
|
||
|
|
"vsub.s32 q12, q12, q8 \n"// interpolate (step1)
|
||
|
|
"vsub.s32 q13, q13, q9 \n"// interpolate (step1)
|
||
|
|
"vsub.s32 q14, q14, q10 \n"// interpolate (step1)
|
||
|
|
"vsub.s32 q15, q15, q11 \n"// interpolate (step1)
|
||
|
|
|
||
|
|
"vqrdmulh.s32 q12, q12, d2[0] \n"// interpolate (step2)
|
||
|
|
"vqrdmulh.s32 q13, q13, d2[0] \n"// interpolate (step2)
|
||
|
|
"vqrdmulh.s32 q14, q14, d2[0] \n"// interpolate (step2)
|
||
|
|
"vqrdmulh.s32 q15, q15, d2[0] \n"// interpolate (step2)
|
||
|
|
|
||
|
|
"vadd.s32 q8, q8, q12 \n"// interpolate (step3)
|
||
|
|
"vadd.s32 q9, q9, q13 \n"// interpolate (step3)
|
||
|
|
"vadd.s32 q10, q10, q14 \n"// interpolate (step3)
|
||
|
|
"vadd.s32 q11, q11, q15 \n"// interpolate (step3)
|
||
|
|
|
||
|
|
"vrev64.16 q2, q2 \n"// reverse 8 samples of the positive side
|
||
|
|
|
||
|
|
"vshll.s16 q12, d4, #15 \n"// extend samples to 31 bits
|
||
|
|
"vshll.s16 q13, d5, #15 \n"// extend samples to 31 bits
|
||
|
|
|
||
|
|
"vshll.s16 q14, d6, #15 \n"// extend samples to 31 bits
|
||
|
|
"vshll.s16 q15, d7, #15 \n"// extend samples to 31 bits
|
||
|
|
|
||
|
|
"vqrdmulh.s32 q12, q12, q9 \n"// multiply samples by interpolated coef
|
||
|
|
"vqrdmulh.s32 q13, q13, q8 \n"// multiply samples by interpolated coef
|
||
|
|
"vqrdmulh.s32 q14, q14, q10 \n"// multiply samples by interpolated coef
|
||
|
|
"vqrdmulh.s32 q15, q15, q11 \n"// multiply samples by interpolated coef
|
||
|
|
|
||
|
|
"vadd.s32 q0, q0, q12 \n"// accumulate result
|
||
|
|
"vadd.s32 q13, q13, q14 \n"// accumulate result
|
||
|
|
"vadd.s32 q0, q0, q15 \n"// accumulate result
|
||
|
|
"vadd.s32 q0, q0, q13 \n"// accumulate result
|
||
|
|
|
||
|
|
"sub %[sP], %[sP], #16 \n"// move pointer to next set of samples
|
||
|
|
"subs %[count], %[count], #8 \n"// update loop counter
|
||
|
|
|
||
|
|
"bne 1b \n"// loop
|
||
|
|
|
||
|
|
ASSEMBLY_ACCUMULATE_MONO
|
||
|
|
|
||
|
|
: [out] "=Uv" (out[0]),
|
||
|
|
[count] "+r" (count),
|
||
|
|
[coefsP0] "+r" (coefsP),
|
||
|
|
[coefsN0] "+r" (coefsN),
|
||
|
|
[coefsP1] "+r" (coefsP1),
|
||
|
|
[coefsN1] "+r" (coefsN1),
|
||
|
|
[sP] "+r" (sP),
|
||
|
|
[sN] "+r" (sN)
|
||
|
|
: [lerpP] "r" (lerpP),
|
||
|
|
[vLR] "r" (volumeLR)
|
||
|
|
: "cc", "memory",
|
||
|
|
"q0", "q1", "q2", "q3",
|
||
|
|
"q8", "q9", "q10", "q11",
|
||
|
|
"q12", "q13", "q14", "q15"
|
||
|
|
);
|
||
|
|
#endif
|
||
|
|
}
|
||
|
|
|
||
|
|
template <>
|
||
|
|
inline void Process<2, 16>(int32_t* const out,
|
||
|
|
int count,
|
||
|
|
const int32_t* coefsP,
|
||
|
|
const int32_t* coefsN,
|
||
|
|
const int32_t* coefsP1,
|
||
|
|
const int32_t* coefsN1,
|
||
|
|
const int16_t* sP,
|
||
|
|
const int16_t* sN,
|
||
|
|
uint32_t lerpP,
|
||
|
|
const int32_t* const volumeLR)
|
||
|
|
{
|
||
|
|
#ifdef USE_INTRINSIC
|
||
|
|
ProcessNeonIntrinsic<2, 16, false>(out, count, coefsP, coefsN, sP, sN, volumeLR,
|
||
|
|
lerpP, coefsP1, coefsN1);
|
||
|
|
#else
|
||
|
|
const int CHANNELS = 2; // template specialization does not preserve params
|
||
|
|
const int STRIDE = 16;
|
||
|
|
sP -= CHANNELS*((STRIDE>>1)-1);
|
||
|
|
asm (
|
||
|
|
"vmov.32 d2[0], %[lerpP] \n"// load the positive phase
|
||
|
|
"veor q0, q0, q0 \n"// result, initialize to 0
|
||
|
|
"veor q4, q4, q4 \n"// result, initialize to 0
|
||
|
|
|
||
|
|
"1: \n"
|
||
|
|
|
||
|
|
"vld2.16 {q2, q3}, [%[sP]] \n"// load 8 16-bits stereo frames
|
||
|
|
"vld2.16 {q5, q6}, [%[sN]]! \n"// load 8 16-bits stereo frames
|
||
|
|
"vld1.32 {q8, q9}, [%[coefsP0]:128]! \n"// load 8 32-bits coefs
|
||
|
|
"vld1.32 {q12, q13}, [%[coefsP1]:128]! \n"// load 8 32-bits coefs
|
||
|
|
"vld1.32 {q10, q11}, [%[coefsN1]:128]! \n"// load 8 32-bits coefs
|
||
|
|
"vld1.32 {q14, q15}, [%[coefsN0]:128]! \n"// load 8 32-bits coefs
|
||
|
|
|
||
|
|
"vsub.s32 q12, q12, q8 \n"// interpolate (step1)
|
||
|
|
"vsub.s32 q13, q13, q9 \n"// interpolate (step1)
|
||
|
|
"vsub.s32 q14, q14, q10 \n"// interpolate (step1)
|
||
|
|
"vsub.s32 q15, q15, q11 \n"// interpolate (step1)
|
||
|
|
|
||
|
|
"vqrdmulh.s32 q12, q12, d2[0] \n"// interpolate (step2)
|
||
|
|
"vqrdmulh.s32 q13, q13, d2[0] \n"// interpolate (step2)
|
||
|
|
"vqrdmulh.s32 q14, q14, d2[0] \n"// interpolate (step2)
|
||
|
|
"vqrdmulh.s32 q15, q15, d2[0] \n"// interpolate (step2)
|
||
|
|
|
||
|
|
"vadd.s32 q8, q8, q12 \n"// interpolate (step3)
|
||
|
|
"vadd.s32 q9, q9, q13 \n"// interpolate (step3)
|
||
|
|
"vadd.s32 q10, q10, q14 \n"// interpolate (step3)
|
||
|
|
"vadd.s32 q11, q11, q15 \n"// interpolate (step3)
|
||
|
|
|
||
|
|
"vrev64.16 q2, q2 \n"// reverse 8 samples of positive left
|
||
|
|
"vrev64.16 q3, q3 \n"// reverse 8 samples of positive right
|
||
|
|
|
||
|
|
"vshll.s16 q12, d4, #15 \n"// extend samples to 31 bits
|
||
|
|
"vshll.s16 q13, d5, #15 \n"// extend samples to 31 bits
|
||
|
|
|
||
|
|
"vshll.s16 q14, d10, #15 \n"// extend samples to 31 bits
|
||
|
|
"vshll.s16 q15, d11, #15 \n"// extend samples to 31 bits
|
||
|
|
|
||
|
|
"vqrdmulh.s32 q12, q12, q9 \n"// multiply samples by interpolated coef
|
||
|
|
"vqrdmulh.s32 q13, q13, q8 \n"// multiply samples by interpolated coef
|
||
|
|
"vqrdmulh.s32 q14, q14, q10 \n"// multiply samples by interpolated coef
|
||
|
|
"vqrdmulh.s32 q15, q15, q11 \n"// multiply samples by interpolated coef
|
||
|
|
|
||
|
|
"vadd.s32 q0, q0, q12 \n"// accumulate result
|
||
|
|
"vadd.s32 q13, q13, q14 \n"// accumulate result
|
||
|
|
"vadd.s32 q0, q0, q15 \n"// accumulate result
|
||
|
|
"vadd.s32 q0, q0, q13 \n"// accumulate result
|
||
|
|
|
||
|
|
"vshll.s16 q12, d6, #15 \n"// extend samples to 31 bits
|
||
|
|
"vshll.s16 q13, d7, #15 \n"// extend samples to 31 bits
|
||
|
|
|
||
|
|
"vshll.s16 q14, d12, #15 \n"// extend samples to 31 bits
|
||
|
|
"vshll.s16 q15, d13, #15 \n"// extend samples to 31 bits
|
||
|
|
|
||
|
|
"vqrdmulh.s32 q12, q12, q9 \n"// multiply samples by interpolated coef
|
||
|
|
"vqrdmulh.s32 q13, q13, q8 \n"// multiply samples by interpolated coef
|
||
|
|
"vqrdmulh.s32 q14, q14, q10 \n"// multiply samples by interpolated coef
|
||
|
|
"vqrdmulh.s32 q15, q15, q11 \n"// multiply samples by interpolated coef
|
||
|
|
|
||
|
|
"vadd.s32 q4, q4, q12 \n"// accumulate result
|
||
|
|
"vadd.s32 q13, q13, q14 \n"// accumulate result
|
||
|
|
"vadd.s32 q4, q4, q15 \n"// accumulate result
|
||
|
|
"vadd.s32 q4, q4, q13 \n"// accumulate result
|
||
|
|
|
||
|
|
"subs %[count], %[count], #8 \n"// update loop counter
|
||
|
|
"sub %[sP], %[sP], #32 \n"// move pointer to next set of samples
|
||
|
|
|
||
|
|
"bne 1b \n"// loop
|
||
|
|
|
||
|
|
ASSEMBLY_ACCUMULATE_STEREO
|
||
|
|
|
||
|
|
: [out] "=Uv" (out[0]),
|
||
|
|
[count] "+r" (count),
|
||
|
|
[coefsP0] "+r" (coefsP),
|
||
|
|
[coefsN0] "+r" (coefsN),
|
||
|
|
[coefsP1] "+r" (coefsP1),
|
||
|
|
[coefsN1] "+r" (coefsN1),
|
||
|
|
[sP] "+r" (sP),
|
||
|
|
[sN] "+r" (sN)
|
||
|
|
: [lerpP] "r" (lerpP),
|
||
|
|
[vLR] "r" (volumeLR)
|
||
|
|
: "cc", "memory",
|
||
|
|
"q0", "q1", "q2", "q3",
|
||
|
|
"q4", "q5", "q6",
|
||
|
|
"q8", "q9", "q10", "q11",
|
||
|
|
"q12", "q13", "q14", "q15"
|
||
|
|
);
|
||
|
|
#endif
|
||
|
|
}
|
||
|
|
|
||
|
|
template<>
|
||
|
|
inline void ProcessL<1, 16>(float* const out,
|
||
|
|
int count,
|
||
|
|
const float* coefsP,
|
||
|
|
const float* coefsN,
|
||
|
|
const float* sP,
|
||
|
|
const float* sN,
|
||
|
|
const float* const volumeLR)
|
||
|
|
{
|
||
|
|
ProcessNeonIntrinsic<1, 16, true>(out, count, coefsP, coefsN, sP, sN, volumeLR,
|
||
|
|
0 /*lerpP*/, NULL /*coefsP1*/, NULL /*coefsN1*/);
|
||
|
|
}
|
||
|
|
|
||
|
|
template<>
|
||
|
|
inline void ProcessL<2, 16>(float* const out,
|
||
|
|
int count,
|
||
|
|
const float* coefsP,
|
||
|
|
const float* coefsN,
|
||
|
|
const float* sP,
|
||
|
|
const float* sN,
|
||
|
|
const float* const volumeLR)
|
||
|
|
{
|
||
|
|
ProcessNeonIntrinsic<2, 16, true>(out, count, coefsP, coefsN, sP, sN, volumeLR,
|
||
|
|
0 /*lerpP*/, NULL /*coefsP1*/, NULL /*coefsN1*/);
|
||
|
|
}
|
||
|
|
|
||
|
|
template<>
|
||
|
|
inline void Process<1, 16>(float* const out,
|
||
|
|
int count,
|
||
|
|
const float* coefsP,
|
||
|
|
const float* coefsN,
|
||
|
|
const float* coefsP1,
|
||
|
|
const float* coefsN1,
|
||
|
|
const float* sP,
|
||
|
|
const float* sN,
|
||
|
|
float lerpP,
|
||
|
|
const float* const volumeLR)
|
||
|
|
{
|
||
|
|
ProcessNeonIntrinsic<1, 16, false>(out, count, coefsP, coefsN, sP, sN, volumeLR,
|
||
|
|
lerpP, coefsP1, coefsN1);
|
||
|
|
}
|
||
|
|
|
||
|
|
template<>
|
||
|
|
inline void Process<2, 16>(float* const out,
|
||
|
|
int count,
|
||
|
|
const float* coefsP,
|
||
|
|
const float* coefsN,
|
||
|
|
const float* coefsP1,
|
||
|
|
const float* coefsN1,
|
||
|
|
const float* sP,
|
||
|
|
const float* sN,
|
||
|
|
float lerpP,
|
||
|
|
const float* const volumeLR)
|
||
|
|
{
|
||
|
|
ProcessNeonIntrinsic<2, 16, false>(out, count, coefsP, coefsN, sP, sN, volumeLR,
|
||
|
|
lerpP, coefsP1, coefsN1);
|
||
|
|
}
|
||
|
|
|
||
|
|
#endif //USE_NEON
|
||
|
|
|
||
|
|
} // namespace android
|
||
|
|
|
||
|
|
#endif /*ANDROID_AUDIO_RESAMPLER_FIR_PROCESS_NEON_H*/
|