/*
* Copyright (C) 2010-2020 Arm Limited or its affiliates. All rights reserved.
*
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the License); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an AS IS BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/* ----------------------------------------------------------------------
* Project: CMSIS NN Library
* Title: arm_softmax_s8.c
* Description: S8 softmax function
*
* $Date: 01. March 2021
* $Revision: V.2.0.2
*
* Target Processor: Cortex-M cores
*
* -------------------------------------------------------------------- */
#include "arm_nnfunctions.h"
#include "arm_nnsupportfunctions.h"
#define ACCUM_BITS 12
#ifdef ARM_MATH_MVEI
static int32x4_t arm_exp_on_negative_values_mve_32x4(int32x4_t val)
{
#define SHIFT_START (24)
int32_t shift = SHIFT_START;
int32x4_t mask;
const int32x4_t val_mod_minus_quarter =
vandq_s32(val, vdupq_n_s32((1 << SHIFT_START) - 1)) - vdupq_n_s32(1 << SHIFT_START);
const int32x4_t remainder = vsubq_s32(val_mod_minus_quarter, val);
const int32x4_t x = vaddq_n_s32(val_mod_minus_quarter << 5, 1 << 28);
const int32x4_t x2 = MUL_SAT_MVE(x, x);
const int32x4_t op_1 = DIV_POW2_MVE(MUL_SAT_MVE(x2, x2), 2) + MUL_SAT_MVE(x2, x);
const int32x4_t op_2 = x + DIV_POW2_MVE(MUL_SAT_MVE(op_1, vdupq_n_s32(715827883)) + x2, 1);
int32x4_t result = vdupq_n_s32(1895147668) + MUL_SAT_MVE(vdupq_n_s32(1895147668), op_2);
#define SELECT_IF_NON_ZERO(x) \
{ \
mve_pred16_t p = vcmpneq_n_s32(remainder & vdupq_n_s32(1 << shift++), 0); \
mask = vmvnq_m_s32(vdupq_n_s32(0), vdupq_n_s32(0), p); \
result = SELECT_USING_MASK(mask, MUL_SAT_MVE(result, vdupq_n_s32(x)), result); \
}
SELECT_IF_NON_ZERO(1672461947)
SELECT_IF_NON_ZERO(1302514674)
SELECT_IF_NON_ZERO(790015084)
SELECT_IF_NON_ZERO(290630308)
SELECT_IF_NON_ZERO(39332535)
SELECT_IF_NON_ZERO(720401)
SELECT_IF_NON_ZERO(242)
#undef SELECT_IF_NON_ZERO
mve_pred16_t p = vcmpeqq_n_s32(val, 0);
mask = vmvnq_m_s32(vdupq_n_s32(0), vdupq_n_s32(0), p);
result = SELECT_USING_MASK(mask, vdupq_n_s32(Q31_MAX), result);
return result;
}
#endif
/**
* @ingroup groupNN
*/
/**
* @addtogroup Softmax
* @{
*/
void arm_softmax_s8(const int8_t *input,
const int32_t num_rows,
const int32_t row_size,
const int32_t mult,
const int32_t shift,
const int32_t diff_min,
int8_t *output)
{
#ifdef ARM_MATH_MVEI
#define ACT_MIN ((int8_t)Q7_MIN)
#define ACT_MAX ((int8_t)Q7_MAX)
const int32_t mask = (1 << shift);
for (int i_num_rows = 0; i_num_rows < num_rows; ++i_num_rows)
{
int8_t max = ACT_MIN;
int32_t vec_count = (row_size + 15) / 16;
uint32_t r_count = (uint32_t)row_size;
for (int i = 0; i < vec_count; i++)
{
mve_pred16_t p = vctp8q(r_count);
const int8x16_t ip = vldrbq_z_s8(&input[i * 16], p);
max = vmaxvq_p_s8(max, ip, p);
r_count -= 16;
}
vec_count = row_size / 4;
int32_t idx = 0;
int32_t sum = 0;
while (vec_count)
{
int32x4_t ip = vldrbq_s32(&input[idx * 4]);
ip = vsubq_n_s32(ip, max);
mve_pred16_t p = vcmpgeq_n_s32(ip, diff_min);
if (p != 0)
{
ip = vmulq_n_s32(ip, mask);
int32x4_t res = MUL_SAT_MVE(ip, vdupq_n_s32(mult));
res = arm_exp_on_negative_values_mve_32x4(res);
res = DIV_POW2_MVE(res, ACCUM_BITS);
res = vpselq_s32(res, vdupq_n_s32(0), p);
sum += vaddvq_s32(res);
}
vec_count--;
idx++;
}
const int32_t tail_idx = row_size & ~3;
for (int i = 0; i < (row_size & 3); i++)
{
const int32_t diff = input[tail_idx + i] - max;
if (diff >= diff_min)
{
sum += DIV_POW2(EXP_ON_NEG(MUL_SAT(diff * mask, mult)), ACCUM_BITS);
}
}
const int32_t headroom = __CLZ((uint32_t)sum);
const int32_t bits_over_unit = ACCUM_BITS - headroom + 23;
const int32_t shifted_scale = ONE_OVER1((sum > 0 ? sum << headroom : 0) - (1 << 31));
vec_count = row_size / 4;
idx = 0;
while (vec_count)
{
int32x4_t ip = vldrbq_s32(&input[idx]);
ip = vsubq_n_s32(ip, max);
mve_pred16_t p = vcmpgeq_n_s32(ip, diff_min);
int32x4_t tmp_res;
if (p != 0)
{
ip = vmulq_n_s32(ip, mask);
tmp_res = MUL_SAT_MVE(ip, vdupq_n_s32(mult));
tmp_res = arm_exp_on_negative_values_mve_32x4(tmp_res);
tmp_res = MUL_SAT_MVE(vdupq_n_s32(shifted_scale), tmp_res);
tmp_res = DIV_POW2_MVE(tmp_res, bits_over_unit);
tmp_res += vdupq_n_s32(ACT_MIN);
tmp_res = vmaxq_s32(tmp_res, vdupq_n_s32(ACT_MIN));
tmp_res = vminq_s32(tmp_res, vdupq_n_s32(ACT_MAX));
tmp_res = vpselq_s32(tmp_res, vdupq_n_s32(ACT_MIN), p);
}
else
{
tmp_res = vdupq_n_s32(ACT_MIN);
}
vstrbq_s32(&output[idx], tmp_res);
vec_count--;
idx += 4;
}
for (int i = 0; i < (row_size & 3); i++)
{
int32_t diff = input[tail_idx + i] - max;
if (diff >= diff_min)
{
const int32_t res =
DIV_POW2(MUL_SAT(shifted_scale, EXP_ON_NEG(MUL_SAT(diff * mask, mult))), bits_over_unit) - 128;
output[tail_idx + i] = (int8_t)CLAMP(res, (int32_t)ACT_MAX, (int32_t)ACT_MIN);
}
else
{
output[tail_idx + i] = ACT_MIN;
}
}
input += row_size;
output += row_size;
}
#else
const int32_t mask = (1 << shift);
int32_t col = 0;
int32_t row_idx;
for (row_idx = 0; row_idx < num_rows; ++row_idx)
{
// Find the maximum value in order to ensure numerical stability
int8_t max = *input;
for (col = 1; col < row_size; ++col)
{
max = MAX(max, input[col]);
}
int32_t diff = 0;
int32_t sum = 0;
for (col = 0; col < row_size; ++col)
{
diff = input[col] - max;
if (diff >= diff_min)
{
sum += DIV_POW2(EXP_ON_NEG(MUL_SAT(diff * mask, mult)), ACCUM_BITS);
}
}
const int32_t headroom = __CLZ(sum);
const int32_t bits_over_unit = ACCUM_BITS - headroom + 23;
const int32_t shifted_scale = ONE_OVER1((sum > 0 ? sum << headroom : 0) - (1 << 31));
for (col = 0; col < row_size; ++col)
{
diff = input[col] - max;
if (diff >= diff_min)
{
const int32_t res =
DIV_POW2(MUL_SAT(shifted_scale, EXP_ON_NEG(MUL_SAT(diff * mask, mult))), bits_over_unit) - 128;
output[col] = (int8_t)CLAMP(res, (int32_t)127, (int32_t)-128);
}
else
{
output[col] = -128;
}
}
input += row_size;
output += row_size;
}
#endif
}
/**
* @} end of Softmax group
*/