/*      $NetBSD: chacha_neon.c,v 1.9 2023/08/07 01:07:36 rin Exp $      */

/*-
* Copyright (c) 2020 The NetBSD Foundation, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
*    notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
*    notice, this list of conditions and the following disclaimer in the
*    documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

#include <sys/types.h>
#include <sys/endian.h>

#include <crypto/arch/arm/arm_neon.h>
#include <crypto/arch/arm/arm_neon_imm.h>
#include "chacha_neon.h"

/*
* Tempting to use VSHL/VSRI instead of VSHL/VSHR/VORR, but in practice
* it hurts performance at least on Cortex-A8.
*/
#if 1
#define vrolq_n_u32(x, n)       (vshlq_n_u32(x, n) | vshrq_n_u32(x, 32 - (n)))
#else
#define vrolq_n_u32(x, n)       vsriq_n_u32(vshlq_n_u32(x, n), x, 32 - (n))
#endif

static inline uint32x4_t
rol16(uint32x4_t x)
{
       uint16x8_t y16, x16 = vreinterpretq_u16_u32(x);

       y16 = vrev32q_u16(x16);

       return vreinterpretq_u32_u16(y16);
}

static inline uint32x4_t
rol12(uint32x4_t x)
{

       return vrolq_n_u32(x, 12);
}

static inline uint32x4_t
rol8(uint32x4_t x)
{
#if defined(__aarch64__)
       static const uint8x16_t rol8_tab = VQ_N_U8(
                 3, 0, 1, 2,  7, 4, 5, 6,
                11, 8, 9,10, 15,12,13,14
       );
       uint8x16_t y8, x8 = vreinterpretq_u8_u32(x);

       y8 = vqtbl1q_u8(x8, rol8_tab);

       return vreinterpretq_u32_u8(y8);
#elif 0
       /*
        * GCC does a lousy job with this, spilling two 64-bit vector
        * registers to the stack every time.  There should be plenty
        * of vector registers free, requiring no spills at all, and
        * GCC should be able to hoist the load of rol8_tab out of any
        * loops, but it doesn't and so attempting to use VTBL hurts
        * more than it helps.
        */
       static const uint8x8_t rol8_tab = V_N_U8(
                3, 0, 1, 2,  7, 4, 5, 6
       );

       uint64x2_t y64, x64 = vreinterpretq_u64_u32(x);

       y64 = (uint64x2_t) {
               (uint64_t)vtbl1_u8((uint8x8_t)x64[0], rol8_tab),
               (uint64_t)vtbl1_u8((uint8x8_t)x64[1], rol8_tab),
       };

       return vreinterpretq_u32_u64(y64);
#else
       return vrolq_n_u32(x, 8);
#endif
}

static inline uint32x4_t
rol7(uint32x4_t x)
{

       return vrolq_n_u32(x, 7);
}

static inline void
chacha_permute(uint32x4_t *p0, uint32x4_t *p1, uint32x4_t *p2, uint32x4_t *p3,
   unsigned nr)
{
       uint32x4_t r0, r1, r2, r3;
       uint32x4_t c0, c1, c2, c3;

       r0 = *p0;
       r1 = *p1;
       r2 = *p2;
       r3 = *p3;

       for (; nr > 0; nr -= 2) {
               r0 = vaddq_u32(r0, r1); r3 ^= r0; r3 = rol16(r3);
               r2 = vaddq_u32(r2, r3); r1 ^= r2; r1 = rol12(r1);
               r0 = vaddq_u32(r0, r1); r3 ^= r0; r3 = rol8(r3);
               r2 = vaddq_u32(r2, r3); r1 ^= r2; r1 = rol7(r1);

               c0 = r0;
               c1 = vextq_u32(r1, r1, 1);
               c2 = vextq_u32(r2, r2, 2);
               c3 = vextq_u32(r3, r3, 3);

               c0 = vaddq_u32(c0, c1); c3 ^= c0; c3 = rol16(c3);
               c2 = vaddq_u32(c2, c3); c1 ^= c2; c1 = rol12(c1);
               c0 = vaddq_u32(c0, c1); c3 ^= c0; c3 = rol8(c3);
               c2 = vaddq_u32(c2, c3); c1 ^= c2; c1 = rol7(c1);

               r0 = c0;
               r1 = vextq_u32(c1, c1, 3);
               r2 = vextq_u32(c2, c2, 2);
               r3 = vextq_u32(c3, c3, 1);
       }

       *p0 = r0;
       *p1 = r1;
       *p2 = r2;
       *p3 = r3;
}

void
chacha_core_neon(uint8_t out[restrict static 64],
   const uint8_t in[static 16],
   const uint8_t k[static 32],
   const uint8_t c[static 16],
   unsigned nr)
{
       uint32x4_t in0, in1, in2, in3;
       uint32x4_t r0, r1, r2, r3;

       r0 = in0 = vreinterpretq_u32_u8(vld1q_u8(c));
       r1 = in1 = vreinterpretq_u32_u8(vld1q_u8(k + 0));
       r2 = in2 = vreinterpretq_u32_u8(vld1q_u8(k + 16));
       r3 = in3 = vreinterpretq_u32_u8(vld1q_u8(in));

       chacha_permute(&r0, &r1, &r2, &r3, nr);

       vst1q_u8(out + 0, vreinterpretq_u8_u32(vaddq_u32(r0, in0)));
       vst1q_u8(out + 16, vreinterpretq_u8_u32(vaddq_u32(r1, in1)));
       vst1q_u8(out + 32, vreinterpretq_u8_u32(vaddq_u32(r2, in2)));
       vst1q_u8(out + 48, vreinterpretq_u8_u32(vaddq_u32(r3, in3)));
}

void
hchacha_neon(uint8_t out[restrict static 32],
   const uint8_t in[static 16],
   const uint8_t k[static 32],
   const uint8_t c[static 16],
   unsigned nr)
{
       uint32x4_t r0, r1, r2, r3;

       r0 = vreinterpretq_u32_u8(vld1q_u8(c));
       r1 = vreinterpretq_u32_u8(vld1q_u8(k + 0));
       r2 = vreinterpretq_u32_u8(vld1q_u8(k + 16));
       r3 = vreinterpretq_u32_u8(vld1q_u8(in));

       chacha_permute(&r0, &r1, &r2, &r3, nr);

       vst1q_u8(out + 0, vreinterpretq_u8_u32(r0));
       vst1q_u8(out + 16, vreinterpretq_u8_u32(r3));
}

void
chacha_stream_neon(uint8_t *restrict s, size_t n,
   uint32_t blkno,
   const uint8_t nonce[static 12],
   const uint8_t k[static 32],
   unsigned nr)
{

       for (; n >= 256; s += 256, n -= 256, blkno += 4)
               chacha_stream256_neon(s, blkno, nonce, k, chacha_const32, nr);

       if (n) {
               const uint32x4_t blkno_inc = /* (1,0,0,0) */
                   vsetq_lane_u32(1, vdupq_n_u32(0), 0);
               uint32x4_t in0, in1, in2, in3;
               uint32x4_t r0, r1, r2, r3;

               in0 = vreinterpretq_u32_u8(vld1q_u8(chacha_const32));
               in1 = vreinterpretq_u32_u8(vld1q_u8(k + 0));
               in2 = vreinterpretq_u32_u8(vld1q_u8(k + 16));
               in3 = (uint32x4_t) VQ_N_U32(
                       blkno,
                       le32dec(nonce),
                       le32dec(nonce + 4),
                       le32dec(nonce + 8)
               );

               for (; n; s += 64, n -= 64) {
                       r0 = in0;
                       r1 = in1;
                       r2 = in2;
                       r3 = in3;
                       chacha_permute(&r0, &r1, &r2, &r3, nr);
                       r0 = vaddq_u32(r0, in0);
                       r1 = vaddq_u32(r1, in1);
                       r2 = vaddq_u32(r2, in2);
                       r3 = vaddq_u32(r3, in3);

                       if (n < 64) {
                               uint8_t buf[64] __aligned(16);

                               vst1q_u8(buf + 0, vreinterpretq_u8_u32(r0));
                               vst1q_u8(buf + 16, vreinterpretq_u8_u32(r1));
                               vst1q_u8(buf + 32, vreinterpretq_u8_u32(r2));
                               vst1q_u8(buf + 48, vreinterpretq_u8_u32(r3));
                               memcpy(s, buf, n);

                               break;
                       }

                       vst1q_u8(s + 0, vreinterpretq_u8_u32(r0));
                       vst1q_u8(s + 16, vreinterpretq_u8_u32(r1));
                       vst1q_u8(s + 32, vreinterpretq_u8_u32(r2));
                       vst1q_u8(s + 48, vreinterpretq_u8_u32(r3));
                       in3 = vaddq_u32(in3, blkno_inc);
               }
       }
}

void
chacha_stream_xor_neon(uint8_t *s, const uint8_t *p, size_t n,
   uint32_t blkno,
   const uint8_t nonce[static 12],
   const uint8_t k[static 32],
   unsigned nr)
{

       for (; n >= 256; s += 256, p += 256, n -= 256, blkno += 4)
               chacha_stream_xor256_neon(s, p, blkno, nonce, k,
                   chacha_const32, nr);

       if (n) {
               const uint32x4_t blkno_inc = /* (1,0,0,0) */
                   vsetq_lane_u32(1, vdupq_n_u32(0), 0);
               uint32x4_t in0, in1, in2, in3;
               uint32x4_t r0, r1, r2, r3;

               in0 = vreinterpretq_u32_u8(vld1q_u8(chacha_const32));
               in1 = vreinterpretq_u32_u8(vld1q_u8(k + 0));
               in2 = vreinterpretq_u32_u8(vld1q_u8(k + 16));
               in3 = (uint32x4_t) VQ_N_U32(
                       blkno,
                       le32dec(nonce),
                       le32dec(nonce + 4),
                       le32dec(nonce + 8)
               );

               for (; n; s += 64, p += 64, n -= 64) {
                       r0 = in0;
                       r1 = in1;
                       r2 = in2;
                       r3 = in3;
                       chacha_permute(&r0, &r1, &r2, &r3, nr);
                       r0 = vaddq_u32(r0, in0);
                       r1 = vaddq_u32(r1, in1);
                       r2 = vaddq_u32(r2, in2);
                       r3 = vaddq_u32(r3, in3);

                       if (n < 64) {
                               uint8_t buf[64] __aligned(16);
                               unsigned i;

                               vst1q_u8(buf + 0, vreinterpretq_u8_u32(r0));
                               vst1q_u8(buf + 16, vreinterpretq_u8_u32(r1));
                               vst1q_u8(buf + 32, vreinterpretq_u8_u32(r2));
                               vst1q_u8(buf + 48, vreinterpretq_u8_u32(r3));

                               for (i = 0; i < n - n%4; i += 4)
                                       le32enc(s + i,
                                           le32dec(p + i) ^ le32dec(buf + i));
                               for (; i < n; i++)
                                       s[i] = p[i] ^ buf[i];

                               break;
                       }

                       r0 ^= vreinterpretq_u32_u8(vld1q_u8(p + 0));
                       r1 ^= vreinterpretq_u32_u8(vld1q_u8(p + 16));
                       r2 ^= vreinterpretq_u32_u8(vld1q_u8(p + 32));
                       r3 ^= vreinterpretq_u32_u8(vld1q_u8(p + 48));
                       vst1q_u8(s + 0, vreinterpretq_u8_u32(r0));
                       vst1q_u8(s + 16, vreinterpretq_u8_u32(r1));
                       vst1q_u8(s + 32, vreinterpretq_u8_u32(r2));
                       vst1q_u8(s + 48, vreinterpretq_u8_u32(r3));
                       in3 = vaddq_u32(in3, blkno_inc);
               }
       }
}

void
xchacha_stream_neon(uint8_t *restrict s, size_t nbytes,
   uint32_t blkno,
   const uint8_t nonce[static 24],
   const uint8_t k[static 32],
   unsigned nr)
{
       uint8_t subkey[32];
       uint8_t subnonce[12];

       hchacha_neon(subkey, nonce/*[0:16)*/, k, chacha_const32, nr);
       memset(subnonce, 0, 4);
       memcpy(subnonce + 4, nonce + 16, 8);
       chacha_stream_neon(s, nbytes, blkno, subnonce, subkey, nr);
}

void
xchacha_stream_xor_neon(uint8_t *restrict c, const uint8_t *p, size_t nbytes,
   uint32_t blkno,
   const uint8_t nonce[static 24],
   const uint8_t k[static 32],
   unsigned nr)
{
       uint8_t subkey[32];
       uint8_t subnonce[12];

       hchacha_neon(subkey, nonce/*[0:16)*/, k, chacha_const32, nr);
       memset(subnonce, 0, 4);
       memcpy(subnonce + 4, nonce + 16, 8);
       chacha_stream_xor_neon(c, p, nbytes, blkno, subnonce, subkey, nr);
}

You are viewing proxied material from ftp.icm.edu.pl. The copyright of proxied material belongs to its original authors. Any comments or complaints in relation to proxied material should be directed to the original authors of the content concerned. Please see the disclaimer for more details.