mirror of
https://github.com/rocky-linux/peridot.git
synced 2024-11-14 17:51:25 +00:00
226 lines
5.4 KiB
ArmAsm
226 lines
5.4 KiB
ArmAsm
// Copyright 2018 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
//go:build gc && !purego
|
|
// +build gc,!purego
|
|
|
|
#include "go_asm.h"
|
|
#include "textflag.h"
|
|
|
|
// This is an implementation of the ChaCha20 encryption algorithm as
|
|
// specified in RFC 7539. It uses vector instructions to compute
|
|
// 4 keystream blocks in parallel (256 bytes) which are then XORed
|
|
// with the bytes in the input slice.
|
|
|
|
GLOBL ·constants<>(SB), RODATA|NOPTR, $32
|
|
// BSWAP: swap bytes in each 4-byte element
|
|
DATA ·constants<>+0x00(SB)/4, $0x03020100
|
|
DATA ·constants<>+0x04(SB)/4, $0x07060504
|
|
DATA ·constants<>+0x08(SB)/4, $0x0b0a0908
|
|
DATA ·constants<>+0x0c(SB)/4, $0x0f0e0d0c
|
|
// J0: [j0, j1, j2, j3]
|
|
DATA ·constants<>+0x10(SB)/4, $0x61707865
|
|
DATA ·constants<>+0x14(SB)/4, $0x3320646e
|
|
DATA ·constants<>+0x18(SB)/4, $0x79622d32
|
|
DATA ·constants<>+0x1c(SB)/4, $0x6b206574
|
|
|
|
#define BSWAP V5
|
|
#define J0 V6
|
|
#define KEY0 V7
|
|
#define KEY1 V8
|
|
#define NONCE V9
|
|
#define CTR V10
|
|
#define M0 V11
|
|
#define M1 V12
|
|
#define M2 V13
|
|
#define M3 V14
|
|
#define INC V15
|
|
#define X0 V16
|
|
#define X1 V17
|
|
#define X2 V18
|
|
#define X3 V19
|
|
#define X4 V20
|
|
#define X5 V21
|
|
#define X6 V22
|
|
#define X7 V23
|
|
#define X8 V24
|
|
#define X9 V25
|
|
#define X10 V26
|
|
#define X11 V27
|
|
#define X12 V28
|
|
#define X13 V29
|
|
#define X14 V30
|
|
#define X15 V31
|
|
|
|
#define NUM_ROUNDS 20
|
|
|
|
#define ROUND4(a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3) \
|
|
VAF a1, a0, a0 \
|
|
VAF b1, b0, b0 \
|
|
VAF c1, c0, c0 \
|
|
VAF d1, d0, d0 \
|
|
VX a0, a2, a2 \
|
|
VX b0, b2, b2 \
|
|
VX c0, c2, c2 \
|
|
VX d0, d2, d2 \
|
|
VERLLF $16, a2, a2 \
|
|
VERLLF $16, b2, b2 \
|
|
VERLLF $16, c2, c2 \
|
|
VERLLF $16, d2, d2 \
|
|
VAF a2, a3, a3 \
|
|
VAF b2, b3, b3 \
|
|
VAF c2, c3, c3 \
|
|
VAF d2, d3, d3 \
|
|
VX a3, a1, a1 \
|
|
VX b3, b1, b1 \
|
|
VX c3, c1, c1 \
|
|
VX d3, d1, d1 \
|
|
VERLLF $12, a1, a1 \
|
|
VERLLF $12, b1, b1 \
|
|
VERLLF $12, c1, c1 \
|
|
VERLLF $12, d1, d1 \
|
|
VAF a1, a0, a0 \
|
|
VAF b1, b0, b0 \
|
|
VAF c1, c0, c0 \
|
|
VAF d1, d0, d0 \
|
|
VX a0, a2, a2 \
|
|
VX b0, b2, b2 \
|
|
VX c0, c2, c2 \
|
|
VX d0, d2, d2 \
|
|
VERLLF $8, a2, a2 \
|
|
VERLLF $8, b2, b2 \
|
|
VERLLF $8, c2, c2 \
|
|
VERLLF $8, d2, d2 \
|
|
VAF a2, a3, a3 \
|
|
VAF b2, b3, b3 \
|
|
VAF c2, c3, c3 \
|
|
VAF d2, d3, d3 \
|
|
VX a3, a1, a1 \
|
|
VX b3, b1, b1 \
|
|
VX c3, c1, c1 \
|
|
VX d3, d1, d1 \
|
|
VERLLF $7, a1, a1 \
|
|
VERLLF $7, b1, b1 \
|
|
VERLLF $7, c1, c1 \
|
|
VERLLF $7, d1, d1
|
|
|
|
#define PERMUTE(mask, v0, v1, v2, v3) \
|
|
VPERM v0, v0, mask, v0 \
|
|
VPERM v1, v1, mask, v1 \
|
|
VPERM v2, v2, mask, v2 \
|
|
VPERM v3, v3, mask, v3
|
|
|
|
#define ADDV(x, v0, v1, v2, v3) \
|
|
VAF x, v0, v0 \
|
|
VAF x, v1, v1 \
|
|
VAF x, v2, v2 \
|
|
VAF x, v3, v3
|
|
|
|
#define XORV(off, dst, src, v0, v1, v2, v3) \
|
|
VLM off(src), M0, M3 \
|
|
PERMUTE(BSWAP, v0, v1, v2, v3) \
|
|
VX v0, M0, M0 \
|
|
VX v1, M1, M1 \
|
|
VX v2, M2, M2 \
|
|
VX v3, M3, M3 \
|
|
VSTM M0, M3, off(dst)
|
|
|
|
#define SHUFFLE(a, b, c, d, t, u, v, w) \
|
|
VMRHF a, c, t \ // t = {a[0], c[0], a[1], c[1]}
|
|
VMRHF b, d, u \ // u = {b[0], d[0], b[1], d[1]}
|
|
VMRLF a, c, v \ // v = {a[2], c[2], a[3], c[3]}
|
|
VMRLF b, d, w \ // w = {b[2], d[2], b[3], d[3]}
|
|
VMRHF t, u, a \ // a = {a[0], b[0], c[0], d[0]}
|
|
VMRLF t, u, b \ // b = {a[1], b[1], c[1], d[1]}
|
|
VMRHF v, w, c \ // c = {a[2], b[2], c[2], d[2]}
|
|
VMRLF v, w, d // d = {a[3], b[3], c[3], d[3]}
|
|
|
|
// func xorKeyStreamVX(dst, src []byte, key *[8]uint32, nonce *[3]uint32, counter *uint32)
|
|
TEXT ·xorKeyStreamVX(SB), NOSPLIT, $0
|
|
MOVD $·constants<>(SB), R1
|
|
MOVD dst+0(FP), R2 // R2=&dst[0]
|
|
LMG src+24(FP), R3, R4 // R3=&src[0] R4=len(src)
|
|
MOVD key+48(FP), R5 // R5=key
|
|
MOVD nonce+56(FP), R6 // R6=nonce
|
|
MOVD counter+64(FP), R7 // R7=counter
|
|
|
|
// load BSWAP and J0
|
|
VLM (R1), BSWAP, J0
|
|
|
|
// setup
|
|
MOVD $95, R0
|
|
VLM (R5), KEY0, KEY1
|
|
VLL R0, (R6), NONCE
|
|
VZERO M0
|
|
VLEIB $7, $32, M0
|
|
VSRLB M0, NONCE, NONCE
|
|
|
|
// initialize counter values
|
|
VLREPF (R7), CTR
|
|
VZERO INC
|
|
VLEIF $1, $1, INC
|
|
VLEIF $2, $2, INC
|
|
VLEIF $3, $3, INC
|
|
VAF INC, CTR, CTR
|
|
VREPIF $4, INC
|
|
|
|
chacha:
|
|
VREPF $0, J0, X0
|
|
VREPF $1, J0, X1
|
|
VREPF $2, J0, X2
|
|
VREPF $3, J0, X3
|
|
VREPF $0, KEY0, X4
|
|
VREPF $1, KEY0, X5
|
|
VREPF $2, KEY0, X6
|
|
VREPF $3, KEY0, X7
|
|
VREPF $0, KEY1, X8
|
|
VREPF $1, KEY1, X9
|
|
VREPF $2, KEY1, X10
|
|
VREPF $3, KEY1, X11
|
|
VLR CTR, X12
|
|
VREPF $1, NONCE, X13
|
|
VREPF $2, NONCE, X14
|
|
VREPF $3, NONCE, X15
|
|
|
|
MOVD $(NUM_ROUNDS/2), R1
|
|
|
|
loop:
|
|
ROUND4(X0, X4, X12, X8, X1, X5, X13, X9, X2, X6, X14, X10, X3, X7, X15, X11)
|
|
ROUND4(X0, X5, X15, X10, X1, X6, X12, X11, X2, X7, X13, X8, X3, X4, X14, X9)
|
|
|
|
ADD $-1, R1
|
|
BNE loop
|
|
|
|
// decrement length
|
|
ADD $-256, R4
|
|
|
|
// rearrange vectors
|
|
SHUFFLE(X0, X1, X2, X3, M0, M1, M2, M3)
|
|
ADDV(J0, X0, X1, X2, X3)
|
|
SHUFFLE(X4, X5, X6, X7, M0, M1, M2, M3)
|
|
ADDV(KEY0, X4, X5, X6, X7)
|
|
SHUFFLE(X8, X9, X10, X11, M0, M1, M2, M3)
|
|
ADDV(KEY1, X8, X9, X10, X11)
|
|
VAF CTR, X12, X12
|
|
SHUFFLE(X12, X13, X14, X15, M0, M1, M2, M3)
|
|
ADDV(NONCE, X12, X13, X14, X15)
|
|
|
|
// increment counters
|
|
VAF INC, CTR, CTR
|
|
|
|
// xor keystream with plaintext
|
|
XORV(0*64, R2, R3, X0, X4, X8, X12)
|
|
XORV(1*64, R2, R3, X1, X5, X9, X13)
|
|
XORV(2*64, R2, R3, X2, X6, X10, X14)
|
|
XORV(3*64, R2, R3, X3, X7, X11, X15)
|
|
|
|
// increment pointers
|
|
MOVD $256(R2), R2
|
|
MOVD $256(R3), R3
|
|
|
|
CMPBNE R4, $0, chacha
|
|
|
|
VSTEF $0, CTR, (R7)
|
|
RET
|