peridot/vendor/golang.org/x/crypto/chacha20/chacha_s390x.s

// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build gc && !purego
// +build gc,!purego

#include "go_asm.h"
#include "textflag.h"

// This is an implementation of the ChaCha20 encryption algorithm as
// specified in RFC 7539. It uses vector instructions to compute
// 4 keystream blocks in parallel (256 bytes) which are then XORed
// with the bytes in the input slice.

GLOBL ·constants<>(SB), RODATA|NOPTR, $32
// BSWAP: swap bytes in each 4-byte element
DATA ·constants<>+0x00(SB)/4, $0x03020100
DATA ·constants<>+0x04(SB)/4, $0x07060504
DATA ·constants<>+0x08(SB)/4, $0x0b0a0908
DATA ·constants<>+0x0c(SB)/4, $0x0f0e0d0c
// J0: [j0, j1, j2, j3]
DATA ·constants<>+0x10(SB)/4, $0x61707865
DATA ·constants<>+0x14(SB)/4, $0x3320646e
DATA ·constants<>+0x18(SB)/4, $0x79622d32
DATA ·constants<>+0x1c(SB)/4, $0x6b206574

#define BSWAP V5
#define J0    V6
#define KEY0  V7
#define KEY1  V8
#define NONCE V9
#define CTR   V10
#define M0    V11
#define M1    V12
#define M2    V13
#define M3    V14
#define INC   V15
#define X0    V16
#define X1    V17
#define X2    V18
#define X3    V19
#define X4    V20
#define X5    V21
#define X6    V22
#define X7    V23
#define X8    V24
#define X9    V25
#define X10   V26
#define X11   V27
#define X12   V28
#define X13   V29
#define X14   V30
#define X15   V31

#define NUM_ROUNDS 20

#define ROUND4(a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3) \
	VAF    a1, a0, a0  \
	VAF    b1, b0, b0  \
	VAF    c1, c0, c0  \
	VAF    d1, d0, d0  \
	VX     a0, a2, a2  \
	VX     b0, b2, b2  \
	VX     c0, c2, c2  \
	VX     d0, d2, d2  \
	VERLLF $16, a2, a2 \
	VERLLF $16, b2, b2 \
	VERLLF $16, c2, c2 \
	VERLLF $16, d2, d2 \
	VAF    a2, a3, a3  \
	VAF    b2, b3, b3  \
	VAF    c2, c3, c3  \
	VAF    d2, d3, d3  \
	VX     a3, a1, a1  \
	VX     b3, b1, b1  \
	VX     c3, c1, c1  \
	VX     d3, d1, d1  \
	VERLLF $12, a1, a1 \
	VERLLF $12, b1, b1 \
	VERLLF $12, c1, c1 \
	VERLLF $12, d1, d1 \
	VAF    a1, a0, a0  \
	VAF    b1, b0, b0  \
	VAF    c1, c0, c0  \
	VAF    d1, d0, d0  \
	VX     a0, a2, a2  \
	VX     b0, b2, b2  \
	VX     c0, c2, c2  \
	VX     d0, d2, d2  \
	VERLLF $8, a2, a2  \
	VERLLF $8, b2, b2  \
	VERLLF $8, c2, c2  \
	VERLLF $8, d2, d2  \
	VAF    a2, a3, a3  \
	VAF    b2, b3, b3  \
	VAF    c2, c3, c3  \
	VAF    d2, d3, d3  \
	VX     a3, a1, a1  \
	VX     b3, b1, b1  \
	VX     c3, c1, c1  \
	VX     d3, d1, d1  \
	VERLLF $7, a1, a1  \
	VERLLF $7, b1, b1  \
	VERLLF $7, c1, c1  \
	VERLLF $7, d1, d1

#define PERMUTE(mask, v0, v1, v2, v3) \
	VPERM v0, v0, mask, v0 \
	VPERM v1, v1, mask, v1 \
	VPERM v2, v2, mask, v2 \
	VPERM v3, v3, mask, v3

#define ADDV(x, v0, v1, v2, v3) \
	VAF x, v0, v0 \
	VAF x, v1, v1 \
	VAF x, v2, v2 \
	VAF x, v3, v3

#define XORV(off, dst, src, v0, v1, v2, v3) \
	VLM  off(src), M0, M3          \
	PERMUTE(BSWAP, v0, v1, v2, v3) \
	VX   v0, M0, M0                \
	VX   v1, M1, M1                \
	VX   v2, M2, M2                \
	VX   v3, M3, M3                \
	VSTM M0, M3, off(dst)

#define SHUFFLE(a, b, c, d, t, u, v, w) \
	VMRHF a, c, t \ // t = {a[0], c[0], a[1], c[1]}
	VMRHF b, d, u \ // u = {b[0], d[0], b[1], d[1]}
	VMRLF a, c, v \ // v = {a[2], c[2], a[3], c[3]}
	VMRLF b, d, w \ // w = {b[2], d[2], b[3], d[3]}
	VMRHF t, u, a \ // a = {a[0], b[0], c[0], d[0]}
	VMRLF t, u, b \ // b = {a[1], b[1], c[1], d[1]}
	VMRHF v, w, c \ // c = {a[2], b[2], c[2], d[2]}
	VMRLF v, w, d // d = {a[3], b[3], c[3], d[3]}

// func xorKeyStreamVX(dst, src []byte, key *[8]uint32, nonce *[3]uint32, counter *uint32)
TEXT ·xorKeyStreamVX(SB), NOSPLIT, $0
	MOVD $·constants<>(SB), R1
	MOVD dst+0(FP), R2         // R2=&dst[0]
	LMG  src+24(FP), R3, R4    // R3=&src[0] R4=len(src)
	MOVD key+48(FP), R5        // R5=key
	MOVD nonce+56(FP), R6      // R6=nonce
	MOVD counter+64(FP), R7    // R7=counter

	// load BSWAP and J0
	VLM (R1), BSWAP, J0

	// setup
	MOVD  $95, R0
	VLM   (R5), KEY0, KEY1
	VLL   R0, (R6), NONCE
	VZERO M0
	VLEIB $7, $32, M0
	VSRLB M0, NONCE, NONCE

	// initialize counter values
	VLREPF (R7), CTR
	VZERO  INC
	VLEIF  $1, $1, INC
	VLEIF  $2, $2, INC
	VLEIF  $3, $3, INC
	VAF    INC, CTR, CTR
	VREPIF $4, INC

chacha:
	VREPF $0, J0, X0
	VREPF $1, J0, X1
	VREPF $2, J0, X2
	VREPF $3, J0, X3
	VREPF $0, KEY0, X4
	VREPF $1, KEY0, X5
	VREPF $2, KEY0, X6
	VREPF $3, KEY0, X7
	VREPF $0, KEY1, X8
	VREPF $1, KEY1, X9
	VREPF $2, KEY1, X10
	VREPF $3, KEY1, X11
	VLR   CTR, X12
	VREPF $1, NONCE, X13
	VREPF $2, NONCE, X14
	VREPF $3, NONCE, X15

	MOVD $(NUM_ROUNDS/2), R1

loop:
	ROUND4(X0, X4, X12,  X8, X1, X5, X13,  X9, X2, X6, X14, X10, X3, X7, X15, X11)
	ROUND4(X0, X5, X15, X10, X1, X6, X12, X11, X2, X7, X13, X8,  X3, X4, X14, X9)

	ADD $-1, R1
	BNE loop

	// decrement length
	ADD $-256, R4

	// rearrange vectors
	SHUFFLE(X0, X1, X2, X3, M0, M1, M2, M3)
	ADDV(J0, X0, X1, X2, X3)
	SHUFFLE(X4, X5, X6, X7, M0, M1, M2, M3)
	ADDV(KEY0, X4, X5, X6, X7)
	SHUFFLE(X8, X9, X10, X11, M0, M1, M2, M3)
	ADDV(KEY1, X8, X9, X10, X11)
	VAF CTR, X12, X12
	SHUFFLE(X12, X13, X14, X15, M0, M1, M2, M3)
	ADDV(NONCE, X12, X13, X14, X15)

	// increment counters
	VAF INC, CTR, CTR

	// xor keystream with plaintext
	XORV(0*64, R2, R3, X0, X4,  X8, X12)
	XORV(1*64, R2, R3, X1, X5,  X9, X13)
	XORV(2*64, R2, R3, X2, X6, X10, X14)
	XORV(3*64, R2, R3, X3, X7, X11, X15)

	// increment pointers
	MOVD $256(R2), R2
	MOVD $256(R3), R3

	CMPBNE  R4, $0, chacha

	VSTEF $0, CTR, (R7)
	RET