/* SPDX-License-Identifier: GPL-2.0 */
/*
 * sha3-ce-core.S - core SHA-3 transform using v8.2 Crypto Extensions
 *
 * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#include <linux/linkage.h>
#include <asm/assembler.h>

	.irp	b,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
	.set	.Lv\b\().2d, \b
	.set	.Lv\b\().16b, \b
	.endr

	/*
	 * ARMv8.2 Crypto Extensions instructions
	 */
	.macro	eor3, rd, rn, rm, ra
	.inst	0xce000000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16)
	.endm

	.macro	rax1, rd, rn, rm
	.inst	0xce608c00 | .L\rd | (.L\rn << 5) | (.L\rm << 16)
	.endm

	.macro	bcax, rd, rn, rm, ra
	.inst	0xce200000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16)
	.endm

	.macro	xar, rd, rn, rm, imm6
	.inst	0xce800000 | .L\rd | (.L\rn << 5) | ((\imm6) << 10) | (.L\rm << 16)
	.endm

	/*
	 * sha3_ce_transform(u64 *st, const u8 *data, int blocks, int dg_size)
	 */
	.text
SYM_FUNC_START(sha3_ce_transform)
	frame_push	4

	mov	x19, x0
	mov	x20, x1
	mov	x21, x2
	mov	x22, x3

0:	/* load state */
	add	x8, x19, #32
	ld1	{ v0.1d- v3.1d}, [x19]
	ld1	{ v4.1d- v7.1d}, [x8], #32
	ld1	{ v8.1d-v11.1d}, [x8], #32
	ld1	{v12.1d-v15.1d}, [x8], #32
	ld1	{v16.1d-v19.1d}, [x8], #32
	ld1	{v20.1d-v23.1d}, [x8], #32
	ld1	{v24.1d}, [x8]

1:	sub	w21, w21, #1
	mov	w8, #24
	adr_l	x9, .Lsha3_rcon

	/* load input */
	ld1	{v25.8b-v28.8b}, [x20], #32
	ld1	{v29.8b-v31.8b}, [x20], #24
	eor	v0.8b, v0.8b, v25.8b
	eor	v1.8b, v1.8b, v26.8b
	eor	v2.8b, v2.8b, v27.8b
	eor	v3.8b, v3.8b, v28.8b
	eor	v4.8b, v4.8b, v29.8b
	eor	v5.8b, v5.8b, v30.8b
	eor	v6.8b, v6.8b, v31.8b

	tbnz	x22, #6, 3f		// SHA3-512

	ld1	{v25.8b-v28.8b}, [x20], #32
	ld1	{v29.8b-v30.8b}, [x20], #16
	eor	 v7.8b,  v7.8b, v25.8b
	eor	 v8.8b,  v8.8b, v26.8b
	eor	 v9.8b,  v9.8b, v27.8b
	eor	v10.8b, v10.8b, v28.8b
	eor	v11.8b, v11.8b, v29.8b
	eor	v12.8b, v12.8b, v30.8b

	tbnz	x22, #4, 2f		// SHA3-384 or SHA3-224

	// SHA3-256
	ld1	{v25.8b-v28.8b}, [x20], #32
	eor	v13.8b, v13.8b, v25.8b
	eor	v14.8b, v14.8b, v26.8b
	eor	v15.8b, v15.8b, v27.8b
	eor	v16.8b, v16.8b, v28.8b
	b	4f

2:	tbz	x22, #2, 4f		// bit 2 cleared? SHA-384

	// SHA3-224
	ld1	{v25.8b-v28.8b}, [x20], #32
	ld1	{v29.8b}, [x20], #8
	eor	v13.8b, v13.8b, v25.8b
	eor	v14.8b, v14.8b, v26.8b
	eor	v15.8b, v15.8b, v27.8b
	eor	v16.8b, v16.8b, v28.8b
	eor	v17.8b, v17.8b, v29.8b
	b	4f

	// SHA3-512
3:	ld1	{v25.8b-v26.8b}, [x20], #16
	eor	 v7.8b,  v7.8b, v25.8b
	eor	 v8.8b,  v8.8b, v26.8b

4:	sub	w8, w8, #1

	eor3	v29.16b,  v4.16b,  v9.16b, v14.16b
	eor3	v26.16b,  v1.16b,  v6.16b, v11.16b
	eor3	v28.16b,  v3.16b,  v8.16b, v13.16b
	eor3	v25.16b,  v0.16b,  v5.16b, v10.16b
	eor3	v27.16b,  v2.16b,  v7.16b, v12.16b
	eor3	v29.16b, v29.16b, v19.16b, v24.16b
	eor3	v26.16b, v26.16b, v16.16b, v21.16b
	eor3	v28.16b, v28.16b, v18.16b, v23.16b
	eor3	v25.16b, v25.16b, v15.16b, v20.16b
	eor3	v27.16b, v27.16b, v17.16b, v22.16b

	rax1	v30.2d, v29.2d, v26.2d	// bc[0]
	rax1	v26.2d, v26.2d, v28.2d	// bc[2]
	rax1	v28.2d, v28.2d, v25.2d	// bc[4]
	rax1	v25.2d, v25.2d, v27.2d	// bc[1]
	rax1	v27.2d, v27.2d, v29.2d	// bc[3]

	eor	 v0.16b,  v0.16b, v30.16b
	xar	 v29.2d,   v1.2d,  v25.2d, (64 - 1)
	xar	  v1.2d,   v6.2d,  v25.2d, (64 - 44)
	xar	  v6.2d,   v9.2d,  v28.2d, (64 - 20)
	xar	  v9.2d,  v22.2d,  v26.2d, (64 - 61)
	xar	 v22.2d,  v14.2d,  v28.2d, (64 - 39)
	xar	 v14.2d,  v20.2d,  v30.2d, (64 - 18)
	xar	 v31.2d,   v2.2d,  v26.2d, (64 - 62)
	xar	  v2.2d,  v12.2d,  v26.2d, (64 - 43)
	xar	 v12.2d,  v13.2d,  v27.2d, (64 - 25)
	xar	 v13.2d,  v19.2d,  v28.2d, (64 - 8)
	xar	 v19.2d,  v23.2d,  v27.2d, (64 - 56)
	xar	 v23.2d,  v15.2d,  v30.2d, (64 - 41)
	xar	 v15.2d,   v4.2d,  v28.2d, (64 - 27)
	xar	 v28.2d,  v24.2d,  v28.2d, (64 - 14)
	xar	 v24.2d,  v21.2d,  v25.2d, (64 - 2)
	xar	  v8.2d,   v8.2d,  v27.2d, (64 - 55)
	xar	  v4.2d,  v16.2d,  v25.2d, (64 - 45)
	xar	 v16.2d,   v5.2d,  v30.2d, (64 - 36)
	xar	  v5.2d,   v3.2d,  v27.2d, (64 - 28)
	xar	 v27.2d,  v18.2d,  v27.2d, (64 - 21)
	xar	  v3.2d,  v17.2d,  v26.2d, (64 - 15)
	xar	 v25.2d,  v11.2d,  v25.2d, (64 - 10)
	xar	 v26.2d,   v7.2d,  v26.2d, (64 - 6)
	xar	 v30.2d,  v10.2d,  v30.2d, (64 - 3)

	bcax	v20.16b, v31.16b, v22.16b,  v8.16b
	bcax	v21.16b,  v8.16b, v23.16b, v22.16b
	bcax	v22.16b, v22.16b, v24.16b, v23.16b
	bcax	v23.16b, v23.16b, v31.16b, v24.16b
	bcax	v24.16b, v24.16b,  v8.16b, v31.16b

	ld1r	{v31.2d}, [x9], #8

	bcax	v17.16b, v25.16b, v19.16b,  v3.16b
	bcax	v18.16b,  v3.16b, v15.16b, v19.16b
	bcax	v19.16b, v19.16b, v16.16b, v15.16b
	bcax	v15.16b, v15.16b, v25.16b, v16.16b
	bcax	v16.16b, v16.16b,  v3.16b, v25.16b

	bcax	v10.16b, v29.16b, v12.16b, v26.16b
	bcax	v11.16b, v26.16b, v13.16b, v12.16b
	bcax	v12.16b, v12.16b, v14.16b, v13.16b
	bcax	v13.16b, v13.16b, v29.16b, v14.16b
	bcax	v14.16b, v14.16b, v26.16b, v29.16b

	bcax	 v7.16b, v30.16b,  v9.16b,  v4.16b
	bcax	 v8.16b,  v4.16b,  v5.16b,  v9.16b
	bcax	 v9.16b,  v9.16b,  v6.16b,  v5.16b
	bcax	 v5.16b,  v5.16b, v30.16b,  v6.16b
	bcax	 v6.16b,  v6.16b,  v4.16b, v30.16b

	bcax	 v3.16b, v27.16b,  v0.16b, v28.16b
	bcax	 v4.16b, v28.16b,  v1.16b,  v0.16b
	bcax	 v0.16b,  v0.16b,  v2.16b,  v1.16b
	bcax	 v1.16b,  v1.16b, v27.16b,  v2.16b
	bcax	 v2.16b,  v2.16b, v28.16b, v27.16b

	eor	 v0.16b,  v0.16b, v31.16b

	cbnz	w8, 4b
	cbz	w21, 5f

	if_will_cond_yield_neon
	add	x8, x19, #32
	st1	{ v0.1d- v3.1d}, [x19]
	st1	{ v4.1d- v7.1d}, [x8], #32
	st1	{ v8.1d-v11.1d}, [x8], #32
	st1	{v12.1d-v15.1d}, [x8], #32
	st1	{v16.1d-v19.1d}, [x8], #32
	st1	{v20.1d-v23.1d}, [x8], #32
	st1	{v24.1d}, [x8]
	do_cond_yield_neon
	b		0b
	endif_yield_neon

	b	1b

	/* save state */
5:	st1	{ v0.1d- v3.1d}, [x19], #32
	st1	{ v4.1d- v7.1d}, [x19], #32
	st1	{ v8.1d-v11.1d}, [x19], #32
	st1	{v12.1d-v15.1d}, [x19], #32
	st1	{v16.1d-v19.1d}, [x19], #32
	st1	{v20.1d-v23.1d}, [x19], #32
	st1	{v24.1d}, [x19]
	frame_pop
	ret
SYM_FUNC_END(sha3_ce_transform)

	.section	".rodata", "a"
	.align		8
.Lsha3_rcon:
	.quad	0x0000000000000001, 0x0000000000008082, 0x800000000000808a
	.quad	0x8000000080008000, 0x000000000000808b, 0x0000000080000001
	.quad	0x8000000080008081, 0x8000000000008009, 0x000000000000008a
	.quad	0x0000000000000088, 0x0000000080008009, 0x000000008000000a
	.quad	0x000000008000808b, 0x800000000000008b, 0x8000000000008089
	.quad	0x8000000000008003, 0x8000000000008002, 0x8000000000000080
	.quad	0x000000000000800a, 0x800000008000000a, 0x8000000080008081
	.quad	0x8000000000008080, 0x0000000080000001, 0x8000000080008008