/* SPDX-License-Identifier: GPL-2.0-only */
/*
* aes-ce-ccm-core.S - AES-CCM transform for ARMv8 with Crypto Extensions
*
* Copyright (C) 2013 - 2017 Linaro Ltd.
* Copyright (C) 2024 Google LLC
*
* Author: Ard Biesheuvel <ardb@kernel.org>
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
.text
.arch armv8-a+crypto
.macro load_round_keys, rk, nr, tmp
sub w\tmp, \nr, #10
add \tmp, \rk, w\tmp, sxtw #4
ld1 {v10.4s-v13.4s}, [\rk]
ld1 {v14.4s-v17.4s}, [\tmp], #64
ld1 {v18.4s-v21.4s}, [\tmp], #64
ld1 {v3.4s-v5.4s}, [\tmp]
.endm
.macro dround, va, vb, vk
aese \va\().16b, \vk\().16b
aesmc \va\().16b, \va\().16b
aese \vb\().16b, \vk\().16b
aesmc \vb\().16b, \vb\().16b
.endm
.macro aes_encrypt, va, vb, nr
tbz \nr, #2, .L\@
dround \va, \vb, v10
dround \va, \vb, v11
tbz \nr, #1, .L\@
dround \va, \vb, v12
dround \va, \vb, v13
.L\@: .irp v, v14, v15, v16, v17, v18, v19, v20, v21, v3
dround \va, \vb, \v
.endr
aese \va\().16b, v4.16b
aese \vb\().16b, v4.16b
.endm
.macro aes_ccm_do_crypt,enc
load_round_keys x3, w4, x10
ld1 {v0.16b}, [x5]
/* load mac */
cbz x2, ce_aes_ccm_final
ldr x8, [x6, #8]
/* load lower ctr */
CPU_LE( rev x8, x8 )
/* keep swabbed ctr in reg */
0:
/* outer loop */
ld1 {v1.8b}, [x6]
/* load upper ctr */
prfm pldl1strm, [x1]
add x8, x8, #1
rev x9, x8
ins v1.d[1], x9
/* no carry in lower ctr */
aes_encrypt v0, v1, w4
subs w2, w2, #16
bmi ce_aes_ccm_crypt_tail
ld1 {v2.16b}, [x1], #16
/* load next input block */
.if \enc == 1
eor v2.16b, v2.16b, v5.16b
/* final round enc+mac */
eor v6.16b, v1.16b, v2.16b
/* xor with crypted ctr */
.else
eor v2.16b, v2.16b, v1.16b
/* xor with crypted ctr */
eor v6.16b, v2.16b, v5.16b
/* final round enc */
.endif
eor v0.16b, v0.16b, v2.16b
/* xor mac with pt ^ rk[last] */
st1 {v6.16b}, [x0], #16
/* write output block */
bne 0b
CPU_LE( rev x8, x8 )
str x8, [x6, #8]
/* store lsb end of ctr (BE) */
cbnz x7, ce_aes_ccm_final
st1 {v0.16b}, [x5]
/* store mac */
ret
.endm
SYM_FUNC_START_LOCAL(ce_aes_ccm_crypt_tail)
eor v0.16b, v0.16b, v5.16b
/* final round mac */
eor v1.16b, v1.16b, v5.16b
/* final round enc */
add x1, x1, w2, sxtw
/* rewind the input pointer (w2 < 0) */
add x0, x0, w2, sxtw
/* rewind the output pointer */
adr_l x8, .Lpermute
/* load permute vectors */
add x9, x8, w2, sxtw
sub x8, x8, w2, sxtw
ld1 {v7.16b-v8.16b}, [x9]
ld1 {v9.16b}, [x8]
ld1 {v2.16b}, [x1]
/* load a full block of input */
tbl v1.16b, {v1.16b}, v7.16b
/* move keystream to end of register */
eor v7.16b, v2.16b, v1.16b
/* encrypt partial input block */
bif v2.16b, v7.16b, v22.16b
/* select plaintext */
tbx v7.16b, {v6.16b}, v8.16b
/* insert output from previous iteration */
tbl v2.16b, {v2.16b}, v9.16b
/* copy plaintext to start of v2 */
eor v0.16b, v0.16b, v2.16b
/* fold plaintext into mac */
st1 {v7.16b}, [x0]
/* store output block */
cbz x7, 0f
SYM_INNER_LABEL(ce_aes_ccm_final, SYM_L_LOCAL)
ld1 {v1.16b}, [x7]
/* load 1st ctriv */
aes_encrypt v0, v1, w4
/* final round key cancels out */
eor v0.16b, v0.16b, v1.16b
/* en-/decrypt the mac */
0: st1 {v0.16b}, [x5]
/* store result */
ret
SYM_FUNC_END(ce_aes_ccm_crypt_tail)
/*
* void ce_aes_ccm_encrypt(u8 out[], u8 const in[], u32 cbytes,
* u8 const rk[], u32 rounds, u8 mac[],
* u8 ctr[], u8 const final_iv[]);
* void ce_aes_ccm_decrypt(u8 out[], u8 const in[], u32 cbytes,
* u8 const rk[], u32 rounds, u8 mac[],
* u8 ctr[], u8 const final_iv[]);
*/
SYM_FUNC_START(ce_aes_ccm_encrypt)
movi v22.16b, #255
aes_ccm_do_crypt 1
SYM_FUNC_END(ce_aes_ccm_encrypt)
SYM_FUNC_START(ce_aes_ccm_decrypt)
movi v22.16b, #0
aes_ccm_do_crypt 0
SYM_FUNC_END(ce_aes_ccm_decrypt)
.
section ".rodata",
"a"
.
align 6
.fill 15, 1, 0xff
.Lpermute:
.byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7
.byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf
.fill 15, 1, 0xff
