/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */
//
// Copyright 2025 Google LLC
//
// Author: Eric Biggers <ebiggers@google.com>
//
// This
file is dual-licensed, meaning that you can use it under your choice of
// either of the following two licenses:
//
// Licensed under the Apache License 2.0 (the
"License"). You may obtain a copy
// of the License at
//
//
http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an
"AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// or
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
//
//------------------------------------------------------------------------------
//
// This
file contains x86_64 assembly implementations of AES-CTR and AES-XCTR
// using the following sets of CPU features:
// - AES-NI && AVX
// - VAES && AVX2
// - VAES && AVX512BW && AVX512VL && BMI2
//
// See the function definitions at the bottom of the
file for more information.
#include <linux/linkage.h>
#include <linux/cfi_types.h>
.
section .rodata
.p2align 4
.Lbswap_mask:
.octa 0x000102030405060708090a0b0c0d0e0f
.Lctr_pattern:
.quad 0, 0
.Lone:
.quad 1, 0
.Ltwo:
.quad 2, 0
.quad 3, 0
.Lfour:
.quad 4, 0
.text
// Move a vector between memory and a register.
.macro _vmovdqu src, dst
.if VL < 64
vmovdqu \src, \dst
.else
vmovdqu8 \src, \dst
.endif
.endm
// Move a vector between registers.
.macro _vmovdqa src, dst
.if VL < 64
vmovdqa \src, \dst
.else
vmovdqa64 \src, \dst
.endif
.endm
// Broadcast a 128-bit value from memory to all 128-bit lanes of a vector
// register.
.macro _vbroadcast128 src, dst
.if VL == 16
vmovdqu \src, \dst
.elseif VL == 32
vbroadcasti128 \src, \dst
.else
vbroadcasti32x4 \src, \dst
.endif
.endm
// XOR two vectors together.
.macro _vpxor src1, src2, dst
.if VL < 64
vpxor \src1, \src2, \dst
.else
vpxord \src1, \src2, \dst
.endif
.endm
// Load 1 <= %ecx <= 15 bytes from the pointer \src into the xmm register \dst
// and zeroize any remaining bytes. Clobbers %rax, %rcx, and \tmp{64,32}.
.macro _load_partial_block src, dst, tmp64, tmp32
sub $8, %ecx // LEN - 8
jle .Lle8\@
// Load 9 <= LEN <= 15 bytes.
vmovq (\src), \dst // Load first 8 bytes
mov (\src, %rcx), %rax // Load last 8 bytes
neg %ecx
shl $3, %ecx
shr %cl, %rax // Discard overlapping bytes
vpinsrq $1, %rax, \dst, \dst
jmp .Ldone\@
.Lle8\@:
add $4, %ecx // LEN - 4
jl .Llt4\@
// Load 4 <= LEN <= 8 bytes.
mov (\src), %eax // Load first 4 bytes
mov (\src, %rcx), \tmp32 // Load last 4 bytes
jmp .Lcombine\@
.Llt4\@:
// Load 1 <= LEN <= 3 bytes.
add $2, %ecx // LEN - 2
movzbl (\src), %eax // Load first byte
jl .Lmovq\@
movzwl (\src, %rcx), \tmp32 // Load last 2 bytes
.Lcombine\@:
shl $3, %ecx
shl %cl, \tmp64
or \tmp64, %rax // Combine the two parts
.Lmovq\@:
vmovq %rax, \dst
.Ldone\@:
.endm
// Store 1 <= %ecx <= 15 bytes from the xmm register \src to the pointer \dst.
// Clobbers %rax, %rcx, and \tmp{64,32}.
.macro _store_partial_block src, dst, tmp64, tmp32
sub $8, %ecx // LEN - 8
jl .Llt8\@
// Store 8 <= LEN <= 15 bytes.
vpextrq $1, \src, %rax
mov %ecx, \tmp32
shl $3, %ecx
ror %cl, %rax
mov %rax, (\dst, \tmp64) // Store last LEN - 8 bytes
vmovq \src, (\dst) // Store first 8 bytes
jmp .Ldone\@
.Llt8\@:
add $4, %ecx // LEN - 4
jl .Llt4\@
// Store 4 <= LEN <= 7 bytes.
vpextrd $1, \src, %eax
mov %ecx, \tmp32
shl $3, %ecx
ror %cl, %eax
mov %eax, (\dst, \tmp64) // Store last LEN - 4 bytes
vmovd \src, (\dst) // Store first 4 bytes
jmp .Ldone\@
.Llt4\@:
// Store 1 <= LEN <= 3 bytes.
vpextrb $0, \src, 0(\dst)
cmp $-2, %ecx // LEN - 4 == -2, i.e. LEN == 2?
jl .Ldone\@
vpextrb $1, \src, 1(\dst)
je .Ldone\@
vpextrb $2, \src, 2(\dst)
.Ldone\@:
.endm
// Prepare the next two vectors of AES inputs in AESDATA\i0 and AESDATA\i1, and
// XOR each with the zero-th round key. Also update LE_CTR if
!\final.
.macro _prepare_2_ctr_vecs is_xctr, i0, i1, final=0
.if \is_xctr
.if USE_AVX512
vmovdqa64 LE_CTR, AESDATA\i0
vpternlogd $0x96, XCTR_IV, RNDKEY0, AESDATA\i0
.else
vpxor XCTR_IV, LE_CTR, AESDATA\i0
vpxor RNDKEY0, AESDATA\i0, AESDATA\i0
.endif
vpaddq LE_CTR_INC1, LE_CTR, AESDATA\i1
.if USE_AVX512
vpternlogd $0x96, XCTR_IV, RNDKEY0, AESDATA\i1
.else
vpxor XCTR_IV, AESDATA\i1, AESDATA\i1
vpxor RNDKEY0, AESDATA\i1, AESDATA\i1
.endif
.else
vpshufb BSWAP_MASK, LE_CTR, AESDATA\i0
_vpxor RNDKEY0, AESDATA\i0, AESDATA\i0
vpaddq LE_CTR_INC1, LE_CTR, AESDATA\i1
vpshufb BSWAP_MASK, AESDATA\i1, AESDATA\i1
_vpxor RNDKEY0, AESDATA\i1, AESDATA\i1
.endif
.if
!\final
vpaddq LE_CTR_INC2, LE_CTR, LE_CTR
.endif
.endm
// Do all AES rounds on the
data in the given AESDATA vectors, excluding the
// zero-th and last rounds.
.macro _aesenc_loop vecs:vararg
mov KEY, %rax
1:
_vbroadcast128 (%rax), RNDKEY
.irp i, \vecs
vaesenc RNDKEY, AESDATA\i, AESDATA\i
.endr
add $16, %rax
cmp %rax, RNDKEYLAST_PTR
jne 1b
.endm
// Finalize the keystream blocks in the given AESDATA vectors by doing the last
// AES round, then XOR those keystream blocks with the corresponding
data.
// Reduce latency by doing the XOR before the vaesenclast, utilizing the
// property vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a).
.macro _aesenclast_and_xor vecs:vararg
.irp i, \vecs
_vpxor \i*VL(SRC), RNDKEYLAST, RNDKEY
vaesenclast RNDKEY, AESDATA\i, AESDATA\i
.endr
.irp i, \vecs
_vmovdqu AESDATA\i, \i*VL(DST)
.endr
.endm
// XOR the keystream blocks in the specified AESDATA vectors with the
// corresponding
data.
.macro _xor_data vecs:vararg
.irp i, \vecs
_vpxor \i*VL(SRC), AESDATA\i, AESDATA\i
.endr
.irp i, \vecs
_vmovdqu AESDATA\i, \i*VL(DST)
.endr
.endm
.macro _aes_ctr_crypt is_xctr
// Define register aliases V0-V15 that map to the xmm, ymm, or zmm
// registers according to the selected Vector Length (VL).
.irp i, 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
.if VL == 16
.
set V\i, %xmm\i
.elseif VL == 32
.
set V\i, %ymm\i
.elseif VL == 64
.
set V\i, %zmm\i
.else
.error
"Unsupported Vector Length (VL)"
.endif
.endr
// Function arguments
.
set KEY, %rdi // Initially points to the start of the
// crypto_aes_ctx, then is advanced to
// point to the index 1 round key
.
set KEY32, %edi // Available as temp register after all
// keystream blocks have been generated
.
set SRC, %rsi // Pointer to next source
data
.
set DST, %rdx // Pointer to next destination
data
.
set LEN, %ecx // Remaining length in bytes.
// Note: _load_partial_block relies on
// this being in %ecx.
.
set LEN64, %rcx // Zero-extend LEN before using
!
.
set LEN8, %cl
.if \is_xctr
.
set XCTR_IV_PTR, %r8 // const u8 iv[AES_BLOCK_SIZE];
.
set XCTR_CTR, %r9 // u64 ctr;
.else
.
set LE_CTR_PTR, %r8 // const u64 le_ctr[2];
.endif
// Additional local variables
.
set RNDKEYLAST_PTR, %r10
.
set AESDATA0, V0
.
set AESDATA0_XMM, %xmm0
.
set AESDATA1, V1
.
set AESDATA1_XMM, %xmm1
.
set AESDATA2, V2
.
set AESDATA3, V3
.
set AESDATA4, V4
.
set AESDATA5, V5
.
set AESDATA6, V6
.
set AESDATA7, V7
.if \is_xctr
.
set XCTR_IV, V8
.else
.
set BSWAP_MASK, V8
.endif
.
set LE_CTR, V9
.
set LE_CTR_XMM, %xmm9
.
set LE_CTR_INC1, V10
.
set LE_CTR_INC2, V11
.
set RNDKEY0, V12
.
set RNDKEYLAST, V13
.
set RNDKEY, V14
// Create the first vector of counters.
.if \is_xctr
.if VL == 16
vmovq XCTR_CTR, LE_CTR
.elseif VL == 32
vmovq XCTR_CTR, LE_CTR_XMM
inc XCTR_CTR
vmovq XCTR_CTR, AESDATA0_XMM
vinserti128 $1, AESDATA0_XMM, LE_CTR, LE_CTR
.else
vpbroadcastq XCTR_CTR, LE_CTR
vpsrldq $8, LE_CTR, LE_CTR
vpaddq .Lctr_pattern(%rip), LE_CTR, LE_CTR
.endif
_vbroadcast128 (XCTR_IV_PTR), XCTR_IV
.else
_vbroadcast128 (LE_CTR_PTR), LE_CTR
.if VL > 16
vpaddq .Lctr_pattern(%rip), LE_CTR, LE_CTR
.endif
_vbroadcast128 .Lbswap_mask(%rip), BSWAP_MASK
.endif
.if VL == 16
_vbroadcast128 .Lone(%rip), LE_CTR_INC1
.elseif VL == 32
_vbroadcast128 .Ltwo(%rip), LE_CTR_INC1
.else
_vbroadcast128 .Lfour(%rip), LE_CTR_INC1
.endif
vpsllq $1, LE_CTR_INC1, LE_CTR_INC2
// Load the AES key length: 16 (AES-128), 24 (AES-192), or 32 (AES-256).
movl 480(KEY), %eax
// Compute the pointer to the last round key.
lea 6*16(KEY, %rax, 4), RNDKEYLAST_PTR
// Load the zero-th and last round keys.
_vbroadcast128 (KEY), RNDKEY0
_vbroadcast128 (RNDKEYLAST_PTR), RNDKEYLAST
// Make KEY point to the first round key.
add $16, KEY
// This is the main loop, which encrypts 8 vectors of
data at a time.
add $-8*VL, LEN
jl .Lloop_8x_done\@
.Lloop_8x\@:
_prepare_2_ctr_vecs \is_xctr, 0, 1
_prepare_2_ctr_vecs \is_xctr, 2, 3
_prepare_2_ctr_vecs \is_xctr, 4, 5
_prepare_2_ctr_vecs \is_xctr, 6, 7
_aesenc_loop 0,1,2,3,4,5,6,7
_aesenclast_and_xor 0,1,2,3,4,5,6,7
sub $-8*VL, SRC
sub $-8*VL, DST
add $-8*VL, LEN
jge .Lloop_8x\@
.Lloop_8x_done\@:
sub $-8*VL, LEN
jz .Ldone\@
// 1 <= LEN < 8*VL. Generate 2, 4, or 8 more vectors of keystream
// blocks, depending on the remaining LEN.
_prepare_2_ctr_vecs \is_xctr, 0, 1
_prepare_2_ctr_vecs \is_xctr, 2, 3
cmp $4*VL, LEN
jle .Lenc_tail_atmost4vecs\@
// 4*VL < LEN < 8*VL. Generate 8 vectors of keystream blocks. Use the
// first 4 to XOR 4 full vectors of
data. Then XOR the remaining
data.
_prepare_2_ctr_vecs \is_xctr, 4, 5
_prepare_2_ctr_vecs \is_xctr, 6, 7, final=1
_aesenc_loop 0,1,2,3,4,5,6,7
_aesenclast_and_xor 0,1,2,3
vaesenclast RNDKEYLAST, AESDATA4, AESDATA0
vaesenclast RNDKEYLAST, AESDATA5, AESDATA1
vaesenclast RNDKEYLAST, AESDATA6, AESDATA2
vaesenclast RNDKEYLAST, AESDATA7, AESDATA3
sub $-4*VL, SRC
sub $-4*VL, DST
add $-4*VL, LEN
cmp $1*VL-1, LEN
jle .Lxor_tail_partial_vec_0\@
_xor_data 0
cmp $2*VL-1, LEN
jle .Lxor_tail_partial_vec_1\@
_xor_data 1
cmp $3*VL-1, LEN
jle .Lxor_tail_partial_vec_2\@
_xor_data 2
cmp $4*VL-1, LEN
jle .Lxor_tail_partial_vec_3\@
_xor_data 3
jmp .Ldone\@
.Lenc_tail_atmost4vecs\@:
cmp $2*VL, LEN
jle .Lenc_tail_atmost2vecs\@
// 2*VL < LEN <= 4*VL. Generate 4 vectors of keystream blocks. Use the
// first 2 to XOR 2 full vectors of
data. Then XOR the remaining
data.
_aesenc_loop 0,1,2,3
_aesenclast_and_xor 0,1
vaesenclast RNDKEYLAST, AESDATA2, AESDATA0
vaesenclast RNDKEYLAST, AESDATA3, AESDATA1
sub $-2*VL, SRC
sub $-2*VL, DST
add $-2*VL, LEN
jmp .Lxor_tail_upto2vecs\@
.Lenc_tail_atmost2vecs\@:
// 1 <= LEN <= 2*VL. Generate 2 vectors of keystream blocks. Then XOR
// the remaining
data.
_aesenc_loop 0,1
vaesenclast RNDKEYLAST, AESDATA0, AESDATA0
vaesenclast RNDKEYLAST, AESDATA1, AESDATA1
.Lxor_tail_upto2vecs\@:
cmp $1*VL-1, LEN
jle .Lxor_tail_partial_vec_0\@
_xor_data 0
cmp $2*VL-1, LEN
jle .Lxor_tail_partial_vec_1\@
_xor_data 1
jmp .Ldone\@
.Lxor_tail_partial_vec_1\@:
add $-1*VL, LEN
jz .Ldone\@
sub $-1*VL, SRC
sub $-1*VL, DST
_vmovdqa AESDATA1, AESDATA0
jmp .Lxor_tail_partial_vec_0\@
.Lxor_tail_partial_vec_2\@:
add $-2*VL, LEN
jz .Ldone\@
sub $-2*VL, SRC
sub $-2*VL, DST
_vmovdqa AESDATA2, AESDATA0
jmp .Lxor_tail_partial_vec_0\@
.Lxor_tail_partial_vec_3\@:
add $-3*VL, LEN
jz .Ldone\@
sub $-3*VL, SRC
sub $-3*VL, DST
_vmovdqa AESDATA3, AESDATA0
.Lxor_tail_partial_vec_0\@:
// XOR the remaining 1 <= LEN < VL bytes. It
's easy if masked
// loads/stores are available; otherwise it
's a bit harder...
.if USE_AVX512
mov $-1, %rax
bzhi LEN64, %rax, %rax
kmovq %rax, %k1
vmovdqu8 (SRC), AESDATA1{%k1}{z}
vpxord AESDATA1, AESDATA0, AESDATA0
vmovdqu8 AESDATA0, (DST){%k1}
.else
.if VL == 32
cmp $16, LEN
jl 1f
vpxor (SRC), AESDATA0_XMM, AESDATA1_XMM
vmovdqu AESDATA1_XMM, (DST)
add $16, SRC
add $16, DST
sub $16, LEN
jz .Ldone\@
vextracti128 $1, AESDATA0, AESDATA0_XMM
1:
.endif
mov LEN, %r10d
_load_partial_block SRC, AESDATA1_XMM, KEY, KEY32
vpxor AESDATA1_XMM, AESDATA0_XMM, AESDATA0_XMM
mov %r10d, %ecx
_store_partial_block AESDATA0_XMM, DST, KEY, KEY32
.endif
.Ldone\@:
.if VL > 16
vzeroupper
.endif
RET
.endm
// Below are the definitions of the functions generated by the above macro.
// They have the following prototypes:
//
//
// void aes_ctr64_crypt_##suffix(const struct crypto_aes_ctx *key,
// const u8 *src, u8 *dst, int len,
// const u64 le_ctr[2]);
//
// void aes_xctr_crypt_##suffix(const struct crypto_aes_ctx *key,
// const u8 *src, u8 *dst, int len,
// const u8 iv[AES_BLOCK_SIZE], u64 ctr);
//
// Both functions generate |len| bytes of keystream, XOR it with the
data from
// |src|, and write the result to |dst|. On non-final calls, |len| must be a
// multiple of 16. On the final call, |len| can be any value.
//
// aes_ctr64_crypt_* implement
"regular" CTR, where the keystream is generated
// from a 128-bit big endian counter that increments by 1 for each AES block.
// HOWEVER, to keep the assembly code simple, some of the counter management is
// left to the caller. aes_ctr64_crypt_* take the counter in little endian
// form, only increment the low 64 bits internally, do the conversion to big
// endian internally, and don
't write the updated counter back to memory. The
// caller is responsible for converting the starting IV to the little endian
// le_ctr, detecting the (very rare) case of a carry out of the low 64 bits
// being needed and splitting at that point with a carry done in between, and
// updating le_ctr after each part if the message is multi-part.
//
// aes_xctr_crypt_* implement XCTR as specified in
"Length-preserving encryption
// with HCTR2
" (https://eprint.iacr.org/2021/1441.pdf). XCTR is an
// easier-to-implement variant of CTR that uses little endian byte order and
// eliminates carries. |ctr| is the per-message block counter starting at 1.
.
set VL, 16
.
set USE_AVX512, 0
SYM_TYPED_FUNC_START(aes_ctr64_crypt_aesni_avx)
_aes_ctr_crypt 0
SYM_FUNC_END(aes_ctr64_crypt_aesni_avx)
SYM_TYPED_FUNC_START(aes_xctr_crypt_aesni_avx)
_aes_ctr_crypt 1
SYM_FUNC_END(aes_xctr_crypt_aesni_avx)
#if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ)
.
set VL, 32
.
set USE_AVX512, 0
SYM_TYPED_FUNC_START(aes_ctr64_crypt_vaes_avx2)
_aes_ctr_crypt 0
SYM_FUNC_END(aes_ctr64_crypt_vaes_avx2)
SYM_TYPED_FUNC_START(aes_xctr_crypt_vaes_avx2)
_aes_ctr_crypt 1
SYM_FUNC_END(aes_xctr_crypt_vaes_avx2)
.
set VL, 64
.
set USE_AVX512, 1
SYM_TYPED_FUNC_START(aes_ctr64_crypt_vaes_avx512)
_aes_ctr_crypt 0
SYM_FUNC_END(aes_ctr64_crypt_vaes_avx512)
SYM_TYPED_FUNC_START(aes_xctr_crypt_vaes_avx512)
_aes_ctr_crypt 1
SYM_FUNC_END(aes_xctr_crypt_vaes_avx512)
#endif // CONFIG_AS_VAES && CONFIG_AS_VPCLMULQDQ
