| /* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */ |
| // |
| // Copyright 2025 Google LLC |
| // |
| // Author: Eric Biggers <ebiggers@google.com> |
| // |
| // This file is dual-licensed, meaning that you can use it under your choice of |
| // either of the following two licenses: |
| // |
| // Licensed under the Apache License 2.0 (the "License"). You may obtain a copy |
| // of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| // |
| // or |
| // |
| // Redistribution and use in source and binary forms, with or without |
| // modification, are permitted provided that the following conditions are met: |
| // |
| // 1. Redistributions of source code must retain the above copyright notice, |
| // this list of conditions and the following disclaimer. |
| // |
| // 2. Redistributions in binary form must reproduce the above copyright |
| // notice, this list of conditions and the following disclaimer in the |
| // documentation and/or other materials provided with the distribution. |
| // |
| // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
| // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE |
| // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
| // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
| // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
| // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
| // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
| // POSSIBILITY OF SUCH DAMAGE. |
| // |
| //------------------------------------------------------------------------------ |
| // |
| // This file contains x86_64 assembly implementations of AES-CTR and AES-XCTR |
| // using the following sets of CPU features: |
| // - AES-NI && AVX |
| // - VAES && AVX2 |
| // - VAES && AVX512BW && AVX512VL && BMI2 |
| // |
| // See the function definitions at the bottom of the file for more information. |
| |
| #include <linux/linkage.h> |
| #include <linux/cfi_types.h> |
| |
| .section .rodata |
| .p2align 4 |
| |
| .Lbswap_mask: |
| .octa 0x000102030405060708090a0b0c0d0e0f |
| |
| .Lctr_pattern: |
| .quad 0, 0 |
| .Lone: |
| .quad 1, 0 |
| .Ltwo: |
| .quad 2, 0 |
| .quad 3, 0 |
| |
| .Lfour: |
| .quad 4, 0 |
| |
| .text |
| |
| // Move a vector between memory and a register. |
| .macro _vmovdqu src, dst |
| .if VL < 64 |
| vmovdqu \src, \dst |
| .else |
| vmovdqu8 \src, \dst |
| .endif |
| .endm |
| |
| // Move a vector between registers. |
| .macro _vmovdqa src, dst |
| .if VL < 64 |
| vmovdqa \src, \dst |
| .else |
| vmovdqa64 \src, \dst |
| .endif |
| .endm |
| |
| // Broadcast a 128-bit value from memory to all 128-bit lanes of a vector |
| // register. |
| .macro _vbroadcast128 src, dst |
| .if VL == 16 |
| vmovdqu \src, \dst |
| .elseif VL == 32 |
| vbroadcasti128 \src, \dst |
| .else |
| vbroadcasti32x4 \src, \dst |
| .endif |
| .endm |
| |
| // XOR two vectors together. |
| .macro _vpxor src1, src2, dst |
| .if VL < 64 |
| vpxor \src1, \src2, \dst |
| .else |
| vpxord \src1, \src2, \dst |
| .endif |
| .endm |
| |
| // Load 1 <= %ecx <= 15 bytes from the pointer \src into the xmm register \dst |
| // and zeroize any remaining bytes. Clobbers %rax, %rcx, and \tmp{64,32}. |
| .macro _load_partial_block src, dst, tmp64, tmp32 |
| sub $8, %ecx // LEN - 8 |
| jle .Lle8\@ |
| |
| // Load 9 <= LEN <= 15 bytes. |
| vmovq (\src), \dst // Load first 8 bytes |
| mov (\src, %rcx), %rax // Load last 8 bytes |
| neg %ecx |
| shl $3, %ecx |
| shr %cl, %rax // Discard overlapping bytes |
| vpinsrq $1, %rax, \dst, \dst |
| jmp .Ldone\@ |
| |
| .Lle8\@: |
| add $4, %ecx // LEN - 4 |
| jl .Llt4\@ |
| |
| // Load 4 <= LEN <= 8 bytes. |
| mov (\src), %eax // Load first 4 bytes |
| mov (\src, %rcx), \tmp32 // Load last 4 bytes |
| jmp .Lcombine\@ |
| |
| .Llt4\@: |
| // Load 1 <= LEN <= 3 bytes. |
| add $2, %ecx // LEN - 2 |
| movzbl (\src), %eax // Load first byte |
| jl .Lmovq\@ |
| movzwl (\src, %rcx), \tmp32 // Load last 2 bytes |
| .Lcombine\@: |
| shl $3, %ecx |
| shl %cl, \tmp64 |
| or \tmp64, %rax // Combine the two parts |
| .Lmovq\@: |
| vmovq %rax, \dst |
| .Ldone\@: |
| .endm |
| |
| // Store 1 <= %ecx <= 15 bytes from the xmm register \src to the pointer \dst. |
| // Clobbers %rax, %rcx, and \tmp{64,32}. |
| .macro _store_partial_block src, dst, tmp64, tmp32 |
| sub $8, %ecx // LEN - 8 |
| jl .Llt8\@ |
| |
| // Store 8 <= LEN <= 15 bytes. |
| vpextrq $1, \src, %rax |
| mov %ecx, \tmp32 |
| shl $3, %ecx |
| ror %cl, %rax |
| mov %rax, (\dst, \tmp64) // Store last LEN - 8 bytes |
| vmovq \src, (\dst) // Store first 8 bytes |
| jmp .Ldone\@ |
| |
| .Llt8\@: |
| add $4, %ecx // LEN - 4 |
| jl .Llt4\@ |
| |
| // Store 4 <= LEN <= 7 bytes. |
| vpextrd $1, \src, %eax |
| mov %ecx, \tmp32 |
| shl $3, %ecx |
| ror %cl, %eax |
| mov %eax, (\dst, \tmp64) // Store last LEN - 4 bytes |
| vmovd \src, (\dst) // Store first 4 bytes |
| jmp .Ldone\@ |
| |
| .Llt4\@: |
| // Store 1 <= LEN <= 3 bytes. |
| vpextrb $0, \src, 0(\dst) |
| cmp $-2, %ecx // LEN - 4 == -2, i.e. LEN == 2? |
| jl .Ldone\@ |
| vpextrb $1, \src, 1(\dst) |
| je .Ldone\@ |
| vpextrb $2, \src, 2(\dst) |
| .Ldone\@: |
| .endm |
| |
| // Prepare the next two vectors of AES inputs in AESDATA\i0 and AESDATA\i1, and |
| // XOR each with the zero-th round key. Also update LE_CTR if !\final. |
| .macro _prepare_2_ctr_vecs is_xctr, i0, i1, final=0 |
| .if \is_xctr |
| .if USE_AVX512 |
| vmovdqa64 LE_CTR, AESDATA\i0 |
| vpternlogd $0x96, XCTR_IV, RNDKEY0, AESDATA\i0 |
| .else |
| vpxor XCTR_IV, LE_CTR, AESDATA\i0 |
| vpxor RNDKEY0, AESDATA\i0, AESDATA\i0 |
| .endif |
| vpaddq LE_CTR_INC1, LE_CTR, AESDATA\i1 |
| |
| .if USE_AVX512 |
| vpternlogd $0x96, XCTR_IV, RNDKEY0, AESDATA\i1 |
| .else |
| vpxor XCTR_IV, AESDATA\i1, AESDATA\i1 |
| vpxor RNDKEY0, AESDATA\i1, AESDATA\i1 |
| .endif |
| .else |
| vpshufb BSWAP_MASK, LE_CTR, AESDATA\i0 |
| _vpxor RNDKEY0, AESDATA\i0, AESDATA\i0 |
| vpaddq LE_CTR_INC1, LE_CTR, AESDATA\i1 |
| vpshufb BSWAP_MASK, AESDATA\i1, AESDATA\i1 |
| _vpxor RNDKEY0, AESDATA\i1, AESDATA\i1 |
| .endif |
| .if !\final |
| vpaddq LE_CTR_INC2, LE_CTR, LE_CTR |
| .endif |
| .endm |
| |
| // Do all AES rounds on the data in the given AESDATA vectors, excluding the |
| // zero-th and last rounds. |
| .macro _aesenc_loop vecs:vararg |
| mov KEY, %rax |
| 1: |
| _vbroadcast128 (%rax), RNDKEY |
| .irp i, \vecs |
| vaesenc RNDKEY, AESDATA\i, AESDATA\i |
| .endr |
| add $16, %rax |
| cmp %rax, RNDKEYLAST_PTR |
| jne 1b |
| .endm |
| |
| // Finalize the keystream blocks in the given AESDATA vectors by doing the last |
| // AES round, then XOR those keystream blocks with the corresponding data. |
| // Reduce latency by doing the XOR before the vaesenclast, utilizing the |
| // property vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a). |
| .macro _aesenclast_and_xor vecs:vararg |
| .irp i, \vecs |
| _vpxor \i*VL(SRC), RNDKEYLAST, RNDKEY |
| vaesenclast RNDKEY, AESDATA\i, AESDATA\i |
| .endr |
| .irp i, \vecs |
| _vmovdqu AESDATA\i, \i*VL(DST) |
| .endr |
| .endm |
| |
| // XOR the keystream blocks in the specified AESDATA vectors with the |
| // corresponding data. |
| .macro _xor_data vecs:vararg |
| .irp i, \vecs |
| _vpxor \i*VL(SRC), AESDATA\i, AESDATA\i |
| .endr |
| .irp i, \vecs |
| _vmovdqu AESDATA\i, \i*VL(DST) |
| .endr |
| .endm |
| |
| .macro _aes_ctr_crypt is_xctr |
| |
| // Define register aliases V0-V15 that map to the xmm, ymm, or zmm |
| // registers according to the selected Vector Length (VL). |
| .irp i, 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 |
| .if VL == 16 |
| .set V\i, %xmm\i |
| .elseif VL == 32 |
| .set V\i, %ymm\i |
| .elseif VL == 64 |
| .set V\i, %zmm\i |
| .else |
| .error "Unsupported Vector Length (VL)" |
| .endif |
| .endr |
| |
| // Function arguments |
| .set KEY, %rdi // Initially points to the start of the |
| // crypto_aes_ctx, then is advanced to |
| // point to the index 1 round key |
| .set KEY32, %edi // Available as temp register after all |
| // keystream blocks have been generated |
| .set SRC, %rsi // Pointer to next source data |
| .set DST, %rdx // Pointer to next destination data |
| .set LEN, %ecx // Remaining length in bytes. |
| // Note: _load_partial_block relies on |
| // this being in %ecx. |
| .set LEN64, %rcx // Zero-extend LEN before using! |
| .set LEN8, %cl |
| .if \is_xctr |
| .set XCTR_IV_PTR, %r8 // const u8 iv[AES_BLOCK_SIZE]; |
| .set XCTR_CTR, %r9 // u64 ctr; |
| .else |
| .set LE_CTR_PTR, %r8 // const u64 le_ctr[2]; |
| .endif |
| |
| // Additional local variables |
| .set RNDKEYLAST_PTR, %r10 |
| .set AESDATA0, V0 |
| .set AESDATA0_XMM, %xmm0 |
| .set AESDATA1, V1 |
| .set AESDATA1_XMM, %xmm1 |
| .set AESDATA2, V2 |
| .set AESDATA3, V3 |
| .set AESDATA4, V4 |
| .set AESDATA5, V5 |
| .set AESDATA6, V6 |
| .set AESDATA7, V7 |
| .if \is_xctr |
| .set XCTR_IV, V8 |
| .else |
| .set BSWAP_MASK, V8 |
| .endif |
| .set LE_CTR, V9 |
| .set LE_CTR_XMM, %xmm9 |
| .set LE_CTR_INC1, V10 |
| .set LE_CTR_INC2, V11 |
| .set RNDKEY0, V12 |
| .set RNDKEYLAST, V13 |
| .set RNDKEY, V14 |
| |
| // Create the first vector of counters. |
| .if \is_xctr |
| .if VL == 16 |
| vmovq XCTR_CTR, LE_CTR |
| .elseif VL == 32 |
| vmovq XCTR_CTR, LE_CTR_XMM |
| inc XCTR_CTR |
| vmovq XCTR_CTR, AESDATA0_XMM |
| vinserti128 $1, AESDATA0_XMM, LE_CTR, LE_CTR |
| .else |
| vpbroadcastq XCTR_CTR, LE_CTR |
| vpsrldq $8, LE_CTR, LE_CTR |
| vpaddq .Lctr_pattern(%rip), LE_CTR, LE_CTR |
| .endif |
| _vbroadcast128 (XCTR_IV_PTR), XCTR_IV |
| .else |
| _vbroadcast128 (LE_CTR_PTR), LE_CTR |
| .if VL > 16 |
| vpaddq .Lctr_pattern(%rip), LE_CTR, LE_CTR |
| .endif |
| _vbroadcast128 .Lbswap_mask(%rip), BSWAP_MASK |
| .endif |
| |
| .if VL == 16 |
| _vbroadcast128 .Lone(%rip), LE_CTR_INC1 |
| .elseif VL == 32 |
| _vbroadcast128 .Ltwo(%rip), LE_CTR_INC1 |
| .else |
| _vbroadcast128 .Lfour(%rip), LE_CTR_INC1 |
| .endif |
| vpsllq $1, LE_CTR_INC1, LE_CTR_INC2 |
| |
| // Load the AES key length: 16 (AES-128), 24 (AES-192), or 32 (AES-256). |
| movl 480(KEY), %eax |
| |
| // Compute the pointer to the last round key. |
| lea 6*16(KEY, %rax, 4), RNDKEYLAST_PTR |
| |
| // Load the zero-th and last round keys. |
| _vbroadcast128 (KEY), RNDKEY0 |
| _vbroadcast128 (RNDKEYLAST_PTR), RNDKEYLAST |
| |
| // Make KEY point to the first round key. |
| add $16, KEY |
| |
| // This is the main loop, which encrypts 8 vectors of data at a time. |
| add $-8*VL, LEN |
| jl .Lloop_8x_done\@ |
| .Lloop_8x\@: |
| _prepare_2_ctr_vecs \is_xctr, 0, 1 |
| _prepare_2_ctr_vecs \is_xctr, 2, 3 |
| _prepare_2_ctr_vecs \is_xctr, 4, 5 |
| _prepare_2_ctr_vecs \is_xctr, 6, 7 |
| _aesenc_loop 0,1,2,3,4,5,6,7 |
| _aesenclast_and_xor 0,1,2,3,4,5,6,7 |
| sub $-8*VL, SRC |
| sub $-8*VL, DST |
| add $-8*VL, LEN |
| jge .Lloop_8x\@ |
| .Lloop_8x_done\@: |
| sub $-8*VL, LEN |
| jz .Ldone\@ |
| |
| // 1 <= LEN < 8*VL. Generate 2, 4, or 8 more vectors of keystream |
| // blocks, depending on the remaining LEN. |
| |
| _prepare_2_ctr_vecs \is_xctr, 0, 1 |
| _prepare_2_ctr_vecs \is_xctr, 2, 3 |
| cmp $4*VL, LEN |
| jle .Lenc_tail_atmost4vecs\@ |
| |
| // 4*VL < LEN < 8*VL. Generate 8 vectors of keystream blocks. Use the |
| // first 4 to XOR 4 full vectors of data. Then XOR the remaining data. |
| _prepare_2_ctr_vecs \is_xctr, 4, 5 |
| _prepare_2_ctr_vecs \is_xctr, 6, 7, final=1 |
| _aesenc_loop 0,1,2,3,4,5,6,7 |
| _aesenclast_and_xor 0,1,2,3 |
| vaesenclast RNDKEYLAST, AESDATA4, AESDATA0 |
| vaesenclast RNDKEYLAST, AESDATA5, AESDATA1 |
| vaesenclast RNDKEYLAST, AESDATA6, AESDATA2 |
| vaesenclast RNDKEYLAST, AESDATA7, AESDATA3 |
| sub $-4*VL, SRC |
| sub $-4*VL, DST |
| add $-4*VL, LEN |
| cmp $1*VL-1, LEN |
| jle .Lxor_tail_partial_vec_0\@ |
| _xor_data 0 |
| cmp $2*VL-1, LEN |
| jle .Lxor_tail_partial_vec_1\@ |
| _xor_data 1 |
| cmp $3*VL-1, LEN |
| jle .Lxor_tail_partial_vec_2\@ |
| _xor_data 2 |
| cmp $4*VL-1, LEN |
| jle .Lxor_tail_partial_vec_3\@ |
| _xor_data 3 |
| jmp .Ldone\@ |
| |
| .Lenc_tail_atmost4vecs\@: |
| cmp $2*VL, LEN |
| jle .Lenc_tail_atmost2vecs\@ |
| |
| // 2*VL < LEN <= 4*VL. Generate 4 vectors of keystream blocks. Use the |
| // first 2 to XOR 2 full vectors of data. Then XOR the remaining data. |
| _aesenc_loop 0,1,2,3 |
| _aesenclast_and_xor 0,1 |
| vaesenclast RNDKEYLAST, AESDATA2, AESDATA0 |
| vaesenclast RNDKEYLAST, AESDATA3, AESDATA1 |
| sub $-2*VL, SRC |
| sub $-2*VL, DST |
| add $-2*VL, LEN |
| jmp .Lxor_tail_upto2vecs\@ |
| |
| .Lenc_tail_atmost2vecs\@: |
| // 1 <= LEN <= 2*VL. Generate 2 vectors of keystream blocks. Then XOR |
| // the remaining data. |
| _aesenc_loop 0,1 |
| vaesenclast RNDKEYLAST, AESDATA0, AESDATA0 |
| vaesenclast RNDKEYLAST, AESDATA1, AESDATA1 |
| |
| .Lxor_tail_upto2vecs\@: |
| cmp $1*VL-1, LEN |
| jle .Lxor_tail_partial_vec_0\@ |
| _xor_data 0 |
| cmp $2*VL-1, LEN |
| jle .Lxor_tail_partial_vec_1\@ |
| _xor_data 1 |
| jmp .Ldone\@ |
| |
| .Lxor_tail_partial_vec_1\@: |
| add $-1*VL, LEN |
| jz .Ldone\@ |
| sub $-1*VL, SRC |
| sub $-1*VL, DST |
| _vmovdqa AESDATA1, AESDATA0 |
| jmp .Lxor_tail_partial_vec_0\@ |
| |
| .Lxor_tail_partial_vec_2\@: |
| add $-2*VL, LEN |
| jz .Ldone\@ |
| sub $-2*VL, SRC |
| sub $-2*VL, DST |
| _vmovdqa AESDATA2, AESDATA0 |
| jmp .Lxor_tail_partial_vec_0\@ |
| |
| .Lxor_tail_partial_vec_3\@: |
| add $-3*VL, LEN |
| jz .Ldone\@ |
| sub $-3*VL, SRC |
| sub $-3*VL, DST |
| _vmovdqa AESDATA3, AESDATA0 |
| |
| .Lxor_tail_partial_vec_0\@: |
| // XOR the remaining 1 <= LEN < VL bytes. It's easy if masked |
| // loads/stores are available; otherwise it's a bit harder... |
| .if USE_AVX512 |
| mov $-1, %rax |
| bzhi LEN64, %rax, %rax |
| kmovq %rax, %k1 |
| vmovdqu8 (SRC), AESDATA1{%k1}{z} |
| vpxord AESDATA1, AESDATA0, AESDATA0 |
| vmovdqu8 AESDATA0, (DST){%k1} |
| .else |
| .if VL == 32 |
| cmp $16, LEN |
| jl 1f |
| vpxor (SRC), AESDATA0_XMM, AESDATA1_XMM |
| vmovdqu AESDATA1_XMM, (DST) |
| add $16, SRC |
| add $16, DST |
| sub $16, LEN |
| jz .Ldone\@ |
| vextracti128 $1, AESDATA0, AESDATA0_XMM |
| 1: |
| .endif |
| mov LEN, %r10d |
| _load_partial_block SRC, AESDATA1_XMM, KEY, KEY32 |
| vpxor AESDATA1_XMM, AESDATA0_XMM, AESDATA0_XMM |
| mov %r10d, %ecx |
| _store_partial_block AESDATA0_XMM, DST, KEY, KEY32 |
| .endif |
| |
| .Ldone\@: |
| .if VL > 16 |
| vzeroupper |
| .endif |
| RET |
| .endm |
| |
| // Below are the definitions of the functions generated by the above macro. |
| // They have the following prototypes: |
| // |
| // |
| // void aes_ctr64_crypt_##suffix(const struct crypto_aes_ctx *key, |
| // const u8 *src, u8 *dst, int len, |
| // const u64 le_ctr[2]); |
| // |
| // void aes_xctr_crypt_##suffix(const struct crypto_aes_ctx *key, |
| // const u8 *src, u8 *dst, int len, |
| // const u8 iv[AES_BLOCK_SIZE], u64 ctr); |
| // |
| // Both functions generate |len| bytes of keystream, XOR it with the data from |
| // |src|, and write the result to |dst|. On non-final calls, |len| must be a |
| // multiple of 16. On the final call, |len| can be any value. |
| // |
| // aes_ctr64_crypt_* implement "regular" CTR, where the keystream is generated |
| // from a 128-bit big endian counter that increments by 1 for each AES block. |
| // HOWEVER, to keep the assembly code simple, some of the counter management is |
| // left to the caller. aes_ctr64_crypt_* take the counter in little endian |
| // form, only increment the low 64 bits internally, do the conversion to big |
| // endian internally, and don't write the updated counter back to memory. The |
| // caller is responsible for converting the starting IV to the little endian |
| // le_ctr, detecting the (very rare) case of a carry out of the low 64 bits |
| // being needed and splitting at that point with a carry done in between, and |
| // updating le_ctr after each part if the message is multi-part. |
| // |
| // aes_xctr_crypt_* implement XCTR as specified in "Length-preserving encryption |
| // with HCTR2" (https://eprint.iacr.org/2021/1441.pdf). XCTR is an |
| // easier-to-implement variant of CTR that uses little endian byte order and |
| // eliminates carries. |ctr| is the per-message block counter starting at 1. |
| |
| .set VL, 16 |
| .set USE_AVX512, 0 |
| SYM_TYPED_FUNC_START(aes_ctr64_crypt_aesni_avx) |
| _aes_ctr_crypt 0 |
| SYM_FUNC_END(aes_ctr64_crypt_aesni_avx) |
| SYM_TYPED_FUNC_START(aes_xctr_crypt_aesni_avx) |
| _aes_ctr_crypt 1 |
| SYM_FUNC_END(aes_xctr_crypt_aesni_avx) |
| |
| #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ) |
| .set VL, 32 |
| .set USE_AVX512, 0 |
| SYM_TYPED_FUNC_START(aes_ctr64_crypt_vaes_avx2) |
| _aes_ctr_crypt 0 |
| SYM_FUNC_END(aes_ctr64_crypt_vaes_avx2) |
| SYM_TYPED_FUNC_START(aes_xctr_crypt_vaes_avx2) |
| _aes_ctr_crypt 1 |
| SYM_FUNC_END(aes_xctr_crypt_vaes_avx2) |
| |
| .set VL, 64 |
| .set USE_AVX512, 1 |
| SYM_TYPED_FUNC_START(aes_ctr64_crypt_vaes_avx512) |
| _aes_ctr_crypt 0 |
| SYM_FUNC_END(aes_ctr64_crypt_vaes_avx512) |
| SYM_TYPED_FUNC_START(aes_xctr_crypt_vaes_avx512) |
| _aes_ctr_crypt 1 |
| SYM_FUNC_END(aes_xctr_crypt_vaes_avx512) |
| #endif // CONFIG_AS_VAES && CONFIG_AS_VPCLMULQDQ |
