diff options
| -rw-r--r-- | crypto/Kconfig | 5 | ||||
| -rw-r--r-- | crypto/Makefile | 12 | ||||
| -rw-r--r-- | crypto/aegis128-neon-inner.c | 149 | ||||
| -rw-r--r-- | crypto/aegis128-neon.c | 43 | ||||
| -rw-r--r-- | crypto/aegis128.c (renamed from crypto/aegis128-core.c) | 42 |
5 files changed, 4 insertions, 247 deletions
diff --git a/crypto/Kconfig b/crypto/Kconfig index 455a3354e291..8880c1fc51d8 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -306,11 +306,6 @@ config CRYPTO_AEGIS128 help Support for the AEGIS-128 dedicated AEAD algorithm. -config CRYPTO_AEGIS128_SIMD - bool "Support SIMD acceleration for AEGIS-128" - depends on CRYPTO_AEGIS128 && ((ARM || ARM64) && KERNEL_MODE_NEON) - default y - config CRYPTO_AEGIS128_AESNI_SSE2 tristate "AEGIS-128 AEAD algorithm (x86_64 AESNI+SSE2 implementation)" depends on X86 && 64BIT diff --git a/crypto/Makefile b/crypto/Makefile index b3e16b4fb414..93375e124ff7 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -90,18 +90,6 @@ obj-$(CONFIG_CRYPTO_GCM) += gcm.o obj-$(CONFIG_CRYPTO_CCM) += ccm.o obj-$(CONFIG_CRYPTO_CHACHA20POLY1305) += chacha20poly1305.o obj-$(CONFIG_CRYPTO_AEGIS128) += aegis128.o -aegis128-y := aegis128-core.o - -ifeq ($(ARCH),arm) -CFLAGS_aegis128-neon-inner.o += -ffreestanding -march=armv7-a -mfloat-abi=softfp -mfpu=crypto-neon-fp-armv8 -aegis128-$(CONFIG_CRYPTO_AEGIS128_SIMD) += aegis128-neon.o aegis128-neon-inner.o -endif -ifeq ($(ARCH),arm64) -CFLAGS_aegis128-neon-inner.o += -ffreestanding -mcpu=generic+crypto -CFLAGS_REMOVE_aegis128-neon-inner.o += -mgeneral-regs-only -aegis128-$(CONFIG_CRYPTO_AEGIS128_SIMD) += aegis128-neon.o aegis128-neon-inner.o -endif - obj-$(CONFIG_CRYPTO_PCRYPT) += pcrypt.o obj-$(CONFIG_CRYPTO_CRYPTD) += cryptd.o obj-$(CONFIG_CRYPTO_DES) += des_generic.o diff --git a/crypto/aegis128-neon-inner.c b/crypto/aegis128-neon-inner.c deleted file mode 100644 index 26e9450a5833..000000000000 --- a/crypto/aegis128-neon-inner.c +++ /dev/null @@ -1,149 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2019 Linaro, Ltd. <ard.biesheuvel@linaro.org> - */ - -#ifdef CONFIG_ARM64 -#include <asm/neon-intrinsics.h> - -#define AES_ROUND "aese %0.16b, %1.16b \n\t aesmc %0.16b, %0.16b" -#else -#include <arm_neon.h> - -#define AES_ROUND "aese.8 %q0, %q1 \n\t aesmc.8 %q0, %q0" -#endif - -#define AEGIS_BLOCK_SIZE 16 - -#include <stddef.h> - -void *memcpy(void *dest, const void *src, size_t n); -void *memset(void *s, int c, size_t n); - -struct aegis128_state { - uint8x16_t v[5]; -}; - -static struct aegis128_state aegis128_load_state_neon(const void *state) -{ - return (struct aegis128_state){ { - vld1q_u8(state), - vld1q_u8(state + 16), - vld1q_u8(state + 32), - vld1q_u8(state + 48), - vld1q_u8(state + 64) - } }; -} - -static void aegis128_save_state_neon(struct aegis128_state st, void *state) -{ - vst1q_u8(state, st.v[0]); - vst1q_u8(state + 16, st.v[1]); - vst1q_u8(state + 32, st.v[2]); - vst1q_u8(state + 48, st.v[3]); - vst1q_u8(state + 64, st.v[4]); -} - -static uint8x16_t aegis_aes_round(uint8x16_t w) -{ - uint8x16_t z = {}; - - /* - * We use inline asm here instead of the vaeseq_u8/vaesmcq_u8 intrinsics - * to force the compiler to issue the aese/aesmc instructions in pairs. - * This is much faster on many cores, where the instruction pair can - * execute in a single cycle. - */ - asm(AES_ROUND : "+w"(w) : "w"(z)); - return w; -} - -static struct aegis128_state aegis128_update_neon(struct aegis128_state st, - uint8x16_t m) -{ - uint8x16_t t; - - t = aegis_aes_round(st.v[3]); - st.v[3] ^= aegis_aes_round(st.v[2]); - st.v[2] ^= aegis_aes_round(st.v[1]); - st.v[1] ^= aegis_aes_round(st.v[0]); - st.v[0] ^= aegis_aes_round(st.v[4]) ^ m; - st.v[4] ^= t; - - return st; -} - -void crypto_aegis128_update_neon(void *state, const void *msg) -{ - struct aegis128_state st = aegis128_load_state_neon(state); - - st = aegis128_update_neon(st, vld1q_u8(msg)); - - aegis128_save_state_neon(st, state); -} - -void crypto_aegis128_encrypt_chunk_neon(void *state, void *dst, const void *src, - unsigned int size) -{ - struct aegis128_state st = aegis128_load_state_neon(state); - uint8x16_t tmp; - - while (size >= AEGIS_BLOCK_SIZE) { - uint8x16_t s = vld1q_u8(src); - - tmp = s ^ st.v[1] ^ (st.v[2] & st.v[3]) ^ st.v[4]; - st = aegis128_update_neon(st, s); - vst1q_u8(dst, tmp); - - size -= AEGIS_BLOCK_SIZE; - src += AEGIS_BLOCK_SIZE; - dst += AEGIS_BLOCK_SIZE; - } - - if (size > 0) { - uint8_t buf[AEGIS_BLOCK_SIZE] = {}; - uint8x16_t msg; - - memcpy(buf, src, size); - msg = vld1q_u8(buf); - tmp = msg ^ st.v[1] ^ (st.v[2] & st.v[3]) ^ st.v[4]; - st = aegis128_update_neon(st, msg); - vst1q_u8(buf, tmp); - memcpy(dst, buf, size); - } - - aegis128_save_state_neon(st, state); -} - -void crypto_aegis128_decrypt_chunk_neon(void *state, void *dst, const void *src, - unsigned int size) -{ - struct aegis128_state st = aegis128_load_state_neon(state); - uint8x16_t tmp; - - while (size >= AEGIS_BLOCK_SIZE) { - tmp = vld1q_u8(src) ^ st.v[1] ^ (st.v[2] & st.v[3]) ^ st.v[4]; - st = aegis128_update_neon(st, tmp); - vst1q_u8(dst, tmp); - - size -= AEGIS_BLOCK_SIZE; - src += AEGIS_BLOCK_SIZE; - dst += AEGIS_BLOCK_SIZE; - } - - if (size > 0) { - uint8_t buf[AEGIS_BLOCK_SIZE] = {}; - uint8x16_t msg; - - memcpy(buf, src, size); - msg = vld1q_u8(buf) ^ st.v[1] ^ (st.v[2] & st.v[3]) ^ st.v[4]; - vst1q_u8(buf, msg); - memcpy(dst, buf, size); - - memset(buf + size, 0, AEGIS_BLOCK_SIZE - size); - msg = vld1q_u8(buf); - st = aegis128_update_neon(st, msg); - } - - aegis128_save_state_neon(st, state); -} diff --git a/crypto/aegis128-neon.c b/crypto/aegis128-neon.c deleted file mode 100644 index c1c0a1686f67..000000000000 --- a/crypto/aegis128-neon.c +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2019 Linaro Ltd <ard.biesheuvel@linaro.org> - */ - -#include <asm/cpufeature.h> -#include <asm/neon.h> - -#include "aegis.h" - -void crypto_aegis128_update_neon(void *state, const void *msg); -void crypto_aegis128_encrypt_chunk_neon(void *state, void *dst, const void *src, - unsigned int size); -void crypto_aegis128_decrypt_chunk_neon(void *state, void *dst, const void *src, - unsigned int size); - -bool crypto_aegis128_have_simd(void) -{ - return cpu_have_feature(cpu_feature(AES)); -} - -void crypto_aegis128_update_simd(union aegis_block *state, const void *msg) -{ - kernel_neon_begin(); - crypto_aegis128_update_neon(state, msg); - kernel_neon_end(); -} - -void crypto_aegis128_encrypt_chunk_simd(union aegis_block *state, u8 *dst, - const u8 *src, unsigned int size) -{ - kernel_neon_begin(); - crypto_aegis128_encrypt_chunk_neon(state, dst, src, size); - kernel_neon_end(); -} - -void crypto_aegis128_decrypt_chunk_simd(union aegis_block *state, u8 *dst, - const u8 *src, unsigned int size) -{ - kernel_neon_begin(); - crypto_aegis128_decrypt_chunk_neon(state, dst, src, size); - kernel_neon_end(); -} diff --git a/crypto/aegis128-core.c b/crypto/aegis128.c index f815b4685156..32840d5e7f65 100644 --- a/crypto/aegis128-core.c +++ b/crypto/aegis128.c @@ -8,7 +8,6 @@ #include <crypto/algapi.h> #include <crypto/internal/aead.h> -#include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> #include <linux/err.h> @@ -16,7 +15,6 @@ #include <linux/kernel.h> #include <linux/module.h> #include <linux/scatterlist.h> -#include <asm/simd.h> #include "aegis.h" @@ -42,15 +40,6 @@ struct aegis128_ops { const u8 *src, unsigned int size); }; -static bool have_simd; - -bool crypto_aegis128_have_simd(void); -void crypto_aegis128_update_simd(struct aegis_state *state, const void *msg); -void crypto_aegis128_encrypt_chunk_simd(struct aegis_state *state, u8 *dst, - const u8 *src, unsigned int size); -void crypto_aegis128_decrypt_chunk_simd(struct aegis_state *state, u8 *dst, - const u8 *src, unsigned int size); - static void crypto_aegis128_update(struct aegis_state *state) { union aegis_block tmp; @@ -66,22 +55,12 @@ static void crypto_aegis128_update(struct aegis_state *state) static void crypto_aegis128_update_a(struct aegis_state *state, const union aegis_block *msg) { - if (have_simd && crypto_simd_usable()) { - crypto_aegis128_update_simd(state, msg); - return; - } - crypto_aegis128_update(state); crypto_aegis_block_xor(&state->blocks[0], msg); } static void crypto_aegis128_update_u(struct aegis_state *state, const void *msg) { - if (have_simd && crypto_simd_usable()) { - crypto_aegis128_update_simd(state, msg); - return; - } - crypto_aegis128_update(state); crypto_xor(state->blocks[0].bytes, msg, AEGIS_BLOCK_SIZE); } @@ -386,7 +365,7 @@ static void crypto_aegis128_crypt(struct aead_request *req, static int crypto_aegis128_encrypt(struct aead_request *req) { - const struct aegis128_ops *ops = &(struct aegis128_ops){ + static const struct aegis128_ops ops = { .skcipher_walk_init = skcipher_walk_aead_encrypt, .crypt_chunk = crypto_aegis128_encrypt_chunk, }; @@ -396,12 +375,7 @@ static int crypto_aegis128_encrypt(struct aead_request *req) unsigned int authsize = crypto_aead_authsize(tfm); unsigned int cryptlen = req->cryptlen; - if (have_simd && crypto_simd_usable()) - ops = &(struct aegis128_ops){ - .skcipher_walk_init = skcipher_walk_aead_encrypt, - .crypt_chunk = crypto_aegis128_encrypt_chunk_simd }; - - crypto_aegis128_crypt(req, &tag, cryptlen, ops); + crypto_aegis128_crypt(req, &tag, cryptlen, &ops); scatterwalk_map_and_copy(tag.bytes, req->dst, req->assoclen + cryptlen, authsize, 1); @@ -410,7 +384,7 @@ static int crypto_aegis128_encrypt(struct aead_request *req) static int crypto_aegis128_decrypt(struct aead_request *req) { - const struct aegis128_ops *ops = &(struct aegis128_ops){ + static const struct aegis128_ops ops = { .skcipher_walk_init = skcipher_walk_aead_decrypt, .crypt_chunk = crypto_aegis128_decrypt_chunk, }; @@ -424,12 +398,7 @@ static int crypto_aegis128_decrypt(struct aead_request *req) scatterwalk_map_and_copy(tag.bytes, req->src, req->assoclen + cryptlen, authsize, 0); - if (have_simd && crypto_simd_usable()) - ops = &(struct aegis128_ops){ - .skcipher_walk_init = skcipher_walk_aead_decrypt, - .crypt_chunk = crypto_aegis128_decrypt_chunk_simd }; - - crypto_aegis128_crypt(req, &tag, cryptlen, ops); + crypto_aegis128_crypt(req, &tag, cryptlen, &ops); return crypto_memneq(tag.bytes, zeros, authsize) ? -EBADMSG : 0; } @@ -460,9 +429,6 @@ static struct aead_alg crypto_aegis128_alg = { static int __init crypto_aegis128_module_init(void) { - if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD)) - have_simd = crypto_aegis128_have_simd(); - return crypto_register_aead(&crypto_aegis128_alg); } |