diff options
Diffstat (limited to 'crypto')
42 files changed, 1033 insertions, 587 deletions
diff --git a/crypto/Kconfig b/crypto/Kconfig index 3d056e7da65f..e801450bcb1c 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -61,7 +61,6 @@ config CRYPTO_BLKCIPHER2 tristate select CRYPTO_ALGAPI2 select CRYPTO_RNG2 - select CRYPTO_WORKQUEUE config CRYPTO_HASH tristate @@ -137,10 +136,11 @@ config CRYPTO_USER Userspace configuration for cryptographic instantiations such as cbc(aes). +if CRYPTO_MANAGER2 + config CRYPTO_MANAGER_DISABLE_TESTS bool "Disable run-time self tests" default y - depends on CRYPTO_MANAGER2 help Disable run-time self tests that normally take place at algorithm registration. @@ -155,14 +155,10 @@ config CRYPTO_MANAGER_EXTRA_TESTS This is intended for developer use only, as these tests take much longer to run than the normal self tests. +endif # if CRYPTO_MANAGER2 + config CRYPTO_GF128MUL - tristate "GF(2^128) multiplication functions" - help - Efficient table driven implementation of multiplications in the - field GF(2^128). This is needed by some cypher modes. This - option will be selected automatically if you select such a - cipher mode. Only select this option by hand if you expect to load - an external module that requires these functions. + tristate config CRYPTO_NULL tristate "Null algorithms" @@ -186,15 +182,11 @@ config CRYPTO_PCRYPT This converts an arbitrary crypto algorithm into a parallel algorithm that executes in kernel threads. -config CRYPTO_WORKQUEUE - tristate - config CRYPTO_CRYPTD tristate "Software async crypto daemon" select CRYPTO_BLKCIPHER select CRYPTO_HASH select CRYPTO_MANAGER - select CRYPTO_WORKQUEUE help This is a generic software asynchronous crypto daemon that converts an arbitrary synchronous software crypto algorithm @@ -279,6 +271,7 @@ config CRYPTO_CCM select CRYPTO_CTR select CRYPTO_HASH select CRYPTO_AEAD + select CRYPTO_MANAGER help Support for Counter with CBC MAC. Required for IPsec. @@ -288,6 +281,7 @@ config CRYPTO_GCM select CRYPTO_AEAD select CRYPTO_GHASH select CRYPTO_NULL + select CRYPTO_MANAGER help Support for Galois/Counter Mode (GCM) and Galois Message Authentication Code (GMAC). Required for IPSec. @@ -297,6 +291,7 @@ config CRYPTO_CHACHA20POLY1305 select CRYPTO_CHACHA20 select CRYPTO_POLY1305 select CRYPTO_AEAD + select CRYPTO_MANAGER help ChaCha20-Poly1305 AEAD support, RFC7539. @@ -411,6 +406,7 @@ config CRYPTO_SEQIV select CRYPTO_BLKCIPHER select CRYPTO_NULL select CRYPTO_RNG_DEFAULT + select CRYPTO_MANAGER help This IV generator generates an IV based on a sequence number by xoring it with a salt. This algorithm is mainly useful for CTR @@ -420,7 +416,7 @@ config CRYPTO_ECHAINIV select CRYPTO_AEAD select CRYPTO_NULL select CRYPTO_RNG_DEFAULT - default m + select CRYPTO_MANAGER help This IV generator generates an IV based on the encryption of a sequence number xored with a salt. This is the default @@ -456,6 +452,7 @@ config CRYPTO_CTR config CRYPTO_CTS tristate "CTS support" select CRYPTO_BLKCIPHER + select CRYPTO_MANAGER help CTS: Cipher Text Stealing This is the Cipher Text Stealing mode as described by @@ -521,6 +518,7 @@ config CRYPTO_XTS config CRYPTO_KEYWRAP tristate "Key wrapping support" select CRYPTO_BLKCIPHER + select CRYPTO_MANAGER help Support for key wrapping (NIST SP800-38F / RFC3394) without padding. @@ -551,6 +549,7 @@ config CRYPTO_ADIANTUM select CRYPTO_CHACHA20 select CRYPTO_POLY1305 select CRYPTO_NHPOLY1305 + select CRYPTO_MANAGER help Adiantum is a tweakable, length-preserving encryption mode designed for fast and secure disk encryption, especially on @@ -684,6 +683,14 @@ config CRYPTO_CRC32_MIPS instructions, when available. +config CRYPTO_XXHASH + tristate "xxHash hash algorithm" + select CRYPTO_HASH + select XXHASH + help + xxHash non-cryptographic hash algorithm. Extremely fast, working at + speeds close to RAM limits. + config CRYPTO_CRCT10DIF tristate "CRCT10DIF algorithm" select CRYPTO_HASH @@ -1230,9 +1237,13 @@ config CRYPTO_ANUBIS <https://www.cosic.esat.kuleuven.be/nessie/reports/> <http://www.larc.usp.br/~pbarreto/AnubisPage.html> +config CRYPTO_LIB_ARC4 + tristate + config CRYPTO_ARC4 tristate "ARC4 cipher algorithm" select CRYPTO_BLKCIPHER + select CRYPTO_LIB_ARC4 help ARC4 cipher algorithm. diff --git a/crypto/Makefile b/crypto/Makefile index 266a4cdbb9e2..9479e1a45d8c 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -6,8 +6,6 @@ obj-$(CONFIG_CRYPTO) += crypto.o crypto-y := api.o cipher.o compress.o memneq.o -obj-$(CONFIG_CRYPTO_WORKQUEUE) += crypto_wq.o - obj-$(CONFIG_CRYPTO_ENGINE) += crypto_engine.o obj-$(CONFIG_CRYPTO_FIPS) += fips.o @@ -131,6 +129,7 @@ obj-$(CONFIG_CRYPTO_AUTHENC) += authenc.o authencesn.o obj-$(CONFIG_CRYPTO_LZO) += lzo.o lzo-rle.o obj-$(CONFIG_CRYPTO_LZ4) += lz4.o obj-$(CONFIG_CRYPTO_LZ4HC) += lz4hc.o +obj-$(CONFIG_CRYPTO_XXHASH) += xxhash_generic.o obj-$(CONFIG_CRYPTO_842) += 842.o obj-$(CONFIG_CRYPTO_RNG2) += rng.o obj-$(CONFIG_CRYPTO_ANSI_CPRNG) += ansi_cprng.o diff --git a/crypto/aead.c b/crypto/aead.c index c3c158ba9883..fbf0ec93bc8e 100644 --- a/crypto/aead.c +++ b/crypto/aead.c @@ -84,6 +84,42 @@ int crypto_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) } EXPORT_SYMBOL_GPL(crypto_aead_setauthsize); +int crypto_aead_encrypt(struct aead_request *req) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct crypto_alg *alg = aead->base.__crt_alg; + unsigned int cryptlen = req->cryptlen; + int ret; + + crypto_stats_get(alg); + if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY) + ret = -ENOKEY; + else + ret = crypto_aead_alg(aead)->encrypt(req); + crypto_stats_aead_encrypt(cryptlen, alg, ret); + return ret; +} +EXPORT_SYMBOL_GPL(crypto_aead_encrypt); + +int crypto_aead_decrypt(struct aead_request *req) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct crypto_alg *alg = aead->base.__crt_alg; + unsigned int cryptlen = req->cryptlen; + int ret; + + crypto_stats_get(alg); + if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY) + ret = -ENOKEY; + else if (req->cryptlen < crypto_aead_authsize(aead)) + ret = -EINVAL; + else + ret = crypto_aead_alg(aead)->decrypt(req); + crypto_stats_aead_decrypt(cryptlen, alg, ret); + return ret; +} +EXPORT_SYMBOL_GPL(crypto_aead_decrypt); + static void crypto_aead_exit_tfm(struct crypto_tfm *tfm) { struct crypto_aead *aead = __crypto_aead_cast(tfm); diff --git a/crypto/algapi.c b/crypto/algapi.c index 313a7682cef1..de30ddc952d8 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -21,23 +21,6 @@ static LIST_HEAD(crypto_template_list); -static inline int crypto_set_driver_name(struct crypto_alg *alg) -{ - static const char suffix[] = "-generic"; - char *driver_name = alg->cra_driver_name; - int len; - - if (*driver_name) - return 0; - - len = strlcpy(driver_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); - if (len + sizeof(suffix) > CRYPTO_MAX_ALG_NAME) - return -ENAMETOOLONG; - - memcpy(driver_name + len, suffix, sizeof(suffix)); - return 0; -} - static inline void crypto_check_module_sig(struct module *mod) { if (fips_enabled && mod && !module_sig_ok(mod)) @@ -49,6 +32,9 @@ static int crypto_check_alg(struct crypto_alg *alg) { crypto_check_module_sig(alg->cra_module); + if (!alg->cra_name[0] || !alg->cra_driver_name[0]) + return -EINVAL; + if (alg->cra_alignmask & (alg->cra_alignmask + 1)) return -EINVAL; @@ -74,7 +60,7 @@ static int crypto_check_alg(struct crypto_alg *alg) refcount_set(&alg->cra_refcnt, 1); - return crypto_set_driver_name(alg); + return 0; } static void crypto_free_instance(struct crypto_instance *inst) @@ -947,19 +933,6 @@ struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue) } EXPORT_SYMBOL_GPL(crypto_dequeue_request); -int crypto_tfm_in_queue(struct crypto_queue *queue, struct crypto_tfm *tfm) -{ - struct crypto_async_request *req; - - list_for_each_entry(req, &queue->list, list) { - if (req->tfm == tfm) - return 1; - } - - return 0; -} -EXPORT_SYMBOL_GPL(crypto_tfm_in_queue); - static inline void crypto_inc_byte(u8 *a, unsigned int size) { u8 *b = (a + size); diff --git a/crypto/anubis.c b/crypto/anubis.c index 673927de0eb9..f9ce78fde6ee 100644 --- a/crypto/anubis.c +++ b/crypto/anubis.c @@ -673,6 +673,7 @@ static void anubis_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) static struct crypto_alg anubis_alg = { .cra_name = "anubis", + .cra_driver_name = "anubis-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = ANUBIS_BLOCK_SIZE, .cra_ctxsize = sizeof (struct anubis_ctx), diff --git a/crypto/arc4.c b/crypto/arc4.c index a2120e06bf84..aa79571dbd49 100644 --- a/crypto/arc4.c +++ b/crypto/arc4.c @@ -13,84 +13,15 @@ #include <linux/init.h> #include <linux/module.h> -struct arc4_ctx { - u32 S[256]; - u32 x, y; -}; - -static int arc4_set_key(struct crypto_tfm *tfm, const u8 *in_key, - unsigned int key_len) -{ - struct arc4_ctx *ctx = crypto_tfm_ctx(tfm); - int i, j = 0, k = 0; - - ctx->x = 1; - ctx->y = 0; - - for (i = 0; i < 256; i++) - ctx->S[i] = i; - - for (i = 0; i < 256; i++) { - u32 a = ctx->S[i]; - j = (j + in_key[k] + a) & 0xff; - ctx->S[i] = ctx->S[j]; - ctx->S[j] = a; - if (++k >= key_len) - k = 0; - } - - return 0; -} - -static int arc4_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key, - unsigned int key_len) +static int crypto_arc4_setkey(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) { - return arc4_set_key(&tfm->base, in_key, key_len); -} - -static void arc4_crypt(struct arc4_ctx *ctx, u8 *out, const u8 *in, - unsigned int len) -{ - u32 *const S = ctx->S; - u32 x, y, a, b; - u32 ty, ta, tb; - - if (len == 0) - return; - - x = ctx->x; - y = ctx->y; - - a = S[x]; - y = (y + a) & 0xff; - b = S[y]; - - do { - S[y] = a; - a = (a + b) & 0xff; - S[x] = b; - x = (x + 1) & 0xff; - ta = S[x]; - ty = (y + ta) & 0xff; - tb = S[ty]; - *out++ = *in++ ^ S[a]; - if (--len == 0) - break; - y = ty; - a = ta; - b = tb; - } while (true); - - ctx->x = x; - ctx->y = y; -} + struct arc4_ctx *ctx = crypto_skcipher_ctx(tfm); -static void arc4_crypt_one(struct crypto_tfm *tfm, u8 *out, const u8 *in) -{ - arc4_crypt(crypto_tfm_ctx(tfm), out, in, 1); + return arc4_setkey(ctx, in_key, key_len); } -static int ecb_arc4_crypt(struct skcipher_request *req) +static int crypto_arc4_crypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct arc4_ctx *ctx = crypto_skcipher_ctx(tfm); @@ -108,54 +39,32 @@ static int ecb_arc4_crypt(struct skcipher_request *req) return err; } -static struct crypto_alg arc4_cipher = { - .cra_name = "arc4", - .cra_flags = CRYPTO_ALG_TYPE_CIPHER, - .cra_blocksize = ARC4_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct arc4_ctx), - .cra_module = THIS_MODULE, - .cra_u = { - .cipher = { - .cia_min_keysize = ARC4_MIN_KEY_SIZE, - .cia_max_keysize = ARC4_MAX_KEY_SIZE, - .cia_setkey = arc4_set_key, - .cia_encrypt = arc4_crypt_one, - .cia_decrypt = arc4_crypt_one, - }, - }, -}; - -static struct skcipher_alg arc4_skcipher = { +static struct skcipher_alg arc4_alg = { + /* + * For legacy reasons, this is named "ecb(arc4)", not "arc4". + * Nevertheless it's actually a stream cipher, not a block cipher. + */ .base.cra_name = "ecb(arc4)", + .base.cra_driver_name = "ecb(arc4)-generic", .base.cra_priority = 100, .base.cra_blocksize = ARC4_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct arc4_ctx), .base.cra_module = THIS_MODULE, .min_keysize = ARC4_MIN_KEY_SIZE, .max_keysize = ARC4_MAX_KEY_SIZE, - .setkey = arc4_set_key_skcipher, - .encrypt = ecb_arc4_crypt, - .decrypt = ecb_arc4_crypt, + .setkey = crypto_arc4_setkey, + .encrypt = crypto_arc4_crypt, + .decrypt = crypto_arc4_crypt, }; static int __init arc4_init(void) { - int err; - - err = crypto_register_alg(&arc4_cipher); - if (err) - return err; - - err = crypto_register_skcipher(&arc4_skcipher); - if (err) - crypto_unregister_alg(&arc4_cipher); - return err; + return crypto_register_skcipher(&arc4_alg); } static void __exit arc4_exit(void) { - crypto_unregister_alg(&arc4_cipher); - crypto_unregister_skcipher(&arc4_skcipher); + crypto_unregister_skcipher(&arc4_alg); } subsys_initcall(arc4_init); @@ -164,4 +73,4 @@ module_exit(arc4_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("ARC4 Cipher Algorithm"); MODULE_AUTHOR("Jon Oberheide <jon@oberheide.org>"); -MODULE_ALIAS_CRYPTO("arc4"); +MODULE_ALIAS_CRYPTO("ecb(arc4)"); diff --git a/crypto/asymmetric_keys/Kconfig b/crypto/asymmetric_keys/Kconfig index be70ca6c85d3..1f1f004dc757 100644 --- a/crypto/asymmetric_keys/Kconfig +++ b/crypto/asymmetric_keys/Kconfig @@ -15,6 +15,7 @@ config ASYMMETRIC_PUBLIC_KEY_SUBTYPE select MPILIB select CRYPTO_HASH_INFO select CRYPTO_AKCIPHER + select CRYPTO_HASH help This option provides support for asymmetric public key type handling. If signature generation and/or verification are to be used, @@ -65,6 +66,7 @@ config TPM_KEY_PARSER config PKCS7_MESSAGE_PARSER tristate "PKCS#7 message parser" depends on X509_CERTIFICATE_PARSER + select CRYPTO_HASH select ASN1 select OID_REGISTRY help @@ -87,6 +89,7 @@ config SIGNED_PE_FILE_VERIFICATION bool "Support for PE file signature verification" depends on PKCS7_MESSAGE_PARSER=y depends on SYSTEM_DATA_VERIFICATION + select CRYPTO_HASH select ASN1 select OID_REGISTRY help diff --git a/crypto/asymmetric_keys/asymmetric_type.c b/crypto/asymmetric_keys/asymmetric_type.c index 01945ab46382..6e5fc8e31f01 100644 --- a/crypto/asymmetric_keys/asymmetric_type.c +++ b/crypto/asymmetric_keys/asymmetric_type.c @@ -83,7 +83,7 @@ struct key *find_asymmetric_key(struct key *keyring, pr_debug("Look up: \"%s\"\n", req); ref = keyring_search(make_key_ref(keyring, 1), - &key_type_asymmetric, req); + &key_type_asymmetric, req, true); if (IS_ERR(ref)) pr_debug("Request for key '%s' err %ld\n", req, PTR_ERR(ref)); kfree(req); diff --git a/crypto/chacha20poly1305.c b/crypto/chacha20poly1305.c index 2db7eac4bf3b..74e824e537e6 100644 --- a/crypto/chacha20poly1305.c +++ b/crypto/chacha20poly1305.c @@ -61,6 +61,8 @@ struct chachapoly_req_ctx { unsigned int cryptlen; /* Actual AD, excluding IV */ unsigned int assoclen; + /* request flags, with MAY_SLEEP cleared if needed */ + u32 flags; union { struct poly_req poly; struct chacha_req chacha; @@ -70,8 +72,12 @@ struct chachapoly_req_ctx { static inline void async_done_continue(struct aead_request *req, int err, int (*cont)(struct aead_request *)) { - if (!err) + if (!err) { + struct chachapoly_req_ctx *rctx = aead_request_ctx(req); + + rctx->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; err = cont(req); + } if (err != -EINPROGRESS && err != -EBUSY) aead_request_complete(req, err); @@ -129,16 +135,12 @@ static int chacha_decrypt(struct aead_request *req) chacha_iv(creq->iv, req, 1); - sg_init_table(rctx->src, 2); src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen); dst = src; - - if (req->src != req->dst) { - sg_init_table(rctx->dst, 2); + if (req->src != req->dst) dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen); - } - skcipher_request_set_callback(&creq->req, aead_request_flags(req), + skcipher_request_set_callback(&creq->req, rctx->flags, chacha_decrypt_done, req); skcipher_request_set_tfm(&creq->req, ctx->chacha); skcipher_request_set_crypt(&creq->req, src, dst, @@ -172,17 +174,13 @@ static int poly_tail(struct aead_request *req) struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm); struct chachapoly_req_ctx *rctx = aead_request_ctx(req); struct poly_req *preq = &rctx->u.poly; - __le64 len; int err; - sg_init_table(preq->src, 1); - len = cpu_to_le64(rctx->assoclen); - memcpy(&preq->tail.assoclen, &len, sizeof(len)); - len = cpu_to_le64(rctx->cryptlen); - memcpy(&preq->tail.cryptlen, &len, sizeof(len)); - sg_set_buf(preq->src, &preq->tail, sizeof(preq->tail)); + preq->tail.assoclen = cpu_to_le64(rctx->assoclen); + preq->tail.cryptlen = cpu_to_le64(rctx->cryptlen); + sg_init_one(preq->src, &preq->tail, sizeof(preq->tail)); - ahash_request_set_callback(&preq->req, aead_request_flags(req), + ahash_request_set_callback(&preq->req, rctx->flags, poly_tail_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); ahash_request_set_crypt(&preq->req, preq->src, @@ -205,15 +203,14 @@ static int poly_cipherpad(struct aead_request *req) struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); struct chachapoly_req_ctx *rctx = aead_request_ctx(req); struct poly_req *preq = &rctx->u.poly; - unsigned int padlen, bs = POLY1305_BLOCK_SIZE; + unsigned int padlen; int err; - padlen = (bs - (rctx->cryptlen % bs)) % bs; + padlen = -rctx->cryptlen % POLY1305_BLOCK_SIZE; memset(preq->pad, 0, sizeof(preq->pad)); - sg_init_table(preq->src, 1); - sg_set_buf(preq->src, &preq->pad, padlen); + sg_init_one(preq->src, preq->pad, padlen); - ahash_request_set_callback(&preq->req, aead_request_flags(req), + ahash_request_set_callback(&preq->req, rctx->flags, poly_cipherpad_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen); @@ -241,10 +238,9 @@ static int poly_cipher(struct aead_request *req) if (rctx->cryptlen == req->cryptlen) /* encrypting */ crypt = req->dst; - sg_init_table(rctx->src, 2); crypt = scatterwalk_ffwd(rctx->src, crypt, req->assoclen); - ahash_request_set_callback(&preq->req, aead_request_flags(req), + ahash_request_set_callback(&preq->req, rctx->flags, poly_cipher_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); ahash_request_set_crypt(&preq->req, crypt, NULL, rctx->cryptlen); @@ -266,15 +262,14 @@ static int poly_adpad(struct aead_request *req) struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); struct chachapoly_req_ctx *rctx = aead_request_ctx(req); struct poly_req *preq = &rctx->u.poly; - unsigned int padlen, bs = POLY1305_BLOCK_SIZE; + unsigned int padlen; int err; - padlen = (bs - (rctx->assoclen % bs)) % bs; + padlen = -rctx->assoclen % POLY1305_BLOCK_SIZE; memset(preq->pad, 0, sizeof(preq->pad)); - sg_init_table(preq->src, 1); - sg_set_buf(preq->src, preq->pad, padlen); + sg_init_one(preq->src, preq->pad, padlen); - ahash_request_set_callback(&preq->req, aead_request_flags(req), + ahash_request_set_callback(&preq->req, rctx->flags, poly_adpad_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen); @@ -298,7 +293,7 @@ static int poly_ad(struct aead_request *req) struct poly_req *preq = &rctx->u.poly; int err; - ahash_request_set_callback(&preq->req, aead_request_flags(req), + ahash_request_set_callback(&preq->req, rctx->flags, poly_ad_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); ahash_request_set_crypt(&preq->req, req->src, NULL, rctx->assoclen); @@ -322,10 +317,9 @@ static int poly_setkey(struct aead_request *req) struct poly_req *preq = &rctx->u.poly; int err; - sg_init_table(preq->src, 1); - sg_set_buf(preq->src, rctx->key, sizeof(rctx->key)); + sg_init_one(preq->src, rctx->key, sizeof(rctx->key)); - ahash_request_set_callback(&preq->req, aead_request_flags(req), + ahash_request_set_callback(&preq->req, rctx->flags, poly_setkey_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); ahash_request_set_crypt(&preq->req, preq->src, NULL, sizeof(rctx->key)); @@ -349,7 +343,7 @@ static int poly_init(struct aead_request *req) struct poly_req *preq = &rctx->u.poly; int err; - ahash_request_set_callback(&preq->req, aead_request_flags(req), + ahash_request_set_callback(&preq->req, rctx->flags, poly_init_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); @@ -381,13 +375,12 @@ static int poly_genkey(struct aead_request *req) rctx->assoclen -= 8; } - sg_init_table(creq->src, 1); memset(rctx->key, 0, sizeof(rctx->key)); - sg_set_buf(creq->src, rctx->key, sizeof(rctx->key)); + sg_init_one(creq->src, rctx->key, sizeof(rctx->key)); chacha_iv(creq->iv, req, 0); - skcipher_request_set_callback(&creq->req, aead_request_flags(req), + skcipher_request_set_callback(&creq->req, rctx->flags, poly_genkey_done, req); skcipher_request_set_tfm(&creq->req, ctx->chacha); skcipher_request_set_crypt(&creq->req, creq->src, creq->src, @@ -418,16 +411,12 @@ static int chacha_encrypt(struct aead_request *req) chacha_iv(creq->iv, req, 1); - sg_init_table(rctx->src, 2); src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen); dst = src; - - if (req->src != req->dst) { - sg_init_table(rctx->dst, 2); + if (req->src != req->dst) dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen); - } - skcipher_request_set_callback(&creq->req, aead_request_flags(req), + skcipher_request_set_callback(&creq->req, rctx->flags, chacha_encrypt_done, req); skcipher_request_set_tfm(&creq->req, ctx->chacha); skcipher_request_set_crypt(&creq->req, src, dst, @@ -445,6 +434,7 @@ static int chachapoly_encrypt(struct aead_request *req) struct chachapoly_req_ctx *rctx = aead_request_ctx(req); rctx->cryptlen = req->cryptlen; + rctx->flags = aead_request_flags(req); /* encrypt call chain: * - chacha_encrypt/done() @@ -466,6 +456,7 @@ static int chachapoly_decrypt(struct aead_request *req) struct chachapoly_req_ctx *rctx = aead_request_ctx(req); rctx->cryptlen = req->cryptlen - POLY1305_DIGEST_SIZE; + rctx->flags = aead_request_flags(req); /* decrypt call chain: * - poly_genkey/done() diff --git a/crypto/chacha_generic.c b/crypto/chacha_generic.c index 04404c479e68..085d8d219987 100644 --- a/crypto/chacha_generic.c +++ b/crypto/chacha_generic.c @@ -32,7 +32,7 @@ static void chacha_docrypt(u32 *state, u8 *dst, const u8 *src, } static int chacha_stream_xor(struct skcipher_request *req, - struct chacha_ctx *ctx, u8 *iv) + const struct chacha_ctx *ctx, const u8 *iv) { struct skcipher_walk walk; u32 state[16]; @@ -56,7 +56,7 @@ static int chacha_stream_xor(struct skcipher_request *req, return err; } -void crypto_chacha_init(u32 *state, struct chacha_ctx *ctx, u8 *iv) +void crypto_chacha_init(u32 *state, const struct chacha_ctx *ctx, const u8 *iv) { state[0] = 0x61707865; /* "expa" */ state[1] = 0x3320646e; /* "nd 3" */ diff --git a/crypto/cryptd.c b/crypto/cryptd.c index 1ce1bf6d3bab..3748f9b4516d 100644 --- a/crypto/cryptd.c +++ b/crypto/cryptd.c @@ -16,7 +16,6 @@ #include <crypto/internal/aead.h> #include <crypto/internal/skcipher.h> #include <crypto/cryptd.h> -#include <crypto/crypto_wq.h> #include <linux/atomic.h> #include <linux/err.h> #include <linux/init.h> @@ -26,11 +25,14 @@ #include <linux/scatterlist.h> #include <linux/sched.h> #include <linux/slab.h> +#include <linux/workqueue.h> static unsigned int cryptd_max_cpu_qlen = 1000; module_param(cryptd_max_cpu_qlen, uint, 0); MODULE_PARM_DESC(cryptd_max_cpu_qlen, "Set cryptd Max queue depth"); +static struct workqueue_struct *cryptd_wq; + struct cryptd_cpu_queue { struct crypto_queue queue; struct work_struct work; @@ -136,7 +138,7 @@ static int cryptd_enqueue_request(struct cryptd_queue *queue, if (err == -ENOSPC) goto out_put_cpu; - queue_work_on(cpu, kcrypto_wq, &cpu_queue->work); + queue_work_on(cpu, cryptd_wq, &cpu_queue->work); if (!atomic_read(refcnt)) goto out_put_cpu; @@ -179,7 +181,7 @@ static void cryptd_queue_worker(struct work_struct *work) req->complete(req, 0); if (cpu_queue->queue.qlen) - queue_work(kcrypto_wq, &cpu_queue->work); + queue_work(cryptd_wq, &cpu_queue->work); } static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm) @@ -388,6 +390,7 @@ static void cryptd_skcipher_free(struct skcipher_instance *inst) struct skcipherd_instance_ctx *ctx = skcipher_instance_ctx(inst); crypto_drop_skcipher(&ctx->spawn); + kfree(inst); } static int cryptd_create_skcipher(struct crypto_template *tmpl, @@ -918,7 +921,7 @@ static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb) switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { case CRYPTO_ALG_TYPE_BLKCIPHER: return cryptd_create_skcipher(tmpl, tb, &queue); - case CRYPTO_ALG_TYPE_DIGEST: + case CRYPTO_ALG_TYPE_HASH: return cryptd_create_hash(tmpl, tb, &queue); case CRYPTO_ALG_TYPE_AEAD: return cryptd_create_aead(tmpl, tb, &queue); @@ -1118,19 +1121,31 @@ static int __init cryptd_init(void) { int err; + cryptd_wq = alloc_workqueue("cryptd", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, + 1); + if (!cryptd_wq) + return -ENOMEM; + err = cryptd_init_queue(&queue, cryptd_max_cpu_qlen); if (err) - return err; + goto err_destroy_wq; err = crypto_register_template(&cryptd_tmpl); if (err) - cryptd_fini_queue(&queue); + goto err_fini_queue; + return 0; + +err_fini_queue: + cryptd_fini_queue(&queue); +err_destroy_wq: + destroy_workqueue(cryptd_wq); return err; } static void __exit cryptd_exit(void) { + destroy_workqueue(cryptd_wq); cryptd_fini_queue(&queue); crypto_unregister_template(&cryptd_tmpl); } diff --git a/crypto/crypto_null.c b/crypto/crypto_null.c index 0d341ddecd54..5b84b0f7cc17 100644 --- a/crypto/crypto_null.c +++ b/crypto/crypto_null.c @@ -100,6 +100,7 @@ static struct shash_alg digest_null = { .final = null_final, .base = { .cra_name = "digest_null", + .cra_driver_name = "digest_null-generic", .cra_blocksize = NULL_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -122,6 +123,7 @@ static struct skcipher_alg skcipher_null = { static struct crypto_alg null_algs[] = { { .cra_name = "cipher_null", + .cra_driver_name = "cipher_null-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = NULL_BLOCK_SIZE, .cra_ctxsize = 0, @@ -134,6 +136,7 @@ static struct crypto_alg null_algs[] = { { .cia_decrypt = null_crypt } } }, { .cra_name = "compress_null", + .cra_driver_name = "compress_null-generic", .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, .cra_blocksize = NULL_BLOCK_SIZE, .cra_ctxsize = 0, diff --git a/crypto/crypto_user_base.c b/crypto/crypto_user_base.c index d5d5d155340b..c65e39005ce2 100644 --- a/crypto/crypto_user_base.c +++ b/crypto/crypto_user_base.c @@ -44,6 +44,9 @@ struct crypto_alg *crypto_alg_match(struct crypto_user_alg *p, int exact) list_for_each_entry(q, &crypto_alg_list, cra_list) { int match = 0; + if (crypto_is_larval(q)) + continue; + if ((q->cra_flags ^ p->cru_type) & p->cru_mask) continue; diff --git a/crypto/crypto_wq.c b/crypto/crypto_wq.c deleted file mode 100644 index 80501928e0bb..000000000000 --- a/crypto/crypto_wq.c +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Workqueue for crypto subsystem - * - * Copyright (c) 2009 Intel Corp. - * Author: Huang Ying <ying.huang@intel.com> - */ - -#include <linux/workqueue.h> -#include <linux/module.h> -#include <crypto/algapi.h> -#include <crypto/crypto_wq.h> - -struct workqueue_struct *kcrypto_wq; -EXPORT_SYMBOL_GPL(kcrypto_wq); - -static int __init crypto_wq_init(void) -{ - kcrypto_wq = alloc_workqueue("crypto", - WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, 1); - if (unlikely(!kcrypto_wq)) - return -ENOMEM; - return 0; -} - -static void __exit crypto_wq_exit(void) -{ - destroy_workqueue(kcrypto_wq); -} - -subsys_initcall(crypto_wq_init); -module_exit(crypto_wq_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Workqueue for crypto subsystem"); diff --git a/crypto/deflate.c b/crypto/deflate.c index 65be51456398..4c0e6c9d942a 100644 --- a/crypto/deflate.c +++ b/crypto/deflate.c @@ -275,6 +275,7 @@ static int deflate_sdecompress(struct crypto_scomp *tfm, const u8 *src, static struct crypto_alg alg = { .cra_name = "deflate", + .cra_driver_name = "deflate-generic", .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, .cra_ctxsize = sizeof(struct deflate_ctx), .cra_module = THIS_MODULE, diff --git a/crypto/drbg.c b/crypto/drbg.c index 2a5b16bb000c..b6929eb5f565 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -220,6 +220,57 @@ static inline unsigned short drbg_sec_strength(drbg_flag_t flags) } /* + * FIPS 140-2 continuous self test for the noise source + * The test is performed on the noise source input data. Thus, the function + * implicitly knows the size of the buffer to be equal to the security + * strength. + * + * Note, this function disregards the nonce trailing the entropy data during + * initial seeding. + * + * drbg->drbg_mutex must have been taken. + * + * @drbg DRBG handle + * @entropy buffer of seed data to be checked + * + * return: + * 0 on success + * -EAGAIN on when the CTRNG is not yet primed + * < 0 on error + */ +static int drbg_fips_continuous_test(struct drbg_state *drbg, + const unsigned char *entropy) +{ + unsigned short entropylen = drbg_sec_strength(drbg->core->flags); + int ret = 0; + + if (!IS_ENABLED(CONFIG_CRYPTO_FIPS)) + return 0; + + /* skip test if we test the overall system */ + if (list_empty(&drbg->test_data.list)) + return 0; + /* only perform test in FIPS mode */ + if (!fips_enabled) + return 0; + + if (!drbg->fips_primed) { + /* Priming of FIPS test */ + memcpy(drbg->prev, entropy, entropylen); + drbg->fips_primed = true; + /* priming: another round is needed */ + return -EAGAIN; + } + ret = memcmp(drbg->prev, entropy, entropylen); + if (!ret) + panic("DRBG continuous self test failed\n"); + memcpy(drbg->prev, entropy, entropylen); + + /* the test shall pass when the two values are not equal */ + return 0; +} + +/* * Convert an integer into a byte representation of this integer. * The byte representation is big-endian * @@ -998,6 +1049,22 @@ static inline int __drbg_seed(struct drbg_state *drbg, struct list_head *seed, return ret; } +static inline int drbg_get_random_bytes(struct drbg_state *drbg, + unsigned char *entropy, + unsigned int entropylen) +{ + int ret; + + do { + get_random_bytes(entropy, entropylen); + ret = drbg_fips_continuous_test(drbg, entropy); + if (ret && ret != -EAGAIN) + return ret; + } while (ret); + + return 0; +} + static void drbg_async_seed(struct work_struct *work) { struct drbg_string data; @@ -1006,16 +1073,20 @@ static void drbg_async_seed(struct work_struct *work) seed_work); unsigned int entropylen = drbg_sec_strength(drbg->core->flags); unsigned char entropy[32]; + int ret; BUG_ON(!entropylen); BUG_ON(entropylen > sizeof(entropy)); - get_random_bytes(entropy, entropylen); drbg_string_fill(&data, entropy, entropylen); list_add_tail(&data.list, &seedlist); mutex_lock(&drbg->drbg_mutex); + ret = drbg_get_random_bytes(drbg, entropy, entropylen); + if (ret) + goto unlock; + /* If nonblocking pool is initialized, deactivate Jitter RNG */ crypto_free_rng(drbg->jent); drbg->jent = NULL; @@ -1030,6 +1101,7 @@ static void drbg_async_seed(struct work_struct *work) if (drbg->seeded) drbg->reseed_threshold = drbg_max_requests(drbg); +unlock: mutex_unlock(&drbg->drbg_mutex); memzero_explicit(entropy, entropylen); @@ -1081,7 +1153,9 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, BUG_ON((entropylen * 2) > sizeof(entropy)); /* Get seed from in-kernel /dev/urandom */ - get_random_bytes(entropy, entropylen); + ret = drbg_get_random_bytes(drbg, entropy, entropylen); + if (ret) + goto out; if (!drbg->jent) { drbg_string_fill(&data1, entropy, entropylen); @@ -1094,7 +1168,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, entropylen); if (ret) { pr_devel("DRBG: jent failed with %d\n", ret); - return ret; + goto out; } drbg_string_fill(&data1, entropy, entropylen * 2); @@ -1121,6 +1195,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, ret = __drbg_seed(drbg, &seedlist, reseed); +out: memzero_explicit(entropy, entropylen * 2); return ret; @@ -1142,6 +1217,11 @@ static inline void drbg_dealloc_state(struct drbg_state *drbg) drbg->reseed_ctr = 0; drbg->d_ops = NULL; drbg->core = NULL; + if (IS_ENABLED(CONFIG_CRYPTO_FIPS)) { + kzfree(drbg->prev); + drbg->prev = NULL; + drbg->fips_primed = false; + } } /* @@ -1211,6 +1291,14 @@ static inline int drbg_alloc_state(struct drbg_state *drbg) drbg->scratchpad = PTR_ALIGN(drbg->scratchpadbuf, ret + 1); } + if (IS_ENABLED(CONFIG_CRYPTO_FIPS)) { + drbg->prev = kzalloc(drbg_sec_strength(drbg->core->flags), + GFP_KERNEL); + if (!drbg->prev) + goto fini; + drbg->fips_primed = false; + } + return 0; fini: diff --git a/crypto/fcrypt.c b/crypto/fcrypt.c index 4e8704405a3b..58f935315cf8 100644 --- a/crypto/fcrypt.c +++ b/crypto/fcrypt.c @@ -391,6 +391,7 @@ static int fcrypt_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int key static struct crypto_alg fcrypt_alg = { .cra_name = "fcrypt", + .cra_driver_name = "fcrypt-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = 8, .cra_ctxsize = sizeof(struct fcrypt_ctx), diff --git a/crypto/ghash-generic.c b/crypto/ghash-generic.c index 6425b9cd718e..dad9e1f91a78 100644 --- a/crypto/ghash-generic.c +++ b/crypto/ghash-generic.c @@ -31,6 +31,7 @@ static int ghash_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen) { struct ghash_ctx *ctx = crypto_shash_ctx(tfm); + be128 k; if (keylen != GHASH_BLOCK_SIZE) { crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); @@ -39,7 +40,12 @@ static int ghash_setkey(struct crypto_shash *tfm, if (ctx->gf128) gf128mul_free_4k(ctx->gf128); - ctx->gf128 = gf128mul_init_4k_lle((be128 *)key); + + BUILD_BUG_ON(sizeof(k) != GHASH_BLOCK_SIZE); + memcpy(&k, key, GHASH_BLOCK_SIZE); /* avoid violating alignment rules */ + ctx->gf128 = gf128mul_init_4k_lle(&k); + memzero_explicit(&k, GHASH_BLOCK_SIZE); + if (!ctx->gf128) return -ENOMEM; diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c index 787dccca3715..701b8d86ab49 100644 --- a/crypto/jitterentropy-kcapi.c +++ b/crypto/jitterentropy-kcapi.c @@ -56,11 +56,6 @@ void jent_entropy_collector_free(struct rand_data *entropy_collector); * Helper function ***************************************************************************/ -__u64 jent_rol64(__u64 word, unsigned int shift) -{ - return rol64(word, shift); -} - void *jent_zalloc(unsigned int len) { return kzalloc(len, GFP_KERNEL); diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c index acf44b2d2d1d..77fa2120fe0c 100644 --- a/crypto/jitterentropy.c +++ b/crypto/jitterentropy.c @@ -2,7 +2,7 @@ * Non-physical true random number generator based on timing jitter -- * Jitter RNG standalone code. * - * Copyright Stephan Mueller <smueller@chronox.de>, 2015 + * Copyright Stephan Mueller <smueller@chronox.de>, 2015 - 2019 * * Design * ====== @@ -47,7 +47,7 @@ /* * This Jitterentropy RNG is based on the jitterentropy library - * version 1.1.0 provided at http://www.chronox.de/jent.html + * version 2.1.2 provided at http://www.chronox.de/jent.html */ #ifdef __OPTIMIZE__ @@ -71,10 +71,7 @@ struct rand_data { #define DATA_SIZE_BITS ((sizeof(__u64)) * 8) __u64 last_delta; /* SENSITIVE stuck test */ __s64 last_delta2; /* SENSITIVE stuck test */ - unsigned int stuck:1; /* Time measurement stuck */ unsigned int osr; /* Oversample rate */ - unsigned int stir:1; /* Post-processing stirring */ - unsigned int disable_unbias:1; /* Deactivate Von-Neuman unbias */ #define JENT_MEMORY_BLOCKS 64 #define JENT_MEMORY_BLOCKSIZE 32 #define JENT_MEMORY_ACCESSLOOPS 128 @@ -89,8 +86,6 @@ struct rand_data { }; /* Flags that can be used to initialize the RNG */ -#define JENT_DISABLE_STIR (1<<0) /* Disable stirring the entropy pool */ -#define JENT_DISABLE_UNBIAS (1<<1) /* Disable the Von-Neuman Unbiaser */ #define JENT_DISABLE_MEMORY_ACCESS (1<<2) /* Disable memory access for more * entropy, saves MEMORY_SIZE RAM for * entropy collector */ @@ -99,19 +94,16 @@ struct rand_data { #define JENT_ENOTIME 1 /* Timer service not available */ #define JENT_ECOARSETIME 2 /* Timer too coarse for RNG */ #define JENT_ENOMONOTONIC 3 /* Timer is not monotonic increasing */ -#define JENT_EMINVARIATION 4 /* Timer variations too small for RNG */ #define JENT_EVARVAR 5 /* Timer does not produce variations of * variations (2nd derivation of time is * zero). */ -#define JENT_EMINVARVAR 6 /* Timer variations of variations is tooi - * small. */ +#define JENT_ESTUCK 8 /* Too many stuck results during init. */ /*************************************************************************** * Helper functions ***************************************************************************/ void jent_get_nstime(__u64 *out); -__u64 jent_rol64(__u64 word, unsigned int shift); void *jent_zalloc(unsigned int len); void jent_zfree(void *ptr); int jent_fips_enabled(void); @@ -140,16 +132,16 @@ static __u64 jent_loop_shuffle(struct rand_data *ec, jent_get_nstime(&time); /* - * mix the current state of the random number into the shuffle - * calculation to balance that shuffle a bit more + * Mix the current state of the random number into the shuffle + * calculation to balance that shuffle a bit more. */ if (ec) time ^= ec->data; /* - * we fold the time value as much as possible to ensure that as many - * bits of the time stamp are included as possible + * We fold the time value as much as possible to ensure that as many + * bits of the time stamp are included as possible. */ - for (i = 0; (DATA_SIZE_BITS / bits) > i; i++) { + for (i = 0; ((DATA_SIZE_BITS + bits - 1) / bits) > i; i++) { shuffle ^= time & mask; time = time >> bits; } @@ -169,38 +161,28 @@ static __u64 jent_loop_shuffle(struct rand_data *ec, * CPU Jitter noise source -- this is the noise source based on the CPU * execution time jitter * - * This function folds the time into one bit units by iterating - * through the DATA_SIZE_BITS bit time value as follows: assume our time value - * is 0xabcd - * 1st loop, 1st shift generates 0xd000 - * 1st loop, 2nd shift generates 0x000d - * 2nd loop, 1st shift generates 0xcd00 - * 2nd loop, 2nd shift generates 0x000c - * 3rd loop, 1st shift generates 0xbcd0 - * 3rd loop, 2nd shift generates 0x000b - * 4th loop, 1st shift generates 0xabcd - * 4th loop, 2nd shift generates 0x000a - * Now, the values at the end of the 2nd shifts are XORed together. + * This function injects the individual bits of the time value into the + * entropy pool using an LFSR. * - * The code is deliberately inefficient and shall stay that way. This function - * is the root cause why the code shall be compiled without optimization. This - * function not only acts as folding operation, but this function's execution - * is used to measure the CPU execution time jitter. Any change to the loop in - * this function implies that careful retesting must be done. + * The code is deliberately inefficient with respect to the bit shifting + * and shall stay that way. This function is the root cause why the code + * shall be compiled without optimization. This function not only acts as + * folding operation, but this function's execution is used to measure + * the CPU execution time jitter. Any change to the loop in this function + * implies that careful retesting must be done. * * Input: * @ec entropy collector struct -- may be NULL - * @time time stamp to be folded + * @time time stamp to be injected * @loop_cnt if a value not equal to 0 is set, use the given value as number of * loops to perform the folding * * Output: - * @folded result of folding operation + * updated ec->data * * @return Number of loops the folding operation is performed */ -static __u64 jent_fold_time(struct rand_data *ec, __u64 time, - __u64 *folded, __u64 loop_cnt) +static __u64 jent_lfsr_time(struct rand_data *ec, __u64 time, __u64 loop_cnt) { unsigned int i; __u64 j = 0; @@ -217,15 +199,34 @@ static __u64 jent_fold_time(struct rand_data *ec, __u64 time, if (loop_cnt) fold_loop_cnt = loop_cnt; for (j = 0; j < fold_loop_cnt; j++) { - new = 0; + new = ec->data; for (i = 1; (DATA_SIZE_BITS) >= i; i++) { __u64 tmp = time << (DATA_SIZE_BITS - i); tmp = tmp >> (DATA_SIZE_BITS - 1); + + /* + * Fibonacci LSFR with polynomial of + * x^64 + x^61 + x^56 + x^31 + x^28 + x^23 + 1 which is + * primitive according to + * http://poincare.matf.bg.ac.rs/~ezivkovm/publications/primpol1.pdf + * (the shift values are the polynomial values minus one + * due to counting bits from 0 to 63). As the current + * position is always the LSB, the polynomial only needs + * to shift data in from the left without wrap. + */ + tmp ^= ((new >> 63) & 1); + tmp ^= ((new >> 60) & 1); + tmp ^= ((new >> 55) & 1); + tmp ^= ((new >> 30) & 1); + tmp ^= ((new >> 27) & 1); + tmp ^= ((new >> 22) & 1); + new <<= 1; new ^= tmp; } } - *folded = new; + ec->data = new; + return fold_loop_cnt; } @@ -258,7 +259,6 @@ static __u64 jent_fold_time(struct rand_data *ec, __u64 time, */ static unsigned int jent_memaccess(struct rand_data *ec, __u64 loop_cnt) { - unsigned char *tmpval = NULL; unsigned int wrap = 0; __u64 i = 0; #define MAX_ACC_LOOP_BIT 7 @@ -278,7 +278,7 @@ static unsigned int jent_memaccess(struct rand_data *ec, __u64 loop_cnt) acc_loop_cnt = loop_cnt; for (i = 0; i < (ec->memaccessloops + acc_loop_cnt); i++) { - tmpval = ec->mem + ec->memlocation; + unsigned char *tmpval = ec->mem + ec->memlocation; /* * memory access: just add 1 to one byte, * wrap at 255 -- memory access implies read @@ -316,7 +316,7 @@ static unsigned int jent_memaccess(struct rand_data *ec, __u64 loop_cnt) * 0 jitter measurement not stuck (good bit) * 1 jitter measurement stuck (reject bit) */ -static void jent_stuck(struct rand_data *ec, __u64 current_delta) +static int jent_stuck(struct rand_data *ec, __u64 current_delta) { __s64 delta2 = ec->last_delta - current_delta; __s64 delta3 = delta2 - ec->last_delta2; @@ -325,14 +325,15 @@ static void jent_stuck(struct rand_data *ec, __u64 current_delta) ec->last_delta2 = delta2; if (!current_delta || !delta2 || !delta3) - ec->stuck = 1; + return 1; + + return 0; } /** * This is the heart of the entropy generation: calculate time deltas and - * use the CPU jitter in the time deltas. The jitter is folded into one - * bit. You can call this function the "random bit generator" as it - * produces one random bit per invocation. + * use the CPU jitter in the time deltas. The jitter is injected into the + * entropy pool. * * WARNING: ensure that ->prev_time is primed before using the output * of this function! This can be done by calling this function @@ -341,12 +342,11 @@ static void jent_stuck(struct rand_data *ec, __u64 current_delta) * Input: * @entropy_collector Reference to entropy collector * - * @return One random bit + * @return result of stuck test */ -static __u64 jent_measure_jitter(struct rand_data *ec) +static int jent_measure_jitter(struct rand_data *ec) { __u64 time = 0; - __u64 data = 0; __u64 current_delta = 0; /* Invoke one noise source before time measurement to add variations */ @@ -360,109 +360,11 @@ static __u64 jent_measure_jitter(struct rand_data *ec) current_delta = time - ec->prev_time; ec->prev_time = time; - /* Now call the next noise sources which also folds the data */ - jent_fold_time(ec, current_delta, &data, 0); - - /* - * Check whether we have a stuck measurement. The enforcement - * is performed after the stuck value has been mixed into the - * entropy pool. - */ - jent_stuck(ec, current_delta); - - return data; -} - -/** - * Von Neuman unbias as explained in RFC 4086 section 4.2. As shown in the - * documentation of that RNG, the bits from jent_measure_jitter are considered - * independent which implies that the Von Neuman unbias operation is applicable. - * A proof of the Von-Neumann unbias operation to remove skews is given in the - * document "A proposal for: Functionality classes for random number - * generators", version 2.0 by Werner Schindler, section 5.4.1. - * - * Input: - * @entropy_collector Reference to entropy collector - * - * @return One random bit - */ -static __u64 jent_unbiased_bit(struct rand_data *entropy_collector) -{ - do { - __u64 a = jent_measure_jitter(entropy_collector); - __u64 b = jent_measure_jitter(entropy_collector); - - if (a == b) - continue; - if (1 == a) - return 1; - else - return 0; - } while (1); -} - -/** - * Shuffle the pool a bit by mixing some value with a bijective function (XOR) - * into the pool. - * - * The function generates a mixer value that depends on the bits set and the - * location of the set bits in the random number generated by the entropy - * source. Therefore, based on the generated random number, this mixer value - * can have 2**64 different values. That mixer value is initialized with the - * first two SHA-1 constants. After obtaining the mixer value, it is XORed into - * the random number. - * - * The mixer value is not assumed to contain any entropy. But due to the XOR - * operation, it can also not destroy any entropy present in the entropy pool. - * - * Input: - * @entropy_collector Reference to entropy collector - */ -static void jent_stir_pool(struct rand_data *entropy_collector) -{ - /* - * to shut up GCC on 32 bit, we have to initialize the 64 variable - * with two 32 bit variables - */ - union c { - __u64 u64; - __u32 u32[2]; - }; - /* - * This constant is derived from the first two 32 bit initialization - * vectors of SHA-1 as defined in FIPS 180-4 section 5.3.1 - */ - union c constant; - /* - * The start value of the mixer variable is derived from the third - * and fourth 32 bit initialization vector of SHA-1 as defined in - * FIPS 180-4 section 5.3.1 - */ - union c mixer; - unsigned int i = 0; - - /* - * Store the SHA-1 constants in reverse order to make up the 64 bit - * value -- this applies to a little endian system, on a big endian - * system, it reverses as expected. But this really does not matter - * as we do not rely on the specific numbers. We just pick the SHA-1 - * constants as they have a good mix of bit set and unset. - */ - constant.u32[1] = 0x67452301; - constant.u32[0] = 0xefcdab89; - mixer.u32[1] = 0x98badcfe; - mixer.u32[0] = 0x10325476; + /* Now call the next noise sources which also injects the data */ + jent_lfsr_time(ec, current_delta, 0); - for (i = 0; i < DATA_SIZE_BITS; i++) { - /* - * get the i-th bit of the input random number and only XOR - * the constant into the mixer value when that bit is set - */ - if ((entropy_collector->data >> i) & 1) - mixer.u64 ^= constant.u64; - mixer.u64 = jent_rol64(mixer.u64, 1); - } - entropy_collector->data ^= mixer.u64; + /* Check whether we have a stuck measurement. */ + return jent_stuck(ec, current_delta); } /** @@ -480,48 +382,9 @@ static void jent_gen_entropy(struct rand_data *ec) jent_measure_jitter(ec); while (1) { - __u64 data = 0; - - if (ec->disable_unbias == 1) - data = jent_measure_jitter(ec); - else - data = jent_unbiased_bit(ec); - - /* enforcement of the jent_stuck test */ - if (ec->stuck) { - /* - * We only mix in the bit considered not appropriate - * without the LSFR. The reason is that if we apply - * the LSFR and we do not rotate, the 2nd bit with LSFR - * will cancel out the first LSFR application on the - * bad bit. - * - * And we do not rotate as we apply the next bit to the - * current bit location again. - */ - ec->data ^= data; - ec->stuck = 0; + /* If a stuck measurement is received, repeat measurement */ + if (jent_measure_jitter(ec)) continue; - } - - /* - * Fibonacci LSFR with polynom of - * x^64 + x^61 + x^56 + x^31 + x^28 + x^23 + 1 which is - * primitive according to - * http://poincare.matf.bg.ac.rs/~ezivkovm/publications/primpol1.pdf - * (the shift values are the polynom values minus one - * due to counting bits from 0 to 63). As the current - * position is always the LSB, the polynom only needs - * to shift data in from the left without wrap. - */ - ec->data ^= data; - ec->data ^= ((ec->data >> 63) & 1); - ec->data ^= ((ec->data >> 60) & 1); - ec->data ^= ((ec->data >> 55) & 1); - ec->data ^= ((ec->data >> 30) & 1); - ec->data ^= ((ec->data >> 27) & 1); - ec->data ^= ((ec->data >> 22) & 1); - ec->data = jent_rol64(ec->data, 1); /* * We multiply the loop value with ->osr to obtain the @@ -530,8 +393,6 @@ static void jent_gen_entropy(struct rand_data *ec) if (++k >= (DATA_SIZE_BITS * ec->osr)) break; } - if (ec->stir) - jent_stir_pool(ec); } /** @@ -639,12 +500,6 @@ struct rand_data *jent_entropy_collector_alloc(unsigned int osr, osr = 1; /* minimum sampling rate is 1 */ entropy_collector->osr = osr; - entropy_collector->stir = 1; - if (flags & JENT_DISABLE_STIR) - entropy_collector->stir = 0; - if (flags & JENT_DISABLE_UNBIAS) - entropy_collector->disable_unbias = 1; - /* fill the data pad with non-zero values */ jent_gen_entropy(entropy_collector); @@ -656,7 +511,6 @@ void jent_entropy_collector_free(struct rand_data *entropy_collector) jent_zfree(entropy_collector->mem); entropy_collector->mem = NULL; jent_zfree(entropy_collector); - entropy_collector = NULL; } int jent_entropy_init(void) @@ -665,8 +519,9 @@ int jent_entropy_init(void) __u64 delta_sum = 0; __u64 old_delta = 0; int time_backwards = 0; - int count_var = 0; int count_mod = 0; + int count_stuck = 0; + struct rand_data ec = { 0 }; /* We could perform statistical tests here, but the problem is * that we only have a few loop counts to do testing. These @@ -695,12 +550,14 @@ int jent_entropy_init(void) for (i = 0; (TESTLOOPCOUNT + CLEARCACHE) > i; i++) { __u64 time = 0; __u64 time2 = 0; - __u64 folded = 0; __u64 delta = 0; unsigned int lowdelta = 0; + int stuck; + /* Invoke core entropy collection logic */ jent_get_nstime(&time); - jent_fold_time(NULL, time, &folded, 1<<MIN_FOLD_LOOP_BIT); + ec.prev_time = time; + jent_lfsr_time(&ec, time, 0); jent_get_nstime(&time2); /* test whether timer works */ @@ -715,6 +572,8 @@ int jent_entropy_init(void) if (!delta) return JENT_ECOARSETIME; + stuck = jent_stuck(&ec, delta); + /* * up to here we did not modify any variable that will be * evaluated later, but we already performed some work. Thus we @@ -725,14 +584,14 @@ int jent_entropy_init(void) if (CLEARCACHE > i) continue; + if (stuck) + count_stuck++; + /* test whether we have an increasing timer */ if (!(time2 > time)) time_backwards++; - /* - * Avoid modulo of 64 bit integer to allow code to compile - * on 32 bit architectures. - */ + /* use 32 bit value to ensure compilation on 32 bit arches */ lowdelta = time2 - time; if (!(lowdelta % 100)) count_mod++; @@ -743,14 +602,10 @@ int jent_entropy_init(void) * only after the first loop is executed as we need to prime * the old_data value */ - if (i) { - if (delta != old_delta) - count_var++; - if (delta > old_delta) - delta_sum += (delta - old_delta); - else - delta_sum += (old_delta - delta); - } + if (delta > old_delta) + delta_sum += (delta - old_delta); + else + delta_sum += (old_delta - delta); old_delta = delta; } @@ -763,25 +618,29 @@ int jent_entropy_init(void) */ if (3 < time_backwards) return JENT_ENOMONOTONIC; - /* Error if the time variances are always identical */ - if (!delta_sum) - return JENT_EVARVAR; /* * Variations of deltas of time must on average be larger * than 1 to ensure the entropy estimation * implied with 1 is preserved */ - if (delta_sum <= 1) - return JENT_EMINVARVAR; + if ((delta_sum) <= 1) + return JENT_EVARVAR; /* * Ensure that we have variations in the time stamp below 10 for at - * least 10% of all checks -- on some platforms, the counter - * increments in multiples of 100, but not always + * least 10% of all checks -- on some platforms, the counter increments + * in multiples of 100, but not always */ if ((TESTLOOPCOUNT/10 * 9) < count_mod) return JENT_ECOARSETIME; + /* + * If we have more than 90% stuck results, then this Jitter RNG is + * likely to not work well. + */ + if ((TESTLOOPCOUNT/10 * 9) < count_stuck) + return JENT_ESTUCK; + return 0; } diff --git a/crypto/khazad.c b/crypto/khazad.c index b50aa8a3ab4c..14ca7f1631c7 100644 --- a/crypto/khazad.c +++ b/crypto/khazad.c @@ -848,6 +848,7 @@ static void khazad_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) static struct crypto_alg khazad_alg = { .cra_name = "khazad", + .cra_driver_name = "khazad-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = KHAZAD_BLOCK_SIZE, .cra_ctxsize = sizeof (struct khazad_ctx), diff --git a/crypto/lrw.c b/crypto/lrw.c index 58009cf63a6e..be829f6afc8e 100644 --- a/crypto/lrw.c +++ b/crypto/lrw.c @@ -384,7 +384,7 @@ static int create(struct crypto_template *tmpl, struct rtattr **tb) inst->alg.base.cra_priority = alg->base.cra_priority; inst->alg.base.cra_blocksize = LRW_BLOCK_SIZE; inst->alg.base.cra_alignmask = alg->base.cra_alignmask | - (__alignof__(__be32) - 1); + (__alignof__(be128) - 1); inst->alg.ivsize = LRW_BLOCK_SIZE; inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg) + diff --git a/crypto/lz4.c b/crypto/lz4.c index 54673cf88385..0606f8862e78 100644 --- a/crypto/lz4.c +++ b/crypto/lz4.c @@ -106,6 +106,7 @@ static int lz4_decompress_crypto(struct crypto_tfm *tfm, const u8 *src, static struct crypto_alg alg_lz4 = { .cra_name = "lz4", + .cra_driver_name = "lz4-generic", .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, .cra_ctxsize = sizeof(struct lz4_ctx), .cra_module = THIS_MODULE, diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c index daae7bfb385d..d7cc94aa2fcf 100644 --- a/crypto/lz4hc.c +++ b/crypto/lz4hc.c @@ -107,6 +107,7 @@ static int lz4hc_decompress_crypto(struct crypto_tfm *tfm, const u8 *src, static struct crypto_alg alg_lz4hc = { .cra_name = "lz4hc", + .cra_driver_name = "lz4hc-generic", .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, .cra_ctxsize = sizeof(struct lz4hc_ctx), .cra_module = THIS_MODULE, diff --git a/crypto/lzo-rle.c b/crypto/lzo-rle.c index c4303e96f2b1..0631d975bfac 100644 --- a/crypto/lzo-rle.c +++ b/crypto/lzo-rle.c @@ -109,6 +109,7 @@ static int lzorle_sdecompress(struct crypto_scomp *tfm, const u8 *src, static struct crypto_alg alg = { .cra_name = "lzo-rle", + .cra_driver_name = "lzo-rle-generic", .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, .cra_ctxsize = sizeof(struct lzorle_ctx), .cra_module = THIS_MODULE, diff --git a/crypto/lzo.c b/crypto/lzo.c index 97051a2ca08e..ebda132dd22b 100644 --- a/crypto/lzo.c +++ b/crypto/lzo.c @@ -109,6 +109,7 @@ static int lzo_sdecompress(struct crypto_scomp *tfm, const u8 *src, static struct crypto_alg alg = { .cra_name = "lzo", + .cra_driver_name = "lzo-generic", .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, .cra_ctxsize = sizeof(struct lzo_ctx), .cra_module = THIS_MODULE, diff --git a/crypto/md4.c b/crypto/md4.c index 9a1a228a0c69..2e7f2f319f95 100644 --- a/crypto/md4.c +++ b/crypto/md4.c @@ -216,9 +216,10 @@ static struct shash_alg alg = { .final = md4_final, .descsize = sizeof(struct md4_ctx), .base = { - .cra_name = "md4", - .cra_blocksize = MD4_HMAC_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "md4", + .cra_driver_name = "md4-generic", + .cra_blocksize = MD4_HMAC_BLOCK_SIZE, + .cra_module = THIS_MODULE, } }; diff --git a/crypto/md5.c b/crypto/md5.c index 221c2c0932f8..22dc60bc0437 100644 --- a/crypto/md5.c +++ b/crypto/md5.c @@ -228,9 +228,10 @@ static struct shash_alg alg = { .descsize = sizeof(struct md5_state), .statesize = sizeof(struct md5_state), .base = { - .cra_name = "md5", - .cra_blocksize = MD5_HMAC_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "md5", + .cra_driver_name = "md5-generic", + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_module = THIS_MODULE, } }; diff --git a/crypto/michael_mic.c b/crypto/michael_mic.c index b3d83ff709d3..20e6220f46f6 100644 --- a/crypto/michael_mic.c +++ b/crypto/michael_mic.c @@ -156,6 +156,7 @@ static struct shash_alg alg = { .descsize = sizeof(struct michael_mic_desc_ctx), .base = { .cra_name = "michael_mic", + .cra_driver_name = "michael_mic-generic", .cra_blocksize = 8, .cra_alignmask = 3, .cra_ctxsize = sizeof(struct michael_mic_ctx), diff --git a/crypto/rmd128.c b/crypto/rmd128.c index d6c031a9fd14..29308fb97e7e 100644 --- a/crypto/rmd128.c +++ b/crypto/rmd128.c @@ -298,6 +298,7 @@ static struct shash_alg alg = { .descsize = sizeof(struct rmd128_ctx), .base = { .cra_name = "rmd128", + .cra_driver_name = "rmd128-generic", .cra_blocksize = RMD128_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/crypto/rmd160.c b/crypto/rmd160.c index f3add4d54a22..c5fe4034b153 100644 --- a/crypto/rmd160.c +++ b/crypto/rmd160.c @@ -342,6 +342,7 @@ static struct shash_alg alg = { .descsize = sizeof(struct rmd160_ctx), .base = { .cra_name = "rmd160", + .cra_driver_name = "rmd160-generic", .cra_blocksize = RMD160_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/crypto/rmd256.c b/crypto/rmd256.c index 79ca3029848f..3c730e9de5fd 100644 --- a/crypto/rmd256.c +++ b/crypto/rmd256.c @@ -317,6 +317,7 @@ static struct shash_alg alg = { .descsize = sizeof(struct rmd256_ctx), .base = { .cra_name = "rmd256", + .cra_driver_name = "rmd256-generic", .cra_blocksize = RMD256_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/crypto/rmd320.c b/crypto/rmd320.c index b2392ef7467b..c919ad6c4705 100644 --- a/crypto/rmd320.c +++ b/crypto/rmd320.c @@ -366,6 +366,7 @@ static struct shash_alg alg = { .descsize = sizeof(struct rmd320_ctx), .base = { .cra_name = "rmd320", + .cra_driver_name = "rmd320-generic", .cra_blocksize = RMD320_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/crypto/serpent_generic.c b/crypto/serpent_generic.c index 16f612b6dbca..56fa665a4f01 100644 --- a/crypto/serpent_generic.c +++ b/crypto/serpent_generic.c @@ -225,7 +225,13 @@ x4 ^= x2; \ }) -static void __serpent_setkey_sbox(u32 r0, u32 r1, u32 r2, u32 r3, u32 r4, u32 *k) +/* + * both gcc and clang have misoptimized this function in the past, + * producing horrible object code from spilling temporary variables + * on the stack. Forcing this part out of line avoids that. + */ +static noinline void __serpent_setkey_sbox(u32 r0, u32 r1, u32 r2, + u32 r3, u32 r4, u32 *k) { k += 100; S3(r3, r4, r0, r1, r2); store_and_load_keys(r1, r2, r4, r3, 28, 24); @@ -637,6 +643,7 @@ static struct crypto_alg srp_algs[2] = { { .cia_decrypt = serpent_decrypt } } }, { .cra_name = "tnepres", + .cra_driver_name = "tnepres-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = SERPENT_BLOCK_SIZE, .cra_ctxsize = sizeof(struct serpent_ctx), diff --git a/crypto/skcipher.c b/crypto/skcipher.c index df735148000f..5d836fc3df3e 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -837,6 +837,40 @@ static int skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, return 0; } +int crypto_skcipher_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct crypto_alg *alg = tfm->base.__crt_alg; + unsigned int cryptlen = req->cryptlen; + int ret; + + crypto_stats_get(alg); + if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) + ret = -ENOKEY; + else + ret = tfm->encrypt(req); + crypto_stats_skcipher_encrypt(cryptlen, ret, alg); + return ret; +} +EXPORT_SYMBOL_GPL(crypto_skcipher_encrypt); + +int crypto_skcipher_decrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct crypto_alg *alg = tfm->base.__crt_alg; + unsigned int cryptlen = req->cryptlen; + int ret; + + crypto_stats_get(alg); + if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) + ret = -ENOKEY; + else + ret = tfm->decrypt(req); + crypto_stats_skcipher_decrypt(cryptlen, ret, alg); + return ret; +} +EXPORT_SYMBOL_GPL(crypto_skcipher_decrypt); + static void crypto_skcipher_exit_tfm(struct crypto_tfm *tfm) { struct crypto_skcipher *skcipher = __crypto_skcipher_cast(tfm); diff --git a/crypto/tea.c b/crypto/tea.c index 37a18a9be2f4..02efc5d81690 100644 --- a/crypto/tea.c +++ b/crypto/tea.c @@ -216,6 +216,7 @@ static void xeta_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) static struct crypto_alg tea_algs[3] = { { .cra_name = "tea", + .cra_driver_name = "tea-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = TEA_BLOCK_SIZE, .cra_ctxsize = sizeof (struct tea_ctx), @@ -229,6 +230,7 @@ static struct crypto_alg tea_algs[3] = { { .cia_decrypt = tea_decrypt } } }, { .cra_name = "xtea", + .cra_driver_name = "xtea-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = XTEA_BLOCK_SIZE, .cra_ctxsize = sizeof (struct xtea_ctx), @@ -242,6 +244,7 @@ static struct crypto_alg tea_algs[3] = { { .cia_decrypt = xtea_decrypt } } }, { .cra_name = "xeta", + .cra_driver_name = "xeta-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = XTEA_BLOCK_SIZE, .cra_ctxsize = sizeof (struct xtea_ctx), diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 658a7eeebab2..d0b5b33806a6 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -1032,6 +1032,205 @@ static void crypto_reenable_simd_for_test(void) } #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ +static int build_hash_sglist(struct test_sglist *tsgl, + const struct hash_testvec *vec, + const struct testvec_config *cfg, + unsigned int alignmask, + const struct test_sg_division *divs[XBUFSIZE]) +{ + struct kvec kv; + struct iov_iter input; + + kv.iov_base = (void *)vec->plaintext; + kv.iov_len = vec->psize; + iov_iter_kvec(&input, WRITE, &kv, 1, vec->psize); + return build_test_sglist(tsgl, cfg->src_divs, alignmask, vec->psize, + &input, divs); +} + +static int check_hash_result(const char *type, + const u8 *result, unsigned int digestsize, + const struct hash_testvec *vec, + const char *vec_name, + const char *driver, + const struct testvec_config *cfg) +{ + if (memcmp(result, vec->digest, digestsize) != 0) { + pr_err("alg: %s: %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n", + type, driver, vec_name, cfg->name); + return -EINVAL; + } + if (!testmgr_is_poison(&result[digestsize], TESTMGR_POISON_LEN)) { + pr_err("alg: %s: %s overran result buffer on test vector %s, cfg=\"%s\"\n", + type, driver, vec_name, cfg->name); + return -EOVERFLOW; + } + return 0; +} + +static inline int check_shash_op(const char *op, int err, + const char *driver, const char *vec_name, + const struct testvec_config *cfg) +{ + if (err) + pr_err("alg: shash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n", + driver, op, err, vec_name, cfg->name); + return err; +} + +static inline const void *sg_data(struct scatterlist *sg) +{ + return page_address(sg_page(sg)) + sg->offset; +} + +/* Test one hash test vector in one configuration, using the shash API */ +static int test_shash_vec_cfg(const char *driver, + const struct hash_testvec *vec, + const char *vec_name, + const struct testvec_config *cfg, + struct shash_desc *desc, + struct test_sglist *tsgl, + u8 *hashstate) +{ + struct crypto_shash *tfm = desc->tfm; + const unsigned int alignmask = crypto_shash_alignmask(tfm); + const unsigned int digestsize = crypto_shash_digestsize(tfm); + const unsigned int statesize = crypto_shash_statesize(tfm); + const struct test_sg_division *divs[XBUFSIZE]; + unsigned int i; + u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN]; + int err; + + /* Set the key, if specified */ + if (vec->ksize) { + err = crypto_shash_setkey(tfm, vec->key, vec->ksize); + if (err) { + if (err == vec->setkey_error) + return 0; + pr_err("alg: shash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n", + driver, vec_name, vec->setkey_error, err, + crypto_shash_get_flags(tfm)); + return err; + } + if (vec->setkey_error) { + pr_err("alg: shash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n", + driver, vec_name, vec->setkey_error); + return -EINVAL; + } + } + + /* Build the scatterlist for the source data */ + err = build_hash_sglist(tsgl, vec, cfg, alignmask, divs); + if (err) { + pr_err("alg: shash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n", + driver, vec_name, cfg->name); + return err; + } + + /* Do the actual hashing */ + + testmgr_poison(desc->__ctx, crypto_shash_descsize(tfm)); + testmgr_poison(result, digestsize + TESTMGR_POISON_LEN); + + if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST || + vec->digest_error) { + /* Just using digest() */ + if (tsgl->nents != 1) + return 0; + if (cfg->nosimd) + crypto_disable_simd_for_test(); + err = crypto_shash_digest(desc, sg_data(&tsgl->sgl[0]), + tsgl->sgl[0].length, result); + if (cfg->nosimd) + crypto_reenable_simd_for_test(); + if (err) { + if (err == vec->digest_error) + return 0; + pr_err("alg: shash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n", + driver, vec_name, vec->digest_error, err, + cfg->name); + return err; + } + if (vec->digest_error) { + pr_err("alg: shash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n", + driver, vec_name, vec->digest_error, cfg->name); + return -EINVAL; + } + goto result_ready; + } + + /* Using init(), zero or more update(), then final() or finup() */ + + if (cfg->nosimd) + crypto_disable_simd_for_test(); + err = crypto_shash_init(desc); + if (cfg->nosimd) + crypto_reenable_simd_for_test(); + err = check_shash_op("init", err, driver, vec_name, cfg); + if (err) + return err; + + for (i = 0; i < tsgl->nents; i++) { + if (i + 1 == tsgl->nents && + cfg->finalization_type == FINALIZATION_TYPE_FINUP) { + if (divs[i]->nosimd) + crypto_disable_simd_for_test(); + err = crypto_shash_finup(desc, sg_data(&tsgl->sgl[i]), + tsgl->sgl[i].length, result); + if (divs[i]->nosimd) + crypto_reenable_simd_for_test(); + err = check_shash_op("finup", err, driver, vec_name, + cfg); + if (err) + return err; + goto result_ready; + } + if (divs[i]->nosimd) + crypto_disable_simd_for_test(); + err = crypto_shash_update(desc, sg_data(&tsgl->sgl[i]), + tsgl->sgl[i].length); + if (divs[i]->nosimd) + crypto_reenable_simd_for_test(); + err = check_shash_op("update", err, driver, vec_name, cfg); + if (err) + return err; + if (divs[i]->flush_type == FLUSH_TYPE_REIMPORT) { + /* Test ->export() and ->import() */ + testmgr_poison(hashstate + statesize, + TESTMGR_POISON_LEN); + err = crypto_shash_export(desc, hashstate); + err = check_shash_op("export", err, driver, vec_name, + cfg); + if (err) + return err; + if (!testmgr_is_poison(hashstate + statesize, + TESTMGR_POISON_LEN)) { + pr_err("alg: shash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n", + driver, vec_name, cfg->name); + return -EOVERFLOW; + } + testmgr_poison(desc->__ctx, crypto_shash_descsize(tfm)); + err = crypto_shash_import(desc, hashstate); + err = check_shash_op("import", err, driver, vec_name, + cfg); + if (err) + return err; + } + } + + if (cfg->nosimd) + crypto_disable_simd_for_test(); + err = crypto_shash_final(desc, result); + if (cfg->nosimd) + crypto_reenable_simd_for_test(); + err = check_shash_op("final", err, driver, vec_name, cfg); + if (err) + return err; +result_ready: + return check_hash_result("shash", result, digestsize, vec, vec_name, + driver, cfg); +} + static int do_ahash_op(int (*op)(struct ahash_request *req), struct ahash_request *req, struct crypto_wait *wait, bool nosimd) @@ -1049,31 +1248,32 @@ static int do_ahash_op(int (*op)(struct ahash_request *req), return crypto_wait_req(err, wait); } -static int check_nonfinal_hash_op(const char *op, int err, - u8 *result, unsigned int digestsize, - const char *driver, const char *vec_name, - const struct testvec_config *cfg) +static int check_nonfinal_ahash_op(const char *op, int err, + u8 *result, unsigned int digestsize, + const char *driver, const char *vec_name, + const struct testvec_config *cfg) { if (err) { - pr_err("alg: hash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n", + pr_err("alg: ahash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n", driver, op, err, vec_name, cfg->name); return err; } if (!testmgr_is_poison(result, digestsize)) { - pr_err("alg: hash: %s %s() used result buffer on test vector %s, cfg=\"%s\"\n", + pr_err("alg: ahash: %s %s() used result buffer on test vector %s, cfg=\"%s\"\n", driver, op, vec_name, cfg->name); return -EINVAL; } return 0; } -static int test_hash_vec_cfg(const char *driver, - const struct hash_testvec *vec, - const char *vec_name, - const struct testvec_config *cfg, - struct ahash_request *req, - struct test_sglist *tsgl, - u8 *hashstate) +/* Test one hash test vector in one configuration, using the ahash API */ +static int test_ahash_vec_cfg(const char *driver, + const struct hash_testvec *vec, + const char *vec_name, + const struct testvec_config *cfg, + struct ahash_request *req, + struct test_sglist *tsgl, + u8 *hashstate) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); const unsigned int alignmask = crypto_ahash_alignmask(tfm); @@ -1082,8 +1282,6 @@ static int test_hash_vec_cfg(const char *driver, const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags; const struct test_sg_division *divs[XBUFSIZE]; DECLARE_CRYPTO_WAIT(wait); - struct kvec _input; - struct iov_iter input; unsigned int i; struct scatterlist *pending_sgl; unsigned int pending_len; @@ -1096,26 +1294,22 @@ static int test_hash_vec_cfg(const char *driver, if (err) { if (err == vec->setkey_error) return 0; - pr_err("alg: hash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n", + pr_err("alg: ahash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n", driver, vec_name, vec->setkey_error, err, crypto_ahash_get_flags(tfm)); return err; } if (vec->setkey_error) { - pr_err("alg: hash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n", + pr_err("alg: ahash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n", driver, vec_name, vec->setkey_error); return -EINVAL; } } /* Build the scatterlist for the source data */ - _input.iov_base = (void *)vec->plaintext; - _input.iov_len = vec->psize; - iov_iter_kvec(&input, WRITE, &_input, 1, vec->psize); - err = build_test_sglist(tsgl, cfg->src_divs, alignmask, vec->psize, - &input, divs); + err = build_hash_sglist(tsgl, vec, cfg, alignmask, divs); if (err) { - pr_err("alg: hash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n", + pr_err("alg: ahash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n", driver, vec_name, cfg->name); return err; } @@ -1135,13 +1329,13 @@ static int test_hash_vec_cfg(const char *driver, if (err) { if (err == vec->digest_error) return 0; - pr_err("alg: hash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n", + pr_err("alg: ahash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n", driver, vec_name, vec->digest_error, err, cfg->name); return err; } if (vec->digest_error) { - pr_err("alg: hash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n", + pr_err("alg: ahash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n", driver, vec_name, vec->digest_error, cfg->name); return -EINVAL; } @@ -1153,8 +1347,8 @@ static int test_hash_vec_cfg(const char *driver, ahash_request_set_callback(req, req_flags, crypto_req_done, &wait); ahash_request_set_crypt(req, NULL, result, 0); err = do_ahash_op(crypto_ahash_init, req, &wait, cfg->nosimd); - err = check_nonfinal_hash_op("init", err, result, digestsize, - driver, vec_name, cfg); + err = check_nonfinal_ahash_op("init", err, result, digestsize, + driver, vec_name, cfg); if (err) return err; @@ -1170,9 +1364,9 @@ static int test_hash_vec_cfg(const char *driver, pending_len); err = do_ahash_op(crypto_ahash_update, req, &wait, divs[i]->nosimd); - err = check_nonfinal_hash_op("update", err, - result, digestsize, - driver, vec_name, cfg); + err = check_nonfinal_ahash_op("update", err, + result, digestsize, + driver, vec_name, cfg); if (err) return err; pending_sgl = NULL; @@ -1183,23 +1377,23 @@ static int test_hash_vec_cfg(const char *driver, testmgr_poison(hashstate + statesize, TESTMGR_POISON_LEN); err = crypto_ahash_export(req, hashstate); - err = check_nonfinal_hash_op("export", err, - result, digestsize, - driver, vec_name, cfg); + err = check_nonfinal_ahash_op("export", err, + result, digestsize, + driver, vec_name, cfg); if (err) return err; if (!testmgr_is_poison(hashstate + statesize, TESTMGR_POISON_LEN)) { - pr_err("alg: hash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n", + pr_err("alg: ahash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n", driver, vec_name, cfg->name); return -EOVERFLOW; } testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm)); err = crypto_ahash_import(req, hashstate); - err = check_nonfinal_hash_op("import", err, - result, digestsize, - driver, vec_name, cfg); + err = check_nonfinal_ahash_op("import", err, + result, digestsize, + driver, vec_name, cfg); if (err) return err; } @@ -1213,13 +1407,13 @@ static int test_hash_vec_cfg(const char *driver, if (cfg->finalization_type == FINALIZATION_TYPE_FINAL) { /* finish with update() and final() */ err = do_ahash_op(crypto_ahash_update, req, &wait, cfg->nosimd); - err = check_nonfinal_hash_op("update", err, result, digestsize, - driver, vec_name, cfg); + err = check_nonfinal_ahash_op("update", err, result, digestsize, + driver, vec_name, cfg); if (err) return err; err = do_ahash_op(crypto_ahash_final, req, &wait, cfg->nosimd); if (err) { - pr_err("alg: hash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n", + pr_err("alg: ahash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n", driver, err, vec_name, cfg->name); return err; } @@ -1227,31 +1421,49 @@ static int test_hash_vec_cfg(const char *driver, /* finish with finup() */ err = do_ahash_op(crypto_ahash_finup, req, &wait, cfg->nosimd); if (err) { - pr_err("alg: hash: %s finup() failed with err %d on test vector %s, cfg=\"%s\"\n", + pr_err("alg: ahash: %s finup() failed with err %d on test vector %s, cfg=\"%s\"\n", driver, err, vec_name, cfg->name); return err; } } result_ready: - /* Check that the algorithm produced the correct digest */ - if (memcmp(result, vec->digest, digestsize) != 0) { - pr_err("alg: hash: %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n", - driver, vec_name, cfg->name); - return -EINVAL; - } - if (!testmgr_is_poison(&result[digestsize], TESTMGR_POISON_LEN)) { - pr_err("alg: hash: %s overran result buffer on test vector %s, cfg=\"%s\"\n", - driver, vec_name, cfg->name); - return -EOVERFLOW; + return check_hash_result("ahash", result, digestsize, vec, vec_name, + driver, cfg); +} + +static int test_hash_vec_cfg(const char *driver, + const struct hash_testvec *vec, + const char *vec_name, + const struct testvec_config *cfg, + struct ahash_request *req, + struct shash_desc *desc, + struct test_sglist *tsgl, + u8 *hashstate) +{ + int err; + + /* + * For algorithms implemented as "shash", most bugs will be detected by + * both the shash and ahash tests. Test the shash API first so that the + * failures involve less indirection, so are easier to debug. + */ + + if (desc) { + err = test_shash_vec_cfg(driver, vec, vec_name, cfg, desc, tsgl, + hashstate); + if (err) + return err; } - return 0; + return test_ahash_vec_cfg(driver, vec, vec_name, cfg, req, tsgl, + hashstate); } static int test_hash_vec(const char *driver, const struct hash_testvec *vec, unsigned int vec_num, struct ahash_request *req, - struct test_sglist *tsgl, u8 *hashstate) + struct shash_desc *desc, struct test_sglist *tsgl, + u8 *hashstate) { char vec_name[16]; unsigned int i; @@ -1262,7 +1474,7 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++) { err = test_hash_vec_cfg(driver, vec, vec_name, &default_hash_testvec_configs[i], - req, tsgl, hashstate); + req, desc, tsgl, hashstate); if (err) return err; } @@ -1276,9 +1488,10 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); err = test_hash_vec_cfg(driver, vec, vec_name, &cfg, - req, tsgl, hashstate); + req, desc, tsgl, hashstate); if (err) return err; + cond_resched(); } } #endif @@ -1290,14 +1503,12 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, * Generate a hash test vector from the given implementation. * Assumes the buffers in 'vec' were already allocated. */ -static void generate_random_hash_testvec(struct crypto_shash *tfm, +static void generate_random_hash_testvec(struct shash_desc *desc, struct hash_testvec *vec, unsigned int maxkeysize, unsigned int maxdatasize, char *name, size_t max_namelen) { - SHASH_DESC_ON_STACK(desc, tfm); - /* Data */ vec->psize = generate_random_length(maxdatasize); generate_random_bytes((u8 *)vec->plaintext, vec->psize); @@ -1314,7 +1525,7 @@ static void generate_random_hash_testvec(struct crypto_shash *tfm, vec->ksize = 1 + (prandom_u32() % maxkeysize); generate_random_bytes((u8 *)vec->key, vec->ksize); - vec->setkey_error = crypto_shash_setkey(tfm, vec->key, + vec->setkey_error = crypto_shash_setkey(desc->tfm, vec->key, vec->ksize); /* If the key couldn't be set, no need to continue to digest. */ if (vec->setkey_error) @@ -1322,7 +1533,6 @@ static void generate_random_hash_testvec(struct crypto_shash *tfm, } /* Digest */ - desc->tfm = tfm; vec->digest_error = crypto_shash_digest(desc, vec->plaintext, vec->psize, (u8 *)vec->digest); done: @@ -1338,6 +1548,7 @@ static int test_hash_vs_generic_impl(const char *driver, const char *generic_driver, unsigned int maxkeysize, struct ahash_request *req, + struct shash_desc *desc, struct test_sglist *tsgl, u8 *hashstate) { @@ -1348,10 +1559,11 @@ static int test_hash_vs_generic_impl(const char *driver, const char *algname = crypto_hash_alg_common(tfm)->base.cra_name; char _generic_driver[CRYPTO_MAX_ALG_NAME]; struct crypto_shash *generic_tfm = NULL; + struct shash_desc *generic_desc = NULL; unsigned int i; struct hash_testvec vec = { 0 }; char vec_name[64]; - struct testvec_config cfg; + struct testvec_config *cfg; char cfgname[TESTVEC_CONFIG_NAMELEN]; int err; @@ -1381,6 +1593,20 @@ static int test_hash_vs_generic_impl(const char *driver, return err; } + cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); + if (!cfg) { + err = -ENOMEM; + goto out; + } + + generic_desc = kzalloc(sizeof(*desc) + + crypto_shash_descsize(generic_tfm), GFP_KERNEL); + if (!generic_desc) { + err = -ENOMEM; + goto out; + } + generic_desc->tfm = generic_tfm; + /* Check the algorithm properties for consistency. */ if (digestsize != crypto_shash_digestsize(generic_tfm)) { @@ -1412,23 +1638,25 @@ static int test_hash_vs_generic_impl(const char *driver, } for (i = 0; i < fuzz_iterations * 8; i++) { - generate_random_hash_testvec(generic_tfm, &vec, + generate_random_hash_testvec(generic_desc, &vec, maxkeysize, maxdatasize, vec_name, sizeof(vec_name)); - generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); + generate_random_testvec_config(cfg, cfgname, sizeof(cfgname)); - err = test_hash_vec_cfg(driver, &vec, vec_name, &cfg, - req, tsgl, hashstate); + err = test_hash_vec_cfg(driver, &vec, vec_name, cfg, + req, desc, tsgl, hashstate); if (err) goto out; cond_resched(); } err = 0; out: + kfree(cfg); kfree(vec.key); kfree(vec.plaintext); kfree(vec.digest); crypto_free_shash(generic_tfm); + kzfree(generic_desc); return err; } #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ @@ -1436,6 +1664,7 @@ static int test_hash_vs_generic_impl(const char *driver, const char *generic_driver, unsigned int maxkeysize, struct ahash_request *req, + struct shash_desc *desc, struct test_sglist *tsgl, u8 *hashstate) { @@ -1443,26 +1672,67 @@ static int test_hash_vs_generic_impl(const char *driver, } #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ +static int alloc_shash(const char *driver, u32 type, u32 mask, + struct crypto_shash **tfm_ret, + struct shash_desc **desc_ret) +{ + struct crypto_shash *tfm; + struct shash_desc *desc; + + tfm = crypto_alloc_shash(driver, type, mask); + if (IS_ERR(tfm)) { + if (PTR_ERR(tfm) == -ENOENT) { + /* + * This algorithm is only available through the ahash + * API, not the shash API, so skip the shash tests. + */ + return 0; + } + pr_err("alg: hash: failed to allocate shash transform for %s: %ld\n", + driver, PTR_ERR(tfm)); + return PTR_ERR(tfm); + } + + desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(tfm), GFP_KERNEL); + if (!desc) { + crypto_free_shash(tfm); + return -ENOMEM; + } + desc->tfm = tfm; + + *tfm_ret = tfm; + *desc_ret = desc; + return 0; +} + static int __alg_test_hash(const struct hash_testvec *vecs, unsigned int num_vecs, const char *driver, u32 type, u32 mask, const char *generic_driver, unsigned int maxkeysize) { - struct crypto_ahash *tfm; + struct crypto_ahash *atfm = NULL; struct ahash_request *req = NULL; + struct crypto_shash *stfm = NULL; + struct shash_desc *desc = NULL; struct test_sglist *tsgl = NULL; u8 *hashstate = NULL; + unsigned int statesize; unsigned int i; int err; - tfm = crypto_alloc_ahash(driver, type, mask); - if (IS_ERR(tfm)) { + /* + * Always test the ahash API. This works regardless of whether the + * algorithm is implemented as ahash or shash. + */ + + atfm = crypto_alloc_ahash(driver, type, mask); + if (IS_ERR(atfm)) { pr_err("alg: hash: failed to allocate transform for %s: %ld\n", - driver, PTR_ERR(tfm)); - return PTR_ERR(tfm); + driver, PTR_ERR(atfm)); + return PTR_ERR(atfm); } - req = ahash_request_alloc(tfm, GFP_KERNEL); + req = ahash_request_alloc(atfm, GFP_KERNEL); if (!req) { pr_err("alg: hash: failed to allocate request for %s\n", driver); @@ -1470,6 +1740,14 @@ static int __alg_test_hash(const struct hash_testvec *vecs, goto out; } + /* + * If available also test the shash API, to cover corner cases that may + * be missed by testing the ahash API only. + */ + err = alloc_shash(driver, type, mask, &stfm, &desc); + if (err) + goto out; + tsgl = kmalloc(sizeof(*tsgl), GFP_KERNEL); if (!tsgl || init_test_sglist(tsgl) != 0) { pr_err("alg: hash: failed to allocate test buffers for %s\n", @@ -1480,8 +1758,10 @@ static int __alg_test_hash(const struct hash_testvec *vecs, goto out; } - hashstate = kmalloc(crypto_ahash_statesize(tfm) + TESTMGR_POISON_LEN, - GFP_KERNEL); + statesize = crypto_ahash_statesize(atfm); + if (stfm) + statesize = max(statesize, crypto_shash_statesize(stfm)); + hashstate = kmalloc(statesize + TESTMGR_POISON_LEN, GFP_KERNEL); if (!hashstate) { pr_err("alg: hash: failed to allocate hash state buffer for %s\n", driver); @@ -1490,20 +1770,24 @@ static int __alg_test_hash(const struct hash_testvec *vecs, } for (i = 0; i < num_vecs; i++) { - err = test_hash_vec(driver, &vecs[i], i, req, tsgl, hashstate); + err = test_hash_vec(driver, &vecs[i], i, req, desc, tsgl, + hashstate); if (err) goto out; + cond_resched(); } err = test_hash_vs_generic_impl(driver, generic_driver, maxkeysize, req, - tsgl, hashstate); + desc, tsgl, hashstate); out: kfree(hashstate); if (tsgl) { destroy_test_sglist(tsgl); kfree(tsgl); } + kfree(desc); + crypto_free_shash(stfm); ahash_request_free(req); - crypto_free_ahash(tfm); + crypto_free_ahash(atfm); return err; } @@ -1755,6 +2039,7 @@ static int test_aead_vec(const char *driver, int enc, &cfg, req, tsgls); if (err) return err; + cond_resched(); } } #endif @@ -1864,7 +2149,7 @@ static int test_aead_vs_generic_impl(const char *driver, unsigned int i; struct aead_testvec vec = { 0 }; char vec_name[64]; - struct testvec_config cfg; + struct testvec_config *cfg; char cfgname[TESTVEC_CONFIG_NAMELEN]; int err; @@ -1894,6 +2179,12 @@ static int test_aead_vs_generic_impl(const char *driver, return err; } + cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); + if (!cfg) { + err = -ENOMEM; + goto out; + } + generic_req = aead_request_alloc(generic_tfm, GFP_KERNEL); if (!generic_req) { err = -ENOMEM; @@ -1948,13 +2239,13 @@ static int test_aead_vs_generic_impl(const char *driver, generate_random_aead_testvec(generic_req, &vec, maxkeysize, maxdatasize, vec_name, sizeof(vec_name)); - generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); + generate_random_testvec_config(cfg, cfgname, sizeof(cfgname)); - err = test_aead_vec_cfg(driver, ENCRYPT, &vec, vec_name, &cfg, + err = test_aead_vec_cfg(driver, ENCRYPT, &vec, vec_name, cfg, req, tsgls); if (err) goto out; - err = test_aead_vec_cfg(driver, DECRYPT, &vec, vec_name, &cfg, + err = test_aead_vec_cfg(driver, DECRYPT, &vec, vec_name, cfg, req, tsgls); if (err) goto out; @@ -1962,6 +2253,7 @@ static int test_aead_vs_generic_impl(const char *driver, } err = 0; out: + kfree(cfg); kfree(vec.key); kfree(vec.iv); kfree(vec.assoc); @@ -1994,6 +2286,7 @@ static int test_aead(const char *driver, int enc, tsgls); if (err) return err; + cond_resched(); } return 0; } @@ -2336,6 +2629,7 @@ static int test_skcipher_vec(const char *driver, int enc, &cfg, req, tsgls); if (err) return err; + cond_resched(); } } #endif @@ -2409,7 +2703,7 @@ static int test_skcipher_vs_generic_impl(const char *driver, unsigned int i; struct cipher_testvec vec = { 0 }; char vec_name[64]; - struct testvec_config cfg; + struct testvec_config *cfg; char cfgname[TESTVEC_CONFIG_NAMELEN]; int err; @@ -2443,6 +2737,12 @@ static int test_skcipher_vs_generic_impl(const char *driver, return err; } + cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); + if (!cfg) { + err = -ENOMEM; + goto out; + } + generic_req = skcipher_request_alloc(generic_tfm, GFP_KERNEL); if (!generic_req) { err = -ENOMEM; @@ -2490,20 +2790,21 @@ static int test_skcipher_vs_generic_impl(const char *driver, for (i = 0; i < fuzz_iterations * 8; i++) { generate_random_cipher_testvec(generic_req, &vec, maxdatasize, vec_name, sizeof(vec_name)); - generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); + generate_random_testvec_config(cfg, cfgname, sizeof(cfgname)); err = test_skcipher_vec_cfg(driver, ENCRYPT, &vec, vec_name, - &cfg, req, tsgls); + cfg, req, tsgls); if (err) goto out; err = test_skcipher_vec_cfg(driver, DECRYPT, &vec, vec_name, - &cfg, req, tsgls); + cfg, req, tsgls); if (err) goto out; cond_resched(); } err = 0; out: + kfree(cfg); kfree(vec.key); kfree(vec.iv); kfree(vec.ptext); @@ -2535,6 +2836,7 @@ static int test_skcipher(const char *driver, int enc, tsgls); if (err) return err; + cond_resched(); } return 0; } @@ -4125,6 +4427,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "ecb(arc4)", + .generic_driver = "ecb(arc4)-generic", .test = alg_test_skcipher, .suite = { .cipher = __VECS(arc4_tv_template) @@ -4790,6 +5093,13 @@ static const struct alg_test_desc alg_test_descs[] = { .test = alg_test_null, .fips_allowed = 1, }, { + .alg = "xxhash64", + .test = alg_test_hash, + .fips_allowed = 1, + .suite = { + .hash = __VECS(xxhash64_tv_template) + } + }, { .alg = "zlib-deflate", .test = alg_test_comp, .fips_allowed = 1, diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 1fdae5993bc3..073bd2efafca 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -38,7 +38,7 @@ struct hash_testvec { const char *key; const char *plaintext; const char *digest; - unsigned short psize; + unsigned int psize; unsigned short ksize; int setkey_error; int digest_error; @@ -69,7 +69,7 @@ struct cipher_testvec { const char *ctext; unsigned char wk; /* weak key flag */ unsigned short klen; - unsigned short len; + unsigned int len; bool fips_skip; bool generates_iv; int setkey_error; @@ -105,9 +105,9 @@ struct aead_testvec { unsigned char novrfy; unsigned char wk; unsigned char klen; - unsigned short plen; - unsigned short clen; - unsigned short alen; + unsigned int plen; + unsigned int clen; + unsigned int alen; int setkey_error; int setauthsize_error; int crypt_error; @@ -33382,6 +33382,112 @@ static const struct hash_testvec crc32c_tv_template[] = { } }; +static const struct hash_testvec xxhash64_tv_template[] = { + { + .psize = 0, + .digest = "\x99\xe9\xd8\x51\x37\xdb\x46\xef", + }, + { + .plaintext = "\x40", + .psize = 1, + .digest = "\x20\x5c\x91\xaa\x88\xeb\x59\xd0", + }, + { + .plaintext = "\x40\x8b\xb8\x41\xe4\x42\x15\x2d" + "\x88\xc7\x9a\x09\x1a\x9b", + .psize = 14, + .digest = "\xa8\xe8\x2b\xa9\x92\xa1\x37\x4a", + }, + { + .plaintext = "\x40\x8b\xb8\x41\xe4\x42\x15\x2d" + "\x88\xc7\x9a\x09\x1a\x9b\x42\xe0" + "\xd4\x38\xa5\x2a\x26\xa5\x19\x4b" + "\x57\x65\x7f\xad\xc3\x7d\xca\x40" + "\x31\x65\x05\xbb\x31\xae\x51\x11" + "\xa8\xc0\xb3\x28\x42\xeb\x3c\x46" + "\xc8\xed\xed\x0f\x8d\x0b\xfa\x6e" + "\xbc\xe3\x88\x53\xca\x8f\xc8\xd9" + "\x41\x26\x7a\x3d\x21\xdb\x1a\x3c" + "\x01\x1d\xc9\xe9\xb7\x3a\x78\x67" + "\x57\x20\x94\xf1\x1e\xfd\xce\x39" + "\x99\x57\x69\x39\xa5\xd0\x8d\xd9" + "\x43\xfe\x1d\x66\x04\x3c\x27\x6a" + "\xe1\x0d\xe7\xc9\xfa\xc9\x07\x56" + "\xa5\xb3\xec\xd9\x1f\x42\x65\x66" + "\xaa\xbf\x87\x9b\xc5\x41\x9c\x27" + "\x3f\x2f\xa9\x55\x93\x01\x27\x33" + "\x43\x99\x4d\x81\x85\xae\x82\x00" + "\x6c\xd0\xd1\xa3\x57\x18\x06\xcc" + "\xec\x72\xf7\x8e\x87\x2d\x1f\x5e" + "\xd7\x5b\x1f\x36\x4c\xfa\xfd\x18" + "\x89\x76\xd3\x5e\xb5\x5a\xc0\x01" + "\xd2\xa1\x9a\x50\xe6\x08\xb4\x76" + "\x56\x4f\x0e\xbc\x54\xfc\x67\xe6" + "\xb9\xc0\x28\x4b\xb5\xc3\xff\x79" + "\x52\xea\xa1\x90\xc3\xaf\x08\x70" + "\x12\x02\x0c\xdb\x94\x00\x38\x95" + "\xed\xfd\x08\xf7\xe8\x04", + .psize = 222, + .digest = "\x41\xfc\xd4\x29\xfe\xe7\x85\x17", + }, + { + .psize = 0, + .key = "\xb1\x79\x37\x9e\x00\x00\x00\x00", + .ksize = 8, + .digest = "\xef\x17\x9b\x92\xa2\xfd\x75\xac", + }, + + { + .plaintext = "\x40", + .psize = 1, + .key = "\xb1\x79\x37\x9e\x00\x00\x00\x00", + .ksize = 8, + .digest = "\xd1\x70\x4f\x14\x02\xc4\x9e\x71", + }, + { + .plaintext = "\x40\x8b\xb8\x41\xe4\x42\x15\x2d" + "\x88\xc7\x9a\x09\x1a\x9b", + .psize = 14, + .key = "\xb1\x79\x37\x9e\x00\x00\x00\x00", + .ksize = 8, + .digest = "\xa4\xcd\xfe\x8e\x37\xe2\x1c\x64" + }, + { + .plaintext = "\x40\x8b\xb8\x41\xe4\x42\x15\x2d" + "\x88\xc7\x9a\x09\x1a\x9b\x42\xe0" + "\xd4\x38\xa5\x2a\x26\xa5\x19\x4b" + "\x57\x65\x7f\xad\xc3\x7d\xca\x40" + "\x31\x65\x05\xbb\x31\xae\x51\x11" + "\xa8\xc0\xb3\x28\x42\xeb\x3c\x46" + "\xc8\xed\xed\x0f\x8d\x0b\xfa\x6e" + "\xbc\xe3\x88\x53\xca\x8f\xc8\xd9" + "\x41\x26\x7a\x3d\x21\xdb\x1a\x3c" + "\x01\x1d\xc9\xe9\xb7\x3a\x78\x67" + "\x57\x20\x94\xf1\x1e\xfd\xce\x39" + "\x99\x57\x69\x39\xa5\xd0\x8d\xd9" + "\x43\xfe\x1d\x66\x04\x3c\x27\x6a" + "\xe1\x0d\xe7\xc9\xfa\xc9\x07\x56" + "\xa5\xb3\xec\xd9\x1f\x42\x65\x66" + "\xaa\xbf\x87\x9b\xc5\x41\x9c\x27" + "\x3f\x2f\xa9\x55\x93\x01\x27\x33" + "\x43\x99\x4d\x81\x85\xae\x82\x00" + "\x6c\xd0\xd1\xa3\x57\x18\x06\xcc" + "\xec\x72\xf7\x8e\x87\x2d\x1f\x5e" + "\xd7\x5b\x1f\x36\x4c\xfa\xfd\x18" + "\x89\x76\xd3\x5e\xb5\x5a\xc0\x01" + "\xd2\xa1\x9a\x50\xe6\x08\xb4\x76" + "\x56\x4f\x0e\xbc\x54\xfc\x67\xe6" + "\xb9\xc0\x28\x4b\xb5\xc3\xff\x79" + "\x52\xea\xa1\x90\xc3\xaf\x08\x70" + "\x12\x02\x0c\xdb\x94\x00\x38\x95" + "\xed\xfd\x08\xf7\xe8\x04", + .psize = 222, + .key = "\xb1\x79\x37\x9e\x00\x00\x00\x00", + .ksize = 8, + .digest = "\x58\xbc\x55\xf2\x42\x81\x5c\xf0" + }, +}; + static const struct comp_testvec lz4_comp_tv_template[] = { { .inlen = 255, diff --git a/crypto/tgr192.c b/crypto/tgr192.c index 702c2c89c7a1..052648e24909 100644 --- a/crypto/tgr192.c +++ b/crypto/tgr192.c @@ -630,9 +630,10 @@ static struct shash_alg tgr_algs[3] = { { .final = tgr192_final, .descsize = sizeof(struct tgr192_ctx), .base = { - .cra_name = "tgr192", - .cra_blocksize = TGR192_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "tgr192", + .cra_driver_name = "tgr192-generic", + .cra_blocksize = TGR192_BLOCK_SIZE, + .cra_module = THIS_MODULE, } }, { .digestsize = TGR160_DIGEST_SIZE, @@ -641,9 +642,10 @@ static struct shash_alg tgr_algs[3] = { { .final = tgr160_final, .descsize = sizeof(struct tgr192_ctx), .base = { - .cra_name = "tgr160", - .cra_blocksize = TGR192_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "tgr160", + .cra_driver_name = "tgr160-generic", + .cra_blocksize = TGR192_BLOCK_SIZE, + .cra_module = THIS_MODULE, } }, { .digestsize = TGR128_DIGEST_SIZE, @@ -652,9 +654,10 @@ static struct shash_alg tgr_algs[3] = { { .final = tgr128_final, .descsize = sizeof(struct tgr192_ctx), .base = { - .cra_name = "tgr128", - .cra_blocksize = TGR192_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "tgr128", + .cra_driver_name = "tgr128-generic", + .cra_blocksize = TGR192_BLOCK_SIZE, + .cra_module = THIS_MODULE, } } }; diff --git a/crypto/wp512.c b/crypto/wp512.c index 1b8e502d999f..feadc13ccae0 100644 --- a/crypto/wp512.c +++ b/crypto/wp512.c @@ -1126,9 +1126,10 @@ static struct shash_alg wp_algs[3] = { { .final = wp512_final, .descsize = sizeof(struct wp512_ctx), .base = { - .cra_name = "wp512", - .cra_blocksize = WP512_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "wp512", + .cra_driver_name = "wp512-generic", + .cra_blocksize = WP512_BLOCK_SIZE, + .cra_module = THIS_MODULE, } }, { .digestsize = WP384_DIGEST_SIZE, @@ -1137,9 +1138,10 @@ static struct shash_alg wp_algs[3] = { { .final = wp384_final, .descsize = sizeof(struct wp512_ctx), .base = { - .cra_name = "wp384", - .cra_blocksize = WP512_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "wp384", + .cra_driver_name = "wp384-generic", + .cra_blocksize = WP512_BLOCK_SIZE, + .cra_module = THIS_MODULE, } }, { .digestsize = WP256_DIGEST_SIZE, @@ -1148,9 +1150,10 @@ static struct shash_alg wp_algs[3] = { { .final = wp256_final, .descsize = sizeof(struct wp512_ctx), .base = { - .cra_name = "wp256", - .cra_blocksize = WP512_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "wp256", + .cra_driver_name = "wp256-generic", + .cra_blocksize = WP512_BLOCK_SIZE, + .cra_module = THIS_MODULE, } } }; diff --git a/crypto/xxhash_generic.c b/crypto/xxhash_generic.c new file mode 100644 index 000000000000..4aad2c0f40a9 --- /dev/null +++ b/crypto/xxhash_generic.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <crypto/internal/hash.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/xxhash.h> +#include <asm/unaligned.h> + +#define XXHASH64_BLOCK_SIZE 32 +#define XXHASH64_DIGEST_SIZE 8 + +struct xxhash64_tfm_ctx { + u64 seed; +}; + +struct xxhash64_desc_ctx { + struct xxh64_state xxhstate; +}; + +static int xxhash64_setkey(struct crypto_shash *tfm, const u8 *key, + unsigned int keylen) +{ + struct xxhash64_tfm_ctx *tctx = crypto_shash_ctx(tfm); + + if (keylen != sizeof(tctx->seed)) { + crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + tctx->seed = get_unaligned_le64(key); + return 0; +} + +static int xxhash64_init(struct shash_desc *desc) +{ + struct xxhash64_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + struct xxhash64_desc_ctx *dctx = shash_desc_ctx(desc); + + xxh64_reset(&dctx->xxhstate, tctx->seed); + + return 0; +} + +static int xxhash64_update(struct shash_desc *desc, const u8 *data, + unsigned int length) +{ + struct xxhash64_desc_ctx *dctx = shash_desc_ctx(desc); + + xxh64_update(&dctx->xxhstate, data, length); + + return 0; +} + +static int xxhash64_final(struct shash_desc *desc, u8 *out) +{ + struct xxhash64_desc_ctx *dctx = shash_desc_ctx(desc); + + put_unaligned_le64(xxh64_digest(&dctx->xxhstate), out); + + return 0; +} + +static int xxhash64_digest(struct shash_desc *desc, const u8 *data, + unsigned int length, u8 *out) +{ + struct xxhash64_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + + put_unaligned_le64(xxh64(data, length, tctx->seed), out); + + return 0; +} + +static struct shash_alg alg = { + .digestsize = XXHASH64_DIGEST_SIZE, + .setkey = xxhash64_setkey, + .init = xxhash64_init, + .update = xxhash64_update, + .final = xxhash64_final, + .digest = xxhash64_digest, + .descsize = sizeof(struct xxhash64_desc_ctx), + .base = { + .cra_name = "xxhash64", + .cra_driver_name = "xxhash64-generic", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, + .cra_blocksize = XXHASH64_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct xxhash64_tfm_ctx), + .cra_module = THIS_MODULE, + } +}; + +static int __init xxhash_mod_init(void) +{ + return crypto_register_shash(&alg); +} + +static void __exit xxhash_mod_fini(void) +{ + crypto_unregister_shash(&alg); +} + +subsys_initcall(xxhash_mod_init); +module_exit(xxhash_mod_fini); + +MODULE_AUTHOR("Nikolay Borisov <nborisov@suse.com>"); +MODULE_DESCRIPTION("xxhash calculations wrapper for lib/xxhash.c"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS_CRYPTO("xxhash64"); +MODULE_ALIAS_CRYPTO("xxhash64-generic"); diff --git a/crypto/zstd.c b/crypto/zstd.c index f1e4c70c9d24..5a3ff258d8f7 100644 --- a/crypto/zstd.c +++ b/crypto/zstd.c @@ -206,6 +206,7 @@ static int zstd_sdecompress(struct crypto_scomp *tfm, const u8 *src, static struct crypto_alg alg = { .cra_name = "zstd", + .cra_driver_name = "zstd-generic", .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, .cra_ctxsize = sizeof(struct zstd_ctx), .cra_module = THIS_MODULE, |