From be95f0fa0cde56bb05ed62488a25a73a5b21b2d9 Mon Sep 17 00:00:00 2001 From: Svenning Sørensen Date: Fri, 5 Dec 2014 01:18:57 +0100 Subject: crypto: atmel_sha - remove unused shash fallback instance. The fallback is never used, so there is no point in having it. The cra_exit routine can also be removed, since all it did was releasing the fallback, along with the stub around cra_init, which just added an unused NULL argument. Signed-off-by: Svenning Soerensen Signed-off-by: Herbert Xu --- drivers/crypto/atmel-sha.c | 50 ++++++---------------------------------------- 1 file changed, 6 insertions(+), 44 deletions(-) diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index d94f07c78e19..34db04addc18 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -102,10 +102,6 @@ struct atmel_sha_ctx { struct atmel_sha_dev *dd; unsigned long flags; - - /* fallback stuff */ - struct crypto_shash *fallback; - }; #define ATMEL_SHA_QUEUE_LENGTH 50 @@ -974,19 +970,8 @@ static int atmel_sha_digest(struct ahash_request *req) return atmel_sha_init(req) ?: atmel_sha_finup(req); } -static int atmel_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) +static int atmel_sha_cra_init(struct crypto_tfm *tfm) { - struct atmel_sha_ctx *tctx = crypto_tfm_ctx(tfm); - const char *alg_name = crypto_tfm_alg_name(tfm); - - /* Allocate a fallback and abort if it failed. */ - tctx->fallback = crypto_alloc_shash(alg_name, 0, - CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(tctx->fallback)) { - pr_err("atmel-sha: fallback driver '%s' could not be loaded.\n", - alg_name); - return PTR_ERR(tctx->fallback); - } crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), sizeof(struct atmel_sha_reqctx) + SHA_BUFFER_LEN + SHA512_BLOCK_SIZE); @@ -994,19 +979,6 @@ static int atmel_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) return 0; } -static int atmel_sha_cra_init(struct crypto_tfm *tfm) -{ - return atmel_sha_cra_init_alg(tfm, NULL); -} - -static void atmel_sha_cra_exit(struct crypto_tfm *tfm) -{ - struct atmel_sha_ctx *tctx = crypto_tfm_ctx(tfm); - - crypto_free_shash(tctx->fallback); - tctx->fallback = NULL; -} - static struct ahash_alg sha_1_256_algs[] = { { .init = atmel_sha_init, @@ -1020,14 +992,12 @@ static struct ahash_alg sha_1_256_algs[] = { .cra_name = "sha1", .cra_driver_name = "atmel-sha1", .cra_priority = 100, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_sha_ctx), .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = atmel_sha_cra_init, - .cra_exit = atmel_sha_cra_exit, } } }, @@ -1043,14 +1013,12 @@ static struct ahash_alg sha_1_256_algs[] = { .cra_name = "sha256", .cra_driver_name = "atmel-sha256", .cra_priority = 100, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_sha_ctx), .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = atmel_sha_cra_init, - .cra_exit = atmel_sha_cra_exit, } } }, @@ -1068,14 +1036,12 @@ static struct ahash_alg sha_224_alg = { .cra_name = "sha224", .cra_driver_name = "atmel-sha224", .cra_priority = 100, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_sha_ctx), .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = atmel_sha_cra_init, - .cra_exit = atmel_sha_cra_exit, } } }; @@ -1093,14 +1059,12 @@ static struct ahash_alg sha_384_512_algs[] = { .cra_name = "sha384", .cra_driver_name = "atmel-sha384", .cra_priority = 100, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_sha_ctx), .cra_alignmask = 0x3, .cra_module = THIS_MODULE, .cra_init = atmel_sha_cra_init, - .cra_exit = atmel_sha_cra_exit, } } }, @@ -1116,14 +1080,12 @@ static struct ahash_alg sha_384_512_algs[] = { .cra_name = "sha512", .cra_driver_name = "atmel-sha512", .cra_priority = 100, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_sha_ctx), .cra_alignmask = 0x3, .cra_module = THIS_MODULE, .cra_init = atmel_sha_cra_init, - .cra_exit = atmel_sha_cra_exit, } } }, -- cgit v1.2.3-58-ga151 From a861afbc931b489b3b2362f8011cccd2a071ec37 Mon Sep 17 00:00:00 2001 From: "Joshua I. James" Date: Fri, 5 Dec 2014 14:06:16 +0900 Subject: crypto: ablkcipher - fixed style errors in ablkcipher.c Fixed style errors reported by checkpatch. WARNING: Missing a blank line after declarations + u8 *end_page = (u8 *)(((unsigned long)(start + len - 1)) & PAGE_MASK); + return max(start, end_page); WARNING: line over 80 characters + scatterwalk_start(&walk->out, scatterwalk_sg_next(walk->out.sg)); WARNING: Missing a blank line after declarations + int err = ablkcipher_copy_iv(walk, tfm, alignmask); + if (err) ERROR: do not use assignment in if condition + if ((err = crypto_register_instance(tmpl, inst))) { Signed-off-by: Joshua I. James Signed-off-by: Herbert Xu --- crypto/ablkcipher.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crypto/ablkcipher.c b/crypto/ablkcipher.c index 40886c489903..7bbc8b4ef2e9 100644 --- a/crypto/ablkcipher.c +++ b/crypto/ablkcipher.c @@ -69,6 +69,7 @@ static inline void ablkcipher_queue_write(struct ablkcipher_walk *walk, static inline u8 *ablkcipher_get_spot(u8 *start, unsigned int len) { u8 *end_page = (u8 *)(((unsigned long)(start + len - 1)) & PAGE_MASK); + return max(start, end_page); } @@ -86,7 +87,8 @@ static inline unsigned int ablkcipher_done_slow(struct ablkcipher_walk *walk, if (n == len_this_page) break; n -= len_this_page; - scatterwalk_start(&walk->out, scatterwalk_sg_next(walk->out.sg)); + scatterwalk_start(&walk->out, scatterwalk_sg_next( + walk->out.sg)); } return bsize; @@ -284,6 +286,7 @@ static int ablkcipher_walk_first(struct ablkcipher_request *req, walk->iv = req->info; if (unlikely(((unsigned long)walk->iv & alignmask))) { int err = ablkcipher_copy_iv(walk, tfm, alignmask); + if (err) return err; } @@ -589,7 +592,8 @@ static int crypto_givcipher_default(struct crypto_alg *alg, u32 type, u32 mask) if (IS_ERR(inst)) goto put_tmpl; - if ((err = crypto_register_instance(tmpl, inst))) { + err = crypto_register_instance(tmpl, inst); + if (err) { tmpl->free(inst); goto put_tmpl; } -- cgit v1.2.3-58-ga151 From 4fad478ae0ff7bd27c853a085555317c0dc68704 Mon Sep 17 00:00:00 2001 From: "Joshua I. James" Date: Fri, 5 Dec 2014 14:24:44 +0900 Subject: crypto: aead - fixed style error in aead.c Fixed style error identified by checkpatch. ERROR: do not use assignment in if condition + if ((err = crypto_register_instance(tmpl, inst))) { Signed-off-by: Joshua I. James Signed-off-by: Herbert Xu --- crypto/aead.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crypto/aead.c b/crypto/aead.c index 547491e35c63..222271070b49 100644 --- a/crypto/aead.c +++ b/crypto/aead.c @@ -448,7 +448,8 @@ static int crypto_nivaead_default(struct crypto_alg *alg, u32 type, u32 mask) if (IS_ERR(inst)) goto put_tmpl; - if ((err = crypto_register_instance(tmpl, inst))) { + err = crypto_register_instance(tmpl, inst); + if (err) { tmpl->free(inst); goto put_tmpl; } -- cgit v1.2.3-58-ga151 From 267c4221ff574ac70ec0b02b923b16f39b54da1a Mon Sep 17 00:00:00 2001 From: "Joshua I. James" Date: Fri, 5 Dec 2014 14:38:40 +0900 Subject: crypto: af_alg - fixed style error in af_alg.c Fixed style error identified by checkpatch. ERROR: space required before the open parenthesis '(' + switch(cmsg->cmsg_type) { Signed-off-by: Joshua I. James Signed-off-by: Herbert Xu --- crypto/af_alg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/af_alg.c b/crypto/af_alg.c index 4665b79c729a..8ffc174a0154 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -405,7 +405,7 @@ int af_alg_cmsg_send(struct msghdr *msg, struct af_alg_control *con) if (cmsg->cmsg_level != SOL_ALG) continue; - switch(cmsg->cmsg_type) { + switch (cmsg->cmsg_type) { case ALG_SET_IV: if (cmsg->cmsg_len < CMSG_LEN(sizeof(*con->iv))) return -EINVAL; -- cgit v1.2.3-58-ga151 From b516d514020f17c83267f76366691e4cc9b7bddf Mon Sep 17 00:00:00 2001 From: "Joshua I. James" Date: Fri, 5 Dec 2014 14:44:54 +0900 Subject: crypto: ahash - fixed style error in ahash.c Fixed style error identified by checkpatch. WARNING: Missing a blank line after declarations + unsigned int unaligned = alignmask + 1 - (offset & alignmask); + if (nbytes > unaligned) Signed-off-by: Joshua I. James Signed-off-by: Herbert Xu --- crypto/ahash.c | 1 + 1 file changed, 1 insertion(+) diff --git a/crypto/ahash.c b/crypto/ahash.c index f6a36a52d738..dd2890608aeb 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -55,6 +55,7 @@ static int hash_walk_next(struct crypto_hash_walk *walk) if (offset & alignmask) { unsigned int unaligned = alignmask + 1 - (offset & alignmask); + if (nbytes > unaligned) nbytes = unaligned; } -- cgit v1.2.3-58-ga151 From 0efcb8d5b2f7af86818179810cc080b326a83e19 Mon Sep 17 00:00:00 2001 From: "Joshua I. James" Date: Fri, 5 Dec 2014 15:00:10 +0900 Subject: crypto: api - fixed style erro in algapi.c Fixed style error identified by checkpatch. WARNING: Missing a blank line after declarations + int err = crypto_remove_alg(&inst->alg, &users); + BUG_ON(err); Signed-off-by: Joshua I. James Signed-off-by: Herbert Xu --- crypto/algapi.c | 1 + 1 file changed, 1 insertion(+) diff --git a/crypto/algapi.c b/crypto/algapi.c index 71a8143e23b1..83b04e0884b1 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -473,6 +473,7 @@ void crypto_unregister_template(struct crypto_template *tmpl) list = &tmpl->instances; hlist_for_each_entry(inst, list, list) { int err = crypto_remove_alg(&inst->alg, &users); + BUG_ON(err); } -- cgit v1.2.3-58-ga151 From 905b42e559fa4952569b3444bc6c054c0103e5a0 Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Fri, 5 Dec 2014 22:40:21 +0100 Subject: crypto: drbg - panic on continuous self test error This patch adds a panic if the FIPS 140-2 self test error failed. Note, that entire code is only executed with fips_enabled (i.e. when the kernel is booted with fips=1. It is therefore not executed for 99.9% of all user base. As mathematically such failure cannot occur, this panic should never be triggered. But to comply with NISTs current requirements, an endless loop must be replaced with the panic. When the new version of FIPS 140 will be released, this entire continuous self test function will be ripped out as it will not be needed any more. This patch is functionally equivalent as implemented in ansi_cprng.c and drivers/char/random.c. Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/drbg.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/crypto/drbg.c b/crypto/drbg.c index d748a1d0ca24..96138396ce01 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -223,15 +223,6 @@ static inline unsigned short drbg_sec_strength(drbg_flag_t flags) * function. Thus, the function implicitly knows the size of the * buffer. * - * The FIPS test can be called in an endless loop until it returns - * true. Although the code looks like a potential for a deadlock, it - * is not the case, because returning a false cannot mathematically - * occur (except once when a reseed took place and the updated state - * would is now set up such that the generation of new value returns - * an identical one -- this is most unlikely and would happen only once). - * Thus, if this function repeatedly returns false and thus would cause - * a deadlock, the integrity of the entire kernel is lost. - * * @drbg DRBG handle * @buf output buffer of random data to be checked * @@ -258,6 +249,8 @@ static bool drbg_fips_continuous_test(struct drbg_state *drbg, return false; } ret = memcmp(drbg->prev, buf, drbg_blocklen(drbg)); + if (!ret) + panic("DRBG continuous self test failed\n"); memcpy(drbg->prev, buf, drbg_blocklen(drbg)); /* the test shall pass when the two compared values are not equal */ return ret != 0; -- cgit v1.2.3-58-ga151 From 25fb8638e919bc7431a73f2fb4a9713818ae2c9d Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Sun, 7 Dec 2014 23:21:42 +0100 Subject: crypto: af_alg - add setsockopt for auth tag size Use setsockopt on the tfm FD to provide the authentication tag size for an AEAD cipher. This is achieved by adding a callback function which is intended to be used by the AEAD AF_ALG implementation. The optlen argument of the setsockopt specifies the authentication tag size to be used with the AEAD tfm. Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/af_alg.c | 7 +++++++ include/crypto/if_alg.h | 1 + 2 files changed, 8 insertions(+) diff --git a/crypto/af_alg.c b/crypto/af_alg.c index 8ffc174a0154..a8ff3c44e13c 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -215,6 +215,13 @@ static int alg_setsockopt(struct socket *sock, int level, int optname, goto unlock; err = alg_setkey(sk, optval, optlen); + break; + case ALG_SET_AEAD_AUTHSIZE: + if (sock->state == SS_CONNECTED) + goto unlock; + if (!type->setauthsize) + goto unlock; + err = type->setauthsize(ask->private, optlen); } unlock: diff --git a/include/crypto/if_alg.h b/include/crypto/if_alg.h index cd62bf4289e9..5c7b6c53e96f 100644 --- a/include/crypto/if_alg.h +++ b/include/crypto/if_alg.h @@ -50,6 +50,7 @@ struct af_alg_type { void (*release)(void *private); int (*setkey)(void *private, const u8 *key, unsigned int keylen); int (*accept)(void *private, struct sock *sk); + int (*setauthsize)(void *private, unsigned int authsize); struct proto_ops *ops; struct module *owner; -- cgit v1.2.3-58-ga151 From 9372b35e11149c5314f56f939775e67d83057604 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 8 Dec 2014 16:50:35 +0800 Subject: hwrng: place mutex around read functions and buffers. There's currently a big lock around everything, and it means that we can't query sysfs (eg /sys/devices/virtual/misc/hw_random/rng_current) while the rng is reading. This is a real problem when the rng is slow, or blocked (eg. virtio_rng with qemu's default /dev/random backend) This doesn't help (it leaves the current lock untouched), just adds a lock to protect the read function and the static buffers, in preparation for transition. Signed-off-by: Rusty Russell Signed-off-by: Herbert Xu --- drivers/char/hw_random/core.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 1500cfd799a7..2dd3144d69b1 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -53,7 +53,10 @@ static struct hwrng *current_rng; static struct task_struct *hwrng_fill; static LIST_HEAD(rng_list); +/* Protects rng_list and current_rng */ static DEFINE_MUTEX(rng_mutex); +/* Protects rng read functions, data_avail, rng_buffer and rng_fillbuf */ +static DEFINE_MUTEX(reading_mutex); static int data_avail; static u8 *rng_buffer, *rng_fillbuf; static unsigned short current_quality; @@ -81,7 +84,9 @@ static void add_early_randomness(struct hwrng *rng) unsigned char bytes[16]; int bytes_read; + mutex_lock(&reading_mutex); bytes_read = rng_get_data(rng, bytes, sizeof(bytes), 1); + mutex_unlock(&reading_mutex); if (bytes_read > 0) add_device_randomness(bytes, bytes_read); } @@ -128,6 +133,7 @@ static inline int rng_get_data(struct hwrng *rng, u8 *buffer, size_t size, int wait) { int present; + BUG_ON(!mutex_is_locked(&reading_mutex)); if (rng->read) return rng->read(rng, (void *)buffer, size, wait); @@ -160,13 +166,14 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf, goto out_unlock; } + mutex_lock(&reading_mutex); if (!data_avail) { bytes_read = rng_get_data(current_rng, rng_buffer, rng_buffer_size(), !(filp->f_flags & O_NONBLOCK)); if (bytes_read < 0) { err = bytes_read; - goto out_unlock; + goto out_unlock_reading; } data_avail = bytes_read; } @@ -174,7 +181,7 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf, if (!data_avail) { if (filp->f_flags & O_NONBLOCK) { err = -EAGAIN; - goto out_unlock; + goto out_unlock_reading; } } else { len = data_avail; @@ -186,7 +193,7 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf, if (copy_to_user(buf + ret, rng_buffer + data_avail, len)) { err = -EFAULT; - goto out_unlock; + goto out_unlock_reading; } size -= len; @@ -194,6 +201,7 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf, } mutex_unlock(&rng_mutex); + mutex_unlock(&reading_mutex); if (need_resched()) schedule_timeout_interruptible(1); @@ -208,6 +216,9 @@ out: out_unlock: mutex_unlock(&rng_mutex); goto out; +out_unlock_reading: + mutex_unlock(&reading_mutex); + goto out_unlock; } @@ -344,13 +355,16 @@ static int hwrng_fillfn(void *unused) while (!kthread_should_stop()) { if (!current_rng) break; + mutex_lock(&reading_mutex); rc = rng_get_data(current_rng, rng_fillbuf, rng_buffer_size(), 1); + mutex_unlock(&reading_mutex); if (rc <= 0) { pr_warn("hwrng: no data available\n"); msleep_interruptible(10000); continue; } + /* Outside lock, sure, but y'know: randomness. */ add_hwgenerator_randomness((void *)rng_fillbuf, rc, rc * current_quality * 8 >> 10); } -- cgit v1.2.3-58-ga151 From 1dacb395d68a14825ee48c0843335e3181aea675 Mon Sep 17 00:00:00 2001 From: Amos Kong Date: Mon, 8 Dec 2014 16:50:36 +0800 Subject: hwrng: move some code out mutex_lock for avoiding underlying deadlock In next patch, we use reference counting for each struct hwrng, changing reference count also needs to take mutex_lock. Before releasing the lock, if we try to stop a kthread that waits to take the lock to reduce the referencing count, deadlock will occur. Signed-off-by: Amos Kong Signed-off-by: Herbert Xu --- drivers/char/hw_random/core.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 2dd3144d69b1..b4c0e873d362 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -470,12 +470,12 @@ void hwrng_unregister(struct hwrng *rng) } } if (list_empty(&rng_list)) { + mutex_unlock(&rng_mutex); unregister_miscdev(); if (hwrng_fill) kthread_stop(hwrng_fill); - } - - mutex_unlock(&rng_mutex); + } else + mutex_unlock(&rng_mutex); } EXPORT_SYMBOL_GPL(hwrng_unregister); -- cgit v1.2.3-58-ga151 From 3a2c0ba5ad00c018c0bef39a2224aca950aa33f2 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 8 Dec 2014 16:50:37 +0800 Subject: hwrng: use reference counts on each struct hwrng. current_rng holds one reference, and we bump it every time we want to do a read from it. This means we only hold the rng_mutex to grab or drop a reference, so accessing /sys/devices/virtual/misc/hw_random/rng_current doesn't block on read of /dev/hwrng. Using a kref is overkill (we're always under the rng_mutex), but a standard pattern. This also solves the problem that the hwrng_fillfn thread was accessing current_rng without a lock, which could change (eg. to NULL) underneath it. Signed-off-by: Rusty Russell Signed-off-by: Amos Kong Signed-off-by: Herbert Xu --- drivers/char/hw_random/core.c | 135 ++++++++++++++++++++++++++++-------------- include/linux/hw_random.h | 2 + 2 files changed, 94 insertions(+), 43 deletions(-) diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index b4c0e873d362..089c18dc579e 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -42,6 +42,7 @@ #include #include #include +#include #include @@ -91,6 +92,60 @@ static void add_early_randomness(struct hwrng *rng) add_device_randomness(bytes, bytes_read); } +static inline void cleanup_rng(struct kref *kref) +{ + struct hwrng *rng = container_of(kref, struct hwrng, ref); + + if (rng->cleanup) + rng->cleanup(rng); +} + +static void set_current_rng(struct hwrng *rng) +{ + BUG_ON(!mutex_is_locked(&rng_mutex)); + kref_get(&rng->ref); + current_rng = rng; +} + +static void drop_current_rng(void) +{ + BUG_ON(!mutex_is_locked(&rng_mutex)); + if (!current_rng) + return; + + /* decrease last reference for triggering the cleanup */ + kref_put(¤t_rng->ref, cleanup_rng); + current_rng = NULL; +} + +/* Returns ERR_PTR(), NULL or refcounted hwrng */ +static struct hwrng *get_current_rng(void) +{ + struct hwrng *rng; + + if (mutex_lock_interruptible(&rng_mutex)) + return ERR_PTR(-ERESTARTSYS); + + rng = current_rng; + if (rng) + kref_get(&rng->ref); + + mutex_unlock(&rng_mutex); + return rng; +} + +static void put_rng(struct hwrng *rng) +{ + /* + * Hold rng_mutex here so we serialize in case they set_current_rng + * on rng again immediately. + */ + mutex_lock(&rng_mutex); + if (rng) + kref_put(&rng->ref, cleanup_rng); + mutex_unlock(&rng_mutex); +} + static inline int hwrng_init(struct hwrng *rng) { if (rng->init) { @@ -113,12 +168,6 @@ static inline int hwrng_init(struct hwrng *rng) return 0; } -static inline void hwrng_cleanup(struct hwrng *rng) -{ - if (rng && rng->cleanup) - rng->cleanup(rng); -} - static int rng_dev_open(struct inode *inode, struct file *filp) { /* enforce read-only access to this chrdev */ @@ -154,21 +203,22 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf, ssize_t ret = 0; int err = 0; int bytes_read, len; + struct hwrng *rng; while (size) { - if (mutex_lock_interruptible(&rng_mutex)) { - err = -ERESTARTSYS; + rng = get_current_rng(); + if (IS_ERR(rng)) { + err = PTR_ERR(rng); goto out; } - - if (!current_rng) { + if (!rng) { err = -ENODEV; - goto out_unlock; + goto out; } mutex_lock(&reading_mutex); if (!data_avail) { - bytes_read = rng_get_data(current_rng, rng_buffer, + bytes_read = rng_get_data(rng, rng_buffer, rng_buffer_size(), !(filp->f_flags & O_NONBLOCK)); if (bytes_read < 0) { @@ -200,8 +250,8 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf, ret += len; } - mutex_unlock(&rng_mutex); mutex_unlock(&reading_mutex); + put_rng(rng); if (need_resched()) schedule_timeout_interruptible(1); @@ -213,12 +263,11 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf, } out: return ret ? : err; -out_unlock: - mutex_unlock(&rng_mutex); - goto out; + out_unlock_reading: mutex_unlock(&reading_mutex); - goto out_unlock; + put_rng(rng); + goto out; } @@ -257,8 +306,8 @@ static ssize_t hwrng_attr_current_store(struct device *dev, err = hwrng_init(rng); if (err) break; - hwrng_cleanup(current_rng); - current_rng = rng; + drop_current_rng(); + set_current_rng(rng); err = 0; break; } @@ -272,17 +321,15 @@ static ssize_t hwrng_attr_current_show(struct device *dev, struct device_attribute *attr, char *buf) { - int err; ssize_t ret; - const char *name = "none"; + struct hwrng *rng; - err = mutex_lock_interruptible(&rng_mutex); - if (err) - return -ERESTARTSYS; - if (current_rng) - name = current_rng->name; - ret = snprintf(buf, PAGE_SIZE, "%s\n", name); - mutex_unlock(&rng_mutex); + rng = get_current_rng(); + if (IS_ERR(rng)) + return PTR_ERR(rng); + + ret = snprintf(buf, PAGE_SIZE, "%s\n", rng ? rng->name : "none"); + put_rng(rng); return ret; } @@ -353,12 +400,16 @@ static int hwrng_fillfn(void *unused) long rc; while (!kthread_should_stop()) { - if (!current_rng) + struct hwrng *rng; + + rng = get_current_rng(); + if (IS_ERR(rng) || !rng) break; mutex_lock(&reading_mutex); - rc = rng_get_data(current_rng, rng_fillbuf, + rc = rng_get_data(rng, rng_fillbuf, rng_buffer_size(), 1); mutex_unlock(&reading_mutex); + put_rng(rng); if (rc <= 0) { pr_warn("hwrng: no data available\n"); msleep_interruptible(10000); @@ -419,14 +470,13 @@ int hwrng_register(struct hwrng *rng) err = hwrng_init(rng); if (err) goto out_unlock; - current_rng = rng; + set_current_rng(rng); } err = 0; if (!old_rng) { err = register_miscdev(); if (err) { - hwrng_cleanup(rng); - current_rng = NULL; + drop_current_rng(); goto out_unlock; } } @@ -453,22 +503,21 @@ EXPORT_SYMBOL_GPL(hwrng_register); void hwrng_unregister(struct hwrng *rng) { - int err; - mutex_lock(&rng_mutex); list_del(&rng->list); if (current_rng == rng) { - hwrng_cleanup(rng); - if (list_empty(&rng_list)) { - current_rng = NULL; - } else { - current_rng = list_entry(rng_list.prev, struct hwrng, list); - err = hwrng_init(current_rng); - if (err) - current_rng = NULL; + drop_current_rng(); + if (!list_empty(&rng_list)) { + struct hwrng *tail; + + tail = list_entry(rng_list.prev, struct hwrng, list); + + if (hwrng_init(tail) == 0) + set_current_rng(tail); } } + if (list_empty(&rng_list)) { mutex_unlock(&rng_mutex); unregister_miscdev(); diff --git a/include/linux/hw_random.h b/include/linux/hw_random.h index 914bb08cd738..c212e71ea886 100644 --- a/include/linux/hw_random.h +++ b/include/linux/hw_random.h @@ -14,6 +14,7 @@ #include #include +#include /** * struct hwrng - Hardware Random Number Generator driver @@ -44,6 +45,7 @@ struct hwrng { /* internal. */ struct list_head list; + struct kref ref; }; /** Register a new Hardware Random Number Generator driver. */ -- cgit v1.2.3-58-ga151 From a027f30d72f2c4d27d6dd9bd053205d3102de7d1 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 8 Dec 2014 16:50:38 +0800 Subject: hwrng: fix unregister race. The previous patch added one potential problem: we can still be reading from a hwrng when it's unregistered. Add a wait for zero in the hwrng_unregister path. Signed-off-by: Rusty Russell Signed-off-by: Amos Kong Signed-off-by: Herbert Xu --- drivers/char/hw_random/core.c | 12 ++++++++++++ include/linux/hw_random.h | 1 + 2 files changed, 13 insertions(+) diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 089c18dc579e..8d609a026465 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -60,6 +60,7 @@ static DEFINE_MUTEX(rng_mutex); static DEFINE_MUTEX(reading_mutex); static int data_avail; static u8 *rng_buffer, *rng_fillbuf; +static DECLARE_WAIT_QUEUE_HEAD(rng_done); static unsigned short current_quality; static unsigned short default_quality; /* = 0; default to "off" */ @@ -98,6 +99,11 @@ static inline void cleanup_rng(struct kref *kref) if (rng->cleanup) rng->cleanup(rng); + + /* cleanup_done should be updated after cleanup finishes */ + smp_wmb(); + rng->cleanup_done = true; + wake_up_all(&rng_done); } static void set_current_rng(struct hwrng *rng) @@ -494,6 +500,8 @@ int hwrng_register(struct hwrng *rng) add_early_randomness(rng); } + rng->cleanup_done = false; + out_unlock: mutex_unlock(&rng_mutex); out: @@ -525,6 +533,10 @@ void hwrng_unregister(struct hwrng *rng) kthread_stop(hwrng_fill); } else mutex_unlock(&rng_mutex); + + /* Just in case rng is reading right now, wait. */ + wait_event(rng_done, rng->cleanup_done && + atomic_read(&rng->ref.refcount) == 0); } EXPORT_SYMBOL_GPL(hwrng_unregister); diff --git a/include/linux/hw_random.h b/include/linux/hw_random.h index c212e71ea886..7832e5008959 100644 --- a/include/linux/hw_random.h +++ b/include/linux/hw_random.h @@ -46,6 +46,7 @@ struct hwrng { /* internal. */ struct list_head list; struct kref ref; + bool cleanup_done; }; /** Register a new Hardware Random Number Generator driver. */ -- cgit v1.2.3-58-ga151 From ebbbfa248389b176e2e62d8cf91814253849ccc9 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 8 Dec 2014 16:50:39 +0800 Subject: hwrng: don't double-check old_rng. Interesting anti-pattern. Signed-off-by: Rusty Russell Signed-off-by: Herbert Xu --- drivers/char/hw_random/core.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 8d609a026465..e384ee3ed604 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -472,14 +472,13 @@ int hwrng_register(struct hwrng *rng) } old_rng = current_rng; + err = 0; if (!old_rng) { err = hwrng_init(rng); if (err) goto out_unlock; set_current_rng(rng); - } - err = 0; - if (!old_rng) { + err = register_miscdev(); if (err) { drop_current_rng(); -- cgit v1.2.3-58-ga151 From 2d2ec0642a85966b6a299bbcd94707982327ace8 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 8 Dec 2014 16:50:40 +0800 Subject: hwrng: don't init list element we're about to add to list. Another interesting anti-pattern. Signed-off-by: Rusty Russell Signed-off-by: Herbert Xu --- drivers/char/hw_random/core.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index e384ee3ed604..6ec42252e46e 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -485,7 +485,6 @@ int hwrng_register(struct hwrng *rng) goto out_unlock; } } - INIT_LIST_HEAD(&rng->list); list_add_tail(&rng->list, &rng_list); if (old_rng && !rng->init) { -- cgit v1.2.3-58-ga151 From 0f477b655a524515ec9a263d70d51f460c05a161 Mon Sep 17 00:00:00 2001 From: Tadeusz Struk Date: Mon, 8 Dec 2014 12:03:42 -0800 Subject: crypto: algif - Mark sgl end at the end of data algif_skcipher sends 127 sgl buffers for encryption regardless of how many buffers acctually have data to process, where the few first with valid len and the rest with zero len. This is not very eficient. This patch marks the last one with data as the last one to process. Signed-off-by: Tadeusz Struk Signed-off-by: Herbert Xu --- crypto/algif_skcipher.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c index c12207c8dde9..38a6757e3ccc 100644 --- a/crypto/algif_skcipher.c +++ b/crypto/algif_skcipher.c @@ -330,6 +330,7 @@ static int skcipher_sendmsg(struct kiocb *unused, struct socket *sock, sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list); sg = sgl->sg; + sg_unmark_end(sg + sgl->cur); do { i = sgl->cur; plen = min_t(int, len, PAGE_SIZE); @@ -355,6 +356,9 @@ static int skcipher_sendmsg(struct kiocb *unused, struct socket *sock, sgl->cur++; } while (len && sgl->cur < MAX_SGL_ENTS); + if (!size) + sg_mark_end(sg + sgl->cur - 1); + ctx->merge = plen & (PAGE_SIZE - 1); } @@ -401,6 +405,10 @@ static ssize_t skcipher_sendpage(struct socket *sock, struct page *page, ctx->merge = 0; sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list); + if (sgl->cur) + sg_unmark_end(sgl->sg + sgl->cur - 1); + + sg_mark_end(sgl->sg + sgl->cur); get_page(page); sg_set_page(sgl->sg + sgl->cur, page, size, offset); sgl->cur++; -- cgit v1.2.3-58-ga151 From 82f82504b8f5f1013678bbc74e0882891114594a Mon Sep 17 00:00:00 2001 From: Tadeusz Struk Date: Mon, 8 Dec 2014 12:05:42 -0800 Subject: crypto: qat - Fix assumption that sg in and out will have the same nents Fixed invalid assumpion that the sgl in and sgl out will always have the same number of entries. Signed-off-by: Tadeusz Struk Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_algs.c | 82 ++++++++++++++++++------------ drivers/crypto/qat/qat_common/qat_crypto.h | 1 + 2 files changed, 50 insertions(+), 33 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index 19eea1c832ac..e4e32d872902 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -557,7 +557,8 @@ static void qat_alg_free_bufl(struct qat_crypto_instance *inst, dma_addr_t blp = qat_req->buf.blp; dma_addr_t blpout = qat_req->buf.bloutp; size_t sz = qat_req->buf.sz; - int i, bufs = bl->num_bufs; + size_t sz_out = qat_req->buf.sz_out; + int i; for (i = 0; i < bl->num_bufs; i++) dma_unmap_single(dev, bl->bufers[i].addr, @@ -567,14 +568,14 @@ static void qat_alg_free_bufl(struct qat_crypto_instance *inst, kfree(bl); if (blp != blpout) { /* If out of place operation dma unmap only data */ - int bufless = bufs - blout->num_mapped_bufs; + int bufless = blout->num_bufs - blout->num_mapped_bufs; - for (i = bufless; i < bufs; i++) { + for (i = bufless; i < blout->num_bufs; i++) { dma_unmap_single(dev, blout->bufers[i].addr, blout->bufers[i].len, DMA_BIDIRECTIONAL); } - dma_unmap_single(dev, blpout, sz, DMA_TO_DEVICE); + dma_unmap_single(dev, blpout, sz_out, DMA_TO_DEVICE); kfree(blout); } } @@ -587,19 +588,20 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, struct qat_crypto_request *qat_req) { struct device *dev = &GET_DEV(inst->accel_dev); - int i, bufs = 0, n = sg_nents(sgl), assoc_n = sg_nents(assoc); + int i, bufs = 0, sg_nctr = 0; + int n = sg_nents(sgl), assoc_n = sg_nents(assoc); struct qat_alg_buf_list *bufl; struct qat_alg_buf_list *buflout = NULL; dma_addr_t blp; dma_addr_t bloutp = 0; struct scatterlist *sg; - size_t sz = sizeof(struct qat_alg_buf_list) + + size_t sz_out, sz = sizeof(struct qat_alg_buf_list) + ((1 + n + assoc_n) * sizeof(struct qat_alg_buf)); if (unlikely(!n)) return -EINVAL; - bufl = kmalloc_node(sz, GFP_ATOMIC, + bufl = kzalloc_node(sz, GFP_ATOMIC, dev_to_node(&GET_DEV(inst->accel_dev))); if (unlikely(!bufl)) return -ENOMEM; @@ -620,15 +622,20 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, goto err; bufs++; } - bufl->bufers[bufs].addr = dma_map_single(dev, iv, ivlen, - DMA_BIDIRECTIONAL); - bufl->bufers[bufs].len = ivlen; - if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr))) - goto err; - bufs++; + if (ivlen) { + bufl->bufers[bufs].addr = dma_map_single(dev, iv, ivlen, + DMA_BIDIRECTIONAL); + bufl->bufers[bufs].len = ivlen; + if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr))) + goto err; + bufs++; + } for_each_sg(sgl, sg, n, i) { - int y = i + bufs; + int y = sg_nctr + bufs; + + if (!sg->length) + continue; bufl->bufers[y].addr = dma_map_single(dev, sg_virt(sg), sg->length, @@ -636,8 +643,9 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, bufl->bufers[y].len = sg->length; if (unlikely(dma_mapping_error(dev, bufl->bufers[y].addr))) goto err; + sg_nctr++; } - bufl->num_bufs = n + bufs; + bufl->num_bufs = sg_nctr + bufs; qat_req->buf.bl = bufl; qat_req->buf.blp = blp; qat_req->buf.sz = sz; @@ -645,11 +653,15 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, if (sgl != sglout) { struct qat_alg_buf *bufers; - buflout = kmalloc_node(sz, GFP_ATOMIC, + n = sg_nents(sglout); + sz_out = sizeof(struct qat_alg_buf_list) + + ((1 + n + assoc_n) * sizeof(struct qat_alg_buf)); + sg_nctr = 0; + buflout = kzalloc_node(sz_out, GFP_ATOMIC, dev_to_node(&GET_DEV(inst->accel_dev))); if (unlikely(!buflout)) goto err; - bloutp = dma_map_single(dev, buflout, sz, DMA_TO_DEVICE); + bloutp = dma_map_single(dev, buflout, sz_out, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, bloutp))) goto err; bufers = buflout->bufers; @@ -660,47 +672,51 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, bufers[i].addr = bufl->bufers[i].addr; } for_each_sg(sglout, sg, n, i) { - int y = i + bufs; + int y = sg_nctr + bufs; + + if (!sg->length) + continue; bufers[y].addr = dma_map_single(dev, sg_virt(sg), sg->length, DMA_BIDIRECTIONAL); - buflout->bufers[y].len = sg->length; if (unlikely(dma_mapping_error(dev, bufers[y].addr))) goto err; + bufers[y].len = sg->length; + sg_nctr++; } - buflout->num_bufs = n + bufs; - buflout->num_mapped_bufs = n; + buflout->num_bufs = sg_nctr + bufs; + buflout->num_mapped_bufs = sg_nctr; qat_req->buf.blout = buflout; qat_req->buf.bloutp = bloutp; + qat_req->buf.sz_out = sz_out; } else { /* Otherwise set the src and dst to the same address */ qat_req->buf.bloutp = qat_req->buf.blp; + qat_req->buf.sz_out = 0; } return 0; err: dev_err(dev, "Failed to map buf for dma\n"); - for_each_sg(sgl, sg, n + bufs, i) { - if (!dma_mapping_error(dev, bufl->bufers[i].addr)) { + sg_nctr = 0; + for (i = 0; i < n + bufs; i++) + if (!dma_mapping_error(dev, bufl->bufers[i].addr)) dma_unmap_single(dev, bufl->bufers[i].addr, bufl->bufers[i].len, DMA_BIDIRECTIONAL); - } - } + if (!dma_mapping_error(dev, blp)) dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE); kfree(bufl); if (sgl != sglout && buflout) { - for_each_sg(sglout, sg, n, i) { - int y = i + bufs; - - if (!dma_mapping_error(dev, buflout->bufers[y].addr)) - dma_unmap_single(dev, buflout->bufers[y].addr, - buflout->bufers[y].len, + n = sg_nents(sglout); + for (i = bufs; i < n + bufs; i++) + if (!dma_mapping_error(dev, buflout->bufers[i].addr)) + dma_unmap_single(dev, buflout->bufers[i].addr, + buflout->bufers[i].len, DMA_BIDIRECTIONAL); - } if (!dma_mapping_error(dev, bloutp)) - dma_unmap_single(dev, bloutp, sz, DMA_TO_DEVICE); + dma_unmap_single(dev, bloutp, sz_out, DMA_TO_DEVICE); kfree(buflout); } return -ENOMEM; diff --git a/drivers/crypto/qat/qat_common/qat_crypto.h b/drivers/crypto/qat/qat_common/qat_crypto.h index ab8468d11ddb..fcb323116e60 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.h +++ b/drivers/crypto/qat/qat_common/qat_crypto.h @@ -72,6 +72,7 @@ struct qat_crypto_request_buffs { struct qat_alg_buf_list *blout; dma_addr_t bloutp; size_t sz; + size_t sz_out; }; struct qat_crypto_request { -- cgit v1.2.3-58-ga151 From 338e84f3a9740ab3582c8b6bc5a1a027794dac72 Mon Sep 17 00:00:00 2001 From: Tadeusz Struk Date: Mon, 8 Dec 2014 12:08:49 -0800 Subject: crypto: qat - add support for cbc(aes) ablkcipher Add support for cbc(aes) ablkcipher. Signed-off-by: Tadeusz Struk Acked-by: Bruce W. Allan Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/icp_qat_hw.h | 2 +- drivers/crypto/qat/qat_common/qat_algs.c | 528 +++++++++++++++++++++++------ drivers/crypto/qat/qat_common/qat_crypto.h | 15 +- 3 files changed, 433 insertions(+), 112 deletions(-) diff --git a/drivers/crypto/qat/qat_common/icp_qat_hw.h b/drivers/crypto/qat/qat_common/icp_qat_hw.h index 5031f8c10d75..68f191b653b0 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hw.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hw.h @@ -301,5 +301,5 @@ struct icp_qat_hw_cipher_aes256_f8 { struct icp_qat_hw_cipher_algo_blk { struct icp_qat_hw_cipher_aes256_f8 aes; -}; +} __aligned(64); #endif diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index e4e32d872902..f32d0a58bcc0 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -63,15 +63,15 @@ #include "icp_qat_fw.h" #include "icp_qat_fw_la.h" -#define QAT_AES_HW_CONFIG_ENC(alg) \ +#define QAT_AES_HW_CONFIG_CBC_ENC(alg) \ ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \ - ICP_QAT_HW_CIPHER_NO_CONVERT, \ - ICP_QAT_HW_CIPHER_ENCRYPT) + ICP_QAT_HW_CIPHER_NO_CONVERT, \ + ICP_QAT_HW_CIPHER_ENCRYPT) -#define QAT_AES_HW_CONFIG_DEC(alg) \ +#define QAT_AES_HW_CONFIG_CBC_DEC(alg) \ ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \ - ICP_QAT_HW_CIPHER_KEY_CONVERT, \ - ICP_QAT_HW_CIPHER_DECRYPT) + ICP_QAT_HW_CIPHER_KEY_CONVERT, \ + ICP_QAT_HW_CIPHER_DECRYPT) static atomic_t active_dev; @@ -108,19 +108,31 @@ struct qat_auth_state { uint8_t data[MAX_AUTH_STATE_SIZE + 64]; } __aligned(64); -struct qat_alg_session_ctx { +struct qat_alg_aead_ctx { struct qat_alg_cd *enc_cd; - dma_addr_t enc_cd_paddr; struct qat_alg_cd *dec_cd; + dma_addr_t enc_cd_paddr; dma_addr_t dec_cd_paddr; - struct icp_qat_fw_la_bulk_req enc_fw_req_tmpl; - struct icp_qat_fw_la_bulk_req dec_fw_req_tmpl; - struct qat_crypto_instance *inst; - struct crypto_tfm *tfm; + struct icp_qat_fw_la_bulk_req enc_fw_req; + struct icp_qat_fw_la_bulk_req dec_fw_req; struct crypto_shash *hash_tfm; enum icp_qat_hw_auth_algo qat_hash_alg; + struct qat_crypto_instance *inst; + struct crypto_tfm *tfm; uint8_t salt[AES_BLOCK_SIZE]; - spinlock_t lock; /* protects qat_alg_session_ctx struct */ + spinlock_t lock; /* protects qat_alg_aead_ctx struct */ +}; + +struct qat_alg_ablkcipher_ctx { + struct icp_qat_hw_cipher_algo_blk *enc_cd; + struct icp_qat_hw_cipher_algo_blk *dec_cd; + dma_addr_t enc_cd_paddr; + dma_addr_t dec_cd_paddr; + struct icp_qat_fw_la_bulk_req enc_fw_req; + struct icp_qat_fw_la_bulk_req dec_fw_req; + struct qat_crypto_instance *inst; + struct crypto_tfm *tfm; + spinlock_t lock; /* protects qat_alg_ablkcipher_ctx struct */ }; static int get_current_node(void) @@ -144,7 +156,7 @@ static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg) } static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash, - struct qat_alg_session_ctx *ctx, + struct qat_alg_aead_ctx *ctx, const uint8_t *auth_key, unsigned int auth_keylen) { @@ -267,8 +279,6 @@ static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header) header->comn_req_flags = ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, QAT_COMN_PTR_TYPE_SGL); - ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_DIGEST_IN_BUFFER); ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags, ICP_QAT_FW_LA_PARTIAL_NONE); ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, @@ -279,8 +289,9 @@ static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header) ICP_QAT_FW_LA_NO_UPDATE_STATE); } -static int qat_alg_init_enc_session(struct qat_alg_session_ctx *ctx, - int alg, struct crypto_authenc_keys *keys) +static int qat_alg_aead_init_enc_session(struct qat_alg_aead_ctx *ctx, + int alg, + struct crypto_authenc_keys *keys) { struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm); unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize; @@ -289,7 +300,7 @@ static int qat_alg_init_enc_session(struct qat_alg_session_ctx *ctx, struct icp_qat_hw_auth_algo_blk *hash = (struct icp_qat_hw_auth_algo_blk *)((char *)enc_ctx + sizeof(struct icp_qat_hw_auth_setup) + keys->enckeylen); - struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req_tmpl; + struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req; struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; void *ptr = &req_tmpl->cd_ctrl; @@ -297,7 +308,7 @@ static int qat_alg_init_enc_session(struct qat_alg_session_ctx *ctx, struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr; /* CD setup */ - cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg); + cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_ENC(alg); memcpy(cipher->aes.key, keys->enckey, keys->enckeylen); hash->sha.inner_setup.auth_config.config = ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1, @@ -311,6 +322,8 @@ static int qat_alg_init_enc_session(struct qat_alg_session_ctx *ctx, /* Request setup */ qat_alg_init_common_hdr(header); header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH; + ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_DIGEST_IN_BUFFER); ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, ICP_QAT_FW_LA_RET_AUTH_RES); ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, @@ -356,8 +369,9 @@ static int qat_alg_init_enc_session(struct qat_alg_session_ctx *ctx, return 0; } -static int qat_alg_init_dec_session(struct qat_alg_session_ctx *ctx, - int alg, struct crypto_authenc_keys *keys) +static int qat_alg_aead_init_dec_session(struct qat_alg_aead_ctx *ctx, + int alg, + struct crypto_authenc_keys *keys) { struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm); unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize; @@ -367,7 +381,7 @@ static int qat_alg_init_dec_session(struct qat_alg_session_ctx *ctx, (struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx + sizeof(struct icp_qat_hw_auth_setup) + roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2); - struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req_tmpl; + struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req; struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; void *ptr = &req_tmpl->cd_ctrl; @@ -379,7 +393,7 @@ static int qat_alg_init_dec_session(struct qat_alg_session_ctx *ctx, sizeof(struct icp_qat_fw_la_cipher_req_params)); /* CD setup */ - cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_DEC(alg); + cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_DEC(alg); memcpy(cipher->aes.key, keys->enckey, keys->enckeylen); hash->sha.inner_setup.auth_config.config = ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1, @@ -394,6 +408,8 @@ static int qat_alg_init_dec_session(struct qat_alg_session_ctx *ctx, /* Request setup */ qat_alg_init_common_hdr(header); header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER; + ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_DIGEST_IN_BUFFER); ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, ICP_QAT_FW_LA_NO_RET_AUTH_RES); ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, @@ -444,36 +460,91 @@ static int qat_alg_init_dec_session(struct qat_alg_session_ctx *ctx, return 0; } -static int qat_alg_init_sessions(struct qat_alg_session_ctx *ctx, - const uint8_t *key, unsigned int keylen) +static void qat_alg_ablkcipher_init_com(struct qat_alg_ablkcipher_ctx *ctx, + struct icp_qat_fw_la_bulk_req *req, + struct icp_qat_hw_cipher_algo_blk *cd, + const uint8_t *key, unsigned int keylen) { - struct crypto_authenc_keys keys; - int alg; + struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; + struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr; + struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl; - if (crypto_rng_get_bytes(crypto_default_rng, ctx->salt, AES_BLOCK_SIZE)) - return -EFAULT; + memcpy(cd->aes.key, key, keylen); + qat_alg_init_common_hdr(header); + header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER; + cd_pars->u.s.content_desc_params_sz = + sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3; + /* Cipher CD config setup */ + cd_ctrl->cipher_key_sz = keylen >> 3; + cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3; + cd_ctrl->cipher_cfg_offset = 0; + ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); + ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); +} - if (crypto_authenc_extractkeys(&keys, key, keylen)) - goto bad_key; +static void qat_alg_ablkcipher_init_enc(struct qat_alg_ablkcipher_ctx *ctx, + int alg, const uint8_t *key, + unsigned int keylen) +{ + struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd; + struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req; + struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; - switch (keys.enckeylen) { + qat_alg_ablkcipher_init_com(ctx, req, enc_cd, key, keylen); + cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr; + enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_ENC(alg); +} + +static void qat_alg_ablkcipher_init_dec(struct qat_alg_ablkcipher_ctx *ctx, + int alg, const uint8_t *key, + unsigned int keylen) +{ + struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd; + struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req; + struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; + + qat_alg_ablkcipher_init_com(ctx, req, dec_cd, key, keylen); + cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr; + dec_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_DEC(alg); +} + +static int qat_alg_validate_key(int key_len, int *alg) +{ + switch (key_len) { case AES_KEYSIZE_128: - alg = ICP_QAT_HW_CIPHER_ALGO_AES128; + *alg = ICP_QAT_HW_CIPHER_ALGO_AES128; break; case AES_KEYSIZE_192: - alg = ICP_QAT_HW_CIPHER_ALGO_AES192; + *alg = ICP_QAT_HW_CIPHER_ALGO_AES192; break; case AES_KEYSIZE_256: - alg = ICP_QAT_HW_CIPHER_ALGO_AES256; + *alg = ICP_QAT_HW_CIPHER_ALGO_AES256; break; default: - goto bad_key; + return -EINVAL; } + return 0; +} - if (qat_alg_init_enc_session(ctx, alg, &keys)) +static int qat_alg_aead_init_sessions(struct qat_alg_aead_ctx *ctx, + const uint8_t *key, unsigned int keylen) +{ + struct crypto_authenc_keys keys; + int alg; + + if (crypto_rng_get_bytes(crypto_default_rng, ctx->salt, AES_BLOCK_SIZE)) + return -EFAULT; + + if (crypto_authenc_extractkeys(&keys, key, keylen)) + goto bad_key; + + if (qat_alg_validate_key(keys.enckeylen, &alg)) + goto bad_key; + + if (qat_alg_aead_init_enc_session(ctx, alg, &keys)) goto error; - if (qat_alg_init_dec_session(ctx, alg, &keys)) + if (qat_alg_aead_init_dec_session(ctx, alg, &keys)) goto error; return 0; @@ -484,22 +555,37 @@ error: return -EFAULT; } -static int qat_alg_setkey(struct crypto_aead *tfm, const uint8_t *key, - unsigned int keylen) +static int qat_alg_ablkcipher_init_sessions(struct qat_alg_ablkcipher_ctx *ctx, + const uint8_t *key, + unsigned int keylen) { - struct qat_alg_session_ctx *ctx = crypto_aead_ctx(tfm); + int alg; + + if (qat_alg_validate_key(keylen, &alg)) + goto bad_key; + + qat_alg_ablkcipher_init_enc(ctx, alg, key, keylen); + qat_alg_ablkcipher_init_dec(ctx, alg, key, keylen); + return 0; +bad_key: + crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; +} + +static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key, + unsigned int keylen) +{ + struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); struct device *dev; spin_lock(&ctx->lock); if (ctx->enc_cd) { /* rekeying */ dev = &GET_DEV(ctx->inst->accel_dev); - memzero_explicit(ctx->enc_cd, sizeof(struct qat_alg_cd)); - memzero_explicit(ctx->dec_cd, sizeof(struct qat_alg_cd)); - memzero_explicit(&ctx->enc_fw_req_tmpl, - sizeof(struct icp_qat_fw_la_bulk_req)); - memzero_explicit(&ctx->dec_fw_req_tmpl, - sizeof(struct icp_qat_fw_la_bulk_req)); + memzero_explicit(ctx->enc_cd, sizeof(*ctx->enc_cd)); + memzero_explicit(ctx->dec_cd, sizeof(*ctx->dec_cd)); + memzero_explicit(&ctx->enc_fw_req, sizeof(ctx->enc_fw_req)); + memzero_explicit(&ctx->dec_fw_req, sizeof(ctx->dec_fw_req)); } else { /* new key */ int node = get_current_node(); @@ -512,16 +598,14 @@ static int qat_alg_setkey(struct crypto_aead *tfm, const uint8_t *key, dev = &GET_DEV(inst->accel_dev); ctx->inst = inst; - ctx->enc_cd = dma_zalloc_coherent(dev, - sizeof(struct qat_alg_cd), + ctx->enc_cd = dma_zalloc_coherent(dev, sizeof(*ctx->enc_cd), &ctx->enc_cd_paddr, GFP_ATOMIC); if (!ctx->enc_cd) { spin_unlock(&ctx->lock); return -ENOMEM; } - ctx->dec_cd = dma_zalloc_coherent(dev, - sizeof(struct qat_alg_cd), + ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd), &ctx->dec_cd_paddr, GFP_ATOMIC); if (!ctx->dec_cd) { @@ -530,7 +614,7 @@ static int qat_alg_setkey(struct crypto_aead *tfm, const uint8_t *key, } } spin_unlock(&ctx->lock); - if (qat_alg_init_sessions(ctx, key, keylen)) + if (qat_alg_aead_init_sessions(ctx, key, keylen)) goto out_free_all; return 0; @@ -722,14 +806,12 @@ err: return -ENOMEM; } -void qat_alg_callback(void *resp) +static void qat_aead_alg_callback(struct icp_qat_fw_la_resp *qat_resp, + struct qat_crypto_request *qat_req) { - struct icp_qat_fw_la_resp *qat_resp = resp; - struct qat_crypto_request *qat_req = - (void *)(__force long)qat_resp->opaque_data; - struct qat_alg_session_ctx *ctx = qat_req->ctx; + struct qat_alg_aead_ctx *ctx = qat_req->aead_ctx; struct qat_crypto_instance *inst = ctx->inst; - struct aead_request *areq = qat_req->areq; + struct aead_request *areq = qat_req->aead_req; uint8_t stat_filed = qat_resp->comn_resp.comn_status; int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed); @@ -739,11 +821,35 @@ void qat_alg_callback(void *resp) areq->base.complete(&areq->base, res); } -static int qat_alg_dec(struct aead_request *areq) +static void qat_ablkcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp, + struct qat_crypto_request *qat_req) +{ + struct qat_alg_ablkcipher_ctx *ctx = qat_req->ablkcipher_ctx; + struct qat_crypto_instance *inst = ctx->inst; + struct ablkcipher_request *areq = qat_req->ablkcipher_req; + uint8_t stat_filed = qat_resp->comn_resp.comn_status; + int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed); + + qat_alg_free_bufl(inst, qat_req); + if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK)) + res = -EINVAL; + areq->base.complete(&areq->base, res); +} + +void qat_alg_callback(void *resp) +{ + struct icp_qat_fw_la_resp *qat_resp = resp; + struct qat_crypto_request *qat_req = + (void *)(__force long)qat_resp->opaque_data; + + qat_req->cb(qat_resp, qat_req); +} + +static int qat_alg_aead_dec(struct aead_request *areq) { struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq); struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm); - struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); struct qat_crypto_request *qat_req = aead_request_ctx(areq); struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_auth_req_params *auth_param; @@ -757,9 +863,10 @@ static int qat_alg_dec(struct aead_request *areq) return ret; msg = &qat_req->req; - *msg = ctx->dec_fw_req_tmpl; - qat_req->ctx = ctx; - qat_req->areq = areq; + *msg = ctx->dec_fw_req; + qat_req->aead_ctx = ctx; + qat_req->aead_req = areq; + qat_req->cb = qat_aead_alg_callback; qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req; qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; @@ -782,12 +889,12 @@ static int qat_alg_dec(struct aead_request *areq) return -EINPROGRESS; } -static int qat_alg_enc_internal(struct aead_request *areq, uint8_t *iv, - int enc_iv) +static int qat_alg_aead_enc_internal(struct aead_request *areq, uint8_t *iv, + int enc_iv) { struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq); struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm); - struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); struct qat_crypto_request *qat_req = aead_request_ctx(areq); struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_auth_req_params *auth_param; @@ -800,9 +907,10 @@ static int qat_alg_enc_internal(struct aead_request *areq, uint8_t *iv, return ret; msg = &qat_req->req; - *msg = ctx->enc_fw_req_tmpl; - qat_req->ctx = ctx; - qat_req->areq = areq; + *msg = ctx->enc_fw_req; + qat_req->aead_ctx = ctx; + qat_req->aead_req = areq; + qat_req->cb = qat_aead_alg_callback; qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req; qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; @@ -831,29 +939,167 @@ static int qat_alg_enc_internal(struct aead_request *areq, uint8_t *iv, return -EINPROGRESS; } -static int qat_alg_enc(struct aead_request *areq) +static int qat_alg_aead_enc(struct aead_request *areq) { - return qat_alg_enc_internal(areq, areq->iv, 0); + return qat_alg_aead_enc_internal(areq, areq->iv, 0); } -static int qat_alg_genivenc(struct aead_givcrypt_request *req) +static int qat_alg_aead_genivenc(struct aead_givcrypt_request *req) { struct crypto_aead *aead_tfm = crypto_aead_reqtfm(&req->areq); struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm); - struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); __be64 seq; memcpy(req->giv, ctx->salt, AES_BLOCK_SIZE); seq = cpu_to_be64(req->seq); memcpy(req->giv + AES_BLOCK_SIZE - sizeof(uint64_t), &seq, sizeof(uint64_t)); - return qat_alg_enc_internal(&req->areq, req->giv, 1); + return qat_alg_aead_enc_internal(&req->areq, req->giv, 1); } -static int qat_alg_init(struct crypto_tfm *tfm, - enum icp_qat_hw_auth_algo hash, const char *hash_name) +static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm, + const uint8_t *key, + unsigned int keylen) { - struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct device *dev; + + spin_lock(&ctx->lock); + if (ctx->enc_cd) { + /* rekeying */ + dev = &GET_DEV(ctx->inst->accel_dev); + memzero_explicit(ctx->enc_cd, sizeof(*ctx->enc_cd)); + memzero_explicit(ctx->dec_cd, sizeof(*ctx->dec_cd)); + memzero_explicit(&ctx->enc_fw_req, sizeof(ctx->enc_fw_req)); + memzero_explicit(&ctx->dec_fw_req, sizeof(ctx->dec_fw_req)); + } else { + /* new key */ + int node = get_current_node(); + struct qat_crypto_instance *inst = + qat_crypto_get_instance_node(node); + if (!inst) { + spin_unlock(&ctx->lock); + return -EINVAL; + } + + dev = &GET_DEV(inst->accel_dev); + ctx->inst = inst; + ctx->enc_cd = dma_zalloc_coherent(dev, sizeof(*ctx->enc_cd), + &ctx->enc_cd_paddr, + GFP_ATOMIC); + if (!ctx->enc_cd) { + spin_unlock(&ctx->lock); + return -ENOMEM; + } + ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd), + &ctx->dec_cd_paddr, + GFP_ATOMIC); + if (!ctx->dec_cd) { + spin_unlock(&ctx->lock); + goto out_free_enc; + } + } + spin_unlock(&ctx->lock); + if (qat_alg_ablkcipher_init_sessions(ctx, key, keylen)) + goto out_free_all; + + return 0; + +out_free_all: + memzero_explicit(ctx->dec_cd, sizeof(*ctx->enc_cd)); + dma_free_coherent(dev, sizeof(*ctx->enc_cd), + ctx->dec_cd, ctx->dec_cd_paddr); + ctx->dec_cd = NULL; +out_free_enc: + memzero_explicit(ctx->enc_cd, sizeof(*ctx->dec_cd)); + dma_free_coherent(dev, sizeof(*ctx->dec_cd), + ctx->enc_cd, ctx->enc_cd_paddr); + ctx->enc_cd = NULL; + return -ENOMEM; +} + +static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req); + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(atfm); + struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req); + struct icp_qat_fw_la_cipher_req_params *cipher_param; + struct icp_qat_fw_la_bulk_req *msg; + int ret, ctr = 0; + + ret = qat_alg_sgl_to_bufl(ctx->inst, NULL, req->src, req->dst, + NULL, 0, qat_req); + if (unlikely(ret)) + return ret; + + msg = &qat_req->req; + *msg = ctx->enc_fw_req; + qat_req->ablkcipher_ctx = ctx; + qat_req->ablkcipher_req = req; + qat_req->cb = qat_ablkcipher_alg_callback; + qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req; + qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; + qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; + cipher_param = (void *)&qat_req->req.serv_specif_rqpars; + cipher_param->cipher_length = req->nbytes; + cipher_param->cipher_offset = 0; + memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE); + do { + ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg); + } while (ret == -EAGAIN && ctr++ < 10); + + if (ret == -EAGAIN) { + qat_alg_free_bufl(ctx->inst, qat_req); + return -EBUSY; + } + return -EINPROGRESS; +} + +static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req); + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(atfm); + struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req); + struct icp_qat_fw_la_cipher_req_params *cipher_param; + struct icp_qat_fw_la_bulk_req *msg; + int ret, ctr = 0; + + ret = qat_alg_sgl_to_bufl(ctx->inst, NULL, req->src, req->dst, + NULL, 0, qat_req); + if (unlikely(ret)) + return ret; + + msg = &qat_req->req; + *msg = ctx->dec_fw_req; + qat_req->ablkcipher_ctx = ctx; + qat_req->ablkcipher_req = req; + qat_req->cb = qat_ablkcipher_alg_callback; + qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req; + qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; + qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; + cipher_param = (void *)&qat_req->req.serv_specif_rqpars; + cipher_param->cipher_length = req->nbytes; + cipher_param->cipher_offset = 0; + memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE); + do { + ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg); + } while (ret == -EAGAIN && ctr++ < 10); + + if (ret == -EAGAIN) { + qat_alg_free_bufl(ctx->inst, qat_req); + return -EBUSY; + } + return -EINPROGRESS; +} + +static int qat_alg_aead_init(struct crypto_tfm *tfm, + enum icp_qat_hw_auth_algo hash, + const char *hash_name) +{ + struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); memzero_explicit(ctx, sizeof(*ctx)); ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); @@ -867,24 +1113,24 @@ static int qat_alg_init(struct crypto_tfm *tfm, return 0; } -static int qat_alg_sha1_init(struct crypto_tfm *tfm) +static int qat_alg_aead_sha1_init(struct crypto_tfm *tfm) { - return qat_alg_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1"); + return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1"); } -static int qat_alg_sha256_init(struct crypto_tfm *tfm) +static int qat_alg_aead_sha256_init(struct crypto_tfm *tfm) { - return qat_alg_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256"); + return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256"); } -static int qat_alg_sha512_init(struct crypto_tfm *tfm) +static int qat_alg_aead_sha512_init(struct crypto_tfm *tfm) { - return qat_alg_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512"); + return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512"); } -static void qat_alg_exit(struct crypto_tfm *tfm) +static void qat_alg_aead_exit(struct crypto_tfm *tfm) { - struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); struct qat_crypto_instance *inst = ctx->inst; struct device *dev; @@ -908,24 +1154,63 @@ static void qat_alg_exit(struct crypto_tfm *tfm) qat_crypto_put_instance(inst); } +static int qat_alg_ablkcipher_init(struct crypto_tfm *tfm) +{ + struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm); + + memzero_explicit(ctx, sizeof(*ctx)); + spin_lock_init(&ctx->lock); + tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) + + sizeof(struct qat_crypto_request); + ctx->tfm = tfm; + return 0; +} + +static void qat_alg_ablkcipher_exit(struct crypto_tfm *tfm) +{ + struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev; + + if (!inst) + return; + + dev = &GET_DEV(inst->accel_dev); + if (ctx->enc_cd) { + memzero_explicit(ctx->enc_cd, + sizeof(struct icp_qat_hw_cipher_algo_blk)); + dma_free_coherent(dev, + sizeof(struct icp_qat_hw_cipher_algo_blk), + ctx->enc_cd, ctx->enc_cd_paddr); + } + if (ctx->dec_cd) { + memzero_explicit(ctx->dec_cd, + sizeof(struct icp_qat_hw_cipher_algo_blk)); + dma_free_coherent(dev, + sizeof(struct icp_qat_hw_cipher_algo_blk), + ctx->dec_cd, ctx->dec_cd_paddr); + } + qat_crypto_put_instance(inst); +} + static struct crypto_alg qat_algs[] = { { .cra_name = "authenc(hmac(sha1),cbc(aes))", .cra_driver_name = "qat_aes_cbc_hmac_sha1", .cra_priority = 4001, .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct qat_alg_session_ctx), + .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), .cra_alignmask = 0, .cra_type = &crypto_aead_type, .cra_module = THIS_MODULE, - .cra_init = qat_alg_sha1_init, - .cra_exit = qat_alg_exit, + .cra_init = qat_alg_aead_sha1_init, + .cra_exit = qat_alg_aead_exit, .cra_u = { .aead = { - .setkey = qat_alg_setkey, - .decrypt = qat_alg_dec, - .encrypt = qat_alg_enc, - .givencrypt = qat_alg_genivenc, + .setkey = qat_alg_aead_setkey, + .decrypt = qat_alg_aead_dec, + .encrypt = qat_alg_aead_enc, + .givencrypt = qat_alg_aead_genivenc, .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, }, @@ -936,18 +1221,18 @@ static struct crypto_alg qat_algs[] = { { .cra_priority = 4001, .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct qat_alg_session_ctx), + .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), .cra_alignmask = 0, .cra_type = &crypto_aead_type, .cra_module = THIS_MODULE, - .cra_init = qat_alg_sha256_init, - .cra_exit = qat_alg_exit, + .cra_init = qat_alg_aead_sha256_init, + .cra_exit = qat_alg_aead_exit, .cra_u = { .aead = { - .setkey = qat_alg_setkey, - .decrypt = qat_alg_dec, - .encrypt = qat_alg_enc, - .givencrypt = qat_alg_genivenc, + .setkey = qat_alg_aead_setkey, + .decrypt = qat_alg_aead_dec, + .encrypt = qat_alg_aead_enc, + .givencrypt = qat_alg_aead_genivenc, .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA256_DIGEST_SIZE, }, @@ -958,22 +1243,44 @@ static struct crypto_alg qat_algs[] = { { .cra_priority = 4001, .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct qat_alg_session_ctx), + .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), .cra_alignmask = 0, .cra_type = &crypto_aead_type, .cra_module = THIS_MODULE, - .cra_init = qat_alg_sha512_init, - .cra_exit = qat_alg_exit, + .cra_init = qat_alg_aead_sha512_init, + .cra_exit = qat_alg_aead_exit, .cra_u = { .aead = { - .setkey = qat_alg_setkey, - .decrypt = qat_alg_dec, - .encrypt = qat_alg_enc, - .givencrypt = qat_alg_genivenc, + .setkey = qat_alg_aead_setkey, + .decrypt = qat_alg_aead_dec, + .encrypt = qat_alg_aead_enc, + .givencrypt = qat_alg_aead_genivenc, .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA512_DIGEST_SIZE, }, }, +}, { + .cra_name = "cbc(aes)", + .cra_driver_name = "qat_aes_cbc", + .cra_priority = 4001, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct qat_alg_ablkcipher_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = qat_alg_ablkcipher_init, + .cra_exit = qat_alg_ablkcipher_exit, + .cra_u = { + .ablkcipher = { + .setkey = qat_alg_ablkcipher_setkey, + .decrypt = qat_alg_ablkcipher_decrypt, + .encrypt = qat_alg_ablkcipher_encrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + }, } }; int qat_algs_register(void) @@ -982,8 +1289,11 @@ int qat_algs_register(void) int i; for (i = 0; i < ARRAY_SIZE(qat_algs); i++) - qat_algs[i].cra_flags = CRYPTO_ALG_TYPE_AEAD | - CRYPTO_ALG_ASYNC; + qat_algs[i].cra_flags = + (qat_algs[i].cra_type == &crypto_aead_type) ? + CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC : + CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC; + return crypto_register_algs(qat_algs, ARRAY_SIZE(qat_algs)); } return 0; diff --git a/drivers/crypto/qat/qat_common/qat_crypto.h b/drivers/crypto/qat/qat_common/qat_crypto.h index fcb323116e60..d503007b49e6 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.h +++ b/drivers/crypto/qat/qat_common/qat_crypto.h @@ -75,10 +75,21 @@ struct qat_crypto_request_buffs { size_t sz_out; }; +struct qat_crypto_request; + struct qat_crypto_request { struct icp_qat_fw_la_bulk_req req; - struct qat_alg_session_ctx *ctx; - struct aead_request *areq; + union { + struct qat_alg_aead_ctx *aead_ctx; + struct qat_alg_ablkcipher_ctx *ablkcipher_ctx; + }; + union { + struct aead_request *aead_req; + struct ablkcipher_request *ablkcipher_req; + }; struct qat_crypto_request_buffs buf; + void (*cb)(struct icp_qat_fw_la_resp *resp, + struct qat_crypto_request *req); }; + #endif -- cgit v1.2.3-58-ga151 From d3f6c142865badc82fa4d151766634b895d693e8 Mon Sep 17 00:00:00 2001 From: Asaf Vertz Date: Wed, 10 Dec 2014 14:55:10 +0200 Subject: crypto: ux500 - fix checkpatch errors Fixed a coding style error, code indent should use tabs where possible Signed-off-by: Asaf Vertz Signed-off-by: Herbert Xu --- drivers/crypto/ux500/cryp/cryp_core.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/ux500/cryp/cryp_core.c b/drivers/crypto/ux500/cryp/cryp_core.c index f831bb952b2f..f087f37b2c67 100644 --- a/drivers/crypto/ux500/cryp/cryp_core.c +++ b/drivers/crypto/ux500/cryp/cryp_core.c @@ -479,13 +479,13 @@ static void cryp_dma_setup_channel(struct cryp_device_data *device_data, .dst_addr = device_data->phybase + CRYP_DMA_TX_FIFO, .dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES, .dst_maxburst = 4, - }; + }; struct dma_slave_config cryp2mem = { .direction = DMA_DEV_TO_MEM, .src_addr = device_data->phybase + CRYP_DMA_RX_FIFO, .src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES, .src_maxburst = 4, - }; + }; dma_cap_zero(device_data->dma.mask); dma_cap_set(DMA_SLAVE, device_data->dma.mask); @@ -1774,8 +1774,8 @@ static int ux500_cryp_resume(struct device *dev) static SIMPLE_DEV_PM_OPS(ux500_cryp_pm, ux500_cryp_suspend, ux500_cryp_resume); static const struct of_device_id ux500_cryp_match[] = { - { .compatible = "stericsson,ux500-cryp" }, - { }, + { .compatible = "stericsson,ux500-cryp" }, + { }, }; static struct platform_driver cryp_driver = { -- cgit v1.2.3-58-ga151 From f421258d5bf883b68b1cdaa299a8a1da3eb92e0f Mon Sep 17 00:00:00 2001 From: Aaro Koskinen Date: Sun, 21 Dec 2014 22:53:58 +0200 Subject: MIPS: OCTEON: add crypto helper functions Add crypto helper functions which are needed for kernel level usage. The code for these has been extracted from the EdgeRouter Pro GPL tarball. While at it, also delete duplicate definitions of the functions. Signed-off-by: Aaro Koskinen Signed-off-by: Herbert Xu --- arch/mips/cavium-octeon/Makefile | 1 + arch/mips/cavium-octeon/crypto/Makefile | 5 ++ arch/mips/cavium-octeon/crypto/octeon-crypto.c | 66 ++++++++++++++++++++++++++ arch/mips/cavium-octeon/crypto/octeon-crypto.h | 17 +++++++ arch/mips/include/asm/octeon/octeon.h | 5 -- 5 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 arch/mips/cavium-octeon/crypto/Makefile create mode 100644 arch/mips/cavium-octeon/crypto/octeon-crypto.c create mode 100644 arch/mips/cavium-octeon/crypto/octeon-crypto.h diff --git a/arch/mips/cavium-octeon/Makefile b/arch/mips/cavium-octeon/Makefile index 42f5f1a4b40a..69a8a8dabc2b 100644 --- a/arch/mips/cavium-octeon/Makefile +++ b/arch/mips/cavium-octeon/Makefile @@ -16,6 +16,7 @@ obj-y := cpu.o setup.o octeon-platform.o octeon-irq.o csrc-octeon.o obj-y += dma-octeon.o obj-y += octeon-memcpy.o obj-y += executive/ +obj-y += crypto/ obj-$(CONFIG_MTD) += flash_setup.o obj-$(CONFIG_SMP) += smp.o diff --git a/arch/mips/cavium-octeon/crypto/Makefile b/arch/mips/cavium-octeon/crypto/Makefile new file mode 100644 index 000000000000..739b80366c25 --- /dev/null +++ b/arch/mips/cavium-octeon/crypto/Makefile @@ -0,0 +1,5 @@ +# +# OCTEON-specific crypto modules. +# + +obj-y += octeon-crypto.o diff --git a/arch/mips/cavium-octeon/crypto/octeon-crypto.c b/arch/mips/cavium-octeon/crypto/octeon-crypto.c new file mode 100644 index 000000000000..7c82ff463b65 --- /dev/null +++ b/arch/mips/cavium-octeon/crypto/octeon-crypto.c @@ -0,0 +1,66 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2004-2012 Cavium Networks + */ + +#include +#include +#include + +#include "octeon-crypto.h" + +/** + * Enable access to Octeon's COP2 crypto hardware for kernel use. Wrap any + * crypto operations in calls to octeon_crypto_enable/disable in order to make + * sure the state of COP2 isn't corrupted if userspace is also performing + * hardware crypto operations. Allocate the state parameter on the stack. + * Preemption must be disabled to prevent context switches. + * + * @state: Pointer to state structure to store current COP2 state in. + * + * Returns: Flags to be passed to octeon_crypto_disable() + */ +unsigned long octeon_crypto_enable(struct octeon_cop2_state *state) +{ + int status; + unsigned long flags; + + local_irq_save(flags); + status = read_c0_status(); + write_c0_status(status | ST0_CU2); + if (KSTK_STATUS(current) & ST0_CU2) { + octeon_cop2_save(&(current->thread.cp2)); + KSTK_STATUS(current) &= ~ST0_CU2; + status &= ~ST0_CU2; + } else if (status & ST0_CU2) { + octeon_cop2_save(state); + } + local_irq_restore(flags); + return status & ST0_CU2; +} +EXPORT_SYMBOL_GPL(octeon_crypto_enable); + +/** + * Disable access to Octeon's COP2 crypto hardware in the kernel. This must be + * called after an octeon_crypto_enable() before any context switch or return to + * userspace. + * + * @state: Pointer to COP2 state to restore + * @flags: Return value from octeon_crypto_enable() + */ +void octeon_crypto_disable(struct octeon_cop2_state *state, + unsigned long crypto_flags) +{ + unsigned long flags; + + local_irq_save(flags); + if (crypto_flags & ST0_CU2) + octeon_cop2_restore(state); + else + write_c0_status(read_c0_status() & ~ST0_CU2); + local_irq_restore(flags); +} +EXPORT_SYMBOL_GPL(octeon_crypto_disable); diff --git a/arch/mips/cavium-octeon/crypto/octeon-crypto.h b/arch/mips/cavium-octeon/crypto/octeon-crypto.h new file mode 100644 index 000000000000..5ca86d4ead52 --- /dev/null +++ b/arch/mips/cavium-octeon/crypto/octeon-crypto.h @@ -0,0 +1,17 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2012-2013 Cavium Inc., All Rights Reserved. + */ +#ifndef __LINUX_OCTEON_CRYPTO_H +#define __LINUX_OCTEON_CRYPTO_H + +#include + +extern unsigned long octeon_crypto_enable(struct octeon_cop2_state *state); +extern void octeon_crypto_disable(struct octeon_cop2_state *state, + unsigned long flags); + +#endif /* __LINUX_OCTEON_CRYPTO_H */ diff --git a/arch/mips/include/asm/octeon/octeon.h b/arch/mips/include/asm/octeon/octeon.h index d781f9e66884..6dfefd2d5cdf 100644 --- a/arch/mips/include/asm/octeon/octeon.h +++ b/arch/mips/include/asm/octeon/octeon.h @@ -44,11 +44,6 @@ extern int octeon_get_boot_num_arguments(void); extern const char *octeon_get_boot_argument(int arg); extern void octeon_hal_setup_reserved32(void); extern void octeon_user_io_init(void); -struct octeon_cop2_state; -extern unsigned long octeon_crypto_enable(struct octeon_cop2_state *state); -extern void octeon_crypto_disable(struct octeon_cop2_state *state, - unsigned long flags); -extern asmlinkage void octeon_cop2_restore(struct octeon_cop2_state *task); extern void octeon_init_cvmcount(void); extern void octeon_setup_delays(void); -- cgit v1.2.3-58-ga151 From 1e585ef51c3fdcf296d2ce5cb8c9e74b1a36cfb4 Mon Sep 17 00:00:00 2001 From: Aaro Koskinen Date: Sun, 21 Dec 2014 22:53:59 +0200 Subject: crypto: octeon - add instruction definitions for MD5 Add instruction definitions for MD5. Based on information extracted from EdgeRouter Pro GPL source tarball. Signed-off-by: Aaro Koskinen Signed-off-by: Herbert Xu --- arch/mips/cavium-octeon/crypto/octeon-crypto.h | 56 ++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/arch/mips/cavium-octeon/crypto/octeon-crypto.h b/arch/mips/cavium-octeon/crypto/octeon-crypto.h index 5ca86d4ead52..3f65bc6ccb83 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-crypto.h +++ b/arch/mips/cavium-octeon/crypto/octeon-crypto.h @@ -4,14 +4,70 @@ * for more details. * * Copyright (C) 2012-2013 Cavium Inc., All Rights Reserved. + * + * MD5 instruction definitions added by Aaro Koskinen . + * */ #ifndef __LINUX_OCTEON_CRYPTO_H #define __LINUX_OCTEON_CRYPTO_H #include +#include extern unsigned long octeon_crypto_enable(struct octeon_cop2_state *state); extern void octeon_crypto_disable(struct octeon_cop2_state *state, unsigned long flags); +/* + * Macros needed to implement MD5: + */ + +/* + * The index can be 0-1. + */ +#define write_octeon_64bit_hash_dword(value, index) \ +do { \ + __asm__ __volatile__ ( \ + "dmtc2 %[rt],0x0048+" STR(index) \ + : \ + : [rt] "d" (value)); \ +} while (0) + +/* + * The index can be 0-1. + */ +#define read_octeon_64bit_hash_dword(index) \ +({ \ + u64 __value; \ + \ + __asm__ __volatile__ ( \ + "dmfc2 %[rt],0x0048+" STR(index) \ + : [rt] "=d" (__value) \ + : ); \ + \ + __value; \ +}) + +/* + * The index can be 0-6. + */ +#define write_octeon_64bit_block_dword(value, index) \ +do { \ + __asm__ __volatile__ ( \ + "dmtc2 %[rt],0x0040+" STR(index) \ + : \ + : [rt] "d" (value)); \ +} while (0) + +/* + * The value is the final block dword (64-bit). + */ +#define octeon_md5_start(value) \ +do { \ + __asm__ __volatile__ ( \ + "dmtc2 %[rt],0x4047" \ + : \ + : [rt] "d" (value)); \ +} while (0) + #endif /* __LINUX_OCTEON_CRYPTO_H */ -- cgit v1.2.3-58-ga151 From 011f3c6cbb97859860e451f2a75767cd4c1ffc03 Mon Sep 17 00:00:00 2001 From: Aaro Koskinen Date: Sun, 21 Dec 2014 22:54:00 +0200 Subject: MIPS: OCTEON: reintroduce crypto features check Reintroduce run-time check for crypto features. The old one was deleted because it was unreliable, now decide the crypto availability on early boot when the model string is constructed. Signed-off-by: Aaro Koskinen Signed-off-by: Herbert Xu --- arch/mips/cavium-octeon/executive/octeon-model.c | 6 ++++++ arch/mips/include/asm/octeon/octeon-feature.h | 17 +++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/arch/mips/cavium-octeon/executive/octeon-model.c b/arch/mips/cavium-octeon/executive/octeon-model.c index e15b049b3bd7..b2104bd9ab3b 100644 --- a/arch/mips/cavium-octeon/executive/octeon-model.c +++ b/arch/mips/cavium-octeon/executive/octeon-model.c @@ -27,6 +27,9 @@ #include +enum octeon_feature_bits __octeon_feature_bits __read_mostly; +EXPORT_SYMBOL_GPL(__octeon_feature_bits); + /** * Read a byte of fuse data * @byte_addr: address to read @@ -103,6 +106,9 @@ static const char *__init octeon_model_get_string_buffer(uint32_t chip_id, else suffix = "NSP"; + if (!fus_dat2.s.nocrypto) + __octeon_feature_bits |= OCTEON_HAS_CRYPTO; + /* * Assume pass number is encoded using <5:3><2:0>. Exceptions * will be fixed later. diff --git a/arch/mips/include/asm/octeon/octeon-feature.h b/arch/mips/include/asm/octeon/octeon-feature.h index c4fe81f47f53..8ebd3f579b84 100644 --- a/arch/mips/include/asm/octeon/octeon-feature.h +++ b/arch/mips/include/asm/octeon/octeon-feature.h @@ -46,8 +46,6 @@ enum octeon_feature { OCTEON_FEATURE_SAAD, /* Does this Octeon support the ZIP offload engine? */ OCTEON_FEATURE_ZIP, - /* Does this Octeon support crypto acceleration using COP2? */ - OCTEON_FEATURE_CRYPTO, OCTEON_FEATURE_DORM_CRYPTO, /* Does this Octeon support PCI express? */ OCTEON_FEATURE_PCIE, @@ -86,6 +84,21 @@ enum octeon_feature { OCTEON_MAX_FEATURE }; +enum octeon_feature_bits { + OCTEON_HAS_CRYPTO = 0x0001, /* Crypto acceleration using COP2 */ +}; +extern enum octeon_feature_bits __octeon_feature_bits; + +/** + * octeon_has_crypto() - Check if this OCTEON has crypto acceleration support. + * + * Returns: Non-zero if the feature exists. Zero if the feature does not exist. + */ +static inline int octeon_has_crypto(void) +{ + return __octeon_feature_bits & OCTEON_HAS_CRYPTO; +} + /** * Determine if the current Octeon supports a specific feature. These * checks have been optimized to be fairly quick, but they should still -- cgit v1.2.3-58-ga151 From 1953c22f53747035b28c36dbb337ac3c10902644 Mon Sep 17 00:00:00 2001 From: Aaro Koskinen Date: Sun, 21 Dec 2014 22:54:01 +0200 Subject: crypto: octeon - add MD5 module Add OCTEON MD5 module. Signed-off-by: Aaro Koskinen Signed-off-by: Herbert Xu --- arch/mips/cavium-octeon/crypto/Makefile | 2 + arch/mips/cavium-octeon/crypto/octeon-crypto.h | 2 + arch/mips/cavium-octeon/crypto/octeon-md5.c | 216 +++++++++++++++++++++++++ 3 files changed, 220 insertions(+) create mode 100644 arch/mips/cavium-octeon/crypto/octeon-md5.c diff --git a/arch/mips/cavium-octeon/crypto/Makefile b/arch/mips/cavium-octeon/crypto/Makefile index 739b80366c25..a74f76d85a2f 100644 --- a/arch/mips/cavium-octeon/crypto/Makefile +++ b/arch/mips/cavium-octeon/crypto/Makefile @@ -3,3 +3,5 @@ # obj-y += octeon-crypto.o + +obj-$(CONFIG_CRYPTO_MD5_OCTEON) += octeon-md5.o diff --git a/arch/mips/cavium-octeon/crypto/octeon-crypto.h b/arch/mips/cavium-octeon/crypto/octeon-crypto.h index 3f65bc6ccb83..e2a4aece9c24 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-crypto.h +++ b/arch/mips/cavium-octeon/crypto/octeon-crypto.h @@ -14,6 +14,8 @@ #include #include +#define OCTEON_CR_OPCODE_PRIORITY 300 + extern unsigned long octeon_crypto_enable(struct octeon_cop2_state *state); extern void octeon_crypto_disable(struct octeon_cop2_state *state, unsigned long flags); diff --git a/arch/mips/cavium-octeon/crypto/octeon-md5.c b/arch/mips/cavium-octeon/crypto/octeon-md5.c new file mode 100644 index 000000000000..b909881ba6c1 --- /dev/null +++ b/arch/mips/cavium-octeon/crypto/octeon-md5.c @@ -0,0 +1,216 @@ +/* + * Cryptographic API. + * + * MD5 Message Digest Algorithm (RFC1321). + * + * Adapted for OCTEON by Aaro Koskinen . + * + * Based on crypto/md5.c, which is: + * + * Derived from cryptoapi implementation, originally based on the + * public domain implementation written by Colin Plumb in 1993. + * + * Copyright (c) Cryptoapi developers. + * Copyright (c) 2002 James Morris + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "octeon-crypto.h" + +/* + * We pass everything as 64-bit. OCTEON can handle misaligned data. + */ + +static void octeon_md5_store_hash(struct md5_state *ctx) +{ + u64 *hash = (u64 *)ctx->hash; + + write_octeon_64bit_hash_dword(hash[0], 0); + write_octeon_64bit_hash_dword(hash[1], 1); +} + +static void octeon_md5_read_hash(struct md5_state *ctx) +{ + u64 *hash = (u64 *)ctx->hash; + + hash[0] = read_octeon_64bit_hash_dword(0); + hash[1] = read_octeon_64bit_hash_dword(1); +} + +static void octeon_md5_transform(const void *_block) +{ + const u64 *block = _block; + + write_octeon_64bit_block_dword(block[0], 0); + write_octeon_64bit_block_dword(block[1], 1); + write_octeon_64bit_block_dword(block[2], 2); + write_octeon_64bit_block_dword(block[3], 3); + write_octeon_64bit_block_dword(block[4], 4); + write_octeon_64bit_block_dword(block[5], 5); + write_octeon_64bit_block_dword(block[6], 6); + octeon_md5_start(block[7]); +} + +static int octeon_md5_init(struct shash_desc *desc) +{ + struct md5_state *mctx = shash_desc_ctx(desc); + + mctx->hash[0] = cpu_to_le32(0x67452301); + mctx->hash[1] = cpu_to_le32(0xefcdab89); + mctx->hash[2] = cpu_to_le32(0x98badcfe); + mctx->hash[3] = cpu_to_le32(0x10325476); + mctx->byte_count = 0; + + return 0; +} + +static int octeon_md5_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct md5_state *mctx = shash_desc_ctx(desc); + const u32 avail = sizeof(mctx->block) - (mctx->byte_count & 0x3f); + struct octeon_cop2_state state; + unsigned long flags; + + mctx->byte_count += len; + + if (avail > len) { + memcpy((char *)mctx->block + (sizeof(mctx->block) - avail), + data, len); + return 0; + } + + memcpy((char *)mctx->block + (sizeof(mctx->block) - avail), data, + avail); + + local_bh_disable(); + preempt_disable(); + flags = octeon_crypto_enable(&state); + octeon_md5_store_hash(mctx); + + octeon_md5_transform(mctx->block); + data += avail; + len -= avail; + + while (len >= sizeof(mctx->block)) { + octeon_md5_transform(data); + data += sizeof(mctx->block); + len -= sizeof(mctx->block); + } + + octeon_md5_read_hash(mctx); + octeon_crypto_disable(&state, flags); + preempt_enable(); + local_bh_enable(); + + memcpy(mctx->block, data, len); + + return 0; +} + +static int octeon_md5_final(struct shash_desc *desc, u8 *out) +{ + struct md5_state *mctx = shash_desc_ctx(desc); + const unsigned int offset = mctx->byte_count & 0x3f; + char *p = (char *)mctx->block + offset; + int padding = 56 - (offset + 1); + struct octeon_cop2_state state; + unsigned long flags; + + *p++ = 0x80; + + local_bh_disable(); + preempt_disable(); + flags = octeon_crypto_enable(&state); + octeon_md5_store_hash(mctx); + + if (padding < 0) { + memset(p, 0x00, padding + sizeof(u64)); + octeon_md5_transform(mctx->block); + p = (char *)mctx->block; + padding = 56; + } + + memset(p, 0, padding); + mctx->block[14] = cpu_to_le32(mctx->byte_count << 3); + mctx->block[15] = cpu_to_le32(mctx->byte_count >> 29); + octeon_md5_transform(mctx->block); + + octeon_md5_read_hash(mctx); + octeon_crypto_disable(&state, flags); + preempt_enable(); + local_bh_enable(); + + memcpy(out, mctx->hash, sizeof(mctx->hash)); + memset(mctx, 0, sizeof(*mctx)); + + return 0; +} + +static int octeon_md5_export(struct shash_desc *desc, void *out) +{ + struct md5_state *ctx = shash_desc_ctx(desc); + + memcpy(out, ctx, sizeof(*ctx)); + return 0; +} + +static int octeon_md5_import(struct shash_desc *desc, const void *in) +{ + struct md5_state *ctx = shash_desc_ctx(desc); + + memcpy(ctx, in, sizeof(*ctx)); + return 0; +} + +static struct shash_alg alg = { + .digestsize = MD5_DIGEST_SIZE, + .init = octeon_md5_init, + .update = octeon_md5_update, + .final = octeon_md5_final, + .export = octeon_md5_export, + .import = octeon_md5_import, + .descsize = sizeof(struct md5_state), + .statesize = sizeof(struct md5_state), + .base = { + .cra_name = "md5", + .cra_driver_name= "octeon-md5", + .cra_priority = OCTEON_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static int __init md5_mod_init(void) +{ + if (!octeon_has_crypto()) + return -ENOTSUPP; + return crypto_register_shash(&alg); +} + +static void __exit md5_mod_fini(void) +{ + crypto_unregister_shash(&alg); +} + +module_init(md5_mod_init); +module_exit(md5_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("MD5 Message Digest Algorithm (OCTEON)"); +MODULE_AUTHOR("Aaro Koskinen "); -- cgit v1.2.3-58-ga151 From d69e75deff2377b46b2b357ac3781cc93cd7ffd6 Mon Sep 17 00:00:00 2001 From: Aaro Koskinen Date: Sun, 21 Dec 2014 22:54:02 +0200 Subject: crypto: octeon - enable OCTEON MD5 module selection Enable user to select OCTEON MD5 module. Signed-off-by: Aaro Koskinen Signed-off-by: Herbert Xu --- crypto/Kconfig | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crypto/Kconfig b/crypto/Kconfig index 87bbc9c1e681..1618468b1fcb 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -427,6 +427,15 @@ config CRYPTO_MD5 help MD5 message digest algorithm (RFC1321). +config CRYPTO_MD5_OCTEON + tristate "MD5 digest algorithm (OCTEON)" + depends on CPU_CAVIUM_OCTEON + select CRYPTO_MD5 + select CRYPTO_HASH + help + MD5 message digest algorithm (RFC1321) implemented + using OCTEON crypto instructions, when available. + config CRYPTO_MD5_SPARC64 tristate "MD5 digest algorithm (SPARC64)" depends on SPARC64 -- cgit v1.2.3-58-ga151 From 77584ee57434813b50fc85cde995a6271a5081b7 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 23 Dec 2014 16:40:17 +1100 Subject: hwrng: core - Use struct completion for cleanup_done There is no point in doing a manual completion for cleanup_done when struct completion fits in perfectly. Signed-off-by: Herbert Xu --- drivers/char/hw_random/core.c | 12 +++--------- include/linux/hw_random.h | 3 ++- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 6ec42252e46e..3dba2cf50241 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -60,7 +60,6 @@ static DEFINE_MUTEX(rng_mutex); static DEFINE_MUTEX(reading_mutex); static int data_avail; static u8 *rng_buffer, *rng_fillbuf; -static DECLARE_WAIT_QUEUE_HEAD(rng_done); static unsigned short current_quality; static unsigned short default_quality; /* = 0; default to "off" */ @@ -100,10 +99,7 @@ static inline void cleanup_rng(struct kref *kref) if (rng->cleanup) rng->cleanup(rng); - /* cleanup_done should be updated after cleanup finishes */ - smp_wmb(); - rng->cleanup_done = true; - wake_up_all(&rng_done); + complete(&rng->cleanup_done); } static void set_current_rng(struct hwrng *rng) @@ -498,7 +494,7 @@ int hwrng_register(struct hwrng *rng) add_early_randomness(rng); } - rng->cleanup_done = false; + init_completion(&rng->cleanup_done); out_unlock: mutex_unlock(&rng_mutex); @@ -532,9 +528,7 @@ void hwrng_unregister(struct hwrng *rng) } else mutex_unlock(&rng_mutex); - /* Just in case rng is reading right now, wait. */ - wait_event(rng_done, rng->cleanup_done && - atomic_read(&rng->ref.refcount) == 0); + wait_for_completion(&rng->cleanup_done); } EXPORT_SYMBOL_GPL(hwrng_unregister); diff --git a/include/linux/hw_random.h b/include/linux/hw_random.h index 7832e5008959..eb7b414d232b 100644 --- a/include/linux/hw_random.h +++ b/include/linux/hw_random.h @@ -12,6 +12,7 @@ #ifndef LINUX_HWRANDOM_H_ #define LINUX_HWRANDOM_H_ +#include #include #include #include @@ -46,7 +47,7 @@ struct hwrng { /* internal. */ struct list_head list; struct kref ref; - bool cleanup_done; + struct completion cleanup_done; }; /** Register a new Hardware Random Number Generator driver. */ -- cgit v1.2.3-58-ga151 From 15b66cd54291186011f733cc750263f320b8a0a4 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 23 Dec 2014 16:40:18 +1100 Subject: hwrng: core - Fix current_rng init/cleanup race yet again The kref solution is still buggy because we were only focusing on the register/unregister race. The same race affects the setting of current_rng through sysfs. This patch fixes it by using kref_get_unless_zero. Signed-off-by: Herbert Xu --- drivers/char/hw_random/core.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 3dba2cf50241..42827fd5f38d 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -105,7 +105,6 @@ static inline void cleanup_rng(struct kref *kref) static void set_current_rng(struct hwrng *rng) { BUG_ON(!mutex_is_locked(&rng_mutex)); - kref_get(&rng->ref); current_rng = rng; } @@ -150,6 +149,9 @@ static void put_rng(struct hwrng *rng) static inline int hwrng_init(struct hwrng *rng) { + if (kref_get_unless_zero(&rng->ref)) + goto skip_init; + if (rng->init) { int ret; @@ -157,6 +159,11 @@ static inline int hwrng_init(struct hwrng *rng) if (ret) return ret; } + + kref_init(&rng->ref); + reinit_completion(&rng->cleanup_done); + +skip_init: add_early_randomness(rng); current_quality = rng->quality ? : default_quality; @@ -467,6 +474,9 @@ int hwrng_register(struct hwrng *rng) goto out_unlock; } + init_completion(&rng->cleanup_done); + complete(&rng->cleanup_done); + old_rng = current_rng; err = 0; if (!old_rng) { @@ -494,8 +504,6 @@ int hwrng_register(struct hwrng *rng) add_early_randomness(rng); } - init_completion(&rng->cleanup_done); - out_unlock: mutex_unlock(&rng_mutex); out: -- cgit v1.2.3-58-ga151 From ac3a497f13e42a99ed6fe188ebf2dcc39eb7ac20 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 23 Dec 2014 16:40:19 +1100 Subject: hwrng: core - Do not register device opportunistically Currently we only register the device when a valid RNG is added. However the way it's done is buggy because we test whether there is a current RNG to determine whether we need to register. As the current RNG may be missing due to a reinitialisation error this can lead to a reregistration of the device. As the device already has to handle a NULL current RNG anyway, let's just register the device always and remove the complexity. Signed-off-by: Herbert Xu --- drivers/char/hw_random/core.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 42827fd5f38d..1d342f0cb0c1 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -372,14 +372,14 @@ static DEVICE_ATTR(rng_available, S_IRUGO, NULL); -static void unregister_miscdev(void) +static void __exit unregister_miscdev(void) { device_remove_file(rng_miscdev.this_device, &dev_attr_rng_available); device_remove_file(rng_miscdev.this_device, &dev_attr_rng_current); misc_deregister(&rng_miscdev); } -static int register_miscdev(void) +static int __init register_miscdev(void) { int err; @@ -484,12 +484,6 @@ int hwrng_register(struct hwrng *rng) if (err) goto out_unlock; set_current_rng(rng); - - err = register_miscdev(); - if (err) { - drop_current_rng(); - goto out_unlock; - } } list_add_tail(&rng->list, &rng_list); @@ -530,7 +524,6 @@ void hwrng_unregister(struct hwrng *rng) if (list_empty(&rng_list)) { mutex_unlock(&rng_mutex); - unregister_miscdev(); if (hwrng_fill) kthread_stop(hwrng_fill); } else @@ -540,16 +533,24 @@ void hwrng_unregister(struct hwrng *rng) } EXPORT_SYMBOL_GPL(hwrng_unregister); -static void __exit hwrng_exit(void) +static int __init hwrng_modinit(void) +{ + return register_miscdev(); +} + +static void __exit hwrng_modexit(void) { mutex_lock(&rng_mutex); BUG_ON(current_rng); kfree(rng_buffer); kfree(rng_fillbuf); mutex_unlock(&rng_mutex); + + unregister_miscdev(); } -module_exit(hwrng_exit); +module_init(hwrng_modinit); +module_exit(hwrng_modexit); MODULE_DESCRIPTION("H/W Random Number Generator (RNG) driver"); MODULE_LICENSE("GPL"); -- cgit v1.2.3-58-ga151 From ff77c150f71b761dcf29b9d1947df3165d2dc83e Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 23 Dec 2014 16:40:21 +1100 Subject: hwrng: core - Drop current rng in set_current_rng Rather than having callers of set_current_rng call drop_current_rng, we can do it directly in set_current_rng. Signed-off-by: Herbert Xu --- drivers/char/hw_random/core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 1d342f0cb0c1..787ef42a77b9 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -70,6 +70,7 @@ module_param(default_quality, ushort, 0644); MODULE_PARM_DESC(default_quality, "default entropy content of hwrng per mill"); +static void drop_current_rng(void); static void start_khwrngd(void); static inline int rng_get_data(struct hwrng *rng, u8 *buffer, size_t size, @@ -105,6 +106,7 @@ static inline void cleanup_rng(struct kref *kref) static void set_current_rng(struct hwrng *rng) { BUG_ON(!mutex_is_locked(&rng_mutex)); + drop_current_rng(); current_rng = rng; } @@ -315,7 +317,6 @@ static ssize_t hwrng_attr_current_store(struct device *dev, err = hwrng_init(rng); if (err) break; - drop_current_rng(); set_current_rng(rng); err = 0; break; -- cgit v1.2.3-58-ga151 From 90ac41bd40ad0571a10826eb26d53c84bd791f29 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 23 Dec 2014 16:40:22 +1100 Subject: hwrng: core - Move hwrng_init call into set_current_rng We always do hwrng_init in set_current_rng. In fact, our current reference count system relies on this. So make this explicit by moving hwrng_init into set_current_rng. Signed-off-by: Herbert Xu --- drivers/char/hw_random/core.c | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 787ef42a77b9..32a8a867f7f8 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -71,6 +71,7 @@ MODULE_PARM_DESC(default_quality, "default entropy content of hwrng per mill"); static void drop_current_rng(void); +static int hwrng_init(struct hwrng *rng); static void start_khwrngd(void); static inline int rng_get_data(struct hwrng *rng, u8 *buffer, size_t size, @@ -103,11 +104,20 @@ static inline void cleanup_rng(struct kref *kref) complete(&rng->cleanup_done); } -static void set_current_rng(struct hwrng *rng) +static int set_current_rng(struct hwrng *rng) { + int err; + BUG_ON(!mutex_is_locked(&rng_mutex)); + + err = hwrng_init(rng); + if (err) + return err; + drop_current_rng(); current_rng = rng; + + return 0; } static void drop_current_rng(void) @@ -149,7 +159,7 @@ static void put_rng(struct hwrng *rng) mutex_unlock(&rng_mutex); } -static inline int hwrng_init(struct hwrng *rng) +static int hwrng_init(struct hwrng *rng) { if (kref_get_unless_zero(&rng->ref)) goto skip_init; @@ -310,15 +320,9 @@ static ssize_t hwrng_attr_current_store(struct device *dev, err = -ENODEV; list_for_each_entry(rng, &rng_list, list) { if (strcmp(rng->name, buf) == 0) { - if (rng == current_rng) { - err = 0; - break; - } - err = hwrng_init(rng); - if (err) - break; - set_current_rng(rng); err = 0; + if (rng != current_rng) + err = set_current_rng(rng); break; } } @@ -481,10 +485,9 @@ int hwrng_register(struct hwrng *rng) old_rng = current_rng; err = 0; if (!old_rng) { - err = hwrng_init(rng); + err = set_current_rng(rng); if (err) goto out_unlock; - set_current_rng(rng); } list_add_tail(&rng->list, &rng_list); @@ -518,8 +521,7 @@ void hwrng_unregister(struct hwrng *rng) tail = list_entry(rng_list.prev, struct hwrng, list); - if (hwrng_init(tail) == 0) - set_current_rng(tail); + set_current_rng(tail); } } -- cgit v1.2.3-58-ga151 From ad202c8c1563da4dda9416ca0ea1e0b94430f759 Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Tue, 23 Dec 2014 09:34:03 +0100 Subject: crypto: af_alg - zeroize key data alg_setkey should zeroize the sensitive data after use. Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/af_alg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/af_alg.c b/crypto/af_alg.c index a8ff3c44e13c..76d739d08211 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -188,7 +188,7 @@ static int alg_setkey(struct sock *sk, char __user *ukey, err = type->setkey(ask->private, key, keylen); out: - sock_kfree_s(sk, key, keylen); + sock_kzfree_s(sk, key, keylen); return err; } -- cgit v1.2.3-58-ga151 From 5afdfd22e6ba2260129a2a7113ab0916339c4205 Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Thu, 25 Dec 2014 23:00:06 +0100 Subject: crypto: algif_rng - add random number generator support This patch adds the random number generator support for AF_ALG. A random number generator's purpose is to generate data without requiring the caller to provide any data. Therefore, the AF_ALG interface handler for RNGs only implements a callback handler for recvmsg. The following parameters provided with a recvmsg are processed by the RNG callback handler: * sock - to resolve the RNG context data structure accessing the RNG instance private to the socket * len - this parameter allows userspace callers to specify how many random bytes the RNG shall produce and return. As the kernel context for the RNG allocates a buffer of 128 bytes to store random numbers before copying them to userspace, the len parameter is checked that it is not larger than 128. If a caller wants more random numbers, a new request for recvmsg shall be made. The size of 128 bytes is chose because of the following considerations: * to increase the memory footprint of the kernel too much (note, that would be 128 bytes per open socket) * 128 is divisible by any typical cryptographic block size an RNG may have * A request for random numbers typically only shall supply small amount of data like for keys or IVs that should only require one invocation of the recvmsg function. Note, during instantiation of the RNG, the code checks whether the RNG implementation requires seeding. If so, the RNG is seeded with output from get_random_bytes. A fully working example using all aspects of the RNG interface is provided at http://www.chronox.de/libkcapi.html Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/algif_rng.c | 192 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 crypto/algif_rng.c diff --git a/crypto/algif_rng.c b/crypto/algif_rng.c new file mode 100644 index 000000000000..91c06f5673dd --- /dev/null +++ b/crypto/algif_rng.c @@ -0,0 +1,192 @@ +/* + * algif_rng: User-space interface for random number generators + * + * This file provides the user-space API for random number generators. + * + * Copyright (C) 2014, Stephan Mueller + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, and the entire permission notice in its entirety, + * including the disclaimer of warranties. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * ALTERNATIVELY, this product may be distributed under the terms of + * the GNU General Public License, in which case the provisions of the GPL2 + * are required INSTEAD OF the above restrictions. (This clause is + * necessary due to a potential bad interaction between the GPL and + * the restrictions contained in a BSD-style copyright.) + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF + * WHICH ARE HEREBY DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT + * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE + * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH + * DAMAGE. + */ + +#include +#include +#include +#include +#include +#include + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Stephan Mueller "); +MODULE_DESCRIPTION("User-space interface for random number generators"); + +struct rng_ctx { +#define MAXSIZE 128 + unsigned int len; + struct crypto_rng *drng; +}; + +static int rng_recvmsg(struct kiocb *unused, struct socket *sock, + struct msghdr *msg, size_t len, int flags) +{ + struct sock *sk = sock->sk; + struct alg_sock *ask = alg_sk(sk); + struct rng_ctx *ctx = ask->private; + int err = -EFAULT; + int genlen = 0; + u8 result[MAXSIZE]; + + if (len == 0) + return 0; + if (len > MAXSIZE) + len = MAXSIZE; + + /* + * although not strictly needed, this is a precaution against coding + * errors + */ + memset(result, 0, len); + + /* + * The enforcement of a proper seeding of an RNG is done within an + * RNG implementation. Some RNGs (DRBG, krng) do not need specific + * seeding as they automatically seed. The X9.31 DRNG will return + * an error if it was not seeded properly. + */ + genlen = crypto_rng_get_bytes(ctx->drng, result, len); + if (genlen < 0) + return genlen; + + err = memcpy_to_msg(msg, result, len); + memzero_explicit(result, genlen); + + return err ? err : len; +} + +static struct proto_ops algif_rng_ops = { + .family = PF_ALG, + + .connect = sock_no_connect, + .socketpair = sock_no_socketpair, + .getname = sock_no_getname, + .ioctl = sock_no_ioctl, + .listen = sock_no_listen, + .shutdown = sock_no_shutdown, + .getsockopt = sock_no_getsockopt, + .mmap = sock_no_mmap, + .bind = sock_no_bind, + .accept = sock_no_accept, + .setsockopt = sock_no_setsockopt, + .poll = sock_no_poll, + .sendmsg = sock_no_sendmsg, + .sendpage = sock_no_sendpage, + + .release = af_alg_release, + .recvmsg = rng_recvmsg, +}; + +static void *rng_bind(const char *name, u32 type, u32 mask) +{ + return crypto_alloc_rng(name, type, mask); +} + +static void rng_release(void *private) +{ + crypto_free_rng(private); +} + +static void rng_sock_destruct(struct sock *sk) +{ + struct alg_sock *ask = alg_sk(sk); + struct rng_ctx *ctx = ask->private; + + sock_kfree_s(sk, ctx, ctx->len); + af_alg_release_parent(sk); +} + +static int rng_accept_parent(void *private, struct sock *sk) +{ + struct rng_ctx *ctx; + struct alg_sock *ask = alg_sk(sk); + unsigned int len = sizeof(*ctx); + + ctx = sock_kmalloc(sk, len, GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + ctx->len = len; + + /* + * No seeding done at that point -- if multiple accepts are + * done on one RNG instance, each resulting FD points to the same + * state of the RNG. + */ + + ctx->drng = private; + ask->private = ctx; + sk->sk_destruct = rng_sock_destruct; + + return 0; +} + +static int rng_setkey(void *private, const u8 *seed, unsigned int seedlen) +{ + /* + * Check whether seedlen is of sufficient size is done in RNG + * implementations. + */ + return crypto_rng_reset(private, (u8 *)seed, seedlen); +} + +static const struct af_alg_type algif_type_rng = { + .bind = rng_bind, + .release = rng_release, + .accept = rng_accept_parent, + .setkey = rng_setkey, + .ops = &algif_rng_ops, + .name = "rng", + .owner = THIS_MODULE +}; + +static int __init rng_init(void) +{ + return af_alg_register_type(&algif_type_rng); +} + +void __exit rng_exit(void) +{ + int err = af_alg_unregister_type(&algif_type_rng); + BUG_ON(err); +} + +module_init(rng_init); +module_exit(rng_exit); -- cgit v1.2.3-58-ga151 From 2f3755381da8d592656f1ef6868fa9f96c450ba9 Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Thu, 25 Dec 2014 23:00:39 +0100 Subject: crypto: algif_rng - enable RNG interface compilation Enable compilation of the RNG AF_ALG support and provide a Kconfig option to compile the RNG AF_ALG support. Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/Kconfig | 9 +++++++++ crypto/Makefile | 1 + 2 files changed, 10 insertions(+) diff --git a/crypto/Kconfig b/crypto/Kconfig index 1618468b1fcb..50f4da44a304 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1514,6 +1514,15 @@ config CRYPTO_USER_API_SKCIPHER This option enables the user-spaces interface for symmetric key cipher algorithms. +config CRYPTO_USER_API_RNG + tristate "User-space interface for random number generator algorithms" + depends on NET + select CRYPTO_RNG + select CRYPTO_USER_API + help + This option enables the user-spaces interface for random + number generator algorithms. + config CRYPTO_HASH_INFO bool diff --git a/crypto/Makefile b/crypto/Makefile index 1445b9100c05..ba19465f9ad3 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -99,6 +99,7 @@ obj-$(CONFIG_CRYPTO_GHASH) += ghash-generic.o obj-$(CONFIG_CRYPTO_USER_API) += af_alg.o obj-$(CONFIG_CRYPTO_USER_API_HASH) += algif_hash.o obj-$(CONFIG_CRYPTO_USER_API_SKCIPHER) += algif_skcipher.o +obj-$(CONFIG_CRYPTO_USER_API_RNG) += algif_rng.o # # generic algorithms and the async_tx api -- cgit v1.2.3-58-ga151 From 1471f09f9b874e3bd6a439cae7fc34261dc6f7dd Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 5 Jan 2015 10:44:09 +1100 Subject: Revert "crypto: drbg - use memzero_explicit() for clearing sensitive data" This reverts commit 421d82f5b3e75f94e31875e37d45cdf6a557c120. None of the data zeroed are on the stack so the compiler cannot optimise them away. Signed-off-by: Herbert Xu --- crypto/drbg.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/crypto/drbg.c b/crypto/drbg.c index 96138396ce01..d8ff16e5c322 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -98,7 +98,6 @@ */ #include -#include /*************************************************************** * Backend cipher definitions available to DRBG @@ -491,9 +490,9 @@ static int drbg_ctr_df(struct drbg_state *drbg, ret = 0; out: - memzero_explicit(iv, drbg_blocklen(drbg)); - memzero_explicit(temp, drbg_statelen(drbg)); - memzero_explicit(pad, drbg_blocklen(drbg)); + memset(iv, 0, drbg_blocklen(drbg)); + memset(temp, 0, drbg_statelen(drbg)); + memset(pad, 0, drbg_blocklen(drbg)); return ret; } @@ -567,9 +566,9 @@ static int drbg_ctr_update(struct drbg_state *drbg, struct list_head *seed, ret = 0; out: - memzero_explicit(temp, drbg_statelen(drbg) + drbg_blocklen(drbg)); + memset(temp, 0, drbg_statelen(drbg) + drbg_blocklen(drbg)); if (2 != reseed) - memzero_explicit(df_data, drbg_statelen(drbg)); + memset(df_data, 0, drbg_statelen(drbg)); return ret; } @@ -627,7 +626,7 @@ static int drbg_ctr_generate(struct drbg_state *drbg, len = ret; out: - memzero_explicit(drbg->scratchpad, drbg_blocklen(drbg)); + memset(drbg->scratchpad, 0, drbg_blocklen(drbg)); return len; } @@ -865,7 +864,7 @@ static int drbg_hash_df(struct drbg_state *drbg, } out: - memzero_explicit(tmp, drbg_blocklen(drbg)); + memset(tmp, 0, drbg_blocklen(drbg)); return ret; } @@ -909,7 +908,7 @@ static int drbg_hash_update(struct drbg_state *drbg, struct list_head *seed, ret = drbg_hash_df(drbg, drbg->C, drbg_statelen(drbg), &datalist2); out: - memzero_explicit(drbg->scratchpad, drbg_statelen(drbg)); + memset(drbg->scratchpad, 0, drbg_statelen(drbg)); return ret; } @@ -944,7 +943,7 @@ static int drbg_hash_process_addtl(struct drbg_state *drbg, drbg->scratchpad, drbg_blocklen(drbg)); out: - memzero_explicit(drbg->scratchpad, drbg_blocklen(drbg)); + memset(drbg->scratchpad, 0, drbg_blocklen(drbg)); return ret; } @@ -991,7 +990,7 @@ static int drbg_hash_hashgen(struct drbg_state *drbg, } out: - memzero_explicit(drbg->scratchpad, + memset(drbg->scratchpad, 0, (drbg_statelen(drbg) + drbg_blocklen(drbg))); return len; } @@ -1040,7 +1039,7 @@ static int drbg_hash_generate(struct drbg_state *drbg, drbg_add_buf(drbg->V, drbg_statelen(drbg), u.req, 8); out: - memzero_explicit(drbg->scratchpad, drbg_blocklen(drbg)); + memset(drbg->scratchpad, 0, drbg_blocklen(drbg)); return len; } -- cgit v1.2.3-58-ga151 From 4c1ada872d3217c506c76f48a21decffe16c4ed4 Mon Sep 17 00:00:00 2001 From: Rickard Strandqvist Date: Thu, 1 Jan 2015 16:13:47 +0100 Subject: crypto: amcc - Remove unused function Remove the function get_dynamic_sa_offset_iv_field() that is not used anywhere. This was partially found by using a static code analysis program called cppcheck. Signed-off-by: Rickard Strandqvist Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_sa.c | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/drivers/crypto/amcc/crypto4xx_sa.c b/drivers/crypto/amcc/crypto4xx_sa.c index de8a7a48775a..69182e2cc3ea 100644 --- a/drivers/crypto/amcc/crypto4xx_sa.c +++ b/drivers/crypto/amcc/crypto4xx_sa.c @@ -34,29 +34,6 @@ #include "crypto4xx_sa.h" #include "crypto4xx_core.h" -u32 get_dynamic_sa_offset_iv_field(struct crypto4xx_ctx *ctx) -{ - u32 offset; - union dynamic_sa_contents cts; - - if (ctx->direction == DIR_INBOUND) - cts.w = ((struct dynamic_sa_ctl *)(ctx->sa_in))->sa_contents; - else - cts.w = ((struct dynamic_sa_ctl *)(ctx->sa_out))->sa_contents; - offset = cts.bf.key_size - + cts.bf.inner_size - + cts.bf.outer_size - + cts.bf.spi - + cts.bf.seq_num0 - + cts.bf.seq_num1 - + cts.bf.seq_num_mask0 - + cts.bf.seq_num_mask1 - + cts.bf.seq_num_mask2 - + cts.bf.seq_num_mask3; - - return sizeof(struct dynamic_sa_ctl) + offset * 4; -} - u32 get_dynamic_sa_offset_state_ptr_field(struct crypto4xx_ctx *ctx) { u32 offset; -- cgit v1.2.3-58-ga151 From 28c29f5657e886cf89778622af3f6569329508df Mon Sep 17 00:00:00 2001 From: Masanari Iida Date: Fri, 2 Jan 2015 12:40:46 +0900 Subject: crypto: bfin_crc - Remove unnecessary KERN_ERR in bfin_crc.c This patch removes unnecessary KERN_ERR from bfin_crypto_crc_mod_init(). Signed-off-by: Masanari Iida Signed-off-by: Herbert Xu --- drivers/crypto/bfin_crc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/bfin_crc.c b/drivers/crypto/bfin_crc.c index 9ae149bddb6e..33cf9eb87344 100644 --- a/drivers/crypto/bfin_crc.c +++ b/drivers/crypto/bfin_crc.c @@ -744,7 +744,7 @@ static int __init bfin_crypto_crc_mod_init(void) ret = platform_driver_register(&bfin_crypto_crc_driver); if (ret) { - pr_info(KERN_ERR "unable to register driver\n"); + pr_err("unable to register driver\n"); return ret; } -- cgit v1.2.3-58-ga151 From 42d2e780c6239040ec29d404f87e752de3627e08 Mon Sep 17 00:00:00 2001 From: Asaf Vertz Date: Mon, 5 Jan 2015 10:23:10 +0200 Subject: crypto: omap-des - fix BUG_ON condition dd->total is unsigned so it won't do any good to check for negative value after subtracting instead of that we should check if the subtracted value is bigger than him This was partially found by using a static code analysis program called cppcheck. Signed-off-by: Asaf Vertz Signed-off-by: Herbert Xu --- drivers/crypto/omap-des.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c index e350f5be4d2e..0b8dcf5eb74e 100644 --- a/drivers/crypto/omap-des.c +++ b/drivers/crypto/omap-des.c @@ -965,9 +965,9 @@ static irqreturn_t omap_des_irq(int irq, void *dev_id) } } - dd->total -= DES_BLOCK_SIZE; + BUG_ON(dd->total < DES_BLOCK_SIZE); - BUG_ON(dd->total < 0); + dd->total -= DES_BLOCK_SIZE; /* Clear IRQ status */ status &= ~DES_REG_IRQ_DATA_OUT; -- cgit v1.2.3-58-ga151 From 15acabfd02e35e270360fbe0def898e48754b3d6 Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Mon, 5 Jan 2015 12:21:45 +0100 Subject: crypto: aead - add check for presence of auth tag The AEAD decryption operation requires the authentication tag to be present as part of the cipher text buffer. The added check verifies that the caller provides a cipher text with at least the authentication tag. Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- include/linux/crypto.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/linux/crypto.h b/include/linux/crypto.h index 9c8776d0ada8..90998348e564 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -1412,6 +1412,9 @@ static inline int crypto_aead_encrypt(struct aead_request *req) */ static inline int crypto_aead_decrypt(struct aead_request *req) { + if (req->cryptlen < crypto_aead_authsize(crypto_aead_reqtfm(req))) + return -EINVAL; + return crypto_aead_crt(crypto_aead_reqtfm(req))->decrypt(req); } -- cgit v1.2.3-58-ga151 From ad511e260a27b8e35d273cc0ecfe5a8ff9543181 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 6 Jan 2015 07:54:41 +1100 Subject: crypto: qat - Fix incorrect uses of memzero_explicit memzero_explicit should only be used on stack variables that get zapped just before they go out of scope. This patch replaces all unnecessary uses of memzero_explicit with memset, removes two memzero_explicit calls altogether as the tfm context comes pre-zeroed, and adds a missing memzero_explicit of the stack variable buff in qat_alg_do_precomputes. The memzeros on ipad/opad + digest_size/auth_keylen are also removed as the entire auth_state is already zeroed on entry. Signed-off-by: Herbert Xu Acked-by: Tadeusz Struk --- drivers/crypto/qat/qat_common/qat_algs.c | 45 ++++++++++++++------------------ 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index f32d0a58bcc0..a0d95f329094 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -173,7 +173,7 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash, __be64 *hash512_state_out; int i, offset; - memzero_explicit(auth_state.data, MAX_AUTH_STATE_SIZE + 64); + memset(auth_state.data, 0, sizeof(auth_state.data)); shash->tfm = ctx->hash_tfm; shash->flags = 0x0; @@ -186,13 +186,10 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash, memcpy(ipad, buff, digest_size); memcpy(opad, buff, digest_size); - memzero_explicit(ipad + digest_size, block_size - digest_size); - memzero_explicit(opad + digest_size, block_size - digest_size); + memzero_explicit(buff, sizeof(buff)); } else { memcpy(ipad, auth_key, auth_keylen); memcpy(opad, auth_key, auth_keylen); - memzero_explicit(ipad + auth_keylen, block_size - auth_keylen); - memzero_explicit(opad + auth_keylen, block_size - auth_keylen); } for (i = 0; i < block_size; i++) { @@ -582,10 +579,10 @@ static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key, if (ctx->enc_cd) { /* rekeying */ dev = &GET_DEV(ctx->inst->accel_dev); - memzero_explicit(ctx->enc_cd, sizeof(*ctx->enc_cd)); - memzero_explicit(ctx->dec_cd, sizeof(*ctx->dec_cd)); - memzero_explicit(&ctx->enc_fw_req, sizeof(ctx->enc_fw_req)); - memzero_explicit(&ctx->dec_fw_req, sizeof(ctx->dec_fw_req)); + memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd)); + memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd)); + memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req)); + memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req)); } else { /* new key */ int node = get_current_node(); @@ -620,12 +617,12 @@ static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key, return 0; out_free_all: - memzero_explicit(ctx->dec_cd, sizeof(struct qat_alg_cd)); + memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd)); dma_free_coherent(dev, sizeof(struct qat_alg_cd), ctx->dec_cd, ctx->dec_cd_paddr); ctx->dec_cd = NULL; out_free_enc: - memzero_explicit(ctx->enc_cd, sizeof(struct qat_alg_cd)); + memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd)); dma_free_coherent(dev, sizeof(struct qat_alg_cd), ctx->enc_cd, ctx->enc_cd_paddr); ctx->enc_cd = NULL; @@ -969,10 +966,10 @@ static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm, if (ctx->enc_cd) { /* rekeying */ dev = &GET_DEV(ctx->inst->accel_dev); - memzero_explicit(ctx->enc_cd, sizeof(*ctx->enc_cd)); - memzero_explicit(ctx->dec_cd, sizeof(*ctx->dec_cd)); - memzero_explicit(&ctx->enc_fw_req, sizeof(ctx->enc_fw_req)); - memzero_explicit(&ctx->dec_fw_req, sizeof(ctx->dec_fw_req)); + memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd)); + memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd)); + memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req)); + memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req)); } else { /* new key */ int node = get_current_node(); @@ -1007,12 +1004,12 @@ static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm, return 0; out_free_all: - memzero_explicit(ctx->dec_cd, sizeof(*ctx->enc_cd)); + memset(ctx->dec_cd, 0, sizeof(*ctx->enc_cd)); dma_free_coherent(dev, sizeof(*ctx->enc_cd), ctx->dec_cd, ctx->dec_cd_paddr); ctx->dec_cd = NULL; out_free_enc: - memzero_explicit(ctx->enc_cd, sizeof(*ctx->dec_cd)); + memset(ctx->enc_cd, 0, sizeof(*ctx->dec_cd)); dma_free_coherent(dev, sizeof(*ctx->dec_cd), ctx->enc_cd, ctx->enc_cd_paddr); ctx->enc_cd = NULL; @@ -1101,7 +1098,6 @@ static int qat_alg_aead_init(struct crypto_tfm *tfm, { struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); - memzero_explicit(ctx, sizeof(*ctx)); ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); if (IS_ERR(ctx->hash_tfm)) return -EFAULT; @@ -1142,12 +1138,12 @@ static void qat_alg_aead_exit(struct crypto_tfm *tfm) dev = &GET_DEV(inst->accel_dev); if (ctx->enc_cd) { - memzero_explicit(ctx->enc_cd, sizeof(struct qat_alg_cd)); + memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd)); dma_free_coherent(dev, sizeof(struct qat_alg_cd), ctx->enc_cd, ctx->enc_cd_paddr); } if (ctx->dec_cd) { - memzero_explicit(ctx->dec_cd, sizeof(struct qat_alg_cd)); + memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd)); dma_free_coherent(dev, sizeof(struct qat_alg_cd), ctx->dec_cd, ctx->dec_cd_paddr); } @@ -1158,7 +1154,6 @@ static int qat_alg_ablkcipher_init(struct crypto_tfm *tfm) { struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm); - memzero_explicit(ctx, sizeof(*ctx)); spin_lock_init(&ctx->lock); tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) + sizeof(struct qat_crypto_request); @@ -1177,15 +1172,15 @@ static void qat_alg_ablkcipher_exit(struct crypto_tfm *tfm) dev = &GET_DEV(inst->accel_dev); if (ctx->enc_cd) { - memzero_explicit(ctx->enc_cd, - sizeof(struct icp_qat_hw_cipher_algo_blk)); + memset(ctx->enc_cd, 0, + sizeof(struct icp_qat_hw_cipher_algo_blk)); dma_free_coherent(dev, sizeof(struct icp_qat_hw_cipher_algo_blk), ctx->enc_cd, ctx->enc_cd_paddr); } if (ctx->dec_cd) { - memzero_explicit(ctx->dec_cd, - sizeof(struct icp_qat_hw_cipher_algo_blk)); + memset(ctx->dec_cd, 0, + sizeof(struct icp_qat_hw_cipher_algo_blk)); dma_free_coherent(dev, sizeof(struct icp_qat_hw_cipher_algo_blk), ctx->dec_cd, ctx->dec_cd_paddr); -- cgit v1.2.3-58-ga151 From 8155330aad477c5b1337895a6922df76817f0874 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 6 Jan 2015 00:27:45 +0100 Subject: lib: memzero_explicit: add comment for its usage Lets improve the comment to add a note on when to use memzero_explicit() for those not digging through the git logs. We don't want people to pollute places with memzero_explicit() where it's not really necessary. Reference: https://lkml.org/lkml/2015/1/4/190 Suggested-by: Herbert Xu Signed-off-by: Daniel Borkmann Signed-off-by: Herbert Xu --- lib/string.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/string.c b/lib/string.c index 10063300b830..d984ec4fd3b7 100644 --- a/lib/string.c +++ b/lib/string.c @@ -604,6 +604,11 @@ EXPORT_SYMBOL(memset); * @s: Pointer to the start of the area. * @count: The size of the area. * + * Note: usually using memset() is just fine (!), but in cases + * where clearing out _local_ data at the end of a scope is + * necessary, memzero_explicit() should be used instead in + * order to prevent the compiler from optimising away zeroing. + * * memzero_explicit() doesn't need an arch-specific version as * it just invokes the one of memset() implicitly. */ -- cgit v1.2.3-58-ga151 From 8a45ac12ec5b6ee67f8559c78ae11d9af8b821ee Mon Sep 17 00:00:00 2001 From: Rabin Vincent Date: Fri, 9 Jan 2015 16:25:28 +0100 Subject: crypto: testmgr - don't use interruptible wait in tests tcrypt/testmgr uses wait_for_completion_interruptible() everywhere when it waits for a request to be completed. If it's interrupted, then the test is aborted and the request is freed. However, if any of these calls actually do get interrupted, the result will likely be a kernel crash, when the driver handles the now-freed request. Use wait_for_completion() instead. Signed-off-by: Rabin Vincent Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 10 ++++------ crypto/testmgr.c | 50 ++++++++++++++++++++++---------------------------- 2 files changed, 26 insertions(+), 34 deletions(-) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 1d864e988ea9..004349576ba1 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -764,10 +764,9 @@ static inline int do_one_ahash_op(struct ahash_request *req, int ret) if (ret == -EINPROGRESS || ret == -EBUSY) { struct tcrypt_result *tr = req->base.data; - ret = wait_for_completion_interruptible(&tr->completion); - if (!ret) - ret = tr->err; + wait_for_completion(&tr->completion); reinit_completion(&tr->completion); + ret = tr->err; } return ret; } @@ -993,10 +992,9 @@ static inline int do_one_acipher_op(struct ablkcipher_request *req, int ret) if (ret == -EINPROGRESS || ret == -EBUSY) { struct tcrypt_result *tr = req->base.data; - ret = wait_for_completion_interruptible(&tr->completion); - if (!ret) - ret = tr->err; + wait_for_completion(&tr->completion); reinit_completion(&tr->completion); + ret = tr->err; } return ret; diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 037368d34586..235b1fff04c4 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -181,10 +181,9 @@ static void testmgr_free_buf(char *buf[XBUFSIZE]) static int wait_async_op(struct tcrypt_result *tr, int ret) { if (ret == -EINPROGRESS || ret == -EBUSY) { - ret = wait_for_completion_interruptible(&tr->completion); - if (!ret) - ret = tr->err; + wait_for_completion(&tr->completion); reinit_completion(&tr->completion); + ret = tr->err; } return ret; } @@ -353,12 +352,11 @@ static int __test_hash(struct crypto_ahash *tfm, struct hash_testvec *template, break; case -EINPROGRESS: case -EBUSY: - ret = wait_for_completion_interruptible( - &tresult.completion); - if (!ret && !(ret = tresult.err)) { - reinit_completion(&tresult.completion); + wait_for_completion(&tresult.completion); + reinit_completion(&tresult.completion); + ret = tresult.err; + if (!ret) break; - } /* fall through */ default: printk(KERN_ERR "alg: hash: digest failed " @@ -569,12 +567,11 @@ static int __test_aead(struct crypto_aead *tfm, int enc, break; case -EINPROGRESS: case -EBUSY: - ret = wait_for_completion_interruptible( - &result.completion); - if (!ret && !(ret = result.err)) { - reinit_completion(&result.completion); + wait_for_completion(&result.completion); + reinit_completion(&result.completion); + ret = result.err; + if (!ret) break; - } case -EBADMSG: if (template[i].novrfy) /* verification failure was expected */ @@ -720,12 +717,11 @@ static int __test_aead(struct crypto_aead *tfm, int enc, break; case -EINPROGRESS: case -EBUSY: - ret = wait_for_completion_interruptible( - &result.completion); - if (!ret && !(ret = result.err)) { - reinit_completion(&result.completion); + wait_for_completion(&result.completion); + reinit_completion(&result.completion); + ret = result.err; + if (!ret) break; - } case -EBADMSG: if (template[i].novrfy) /* verification failure was expected */ @@ -1002,12 +998,11 @@ static int __test_skcipher(struct crypto_ablkcipher *tfm, int enc, break; case -EINPROGRESS: case -EBUSY: - ret = wait_for_completion_interruptible( - &result.completion); - if (!ret && !((ret = result.err))) { - reinit_completion(&result.completion); + wait_for_completion(&result.completion); + reinit_completion(&result.completion); + ret = result.err; + if (!ret) break; - } /* fall through */ default: pr_err("alg: skcipher%s: %s failed on test %d for %s: ret=%d\n", @@ -1097,12 +1092,11 @@ static int __test_skcipher(struct crypto_ablkcipher *tfm, int enc, break; case -EINPROGRESS: case -EBUSY: - ret = wait_for_completion_interruptible( - &result.completion); - if (!ret && !((ret = result.err))) { - reinit_completion(&result.completion); + wait_for_completion(&result.completion); + reinit_completion(&result.completion); + ret = result.err; + if (!ret) break; - } /* fall through */ default: pr_err("alg: skcipher%s: %s failed on chunk test %d for %s: ret=%d\n", -- cgit v1.2.3-58-ga151 From 22e4dda06dd0fa2a56e573049411479a1f759cbb Mon Sep 17 00:00:00 2001 From: "Allan, Bruce W" Date: Fri, 9 Jan 2015 11:54:58 -0800 Subject: crypto: qat - fix device reset flow When the device needs a reset, e.g. when an uncorrectable PCIe AER event occurs, various services/data structures need to be cleaned up, the hardware reset and the services/data structures initialized and started. The code to perform the cleanup and initialization was not performed when a device reset was done. This patch moves some of the initialization code out of the .probe entry- point into a separate function that is now called during probe as well as after the hardware has been reset. Similarly, a new function is added for first cleaning up these services/data structures prior to resetting. The new functions are adf_dev_init() and adf_dev_shutdown(), respectively, for which there are already prototypes but no actual functions just yet and are now called when the device is reset and during probe/cleanup of the driver. The down and up flows via ioctl calls has similarly been updated. In addition, there are two other bugs in the reset flow - one in the logic for determining whether to schedule a device reset upon receiving an uncorrectable AER event which prevents the reset flow from being initiated, and another with clearing the status bit indicating a device is configured (when resetting the device the configuration remains across the reset so the bit should not be cleared, otherwise, the necessary services will not be re-started in adf_dev_start() after the reset - clear the bit only when actually deleting the configuration). Signed-off-by: Bruce Allan Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_accel_devices.h | 5 ++ drivers/crypto/qat/qat_common/adf_aer.c | 7 +- drivers/crypto/qat/qat_common/adf_cfg.c | 2 + drivers/crypto/qat/qat_common/adf_common_drv.h | 2 +- drivers/crypto/qat/qat_common/adf_ctl_drv.c | 7 +- drivers/crypto/qat/qat_common/adf_init.c | 91 +++++++++++++++++++--- .../crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 19 +++++ drivers/crypto/qat/qat_dh895xcc/adf_drv.c | 42 +++------- 8 files changed, 128 insertions(+), 47 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 2ed425664a16..ed226cee6e29 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -148,6 +148,11 @@ struct adf_hw_device_data { int (*alloc_irq)(struct adf_accel_dev *accel_dev); void (*free_irq)(struct adf_accel_dev *accel_dev); void (*enable_error_correction)(struct adf_accel_dev *accel_dev); + int (*init_admin_comms)(struct adf_accel_dev *accel_dev); + void (*exit_admin_comms)(struct adf_accel_dev *accel_dev); + int (*init_arb)(struct adf_accel_dev *accel_dev); + void (*exit_arb)(struct adf_accel_dev *accel_dev); + void (*enable_ints)(struct adf_accel_dev *accel_dev); const char *fw_name; uint32_t pci_dev_id; uint32_t fuses; diff --git a/drivers/crypto/qat/qat_common/adf_aer.c b/drivers/crypto/qat/qat_common/adf_aer.c index 10ce4a2854ab..740dc9e1c70c 100644 --- a/drivers/crypto/qat/qat_common/adf_aer.c +++ b/drivers/crypto/qat/qat_common/adf_aer.c @@ -125,8 +125,9 @@ static void adf_device_reset_worker(struct work_struct *work) adf_dev_restarting_notify(accel_dev); adf_dev_stop(accel_dev); + adf_dev_shutdown(accel_dev); adf_dev_restore(accel_dev); - if (adf_dev_start(accel_dev)) { + if (adf_dev_init(accel_dev) || adf_dev_start(accel_dev)) { /* The device hanged and we can't restart it so stop here */ dev_err(&GET_DEV(accel_dev), "Restart device failed\n"); kfree(reset_data); @@ -148,8 +149,8 @@ static int adf_dev_aer_schedule_reset(struct adf_accel_dev *accel_dev, { struct adf_reset_dev_data *reset_data; - if (adf_dev_started(accel_dev) && - !test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) + if (!adf_dev_started(accel_dev) || + test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) return 0; set_bit(ADF_STATUS_RESTARTING, &accel_dev->status); diff --git a/drivers/crypto/qat/qat_common/adf_cfg.c b/drivers/crypto/qat/qat_common/adf_cfg.c index aba7f1d043fb..de16da9070a5 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg.c +++ b/drivers/crypto/qat/qat_common/adf_cfg.c @@ -50,6 +50,7 @@ #include #include "adf_accel_devices.h" #include "adf_cfg.h" +#include "adf_common_drv.h" static DEFINE_MUTEX(qat_cfg_read_lock); @@ -159,6 +160,7 @@ void adf_cfg_del_all(struct adf_accel_dev *accel_dev) down_write(&dev_cfg_data->lock); adf_cfg_section_del_all(&dev_cfg_data->sec_list); up_write(&dev_cfg_data->lock); + clear_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); } /** diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index 5e8f9d431e5d..a62e485c8786 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -93,7 +93,7 @@ int adf_service_unregister(struct service_hndl *service); int adf_dev_init(struct adf_accel_dev *accel_dev); int adf_dev_start(struct adf_accel_dev *accel_dev); int adf_dev_stop(struct adf_accel_dev *accel_dev); -int adf_dev_shutdown(struct adf_accel_dev *accel_dev); +void adf_dev_shutdown(struct adf_accel_dev *accel_dev); int adf_ctl_dev_register(void); void adf_ctl_dev_unregister(void); diff --git a/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/qat/qat_common/adf_ctl_drv.c index 7ee93f881db6..74207a6f0516 100644 --- a/drivers/crypto/qat/qat_common/adf_ctl_drv.c +++ b/drivers/crypto/qat/qat_common/adf_ctl_drv.c @@ -282,6 +282,8 @@ static int adf_ctl_stop_devices(uint32_t id) if (adf_dev_stop(accel_dev)) { pr_err("QAT: Failed to stop qat_dev%d\n", id); ret = -EFAULT; + } else { + adf_dev_shutdown(accel_dev); } } } @@ -343,7 +345,9 @@ static int adf_ctl_ioctl_dev_start(struct file *fp, unsigned int cmd, if (!adf_dev_started(accel_dev)) { pr_info("QAT: Starting acceleration device qat_dev%d.\n", ctl_data->device_id); - ret = adf_dev_start(accel_dev); + ret = adf_dev_init(accel_dev); + if (!ret) + ret = adf_dev_start(accel_dev); } else { pr_info("QAT: Acceleration device qat_dev%d already started.\n", ctl_data->device_id); @@ -351,6 +355,7 @@ static int adf_ctl_ioctl_dev_start(struct file *fp, unsigned int cmd, if (ret) { pr_err("QAT: Failed to start qat_dev%d\n", ctl_data->device_id); adf_dev_stop(accel_dev); + adf_dev_shutdown(accel_dev); } out: kfree(ctl_data); diff --git a/drivers/crypto/qat/qat_common/adf_init.c b/drivers/crypto/qat/qat_common/adf_init.c index 5c0e47a00a87..a3afa0ff9ec6 100644 --- a/drivers/crypto/qat/qat_common/adf_init.c +++ b/drivers/crypto/qat/qat_common/adf_init.c @@ -108,26 +108,47 @@ int adf_service_unregister(struct service_hndl *service) EXPORT_SYMBOL_GPL(adf_service_unregister); /** - * adf_dev_start() - Start acceleration service for the given accel device - * @accel_dev: Pointer to acceleration device. + * adf_dev_init() - Init data structures and services for the given accel device + * @accel_dev: Pointer to acceleration device. * - * Function notifies all the registered services that the acceleration device - * is ready to be used. - * To be used by QAT device specific drivers. + * Initialize the ring data structures and the admin comms and arbitration + * services. * * Return: 0 on success, error code othewise. */ -int adf_dev_start(struct adf_accel_dev *accel_dev) +int adf_dev_init(struct adf_accel_dev *accel_dev) { struct service_hndl *service; struct list_head *list_itr; struct adf_hw_device_data *hw_data = accel_dev->hw_device; + if (!hw_data) { + dev_err(&GET_DEV(accel_dev), + "QAT: Failed to init device - hw_data not set\n"); + return -EFAULT; + } + if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status)) { pr_info("QAT: Device not configured\n"); return -EFAULT; } - set_bit(ADF_STATUS_STARTING, &accel_dev->status); + + if (adf_init_etr_data(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Failed initialize etr\n"); + return -EFAULT; + } + + if (hw_data->init_admin_comms && hw_data->init_admin_comms(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Failed initialize admin comms\n"); + return -EFAULT; + } + + if (hw_data->init_arb && hw_data->init_arb(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Failed initialize hw arbiter\n"); + return -EFAULT; + } + + hw_data->enable_ints(accel_dev); if (adf_ae_init(accel_dev)) { pr_err("QAT: Failed to initialise Acceleration Engine\n"); @@ -178,6 +199,27 @@ int adf_dev_start(struct adf_accel_dev *accel_dev) hw_data->enable_error_correction(accel_dev); + return 0; +} +EXPORT_SYMBOL_GPL(adf_dev_init); + +/** + * adf_dev_start() - Start acceleration service for the given accel device + * @accel_dev: Pointer to acceleration device. + * + * Function notifies all the registered services that the acceleration device + * is ready to be used. + * To be used by QAT device specific drivers. + * + * Return: 0 on success, error code othewise. + */ +int adf_dev_start(struct adf_accel_dev *accel_dev) +{ + struct service_hndl *service; + struct list_head *list_itr; + + set_bit(ADF_STATUS_STARTING, &accel_dev->status); + if (adf_ae_start(accel_dev)) { pr_err("QAT: AE Start Failed\n"); return -EFAULT; @@ -232,7 +274,6 @@ EXPORT_SYMBOL_GPL(adf_dev_start); */ int adf_dev_stop(struct adf_accel_dev *accel_dev) { - struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct service_hndl *service; struct list_head *list_itr; int ret, wait = 0; @@ -241,7 +282,6 @@ int adf_dev_stop(struct adf_accel_dev *accel_dev) !test_bit(ADF_STATUS_STARTING, &accel_dev->status)) { return 0; } - clear_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); clear_bit(ADF_STATUS_STARTING, &accel_dev->status); clear_bit(ADF_STATUS_STARTED, &accel_dev->status); @@ -285,6 +325,29 @@ int adf_dev_stop(struct adf_accel_dev *accel_dev) clear_bit(ADF_STATUS_AE_STARTED, &accel_dev->status); } + return 0; +} +EXPORT_SYMBOL_GPL(adf_dev_stop); + +/** + * adf_dev_shutdown() - shutdown acceleration services and data strucutures + * @accel_dev: Pointer to acceleration device + * + * Cleanup the ring data structures and the admin comms and arbitration + * services. + */ +void adf_dev_shutdown(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct service_hndl *service; + struct list_head *list_itr; + + if (!hw_data) { + dev_err(&GET_DEV(accel_dev), + "QAT: Failed to shutdown device - hw_data not set\n"); + return; + } + if (test_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status)) { if (adf_ae_fw_release(accel_dev)) pr_err("QAT: Failed to release the ucode\n"); @@ -335,9 +398,15 @@ int adf_dev_stop(struct adf_accel_dev *accel_dev) if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) adf_cfg_del_all(accel_dev); - return 0; + if (hw_data->exit_arb) + hw_data->exit_arb(accel_dev); + + if (hw_data->exit_admin_comms) + hw_data->exit_admin_comms(accel_dev); + + adf_cleanup_etr_data(accel_dev); } -EXPORT_SYMBOL_GPL(adf_dev_stop); +EXPORT_SYMBOL_GPL(adf_dev_shutdown); int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev) { diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index ef05825cc651..6a735d5c0e37 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -46,6 +46,7 @@ */ #include #include "adf_dh895xcc_hw_data.h" +#include "adf_common_drv.h" #include "adf_drv.h" /* Worker thread to service arbiter mappings based on dev SKUs */ @@ -182,6 +183,19 @@ static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) } } +static void adf_enable_ints(struct adf_accel_dev *accel_dev) +{ + void __iomem *addr; + + addr = (&GET_BARS(accel_dev)[ADF_DH895XCC_PMISC_BAR])->virt_addr; + + /* Enable bundle and misc interrupts */ + ADF_CSR_WR(addr, ADF_DH895XCC_SMIAPF0_MASK_OFFSET, + ADF_DH895XCC_SMIA0_MASK); + ADF_CSR_WR(addr, ADF_DH895XCC_SMIAPF1_MASK_OFFSET, + ADF_DH895XCC_SMIA1_MASK); +} + void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &dh895xcc_class; @@ -206,6 +220,11 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->get_misc_bar_id = get_misc_bar_id; hw_data->get_sku = get_sku; hw_data->fw_name = ADF_DH895XCC_FW; + hw_data->init_admin_comms = adf_init_admin_comms; + hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->init_arb = adf_init_arb; + hw_data->exit_arb = adf_exit_arb; + hw_data->enable_ints = adf_enable_ints; } void adf_clean_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c index 948f66be262b..8ffdb95c9804 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c @@ -90,9 +90,7 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; int i; - adf_exit_admin_comms(accel_dev); - adf_exit_arb(accel_dev); - adf_cleanup_etr_data(accel_dev); + adf_dev_shutdown(accel_dev); for (i = 0; i < ADF_PCI_MAX_BARS; i++) { struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; @@ -119,7 +117,7 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) kfree(accel_dev); } -static int qat_dev_start(struct adf_accel_dev *accel_dev) +static int adf_dev_configure(struct adf_accel_dev *accel_dev) { int cpus = num_online_cpus(); int banks = GET_MAX_BANKS(accel_dev); @@ -206,7 +204,7 @@ static int qat_dev_start(struct adf_accel_dev *accel_dev) goto err; set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); - return adf_dev_start(accel_dev); + return 0; err: dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n"); return -EINVAL; @@ -217,7 +215,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) struct adf_accel_dev *accel_dev; struct adf_accel_pci *accel_pci_dev; struct adf_hw_device_data *hw_data; - void __iomem *pmisc_bar_addr = NULL; char name[ADF_DEVICE_NAME_LENGTH]; unsigned int i, bar_nr; int ret; @@ -347,8 +344,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) ret = -EFAULT; goto out_err; } - if (i == ADF_DH895XCC_PMISC_BAR) - pmisc_bar_addr = bar->virt_addr; } pci_set_master(pdev); @@ -358,36 +353,21 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err; } - if (adf_init_etr_data(accel_dev)) { - dev_err(&pdev->dev, "Failed initialize etr\n"); - ret = -EFAULT; - goto out_err; - } - - if (adf_init_admin_comms(accel_dev)) { - dev_err(&pdev->dev, "Failed initialize admin comms\n"); - ret = -EFAULT; - goto out_err; - } - - if (adf_init_arb(accel_dev)) { - dev_err(&pdev->dev, "Failed initialize hw arbiter\n"); - ret = -EFAULT; - goto out_err; - } if (pci_save_state(pdev)) { dev_err(&pdev->dev, "Failed to save pci state\n"); ret = -ENOMEM; goto out_err; } - /* Enable bundle and misc interrupts */ - ADF_CSR_WR(pmisc_bar_addr, ADF_DH895XCC_SMIAPF0_MASK_OFFSET, - ADF_DH895XCC_SMIA0_MASK); - ADF_CSR_WR(pmisc_bar_addr, ADF_DH895XCC_SMIAPF1_MASK_OFFSET, - ADF_DH895XCC_SMIA1_MASK); + ret = adf_dev_configure(accel_dev); + if (ret) + goto out_err; + + ret = adf_dev_init(accel_dev); + if (ret) + goto out_err; - ret = qat_dev_start(accel_dev); + ret = adf_dev_start(accel_dev); if (ret) { adf_dev_stop(accel_dev); goto out_err; -- cgit v1.2.3-58-ga151 From 53bc0251b111dce0820e3ae94fe6bb8d1cc16cd9 Mon Sep 17 00:00:00 2001 From: "Allan, Bruce W" Date: Fri, 9 Jan 2015 11:55:04 -0800 Subject: crypto: qat - correctly type a boolean Signed-off-by: Bruce Allan Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_init.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_init.c b/drivers/crypto/qat/qat_common/adf_init.c index a3afa0ff9ec6..edbf85af4ea5 100644 --- a/drivers/crypto/qat/qat_common/adf_init.c +++ b/drivers/crypto/qat/qat_common/adf_init.c @@ -276,7 +276,8 @@ int adf_dev_stop(struct adf_accel_dev *accel_dev) { struct service_hndl *service; struct list_head *list_itr; - int ret, wait = 0; + bool wait = false; + int ret; if (!adf_dev_started(accel_dev) && !test_bit(ADF_STATUS_STARTING, &accel_dev->status)) { @@ -298,7 +299,7 @@ int adf_dev_stop(struct adf_accel_dev *accel_dev) if (!ret) { clear_bit(accel_dev->accel_id, &service->start_status); } else if (ret == -EAGAIN) { - wait = 1; + wait = true; clear_bit(accel_dev->accel_id, &service->start_status); } } -- cgit v1.2.3-58-ga151 From fd98d692bb2bdd22f76bf1449df7131cefddccae Mon Sep 17 00:00:00 2001 From: "Allan, Bruce W" Date: Fri, 9 Jan 2015 11:55:09 -0800 Subject: crypto: qat - adf_ae_stop() is never called In adf_dev_stop(), adf_ae_stop() is never called because adf_dev_started() will always return false since the ADF_STATUS_STARTED bit is cleared earlier in the function. Signed-off-by: Bruce Allan Acked-by: Tadeusz Struk Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/qat/qat_common/adf_init.c b/drivers/crypto/qat/qat_common/adf_init.c index edbf85af4ea5..8f0ca498ab87 100644 --- a/drivers/crypto/qat/qat_common/adf_init.c +++ b/drivers/crypto/qat/qat_common/adf_init.c @@ -319,7 +319,7 @@ int adf_dev_stop(struct adf_accel_dev *accel_dev) if (wait) msleep(100); - if (adf_dev_started(accel_dev)) { + if (test_bit(ADF_STATUS_AE_STARTED, &accel_dev->status)) { if (adf_ae_stop(accel_dev)) pr_err("QAT: failed to stop AE\n"); else -- cgit v1.2.3-58-ga151 From a6bcc1e443420f370881ec0b38aea636c1bf71c7 Mon Sep 17 00:00:00 2001 From: "Allan, Bruce W" Date: Fri, 9 Jan 2015 11:55:14 -0800 Subject: crypto: qat - use pci_wait_for_pending_transaction() Prior to resetting the hardware, use pci_wait_for_pending_transaction() instead of open coding similar functionality. Signed-off-by: Bruce Allan Acked-by: Tadeusz Struk Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_aer.c | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_aer.c b/drivers/crypto/qat/qat_common/adf_aer.c index 740dc9e1c70c..fa1fef824de2 100644 --- a/drivers/crypto/qat/qat_common/adf_aer.c +++ b/drivers/crypto/qat/qat_common/adf_aer.c @@ -82,28 +82,15 @@ struct adf_reset_dev_data { struct work_struct reset_work; }; -#define PPDSTAT_OFFSET 0x7E static void adf_dev_restore(struct adf_accel_dev *accel_dev) { struct pci_dev *pdev = accel_to_pci_dev(accel_dev); struct pci_dev *parent = pdev->bus->self; - uint16_t ppdstat = 0, bridge_ctl = 0; - int pending = 0; + uint16_t bridge_ctl = 0; pr_info("QAT: Resetting device qat_dev%d\n", accel_dev->accel_id); - pci_read_config_word(pdev, PPDSTAT_OFFSET, &ppdstat); - pending = ppdstat & PCI_EXP_DEVSTA_TRPND; - if (pending) { - int ctr = 0; - - do { - msleep(100); - pci_read_config_word(pdev, PPDSTAT_OFFSET, &ppdstat); - pending = ppdstat & PCI_EXP_DEVSTA_TRPND; - } while (pending && ctr++ < 10); - } - if (pending) + if (!pci_wait_for_pending_transaction(pdev)) pr_info("QAT: Transaction still in progress. Proceeding\n"); pci_read_config_word(parent, PCI_BRIDGE_CONTROL, &bridge_ctl); -- cgit v1.2.3-58-ga151 From 6a24efda80f9a434c68e94a1e7853bfe23ed5e70 Mon Sep 17 00:00:00 2001 From: "Allan, Bruce W" Date: Fri, 9 Jan 2015 11:55:20 -0800 Subject: crypto: qat - remove unnecessary include of atomic.h header file Signed-off-by: Bruce Allan Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_accel_devices.h | 1 - drivers/crypto/qat/qat_common/adf_transport_internal.h | 1 - 2 files changed, 2 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index ed226cee6e29..19c0efa29ab3 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -47,7 +47,6 @@ #ifndef ADF_ACCEL_DEVICES_H_ #define ADF_ACCEL_DEVICES_H_ #include -#include #include #include #include diff --git a/drivers/crypto/qat/qat_common/adf_transport_internal.h b/drivers/crypto/qat/qat_common/adf_transport_internal.h index c40546079981..a4869627fd57 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_internal.h +++ b/drivers/crypto/qat/qat_common/adf_transport_internal.h @@ -48,7 +48,6 @@ #define ADF_TRANSPORT_INTRN_H #include -#include #include #include "adf_transport.h" -- cgit v1.2.3-58-ga151 From b01264170cef82f3521db90a0589be72dfc7f6eb Mon Sep 17 00:00:00 2001 From: Mathias Krause Date: Sun, 11 Jan 2015 18:17:43 +0100 Subject: crypto: sparc64/aes - fix module description AES is a block cipher, not a hash. Cc: David S. Miller Signed-off-by: Mathias Krause Acked-by: David S. Miller Signed-off-by: Herbert Xu --- arch/sparc/crypto/aes_glue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sparc/crypto/aes_glue.c b/arch/sparc/crypto/aes_glue.c index 705408766ab0..2e48eb8813ff 100644 --- a/arch/sparc/crypto/aes_glue.c +++ b/arch/sparc/crypto/aes_glue.c @@ -497,7 +497,7 @@ module_init(aes_sparc64_mod_init); module_exit(aes_sparc64_mod_fini); MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("AES Secure Hash Algorithm, sparc64 aes opcode accelerated"); +MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, sparc64 aes opcode accelerated"); MODULE_ALIAS_CRYPTO("aes"); -- cgit v1.2.3-58-ga151 From 7d676fdbb42ddef809011e4cfcc4d11cf64989b5 Mon Sep 17 00:00:00 2001 From: Mathias Krause Date: Sun, 11 Jan 2015 18:17:44 +0100 Subject: crypto: sparc64/camellia - fix module alias The module alias should be "camellia", not "aes". Cc: David S. Miller Signed-off-by: Mathias Krause Acked-by: David S. Miller Signed-off-by: Herbert Xu --- arch/sparc/crypto/camellia_glue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sparc/crypto/camellia_glue.c b/arch/sparc/crypto/camellia_glue.c index 641f55cb61c3..6bf2479a12fb 100644 --- a/arch/sparc/crypto/camellia_glue.c +++ b/arch/sparc/crypto/camellia_glue.c @@ -322,6 +322,6 @@ module_exit(camellia_sparc64_mod_fini); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Camellia Cipher Algorithm, sparc64 camellia opcode accelerated"); -MODULE_ALIAS_CRYPTO("aes"); +MODULE_ALIAS_CRYPTO("camellia"); #include "crop_devid.c" -- cgit v1.2.3-58-ga151 From 5001323b701449459c06a926becb6cf27708dff4 Mon Sep 17 00:00:00 2001 From: Mathias Krause Date: Sun, 11 Jan 2015 18:17:45 +0100 Subject: crypto: sparc64/des - add "des3_ede" module alias This module provides implementations for "des3_ede", too. Announce those via an appropriate crypto module alias so it can be used in favour to the generic C implementation. Cc: David S. Miller Signed-off-by: Mathias Krause Acked-by: David S. Miller Signed-off-by: Herbert Xu --- arch/sparc/crypto/des_glue.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/sparc/crypto/des_glue.c b/arch/sparc/crypto/des_glue.c index d11500972994..dd6a34fa6e19 100644 --- a/arch/sparc/crypto/des_glue.c +++ b/arch/sparc/crypto/des_glue.c @@ -533,5 +533,6 @@ MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated"); MODULE_ALIAS_CRYPTO("des"); +MODULE_ALIAS_CRYPTO("des3_ede"); #include "crop_devid.c" -- cgit v1.2.3-58-ga151 From 97ef8ef5a8fe022308c33c03d24424b579eecb36 Mon Sep 17 00:00:00 2001 From: Mathias Krause Date: Sun, 11 Jan 2015 18:17:46 +0100 Subject: crypto: sparc64/md5 - fix module description MD5 is not SHA1. Cc: David S. Miller Signed-off-by: Mathias Krause Acked-by: David S. Miller Signed-off-by: Herbert Xu --- arch/sparc/crypto/md5_glue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sparc/crypto/md5_glue.c b/arch/sparc/crypto/md5_glue.c index 64c7ff5f72a9..b688731d7ede 100644 --- a/arch/sparc/crypto/md5_glue.c +++ b/arch/sparc/crypto/md5_glue.c @@ -183,7 +183,7 @@ module_init(md5_sparc64_mod_init); module_exit(md5_sparc64_mod_fini); MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("MD5 Secure Hash Algorithm, sparc64 md5 opcode accelerated"); +MODULE_DESCRIPTION("MD5 Message Digest Algorithm, sparc64 md5 opcode accelerated"); MODULE_ALIAS_CRYPTO("md5"); -- cgit v1.2.3-58-ga151 From d8219f52a72033f84c15cde73294d46578fb2d68 Mon Sep 17 00:00:00 2001 From: Mathias Krause Date: Sun, 11 Jan 2015 18:17:47 +0100 Subject: crypto: x86/des3_ede - drop bogus module aliases This module implements variations of "des3_ede" only. Drop the bogus module aliases for "des". Cc: Jussi Kivilinna Signed-off-by: Mathias Krause Signed-off-by: Herbert Xu --- arch/x86/crypto/des3_ede_glue.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/x86/crypto/des3_ede_glue.c b/arch/x86/crypto/des3_ede_glue.c index 38a14f818ef1..d6fc59aaaadf 100644 --- a/arch/x86/crypto/des3_ede_glue.c +++ b/arch/x86/crypto/des3_ede_glue.c @@ -504,6 +504,4 @@ MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Triple DES EDE Cipher Algorithm, asm optimized"); MODULE_ALIAS_CRYPTO("des3_ede"); MODULE_ALIAS_CRYPTO("des3_ede-asm"); -MODULE_ALIAS_CRYPTO("des"); -MODULE_ALIAS_CRYPTO("des-asm"); MODULE_AUTHOR("Jussi Kivilinna "); -- cgit v1.2.3-58-ga151 From e31ac32d3bc27c33f002e0c9ffd6ae08b65474e6 Mon Sep 17 00:00:00 2001 From: Timothy McCaffrey Date: Tue, 13 Jan 2015 13:16:43 -0500 Subject: crypto: aesni - Add support for 192 & 256 bit keys to AESNI RFC4106 These patches fix the RFC4106 implementation in the aesni-intel module so it supports 192 & 256 bit keys. Since the AVX support that was added to this module also only supports 128 bit keys, and this patch only affects the SSE implementation, changes were also made to use the SSE version if key sizes other than 128 are specified. RFC4106 specifies that 192 & 256 bit keys must be supported (section 8.4). Also, this should fix Strongswan issue 341 where the aesni module needs to be unloaded if 256 bit keys are used: http://wiki.strongswan.org/issues/341 This patch has been tested with Sandy Bridge and Haswell processors. With 128 bit keys and input buffers > 512 bytes a slight performance degradation was noticed (~1%). For input buffers of less than 512 bytes there was no performance impact. Compared to 128 bit keys, 256 bit key size performance is approx. .5 cycles per byte slower on Sandy Bridge, and .37 cycles per byte slower on Haswell (vs. SSE code). This patch has also been tested with StrongSwan IPSec connections where it worked correctly. I created this diff from a git clone of crypto-2.6.git. Any questions, please feel free to contact me. Signed-off-by: Timothy McCaffrey Signed-off-by: Jarod Wilson Signed-off-by: Herbert Xu --- arch/x86/crypto/aesni-intel_asm.S | 343 +++++++++++++++++++------------------ arch/x86/crypto/aesni-intel_glue.c | 34 +++- 2 files changed, 205 insertions(+), 172 deletions(-) diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S index 477e9d75149b..6bd2c6c95373 100644 --- a/arch/x86/crypto/aesni-intel_asm.S +++ b/arch/x86/crypto/aesni-intel_asm.S @@ -32,12 +32,23 @@ #include #include +/* + * The following macros are used to move an (un)aligned 16 byte value to/from + * an XMM register. This can done for either FP or integer values, for FP use + * movaps (move aligned packed single) or integer use movdqa (move double quad + * aligned). It doesn't make a performance difference which instruction is used + * since Nehalem (original Core i7) was released. However, the movaps is a byte + * shorter, so that is the one we'll use for now. (same for unaligned). + */ +#define MOVADQ movaps +#define MOVUDQ movups + #ifdef __x86_64__ + .data .align 16 .Lgf128mul_x_ble_mask: .octa 0x00000000000000010000000000000087 - POLY: .octa 0xC2000000000000000000000000000001 TWOONE: .octa 0x00000001000000000000000000000001 @@ -89,6 +100,7 @@ enc: .octa 0x2 #define arg8 STACK_OFFSET+16(%r14) #define arg9 STACK_OFFSET+24(%r14) #define arg10 STACK_OFFSET+32(%r14) +#define keysize 2*15*16(%arg1) #endif @@ -213,10 +225,12 @@ enc: .octa 0x2 .macro INITIAL_BLOCKS_DEC num_initial_blocks TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \ XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation + MOVADQ SHUF_MASK(%rip), %xmm14 mov arg7, %r10 # %r10 = AAD mov arg8, %r12 # %r12 = aadLen mov %r12, %r11 pxor %xmm\i, %xmm\i + _get_AAD_loop\num_initial_blocks\operation: movd (%r10), \TMP1 pslldq $12, \TMP1 @@ -225,16 +239,18 @@ _get_AAD_loop\num_initial_blocks\operation: add $4, %r10 sub $4, %r12 jne _get_AAD_loop\num_initial_blocks\operation + cmp $16, %r11 je _get_AAD_loop2_done\num_initial_blocks\operation + mov $16, %r12 _get_AAD_loop2\num_initial_blocks\operation: psrldq $4, %xmm\i sub $4, %r12 cmp %r11, %r12 jne _get_AAD_loop2\num_initial_blocks\operation + _get_AAD_loop2_done\num_initial_blocks\operation: - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, %xmm\i # byte-reflect the AAD data xor %r11, %r11 # initialise the data pointer offset as zero @@ -243,59 +259,34 @@ _get_AAD_loop2_done\num_initial_blocks\operation: mov %arg5, %rax # %rax = *Y0 movdqu (%rax), \XMM0 # XMM0 = Y0 - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, \XMM0 .if (\i == 5) || (\i == 6) || (\i == 7) + MOVADQ ONE(%RIP),\TMP1 + MOVADQ (%arg1),\TMP2 .irpc index, \i_seq - paddd ONE(%rip), \XMM0 # INCR Y0 + paddd \TMP1, \XMM0 # INCR Y0 movdqa \XMM0, %xmm\index - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, %xmm\index # perform a 16 byte swap - -.endr -.irpc index, \i_seq - pxor 16*0(%arg1), %xmm\index -.endr -.irpc index, \i_seq - movaps 0x10(%rdi), \TMP1 - AESENC \TMP1, %xmm\index # Round 1 -.endr -.irpc index, \i_seq - movaps 0x20(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr -.irpc index, \i_seq - movaps 0x30(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr -.irpc index, \i_seq - movaps 0x40(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr -.irpc index, \i_seq - movaps 0x50(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr -.irpc index, \i_seq - movaps 0x60(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 + pxor \TMP2, %xmm\index .endr -.irpc index, \i_seq - movaps 0x70(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr -.irpc index, \i_seq - movaps 0x80(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr -.irpc index, \i_seq - movaps 0x90(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 + lea 0x10(%arg1),%r10 + mov keysize,%eax + shr $2,%eax # 128->4, 192->6, 256->8 + add $5,%eax # 128->9, 192->11, 256->13 + +aes_loop_initial_dec\num_initial_blocks: + MOVADQ (%r10),\TMP1 +.irpc index, \i_seq + AESENC \TMP1, %xmm\index .endr + add $16,%r10 + sub $1,%eax + jnz aes_loop_initial_dec\num_initial_blocks + + MOVADQ (%r10), \TMP1 .irpc index, \i_seq - movaps 0xa0(%arg1), \TMP1 - AESENCLAST \TMP1, %xmm\index # Round 10 + AESENCLAST \TMP1, %xmm\index # Last Round .endr .irpc index, \i_seq movdqu (%arg3 , %r11, 1), \TMP1 @@ -305,10 +296,8 @@ _get_AAD_loop2_done\num_initial_blocks\operation: add $16, %r11 movdqa \TMP1, %xmm\index - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, %xmm\index - - # prepare plaintext/ciphertext for GHASH computation + # prepare plaintext/ciphertext for GHASH computation .endr .endif GHASH_MUL %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 @@ -338,30 +327,28 @@ _get_AAD_loop2_done\num_initial_blocks\operation: * Precomputations for HashKey parallel with encryption of first 4 blocks. * Haskey_i_k holds XORed values of the low and high parts of the Haskey_i */ - paddd ONE(%rip), \XMM0 # INCR Y0 - movdqa \XMM0, \XMM1 - movdqa SHUF_MASK(%rip), %xmm14 + MOVADQ ONE(%rip), \TMP1 + paddd \TMP1, \XMM0 # INCR Y0 + MOVADQ \XMM0, \XMM1 PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap - paddd ONE(%rip), \XMM0 # INCR Y0 - movdqa \XMM0, \XMM2 - movdqa SHUF_MASK(%rip), %xmm14 + paddd \TMP1, \XMM0 # INCR Y0 + MOVADQ \XMM0, \XMM2 PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap - paddd ONE(%rip), \XMM0 # INCR Y0 - movdqa \XMM0, \XMM3 - movdqa SHUF_MASK(%rip), %xmm14 + paddd \TMP1, \XMM0 # INCR Y0 + MOVADQ \XMM0, \XMM3 PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap - paddd ONE(%rip), \XMM0 # INCR Y0 - movdqa \XMM0, \XMM4 - movdqa SHUF_MASK(%rip), %xmm14 + paddd \TMP1, \XMM0 # INCR Y0 + MOVADQ \XMM0, \XMM4 PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap - pxor 16*0(%arg1), \XMM1 - pxor 16*0(%arg1), \XMM2 - pxor 16*0(%arg1), \XMM3 - pxor 16*0(%arg1), \XMM4 + MOVADQ 0(%arg1),\TMP1 + pxor \TMP1, \XMM1 + pxor \TMP1, \XMM2 + pxor \TMP1, \XMM3 + pxor \TMP1, \XMM4 movdqa \TMP3, \TMP5 pshufd $78, \TMP3, \TMP1 pxor \TMP3, \TMP1 @@ -399,7 +386,23 @@ _get_AAD_loop2_done\num_initial_blocks\operation: pshufd $78, \TMP5, \TMP1 pxor \TMP5, \TMP1 movdqa \TMP1, HashKey_4_k(%rsp) - movaps 0xa0(%arg1), \TMP2 + lea 0xa0(%arg1),%r10 + mov keysize,%eax + shr $2,%eax # 128->4, 192->6, 256->8 + sub $4,%eax # 128->0, 192->2, 256->4 + jz aes_loop_pre_dec_done\num_initial_blocks + +aes_loop_pre_dec\num_initial_blocks: + MOVADQ (%r10),\TMP2 +.irpc index, 1234 + AESENC \TMP2, %xmm\index +.endr + add $16,%r10 + sub $1,%eax + jnz aes_loop_pre_dec\num_initial_blocks + +aes_loop_pre_dec_done\num_initial_blocks: + MOVADQ (%r10), \TMP2 AESENCLAST \TMP2, \XMM1 AESENCLAST \TMP2, \XMM2 AESENCLAST \TMP2, \XMM3 @@ -421,15 +424,11 @@ _get_AAD_loop2_done\num_initial_blocks\operation: movdqu \XMM4, 16*3(%arg2 , %r11 , 1) movdqa \TMP1, \XMM4 add $64, %r11 - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap pxor \XMMDst, \XMM1 # combine GHASHed value with the corresponding ciphertext - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap _initial_blocks_done\num_initial_blocks\operation: @@ -451,6 +450,7 @@ _initial_blocks_done\num_initial_blocks\operation: .macro INITIAL_BLOCKS_ENC num_initial_blocks TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \ XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation + MOVADQ SHUF_MASK(%rip), %xmm14 mov arg7, %r10 # %r10 = AAD mov arg8, %r12 # %r12 = aadLen mov %r12, %r11 @@ -472,7 +472,6 @@ _get_AAD_loop2\num_initial_blocks\operation: cmp %r11, %r12 jne _get_AAD_loop2\num_initial_blocks\operation _get_AAD_loop2_done\num_initial_blocks\operation: - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, %xmm\i # byte-reflect the AAD data xor %r11, %r11 # initialise the data pointer offset as zero @@ -481,59 +480,35 @@ _get_AAD_loop2_done\num_initial_blocks\operation: mov %arg5, %rax # %rax = *Y0 movdqu (%rax), \XMM0 # XMM0 = Y0 - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, \XMM0 .if (\i == 5) || (\i == 6) || (\i == 7) -.irpc index, \i_seq - paddd ONE(%rip), \XMM0 # INCR Y0 - movdqa \XMM0, %xmm\index - movdqa SHUF_MASK(%rip), %xmm14 - PSHUFB_XMM %xmm14, %xmm\index # perform a 16 byte swap -.endr -.irpc index, \i_seq - pxor 16*0(%arg1), %xmm\index -.endr -.irpc index, \i_seq - movaps 0x10(%rdi), \TMP1 - AESENC \TMP1, %xmm\index # Round 1 -.endr -.irpc index, \i_seq - movaps 0x20(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr + MOVADQ ONE(%RIP),\TMP1 + MOVADQ 0(%arg1),\TMP2 .irpc index, \i_seq - movaps 0x30(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 + paddd \TMP1, \XMM0 # INCR Y0 + MOVADQ \XMM0, %xmm\index + PSHUFB_XMM %xmm14, %xmm\index # perform a 16 byte swap + pxor \TMP2, %xmm\index .endr -.irpc index, \i_seq - movaps 0x40(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr -.irpc index, \i_seq - movaps 0x50(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr -.irpc index, \i_seq - movaps 0x60(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr -.irpc index, \i_seq - movaps 0x70(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr -.irpc index, \i_seq - movaps 0x80(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr -.irpc index, \i_seq - movaps 0x90(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 + lea 0x10(%arg1),%r10 + mov keysize,%eax + shr $2,%eax # 128->4, 192->6, 256->8 + add $5,%eax # 128->9, 192->11, 256->13 + +aes_loop_initial_enc\num_initial_blocks: + MOVADQ (%r10),\TMP1 +.irpc index, \i_seq + AESENC \TMP1, %xmm\index .endr + add $16,%r10 + sub $1,%eax + jnz aes_loop_initial_enc\num_initial_blocks + + MOVADQ (%r10), \TMP1 .irpc index, \i_seq - movaps 0xa0(%arg1), \TMP1 - AESENCLAST \TMP1, %xmm\index # Round 10 + AESENCLAST \TMP1, %xmm\index # Last Round .endr .irpc index, \i_seq movdqu (%arg3 , %r11, 1), \TMP1 @@ -541,8 +516,6 @@ _get_AAD_loop2_done\num_initial_blocks\operation: movdqu %xmm\index, (%arg2 , %r11, 1) # write back plaintext/ciphertext for num_initial_blocks add $16, %r11 - - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, %xmm\index # prepare plaintext/ciphertext for GHASH computation @@ -575,30 +548,28 @@ _get_AAD_loop2_done\num_initial_blocks\operation: * Precomputations for HashKey parallel with encryption of first 4 blocks. * Haskey_i_k holds XORed values of the low and high parts of the Haskey_i */ - paddd ONE(%rip), \XMM0 # INCR Y0 - movdqa \XMM0, \XMM1 - movdqa SHUF_MASK(%rip), %xmm14 + MOVADQ ONE(%RIP),\TMP1 + paddd \TMP1, \XMM0 # INCR Y0 + MOVADQ \XMM0, \XMM1 PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap - paddd ONE(%rip), \XMM0 # INCR Y0 - movdqa \XMM0, \XMM2 - movdqa SHUF_MASK(%rip), %xmm14 + paddd \TMP1, \XMM0 # INCR Y0 + MOVADQ \XMM0, \XMM2 PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap - paddd ONE(%rip), \XMM0 # INCR Y0 - movdqa \XMM0, \XMM3 - movdqa SHUF_MASK(%rip), %xmm14 + paddd \TMP1, \XMM0 # INCR Y0 + MOVADQ \XMM0, \XMM3 PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap - paddd ONE(%rip), \XMM0 # INCR Y0 - movdqa \XMM0, \XMM4 - movdqa SHUF_MASK(%rip), %xmm14 + paddd \TMP1, \XMM0 # INCR Y0 + MOVADQ \XMM0, \XMM4 PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap - pxor 16*0(%arg1), \XMM1 - pxor 16*0(%arg1), \XMM2 - pxor 16*0(%arg1), \XMM3 - pxor 16*0(%arg1), \XMM4 + MOVADQ 0(%arg1),\TMP1 + pxor \TMP1, \XMM1 + pxor \TMP1, \XMM2 + pxor \TMP1, \XMM3 + pxor \TMP1, \XMM4 movdqa \TMP3, \TMP5 pshufd $78, \TMP3, \TMP1 pxor \TMP3, \TMP1 @@ -636,7 +607,23 @@ _get_AAD_loop2_done\num_initial_blocks\operation: pshufd $78, \TMP5, \TMP1 pxor \TMP5, \TMP1 movdqa \TMP1, HashKey_4_k(%rsp) - movaps 0xa0(%arg1), \TMP2 + lea 0xa0(%arg1),%r10 + mov keysize,%eax + shr $2,%eax # 128->4, 192->6, 256->8 + sub $4,%eax # 128->0, 192->2, 256->4 + jz aes_loop_pre_enc_done\num_initial_blocks + +aes_loop_pre_enc\num_initial_blocks: + MOVADQ (%r10),\TMP2 +.irpc index, 1234 + AESENC \TMP2, %xmm\index +.endr + add $16,%r10 + sub $1,%eax + jnz aes_loop_pre_enc\num_initial_blocks + +aes_loop_pre_enc_done\num_initial_blocks: + MOVADQ (%r10), \TMP2 AESENCLAST \TMP2, \XMM1 AESENCLAST \TMP2, \XMM2 AESENCLAST \TMP2, \XMM3 @@ -655,15 +642,11 @@ _get_AAD_loop2_done\num_initial_blocks\operation: movdqu \XMM4, 16*3(%arg2 , %r11 , 1) add $64, %r11 - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap pxor \XMMDst, \XMM1 # combine GHASHed value with the corresponding ciphertext - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap _initial_blocks_done\num_initial_blocks\operation: @@ -794,7 +777,23 @@ TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation AESENC \TMP3, \XMM3 AESENC \TMP3, \XMM4 PCLMULQDQ 0x00, \TMP5, \XMM8 # XMM8 = a0*b0 - movaps 0xa0(%arg1), \TMP3 + lea 0xa0(%arg1),%r10 + mov keysize,%eax + shr $2,%eax # 128->4, 192->6, 256->8 + sub $4,%eax # 128->0, 192->2, 256->4 + jz aes_loop_par_enc_done + +aes_loop_par_enc: + MOVADQ (%r10),\TMP3 +.irpc index, 1234 + AESENC \TMP3, %xmm\index +.endr + add $16,%r10 + sub $1,%eax + jnz aes_loop_par_enc + +aes_loop_par_enc_done: + MOVADQ (%r10), \TMP3 AESENCLAST \TMP3, \XMM1 # Round 10 AESENCLAST \TMP3, \XMM2 AESENCLAST \TMP3, \XMM3 @@ -986,8 +985,24 @@ TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation AESENC \TMP3, \XMM3 AESENC \TMP3, \XMM4 PCLMULQDQ 0x00, \TMP5, \XMM8 # XMM8 = a0*b0 - movaps 0xa0(%arg1), \TMP3 - AESENCLAST \TMP3, \XMM1 # Round 10 + lea 0xa0(%arg1),%r10 + mov keysize,%eax + shr $2,%eax # 128->4, 192->6, 256->8 + sub $4,%eax # 128->0, 192->2, 256->4 + jz aes_loop_par_dec_done + +aes_loop_par_dec: + MOVADQ (%r10),\TMP3 +.irpc index, 1234 + AESENC \TMP3, %xmm\index +.endr + add $16,%r10 + sub $1,%eax + jnz aes_loop_par_dec + +aes_loop_par_dec_done: + MOVADQ (%r10), \TMP3 + AESENCLAST \TMP3, \XMM1 # last round AESENCLAST \TMP3, \XMM2 AESENCLAST \TMP3, \XMM3 AESENCLAST \TMP3, \XMM4 @@ -1155,33 +1170,29 @@ TMP7 XMM1 XMM2 XMM3 XMM4 XMMDst pxor \TMP6, \XMMDst # reduced result is in XMMDst .endm -/* Encryption of a single block done*/ -.macro ENCRYPT_SINGLE_BLOCK XMM0 TMP1 - pxor (%arg1), \XMM0 - movaps 16(%arg1), \TMP1 - AESENC \TMP1, \XMM0 - movaps 32(%arg1), \TMP1 - AESENC \TMP1, \XMM0 - movaps 48(%arg1), \TMP1 - AESENC \TMP1, \XMM0 - movaps 64(%arg1), \TMP1 - AESENC \TMP1, \XMM0 - movaps 80(%arg1), \TMP1 - AESENC \TMP1, \XMM0 - movaps 96(%arg1), \TMP1 - AESENC \TMP1, \XMM0 - movaps 112(%arg1), \TMP1 - AESENC \TMP1, \XMM0 - movaps 128(%arg1), \TMP1 - AESENC \TMP1, \XMM0 - movaps 144(%arg1), \TMP1 - AESENC \TMP1, \XMM0 - movaps 160(%arg1), \TMP1 - AESENCLAST \TMP1, \XMM0 -.endm +/* Encryption of a single block +* uses eax & r10 +*/ +.macro ENCRYPT_SINGLE_BLOCK XMM0 TMP1 + pxor (%arg1), \XMM0 + mov keysize,%eax + shr $2,%eax # 128->4, 192->6, 256->8 + add $5,%eax # 128->9, 192->11, 256->13 + lea 16(%arg1), %r10 # get first expanded key address + +_esb_loop_\@: + MOVADQ (%r10),\TMP1 + AESENC \TMP1,\XMM0 + add $16,%r10 + sub $1,%eax + jnz _esb_loop_\@ + + MOVADQ (%r10),\TMP1 + AESENCLAST \TMP1,\XMM0 +.endm /***************************************************************************** * void aesni_gcm_dec(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary. * u8 *out, // Plaintext output. Encrypt in-place is allowed. diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index ae855f4f64b7..947c6bf52c33 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -43,6 +43,7 @@ #include #endif + /* This data is stored at the end of the crypto_tfm struct. * It's a type of per "session" data storage location. * This needs to be 16 byte aligned. @@ -182,7 +183,8 @@ static void aesni_gcm_enc_avx(void *ctx, u8 *out, u8 *hash_subkey, const u8 *aad, unsigned long aad_len, u8 *auth_tag, unsigned long auth_tag_len) { - if (plaintext_len < AVX_GEN2_OPTSIZE) { + struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx; + if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)){ aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad, aad_len, auth_tag, auth_tag_len); } else { @@ -197,7 +199,8 @@ static void aesni_gcm_dec_avx(void *ctx, u8 *out, u8 *hash_subkey, const u8 *aad, unsigned long aad_len, u8 *auth_tag, unsigned long auth_tag_len) { - if (ciphertext_len < AVX_GEN2_OPTSIZE) { + struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx; + if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) { aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, aad, aad_len, auth_tag, auth_tag_len); } else { @@ -231,7 +234,8 @@ static void aesni_gcm_enc_avx2(void *ctx, u8 *out, u8 *hash_subkey, const u8 *aad, unsigned long aad_len, u8 *auth_tag, unsigned long auth_tag_len) { - if (plaintext_len < AVX_GEN2_OPTSIZE) { + struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx; + if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) { aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad, aad_len, auth_tag, auth_tag_len); } else if (plaintext_len < AVX_GEN4_OPTSIZE) { @@ -250,7 +254,8 @@ static void aesni_gcm_dec_avx2(void *ctx, u8 *out, u8 *hash_subkey, const u8 *aad, unsigned long aad_len, u8 *auth_tag, unsigned long auth_tag_len) { - if (ciphertext_len < AVX_GEN2_OPTSIZE) { + struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx; + if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) { aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, aad, aad_len, auth_tag, auth_tag_len); } else if (ciphertext_len < AVX_GEN4_OPTSIZE) { @@ -511,7 +516,7 @@ static int ctr_crypt(struct blkcipher_desc *desc, kernel_fpu_begin(); while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr, - nbytes & AES_BLOCK_MASK, walk.iv); + nbytes & AES_BLOCK_MASK, walk.iv); nbytes &= AES_BLOCK_SIZE - 1; err = blkcipher_walk_done(desc, &walk, nbytes); } @@ -902,7 +907,8 @@ static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key, } /*Account for 4 byte nonce at the end.*/ key_len -= 4; - if (key_len != AES_KEYSIZE_128) { + if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && + key_len != AES_KEYSIZE_256) { crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } @@ -1013,6 +1019,7 @@ static int __driver_rfc4106_encrypt(struct aead_request *req) __be32 counter = cpu_to_be32(1); struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); + u32 key_len = ctx->aes_key_expanded.key_length; void *aes_ctx = &(ctx->aes_key_expanded); unsigned long auth_tag_len = crypto_aead_authsize(tfm); u8 iv_tab[16+AESNI_ALIGN]; @@ -1027,6 +1034,13 @@ static int __driver_rfc4106_encrypt(struct aead_request *req) /* to 8 or 12 bytes */ if (unlikely(req->assoclen != 8 && req->assoclen != 12)) return -EINVAL; + if (unlikely(auth_tag_len != 8 && auth_tag_len != 12 && auth_tag_len != 16)) + return -EINVAL; + if (unlikely(key_len != AES_KEYSIZE_128 && + key_len != AES_KEYSIZE_192 && + key_len != AES_KEYSIZE_256)) + return -EINVAL; + /* IV below built */ for (i = 0; i < 4; i++) *(iv+i) = ctx->nonce[i]; @@ -1091,6 +1105,7 @@ static int __driver_rfc4106_decrypt(struct aead_request *req) int retval = 0; struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); + u32 key_len = ctx->aes_key_expanded.key_length; void *aes_ctx = &(ctx->aes_key_expanded); unsigned long auth_tag_len = crypto_aead_authsize(tfm); u8 iv_and_authTag[32+AESNI_ALIGN]; @@ -1104,6 +1119,13 @@ static int __driver_rfc4106_decrypt(struct aead_request *req) if (unlikely((req->cryptlen < auth_tag_len) || (req->assoclen != 8 && req->assoclen != 12))) return -EINVAL; + if (unlikely(auth_tag_len != 8 && auth_tag_len != 12 && auth_tag_len != 16)) + return -EINVAL; + if (unlikely(key_len != AES_KEYSIZE_128 && + key_len != AES_KEYSIZE_192 && + key_len != AES_KEYSIZE_256)) + return -EINVAL; + /* Assuming we are supporting rfc4106 64-bit extended */ /* sequence numbers We need to have the AAD length */ /* equal to 8 or 12 bytes */ -- cgit v1.2.3-58-ga151 From 48eb3691e8beab349e3a700d45f070be8d4b88ba Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 14 Jan 2015 10:07:03 +1100 Subject: crypto: qat - Ensure ipad and opad are zeroed The patch ad511e260a27b8e35d273cc0ecfe5a8ff9543181 (crypto: qat - Fix incorrect uses of memzero_explicit) broke hashing because the code was in fact overwriting the qat_auth_state variable. In fact there is no reason for the variable to exist anyway since all we are using it for is to store ipad and opad. So we could simply create ipad and opad directly and avoid this whole mess. Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_algs.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index a0d95f329094..e2c4b254ac41 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -160,33 +160,30 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash, const uint8_t *auth_key, unsigned int auth_keylen) { - struct qat_auth_state auth_state; SHASH_DESC_ON_STACK(shash, ctx->hash_tfm); struct sha1_state sha1; struct sha256_state sha256; struct sha512_state sha512; int block_size = crypto_shash_blocksize(ctx->hash_tfm); int digest_size = crypto_shash_digestsize(ctx->hash_tfm); - uint8_t *ipad = auth_state.data; - uint8_t *opad = ipad + block_size; + char ipad[block_size]; + char opad[block_size]; __be32 *hash_state_out; __be64 *hash512_state_out; int i, offset; - memset(auth_state.data, 0, sizeof(auth_state.data)); + memset(ipad, 0, block_size); + memset(opad, 0, block_size); shash->tfm = ctx->hash_tfm; shash->flags = 0x0; if (auth_keylen > block_size) { - char buff[SHA512_BLOCK_SIZE]; int ret = crypto_shash_digest(shash, auth_key, - auth_keylen, buff); + auth_keylen, ipad); if (ret) return ret; - memcpy(ipad, buff, digest_size); - memcpy(opad, buff, digest_size); - memzero_explicit(buff, sizeof(buff)); + memcpy(opad, ipad, digest_size); } else { memcpy(ipad, auth_key, auth_keylen); memcpy(opad, auth_key, auth_keylen); -- cgit v1.2.3-58-ga151 From 598de3695201cc9f722b6e82c0097438d30fd54e Mon Sep 17 00:00:00 2001 From: Wei Yongjun Date: Wed, 14 Jan 2015 09:14:41 +0800 Subject: crypto: algif_rng - fix sparse non static symbol warning Fixes the following sparse warnings: crypto/algif_rng.c:185:13: warning: symbol 'rng_exit' was not declared. Should it be static? Signed-off-by: Wei Yongjun Acked-by: Stephan Mueller Acked-by: Neil Horman Signed-off-by: Herbert Xu --- crypto/algif_rng.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/algif_rng.c b/crypto/algif_rng.c index 91c06f5673dd..67f612cfed97 100644 --- a/crypto/algif_rng.c +++ b/crypto/algif_rng.c @@ -182,7 +182,7 @@ static int __init rng_init(void) return af_alg_register_type(&algif_type_rng); } -void __exit rng_exit(void) +static void __exit rng_exit(void) { int err = af_alg_unregister_type(&algif_type_rng); BUG_ON(err); -- cgit v1.2.3-58-ga151 From b2c3f7cdad7673e00a449b879fb2b20d5c164797 Mon Sep 17 00:00:00 2001 From: Tadeusz Struk Date: Wed, 14 Jan 2015 10:27:35 -0800 Subject: crypto: qat - don't need qat_auth_state struct We don't need the qat_auth_state structure anymore. Signed-off-by: Tadeusz Struk Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_algs.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index e2c4b254ac41..1dc5b0a17cf7 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -102,12 +102,6 @@ struct qat_alg_cd { }; } __aligned(64); -#define MAX_AUTH_STATE_SIZE sizeof(struct icp_qat_hw_auth_algo_blk) - -struct qat_auth_state { - uint8_t data[MAX_AUTH_STATE_SIZE + 64]; -} __aligned(64); - struct qat_alg_aead_ctx { struct qat_alg_cd *enc_cd; struct qat_alg_cd *dec_cd; -- cgit v1.2.3-58-ga151 From 0c5c8e646cd9832b5e307e541264e4ff7de85bd7 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 16 Jan 2015 18:09:21 +1100 Subject: crypto: cts - Remove bogus use of seqiv The seqiv generator is completely inappropriate for cts as it's designed for IPsec algorithms. Since cts users do not actually use the IV generator we can just fall back to the default. Signed-off-by: Herbert Xu Acked-by: Maciej ?enczykowski --- crypto/cts.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/crypto/cts.c b/crypto/cts.c index bd9405820e8a..6a8089c4d4b4 100644 --- a/crypto/cts.c +++ b/crypto/cts.c @@ -307,8 +307,6 @@ static struct crypto_instance *crypto_cts_alloc(struct rtattr **tb) inst->alg.cra_blkcipher.min_keysize = alg->cra_blkcipher.min_keysize; inst->alg.cra_blkcipher.max_keysize = alg->cra_blkcipher.max_keysize; - inst->alg.cra_blkcipher.geniv = "seqiv"; - inst->alg.cra_ctxsize = sizeof(struct crypto_cts_ctx); inst->alg.cra_init = crypto_cts_init_tfm; -- cgit v1.2.3-58-ga151 From 3eda71676bd9d396775c41a8b0f91357b1302cd5 Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Fri, 16 Jan 2015 09:16:00 +0200 Subject: MAINTAINERS: add linux-crypto to hw random hw random is crypto-related, Cc the linux-crypto list on patches. Signed-off-by: Michael S. Tsirkin Signed-off-by: Herbert Xu --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index ddb9ac8d32b3..aa5aaebf0397 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4402,6 +4402,7 @@ F: include/linux/hwmon*.h HARDWARE RANDOM NUMBER GENERATOR CORE M: Matt Mackall M: Herbert Xu +L: linux-crypto@vger.kernel.org S: Odd fixes F: Documentation/hw_random.txt F: drivers/char/hw_random/ -- cgit v1.2.3-58-ga151 From 988dc0174411ad4b98c173f3d7d25aca9e56cd85 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 16 Jan 2015 19:38:17 +1100 Subject: crypto: cts - Weed out non-CBC algorithms The cts algorithm as currently implemented assumes the underlying is a CBC-mode algorithm. So this patch adds a check for that to eliminate bogus combinations of cts with non-CBC modes. Signed-off-by: Herbert Xu --- crypto/cts.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crypto/cts.c b/crypto/cts.c index 6a8089c4d4b4..e467ec0acf9f 100644 --- a/crypto/cts.c +++ b/crypto/cts.c @@ -290,6 +290,9 @@ static struct crypto_instance *crypto_cts_alloc(struct rtattr **tb) if (!is_power_of_2(alg->cra_blocksize)) goto out_put_alg; + if (strncmp(alg->cra_name, "cbc(", 4)) + goto out_put_alg; + inst = crypto_alloc_instance("cts", alg); if (IS_ERR(inst)) goto out_put_alg; -- cgit v1.2.3-58-ga151 From c0ecf8916d28e99fd072514f19fd36ee42a6ad7e Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 16 Jan 2015 19:51:20 +1100 Subject: crypto: seqiv - Ensure that IV size is at least 8 bytes Since seqiv is designed for IPsec we need to be able to accomodate the whole IPsec sequence number in order to ensure the uniqueness of the IV. This patch forbids any algorithm with an IV size of less than 8 from using it. This should have no impact on existing users since they all have an IV size of 8. Reported-by: Maciej ?enczykowski Signed-off-by: Herbert Xu Acked-by: Maciej ?enczykowski --- crypto/seqiv.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crypto/seqiv.c b/crypto/seqiv.c index 9daa854cc485..b7bb9a2f4a31 100644 --- a/crypto/seqiv.c +++ b/crypto/seqiv.c @@ -267,6 +267,12 @@ static struct crypto_instance *seqiv_ablkcipher_alloc(struct rtattr **tb) if (IS_ERR(inst)) goto out; + if (inst->alg.cra_ablkcipher.ivsize < sizeof(u64)) { + skcipher_geniv_free(inst); + inst = ERR_PTR(-EINVAL); + goto out; + } + inst->alg.cra_ablkcipher.givencrypt = seqiv_givencrypt_first; inst->alg.cra_init = seqiv_init; @@ -287,6 +293,12 @@ static struct crypto_instance *seqiv_aead_alloc(struct rtattr **tb) if (IS_ERR(inst)) goto out; + if (inst->alg.cra_aead.ivsize < sizeof(u64)) { + aead_geniv_free(inst); + inst = ERR_PTR(-EINVAL); + goto out; + } + inst->alg.cra_aead.givencrypt = seqiv_aead_givencrypt_first; inst->alg.cra_init = seqiv_aead_init; -- cgit v1.2.3-58-ga151 From 379dcfb406002d855fa56f9c3d290c4048f44e9c Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Mon, 19 Jan 2015 00:13:39 +0100 Subject: crypto: doc - remove colons in comments As documented in Documentation/kernel-doc-nano-HOWTO.txt lines terminated with a colon are treated as headings. The current layout of the documentation when compiling the kernel crypto API DocBook documentation is messed up by by treating some lines as headings. The patch removes colons from comments that shall not be treated as headings. Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- include/linux/crypto.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/linux/crypto.h b/include/linux/crypto.h index 90998348e564..fb5ef16d6a12 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -1147,7 +1147,7 @@ static inline void ablkcipher_request_free(struct ablkcipher_request *req) * cipher operation completes. * * The callback function is registered with the ablkcipher_request handle and - * must comply with the following template: + * must comply with the following template * * void callback_function(struct crypto_async_request *req, int error) */ @@ -1174,7 +1174,7 @@ static inline void ablkcipher_request_set_callback( * * For encryption, the source is treated as the plaintext and the * destination is the ciphertext. For a decryption operation, the use is - * reversed: the source is the ciphertext and the destination is the plaintext. + * reversed - the source is the ciphertext and the destination is the plaintext. */ static inline void ablkcipher_request_set_crypt( struct ablkcipher_request *req, @@ -1509,7 +1509,7 @@ static inline void aead_request_free(struct aead_request *req) * completes * * The callback function is registered with the aead_request handle and - * must comply with the following template: + * must comply with the following template * * void callback_function(struct crypto_async_request *req, int error) */ @@ -1536,7 +1536,7 @@ static inline void aead_request_set_callback(struct aead_request *req, * * For encryption, the source is treated as the plaintext and the * destination is the ciphertext. For a decryption operation, the use is - * reversed: the source is the ciphertext and the destination is the plaintext. + * reversed - the source is the ciphertext and the destination is the plaintext. * * IMPORTANT NOTE AEAD requires an authentication tag (MAC). For decryption, * the caller must concatenate the ciphertext followed by the -- cgit v1.2.3-58-ga151 From 088f628cc0898c4f0da7e91945f9f43a0b18a3bf Mon Sep 17 00:00:00 2001 From: Christophe Jaillet Date: Tue, 20 Jan 2015 08:15:52 +0100 Subject: crypto: atmel - Free memory in error path If only one of the 2 __get_free_pages fails, then there is a memory leak. Signed-off-by: Christophe Jaillet Signed-off-by: Herbert Xu --- drivers/crypto/atmel-aes.c | 2 +- drivers/crypto/atmel-tdes.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index 53d1c330f8a8..6597aac9905d 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -673,9 +673,9 @@ err_map_out: dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE); err_map_in: +err_alloc: free_page((unsigned long)dd->buf_out); free_page((unsigned long)dd->buf_in); -err_alloc: if (err) pr_err("error: %d\n", err); return err; diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index 5e7c896cde30..258772d9b22f 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -376,9 +376,9 @@ err_map_out: dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE); err_map_in: +err_alloc: free_page((unsigned long)dd->buf_out); free_page((unsigned long)dd->buf_in); -err_alloc: if (err) pr_err("error: %d\n", err); return err; -- cgit v1.2.3-58-ga151 From 5be4d4c94b1f98b839344fda7a8752a4a09d0ef5 Mon Sep 17 00:00:00 2001 From: Cristian Stoica Date: Tue, 20 Jan 2015 10:06:16 +0200 Subject: crypto: replace scatterwalk_sg_next with sg_next Modify crypto drivers to use the generic SG helper since both of them are equivalent and the one from crypto is redundant. See also: 468577abe37ff7b453a9ac613e0ea155349203ae reverted in b2ab4a57b018aafbba35bff088218f5cc3d2142e Signed-off-by: Cristian Stoica Signed-off-by: Herbert Xu --- crypto/ablkcipher.c | 3 +-- crypto/ahash.c | 2 +- crypto/scatterwalk.c | 6 +++--- drivers/crypto/bfin_crc.c | 2 +- drivers/crypto/caam/sg_sw_sec4.h | 8 ++++---- drivers/crypto/ixp4xx_crypto.c | 4 ++-- drivers/crypto/nx/nx.c | 6 +++--- drivers/crypto/omap-aes.c | 4 ++-- drivers/crypto/omap-des.c | 4 ++-- drivers/crypto/qce/dma.c | 6 +++--- drivers/crypto/qce/sha.c | 2 +- drivers/crypto/sahara.c | 2 +- drivers/crypto/talitos.c | 8 ++++---- drivers/crypto/ux500/cryp/cryp_core.c | 2 +- include/crypto/scatterwalk.h | 10 +--------- 15 files changed, 30 insertions(+), 39 deletions(-) diff --git a/crypto/ablkcipher.c b/crypto/ablkcipher.c index 7bbc8b4ef2e9..db201bca1581 100644 --- a/crypto/ablkcipher.c +++ b/crypto/ablkcipher.c @@ -87,8 +87,7 @@ static inline unsigned int ablkcipher_done_slow(struct ablkcipher_walk *walk, if (n == len_this_page) break; n -= len_this_page; - scatterwalk_start(&walk->out, scatterwalk_sg_next( - walk->out.sg)); + scatterwalk_start(&walk->out, sg_next(walk->out.sg)); } return bsize; diff --git a/crypto/ahash.c b/crypto/ahash.c index dd2890608aeb..8acb886032ae 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -121,7 +121,7 @@ int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err) if (!walk->total) return 0; - walk->sg = scatterwalk_sg_next(walk->sg); + walk->sg = sg_next(walk->sg); return hash_walk_new_entry(walk); } diff --git a/crypto/scatterwalk.c b/crypto/scatterwalk.c index 79ca2278c2a3..3bd749c7bb70 100644 --- a/crypto/scatterwalk.c +++ b/crypto/scatterwalk.c @@ -62,7 +62,7 @@ static void scatterwalk_pagedone(struct scatter_walk *walk, int out, walk->offset += PAGE_SIZE - 1; walk->offset &= PAGE_MASK; if (walk->offset >= walk->sg->offset + walk->sg->length) - scatterwalk_start(walk, scatterwalk_sg_next(walk->sg)); + scatterwalk_start(walk, sg_next(walk->sg)); } } @@ -116,7 +116,7 @@ void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg, break; offset += sg->length; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } scatterwalk_advance(&walk, start - offset); @@ -136,7 +136,7 @@ int scatterwalk_bytes_sglen(struct scatterlist *sg, int num_bytes) do { offset += sg->length; n++; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); /* num_bytes is too large */ if (unlikely(!sg && (num_bytes < offset))) diff --git a/drivers/crypto/bfin_crc.c b/drivers/crypto/bfin_crc.c index 33cf9eb87344..d9af9403ab6c 100644 --- a/drivers/crypto/bfin_crc.c +++ b/drivers/crypto/bfin_crc.c @@ -110,7 +110,7 @@ static int sg_count(struct scatterlist *sg_list) while (!sg_is_last(sg)) { sg_nents++; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } return sg_nents; diff --git a/drivers/crypto/caam/sg_sw_sec4.h b/drivers/crypto/caam/sg_sw_sec4.h index ce28a563effc..3b918218aa4c 100644 --- a/drivers/crypto/caam/sg_sw_sec4.h +++ b/drivers/crypto/caam/sg_sw_sec4.h @@ -37,7 +37,7 @@ sg_to_sec4_sg(struct scatterlist *sg, int sg_count, dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg), sg_dma_len(sg), offset); sec4_sg_ptr++; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); sg_count--; } return sec4_sg_ptr - 1; @@ -67,7 +67,7 @@ static inline int __sg_count(struct scatterlist *sg_list, int nbytes, nbytes -= sg->length; if (!sg_is_last(sg) && (sg + 1)->length == 0) *chained = true; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } return sg_nents; @@ -93,7 +93,7 @@ static int dma_map_sg_chained(struct device *dev, struct scatterlist *sg, int i; for (i = 0; i < nents; i++) { dma_map_sg(dev, sg, 1, dir); - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } } else { dma_map_sg(dev, sg, nents, dir); @@ -109,7 +109,7 @@ static int dma_unmap_sg_chained(struct device *dev, struct scatterlist *sg, int i; for (i = 0; i < nents; i++) { dma_unmap_sg(dev, sg, 1, dir); - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } } else { dma_unmap_sg(dev, sg, nents, dir); diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c index f757a0f428bd..48f453555f1f 100644 --- a/drivers/crypto/ixp4xx_crypto.c +++ b/drivers/crypto/ixp4xx_crypto.c @@ -784,7 +784,7 @@ static struct buffer_desc *chainup_buffers(struct device *dev, struct buffer_desc *buf, gfp_t flags, enum dma_data_direction dir) { - for (;nbytes > 0; sg = scatterwalk_sg_next(sg)) { + for (; nbytes > 0; sg = sg_next(sg)) { unsigned len = min(nbytes, sg->length); struct buffer_desc *next_buf; u32 next_buf_phys; @@ -982,7 +982,7 @@ static int hmac_inconsistent(struct scatterlist *sg, unsigned start, break; offset += sg->length; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } return (start + nbytes > offset + sg->length); } diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c index a392465d3e3f..1da6dc59d0dd 100644 --- a/drivers/crypto/nx/nx.c +++ b/drivers/crypto/nx/nx.c @@ -177,7 +177,7 @@ struct nx_sg *nx_walk_and_build(struct nx_sg *nx_dst, break; offset += sg_src->length; - sg_src = scatterwalk_sg_next(sg_src); + sg_src = sg_next(sg_src); } /* start - offset is the number of bytes to advance in the scatterlist @@ -187,9 +187,9 @@ struct nx_sg *nx_walk_and_build(struct nx_sg *nx_dst, while (len && (nx_sg - nx_dst) < sglen) { n = scatterwalk_clamp(&walk, len); if (!n) { - /* In cases where we have scatterlist chain scatterwalk_sg_next + /* In cases where we have scatterlist chain sg_next * handles with it properly */ - scatterwalk_start(&walk, scatterwalk_sg_next(walk.sg)); + scatterwalk_start(&walk, sg_next(walk.sg)); n = scatterwalk_clamp(&walk, len); } dst = scatterwalk_map(&walk); diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index f79dd410dede..42f95a4326b0 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -994,7 +994,7 @@ static irqreturn_t omap_aes_irq(int irq, void *dev_id) scatterwalk_advance(&dd->in_walk, 4); if (dd->in_sg->length == _calc_walked(in)) { - dd->in_sg = scatterwalk_sg_next(dd->in_sg); + dd->in_sg = sg_next(dd->in_sg); if (dd->in_sg) { scatterwalk_start(&dd->in_walk, dd->in_sg); @@ -1026,7 +1026,7 @@ static irqreturn_t omap_aes_irq(int irq, void *dev_id) *dst = omap_aes_read(dd, AES_REG_DATA_N(dd, i)); scatterwalk_advance(&dd->out_walk, 4); if (dd->out_sg->length == _calc_walked(out)) { - dd->out_sg = scatterwalk_sg_next(dd->out_sg); + dd->out_sg = sg_next(dd->out_sg); if (dd->out_sg) { scatterwalk_start(&dd->out_walk, dd->out_sg); diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c index 0b8dcf5eb74e..46307098f8ba 100644 --- a/drivers/crypto/omap-des.c +++ b/drivers/crypto/omap-des.c @@ -921,7 +921,7 @@ static irqreturn_t omap_des_irq(int irq, void *dev_id) scatterwalk_advance(&dd->in_walk, 4); if (dd->in_sg->length == _calc_walked(in)) { - dd->in_sg = scatterwalk_sg_next(dd->in_sg); + dd->in_sg = sg_next(dd->in_sg); if (dd->in_sg) { scatterwalk_start(&dd->in_walk, dd->in_sg); @@ -953,7 +953,7 @@ static irqreturn_t omap_des_irq(int irq, void *dev_id) *dst = omap_des_read(dd, DES_REG_DATA_N(dd, i)); scatterwalk_advance(&dd->out_walk, 4); if (dd->out_sg->length == _calc_walked(out)) { - dd->out_sg = scatterwalk_sg_next(dd->out_sg); + dd->out_sg = sg_next(dd->out_sg); if (dd->out_sg) { scatterwalk_start(&dd->out_walk, dd->out_sg); diff --git a/drivers/crypto/qce/dma.c b/drivers/crypto/qce/dma.c index 0fb21e13f247..378cb768647f 100644 --- a/drivers/crypto/qce/dma.c +++ b/drivers/crypto/qce/dma.c @@ -64,7 +64,7 @@ int qce_mapsg(struct device *dev, struct scatterlist *sg, int nents, err = dma_map_sg(dev, sg, 1, dir); if (!err) return -EFAULT; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } } else { err = dma_map_sg(dev, sg, nents, dir); @@ -81,7 +81,7 @@ void qce_unmapsg(struct device *dev, struct scatterlist *sg, int nents, if (chained) while (sg) { dma_unmap_sg(dev, sg, 1, dir); - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } else dma_unmap_sg(dev, sg, nents, dir); @@ -100,7 +100,7 @@ int qce_countsg(struct scatterlist *sglist, int nbytes, bool *chained) nbytes -= sg->length; if (!sg_is_last(sg) && (sg + 1)->length == 0 && chained) *chained = true; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } return nents; diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c index f3385934eed2..5c5df1d17f90 100644 --- a/drivers/crypto/qce/sha.c +++ b/drivers/crypto/qce/sha.c @@ -285,7 +285,7 @@ static int qce_ahash_update(struct ahash_request *req) break; len += sg_dma_len(sg); sg_last = sg; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } if (!sg_last) diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index 220b92f7eabc..290a7f0a681f 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -940,7 +940,7 @@ static int sahara_walk_and_recalc(struct scatterlist *sg, unsigned int nbytes) break; } nbytes -= sg->length; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } return nbytes; diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 067ec2193d71..ebbae8d3ce0d 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -743,7 +743,7 @@ static int talitos_map_sg(struct device *dev, struct scatterlist *sg, if (unlikely(chained)) while (sg) { dma_map_sg(dev, sg, 1, dir); - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } else dma_map_sg(dev, sg, nents, dir); @@ -755,7 +755,7 @@ static void talitos_unmap_sg_chain(struct device *dev, struct scatterlist *sg, { while (sg) { dma_unmap_sg(dev, sg, 1, dir); - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } } @@ -915,7 +915,7 @@ static int sg_to_link_tbl(struct scatterlist *sg, int sg_count, link_tbl_ptr->j_extent = 0; link_tbl_ptr++; cryptlen -= sg_dma_len(sg); - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } /* adjust (decrease) last one (or two) entry's len to cryptlen */ @@ -1102,7 +1102,7 @@ static int sg_count(struct scatterlist *sg_list, int nbytes, bool *chained) nbytes -= sg->length; if (!sg_is_last(sg) && (sg + 1)->length == 0) *chained = true; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } return sg_nents; diff --git a/drivers/crypto/ux500/cryp/cryp_core.c b/drivers/crypto/ux500/cryp/cryp_core.c index f087f37b2c67..d594ae962ed2 100644 --- a/drivers/crypto/ux500/cryp/cryp_core.c +++ b/drivers/crypto/ux500/cryp/cryp_core.c @@ -814,7 +814,7 @@ static int get_nents(struct scatterlist *sg, int nbytes) while (nbytes > 0) { nbytes -= sg->length; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); nents++; } diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h index 7ef512f8631c..20e4226a2e14 100644 --- a/include/crypto/scatterwalk.h +++ b/include/crypto/scatterwalk.h @@ -33,21 +33,13 @@ static inline void scatterwalk_sg_chain(struct scatterlist *sg1, int num, sg1[num - 1].page_link |= 0x01; } -static inline struct scatterlist *scatterwalk_sg_next(struct scatterlist *sg) -{ - if (sg_is_last(sg)) - return NULL; - - return (++sg)->length ? sg : sg_chain_ptr(sg); -} - static inline void scatterwalk_crypto_chain(struct scatterlist *head, struct scatterlist *sg, int chain, int num) { if (chain) { head->length += sg->length; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } if (sg) -- cgit v1.2.3-58-ga151 From e20016a9c68cebeec7d7dddb005656c8134295c4 Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Tue, 20 Jan 2015 14:35:15 +0200 Subject: hwrng: virtio - drop extra empty line makes code look a bit prettier. Cc: linux-crypto@vger.kernel.org. Signed-off-by: Michael S. Tsirkin Signed-off-by: Herbert Xu --- drivers/char/hw_random/virtio-rng.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/char/hw_random/virtio-rng.c b/drivers/char/hw_random/virtio-rng.c index 72295ea2fd1c..3fa2f8a009b3 100644 --- a/drivers/char/hw_random/virtio-rng.c +++ b/drivers/char/hw_random/virtio-rng.c @@ -39,7 +39,6 @@ struct virtrng_info { bool hwrng_removed; }; - static void random_recv_done(struct virtqueue *vq) { struct virtrng_info *vi = vq->vdev->priv; -- cgit v1.2.3-58-ga151 From 3f80be023900790d029a63e27440f1e33c048b73 Mon Sep 17 00:00:00 2001 From: Kim Phillips Date: Tue, 20 Jan 2015 12:43:10 -0600 Subject: crypto: caam - don't emit ICV check failures to dmesg ICV check failures are part of normal operation; leave user notification up to the higher levels, as is done in s/w algorithm implementations. Signed-off-by: Kim Phillips Tested-by: Cristian Stoica Signed-off-by: Herbert Xu --- drivers/crypto/caam/error.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/caam/error.c b/drivers/crypto/caam/error.c index 66d73bf54166..33e41ea83fcc 100644 --- a/drivers/crypto/caam/error.c +++ b/drivers/crypto/caam/error.c @@ -151,10 +151,15 @@ static void report_ccb_status(struct device *jrdev, const u32 status, else snprintf(err_err_code, sizeof(err_err_code), "%02x", err_id); - dev_err(jrdev, "%08x: %s: %s %d: %s%s: %s%s\n", - status, error, idx_str, idx, - cha_str, cha_err_code, - err_str, err_err_code); + /* + * CCB ICV check failures are part of normal operation life; + * we leave the upper layers to do what they want with them. + */ + if (err_id != JRSTA_CCBERR_ERRID_ICVCHK) + dev_err(jrdev, "%08x: %s: %s %d: %s%s: %s%s\n", + status, error, idx_str, idx, + cha_str, cha_err_code, + err_str, err_err_code); } static void report_jump_status(struct device *jrdev, const u32 status, -- cgit v1.2.3-58-ga151 From 35b82e5540eef2c08f607b9794d68849ebfb7d06 Mon Sep 17 00:00:00 2001 From: Cristian Stoica Date: Wed, 21 Jan 2015 11:53:30 +0200 Subject: crypto: caam - remove dead code - assoc_nents and src_nents are never zero when all_contig is false - iv_contig is zero on the else branch Signed-off-by: Cristian Stoica Reviewed-by: Richard Schmitt Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 3187400daf31..29071a156cbe 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -2532,7 +2532,7 @@ static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr, in_options = 0; } else { src_dma = edesc->sec4_sg_dma; - sec4_sg_index += (iv_contig ? 0 : 1) + edesc->src_nents; + sec4_sg_index += edesc->src_nents + 1; in_options = LDST_SGF; } append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options); @@ -2714,10 +2714,10 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, if (!all_contig) { if (!is_gcm) { sg_to_sec4_sg(req->assoc, - (assoc_nents ? : 1), + assoc_nents, edesc->sec4_sg + sec4_sg_index, 0); - sec4_sg_index += assoc_nents ? : 1; + sec4_sg_index += assoc_nents; } dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, @@ -2726,17 +2726,17 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, if (is_gcm) { sg_to_sec4_sg(req->assoc, - (assoc_nents ? : 1), + assoc_nents, edesc->sec4_sg + sec4_sg_index, 0); - sec4_sg_index += assoc_nents ? : 1; + sec4_sg_index += assoc_nents; } sg_to_sec4_sg_last(req->src, - (src_nents ? : 1), + src_nents, edesc->sec4_sg + sec4_sg_index, 0); - sec4_sg_index += src_nents ? : 1; + sec4_sg_index += src_nents; } if (dst_nents) { sg_to_sec4_sg_last(req->dst, dst_nents, -- cgit v1.2.3-58-ga151 From 467707b2b73c34802e8ba2a98858722b23f1ef8e Mon Sep 17 00:00:00 2001 From: Cristian Stoica Date: Wed, 21 Jan 2015 11:53:31 +0200 Subject: crypto: caam - remove unused local variable Signed-off-by: Cristian Stoica Signed-off-by: Herbert Xu --- drivers/crypto/caam/ctrl.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index 70f1e6f37336..efba4ccd4fac 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -175,13 +175,10 @@ static int instantiate_rng(struct device *ctrldev, int state_handle_mask, { struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev); struct caam_ctrl __iomem *ctrl; - struct rng4tst __iomem *r4tst; u32 *desc, status, rdsta_val; int ret = 0, sh_idx; ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl; - r4tst = &ctrl->r4tst[0]; - desc = kmalloc(CAAM_CMD_SZ * 7, GFP_KERNEL); if (!desc) return -ENOMEM; @@ -209,8 +206,7 @@ static int instantiate_rng(struct device *ctrldev, int state_handle_mask, * without any error (HW optimizations for later * CAAM eras), then try again. */ - rdsta_val = - rd_reg32(&ctrl->r4tst[0].rdsta) & RDSTA_IFMASK; + rdsta_val = rd_reg32(&ctrl->r4tst[0].rdsta) & RDSTA_IFMASK; if (status || !(rdsta_val & (1 << sh_idx))) ret = -EAGAIN; if (ret) -- cgit v1.2.3-58-ga151 From 490f702286bc03a2f1538b39c4a19420ce624a73 Mon Sep 17 00:00:00 2001 From: Andrey Ryabinin Date: Wed, 21 Jan 2015 18:06:31 +0300 Subject: crypto: ccp - terminate ccp_support array with empty element x86_match_cpu() expects array of x86_cpu_ids terminated with empty element. Signed-off-by: Andrey Ryabinin Acked-by: Tom Lendacky Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-dev.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c index c6e6171eb6d3..ca29c120b85f 100644 --- a/drivers/crypto/ccp/ccp-dev.c +++ b/drivers/crypto/ccp/ccp-dev.c @@ -583,6 +583,7 @@ bool ccp_queues_suspended(struct ccp_device *ccp) #ifdef CONFIG_X86 static const struct x86_cpu_id ccp_support[] = { { X86_VENDOR_AMD, 22, }, + { }, }; #endif -- cgit v1.2.3-58-ga151 From c6bf62e4f34f0f490ecbba184e57a8c6808b3cb1 Mon Sep 17 00:00:00 2001 From: Cristian Stoica Date: Thu, 22 Jan 2015 16:00:48 +0200 Subject: crypto: caam - pair irq map and dispose in the same function irq_dispose_mapping is not called on all error paths from caam_jr_init. This takes care of several clean-up issues by performing resource clean-up and allocation at the same level. Signed-off-by: Cristian Stoica Signed-off-by: Herbert Xu --- drivers/crypto/caam/jr.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index 9b3ef1bc9bd7..bce2959993eb 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -384,8 +384,6 @@ static int caam_jr_init(struct device *dev) if (error) { dev_err(dev, "can't connect JobR %d interrupt (%d)\n", jrp->ridx, jrp->irq); - irq_dispose_mapping(jrp->irq); - jrp->irq = 0; return -EINVAL; } @@ -484,8 +482,10 @@ static int caam_jr_probe(struct platform_device *pdev) /* Now do the platform independent part */ error = caam_jr_init(jrdev); /* now turn on hardware */ - if (error) + if (error) { + irq_dispose_mapping(jrpriv->irq); return error; + } jrpriv->dev = jrdev; spin_lock(&driver_data.jr_alloc_lock); -- cgit v1.2.3-58-ga151 From cbceeefd872480afe46197633826a424196d8131 Mon Sep 17 00:00:00 2001 From: Cristian Stoica Date: Thu, 22 Jan 2015 16:00:49 +0200 Subject: crypto: caam - fix resource clean-up on error path for caam_jr_init Signed-off-by: Cristian Stoica Signed-off-by: Herbert Xu --- drivers/crypto/caam/jr.c | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index bce2959993eb..b8b5d47acd7a 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -384,28 +384,28 @@ static int caam_jr_init(struct device *dev) if (error) { dev_err(dev, "can't connect JobR %d interrupt (%d)\n", jrp->ridx, jrp->irq); - return -EINVAL; + goto out_kill_deq; } error = caam_reset_hw_jr(dev); if (error) - return error; + goto out_free_irq; + error = -ENOMEM; jrp->inpring = dma_alloc_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH, &inpbusaddr, GFP_KERNEL); + if (!jrp->inpring) + goto out_free_irq; jrp->outring = dma_alloc_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH, &outbusaddr, GFP_KERNEL); + if (!jrp->outring) + goto out_free_inpring; jrp->entinfo = kzalloc(sizeof(struct caam_jrentry_info) * JOBR_DEPTH, GFP_KERNEL); - - if ((jrp->inpring == NULL) || (jrp->outring == NULL) || - (jrp->entinfo == NULL)) { - dev_err(dev, "can't allocate job rings for %d\n", - jrp->ridx); - return -ENOMEM; - } + if (!jrp->entinfo) + goto out_free_outring; for (i = 0; i < JOBR_DEPTH; i++) jrp->entinfo[i].desc_addr_dma = !0; @@ -432,6 +432,19 @@ static int caam_jr_init(struct device *dev) (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT)); return 0; + +out_free_outring: + dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH, + jrp->outring, outbusaddr); +out_free_inpring: + dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH, + jrp->inpring, inpbusaddr); + dev_err(dev, "can't allocate job rings for %d\n", jrp->ridx); +out_free_irq: + free_irq(jrp->irq, dev); +out_kill_deq: + tasklet_kill(&jrp->irqtask); + return error; } -- cgit v1.2.3-58-ga151 From db71f29a1c327f3824c1c0919937965b36d67b80 Mon Sep 17 00:00:00 2001 From: Jarod Wilson Date: Fri, 23 Jan 2015 12:42:15 -0500 Subject: crypto: testmgr - mark rfc4106(gcm(aes)) as fips_allowed This gcm variant is popular for ipsec use, and there are folks who would like to use it while in fips mode. Mark it with fips_allowed=1 to facilitate that. CC: LKML CC: Stephan Mueller Signed-off-by: Jarod Wilson Acked-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/testmgr.c | 1 + 1 file changed, 1 insertion(+) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 235b1fff04c4..758d02847308 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -3293,6 +3293,7 @@ static const struct alg_test_desc alg_test_descs[] = { }, { .alg = "rfc4106(gcm(aes))", .test = alg_test_aead, + .fips_allowed = 1, .suite = { .aead = { .enc = { -- cgit v1.2.3-58-ga151 From c47689931fff5f8882a923bbd8d8590f038fa097 Mon Sep 17 00:00:00 2001 From: Cristian Stoica Date: Tue, 27 Jan 2015 11:54:27 +0200 Subject: crypto: tcrypt - fix buflen reminder calculation - This fixes the intent of the code to limit the last scatterlist to either a full PAGE or a fraction of it, depending on the number of pages needed by buflen and the available space advertised by XBUFLEN. The original code always sets the last scatterlist to a fraction of a PAGE because the first 'if' is never executed. - Rearrange the second part of the code to remove the conditional from the loop Signed-off-by: Cristian Stoica Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 004349576ba1..2b2486ad26ef 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -250,19 +250,19 @@ static void sg_init_aead(struct scatterlist *sg, char *xbuf[XBUFSIZE], int np = (buflen + PAGE_SIZE - 1)/PAGE_SIZE; int k, rem; - np = (np > XBUFSIZE) ? XBUFSIZE : np; - rem = buflen % PAGE_SIZE; if (np > XBUFSIZE) { rem = PAGE_SIZE; np = XBUFSIZE; + } else { + rem = buflen % PAGE_SIZE; } + sg_init_table(sg, np); - for (k = 0; k < np; ++k) { - if (k == (np-1)) - sg_set_buf(&sg[k], xbuf[k], rem); - else - sg_set_buf(&sg[k], xbuf[k], PAGE_SIZE); - } + np--; + for (k = 0; k < np; k++) + sg_set_buf(&sg[k], xbuf[k], PAGE_SIZE); + + sg_set_buf(&sg[k], xbuf[k], rem); } static void test_aead_speed(const char *algo, int enc, unsigned int secs, -- cgit v1.2.3-58-ga151 From 424a5da6919073392c11345d1b7baa9f31c62734 Mon Sep 17 00:00:00 2001 From: Cristian Stoica Date: Wed, 28 Jan 2015 11:03:05 +0200 Subject: crypto: testmgr - limit IV copy length in aead tests The working copy of IV is the same size as the transformation's IV. It is not necessary to copy more than that from the template since iv_len is usually less than MAX_IVLEN and the rest of the copied data is garbage. Signed-off-by: Cristian Stoica Signed-off-by: Herbert Xu --- crypto/testmgr.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 758d02847308..f4ed6d4205e7 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -429,7 +429,7 @@ static int __test_aead(struct crypto_aead *tfm, int enc, struct scatterlist *sgout; const char *e, *d; struct tcrypt_result result; - unsigned int authsize; + unsigned int authsize, iv_len; void *input; void *output; void *assoc; @@ -500,10 +500,11 @@ static int __test_aead(struct crypto_aead *tfm, int enc, memcpy(input, template[i].input, template[i].ilen); memcpy(assoc, template[i].assoc, template[i].alen); + iv_len = crypto_aead_ivsize(tfm); if (template[i].iv) - memcpy(iv, template[i].iv, MAX_IVLEN); + memcpy(iv, template[i].iv, iv_len); else - memset(iv, 0, MAX_IVLEN); + memset(iv, 0, iv_len); crypto_aead_clear_flags(tfm, ~0); if (template[i].wk) -- cgit v1.2.3-58-ga151 From 96692a7305c49845e3cbf5a60cfcb207c5dc4030 Mon Sep 17 00:00:00 2001 From: Cristian Stoica Date: Wed, 28 Jan 2015 13:07:32 +0200 Subject: crypto: tcrypt - do not allocate iv on stack for aead speed tests See also: 9bac019dad8098a77cce555d929f678e22111783 Signed-off-by: Cristian Stoica Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 2b2486ad26ef..4b9e23fa4204 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -280,16 +280,20 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, struct scatterlist *sgout; const char *e; void *assoc; - char iv[MAX_IVLEN]; + char *iv; char *xbuf[XBUFSIZE]; char *xoutbuf[XBUFSIZE]; char *axbuf[XBUFSIZE]; unsigned int *b_size; unsigned int iv_len; + iv = kzalloc(MAX_IVLEN, GFP_KERNEL); + if (!iv) + return; + if (aad_size >= PAGE_SIZE) { pr_err("associate data length (%u) too big\n", aad_size); - return; + goto out_noxbuf; } if (enc == ENCRYPT) @@ -355,7 +359,7 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, iv_len = crypto_aead_ivsize(tfm); if (iv_len) - memset(&iv, 0xff, iv_len); + memset(iv, 0xff, iv_len); crypto_aead_clear_flags(tfm, ~0); printk(KERN_INFO "test %u (%d bit key, %d byte blocks): ", @@ -408,6 +412,7 @@ out_nooutbuf: out_noaxbuf: testmgr_free_buf(xbuf); out_noxbuf: + kfree(iv); return; } -- cgit v1.2.3-58-ga151