From e1e5e5641e6f271321aec257ed26a72715e4a8c2 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 19 Feb 2015 13:39:03 -0700 Subject: NVMe: Metadata format support Adds support for NVMe metadata formats and exposes block devices for all namespaces regardless of their format. Namespace formats that are unusable will have disk capacity set to 0, but a handle to the block device is created to simplify device management. A namespace is not usable when the format requires host interleave block and metadata in single buffer, has no provisioned storage, or has better data but failed to register with blk integrity. The namespace has to be scanned in two phases to support separate metadata formats. The first establishes the sector size and capacity prior to invoking add_disk. If metadata is required, the capacity will be temporarilly set to 0 until it can be revalidated and registered with the integrity extenstions after add_disk completes. The driver relies on the integrity extensions to provide the metadata buffer. NVMe requires this be a single physically contiguous region, so only one integrity segment is allowed per command. If the metadata is used for T10 PI, the driver provides mappings to save and restore the reftag physical block translation. The driver provides no-op functions for generate and verify if metadata is not used for protection information. This way the setup is always provided by the block layer. If a request does not supply a required metadata buffer, the command is failed with bad address. This could only happen if a user manually disables verify/generate on such a disk. The only exception to where this is okay is if the controller is capable of stripping/generating the metadata, which is possible on some types of formats. The metadata scatter gather list now occupies the spot in the nvme_iod that used to be used to link retryable IOD's, but we don't do that anymore, so the field was unused. Signed-off-by: Keith Busch --- include/linux/nvme.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/nvme.h b/include/linux/nvme.h index 19a5d4b23209..cca264db2478 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -121,6 +121,7 @@ struct nvme_ns { unsigned ns_id; int lba_shift; int ms; + int pi_type; u64 mode_select_num_blocks; u32 mode_select_block_len; }; @@ -138,6 +139,7 @@ struct nvme_iod { int nents; /* Used in scatterlist */ int length; /* Of data, in bytes */ dma_addr_t first_dma; + struct scatterlist meta_sg[1]; /* metadata requires single contiguous buffer */ struct scatterlist sg[0]; }; -- cgit v1.2.3-58-ga151 From 4f1982b4e262c45475a91b4253e9bc7f7c991c13 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 19 Feb 2015 13:42:14 -0700 Subject: NVMe: Update SCSI Inquiry VPD 83h translation The original translation created collisions on Inquiry VPD 83 for many existing devices. Newer specifications provide other ways to translate based on the device's version can be used to create unique identifiers. Version 1.1 provides an EUI64 field that uniquely identifies each namespace, and 1.2 added the longer NGUID field for the same reason. Both follow the IEEE EUI format and readily translate to the SCSI device identification EUI designator type 2h. For devices implementing either, the translation will use this type, defaulting to the EUI64 8-byte type if implemented then NGUID's 16 byte version if not. If neither are provided, the 1.0 translation is used, and is updated to use the SCSI String format to guarantee a unique identifier. Knowing when to use the new fields depends on the nvme controller's revision. The NVME_VS macro was not decoding this correctly, so that is fixed in this patch and moved to a more appropriate place. Since the Identify Namespace structure required an update for the NGUID field, this patch adds the remaining new 1.2 fields to the structure. Signed-off-by: Keith Busch --- drivers/block/nvme-scsi.c | 94 ++++++++++++++++++++++++++--------------------- include/linux/nvme.h | 2 - include/uapi/linux/nvme.h | 10 ++++- 3 files changed, 62 insertions(+), 44 deletions(-) (limited to 'include/linux') diff --git a/drivers/block/nvme-scsi.c b/drivers/block/nvme-scsi.c index 5e78568026c3..5cc907648742 100644 --- a/drivers/block/nvme-scsi.c +++ b/drivers/block/nvme-scsi.c @@ -779,10 +779,8 @@ static int nvme_trans_device_id_page(struct nvme_ns *ns, struct sg_io_hdr *hdr, struct nvme_dev *dev = ns->dev; dma_addr_t dma_addr; void *mem; - struct nvme_id_ctrl *id_ctrl; int res = SNTI_TRANSLATION_SUCCESS; int nvme_sc; - u8 ieee[4]; int xfer_len; __be32 tmp_id = cpu_to_be32(ns->ns_id); @@ -793,46 +791,60 @@ static int nvme_trans_device_id_page(struct nvme_ns *ns, struct sg_io_hdr *hdr, goto out_dma; } - /* nvme controller identify */ - nvme_sc = nvme_identify(dev, 0, 1, dma_addr); - res = nvme_trans_status_code(hdr, nvme_sc); - if (res) - goto out_free; - if (nvme_sc) { - res = nvme_sc; - goto out_free; - } - id_ctrl = mem; - - /* Since SCSI tried to save 4 bits... [SPC-4(r34) Table 591] */ - ieee[0] = id_ctrl->ieee[0] << 4; - ieee[1] = id_ctrl->ieee[0] >> 4 | id_ctrl->ieee[1] << 4; - ieee[2] = id_ctrl->ieee[1] >> 4 | id_ctrl->ieee[2] << 4; - ieee[3] = id_ctrl->ieee[2] >> 4; - - memset(inq_response, 0, STANDARD_INQUIRY_LENGTH); + memset(inq_response, 0, alloc_len); inq_response[1] = INQ_DEVICE_IDENTIFICATION_PAGE; /* Page Code */ - inq_response[3] = 20; /* Page Length */ - /* Designation Descriptor start */ - inq_response[4] = 0x01; /* Proto ID=0h | Code set=1h */ - inq_response[5] = 0x03; /* PIV=0b | Asso=00b | Designator Type=3h */ - inq_response[6] = 0x00; /* Rsvd */ - inq_response[7] = 16; /* Designator Length */ - /* Designator start */ - inq_response[8] = 0x60 | ieee[3]; /* NAA=6h | IEEE ID MSB, High nibble*/ - inq_response[9] = ieee[2]; /* IEEE ID */ - inq_response[10] = ieee[1]; /* IEEE ID */ - inq_response[11] = ieee[0]; /* IEEE ID| Vendor Specific ID... */ - inq_response[12] = (dev->pci_dev->vendor & 0xFF00) >> 8; - inq_response[13] = (dev->pci_dev->vendor & 0x00FF); - inq_response[14] = dev->serial[0]; - inq_response[15] = dev->serial[1]; - inq_response[16] = dev->model[0]; - inq_response[17] = dev->model[1]; - memcpy(&inq_response[18], &tmp_id, sizeof(u32)); - /* Last 2 bytes are zero */ + if (readl(&dev->bar->vs) >= NVME_VS(1, 1)) { + struct nvme_id_ns *id_ns = mem; + void *eui = id_ns->eui64; + int len = sizeof(id_ns->eui64); - xfer_len = min(alloc_len, STANDARD_INQUIRY_LENGTH); + nvme_sc = nvme_identify(dev, ns->ns_id, 0, dma_addr); + res = nvme_trans_status_code(hdr, nvme_sc); + if (res) + goto out_free; + if (nvme_sc) { + res = nvme_sc; + goto out_free; + } + + if (readl(&dev->bar->vs) >= NVME_VS(1, 2)) { + if (bitmap_empty(eui, len * 8)) { + eui = id_ns->nguid; + len = sizeof(id_ns->nguid); + } + } + if (bitmap_empty(eui, len * 8)) + goto scsi_string; + + inq_response[3] = 4 + len; /* Page Length */ + /* Designation Descriptor start */ + inq_response[4] = 0x01; /* Proto ID=0h | Code set=1h */ + inq_response[5] = 0x02; /* PIV=0b | Asso=00b | Designator Type=2h */ + inq_response[6] = 0x00; /* Rsvd */ + inq_response[7] = len; /* Designator Length */ + memcpy(&inq_response[8], eui, len); + } else { + scsi_string: + if (alloc_len < 72) { + res = nvme_trans_completion(hdr, + SAM_STAT_CHECK_CONDITION, + ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB, + SCSI_ASCQ_CAUSE_NOT_REPORTABLE); + goto out_free; + } + inq_response[3] = 0x48; /* Page Length */ + /* Designation Descriptor start */ + inq_response[4] = 0x03; /* Proto ID=0h | Code set=3h */ + inq_response[5] = 0x08; /* PIV=0b | Asso=00b | Designator Type=8h */ + inq_response[6] = 0x00; /* Rsvd */ + inq_response[7] = 0x44; /* Designator Length */ + + sprintf(&inq_response[8], "%04x", dev->pci_dev->vendor); + memcpy(&inq_response[12], dev->model, sizeof(dev->model)); + sprintf(&inq_response[52], "%04x", tmp_id); + memcpy(&inq_response[56], dev->serial, sizeof(dev->serial)); + } + xfer_len = alloc_len; res = nvme_trans_copy_to_user(hdr, inq_response, xfer_len); out_free: @@ -2222,7 +2234,7 @@ static int nvme_trans_inquiry(struct nvme_ns *ns, struct sg_io_hdr *hdr, page_code = GET_INQ_PAGE_CODE(cmd); alloc_len = GET_INQ_ALLOC_LENGTH(cmd); - inq_response = kmalloc(STANDARD_INQUIRY_LENGTH, GFP_KERNEL); + inq_response = kmalloc(alloc_len, GFP_KERNEL); if (inq_response == NULL) { res = -ENOMEM; goto out_mem; diff --git a/include/linux/nvme.h b/include/linux/nvme.h index cca264db2478..1f062a9e521d 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -62,8 +62,6 @@ enum { NVME_CSTS_SHST_MASK = 3 << 2, }; -#define NVME_VS(major, minor) (major << 16 | minor) - extern unsigned char nvme_io_timeout; #define NVME_IO_TIMEOUT (nvme_io_timeout * HZ) diff --git a/include/uapi/linux/nvme.h b/include/uapi/linux/nvme.h index 406bfc95652c..aef9a81b2d75 100644 --- a/include/uapi/linux/nvme.h +++ b/include/uapi/linux/nvme.h @@ -115,7 +115,13 @@ struct nvme_id_ns { __le16 nawun; __le16 nawupf; __le16 nacwu; - __u8 rsvd40[80]; + __le16 nabsn; + __le16 nabo; + __le16 nabspf; + __u16 rsvd46; + __le64 nvmcap[2]; + __u8 rsvd64[40]; + __u8 nguid[16]; __u8 eui64[8]; struct nvme_lbaf lbaf[16]; __u8 rsvd192[192]; @@ -565,6 +571,8 @@ struct nvme_passthru_cmd { __u32 result; }; +#define NVME_VS(major, minor) (((major) << 16) | ((minor) << 8)) + #define nvme_admin_cmd nvme_passthru_cmd #define NVME_IOCTL_ID _IO('N', 0x40) -- cgit v1.2.3-58-ga151 From b3fffdefabab266ae5176a136d93b6670b07bb30 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Tue, 3 Feb 2015 11:21:42 -0700 Subject: NVMe: Register management handle under nvme class This creates a new class type for nvme devices to register their management character devices with. This is so we do not rely on miscdev to provide enough minors for as many nvme devices some people plan to use. The previous limit was approximately 60 NVMe controllers, depending on the platform and kernel. Now the limit is 1M, which ought to be enough for anybody. Since we have a new device class, it makes sense to attach the block devices under this as well, so part of this patch moves the management handle initialization prior to the namespaces discovery. Signed-off-by: Keith Busch --- drivers/block/nvme-core.c | 79 +++++++++++++++++++++++++++++++++-------------- include/linux/nvme.h | 3 +- 2 files changed, 57 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/drivers/block/nvme-core.c b/drivers/block/nvme-core.c index 3ffa57a932ea..bb2b861cfed9 100644 --- a/drivers/block/nvme-core.c +++ b/drivers/block/nvme-core.c @@ -42,6 +42,7 @@ #include #include +#define NVME_MINORS (1U << MINORBITS) #define NVME_Q_DEPTH 1024 #define NVME_AQ_DEPTH 64 #define SQ_SIZE(depth) (depth * sizeof(struct nvme_command)) @@ -69,6 +70,9 @@ MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown") static int nvme_major; module_param(nvme_major, int, 0); +static int nvme_char_major; +module_param(nvme_char_major, int, 0); + static int use_threaded_interrupts; module_param(use_threaded_interrupts, int, 0); @@ -79,6 +83,8 @@ static struct workqueue_struct *nvme_workq; static wait_queue_head_t nvme_kthread_wait; static struct notifier_block nvme_nb; +static struct class *nvme_class; + static void nvme_reset_failed_dev(struct work_struct *ws); static int nvme_process_cq(struct nvme_queue *nvmeq); @@ -2189,7 +2195,7 @@ static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid) disk->fops = &nvme_fops; disk->private_data = ns; disk->queue = ns->queue; - disk->driverfs_dev = &dev->pci_dev->dev; + disk->driverfs_dev = dev->device; disk->flags = GENHD_FL_EXT_DEVT; sprintf(disk->disk_name, "nvme%dn%d", dev->instance, nsid); @@ -2775,6 +2781,7 @@ static void nvme_free_dev(struct kref *kref) struct nvme_dev *dev = container_of(kref, struct nvme_dev, kref); pci_dev_put(dev->pci_dev); + put_device(dev->device); nvme_free_namespaces(dev); nvme_release_instance(dev); blk_mq_free_tag_set(&dev->tagset); @@ -2786,11 +2793,23 @@ static void nvme_free_dev(struct kref *kref) static int nvme_dev_open(struct inode *inode, struct file *f) { - struct nvme_dev *dev = container_of(f->private_data, struct nvme_dev, - miscdev); - kref_get(&dev->kref); - f->private_data = dev; - return 0; + struct nvme_dev *dev; + int instance = iminor(inode); + int ret = -ENODEV; + + spin_lock(&dev_list_lock); + list_for_each_entry(dev, &dev_list, node) { + if (dev->instance == instance) { + if (!kref_get_unless_zero(&dev->kref)) + break; + f->private_data = dev; + ret = 0; + break; + } + } + spin_unlock(&dev_list_lock); + + return ret; } static int nvme_dev_release(struct inode *inode, struct file *f) @@ -3002,29 +3021,26 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (result) goto release_pools; - if (dev->online_queues > 1) - result = nvme_dev_add(dev); - if (result) + dev->device = device_create(nvme_class, &pdev->dev, + MKDEV(nvme_char_major, dev->instance), + dev, "nvme%d", dev->instance); + if (IS_ERR(dev->device)) { + result = PTR_ERR(dev->device); goto shutdown; + } + get_device(dev->device); - scnprintf(dev->name, sizeof(dev->name), "nvme%d", dev->instance); - dev->miscdev.minor = MISC_DYNAMIC_MINOR; - dev->miscdev.parent = &pdev->dev; - dev->miscdev.name = dev->name; - dev->miscdev.fops = &nvme_dev_fops; - result = misc_register(&dev->miscdev); + if (dev->online_queues > 1) + result = nvme_dev_add(dev); if (result) - goto remove; + goto device_del; nvme_set_irq_hints(dev); - dev->initialized = 1; return 0; - remove: - nvme_dev_remove(dev); - nvme_dev_remove_admin(dev); - nvme_free_namespaces(dev); + device_del: + device_destroy(nvme_class, MKDEV(nvme_char_major, dev->instance)); shutdown: nvme_dev_shutdown(dev); release_pools: @@ -3067,10 +3083,10 @@ static void nvme_remove(struct pci_dev *pdev) pci_set_drvdata(pdev, NULL); flush_work(&dev->reset_work); - misc_deregister(&dev->miscdev); nvme_dev_shutdown(dev); nvme_dev_remove(dev); nvme_dev_remove_admin(dev); + device_destroy(nvme_class, MKDEV(nvme_char_major, dev->instance)); nvme_free_queues(dev, 0); nvme_release_prp_pools(dev); kref_put(&dev->kref, nvme_free_dev); @@ -3154,11 +3170,26 @@ static int __init nvme_init(void) else if (result > 0) nvme_major = result; + result = __register_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme", + &nvme_dev_fops); + if (result < 0) + goto unregister_blkdev; + else if (result > 0) + nvme_char_major = result; + + nvme_class = class_create(THIS_MODULE, "nvme"); + if (!nvme_class) + goto unregister_chrdev; + result = pci_register_driver(&nvme_driver); if (result) - goto unregister_blkdev; + goto destroy_class; return 0; + destroy_class: + class_destroy(nvme_class); + unregister_chrdev: + __unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme"); unregister_blkdev: unregister_blkdev(nvme_major, "nvme"); kill_workq: @@ -3172,6 +3203,8 @@ static void __exit nvme_exit(void) unregister_hotcpu_notifier(&nvme_nb); unregister_blkdev(nvme_major, "nvme"); destroy_workqueue(nvme_workq); + class_destroy(nvme_class); + __unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme"); BUG_ON(nvme_thread && !IS_ERR(nvme_thread)); _nvme_check_size(); } diff --git a/include/linux/nvme.h b/include/linux/nvme.h index 1f062a9e521d..383d495c5e4c 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -17,7 +17,6 @@ #include #include -#include #include #include @@ -89,7 +88,7 @@ struct nvme_dev { struct nvme_bar __iomem *bar; struct list_head namespaces; struct kref kref; - struct miscdevice miscdev; + struct device *device; work_func_t reset_workfn; struct work_struct reset_work; char name[12]; -- cgit v1.2.3-58-ga151 From 2e1d8448196ba85cd78a18723413a3c92aabe0f3 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 12 Feb 2015 15:33:00 -0700 Subject: NVMe: Asynchronous controller probe This performs the longest parts of nvme device probe in scheduled work. This speeds up probe significantly when multiple devices are in use. Signed-off-by: Keith Busch --- drivers/block/nvme-core.c | 48 ++++++++++++++++++++++++++++++----------------- include/linux/nvme.h | 1 + 2 files changed, 32 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/drivers/block/nvme-core.c b/drivers/block/nvme-core.c index bb2b861cfed9..a57685f74e5e 100644 --- a/drivers/block/nvme-core.c +++ b/drivers/block/nvme-core.c @@ -2800,6 +2800,10 @@ static int nvme_dev_open(struct inode *inode, struct file *f) spin_lock(&dev_list_lock); list_for_each_entry(dev, &dev_list, node) { if (dev->instance == instance) { + if (!dev->admin_q) { + ret = -EWOULDBLOCK; + break; + } if (!kref_get_unless_zero(&dev->kref)) break; f->private_data = dev; @@ -2982,6 +2986,7 @@ static void nvme_reset_workfn(struct work_struct *work) dev->reset_workfn(work); } +static void nvme_async_probe(struct work_struct *work); static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) { int node, result = -ENOMEM; @@ -3017,34 +3022,20 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto release; kref_init(&dev->kref); - result = nvme_dev_start(dev); - if (result) - goto release_pools; - dev->device = device_create(nvme_class, &pdev->dev, MKDEV(nvme_char_major, dev->instance), dev, "nvme%d", dev->instance); if (IS_ERR(dev->device)) { result = PTR_ERR(dev->device); - goto shutdown; + goto release_pools; } get_device(dev->device); - if (dev->online_queues > 1) - result = nvme_dev_add(dev); - if (result) - goto device_del; - - nvme_set_irq_hints(dev); - dev->initialized = 1; + INIT_WORK(&dev->probe_work, nvme_async_probe); + schedule_work(&dev->probe_work); return 0; - device_del: - device_destroy(nvme_class, MKDEV(nvme_char_major, dev->instance)); - shutdown: - nvme_dev_shutdown(dev); release_pools: - nvme_free_queues(dev, 0); nvme_release_prp_pools(dev); release: nvme_release_instance(dev); @@ -3057,6 +3048,28 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) return result; } +static void nvme_async_probe(struct work_struct *work) +{ + struct nvme_dev *dev = container_of(work, struct nvme_dev, probe_work); + int result; + + result = nvme_dev_start(dev); + if (result) + goto reset; + + if (dev->online_queues > 1) + result = nvme_dev_add(dev); + if (result) + goto reset; + + nvme_set_irq_hints(dev); + dev->initialized = 1; + return; + reset: + dev->reset_workfn = nvme_reset_failed_dev; + queue_work(nvme_workq, &dev->reset_work); +} + static void nvme_reset_notify(struct pci_dev *pdev, bool prepare) { struct nvme_dev *dev = pci_get_drvdata(pdev); @@ -3082,6 +3095,7 @@ static void nvme_remove(struct pci_dev *pdev) spin_unlock(&dev_list_lock); pci_set_drvdata(pdev, NULL); + flush_work(&dev->probe_work); flush_work(&dev->reset_work); nvme_dev_shutdown(dev); nvme_dev_remove(dev); diff --git a/include/linux/nvme.h b/include/linux/nvme.h index 383d495c5e4c..e2429e8cdab4 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -91,6 +91,7 @@ struct nvme_dev { struct device *device; work_func_t reset_workfn; struct work_struct reset_work; + struct work_struct probe_work; char name[12]; char serial[20]; char model[40]; -- cgit v1.2.3-58-ga151 From 07836e659c81ec6b0d683dfbf7958339a22a7b69 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 19 Feb 2015 10:34:48 -0700 Subject: NVMe: Fix potential corruption during shutdown The driver has to end unreturned commands at some point even if the controller has not provided a completion. The driver tried to be safe by deleting IO queues prior to ending all unreturned commands. That should cause the controller to internally abort inflight commands, but IO queue deletion request does not have to be successful, so all bets are off. We still have to make progress, so to be extra safe, this patch doesn't clear a queue to release the dma mapping for a command until after the pci device has been disabled. This patch removes the special handling during device initialization so controller recovery can be done all the time. This is possible since initialization is not inlined with pci probe anymore. Reported-by: Nilish Choudhury Signed-off-by: Keith Busch --- drivers/block/nvme-core.c | 49 ++++++++++++++++++----------------------------- include/linux/nvme.h | 1 - 2 files changed, 19 insertions(+), 31 deletions(-) (limited to 'include/linux') diff --git a/drivers/block/nvme-core.c b/drivers/block/nvme-core.c index a57685f74e5e..0f388206b15b 100644 --- a/drivers/block/nvme-core.c +++ b/drivers/block/nvme-core.c @@ -1274,29 +1274,18 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved) struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req); struct nvme_queue *nvmeq = cmd->nvmeq; - /* - * The aborted req will be completed on receiving the abort req. - * We enable the timer again. If hit twice, it'll cause a device reset, - * as the device then is in a faulty state. - */ - int ret = BLK_EH_RESET_TIMER; - dev_warn(nvmeq->q_dmadev, "Timeout I/O %d QID %d\n", req->tag, nvmeq->qid); - spin_lock_irq(&nvmeq->q_lock); - if (!nvmeq->dev->initialized) { - /* - * Force cancelled command frees the request, which requires we - * return BLK_EH_NOT_HANDLED. - */ - nvme_cancel_queue_ios(nvmeq->hctx, req, nvmeq, reserved); - ret = BLK_EH_NOT_HANDLED; - } else - nvme_abort_req(req); + nvme_abort_req(req); spin_unlock_irq(&nvmeq->q_lock); - return ret; + /* + * The aborted req will be completed on receiving the abort req. + * We enable the timer again. If hit twice, it'll cause a device reset, + * as the device then is in a faulty state. + */ + return BLK_EH_RESET_TIMER; } static void nvme_free_queue(struct nvme_queue *nvmeq) @@ -1349,7 +1338,6 @@ static void nvme_clear_queue(struct nvme_queue *nvmeq) struct blk_mq_hw_ctx *hctx = nvmeq->hctx; spin_lock_irq(&nvmeq->q_lock); - nvme_process_cq(nvmeq); if (hctx && hctx->tags) blk_mq_tag_busy_iter(hctx, nvme_cancel_queue_ios, nvmeq); spin_unlock_irq(&nvmeq->q_lock); @@ -1372,7 +1360,10 @@ static void nvme_disable_queue(struct nvme_dev *dev, int qid) } if (!qid && dev->admin_q) blk_mq_freeze_queue_start(dev->admin_q); - nvme_clear_queue(nvmeq); + + spin_lock_irq(&nvmeq->q_lock); + nvme_process_cq(nvmeq); + spin_unlock_irq(&nvmeq->q_lock); } static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid, @@ -2121,8 +2112,7 @@ static int nvme_kthread(void *data) spin_lock(&dev_list_lock); list_for_each_entry_safe(dev, next, &dev_list, node) { int i; - if (readl(&dev->bar->csts) & NVME_CSTS_CFS && - dev->initialized) { + if (readl(&dev->bar->csts) & NVME_CSTS_CFS) { if (work_busy(&dev->reset_work)) continue; list_del_init(&dev->node); @@ -2525,8 +2515,6 @@ static struct nvme_delq_ctx *nvme_get_dq(struct nvme_delq_ctx *dq) static void nvme_del_queue_end(struct nvme_queue *nvmeq) { struct nvme_delq_ctx *dq = nvmeq->cmdinfo.ctx; - - nvme_clear_queue(nvmeq); nvme_put_dq(dq); } @@ -2669,7 +2657,6 @@ static void nvme_dev_shutdown(struct nvme_dev *dev) int i; u32 csts = -1; - dev->initialized = 0; nvme_dev_list_remove(dev); if (dev->bar) { @@ -2680,7 +2667,6 @@ static void nvme_dev_shutdown(struct nvme_dev *dev) for (i = dev->queue_count - 1; i >= 0; i--) { struct nvme_queue *nvmeq = dev->queues[i]; nvme_suspend_queue(nvmeq); - nvme_clear_queue(nvmeq); } } else { nvme_disable_io_queues(dev); @@ -2688,6 +2674,9 @@ static void nvme_dev_shutdown(struct nvme_dev *dev) nvme_disable_queue(dev, 0); } nvme_dev_unmap(dev); + + for (i = dev->queue_count - 1; i >= 0; i--) + nvme_clear_queue(dev->queues[i]); } static void nvme_dev_remove(struct nvme_dev *dev) @@ -2955,7 +2944,6 @@ static int nvme_dev_resume(struct nvme_dev *dev) nvme_unfreeze_queues(dev); nvme_set_irq_hints(dev); } - dev->initialized = 1; return 0; } @@ -3063,11 +3051,12 @@ static void nvme_async_probe(struct work_struct *work) goto reset; nvme_set_irq_hints(dev); - dev->initialized = 1; return; reset: - dev->reset_workfn = nvme_reset_failed_dev; - queue_work(nvme_workq, &dev->reset_work); + if (!work_busy(&dev->reset_work)) { + dev->reset_workfn = nvme_reset_failed_dev; + queue_work(nvme_workq, &dev->reset_work); + } } static void nvme_reset_notify(struct pci_dev *pdev, bool prepare) diff --git a/include/linux/nvme.h b/include/linux/nvme.h index e2429e8cdab4..0adad4a5419b 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -103,7 +103,6 @@ struct nvme_dev { u16 abort_limit; u8 event_limit; u8 vwc; - u8 initialized; }; /* -- cgit v1.2.3-58-ga151