diff options
48 files changed, 2145 insertions, 543 deletions
diff --git a/MAINTAINERS b/MAINTAINERS index ee99476f4b8f..1d57f1faff16 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10786,7 +10786,7 @@ F: net/ieee802154/ F: net/mac802154/ Intel VIRTIO DATA PATH ACCELERATOR -M: Zhu Lingshan <lingshan.zhu@intel.com> +M: Zhu Lingshan <lingshan.zhu@kernel.org> L: virtualization@lists.linux.dev S: Supported F: drivers/vdpa/ifcvf/ @@ -13611,6 +13611,11 @@ S: Supported F: Documentation/devicetree/bindings/mmc/marvell,xenon-sdhci.yaml F: drivers/mmc/host/sdhci-xenon* +MARVELL OCTEON ENDPOINT VIRTIO DATA PATH ACCELERATOR +R: schalla@marvell.com +R: vattunuru@marvell.com +F: drivers/vdpa/octeon_ep/ + MATROX FRAMEBUFFER DRIVER L: linux-fbdev@vger.kernel.org S: Orphan diff --git a/arch/um/drivers/virt-pci.c b/arch/um/drivers/virt-pci.c index 7cb503469bbd..b16a75c94b8b 100644 --- a/arch/um/drivers/virt-pci.c +++ b/arch/um/drivers/virt-pci.c @@ -567,12 +567,14 @@ struct device_node *pcibios_get_phb_of_node(struct pci_bus *bus) static int um_pci_init_vqs(struct um_pci_device *dev) { + struct virtqueue_info vqs_info[] = { + { "cmd", um_pci_cmd_vq_cb }, + { "irq", um_pci_irq_vq_cb }, + }; struct virtqueue *vqs[2]; - static const char *const names[2] = { "cmd", "irq" }; - vq_callback_t *cbs[2] = { um_pci_cmd_vq_cb, um_pci_irq_vq_cb }; int err, i; - err = virtio_find_vqs(dev->vdev, 2, vqs, cbs, names, NULL); + err = virtio_find_vqs(dev->vdev, 2, vqs, vqs_info, NULL); if (err) return err; diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c index 77faa2cf3a13..2b6e701776b6 100644 --- a/arch/um/drivers/virtio_uml.c +++ b/arch/um/drivers/virtio_uml.c @@ -1014,8 +1014,8 @@ error_kzalloc: } static int vu_find_vqs(struct virtio_device *vdev, unsigned nvqs, - struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], const bool *ctx, + struct virtqueue *vqs[], + struct virtqueue_info vqs_info[], struct irq_affinity *desc) { struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev); @@ -1031,13 +1031,15 @@ static int vu_find_vqs(struct virtio_device *vdev, unsigned nvqs, return rc; for (i = 0; i < nvqs; ++i) { - if (!names[i]) { + struct virtqueue_info *vqi = &vqs_info[i]; + + if (!vqi->name) { vqs[i] = NULL; continue; } - vqs[i] = vu_setup_vq(vdev, queue_idx++, callbacks[i], names[i], - ctx ? ctx[i] : false); + vqs[i] = vu_setup_vq(vdev, queue_idx++, vqi->callback, + vqi->name, vqi->ctx); if (IS_ERR(vqs[i])) { rc = PTR_ERR(vqs[i]); goto error_setup; diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index e3147a611151..194417abc105 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -964,8 +964,7 @@ static int init_vq(struct virtio_blk *vblk) { int err; unsigned short i; - vq_callback_t **callbacks; - const char **names; + struct virtqueue_info *vqs_info; struct virtqueue **vqs; unsigned short num_vqs; unsigned short num_poll_vqs; @@ -1002,28 +1001,26 @@ static int init_vq(struct virtio_blk *vblk) if (!vblk->vqs) return -ENOMEM; - names = kmalloc_array(num_vqs, sizeof(*names), GFP_KERNEL); - callbacks = kmalloc_array(num_vqs, sizeof(*callbacks), GFP_KERNEL); + vqs_info = kcalloc(num_vqs, sizeof(*vqs_info), GFP_KERNEL); vqs = kmalloc_array(num_vqs, sizeof(*vqs), GFP_KERNEL); - if (!names || !callbacks || !vqs) { + if (!vqs_info || !vqs) { err = -ENOMEM; goto out; } for (i = 0; i < num_vqs - num_poll_vqs; i++) { - callbacks[i] = virtblk_done; + vqs_info[i].callback = virtblk_done; snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req.%u", i); - names[i] = vblk->vqs[i].name; + vqs_info[i].name = vblk->vqs[i].name; } for (; i < num_vqs; i++) { - callbacks[i] = NULL; snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req_poll.%u", i); - names[i] = vblk->vqs[i].name; + vqs_info[i].name = vblk->vqs[i].name; } /* Discover virtqueues and write information to configuration. */ - err = virtio_find_vqs(vdev, num_vqs, vqs, callbacks, names, &desc); + err = virtio_find_vqs(vdev, num_vqs, vqs, vqs_info, &desc); if (err) goto out; @@ -1035,8 +1032,7 @@ static int init_vq(struct virtio_blk *vblk) out: kfree(vqs); - kfree(callbacks); - kfree(names); + kfree(vqs_info); if (err) kfree(vblk->vqs); return err; diff --git a/drivers/bluetooth/virtio_bt.c b/drivers/bluetooth/virtio_bt.c index 40bd83825c29..756f292df9e8 100644 --- a/drivers/bluetooth/virtio_bt.c +++ b/drivers/bluetooth/virtio_bt.c @@ -254,13 +254,9 @@ static void virtbt_rx_done(struct virtqueue *vq) static int virtbt_probe(struct virtio_device *vdev) { - vq_callback_t *callbacks[VIRTBT_NUM_VQS] = { - [VIRTBT_VQ_TX] = virtbt_tx_done, - [VIRTBT_VQ_RX] = virtbt_rx_done, - }; - const char *names[VIRTBT_NUM_VQS] = { - [VIRTBT_VQ_TX] = "tx", - [VIRTBT_VQ_RX] = "rx", + struct virtqueue_info vqs_info[VIRTBT_NUM_VQS] = { + [VIRTBT_VQ_TX] = { "tx", virtbt_tx_done }, + [VIRTBT_VQ_RX] = { "rx", virtbt_rx_done }, }; struct virtio_bluetooth *vbt; struct hci_dev *hdev; @@ -288,8 +284,7 @@ static int virtbt_probe(struct virtio_device *vdev) INIT_WORK(&vbt->rx, virtbt_rx_work); - err = virtio_find_vqs(vdev, VIRTBT_NUM_VQS, vbt->vqs, callbacks, - names, NULL); + err = virtio_find_vqs(vdev, VIRTBT_NUM_VQS, vbt->vqs, vqs_info, NULL); if (err) return err; diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c index d9ee2dbc7eab..de7d720d99fa 100644 --- a/drivers/char/virtio_console.c +++ b/drivers/char/virtio_console.c @@ -1804,8 +1804,7 @@ static void config_work_handler(struct work_struct *work) static int init_vqs(struct ports_device *portdev) { - vq_callback_t **io_callbacks; - char **io_names; + struct virtqueue_info *vqs_info; struct virtqueue **vqs; u32 i, j, nr_ports, nr_queues; int err; @@ -1814,15 +1813,12 @@ static int init_vqs(struct ports_device *portdev) nr_queues = use_multiport(portdev) ? (nr_ports + 1) * 2 : 2; vqs = kmalloc_array(nr_queues, sizeof(struct virtqueue *), GFP_KERNEL); - io_callbacks = kmalloc_array(nr_queues, sizeof(vq_callback_t *), - GFP_KERNEL); - io_names = kmalloc_array(nr_queues, sizeof(char *), GFP_KERNEL); + vqs_info = kcalloc(nr_queues, sizeof(*vqs_info), GFP_KERNEL); portdev->in_vqs = kmalloc_array(nr_ports, sizeof(struct virtqueue *), GFP_KERNEL); portdev->out_vqs = kmalloc_array(nr_ports, sizeof(struct virtqueue *), GFP_KERNEL); - if (!vqs || !io_callbacks || !io_names || !portdev->in_vqs || - !portdev->out_vqs) { + if (!vqs || !vqs_info || !portdev->in_vqs || !portdev->out_vqs) { err = -ENOMEM; goto free; } @@ -1833,30 +1829,27 @@ static int init_vqs(struct ports_device *portdev) * 0 before others. */ j = 0; - io_callbacks[j] = in_intr; - io_callbacks[j + 1] = out_intr; - io_names[j] = "input"; - io_names[j + 1] = "output"; + vqs_info[j].callback = in_intr; + vqs_info[j + 1].callback = out_intr; + vqs_info[j].name = "input"; + vqs_info[j + 1].name = "output"; j += 2; if (use_multiport(portdev)) { - io_callbacks[j] = control_intr; - io_callbacks[j + 1] = NULL; - io_names[j] = "control-i"; - io_names[j + 1] = "control-o"; + vqs_info[j].callback = control_intr; + vqs_info[j].name = "control-i"; + vqs_info[j + 1].name = "control-o"; for (i = 1; i < nr_ports; i++) { j += 2; - io_callbacks[j] = in_intr; - io_callbacks[j + 1] = out_intr; - io_names[j] = "input"; - io_names[j + 1] = "output"; + vqs_info[j].callback = in_intr; + vqs_info[j + 1].callback = out_intr; + vqs_info[j].name = "input"; + vqs_info[j + 1].name = "output"; } } /* Find the queues. */ - err = virtio_find_vqs(portdev->vdev, nr_queues, vqs, - io_callbacks, - (const char **)io_names, NULL); + err = virtio_find_vqs(portdev->vdev, nr_queues, vqs, vqs_info, NULL); if (err) goto free; @@ -1874,8 +1867,7 @@ static int init_vqs(struct ports_device *portdev) portdev->out_vqs[i] = vqs[j + 1]; } } - kfree(io_names); - kfree(io_callbacks); + kfree(vqs_info); kfree(vqs); return 0; @@ -1883,8 +1875,7 @@ static int init_vqs(struct ports_device *portdev) free: kfree(portdev->out_vqs); kfree(portdev->in_vqs); - kfree(io_names); - kfree(io_callbacks); + kfree(vqs_info); kfree(vqs); return err; diff --git a/drivers/crypto/virtio/virtio_crypto_core.c b/drivers/crypto/virtio/virtio_crypto_core.c index 30cd040aa03b..d0278eb568b9 100644 --- a/drivers/crypto/virtio/virtio_crypto_core.c +++ b/drivers/crypto/virtio/virtio_crypto_core.c @@ -96,11 +96,10 @@ static void virtcrypto_dataq_callback(struct virtqueue *vq) static int virtcrypto_find_vqs(struct virtio_crypto *vi) { - vq_callback_t **callbacks; + struct virtqueue_info *vqs_info; struct virtqueue **vqs; int ret = -ENOMEM; int i, total_vqs; - const char **names; struct device *dev = &vi->vdev->dev; /* @@ -114,26 +113,23 @@ static int virtcrypto_find_vqs(struct virtio_crypto *vi) vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL); if (!vqs) goto err_vq; - callbacks = kcalloc(total_vqs, sizeof(*callbacks), GFP_KERNEL); - if (!callbacks) - goto err_callback; - names = kcalloc(total_vqs, sizeof(*names), GFP_KERNEL); - if (!names) - goto err_names; + vqs_info = kcalloc(total_vqs, sizeof(*vqs_info), GFP_KERNEL); + if (!vqs_info) + goto err_vqs_info; /* Parameters for control virtqueue */ - callbacks[total_vqs - 1] = virtcrypto_ctrlq_callback; - names[total_vqs - 1] = "controlq"; + vqs_info[total_vqs - 1].callback = virtcrypto_ctrlq_callback; + vqs_info[total_vqs - 1].name = "controlq"; /* Allocate/initialize parameters for data virtqueues */ for (i = 0; i < vi->max_data_queues; i++) { - callbacks[i] = virtcrypto_dataq_callback; + vqs_info[i].callback = virtcrypto_dataq_callback; snprintf(vi->data_vq[i].name, sizeof(vi->data_vq[i].name), "dataq.%d", i); - names[i] = vi->data_vq[i].name; + vqs_info[i].name = vi->data_vq[i].name; } - ret = virtio_find_vqs(vi->vdev, total_vqs, vqs, callbacks, names, NULL); + ret = virtio_find_vqs(vi->vdev, total_vqs, vqs, vqs_info, NULL); if (ret) goto err_find; @@ -153,18 +149,15 @@ static int virtcrypto_find_vqs(struct virtio_crypto *vi) (unsigned long)&vi->data_vq[i]); } - kfree(names); - kfree(callbacks); + kfree(vqs_info); kfree(vqs); return 0; err_engine: err_find: - kfree(names); -err_names: - kfree(callbacks); -err_callback: + kfree(vqs_info); +err_vqs_info: kfree(vqs); err_vq: return ret; diff --git a/drivers/firmware/arm_scmi/virtio.c b/drivers/firmware/arm_scmi/virtio.c index 4892058445ce..dd3459bdb9cb 100644 --- a/drivers/firmware/arm_scmi/virtio.c +++ b/drivers/firmware/arm_scmi/virtio.c @@ -354,11 +354,9 @@ static void scmi_vio_deferred_tx_worker(struct work_struct *work) scmi_vio_channel_release(vioch); } -static const char *const scmi_vio_vqueue_names[] = { "tx", "rx" }; - -static vq_callback_t *scmi_vio_complete_callbacks[] = { - scmi_vio_complete_cb, - scmi_vio_complete_cb +static struct virtqueue_info scmi_vio_vqs_info[] = { + { "tx", scmi_vio_complete_cb }, + { "rx", scmi_vio_complete_cb }, }; static unsigned int virtio_get_max_msg(struct scmi_chan_info *base_cinfo) @@ -831,8 +829,7 @@ static int scmi_vio_probe(struct virtio_device *vdev) if (have_vq_rx) channels[VIRTIO_SCMI_VQ_RX].is_rx = true; - ret = virtio_find_vqs(vdev, vq_cnt, vqs, scmi_vio_complete_callbacks, - scmi_vio_vqueue_names, NULL); + ret = virtio_find_vqs(vdev, vq_cnt, vqs, scmi_vio_vqs_info, NULL); if (ret) { dev_err(dev, "Failed to get %d virtqueue(s)\n", vq_cnt); return ret; diff --git a/drivers/gpio/gpio-virtio.c b/drivers/gpio/gpio-virtio.c index 9fae8e396c58..93544ff62513 100644 --- a/drivers/gpio/gpio-virtio.c +++ b/drivers/gpio/gpio-virtio.c @@ -457,15 +457,15 @@ static void virtio_gpio_free_vqs(struct virtio_device *vdev) static int virtio_gpio_alloc_vqs(struct virtio_gpio *vgpio, struct virtio_device *vdev) { - const char * const names[] = { "requestq", "eventq" }; - vq_callback_t *cbs[] = { - virtio_gpio_request_vq, - virtio_gpio_event_vq, + struct virtqueue_info vqs_info[] = { + { "requestq", virtio_gpio_request_vq }, + { "eventq", virtio_gpio_event_vq }, }; struct virtqueue *vqs[2] = { NULL, NULL }; int ret; - ret = virtio_find_vqs(vdev, vgpio->irq_lines ? 2 : 1, vqs, cbs, names, NULL); + ret = virtio_find_vqs(vdev, vgpio->irq_lines ? 2 : 1, vqs, + vqs_info, NULL); if (ret) { dev_err(&vdev->dev, "failed to find vqs: %d\n", ret); return ret; diff --git a/drivers/gpu/drm/virtio/virtgpu_kms.c b/drivers/gpu/drm/virtio/virtgpu_kms.c index 5a3b5aaed1f3..7dfb2006c561 100644 --- a/drivers/gpu/drm/virtio/virtgpu_kms.c +++ b/drivers/gpu/drm/virtio/virtgpu_kms.c @@ -116,11 +116,10 @@ static void virtio_gpu_get_capsets(struct virtio_gpu_device *vgdev, int virtio_gpu_init(struct virtio_device *vdev, struct drm_device *dev) { - static vq_callback_t *callbacks[] = { - virtio_gpu_ctrl_ack, virtio_gpu_cursor_ack + struct virtqueue_info vqs_info[] = { + { "control", virtio_gpu_ctrl_ack }, + { "cursor", virtio_gpu_cursor_ack }, }; - static const char * const names[] = { "control", "cursor" }; - struct virtio_gpu_device *vgdev; /* this will expand later */ struct virtqueue *vqs[2]; @@ -207,7 +206,7 @@ int virtio_gpu_init(struct virtio_device *vdev, struct drm_device *dev) DRM_INFO("features: %ccontext_init\n", vgdev->has_context_init ? '+' : '-'); - ret = virtio_find_vqs(vgdev->vdev, 2, vqs, callbacks, names, NULL); + ret = virtio_find_vqs(vgdev->vdev, 2, vqs, vqs_info, NULL); if (ret) { DRM_ERROR("failed to find virt queues\n"); goto err_vqs; diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c index 36d680826b57..b85ce6310ddb 100644 --- a/drivers/iommu/virtio-iommu.c +++ b/drivers/iommu/virtio-iommu.c @@ -1084,14 +1084,13 @@ static struct iommu_ops viommu_ops = { static int viommu_init_vqs(struct viommu_dev *viommu) { struct virtio_device *vdev = dev_to_virtio(viommu->dev); - const char *names[] = { "request", "event" }; - vq_callback_t *callbacks[] = { - NULL, /* No async requests */ - viommu_event_handler, + struct virtqueue_info vqs_info[] = { + { "request" }, + { "event", viommu_event_handler }, }; - return virtio_find_vqs(vdev, VIOMMU_NR_VQS, viommu->vqs, callbacks, - names, NULL); + return virtio_find_vqs(vdev, VIOMMU_NR_VQS, viommu->vqs, + vqs_info, NULL); } static int viommu_fill_evtq(struct viommu_dev *viommu) diff --git a/drivers/net/caif/caif_virtio.c b/drivers/net/caif/caif_virtio.c index 99d984851fef..7fea00c7ca8a 100644 --- a/drivers/net/caif/caif_virtio.c +++ b/drivers/net/caif/caif_virtio.c @@ -646,9 +646,7 @@ static inline void debugfs_init(struct cfv_info *cfv) /* Setup CAIF for the a virtio device */ static int cfv_probe(struct virtio_device *vdev) { - vq_callback_t *vq_cbs = cfv_release_cb; vrh_callback_t *vrh_cbs = cfv_recv; - const char *names = "output"; const char *cfv_netdev_name = "cfvrt"; struct net_device *netdev; struct cfv_info *cfv; @@ -675,9 +673,11 @@ static int cfv_probe(struct virtio_device *vdev) goto err; /* Get the TX virtio ring. This is a "guest side vring". */ - err = virtio_find_vqs(vdev, 1, &cfv->vq_tx, &vq_cbs, &names, NULL); - if (err) + cfv->vq_tx = virtio_find_single_vq(vdev, cfv_release_cb, "output"); + if (IS_ERR(cfv->vq_tx)) { + err = PTR_ERR(cfv->vq_tx); goto err; + } /* Get the CAIF configuration from virtio config space, if available */ if (vdev->config->get) { diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index af474cc191d0..0383a3e136d6 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -5946,9 +5946,8 @@ static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqu static int virtnet_find_vqs(struct virtnet_info *vi) { - vq_callback_t **callbacks; + struct virtqueue_info *vqs_info; struct virtqueue **vqs; - const char **names; int ret = -ENOMEM; int total_vqs; bool *ctx; @@ -5965,12 +5964,9 @@ static int virtnet_find_vqs(struct virtnet_info *vi) vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL); if (!vqs) goto err_vq; - callbacks = kmalloc_array(total_vqs, sizeof(*callbacks), GFP_KERNEL); - if (!callbacks) - goto err_callback; - names = kmalloc_array(total_vqs, sizeof(*names), GFP_KERNEL); - if (!names) - goto err_names; + vqs_info = kcalloc(total_vqs, sizeof(*vqs_info), GFP_KERNEL); + if (!vqs_info) + goto err_vqs_info; if (!vi->big_packets || vi->mergeable_rx_bufs) { ctx = kcalloc(total_vqs, sizeof(*ctx), GFP_KERNEL); if (!ctx) @@ -5981,24 +5977,22 @@ static int virtnet_find_vqs(struct virtnet_info *vi) /* Parameters for control virtqueue, if any */ if (vi->has_cvq) { - callbacks[total_vqs - 1] = NULL; - names[total_vqs - 1] = "control"; + vqs_info[total_vqs - 1].name = "control"; } /* Allocate/initialize parameters for send/receive virtqueues */ for (i = 0; i < vi->max_queue_pairs; i++) { - callbacks[rxq2vq(i)] = skb_recv_done; - callbacks[txq2vq(i)] = skb_xmit_done; + vqs_info[rxq2vq(i)].callback = skb_recv_done; + vqs_info[txq2vq(i)].callback = skb_xmit_done; sprintf(vi->rq[i].name, "input.%u", i); sprintf(vi->sq[i].name, "output.%u", i); - names[rxq2vq(i)] = vi->rq[i].name; - names[txq2vq(i)] = vi->sq[i].name; + vqs_info[rxq2vq(i)].name = vi->rq[i].name; + vqs_info[txq2vq(i)].name = vi->sq[i].name; if (ctx) - ctx[rxq2vq(i)] = true; + vqs_info[rxq2vq(i)].ctx = true; } - ret = virtio_find_vqs_ctx(vi->vdev, total_vqs, vqs, callbacks, - names, ctx, NULL); + ret = virtio_find_vqs(vi->vdev, total_vqs, vqs, vqs_info, NULL); if (ret) goto err_find; @@ -6020,10 +6014,8 @@ static int virtnet_find_vqs(struct virtnet_info *vi) err_find: kfree(ctx); err_ctx: - kfree(names); -err_names: - kfree(callbacks); -err_callback: + kfree(vqs_info); +err_vqs_info: kfree(vqs); err_vq: return ret; diff --git a/drivers/net/wireless/virtual/mac80211_hwsim.c b/drivers/net/wireless/virtual/mac80211_hwsim.c index 714e1f04b0cb..d86e6ff4523d 100644 --- a/drivers/net/wireless/virtual/mac80211_hwsim.c +++ b/drivers/net/wireless/virtual/mac80211_hwsim.c @@ -6626,17 +6626,13 @@ static void hwsim_virtio_rx_done(struct virtqueue *vq) static int init_vqs(struct virtio_device *vdev) { - vq_callback_t *callbacks[HWSIM_NUM_VQS] = { - [HWSIM_VQ_TX] = hwsim_virtio_tx_done, - [HWSIM_VQ_RX] = hwsim_virtio_rx_done, - }; - const char *names[HWSIM_NUM_VQS] = { - [HWSIM_VQ_TX] = "tx", - [HWSIM_VQ_RX] = "rx", + struct virtqueue_info vqs_info[HWSIM_NUM_VQS] = { + [HWSIM_VQ_TX] = { "tx", hwsim_virtio_tx_done }, + [HWSIM_VQ_RX] = { "rx", hwsim_virtio_rx_done }, }; return virtio_find_vqs(vdev, HWSIM_NUM_VQS, - hwsim_vqs, callbacks, names, NULL); + hwsim_vqs, vqs_info, NULL); } static int fill_vq(struct virtqueue *vq) diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c b/drivers/platform/mellanox/mlxbf-tmfifo.c index b8d1e32e97eb..6c834e39352d 100644 --- a/drivers/platform/mellanox/mlxbf-tmfifo.c +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c @@ -1058,9 +1058,7 @@ static void mlxbf_tmfifo_virtio_del_vqs(struct virtio_device *vdev) static int mlxbf_tmfifo_virtio_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], - vq_callback_t *callbacks[], - const char * const names[], - const bool *ctx, + struct virtqueue_info vqs_info[], struct irq_affinity *desc) { struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev); @@ -1072,7 +1070,9 @@ static int mlxbf_tmfifo_virtio_find_vqs(struct virtio_device *vdev, return -EINVAL; for (i = 0; i < nvqs; ++i) { - if (!names[i]) { + struct virtqueue_info *vqi = &vqs_info[i]; + + if (!vqi->name) { ret = -EINVAL; goto error; } @@ -1084,7 +1084,7 @@ static int mlxbf_tmfifo_virtio_find_vqs(struct virtio_device *vdev, vq = vring_new_virtqueue(i, vring->num, vring->align, vdev, false, false, vring->va, mlxbf_tmfifo_virtio_notify, - callbacks[i], names[i]); + vqi->callback, vqi->name); if (!vq) { dev_err(&vdev->dev, "vring_new_virtqueue failed\n"); ret = -ENOMEM; diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c index 25b66b113b69..d3f39009b28e 100644 --- a/drivers/remoteproc/remoteproc_virtio.c +++ b/drivers/remoteproc/remoteproc_virtio.c @@ -182,21 +182,21 @@ static void rproc_virtio_del_vqs(struct virtio_device *vdev) static int rproc_virtio_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], - vq_callback_t *callbacks[], - const char * const names[], - const bool * ctx, + struct virtqueue_info vqs_info[], struct irq_affinity *desc) { int i, ret, queue_idx = 0; for (i = 0; i < nvqs; ++i) { - if (!names[i]) { + struct virtqueue_info *vqi = &vqs_info[i]; + + if (!vqi->name) { vqs[i] = NULL; continue; } - vqs[i] = rp_find_vq(vdev, queue_idx++, callbacks[i], names[i], - ctx ? ctx[i] : false); + vqs[i] = rp_find_vq(vdev, queue_idx++, vqi->callback, + vqi->name, vqi->ctx); if (IS_ERR(vqs[i])) { ret = PTR_ERR(vqs[i]); goto error; diff --git a/drivers/rpmsg/virtio_rpmsg_bus.c b/drivers/rpmsg/virtio_rpmsg_bus.c index e9e8c1f7829f..89d7a3b8c48b 100644 --- a/drivers/rpmsg/virtio_rpmsg_bus.c +++ b/drivers/rpmsg/virtio_rpmsg_bus.c @@ -868,8 +868,10 @@ static void rpmsg_virtio_del_ctrl_dev(struct rpmsg_device *rpdev_ctrl) static int rpmsg_probe(struct virtio_device *vdev) { - vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done }; - static const char * const names[] = { "input", "output" }; + struct virtqueue_info vqs_info[] = { + { "input", rpmsg_recv_done }, + { "output", rpmsg_xmit_done }, + }; struct virtqueue *vqs[2]; struct virtproc_info *vrp; struct virtio_rpmsg_channel *vch = NULL; @@ -891,7 +893,7 @@ static int rpmsg_probe(struct virtio_device *vdev) init_waitqueue_head(&vrp->sendq); /* We expect two virtqueues, rx and tx (and in this order) */ - err = virtio_find_vqs(vdev, 2, vqs, vq_cbs, names, NULL); + err = virtio_find_vqs(vdev, 2, vqs, vqs_info, NULL); if (err) goto free_vrp; diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c index d6491fc84e8c..62eca9419ad7 100644 --- a/drivers/s390/virtio/virtio_ccw.c +++ b/drivers/s390/virtio/virtio_ccw.c @@ -689,9 +689,7 @@ out: static int virtio_ccw_find_vqs(struct virtio_device *vdev, unsigned nvqs, struct virtqueue *vqs[], - vq_callback_t *callbacks[], - const char * const names[], - const bool *ctx, + struct virtqueue_info vqs_info[], struct irq_affinity *desc) { struct virtio_ccw_device *vcdev = to_vc_device(vdev); @@ -705,14 +703,15 @@ static int virtio_ccw_find_vqs(struct virtio_device *vdev, unsigned nvqs, return -ENOMEM; for (i = 0; i < nvqs; ++i) { - if (!names[i]) { + struct virtqueue_info *vqi = &vqs_info[i]; + + if (!vqi->name) { vqs[i] = NULL; continue; } - vqs[i] = virtio_ccw_setup_vq(vdev, queue_idx++, callbacks[i], - names[i], ctx ? ctx[i] : false, - ccw); + vqs[i] = virtio_ccw_setup_vq(vdev, queue_idx++, vqi->callback, + vqi->name, vqi->ctx, ccw); if (IS_ERR(vqs[i])) { ret = PTR_ERR(vqs[i]); vqs[i] = NULL; diff --git a/drivers/scsi/virtio_scsi.c b/drivers/scsi/virtio_scsi.c index 89ca26945721..8471f38b730e 100644 --- a/drivers/scsi/virtio_scsi.c +++ b/drivers/scsi/virtio_scsi.c @@ -841,19 +841,16 @@ static int virtscsi_init(struct virtio_device *vdev, int err; u32 i; u32 num_vqs, num_poll_vqs, num_req_vqs; - vq_callback_t **callbacks; - const char **names; + struct virtqueue_info *vqs_info; struct virtqueue **vqs; struct irq_affinity desc = { .pre_vectors = 2 }; num_req_vqs = vscsi->num_queues; num_vqs = num_req_vqs + VIRTIO_SCSI_VQ_BASE; vqs = kmalloc_array(num_vqs, sizeof(struct virtqueue *), GFP_KERNEL); - callbacks = kmalloc_array(num_vqs, sizeof(vq_callback_t *), - GFP_KERNEL); - names = kmalloc_array(num_vqs, sizeof(char *), GFP_KERNEL); + vqs_info = kcalloc(num_vqs, sizeof(*vqs_info), GFP_KERNEL); - if (!callbacks || !vqs || !names) { + if (!vqs || !vqs_info) { err = -ENOMEM; goto out; } @@ -869,22 +866,20 @@ static int virtscsi_init(struct virtio_device *vdev, vscsi->io_queues[HCTX_TYPE_READ], vscsi->io_queues[HCTX_TYPE_POLL]); - callbacks[0] = virtscsi_ctrl_done; - callbacks[1] = virtscsi_event_done; - names[0] = "control"; - names[1] = "event"; + vqs_info[0].callback = virtscsi_ctrl_done; + vqs_info[0].name = "control"; + vqs_info[1].callback = virtscsi_event_done; + vqs_info[1].name = "event"; for (i = VIRTIO_SCSI_VQ_BASE; i < num_vqs - num_poll_vqs; i++) { - callbacks[i] = virtscsi_req_done; - names[i] = "request"; + vqs_info[i].callback = virtscsi_req_done; + vqs_info[i].name = "request"; } - for (; i < num_vqs; i++) { - callbacks[i] = NULL; - names[i] = "request_poll"; - } + for (; i < num_vqs; i++) + vqs_info[i].name = "request_poll"; /* Discover virtqueues and write information to configuration. */ - err = virtio_find_vqs(vdev, num_vqs, vqs, callbacks, names, &desc); + err = virtio_find_vqs(vdev, num_vqs, vqs, vqs_info, &desc); if (err) goto out; @@ -900,8 +895,7 @@ static int virtscsi_init(struct virtio_device *vdev, err = 0; out: - kfree(names); - kfree(callbacks); + kfree(vqs_info); kfree(vqs); if (err) virtscsi_remove_vqs(vdev); diff --git a/drivers/vdpa/Kconfig b/drivers/vdpa/Kconfig index 656c1cb541de..5265d09fc1c4 100644 --- a/drivers/vdpa/Kconfig +++ b/drivers/vdpa/Kconfig @@ -126,4 +126,15 @@ config PDS_VDPA With this driver, the VirtIO dataplane can be offloaded to an AMD/Pensando DSC device. +config OCTEONEP_VDPA + tristate "vDPA driver for Octeon DPU devices" + depends on m + depends on PCI_MSI + help + This is a vDPA driver designed for Marvell's Octeon DPU devices. + This driver enables the offloading of the VirtIO dataplane to an + Octeon DPU device. + Please note that this driver must be built as a module and it + cannot be loaded until the Octeon emulation software is running. + endif # VDPA diff --git a/drivers/vdpa/Makefile b/drivers/vdpa/Makefile index 8f53c6f3cca7..5654d36707af 100644 --- a/drivers/vdpa/Makefile +++ b/drivers/vdpa/Makefile @@ -8,3 +8,4 @@ obj-$(CONFIG_VP_VDPA) += virtio_pci/ obj-$(CONFIG_ALIBABA_ENI_VDPA) += alibaba/ obj-$(CONFIG_SNET_VDPA) += solidrun/ obj-$(CONFIG_PDS_VDPA) += pds/ +obj-$(CONFIG_OCTEONEP_VDPA) += octeon_ep/ diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c index 80d0a0460885..ccf64d7bbfaa 100644 --- a/drivers/vdpa/ifcvf/ifcvf_main.c +++ b/drivers/vdpa/ifcvf/ifcvf_main.c @@ -894,4 +894,5 @@ static struct pci_driver ifcvf_driver = { module_pci_driver(ifcvf_driver); +MODULE_DESCRIPTION("Intel IFC VF NIC driver for virtio dataplane offloading"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c index ecfc16151d61..fa78e8288ebb 100644 --- a/drivers/vdpa/mlx5/net/mlx5_vnet.c +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c @@ -48,6 +48,18 @@ MODULE_LICENSE("Dual BSD/GPL"); #define MLX5V_UNTAGGED 0x1000 +/* Device must start with 1 queue pair, as per VIRTIO v1.2 spec, section + * 5.1.6.5.5 "Device operation in multiqueue mode": + * + * Multiqueue is disabled by default. + * The driver enables multiqueue by sending a command using class + * VIRTIO_NET_CTRL_MQ. The command selects the mode of multiqueue + * operation, as follows: ... + */ +#define MLX5V_DEFAULT_VQ_COUNT 2 + +#define MLX5V_DEFAULT_VQ_SIZE 256 + struct mlx5_vdpa_cq_buf { struct mlx5_frag_buf_ctrl fbc; struct mlx5_frag_buf frag_buf; @@ -144,10 +156,11 @@ static bool is_index_valid(struct mlx5_vdpa_dev *mvdev, u16 idx) return idx <= mvdev->max_idx; } -static void free_resources(struct mlx5_vdpa_net *ndev); -static void init_mvqs(struct mlx5_vdpa_net *ndev); -static int setup_driver(struct mlx5_vdpa_dev *mvdev); -static void teardown_driver(struct mlx5_vdpa_net *ndev); +static void free_fixed_resources(struct mlx5_vdpa_net *ndev); +static void mvqs_set_defaults(struct mlx5_vdpa_net *ndev); +static int setup_vq_resources(struct mlx5_vdpa_net *ndev, bool filled); +static void teardown_vq_resources(struct mlx5_vdpa_net *ndev); +static int resume_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq); static bool mlx5_vdpa_debug; @@ -862,13 +875,16 @@ static bool msix_mode_supported(struct mlx5_vdpa_dev *mvdev) pci_msix_can_alloc_dyn(mvdev->mdev->pdev); } -static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) +static int create_virtqueue(struct mlx5_vdpa_net *ndev, + struct mlx5_vdpa_virtqueue *mvq, + bool filled) { int inlen = MLX5_ST_SZ_BYTES(create_virtio_net_q_in); u32 out[MLX5_ST_SZ_DW(create_virtio_net_q_out)] = {}; struct mlx5_vdpa_dev *mvdev = &ndev->mvdev; struct mlx5_vdpa_mr *vq_mr; struct mlx5_vdpa_mr *vq_desc_mr; + u64 features = filled ? mvdev->actual_features : mvdev->mlx_features; void *obj_context; u16 mlx_features; void *cmd_hdr; @@ -886,7 +902,7 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque goto err_alloc; } - mlx_features = get_features(ndev->mvdev.actual_features); + mlx_features = get_features(features); cmd_hdr = MLX5_ADDR_OF(create_virtio_net_q_in, in, general_obj_in_cmd_hdr); MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); @@ -894,8 +910,6 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); obj_context = MLX5_ADDR_OF(create_virtio_net_q_in, in, obj_context); - MLX5_SET(virtio_net_q_object, obj_context, hw_available_index, mvq->avail_idx); - MLX5_SET(virtio_net_q_object, obj_context, hw_used_index, mvq->used_idx); MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_12_3, mlx_features >> 3); MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_2_0, @@ -917,17 +931,36 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque MLX5_SET(virtio_q, vq_ctx, queue_index, mvq->index); MLX5_SET(virtio_q, vq_ctx, queue_size, mvq->num_ent); MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0, - !!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_F_VERSION_1))); - MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr); - MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr); - MLX5_SET64(virtio_q, vq_ctx, available_addr, mvq->driver_addr); - vq_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP]]; - if (vq_mr) - MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, vq_mr->mkey); - - vq_desc_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_DESC_GROUP]]; - if (vq_desc_mr && MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, desc_group_mkey_supported)) - MLX5_SET(virtio_q, vq_ctx, desc_group_mkey, vq_desc_mr->mkey); + !!(features & BIT_ULL(VIRTIO_F_VERSION_1))); + + if (filled) { + MLX5_SET(virtio_net_q_object, obj_context, hw_available_index, mvq->avail_idx); + MLX5_SET(virtio_net_q_object, obj_context, hw_used_index, mvq->used_idx); + + MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr); + MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr); + MLX5_SET64(virtio_q, vq_ctx, available_addr, mvq->driver_addr); + + vq_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP]]; + if (vq_mr) + MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, vq_mr->mkey); + + vq_desc_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_DESC_GROUP]]; + if (vq_desc_mr && + MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, desc_group_mkey_supported)) + MLX5_SET(virtio_q, vq_ctx, desc_group_mkey, vq_desc_mr->mkey); + } else { + /* If there is no mr update, make sure that the existing ones are set + * modify to ready. + */ + vq_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP]]; + if (vq_mr) + mvq->modified_fields |= MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_MKEY; + + vq_desc_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_DESC_GROUP]]; + if (vq_desc_mr) + mvq->modified_fields |= MLX5_VIRTQ_MODIFY_MASK_DESC_GROUP_MKEY; + } MLX5_SET(virtio_q, vq_ctx, umem_1_id, mvq->umem1.id); MLX5_SET(virtio_q, vq_ctx, umem_1_size, mvq->umem1.size); @@ -947,12 +980,15 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque kfree(in); mvq->virtq_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); - mlx5_vdpa_get_mr(mvdev, vq_mr); - mvq->vq_mr = vq_mr; + if (filled) { + mlx5_vdpa_get_mr(mvdev, vq_mr); + mvq->vq_mr = vq_mr; - if (vq_desc_mr && MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, desc_group_mkey_supported)) { - mlx5_vdpa_get_mr(mvdev, vq_desc_mr); - mvq->desc_mr = vq_desc_mr; + if (vq_desc_mr && + MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, desc_group_mkey_supported)) { + mlx5_vdpa_get_mr(mvdev, vq_desc_mr); + mvq->desc_mr = vq_desc_mr; + } } return 0; @@ -1273,6 +1309,19 @@ static int modify_virtqueue(struct mlx5_vdpa_net *ndev, if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_USED_IDX) MLX5_SET(virtio_net_q_object, obj_context, hw_used_index, mvq->used_idx); + if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_QUEUE_VIRTIO_VERSION) + MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0, + !!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_F_VERSION_1))); + + if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_QUEUE_FEATURES) { + u16 mlx_features = get_features(ndev->mvdev.actual_features); + + MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_12_3, + mlx_features >> 3); + MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_2_0, + mlx_features & 7); + } + if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_MKEY) { vq_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP]]; @@ -1417,14 +1466,13 @@ static void dealloc_vector(struct mlx5_vdpa_net *ndev, } } -static int setup_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) +static int setup_vq(struct mlx5_vdpa_net *ndev, + struct mlx5_vdpa_virtqueue *mvq, + bool filled) { u16 idx = mvq->index; int err; - if (!mvq->num_ent) - return 0; - if (mvq->initialized) return 0; @@ -1449,20 +1497,18 @@ static int setup_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) goto err_connect; alloc_vector(ndev, mvq); - err = create_virtqueue(ndev, mvq); + err = create_virtqueue(ndev, mvq, filled); if (err) goto err_vq; + mvq->initialized = true; + if (mvq->ready) { - err = modify_virtqueue_state(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY); - if (err) { - mlx5_vdpa_warn(&ndev->mvdev, "failed to modify to ready vq idx %d(%d)\n", - idx, err); + err = resume_vq(ndev, mvq); + if (err) goto err_modify; - } } - mvq->initialized = true; return 0; err_modify: @@ -1479,51 +1525,105 @@ err_fwqp: return err; } -static void suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) +static int suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) { struct mlx5_virtq_attr attr; + int err; if (!mvq->initialized) - return; + return 0; if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) - return; + return 0; - if (modify_virtqueue_state(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND)) - mlx5_vdpa_warn(&ndev->mvdev, "modify to suspend failed\n"); + err = modify_virtqueue_state(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND); + if (err) { + mlx5_vdpa_warn(&ndev->mvdev, "modify to suspend failed, err: %d\n", err); + return err; + } - if (query_virtqueue(ndev, mvq, &attr)) { - mlx5_vdpa_warn(&ndev->mvdev, "failed to query virtqueue\n"); - return; + err = query_virtqueue(ndev, mvq, &attr); + if (err) { + mlx5_vdpa_warn(&ndev->mvdev, "failed to query virtqueue, err: %d\n", err); + return err; } + mvq->avail_idx = attr.available_index; mvq->used_idx = attr.used_index; + + return 0; } -static void suspend_vqs(struct mlx5_vdpa_net *ndev) +static int suspend_vqs(struct mlx5_vdpa_net *ndev) { + int err = 0; int i; - for (i = 0; i < ndev->mvdev.max_vqs; i++) - suspend_vq(ndev, &ndev->vqs[i]); + for (i = 0; i < ndev->cur_num_vqs; i++) { + int local_err = suspend_vq(ndev, &ndev->vqs[i]); + + err = local_err ? local_err : err; + } + + return err; } -static void resume_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) +static int resume_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) { - if (!mvq->initialized || !is_resumable(ndev)) - return; + int err; - if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND) - return; + if (!mvq->initialized) + return 0; + + if (mvq->index >= ndev->cur_num_vqs) + return 0; - if (modify_virtqueue_state(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY)) - mlx5_vdpa_warn(&ndev->mvdev, "modify to resume failed for vq %u\n", mvq->index); + switch (mvq->fw_state) { + case MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT: + /* Due to a FW quirk we need to modify the VQ fields first then change state. + * This should be fixed soon. After that, a single command can be used. + */ + err = modify_virtqueue(ndev, mvq, 0); + if (err) { + mlx5_vdpa_warn(&ndev->mvdev, + "modify vq properties failed for vq %u, err: %d\n", + mvq->index, err); + return err; + } + break; + case MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND: + if (!is_resumable(ndev)) { + mlx5_vdpa_warn(&ndev->mvdev, "vq %d is not resumable\n", mvq->index); + return -EINVAL; + } + break; + case MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY: + return 0; + default: + mlx5_vdpa_warn(&ndev->mvdev, "resume vq %u called from bad state %d\n", + mvq->index, mvq->fw_state); + return -EINVAL; + } + + err = modify_virtqueue_state(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY); + if (err) + mlx5_vdpa_warn(&ndev->mvdev, "modify to resume failed for vq %u, err: %d\n", + mvq->index, err); + + return err; } -static void resume_vqs(struct mlx5_vdpa_net *ndev) +static int resume_vqs(struct mlx5_vdpa_net *ndev) { - for (int i = 0; i < ndev->mvdev.max_vqs; i++) - resume_vq(ndev, &ndev->vqs[i]); + int err = 0; + + for (int i = 0; i < ndev->cur_num_vqs; i++) { + int local_err = resume_vq(ndev, &ndev->vqs[i]); + + err = local_err ? local_err : err; + } + + return err; } static void teardown_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) @@ -2033,14 +2133,22 @@ static int change_num_qps(struct mlx5_vdpa_dev *mvdev, int newqps) if (err) return err; - for (i = ndev->cur_num_vqs - 1; i >= 2 * newqps; i--) - teardown_vq(ndev, &ndev->vqs[i]); + for (i = ndev->cur_num_vqs - 1; i >= 2 * newqps; i--) { + struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[i]; + + if (is_resumable(ndev)) + suspend_vq(ndev, mvq); + else + teardown_vq(ndev, mvq); + } ndev->cur_num_vqs = 2 * newqps; } else { ndev->cur_num_vqs = 2 * newqps; for (i = cur_qps * 2; i < 2 * newqps; i++) { - err = setup_vq(ndev, &ndev->vqs[i]); + struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[i]; + + err = mvq->initialized ? resume_vq(ndev, mvq) : setup_vq(ndev, mvq, true); if (err) goto clean_added; } @@ -2285,6 +2393,7 @@ static void mlx5_vdpa_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num) } mvq = &ndev->vqs[idx]; + ndev->needs_teardown = num != mvq->num_ent; mvq->num_ent = num; } @@ -2324,7 +2433,6 @@ static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); struct mlx5_vdpa_virtqueue *mvq; - int err; if (!mvdev->actual_features) return; @@ -2340,15 +2448,11 @@ static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready mvq = &ndev->vqs[idx]; if (!ready) { suspend_vq(ndev, mvq); - } else { - err = modify_virtqueue_state(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY); - if (err) { - mlx5_vdpa_warn(mvdev, "modify VQ %d to ready failed (%d)\n", idx, err); + } else if (mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK) { + if (resume_vq(ndev, mvq)) ready = false; - } } - mvq->ready = ready; } @@ -2536,14 +2640,14 @@ static int verify_driver_features(struct mlx5_vdpa_dev *mvdev, u64 features) return 0; } -static int setup_virtqueues(struct mlx5_vdpa_dev *mvdev) +static int setup_virtqueues(struct mlx5_vdpa_dev *mvdev, bool filled) { struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); int err; int i; for (i = 0; i < mvdev->max_vqs; i++) { - err = setup_vq(ndev, &ndev->vqs[i]); + err = setup_vq(ndev, &ndev->vqs[i], filled); if (err) goto err_vq; } @@ -2559,16 +2663,10 @@ err_vq: static void teardown_virtqueues(struct mlx5_vdpa_net *ndev) { - struct mlx5_vdpa_virtqueue *mvq; int i; - for (i = ndev->mvdev.max_vqs - 1; i >= 0; i--) { - mvq = &ndev->vqs[i]; - if (!mvq->initialized) - continue; - - teardown_vq(ndev, mvq); - } + for (i = ndev->mvdev.max_vqs - 1; i >= 0; i--) + teardown_vq(ndev, &ndev->vqs[i]); } static void update_cvq_info(struct mlx5_vdpa_dev *mvdev) @@ -2705,6 +2803,8 @@ static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features) { struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); + u64 old_features = mvdev->actual_features; + u64 diff_features; int err; print_features(mvdev, features, true); @@ -2714,20 +2814,26 @@ static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features) return err; ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features; - if (ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_MQ)) - ndev->rqt_size = mlx5vdpa16_to_cpu(mvdev, ndev->config.max_virtqueue_pairs); - else - ndev->rqt_size = 1; - /* Device must start with 1 queue pair, as per VIRTIO v1.2 spec, section - * 5.1.6.5.5 "Device operation in multiqueue mode": - * - * Multiqueue is disabled by default. - * The driver enables multiqueue by sending a command using class - * VIRTIO_NET_CTRL_MQ. The command selects the mode of multiqueue - * operation, as follows: ... - */ - ndev->cur_num_vqs = 2; + /* Interested in changes of vq features only. */ + if (get_features(old_features) != get_features(mvdev->actual_features)) { + for (int i = 0; i < mvdev->max_vqs; ++i) { + struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[i]; + + mvq->modified_fields |= ( + MLX5_VIRTQ_MODIFY_MASK_QUEUE_VIRTIO_VERSION | + MLX5_VIRTQ_MODIFY_MASK_QUEUE_FEATURES + ); + } + } + + /* When below features diverge from initial device features, VQs need a full teardown. */ +#define NEEDS_TEARDOWN_MASK (BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) | \ + BIT_ULL(VIRTIO_NET_F_CSUM) | \ + BIT_ULL(VIRTIO_F_VERSION_1)) + + diff_features = mvdev->mlx_features ^ mvdev->actual_features; + ndev->needs_teardown = !!(diff_features & NEEDS_TEARDOWN_MASK); update_cvq_info(mvdev); return err; @@ -2816,7 +2922,7 @@ static void restore_channels_info(struct mlx5_vdpa_net *ndev) int i; mlx5_clear_vqs(ndev); - init_mvqs(ndev); + mvqs_set_defaults(ndev); for (i = 0; i < ndev->mvdev.max_vqs; i++) { mvq = &ndev->vqs[i]; ri = &mvq->ri; @@ -2848,12 +2954,12 @@ static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev, if (err) return err; - teardown_driver(ndev); + teardown_vq_resources(ndev); } mlx5_vdpa_update_mr(mvdev, new_mr, asid); - for (int i = 0; i < ndev->cur_num_vqs; i++) + for (int i = 0; i < mvdev->max_vqs; i++) ndev->vqs[i].modified_fields |= MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_MKEY | MLX5_VIRTQ_MODIFY_MASK_DESC_GROUP_MKEY; @@ -2862,7 +2968,7 @@ static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev, if (teardown) { restore_channels_info(ndev); - err = setup_driver(mvdev); + err = setup_vq_resources(ndev, true); if (err) return err; } @@ -2873,9 +2979,9 @@ static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev, } /* reslock must be held for this function */ -static int setup_driver(struct mlx5_vdpa_dev *mvdev) +static int setup_vq_resources(struct mlx5_vdpa_net *ndev, bool filled) { - struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); + struct mlx5_vdpa_dev *mvdev = &ndev->mvdev; int err; WARN_ON(!rwsem_is_locked(&ndev->reslock)); @@ -2891,7 +2997,7 @@ static int setup_driver(struct mlx5_vdpa_dev *mvdev) if (err) goto err_setup; - err = setup_virtqueues(mvdev); + err = setup_virtqueues(mvdev, filled); if (err) { mlx5_vdpa_warn(mvdev, "setup_virtqueues\n"); goto err_setup; @@ -2931,7 +3037,7 @@ out: } /* reslock must be held for this function */ -static void teardown_driver(struct mlx5_vdpa_net *ndev) +static void teardown_vq_resources(struct mlx5_vdpa_net *ndev) { WARN_ON(!rwsem_is_locked(&ndev->reslock)); @@ -2945,18 +3051,7 @@ static void teardown_driver(struct mlx5_vdpa_net *ndev) destroy_rqt(ndev); teardown_virtqueues(ndev); ndev->setup = false; -} - -static void clear_vqs_ready(struct mlx5_vdpa_net *ndev) -{ - int i; - - for (i = 0; i < ndev->mvdev.max_vqs; i++) { - ndev->vqs[i].ready = false; - ndev->vqs[i].modified_fields = 0; - } - - ndev->mvdev.cvq.ready = false; + ndev->needs_teardown = false; } static int setup_cvq_vring(struct mlx5_vdpa_dev *mvdev) @@ -2997,10 +3092,22 @@ static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status) goto err_setup; } register_link_notifier(ndev); - err = setup_driver(mvdev); - if (err) { - mlx5_vdpa_warn(mvdev, "failed to setup driver\n"); - goto err_driver; + + if (ndev->needs_teardown) + teardown_vq_resources(ndev); + + if (ndev->setup) { + err = resume_vqs(ndev); + if (err) { + mlx5_vdpa_warn(mvdev, "failed to resume VQs\n"); + goto err_driver; + } + } else { + err = setup_vq_resources(ndev, true); + if (err) { + mlx5_vdpa_warn(mvdev, "failed to setup driver\n"); + goto err_driver; + } } } else { mlx5_vdpa_warn(mvdev, "did not expect DRIVER_OK to be cleared\n"); @@ -3030,23 +3137,48 @@ static void init_group_to_asid_map(struct mlx5_vdpa_dev *mvdev) mvdev->group2asid[i] = 0; } +static bool needs_vqs_reset(const struct mlx5_vdpa_dev *mvdev) +{ + struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); + struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[0]; + + if (mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK) + return true; + + if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT) + return true; + + return mvq->modified_fields & ( + MLX5_VIRTQ_MODIFY_MASK_STATE | + MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_ADDRS | + MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_AVAIL_IDX | + MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_USED_IDX + ); +} + static int mlx5_vdpa_compat_reset(struct vdpa_device *vdev, u32 flags) { struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); + bool vq_reset; print_status(mvdev, 0, true); mlx5_vdpa_info(mvdev, "performing device reset\n"); down_write(&ndev->reslock); unregister_link_notifier(ndev); - teardown_driver(ndev); - clear_vqs_ready(ndev); + vq_reset = needs_vqs_reset(mvdev); + if (vq_reset) { + teardown_vq_resources(ndev); + mvqs_set_defaults(ndev); + } + if (flags & VDPA_RESET_F_CLEAN_MAP) mlx5_vdpa_destroy_mr_resources(&ndev->mvdev); ndev->mvdev.status = 0; ndev->mvdev.suspended = false; - ndev->cur_num_vqs = 0; + ndev->cur_num_vqs = MLX5V_DEFAULT_VQ_COUNT; + ndev->mvdev.cvq.ready = false; ndev->mvdev.cvq.received_desc = 0; ndev->mvdev.cvq.completed_desc = 0; memset(ndev->event_cbs, 0, sizeof(*ndev->event_cbs) * (mvdev->max_vqs + 1)); @@ -3059,6 +3191,8 @@ static int mlx5_vdpa_compat_reset(struct vdpa_device *vdev, u32 flags) if (mlx5_vdpa_create_dma_mr(mvdev)) mlx5_vdpa_warn(mvdev, "create MR failed\n"); } + if (vq_reset) + setup_vq_resources(ndev, false); up_write(&ndev->reslock); return 0; @@ -3197,7 +3331,7 @@ static void mlx5_vdpa_free(struct vdpa_device *vdev) ndev = to_mlx5_vdpa_ndev(mvdev); - free_resources(ndev); + free_fixed_resources(ndev); mlx5_vdpa_destroy_mr_resources(mvdev); if (!is_zero_ether_addr(ndev->config.mac)) { pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev)); @@ -3361,27 +3495,25 @@ static int mlx5_vdpa_suspend(struct vdpa_device *vdev) { struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); - struct mlx5_vdpa_virtqueue *mvq; - int i; + int err; mlx5_vdpa_info(mvdev, "suspending device\n"); down_write(&ndev->reslock); unregister_link_notifier(ndev); - for (i = 0; i < ndev->cur_num_vqs; i++) { - mvq = &ndev->vqs[i]; - suspend_vq(ndev, mvq); - } + err = suspend_vqs(ndev); mlx5_vdpa_cvq_suspend(mvdev); mvdev->suspended = true; up_write(&ndev->reslock); - return 0; + + return err; } static int mlx5_vdpa_resume(struct vdpa_device *vdev) { struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); struct mlx5_vdpa_net *ndev; + int err; ndev = to_mlx5_vdpa_ndev(mvdev); @@ -3389,10 +3521,11 @@ static int mlx5_vdpa_resume(struct vdpa_device *vdev) down_write(&ndev->reslock); mvdev->suspended = false; - resume_vqs(ndev); + err = resume_vqs(ndev); register_link_notifier(ndev); up_write(&ndev->reslock); - return 0; + + return err; } static int mlx5_set_group_asid(struct vdpa_device *vdev, u32 group, @@ -3467,7 +3600,7 @@ static int query_mtu(struct mlx5_core_dev *mdev, u16 *mtu) return 0; } -static int alloc_resources(struct mlx5_vdpa_net *ndev) +static int alloc_fixed_resources(struct mlx5_vdpa_net *ndev) { struct mlx5_vdpa_net_resources *res = &ndev->res; int err; @@ -3494,7 +3627,7 @@ err_tis: return err; } -static void free_resources(struct mlx5_vdpa_net *ndev) +static void free_fixed_resources(struct mlx5_vdpa_net *ndev) { struct mlx5_vdpa_net_resources *res = &ndev->res; @@ -3506,7 +3639,7 @@ static void free_resources(struct mlx5_vdpa_net *ndev) res->valid = false; } -static void init_mvqs(struct mlx5_vdpa_net *ndev) +static void mvqs_set_defaults(struct mlx5_vdpa_net *ndev) { struct mlx5_vdpa_virtqueue *mvq; int i; @@ -3518,12 +3651,7 @@ static void init_mvqs(struct mlx5_vdpa_net *ndev) mvq->ndev = ndev; mvq->fwqp.fw = true; mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_NONE; - } - for (; i < ndev->mvdev.max_vqs; i++) { - mvq = &ndev->vqs[i]; - memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); - mvq->index = i; - mvq->ndev = ndev; + mvq->num_ent = MLX5V_DEFAULT_VQ_SIZE; } } @@ -3660,8 +3788,9 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, err = -ENOMEM; goto err_alloc; } + ndev->cur_num_vqs = MLX5V_DEFAULT_VQ_COUNT; - init_mvqs(ndev); + mvqs_set_defaults(ndev); allocate_irqs(ndev); init_rwsem(&ndev->reslock); config = &ndev->config; @@ -3718,8 +3847,12 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, goto err_alloc; } - if (device_features & BIT_ULL(VIRTIO_NET_F_MQ)) + if (device_features & BIT_ULL(VIRTIO_NET_F_MQ)) { config->max_virtqueue_pairs = cpu_to_mlx5vdpa16(mvdev, max_vqs / 2); + ndev->rqt_size = max_vqs / 2; + } else { + ndev->rqt_size = 1; + } ndev->mvdev.mlx_features = device_features; mvdev->vdev.dma_dev = &mdev->pdev->dev; @@ -3735,7 +3868,7 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, goto err_res; } - err = alloc_resources(ndev); + err = alloc_fixed_resources(ndev); if (err) goto err_mr; @@ -3753,12 +3886,25 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, goto err_reg; mgtdev->ndev = ndev; + + /* For virtio-vdpa, the device was set up during device register. */ + if (ndev->setup) + return 0; + + down_write(&ndev->reslock); + err = setup_vq_resources(ndev, false); + up_write(&ndev->reslock); + if (err) + goto err_setup_vq_res; + return 0; +err_setup_vq_res: + _vdpa_unregister_device(&mvdev->vdev); err_reg: destroy_workqueue(mvdev->wq); err_res2: - free_resources(ndev); + free_fixed_resources(ndev); err_mr: mlx5_vdpa_destroy_mr_resources(mvdev); err_res: @@ -3780,6 +3926,11 @@ static void mlx5_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev, struct vdpa_device * unregister_link_notifier(ndev); _vdpa_unregister_device(dev); + + down_write(&ndev->reslock); + teardown_vq_resources(ndev); + up_write(&ndev->reslock); + wq = mvdev->wq; mvdev->wq = NULL; destroy_workqueue(wq); diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.h b/drivers/vdpa/mlx5/net/mlx5_vnet.h index 90b556a57971..00e79a7d0be8 100644 --- a/drivers/vdpa/mlx5/net/mlx5_vnet.h +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.h @@ -56,6 +56,7 @@ struct mlx5_vdpa_net { struct dentry *rx_dent; struct dentry *rx_table_dent; bool setup; + bool needs_teardown; u32 cur_num_vqs; u32 rqt_size; bool nb_registered; diff --git a/drivers/vdpa/octeon_ep/Makefile b/drivers/vdpa/octeon_ep/Makefile new file mode 100644 index 000000000000..e23e2ff14f33 --- /dev/null +++ b/drivers/vdpa/octeon_ep/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_OCTEONEP_VDPA) += octep_vdpa.o +octep_vdpa-$(CONFIG_OCTEONEP_VDPA) += octep_vdpa_main.o +octep_vdpa-$(CONFIG_OCTEONEP_VDPA) += octep_vdpa_hw.o diff --git a/drivers/vdpa/octeon_ep/octep_vdpa.h b/drivers/vdpa/octeon_ep/octep_vdpa.h new file mode 100644 index 000000000000..046710ec4d42 --- /dev/null +++ b/drivers/vdpa/octeon_ep/octep_vdpa.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2024 Marvell. + */ +#ifndef __OCTEP_VDPA_H__ +#define __OCTEP_VDPA_H__ + +#include <linux/pci.h> +#include <linux/pci_regs.h> +#include <linux/vdpa.h> +#include <linux/virtio_pci_modern.h> +#include <uapi/linux/virtio_net.h> +#include <uapi/linux/virtio_blk.h> +#include <uapi/linux/virtio_config.h> +#include <uapi/linux/virtio_pci.h> +#include <uapi/linux/vdpa.h> + +#define OCTEP_VDPA_DEVID_CN106K_PF 0xb900 +#define OCTEP_VDPA_DEVID_CN106K_VF 0xb903 +#define OCTEP_VDPA_DEVID_CN105K_PF 0xba00 +#define OCTEP_VDPA_DEVID_CN105K_VF 0xba03 +#define OCTEP_VDPA_DEVID_CN103K_PF 0xbd00 +#define OCTEP_VDPA_DEVID_CN103K_VF 0xbd03 + +#define OCTEP_HW_MBOX_BAR 0 +#define OCTEP_HW_CAPS_BAR 4 + +#define OCTEP_DEV_READY_SIGNATURE 0xBABABABA + +#define OCTEP_EPF_RINFO(x) (0x000209f0 | ((x) << 25)) +#define OCTEP_VF_MBOX_DATA(x) (0x00010210 | ((x) << 17)) +#define OCTEP_PF_MBOX_DATA(x) (0x00022000 | ((x) << 4)) + +#define OCTEP_EPF_RINFO_RPVF(val) (((val) >> 32) & 0xF) +#define OCTEP_EPF_RINFO_NVFS(val) (((val) >> 48) & 0x7F) + +#define OCTEP_FW_READY_SIGNATURE0 0xFEEDFEED +#define OCTEP_FW_READY_SIGNATURE1 0x3355ffaa + +enum octep_vdpa_dev_status { + OCTEP_VDPA_DEV_STATUS_INVALID, + OCTEP_VDPA_DEV_STATUS_ALLOC, + OCTEP_VDPA_DEV_STATUS_WAIT_FOR_BAR_INIT, + OCTEP_VDPA_DEV_STATUS_INIT, + OCTEP_VDPA_DEV_STATUS_READY, + OCTEP_VDPA_DEV_STATUS_UNINIT +}; + +struct octep_vring_info { + struct vdpa_callback cb; + void __iomem *notify_addr; + u32 __iomem *cb_notify_addr; + phys_addr_t notify_pa; + char msix_name[256]; +}; + +struct octep_hw { + struct pci_dev *pdev; + u8 __iomem *base[PCI_STD_NUM_BARS]; + struct virtio_pci_common_cfg __iomem *common_cfg; + u8 __iomem *dev_cfg; + u8 __iomem *isr; + void __iomem *notify_base; + phys_addr_t notify_base_pa; + u32 notify_off_multiplier; + u8 notify_bar; + struct octep_vring_info *vqs; + struct vdpa_callback config_cb; + u64 features; + u16 nr_vring; + u32 config_size; + int irq; +}; + +u8 octep_hw_get_status(struct octep_hw *oct_hw); +void octep_hw_set_status(struct octep_hw *dev, uint8_t status); +void octep_hw_reset(struct octep_hw *oct_hw); +void octep_write_queue_select(struct octep_hw *oct_hw, u16 queue_id); +void octep_notify_queue(struct octep_hw *oct_hw, u16 qid); +void octep_read_dev_config(struct octep_hw *oct_hw, u64 offset, void *dst, int length); +int octep_set_vq_address(struct octep_hw *oct_hw, u16 qid, u64 desc_area, u64 driver_area, + u64 device_area); +void octep_set_vq_num(struct octep_hw *oct_hw, u16 qid, u32 num); +void octep_set_vq_ready(struct octep_hw *oct_hw, u16 qid, bool ready); +bool octep_get_vq_ready(struct octep_hw *oct_hw, u16 qid); +int octep_set_vq_state(struct octep_hw *oct_hw, u16 qid, const struct vdpa_vq_state *state); +int octep_get_vq_state(struct octep_hw *oct_hw, u16 qid, struct vdpa_vq_state *state); +u16 octep_get_vq_size(struct octep_hw *oct_hw); +int octep_hw_caps_read(struct octep_hw *oct_hw, struct pci_dev *pdev); +u64 octep_hw_get_dev_features(struct octep_hw *oct_hw); +void octep_hw_set_drv_features(struct octep_hw *oct_hw, u64 features); +u64 octep_hw_get_drv_features(struct octep_hw *oct_hw); +int octep_verify_features(u64 features); + +#endif /* __OCTEP_VDPA_H__ */ diff --git a/drivers/vdpa/octeon_ep/octep_vdpa_hw.c b/drivers/vdpa/octeon_ep/octep_vdpa_hw.c new file mode 100644 index 000000000000..7fa0491bb201 --- /dev/null +++ b/drivers/vdpa/octeon_ep/octep_vdpa_hw.c @@ -0,0 +1,517 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2024 Marvell. */ + +#include <linux/iopoll.h> + +#include "octep_vdpa.h" + +enum octep_mbox_ids { + OCTEP_MBOX_MSG_SET_VQ_STATE = 1, + OCTEP_MBOX_MSG_GET_VQ_STATE, +}; + +#define OCTEP_HW_TIMEOUT 10000000 + +#define MBOX_OFFSET 64 +#define MBOX_RSP_MASK 0x00000001 +#define MBOX_RC_MASK 0x0000FFFE + +#define MBOX_RSP_TO_ERR(val) (-(((val) & MBOX_RC_MASK) >> 2)) +#define MBOX_AVAIL(val) (((val) & MBOX_RSP_MASK)) +#define MBOX_RSP(val) ((val) & (MBOX_RC_MASK | MBOX_RSP_MASK)) + +#define DEV_RST_ACK_BIT 7 +#define FEATURE_SEL_ACK_BIT 15 +#define QUEUE_SEL_ACK_BIT 15 + +struct octep_mbox_hdr { + u8 ver; + u8 rsvd1; + u16 id; + u16 rsvd2; +#define MBOX_REQ_SIG (0xdead) +#define MBOX_RSP_SIG (0xbeef) + u16 sig; +}; + +struct octep_mbox_sts { + u16 rsp:1; + u16 rc:15; + u16 rsvd; +}; + +struct octep_mbox { + struct octep_mbox_hdr hdr; + struct octep_mbox_sts sts; + u64 rsvd; + u32 data[]; +}; + +static inline struct octep_mbox __iomem *octep_get_mbox(struct octep_hw *oct_hw) +{ + return (struct octep_mbox __iomem *)(oct_hw->dev_cfg + MBOX_OFFSET); +} + +static inline int octep_wait_for_mbox_avail(struct octep_mbox __iomem *mbox) +{ + u32 val; + + return readx_poll_timeout(ioread32, &mbox->sts, val, MBOX_AVAIL(val), 10, + OCTEP_HW_TIMEOUT); +} + +static inline int octep_wait_for_mbox_rsp(struct octep_mbox __iomem *mbox) +{ + u32 val; + + return readx_poll_timeout(ioread32, &mbox->sts, val, MBOX_RSP(val), 10, + OCTEP_HW_TIMEOUT); +} + +static inline void octep_write_hdr(struct octep_mbox __iomem *mbox, u16 id, u16 sig) +{ + iowrite16(id, &mbox->hdr.id); + iowrite16(sig, &mbox->hdr.sig); +} + +static inline u32 octep_read_sig(struct octep_mbox __iomem *mbox) +{ + return ioread16(&mbox->hdr.sig); +} + +static inline void octep_write_sts(struct octep_mbox __iomem *mbox, u32 sts) +{ + iowrite32(sts, &mbox->sts); +} + +static inline u32 octep_read_sts(struct octep_mbox __iomem *mbox) +{ + return ioread32(&mbox->sts); +} + +static inline u32 octep_read32_word(struct octep_mbox __iomem *mbox, u16 word_idx) +{ + return ioread32(&mbox->data[word_idx]); +} + +static inline void octep_write32_word(struct octep_mbox __iomem *mbox, u16 word_idx, u32 word) +{ + return iowrite32(word, &mbox->data[word_idx]); +} + +static int octep_process_mbox(struct octep_hw *oct_hw, u16 id, u16 qid, void *buffer, + u32 buf_size, bool write) +{ + struct octep_mbox __iomem *mbox = octep_get_mbox(oct_hw); + struct pci_dev *pdev = oct_hw->pdev; + u32 *p = (u32 *)buffer; + u16 data_wds; + int ret, i; + u32 val; + + if (!IS_ALIGNED(buf_size, 4)) + return -EINVAL; + + /* Make sure mbox space is available */ + ret = octep_wait_for_mbox_avail(mbox); + if (ret) { + dev_warn(&pdev->dev, "Timeout waiting for previous mbox data to be consumed\n"); + return ret; + } + data_wds = buf_size / 4; + + if (write) { + for (i = 1; i <= data_wds; i++) { + octep_write32_word(mbox, i, *p); + p++; + } + } + octep_write32_word(mbox, 0, (u32)qid); + octep_write_sts(mbox, 0); + + octep_write_hdr(mbox, id, MBOX_REQ_SIG); + + ret = octep_wait_for_mbox_rsp(mbox); + if (ret) { + dev_warn(&pdev->dev, "Timeout waiting for mbox : %d response\n", id); + return ret; + } + + val = octep_read_sig(mbox); + if ((val & 0xFFFF) != MBOX_RSP_SIG) { + dev_warn(&pdev->dev, "Invalid Signature from mbox : %d response\n", id); + return ret; + } + + val = octep_read_sts(mbox); + if (val & MBOX_RC_MASK) { + ret = MBOX_RSP_TO_ERR(val); + dev_warn(&pdev->dev, "Error while processing mbox : %d, err %d\n", id, ret); + return ret; + } + + if (!write) + for (i = 1; i <= data_wds; i++) + *p++ = octep_read32_word(mbox, i); + + return 0; +} + +static void octep_mbox_init(struct octep_mbox __iomem *mbox) +{ + iowrite32(1, &mbox->sts); +} + +int octep_verify_features(u64 features) +{ + /* Minimum features to expect */ + if (!(features & BIT_ULL(VIRTIO_F_VERSION_1))) + return -EOPNOTSUPP; + + if (!(features & BIT_ULL(VIRTIO_F_NOTIFICATION_DATA))) + return -EOPNOTSUPP; + + if (!(features & BIT_ULL(VIRTIO_F_RING_PACKED))) + return -EOPNOTSUPP; + + return 0; +} + +u8 octep_hw_get_status(struct octep_hw *oct_hw) +{ + return ioread8(&oct_hw->common_cfg->device_status); +} + +void octep_hw_set_status(struct octep_hw *oct_hw, u8 status) +{ + iowrite8(status, &oct_hw->common_cfg->device_status); +} + +void octep_hw_reset(struct octep_hw *oct_hw) +{ + u8 val; + + octep_hw_set_status(oct_hw, 0 | BIT(DEV_RST_ACK_BIT)); + if (readx_poll_timeout(ioread8, &oct_hw->common_cfg->device_status, val, !val, 10, + OCTEP_HW_TIMEOUT)) { + dev_warn(&oct_hw->pdev->dev, "Octeon device reset timeout\n"); + return; + } +} + +static int feature_sel_write_with_timeout(struct octep_hw *oct_hw, u32 select, void __iomem *addr) +{ + u32 val; + + iowrite32(select | BIT(FEATURE_SEL_ACK_BIT), addr); + + if (readx_poll_timeout(ioread32, addr, val, val == select, 10, OCTEP_HW_TIMEOUT)) { + dev_warn(&oct_hw->pdev->dev, "Feature select%d write timeout\n", select); + return -1; + } + return 0; +} + +u64 octep_hw_get_dev_features(struct octep_hw *oct_hw) +{ + u32 features_lo, features_hi; + + if (feature_sel_write_with_timeout(oct_hw, 0, &oct_hw->common_cfg->device_feature_select)) + return 0; + + features_lo = ioread32(&oct_hw->common_cfg->device_feature); + + if (feature_sel_write_with_timeout(oct_hw, 1, &oct_hw->common_cfg->device_feature_select)) + return 0; + + features_hi = ioread32(&oct_hw->common_cfg->device_feature); + + return ((u64)features_hi << 32) | features_lo; +} + +u64 octep_hw_get_drv_features(struct octep_hw *oct_hw) +{ + u32 features_lo, features_hi; + + if (feature_sel_write_with_timeout(oct_hw, 0, &oct_hw->common_cfg->guest_feature_select)) + return 0; + + features_lo = ioread32(&oct_hw->common_cfg->guest_feature); + + if (feature_sel_write_with_timeout(oct_hw, 1, &oct_hw->common_cfg->guest_feature_select)) + return 0; + + features_hi = ioread32(&oct_hw->common_cfg->guest_feature); + + return ((u64)features_hi << 32) | features_lo; +} + +void octep_hw_set_drv_features(struct octep_hw *oct_hw, u64 features) +{ + if (feature_sel_write_with_timeout(oct_hw, 0, &oct_hw->common_cfg->guest_feature_select)) + return; + + iowrite32(features & (BIT_ULL(32) - 1), &oct_hw->common_cfg->guest_feature); + + if (feature_sel_write_with_timeout(oct_hw, 1, &oct_hw->common_cfg->guest_feature_select)) + return; + + iowrite32(features >> 32, &oct_hw->common_cfg->guest_feature); +} + +void octep_write_queue_select(struct octep_hw *oct_hw, u16 queue_id) +{ + u16 val; + + iowrite16(queue_id | BIT(QUEUE_SEL_ACK_BIT), &oct_hw->common_cfg->queue_select); + + if (readx_poll_timeout(ioread16, &oct_hw->common_cfg->queue_select, val, val == queue_id, + 10, OCTEP_HW_TIMEOUT)) { + dev_warn(&oct_hw->pdev->dev, "Queue select write timeout\n"); + return; + } +} + +void octep_notify_queue(struct octep_hw *oct_hw, u16 qid) +{ + iowrite16(qid, oct_hw->vqs[qid].notify_addr); +} + +void octep_read_dev_config(struct octep_hw *oct_hw, u64 offset, void *dst, int length) +{ + u8 old_gen, new_gen, *p; + int i; + + if (WARN_ON(offset + length > oct_hw->config_size)) + return; + + do { + old_gen = ioread8(&oct_hw->common_cfg->config_generation); + p = dst; + for (i = 0; i < length; i++) + *p++ = ioread8(oct_hw->dev_cfg + offset + i); + + new_gen = ioread8(&oct_hw->common_cfg->config_generation); + } while (old_gen != new_gen); +} + +int octep_set_vq_address(struct octep_hw *oct_hw, u16 qid, u64 desc_area, u64 driver_area, + u64 device_area) +{ + struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg; + + octep_write_queue_select(oct_hw, qid); + vp_iowrite64_twopart(desc_area, &cfg->queue_desc_lo, + &cfg->queue_desc_hi); + vp_iowrite64_twopart(driver_area, &cfg->queue_avail_lo, + &cfg->queue_avail_hi); + vp_iowrite64_twopart(device_area, &cfg->queue_used_lo, + &cfg->queue_used_hi); + + return 0; +} + +int octep_get_vq_state(struct octep_hw *oct_hw, u16 qid, struct vdpa_vq_state *state) +{ + return octep_process_mbox(oct_hw, OCTEP_MBOX_MSG_GET_VQ_STATE, qid, state, + sizeof(*state), 0); +} + +int octep_set_vq_state(struct octep_hw *oct_hw, u16 qid, const struct vdpa_vq_state *state) +{ + struct vdpa_vq_state q_state; + + memcpy(&q_state, state, sizeof(struct vdpa_vq_state)); + return octep_process_mbox(oct_hw, OCTEP_MBOX_MSG_SET_VQ_STATE, qid, &q_state, + sizeof(*state), 1); +} + +void octep_set_vq_num(struct octep_hw *oct_hw, u16 qid, u32 num) +{ + struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg; + + octep_write_queue_select(oct_hw, qid); + iowrite16(num, &cfg->queue_size); +} + +void octep_set_vq_ready(struct octep_hw *oct_hw, u16 qid, bool ready) +{ + struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg; + + octep_write_queue_select(oct_hw, qid); + iowrite16(ready, &cfg->queue_enable); +} + +bool octep_get_vq_ready(struct octep_hw *oct_hw, u16 qid) +{ + struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg; + + octep_write_queue_select(oct_hw, qid); + return ioread16(&cfg->queue_enable); +} + +u16 octep_get_vq_size(struct octep_hw *oct_hw) +{ + octep_write_queue_select(oct_hw, 0); + return ioread16(&oct_hw->common_cfg->queue_size); +} + +static u32 octep_get_config_size(struct octep_hw *oct_hw) +{ + return sizeof(struct virtio_net_config); +} + +static void __iomem *octep_get_cap_addr(struct octep_hw *oct_hw, struct virtio_pci_cap *cap) +{ + struct device *dev = &oct_hw->pdev->dev; + u32 length = le32_to_cpu(cap->length); + u32 offset = le32_to_cpu(cap->offset); + u8 bar = cap->bar; + u32 len; + + if (bar != OCTEP_HW_CAPS_BAR) { + dev_err(dev, "Invalid bar: %u\n", bar); + return NULL; + } + if (offset + length < offset) { + dev_err(dev, "offset(%u) + length(%u) overflows\n", + offset, length); + return NULL; + } + len = pci_resource_len(oct_hw->pdev, bar); + if (offset + length > len) { + dev_err(dev, "invalid cap: overflows bar space: %u > %u\n", + offset + length, len); + return NULL; + } + return oct_hw->base[bar] + offset; +} + +/* In Octeon DPU device, the virtio config space is completely + * emulated by the device's firmware. So, the standard pci config + * read apis can't be used for reading the virtio capability. + */ +static void octep_pci_caps_read(struct octep_hw *oct_hw, void *buf, size_t len, off_t offset) +{ + u8 __iomem *bar = oct_hw->base[OCTEP_HW_CAPS_BAR]; + u8 *p = buf; + size_t i; + + for (i = 0; i < len; i++) + *p++ = ioread8(bar + offset + i); +} + +static int octep_pci_signature_verify(struct octep_hw *oct_hw) +{ + u32 signature[2]; + + octep_pci_caps_read(oct_hw, &signature, sizeof(signature), 0); + + if (signature[0] != OCTEP_FW_READY_SIGNATURE0) + return -1; + + if (signature[1] != OCTEP_FW_READY_SIGNATURE1) + return -1; + + return 0; +} + +int octep_hw_caps_read(struct octep_hw *oct_hw, struct pci_dev *pdev) +{ + struct octep_mbox __iomem *mbox; + struct device *dev = &pdev->dev; + struct virtio_pci_cap cap; + u16 notify_off; + int i, ret; + u8 pos; + + oct_hw->pdev = pdev; + ret = octep_pci_signature_verify(oct_hw); + if (ret) { + dev_err(dev, "Octeon Virtio FW is not initialized\n"); + return -EIO; + } + + octep_pci_caps_read(oct_hw, &pos, 1, PCI_CAPABILITY_LIST); + + while (pos) { + octep_pci_caps_read(oct_hw, &cap, 2, pos); + + if (cap.cap_vndr != PCI_CAP_ID_VNDR) { + dev_err(dev, "Found invalid capability vndr id: %d\n", cap.cap_vndr); + break; + } + + octep_pci_caps_read(oct_hw, &cap, sizeof(cap), pos); + + dev_info(dev, "[%2x] cfg type: %u, bar: %u, offset: %04x, len: %u\n", + pos, cap.cfg_type, cap.bar, cap.offset, cap.length); + + switch (cap.cfg_type) { + case VIRTIO_PCI_CAP_COMMON_CFG: + oct_hw->common_cfg = octep_get_cap_addr(oct_hw, &cap); + break; + case VIRTIO_PCI_CAP_NOTIFY_CFG: + octep_pci_caps_read(oct_hw, &oct_hw->notify_off_multiplier, + 4, pos + sizeof(cap)); + + oct_hw->notify_base = octep_get_cap_addr(oct_hw, &cap); + oct_hw->notify_bar = cap.bar; + oct_hw->notify_base_pa = pci_resource_start(pdev, cap.bar) + + le32_to_cpu(cap.offset); + break; + case VIRTIO_PCI_CAP_DEVICE_CFG: + oct_hw->dev_cfg = octep_get_cap_addr(oct_hw, &cap); + break; + case VIRTIO_PCI_CAP_ISR_CFG: + oct_hw->isr = octep_get_cap_addr(oct_hw, &cap); + break; + } + + pos = cap.cap_next; + } + if (!oct_hw->common_cfg || !oct_hw->notify_base || + !oct_hw->dev_cfg || !oct_hw->isr) { + dev_err(dev, "Incomplete PCI capabilities"); + return -EIO; + } + dev_info(dev, "common cfg mapped at: 0x%016llx\n", (u64)(uintptr_t)oct_hw->common_cfg); + dev_info(dev, "device cfg mapped at: 0x%016llx\n", (u64)(uintptr_t)oct_hw->dev_cfg); + dev_info(dev, "isr cfg mapped at: 0x%016llx\n", (u64)(uintptr_t)oct_hw->isr); + dev_info(dev, "notify base: 0x%016llx, notify off multiplier: %u\n", + (u64)(uintptr_t)oct_hw->notify_base, oct_hw->notify_off_multiplier); + + oct_hw->config_size = octep_get_config_size(oct_hw); + oct_hw->features = octep_hw_get_dev_features(oct_hw); + + ret = octep_verify_features(oct_hw->features); + if (ret) { + dev_err(&pdev->dev, "Couldn't read features from the device FW\n"); + return ret; + } + oct_hw->nr_vring = vp_ioread16(&oct_hw->common_cfg->num_queues); + + oct_hw->vqs = devm_kcalloc(&pdev->dev, oct_hw->nr_vring, sizeof(*oct_hw->vqs), GFP_KERNEL); + if (!oct_hw->vqs) + return -ENOMEM; + + oct_hw->irq = -1; + + dev_info(&pdev->dev, "Device features : %llx\n", oct_hw->features); + dev_info(&pdev->dev, "Maximum queues : %u\n", oct_hw->nr_vring); + + for (i = 0; i < oct_hw->nr_vring; i++) { + octep_write_queue_select(oct_hw, i); + notify_off = vp_ioread16(&oct_hw->common_cfg->queue_notify_off); + oct_hw->vqs[i].notify_addr = oct_hw->notify_base + + notify_off * oct_hw->notify_off_multiplier; + oct_hw->vqs[i].cb_notify_addr = (u32 __iomem *)oct_hw->vqs[i].notify_addr + 1; + oct_hw->vqs[i].notify_pa = oct_hw->notify_base_pa + + notify_off * oct_hw->notify_off_multiplier; + } + mbox = octep_get_mbox(oct_hw); + octep_mbox_init(mbox); + dev_info(dev, "mbox mapped at: 0x%016llx\n", (u64)(uintptr_t)mbox); + + return 0; +} diff --git a/drivers/vdpa/octeon_ep/octep_vdpa_main.c b/drivers/vdpa/octeon_ep/octep_vdpa_main.c new file mode 100644 index 000000000000..cd55b1aac151 --- /dev/null +++ b/drivers/vdpa/octeon_ep/octep_vdpa_main.c @@ -0,0 +1,857 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2024 Marvell. */ + +#include <linux/interrupt.h> +#include <linux/io-64-nonatomic-lo-hi.h> +#include <linux/module.h> +#include <linux/iommu.h> +#include "octep_vdpa.h" + +#define OCTEP_VDPA_DRIVER_NAME "octep_vdpa" + +struct octep_pf { + u8 __iomem *base[PCI_STD_NUM_BARS]; + struct pci_dev *pdev; + struct resource res; + u64 vf_base; + int enabled_vfs; + u32 vf_stride; + u16 vf_devid; +}; + +struct octep_vdpa { + struct vdpa_device vdpa; + struct octep_hw *oct_hw; + struct pci_dev *pdev; +}; + +struct octep_vdpa_mgmt_dev { + struct vdpa_mgmt_dev mdev; + struct octep_hw oct_hw; + struct pci_dev *pdev; + /* Work entry to handle device setup */ + struct work_struct setup_task; + /* Device status */ + atomic_t status; +}; + +static struct octep_hw *vdpa_to_octep_hw(struct vdpa_device *vdpa_dev) +{ + struct octep_vdpa *oct_vdpa; + + oct_vdpa = container_of(vdpa_dev, struct octep_vdpa, vdpa); + + return oct_vdpa->oct_hw; +} + +static irqreturn_t octep_vdpa_intr_handler(int irq, void *data) +{ + struct octep_hw *oct_hw = data; + int i; + + for (i = 0; i < oct_hw->nr_vring; i++) { + if (oct_hw->vqs[i].cb.callback && ioread32(oct_hw->vqs[i].cb_notify_addr)) { + /* Acknowledge the per queue notification to the device */ + iowrite32(0, oct_hw->vqs[i].cb_notify_addr); + oct_hw->vqs[i].cb.callback(oct_hw->vqs[i].cb.private); + } + } + + return IRQ_HANDLED; +} + +static void octep_free_irqs(struct octep_hw *oct_hw) +{ + struct pci_dev *pdev = oct_hw->pdev; + + if (oct_hw->irq != -1) { + devm_free_irq(&pdev->dev, oct_hw->irq, oct_hw); + oct_hw->irq = -1; + } + pci_free_irq_vectors(pdev); +} + +static int octep_request_irqs(struct octep_hw *oct_hw) +{ + struct pci_dev *pdev = oct_hw->pdev; + int ret, irq; + + /* Currently HW device provisions one IRQ per VF, hence + * allocate one IRQ for all virtqueues call interface. + */ + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSIX); + if (ret < 0) { + dev_err(&pdev->dev, "Failed to alloc msix vector"); + return ret; + } + + snprintf(oct_hw->vqs->msix_name, sizeof(oct_hw->vqs->msix_name), + OCTEP_VDPA_DRIVER_NAME "-vf-%d", pci_iov_vf_id(pdev)); + + irq = pci_irq_vector(pdev, 0); + ret = devm_request_irq(&pdev->dev, irq, octep_vdpa_intr_handler, 0, + oct_hw->vqs->msix_name, oct_hw); + if (ret) { + dev_err(&pdev->dev, "Failed to register interrupt handler\n"); + goto free_irq_vec; + } + oct_hw->irq = irq; + + return 0; + +free_irq_vec: + pci_free_irq_vectors(pdev); + return ret; +} + +static u64 octep_vdpa_get_device_features(struct vdpa_device *vdpa_dev) +{ + struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); + + return oct_hw->features; +} + +static int octep_vdpa_set_driver_features(struct vdpa_device *vdpa_dev, u64 features) +{ + struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); + int ret; + + pr_debug("Driver Features: %llx\n", features); + + ret = octep_verify_features(features); + if (ret) { + dev_warn(&oct_hw->pdev->dev, + "Must negotiate minimum features 0x%llx for this device", + BIT_ULL(VIRTIO_F_VERSION_1) | BIT_ULL(VIRTIO_F_NOTIFICATION_DATA) | + BIT_ULL(VIRTIO_F_RING_PACKED)); + return ret; + } + octep_hw_set_drv_features(oct_hw, features); + + return 0; +} + +static u64 octep_vdpa_get_driver_features(struct vdpa_device *vdpa_dev) +{ + struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); + + return octep_hw_get_drv_features(oct_hw); +} + +static u8 octep_vdpa_get_status(struct vdpa_device *vdpa_dev) +{ + struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); + + return octep_hw_get_status(oct_hw); +} + +static void octep_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status) +{ + struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); + u8 status_old; + + status_old = octep_hw_get_status(oct_hw); + + if (status_old == status) + return; + + if ((status & VIRTIO_CONFIG_S_DRIVER_OK) && + !(status_old & VIRTIO_CONFIG_S_DRIVER_OK)) { + if (octep_request_irqs(oct_hw)) + status = status_old | VIRTIO_CONFIG_S_FAILED; + } + octep_hw_set_status(oct_hw, status); +} + +static int octep_vdpa_reset(struct vdpa_device *vdpa_dev) +{ + struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); + u8 status = octep_hw_get_status(oct_hw); + u16 qid; + + if (status == 0) + return 0; + + for (qid = 0; qid < oct_hw->nr_vring; qid++) { + oct_hw->vqs[qid].cb.callback = NULL; + oct_hw->vqs[qid].cb.private = NULL; + oct_hw->config_cb.callback = NULL; + oct_hw->config_cb.private = NULL; + } + octep_hw_reset(oct_hw); + + if (status & VIRTIO_CONFIG_S_DRIVER_OK) + octep_free_irqs(oct_hw); + + return 0; +} + +static u16 octep_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev) +{ + struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); + + return octep_get_vq_size(oct_hw); +} + +static int octep_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid, + struct vdpa_vq_state *state) +{ + struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); + + return octep_get_vq_state(oct_hw, qid, state); +} + +static int octep_vdpa_set_vq_state(struct vdpa_device *vdpa_dev, u16 qid, + const struct vdpa_vq_state *state) +{ + struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); + + return octep_set_vq_state(oct_hw, qid, state); +} + +static void octep_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid, struct vdpa_callback *cb) +{ + struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); + + oct_hw->vqs[qid].cb = *cb; +} + +static void octep_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev, u16 qid, bool ready) +{ + struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); + + octep_set_vq_ready(oct_hw, qid, ready); +} + +static bool octep_vdpa_get_vq_ready(struct vdpa_device *vdpa_dev, u16 qid) +{ + struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); + + return octep_get_vq_ready(oct_hw, qid); +} + +static void octep_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid, u32 num) +{ + struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); + + octep_set_vq_num(oct_hw, qid, num); +} + +static int octep_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid, u64 desc_area, + u64 driver_area, u64 device_area) +{ + struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); + + pr_debug("qid[%d]: desc_area: %llx\n", qid, desc_area); + pr_debug("qid[%d]: driver_area: %llx\n", qid, driver_area); + pr_debug("qid[%d]: device_area: %llx\n\n", qid, device_area); + + return octep_set_vq_address(oct_hw, qid, desc_area, driver_area, device_area); +} + +static void octep_vdpa_kick_vq(struct vdpa_device *vdpa_dev, u16 qid) +{ + /* Not supported */ +} + +static void octep_vdpa_kick_vq_with_data(struct vdpa_device *vdpa_dev, u32 data) +{ + struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); + u16 idx = data & 0xFFFF; + + vp_iowrite32(data, oct_hw->vqs[idx].notify_addr); +} + +static u32 octep_vdpa_get_generation(struct vdpa_device *vdpa_dev) +{ + struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); + + return vp_ioread8(&oct_hw->common_cfg->config_generation); +} + +static u32 octep_vdpa_get_device_id(struct vdpa_device *vdpa_dev) +{ + return VIRTIO_ID_NET; +} + +static u32 octep_vdpa_get_vendor_id(struct vdpa_device *vdpa_dev) +{ + return PCI_VENDOR_ID_CAVIUM; +} + +static u32 octep_vdpa_get_vq_align(struct vdpa_device *vdpa_dev) +{ + return PAGE_SIZE; +} + +static size_t octep_vdpa_get_config_size(struct vdpa_device *vdpa_dev) +{ + struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); + + return oct_hw->config_size; +} + +static void octep_vdpa_get_config(struct vdpa_device *vdpa_dev, unsigned int offset, void *buf, + unsigned int len) +{ + struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); + + octep_read_dev_config(oct_hw, offset, buf, len); +} + +static void octep_vdpa_set_config(struct vdpa_device *vdpa_dev, unsigned int offset, + const void *buf, unsigned int len) +{ + /* Not supported */ +} + +static void octep_vdpa_set_config_cb(struct vdpa_device *vdpa_dev, struct vdpa_callback *cb) +{ + struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); + + oct_hw->config_cb.callback = cb->callback; + oct_hw->config_cb.private = cb->private; +} + +static struct vdpa_notification_area octep_get_vq_notification(struct vdpa_device *vdpa_dev, + u16 idx) +{ + struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); + struct vdpa_notification_area area; + + area.addr = oct_hw->vqs[idx].notify_pa; + area.size = PAGE_SIZE; + + return area; +} + +static struct vdpa_config_ops octep_vdpa_ops = { + .get_device_features = octep_vdpa_get_device_features, + .set_driver_features = octep_vdpa_set_driver_features, + .get_driver_features = octep_vdpa_get_driver_features, + .get_status = octep_vdpa_get_status, + .set_status = octep_vdpa_set_status, + .reset = octep_vdpa_reset, + .get_vq_num_max = octep_vdpa_get_vq_num_max, + .get_vq_state = octep_vdpa_get_vq_state, + .set_vq_state = octep_vdpa_set_vq_state, + .set_vq_cb = octep_vdpa_set_vq_cb, + .set_vq_ready = octep_vdpa_set_vq_ready, + .get_vq_ready = octep_vdpa_get_vq_ready, + .set_vq_num = octep_vdpa_set_vq_num, + .set_vq_address = octep_vdpa_set_vq_address, + .get_vq_irq = NULL, + .kick_vq = octep_vdpa_kick_vq, + .kick_vq_with_data = octep_vdpa_kick_vq_with_data, + .get_generation = octep_vdpa_get_generation, + .get_device_id = octep_vdpa_get_device_id, + .get_vendor_id = octep_vdpa_get_vendor_id, + .get_vq_align = octep_vdpa_get_vq_align, + .get_config_size = octep_vdpa_get_config_size, + .get_config = octep_vdpa_get_config, + .set_config = octep_vdpa_set_config, + .set_config_cb = octep_vdpa_set_config_cb, + .get_vq_notification = octep_get_vq_notification, +}; + +static int octep_iomap_region(struct pci_dev *pdev, u8 __iomem **tbl, u8 bar) +{ + int ret; + + ret = pci_request_region(pdev, bar, OCTEP_VDPA_DRIVER_NAME); + if (ret) { + dev_err(&pdev->dev, "Failed to request BAR:%u region\n", bar); + return ret; + } + + tbl[bar] = pci_iomap(pdev, bar, pci_resource_len(pdev, bar)); + if (!tbl[bar]) { + dev_err(&pdev->dev, "Failed to iomap BAR:%u\n", bar); + pci_release_region(pdev, bar); + ret = -ENOMEM; + } + + return ret; +} + +static void octep_iounmap_region(struct pci_dev *pdev, u8 __iomem **tbl, u8 bar) +{ + pci_iounmap(pdev, tbl[bar]); + pci_release_region(pdev, bar); +} + +static void octep_vdpa_pf_bar_shrink(struct octep_pf *octpf) +{ + struct pci_dev *pf_dev = octpf->pdev; + struct resource *res = pf_dev->resource + PCI_STD_RESOURCES + 4; + struct pci_bus_region bus_region; + + octpf->res.start = res->start; + octpf->res.end = res->end; + octpf->vf_base = res->start; + + bus_region.start = res->start; + bus_region.end = res->start - 1; + + pcibios_bus_to_resource(pf_dev->bus, res, &bus_region); +} + +static void octep_vdpa_pf_bar_expand(struct octep_pf *octpf) +{ + struct pci_dev *pf_dev = octpf->pdev; + struct resource *res = pf_dev->resource + PCI_STD_RESOURCES + 4; + struct pci_bus_region bus_region; + + bus_region.start = octpf->res.start; + bus_region.end = octpf->res.end; + + pcibios_bus_to_resource(pf_dev->bus, res, &bus_region); +} + +static void octep_vdpa_remove_pf(struct pci_dev *pdev) +{ + struct octep_pf *octpf = pci_get_drvdata(pdev); + + pci_disable_sriov(pdev); + + if (octpf->base[OCTEP_HW_CAPS_BAR]) + octep_iounmap_region(pdev, octpf->base, OCTEP_HW_CAPS_BAR); + + if (octpf->base[OCTEP_HW_MBOX_BAR]) + octep_iounmap_region(pdev, octpf->base, OCTEP_HW_MBOX_BAR); + + octep_vdpa_pf_bar_expand(octpf); +} + +static void octep_vdpa_vf_bar_shrink(struct pci_dev *pdev) +{ + struct resource *vf_res = pdev->resource + PCI_STD_RESOURCES + 4; + + memset(vf_res, 0, sizeof(*vf_res)); +} + +static void octep_vdpa_remove_vf(struct pci_dev *pdev) +{ + struct octep_vdpa_mgmt_dev *mgmt_dev = pci_get_drvdata(pdev); + struct octep_hw *oct_hw; + int status; + + oct_hw = &mgmt_dev->oct_hw; + status = atomic_read(&mgmt_dev->status); + atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_UNINIT); + + cancel_work_sync(&mgmt_dev->setup_task); + if (status == OCTEP_VDPA_DEV_STATUS_READY) + vdpa_mgmtdev_unregister(&mgmt_dev->mdev); + + if (oct_hw->base[OCTEP_HW_CAPS_BAR]) + octep_iounmap_region(pdev, oct_hw->base, OCTEP_HW_CAPS_BAR); + + if (oct_hw->base[OCTEP_HW_MBOX_BAR]) + octep_iounmap_region(pdev, oct_hw->base, OCTEP_HW_MBOX_BAR); + + octep_vdpa_vf_bar_shrink(pdev); +} + +static void octep_vdpa_remove(struct pci_dev *pdev) +{ + if (pdev->is_virtfn) + octep_vdpa_remove_vf(pdev); + else + octep_vdpa_remove_pf(pdev); +} + +static int octep_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, + const struct vdpa_dev_set_config *config) +{ + struct octep_vdpa_mgmt_dev *mgmt_dev = container_of(mdev, struct octep_vdpa_mgmt_dev, mdev); + struct octep_hw *oct_hw = &mgmt_dev->oct_hw; + struct pci_dev *pdev = oct_hw->pdev; + struct vdpa_device *vdpa_dev; + struct octep_vdpa *oct_vdpa; + u64 device_features; + int ret; + + oct_vdpa = vdpa_alloc_device(struct octep_vdpa, vdpa, &pdev->dev, &octep_vdpa_ops, 1, 1, + NULL, false); + if (IS_ERR(oct_vdpa)) { + dev_err(&pdev->dev, "Failed to allocate vDPA structure for octep vdpa device"); + return PTR_ERR(oct_vdpa); + } + + oct_vdpa->pdev = pdev; + oct_vdpa->vdpa.dma_dev = &pdev->dev; + oct_vdpa->vdpa.mdev = mdev; + oct_vdpa->oct_hw = oct_hw; + vdpa_dev = &oct_vdpa->vdpa; + + device_features = oct_hw->features; + if (config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) { + if (config->device_features & ~device_features) { + dev_err(&pdev->dev, "The provisioned features 0x%llx are not supported by this device with features 0x%llx\n", + config->device_features, device_features); + ret = -EINVAL; + goto vdpa_dev_put; + } + device_features &= config->device_features; + } + + oct_hw->features = device_features; + dev_info(&pdev->dev, "Vdpa management device features : %llx\n", device_features); + + ret = octep_verify_features(device_features); + if (ret) { + dev_warn(mdev->device, + "Must provision minimum features 0x%llx for this device", + BIT_ULL(VIRTIO_F_VERSION_1) | BIT_ULL(VIRTIO_F_ACCESS_PLATFORM) | + BIT_ULL(VIRTIO_F_NOTIFICATION_DATA) | BIT_ULL(VIRTIO_F_RING_PACKED)); + goto vdpa_dev_put; + } + if (name) + ret = dev_set_name(&vdpa_dev->dev, "%s", name); + else + ret = dev_set_name(&vdpa_dev->dev, "vdpa%u", vdpa_dev->index); + + ret = _vdpa_register_device(&oct_vdpa->vdpa, oct_hw->nr_vring); + if (ret) { + dev_err(&pdev->dev, "Failed to register to vDPA bus"); + goto vdpa_dev_put; + } + return 0; + +vdpa_dev_put: + put_device(&oct_vdpa->vdpa.dev); + return ret; +} + +static void octep_vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *vdpa_dev) +{ + _vdpa_unregister_device(vdpa_dev); +} + +static const struct vdpa_mgmtdev_ops octep_vdpa_mgmt_dev_ops = { + .dev_add = octep_vdpa_dev_add, + .dev_del = octep_vdpa_dev_del +}; + +static bool get_device_ready_status(u8 __iomem *addr) +{ + u64 signature = readq(addr + OCTEP_VF_MBOX_DATA(0)); + + if (signature == OCTEP_DEV_READY_SIGNATURE) { + writeq(0, addr + OCTEP_VF_MBOX_DATA(0)); + return true; + } + + return false; +} + +static struct virtio_device_id id_table[] = { + { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID }, + { 0 }, +}; + +static void octep_vdpa_setup_task(struct work_struct *work) +{ + struct octep_vdpa_mgmt_dev *mgmt_dev = container_of(work, struct octep_vdpa_mgmt_dev, + setup_task); + struct pci_dev *pdev = mgmt_dev->pdev; + struct device *dev = &pdev->dev; + struct octep_hw *oct_hw; + unsigned long timeout; + int ret; + + oct_hw = &mgmt_dev->oct_hw; + + atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_WAIT_FOR_BAR_INIT); + + /* Wait for a maximum of 5 sec */ + timeout = jiffies + msecs_to_jiffies(5000); + while (!time_after(jiffies, timeout)) { + if (get_device_ready_status(oct_hw->base[OCTEP_HW_MBOX_BAR])) { + atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_INIT); + break; + } + + if (atomic_read(&mgmt_dev->status) >= OCTEP_VDPA_DEV_STATUS_READY) { + dev_info(dev, "Stopping vDPA setup task.\n"); + return; + } + + usleep_range(1000, 1500); + } + + if (atomic_read(&mgmt_dev->status) != OCTEP_VDPA_DEV_STATUS_INIT) { + dev_err(dev, "BAR initialization is timed out\n"); + return; + } + + ret = octep_iomap_region(pdev, oct_hw->base, OCTEP_HW_CAPS_BAR); + if (ret) + return; + + ret = octep_hw_caps_read(oct_hw, pdev); + if (ret < 0) + goto unmap_region; + + mgmt_dev->mdev.ops = &octep_vdpa_mgmt_dev_ops; + mgmt_dev->mdev.id_table = id_table; + mgmt_dev->mdev.max_supported_vqs = oct_hw->nr_vring; + mgmt_dev->mdev.supported_features = oct_hw->features; + mgmt_dev->mdev.config_attr_mask = (1 << VDPA_ATTR_DEV_FEATURES); + mgmt_dev->mdev.device = dev; + + ret = vdpa_mgmtdev_register(&mgmt_dev->mdev); + if (ret) { + dev_err(dev, "Failed to register vdpa management interface\n"); + goto unmap_region; + } + + atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_READY); + + return; + +unmap_region: + octep_iounmap_region(pdev, oct_hw->base, OCTEP_HW_CAPS_BAR); + oct_hw->base[OCTEP_HW_CAPS_BAR] = NULL; +} + +static int octep_vdpa_probe_vf(struct pci_dev *pdev) +{ + struct octep_vdpa_mgmt_dev *mgmt_dev; + struct device *dev = &pdev->dev; + int ret; + + ret = pcim_enable_device(pdev); + if (ret) { + dev_err(dev, "Failed to enable device\n"); + return ret; + } + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); + if (ret) { + dev_err(dev, "No usable DMA configuration\n"); + return ret; + } + pci_set_master(pdev); + + mgmt_dev = devm_kzalloc(dev, sizeof(struct octep_vdpa_mgmt_dev), GFP_KERNEL); + if (!mgmt_dev) + return -ENOMEM; + + ret = octep_iomap_region(pdev, mgmt_dev->oct_hw.base, OCTEP_HW_MBOX_BAR); + if (ret) + return ret; + + mgmt_dev->pdev = pdev; + pci_set_drvdata(pdev, mgmt_dev); + + atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_ALLOC); + INIT_WORK(&mgmt_dev->setup_task, octep_vdpa_setup_task); + schedule_work(&mgmt_dev->setup_task); + dev_info(&pdev->dev, "octep vdpa mgmt device setup task is queued\n"); + + return 0; +} + +static void octep_vdpa_assign_barspace(struct pci_dev *vf_dev, struct pci_dev *pf_dev, u8 idx) +{ + struct resource *vf_res = vf_dev->resource + PCI_STD_RESOURCES + 4; + struct resource *pf_res = pf_dev->resource + PCI_STD_RESOURCES + 4; + struct octep_pf *pf = pci_get_drvdata(pf_dev); + struct pci_bus_region bus_region; + + vf_res->name = pci_name(vf_dev); + vf_res->flags = pf_res->flags; + vf_res->parent = (pf_dev->resource + PCI_STD_RESOURCES)->parent; + + bus_region.start = pf->vf_base + idx * pf->vf_stride; + bus_region.end = bus_region.start + pf->vf_stride - 1; + pcibios_bus_to_resource(vf_dev->bus, vf_res, &bus_region); +} + +static int octep_sriov_enable(struct pci_dev *pdev, int num_vfs) +{ + struct octep_pf *pf = pci_get_drvdata(pdev); + u8 __iomem *addr = pf->base[OCTEP_HW_MBOX_BAR]; + struct pci_dev *vf_pdev = NULL; + bool done = false; + int index = 0; + int ret, i; + + ret = pci_enable_sriov(pdev, num_vfs); + if (ret) + return ret; + + pf->enabled_vfs = num_vfs; + + while ((vf_pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM, PCI_ANY_ID, vf_pdev))) { + if (vf_pdev->device != pf->vf_devid) + continue; + + octep_vdpa_assign_barspace(vf_pdev, pdev, index); + if (++index == num_vfs) { + done = true; + break; + } + } + + if (done) { + for (i = 0; i < pf->enabled_vfs; i++) + writeq(OCTEP_DEV_READY_SIGNATURE, addr + OCTEP_PF_MBOX_DATA(i)); + } + + return num_vfs; +} + +static int octep_sriov_disable(struct pci_dev *pdev) +{ + struct octep_pf *pf = pci_get_drvdata(pdev); + + if (!pci_num_vf(pdev)) + return 0; + + pci_disable_sriov(pdev); + pf->enabled_vfs = 0; + + return 0; +} + +static int octep_vdpa_sriov_configure(struct pci_dev *pdev, int num_vfs) +{ + if (num_vfs > 0) + return octep_sriov_enable(pdev, num_vfs); + else + return octep_sriov_disable(pdev); +} + +static u16 octep_get_vf_devid(struct pci_dev *pdev) +{ + u16 did; + + switch (pdev->device) { + case OCTEP_VDPA_DEVID_CN106K_PF: + did = OCTEP_VDPA_DEVID_CN106K_VF; + break; + case OCTEP_VDPA_DEVID_CN105K_PF: + did = OCTEP_VDPA_DEVID_CN105K_VF; + break; + case OCTEP_VDPA_DEVID_CN103K_PF: + did = OCTEP_VDPA_DEVID_CN103K_VF; + break; + default: + did = 0xFFFF; + break; + } + + return did; +} + +static int octep_vdpa_pf_setup(struct octep_pf *octpf) +{ + u8 __iomem *addr = octpf->base[OCTEP_HW_MBOX_BAR]; + struct pci_dev *pdev = octpf->pdev; + int totalvfs; + size_t len; + u64 val; + + totalvfs = pci_sriov_get_totalvfs(pdev); + if (unlikely(!totalvfs)) { + dev_info(&pdev->dev, "Total VFs are %d in PF sriov configuration\n", totalvfs); + return 0; + } + + addr = octpf->base[OCTEP_HW_MBOX_BAR]; + val = readq(addr + OCTEP_EPF_RINFO(0)); + if (val == 0) { + dev_err(&pdev->dev, "Invalid device configuration\n"); + return -EINVAL; + } + + if (OCTEP_EPF_RINFO_RPVF(val) != BIT_ULL(0)) { + val &= ~GENMASK_ULL(35, 32); + val |= BIT_ULL(32); + writeq(val, addr + OCTEP_EPF_RINFO(0)); + } + + len = pci_resource_len(pdev, OCTEP_HW_CAPS_BAR); + + octpf->vf_stride = len / totalvfs; + octpf->vf_devid = octep_get_vf_devid(pdev); + + octep_vdpa_pf_bar_shrink(octpf); + + return 0; +} + +static int octep_vdpa_probe_pf(struct pci_dev *pdev) +{ + struct device *dev = &pdev->dev; + struct octep_pf *octpf; + int ret; + + ret = pcim_enable_device(pdev); + if (ret) { + dev_err(dev, "Failed to enable device\n"); + return ret; + } + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); + if (ret) { + dev_err(dev, "No usable DMA configuration\n"); + return ret; + } + octpf = devm_kzalloc(dev, sizeof(*octpf), GFP_KERNEL); + if (!octpf) + return -ENOMEM; + + ret = octep_iomap_region(pdev, octpf->base, OCTEP_HW_MBOX_BAR); + if (ret) + return ret; + + pci_set_master(pdev); + pci_set_drvdata(pdev, octpf); + octpf->pdev = pdev; + + ret = octep_vdpa_pf_setup(octpf); + if (ret) + goto unmap_region; + + return 0; + +unmap_region: + octep_iounmap_region(pdev, octpf->base, OCTEP_HW_MBOX_BAR); + return ret; +} + +static int octep_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + if (pdev->is_virtfn) + return octep_vdpa_probe_vf(pdev); + else + return octep_vdpa_probe_pf(pdev); +} + +static struct pci_device_id octep_pci_vdpa_map[] = { + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN106K_PF) }, + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN106K_VF) }, + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN105K_PF) }, + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN105K_VF) }, + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN103K_PF) }, + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN103K_VF) }, + { 0 }, +}; + +static struct pci_driver octep_pci_vdpa = { + .name = OCTEP_VDPA_DRIVER_NAME, + .id_table = octep_pci_vdpa_map, + .probe = octep_vdpa_probe, + .remove = octep_vdpa_remove, + .sriov_configure = octep_vdpa_sriov_configure +}; + +module_pci_driver(octep_pci_vdpa); + +MODULE_AUTHOR("Marvell"); +MODULE_DESCRIPTION("Marvell Octeon PCIe endpoint vDPA driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index 8d391947eb8d..1ca445e31acb 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -1538,4 +1538,5 @@ core_initcall(vdpa_init); module_exit(vdpa_exit); MODULE_AUTHOR("Jason Wang <jasowang@redhat.com>"); +MODULE_DESCRIPTION("vDPA bus"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index b60955682474..9ac25d08f473 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -1346,10 +1346,36 @@ static void vhost_dev_unlock_vqs(struct vhost_dev *d) mutex_unlock(&d->vqs[i]->mutex); } -static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq, - __virtio16 *idx) +static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq) { - return vhost_get_avail(vq, *idx, &vq->avail->idx); + __virtio16 idx; + int r; + + r = vhost_get_avail(vq, idx, &vq->avail->idx); + if (unlikely(r < 0)) { + vq_err(vq, "Failed to access available index at %p (%d)\n", + &vq->avail->idx, r); + return r; + } + + /* Check it isn't doing very strange thing with available indexes */ + vq->avail_idx = vhost16_to_cpu(vq, idx); + if (unlikely((u16)(vq->avail_idx - vq->last_avail_idx) > vq->num)) { + vq_err(vq, "Invalid available index change from %u to %u", + vq->last_avail_idx, vq->avail_idx); + return -EINVAL; + } + + /* We're done if there is nothing new */ + if (vq->avail_idx == vq->last_avail_idx) + return 0; + + /* + * We updated vq->avail_idx so we need a memory barrier between + * the index read above and the caller reading avail ring entries. + */ + smp_rmb(); + return 1; } static inline int vhost_get_avail_head(struct vhost_virtqueue *vq, @@ -2554,38 +2580,17 @@ int vhost_get_vq_desc(struct vhost_virtqueue *vq, { struct vring_desc desc; unsigned int i, head, found = 0; - u16 last_avail_idx; - __virtio16 avail_idx; + u16 last_avail_idx = vq->last_avail_idx; __virtio16 ring_head; int ret, access; - /* Check it isn't doing very strange things with descriptor numbers. */ - last_avail_idx = vq->last_avail_idx; - if (vq->avail_idx == vq->last_avail_idx) { - if (unlikely(vhost_get_avail_idx(vq, &avail_idx))) { - vq_err(vq, "Failed to access avail idx at %p\n", - &vq->avail->idx); - return -EFAULT; - } - vq->avail_idx = vhost16_to_cpu(vq, avail_idx); - - if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) { - vq_err(vq, "Guest moved avail index from %u to %u", - last_avail_idx, vq->avail_idx); - return -EFAULT; - } + ret = vhost_get_avail_idx(vq); + if (unlikely(ret < 0)) + return ret; - /* If there's nothing new since last we looked, return - * invalid. - */ - if (vq->avail_idx == last_avail_idx) + if (!ret) return vq->num; - - /* Only get avail ring entries after they have been - * exposed by guest. - */ - smp_rmb(); } /* Grab the next descriptor number they're advertising, and increment @@ -2846,35 +2851,21 @@ EXPORT_SYMBOL_GPL(vhost_add_used_and_signal_n); /* return true if we're sure that avaiable ring is empty */ bool vhost_vq_avail_empty(struct vhost_dev *dev, struct vhost_virtqueue *vq) { - __virtio16 avail_idx; int r; if (vq->avail_idx != vq->last_avail_idx) return false; - r = vhost_get_avail_idx(vq, &avail_idx); - if (unlikely(r)) - return false; - - vq->avail_idx = vhost16_to_cpu(vq, avail_idx); - if (vq->avail_idx != vq->last_avail_idx) { - /* Since we have updated avail_idx, the following - * call to vhost_get_vq_desc() will read available - * ring entries. Make sure that read happens after - * the avail_idx read. - */ - smp_rmb(); - return false; - } + r = vhost_get_avail_idx(vq); - return true; + /* Note: we treat error as non-empty here */ + return r == 0; } EXPORT_SYMBOL_GPL(vhost_vq_avail_empty); /* OK, now we need to know about added descriptors. */ bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) { - __virtio16 avail_idx; int r; if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY)) @@ -2898,25 +2889,13 @@ bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) /* They could have slipped one in as we were doing that: make * sure it's written, then check again. */ smp_mb(); - r = vhost_get_avail_idx(vq, &avail_idx); - if (r) { - vq_err(vq, "Failed to check avail idx at %p: %d\n", - &vq->avail->idx, r); - return false; - } - vq->avail_idx = vhost16_to_cpu(vq, avail_idx); - if (vq->avail_idx != vq->last_avail_idx) { - /* Since we have updated avail_idx, the following - * call to vhost_get_vq_desc() will read available - * ring entries. Make sure that read happens after - * the avail_idx read. - */ - smp_rmb(); - return true; - } + r = vhost_get_avail_idx(vq); + /* Note: we treat error as empty here */ + if (unlikely(r < 0)) + return false; - return false; + return r; } EXPORT_SYMBOL_GPL(vhost_enable_notify); diff --git a/drivers/vhost/vringh.c b/drivers/vhost/vringh.c index 7b8fd977f71c..73e153f9b449 100644 --- a/drivers/vhost/vringh.c +++ b/drivers/vhost/vringh.c @@ -1614,4 +1614,5 @@ EXPORT_SYMBOL(vringh_need_notify_iotlb); #endif +MODULE_DESCRIPTION("host side of a virtio ring"); MODULE_LICENSE("GPL"); diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c index ec20ecff85c7..bf664ec9341b 100644 --- a/drivers/vhost/vsock.c +++ b/drivers/vhost/vsock.c @@ -667,6 +667,7 @@ static int vhost_vsock_dev_open(struct inode *inode, struct file *file) } vsock->guest_cid = 0; /* no CID assigned yet */ + vsock->seqpacket_allow = false; atomic_set(&vsock->queued_replies, 0); @@ -810,8 +811,7 @@ static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features) goto err; } - if (features & (1ULL << VIRTIO_VSOCK_F_SEQPACKET)) - vsock->seqpacket_allow = true; + vsock->seqpacket_allow = features & (1ULL << VIRTIO_VSOCK_F_SEQPACKET); for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) { vq = &vsock->vqs[i]; diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index b968b2aa5f4d..396d3cd49a1b 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c @@ -609,4 +609,5 @@ static void __exit virtio_exit(void) core_initcall(virtio_init); module_exit(virtio_exit); +MODULE_DESCRIPTION("Virtio core interface"); MODULE_LICENSE("GPL"); diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index c0a63638f95e..54469277ca30 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -349,34 +349,49 @@ static inline void update_stat(struct virtio_balloon *vb, int idx, #define pages_to_bytes(x) ((u64)(x) << PAGE_SHIFT) -static unsigned int update_balloon_stats(struct virtio_balloon *vb) +#ifdef CONFIG_VM_EVENT_COUNTERS +/* Return the number of entries filled by vm events */ +static inline unsigned int update_balloon_vm_stats(struct virtio_balloon *vb) { unsigned long events[NR_VM_EVENT_ITEMS]; - struct sysinfo i; unsigned int idx = 0; - long available; - unsigned long caches; all_vm_events(events); - si_meminfo(&i); - - available = si_mem_available(); - caches = global_node_page_state(NR_FILE_PAGES); - -#ifdef CONFIG_VM_EVENT_COUNTERS update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_IN, - pages_to_bytes(events[PSWPIN])); + pages_to_bytes(events[PSWPIN])); update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_OUT, - pages_to_bytes(events[PSWPOUT])); + pages_to_bytes(events[PSWPOUT])); update_stat(vb, idx++, VIRTIO_BALLOON_S_MAJFLT, events[PGMAJFAULT]); update_stat(vb, idx++, VIRTIO_BALLOON_S_MINFLT, events[PGFAULT]); + #ifdef CONFIG_HUGETLB_PAGE update_stat(vb, idx++, VIRTIO_BALLOON_S_HTLB_PGALLOC, events[HTLB_BUDDY_PGALLOC]); update_stat(vb, idx++, VIRTIO_BALLOON_S_HTLB_PGFAIL, events[HTLB_BUDDY_PGALLOC_FAIL]); -#endif -#endif +#endif /* CONFIG_HUGETLB_PAGE */ + + return idx; +} +#else /* CONFIG_VM_EVENT_COUNTERS */ +static inline unsigned int update_balloon_vm_stats(struct virtio_balloon *vb) +{ + return 0; +} +#endif /* CONFIG_VM_EVENT_COUNTERS */ + +static unsigned int update_balloon_stats(struct virtio_balloon *vb) +{ + struct sysinfo i; + unsigned int idx; + long available; + unsigned long caches; + + idx = update_balloon_vm_stats(vb); + + si_meminfo(&i); + available = si_mem_available(); + caches = global_node_page_state(NR_FILE_PAGES); update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMFREE, pages_to_bytes(i.freeram)); update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMTOT, @@ -545,9 +560,8 @@ static void update_balloon_size_func(struct work_struct *work) static int init_vqs(struct virtio_balloon *vb) { + struct virtqueue_info vqs_info[VIRTIO_BALLOON_VQ_MAX] = {}; struct virtqueue *vqs[VIRTIO_BALLOON_VQ_MAX]; - vq_callback_t *callbacks[VIRTIO_BALLOON_VQ_MAX]; - const char *names[VIRTIO_BALLOON_VQ_MAX]; int err; /* @@ -555,33 +569,26 @@ static int init_vqs(struct virtio_balloon *vb) * will be NULL if the related feature is not enabled, which will * cause no allocation for the corresponding virtqueue in find_vqs. */ - callbacks[VIRTIO_BALLOON_VQ_INFLATE] = balloon_ack; - names[VIRTIO_BALLOON_VQ_INFLATE] = "inflate"; - callbacks[VIRTIO_BALLOON_VQ_DEFLATE] = balloon_ack; - names[VIRTIO_BALLOON_VQ_DEFLATE] = "deflate"; - callbacks[VIRTIO_BALLOON_VQ_STATS] = NULL; - names[VIRTIO_BALLOON_VQ_STATS] = NULL; - callbacks[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL; - names[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL; - names[VIRTIO_BALLOON_VQ_REPORTING] = NULL; + vqs_info[VIRTIO_BALLOON_VQ_INFLATE].callback = balloon_ack; + vqs_info[VIRTIO_BALLOON_VQ_INFLATE].name = "inflate"; + vqs_info[VIRTIO_BALLOON_VQ_DEFLATE].callback = balloon_ack; + vqs_info[VIRTIO_BALLOON_VQ_DEFLATE].name = "deflate"; if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) { - names[VIRTIO_BALLOON_VQ_STATS] = "stats"; - callbacks[VIRTIO_BALLOON_VQ_STATS] = stats_request; + vqs_info[VIRTIO_BALLOON_VQ_STATS].name = "stats"; + vqs_info[VIRTIO_BALLOON_VQ_STATS].callback = stats_request; } - if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) { - names[VIRTIO_BALLOON_VQ_FREE_PAGE] = "free_page_vq"; - callbacks[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL; - } + if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) + vqs_info[VIRTIO_BALLOON_VQ_FREE_PAGE].name = "free_page_vq"; if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) { - names[VIRTIO_BALLOON_VQ_REPORTING] = "reporting_vq"; - callbacks[VIRTIO_BALLOON_VQ_REPORTING] = balloon_ack; + vqs_info[VIRTIO_BALLOON_VQ_REPORTING].name = "reporting_vq"; + vqs_info[VIRTIO_BALLOON_VQ_REPORTING].callback = balloon_ack; } err = virtio_find_vqs(vb->vdev, VIRTIO_BALLOON_VQ_MAX, vqs, - callbacks, names, NULL); + vqs_info, NULL); if (err) return err; diff --git a/drivers/virtio/virtio_input.c b/drivers/virtio/virtio_input.c index 1a730d6c0b55..a5d63269f20b 100644 --- a/drivers/virtio/virtio_input.c +++ b/drivers/virtio/virtio_input.c @@ -185,13 +185,14 @@ static void virtinput_cfg_abs(struct virtio_input *vi, int abs) static int virtinput_init_vqs(struct virtio_input *vi) { + struct virtqueue_info vqs_info[] = { + { "events", virtinput_recv_events }, + { "status", virtinput_recv_status }, + }; struct virtqueue *vqs[2]; - vq_callback_t *cbs[] = { virtinput_recv_events, - virtinput_recv_status }; - static const char * const names[] = { "events", "status" }; int err; - err = virtio_find_vqs(vi->vdev, 2, vqs, cbs, names, NULL); + err = virtio_find_vqs(vi->vdev, 2, vqs, vqs_info, NULL); if (err) return err; vi->evt = vqs[0]; diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index 173596589c71..90e784e7b721 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -489,9 +489,7 @@ error_available: static int vm_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], - vq_callback_t *callbacks[], - const char * const names[], - const bool *ctx, + struct virtqueue_info vqs_info[], struct irq_affinity *desc) { struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); @@ -510,13 +508,15 @@ static int vm_find_vqs(struct virtio_device *vdev, unsigned int nvqs, enable_irq_wake(irq); for (i = 0; i < nvqs; ++i) { - if (!names[i]) { + struct virtqueue_info *vqi = &vqs_info[i]; + + if (!vqi->name) { vqs[i] = NULL; continue; } - vqs[i] = vm_setup_vq(vdev, queue_idx++, callbacks[i], names[i], - ctx ? ctx[i] : false); + vqs[i] = vm_setup_vq(vdev, queue_idx++, vqi->callback, + vqi->name, vqi->ctx); if (IS_ERR(vqs[i])) { vm_del_vqs(vdev); return PTR_ERR(vqs[i]); diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c index f6b0b00e4599..7d82facafd75 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -285,12 +285,13 @@ void vp_del_vqs(struct virtio_device *vdev) } static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned int nvqs, - struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], bool per_vq_vectors, - const bool *ctx, - struct irq_affinity *desc) + struct virtqueue *vqs[], + struct virtqueue_info vqs_info[], + bool per_vq_vectors, + struct irq_affinity *desc) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); + struct virtqueue_info *vqi; u16 msix_vec; int i, err, nvectors, allocated_vectors, queue_idx = 0; @@ -301,9 +302,11 @@ static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned int nvqs, if (per_vq_vectors) { /* Best option: one for change interrupt, one per vq. */ nvectors = 1; - for (i = 0; i < nvqs; ++i) - if (names[i] && callbacks[i]) + for (i = 0; i < nvqs; ++i) { + vqi = &vqs_info[i]; + if (vqi->name && vqi->callback) ++nvectors; + } } else { /* Second best: one for change, shared for all vqs. */ nvectors = 2; @@ -317,20 +320,20 @@ static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned int nvqs, vp_dev->per_vq_vectors = per_vq_vectors; allocated_vectors = vp_dev->msix_used_vectors; for (i = 0; i < nvqs; ++i) { - if (!names[i]) { + vqi = &vqs_info[i]; + if (!vqi->name) { vqs[i] = NULL; continue; } - if (!callbacks[i]) + if (!vqi->callback) msix_vec = VIRTIO_MSI_NO_VECTOR; else if (vp_dev->per_vq_vectors) msix_vec = allocated_vectors++; else msix_vec = VP_MSIX_VQ_VECTOR; - vqs[i] = vp_setup_vq(vdev, queue_idx++, callbacks[i], names[i], - ctx ? ctx[i] : false, - msix_vec); + vqs[i] = vp_setup_vq(vdev, queue_idx++, vqi->callback, + vqi->name, vqi->ctx, msix_vec); if (IS_ERR(vqs[i])) { err = PTR_ERR(vqs[i]); goto error_find; @@ -343,7 +346,7 @@ static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned int nvqs, snprintf(vp_dev->msix_names[msix_vec], sizeof *vp_dev->msix_names, "%s-%s", - dev_name(&vp_dev->vdev.dev), names[i]); + dev_name(&vp_dev->vdev.dev), vqi->name); err = request_irq(pci_irq_vector(vp_dev->pci_dev, msix_vec), vring_interrupt, 0, vp_dev->msix_names[msix_vec], @@ -361,8 +364,8 @@ error_find: } static int vp_find_vqs_intx(struct virtio_device *vdev, unsigned int nvqs, - struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], const bool *ctx) + struct virtqueue *vqs[], + struct virtqueue_info vqs_info[]) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); int i, err, queue_idx = 0; @@ -379,12 +382,14 @@ static int vp_find_vqs_intx(struct virtio_device *vdev, unsigned int nvqs, vp_dev->intx_enabled = 1; vp_dev->per_vq_vectors = false; for (i = 0; i < nvqs; ++i) { - if (!names[i]) { + struct virtqueue_info *vqi = &vqs_info[i]; + + if (!vqi->name) { vqs[i] = NULL; continue; } - vqs[i] = vp_setup_vq(vdev, queue_idx++, callbacks[i], names[i], - ctx ? ctx[i] : false, + vqs[i] = vp_setup_vq(vdev, queue_idx++, vqi->callback, + vqi->name, vqi->ctx, VIRTIO_MSI_NO_VECTOR); if (IS_ERR(vqs[i])) { err = PTR_ERR(vqs[i]); @@ -400,25 +405,24 @@ out_del_vqs: /* the config->find_vqs() implementation */ int vp_find_vqs(struct virtio_device *vdev, unsigned int nvqs, - struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], const bool *ctx, + struct virtqueue *vqs[], struct virtqueue_info vqs_info[], struct irq_affinity *desc) { int err; /* Try MSI-X with one vector per queue. */ - err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, true, ctx, desc); + err = vp_find_vqs_msix(vdev, nvqs, vqs, vqs_info, true, desc); if (!err) return 0; /* Fallback: MSI-X with one vector for config, one shared for queues. */ - err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, false, ctx, desc); + err = vp_find_vqs_msix(vdev, nvqs, vqs, vqs_info, false, desc); if (!err) return 0; /* Is there an interrupt? If not give up. */ if (!(to_vp_device(vdev)->pci_dev->irq)) return err; /* Finally fall back to regular interrupts. */ - return vp_find_vqs_intx(vdev, nvqs, vqs, callbacks, names, ctx); + return vp_find_vqs_intx(vdev, nvqs, vqs, vqs_info); } const char *vp_bus_name(struct virtio_device *vdev) diff --git a/drivers/virtio/virtio_pci_common.h b/drivers/virtio/virtio_pci_common.h index 7fef52bee455..3c4bb2d6163a 100644 --- a/drivers/virtio/virtio_pci_common.h +++ b/drivers/virtio/virtio_pci_common.h @@ -127,8 +127,7 @@ bool vp_notify(struct virtqueue *vq); void vp_del_vqs(struct virtio_device *vdev); /* the config->find_vqs() implementation */ int vp_find_vqs(struct virtio_device *vdev, unsigned int nvqs, - struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], const bool *ctx, + struct virtqueue *vqs[], struct virtqueue_info vqs_info[], struct irq_affinity *desc); const char *vp_bus_name(struct virtio_device *vdev); diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index f62b530aa3b5..3b5b9499a53a 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -595,13 +595,12 @@ err: static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], - vq_callback_t *callbacks[], - const char * const names[], const bool *ctx, + struct virtqueue_info vqs_info[], struct irq_affinity *desc) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); struct virtqueue *vq; - int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, ctx, desc); + int rc = vp_find_vqs(vdev, nvqs, vqs, vqs_info, desc); if (rc) return rc; diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 2a972752ff1b..be7309b1e860 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -3121,8 +3121,10 @@ dma_addr_t virtqueue_dma_map_single_attrs(struct virtqueue *_vq, void *ptr, { struct vring_virtqueue *vq = to_vvq(_vq); - if (!vq->use_dma_api) + if (!vq->use_dma_api) { + kmsan_handle_dma(virt_to_page(ptr), offset_in_page(ptr), size, dir); return (dma_addr_t)virt_to_phys(ptr); + } return dma_map_single_attrs(vring_dma_dev(vq), ptr, size, dir, attrs); } @@ -3244,4 +3246,5 @@ void virtqueue_dma_sync_single_range_for_device(struct virtqueue *_vq, } EXPORT_SYMBOL_GPL(virtqueue_dma_sync_single_range_for_device); +MODULE_DESCRIPTION("Virtio ring implementation"); MODULE_LICENSE("GPL"); diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index e803db0da307..7364bd53e38d 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -358,9 +358,7 @@ create_affinity_masks(unsigned int nvecs, struct irq_affinity *affd) static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], - vq_callback_t *callbacks[], - const char * const names[], - const bool *ctx, + struct virtqueue_info vqs_info[], struct irq_affinity *desc) { struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); @@ -379,14 +377,15 @@ static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs, } for (i = 0; i < nvqs; ++i) { - if (!names[i]) { + struct virtqueue_info *vqi = &vqs_info[i]; + + if (!vqi->name) { vqs[i] = NULL; continue; } - vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++, - callbacks[i], names[i], ctx ? - ctx[i] : false); + vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++, vqi->callback, + vqi->name, vqi->ctx); if (IS_ERR(vqs[i])) { err = PTR_ERR(vqs[i]); goto err_setup_vq; diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c index 1a52a51b6b07..dd5260141615 100644 --- a/fs/fuse/virtio_fs.c +++ b/fs/fuse/virtio_fs.c @@ -51,7 +51,7 @@ struct virtio_fs_vq { struct work_struct done_work; struct list_head queued_reqs; struct list_head end_reqs; /* End these requests */ - struct delayed_work dispatch_work; + struct work_struct dispatch_work; struct fuse_dev *fud; bool connected; long in_flight; @@ -233,7 +233,7 @@ static void virtio_fs_drain_queue(struct virtio_fs_vq *fsvq) } flush_work(&fsvq->done_work); - flush_delayed_work(&fsvq->dispatch_work); + flush_work(&fsvq->dispatch_work); } static void virtio_fs_drain_all_queues_locked(struct virtio_fs *fs) @@ -408,6 +408,10 @@ static void virtio_fs_hiprio_done_work(struct work_struct *work) dec_in_flight_req(fsvq); } } while (!virtqueue_enable_cb(vq)); + + if (!list_empty(&fsvq->queued_reqs)) + schedule_work(&fsvq->dispatch_work); + spin_unlock(&fsvq->lock); } @@ -415,7 +419,7 @@ static void virtio_fs_request_dispatch_work(struct work_struct *work) { struct fuse_req *req; struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, - dispatch_work.work); + dispatch_work); int ret; pr_debug("virtio-fs: worker %s called.\n", __func__); @@ -447,11 +451,9 @@ static void virtio_fs_request_dispatch_work(struct work_struct *work) ret = virtio_fs_enqueue_req(fsvq, req, true); if (ret < 0) { - if (ret == -ENOMEM || ret == -ENOSPC) { + if (ret == -ENOSPC) { spin_lock(&fsvq->lock); list_add_tail(&req->list, &fsvq->queued_reqs); - schedule_delayed_work(&fsvq->dispatch_work, - msecs_to_jiffies(1)); spin_unlock(&fsvq->lock); return; } @@ -494,12 +496,10 @@ static int send_forget_request(struct virtio_fs_vq *fsvq, ret = virtqueue_add_outbuf(vq, &sg, 1, forget, GFP_ATOMIC); if (ret < 0) { - if (ret == -ENOMEM || ret == -ENOSPC) { + if (ret == -ENOSPC) { pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later\n", ret); list_add_tail(&forget->list, &fsvq->queued_reqs); - schedule_delayed_work(&fsvq->dispatch_work, - msecs_to_jiffies(1)); if (!in_flight) inc_in_flight_req(fsvq); /* Queue is full */ @@ -531,7 +531,7 @@ static void virtio_fs_hiprio_dispatch_work(struct work_struct *work) { struct virtio_fs_forget *forget; struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, - dispatch_work.work); + dispatch_work); pr_debug("virtio-fs: worker %s called.\n", __func__); while (1) { spin_lock(&fsvq->lock); @@ -709,6 +709,12 @@ static void virtio_fs_requests_done_work(struct work_struct *work) virtio_fs_request_complete(req, fsvq); } } + + /* Try to push previously queued requests, as the queue might no longer be full */ + spin_lock(&fsvq->lock); + if (!list_empty(&fsvq->queued_reqs)) + schedule_work(&fsvq->dispatch_work); + spin_unlock(&fsvq->lock); } static void virtio_fs_map_queues(struct virtio_device *vdev, struct virtio_fs *fs) @@ -770,12 +776,12 @@ static void virtio_fs_init_vq(struct virtio_fs_vq *fsvq, char *name, if (vq_type == VQ_REQUEST) { INIT_WORK(&fsvq->done_work, virtio_fs_requests_done_work); - INIT_DELAYED_WORK(&fsvq->dispatch_work, - virtio_fs_request_dispatch_work); + INIT_WORK(&fsvq->dispatch_work, + virtio_fs_request_dispatch_work); } else { INIT_WORK(&fsvq->done_work, virtio_fs_hiprio_done_work); - INIT_DELAYED_WORK(&fsvq->dispatch_work, - virtio_fs_hiprio_dispatch_work); + INIT_WORK(&fsvq->dispatch_work, + virtio_fs_hiprio_dispatch_work); } } @@ -783,14 +789,13 @@ static void virtio_fs_init_vq(struct virtio_fs_vq *fsvq, char *name, static int virtio_fs_setup_vqs(struct virtio_device *vdev, struct virtio_fs *fs) { + struct virtqueue_info *vqs_info; struct virtqueue **vqs; - vq_callback_t **callbacks; /* Specify pre_vectors to ensure that the queues before the * request queues (e.g. hiprio) don't claim any of the CPUs in * the multi-queue mapping and interrupt affinities */ struct irq_affinity desc = { .pre_vectors = VQ_REQUEST }; - const char **names; unsigned int i; int ret = 0; @@ -808,20 +813,18 @@ static int virtio_fs_setup_vqs(struct virtio_device *vdev, return -ENOMEM; vqs = kmalloc_array(fs->nvqs, sizeof(vqs[VQ_HIPRIO]), GFP_KERNEL); - callbacks = kmalloc_array(fs->nvqs, sizeof(callbacks[VQ_HIPRIO]), - GFP_KERNEL); - names = kmalloc_array(fs->nvqs, sizeof(names[VQ_HIPRIO]), GFP_KERNEL); fs->mq_map = kcalloc_node(nr_cpu_ids, sizeof(*fs->mq_map), GFP_KERNEL, dev_to_node(&vdev->dev)); - if (!vqs || !callbacks || !names || !fs->mq_map) { + vqs_info = kcalloc(fs->nvqs, sizeof(*vqs_info), GFP_KERNEL); + if (!vqs || !vqs_info || !fs->mq_map) { ret = -ENOMEM; goto out; } /* Initialize the hiprio/forget request virtqueue */ - callbacks[VQ_HIPRIO] = virtio_fs_vq_done; + vqs_info[VQ_HIPRIO].callback = virtio_fs_vq_done; virtio_fs_init_vq(&fs->vqs[VQ_HIPRIO], "hiprio", VQ_HIPRIO); - names[VQ_HIPRIO] = fs->vqs[VQ_HIPRIO].name; + vqs_info[VQ_HIPRIO].name = fs->vqs[VQ_HIPRIO].name; /* Initialize the requests virtqueues */ for (i = VQ_REQUEST; i < fs->nvqs; i++) { @@ -829,11 +832,11 @@ static int virtio_fs_setup_vqs(struct virtio_device *vdev, snprintf(vq_name, VQ_NAME_LEN, "requests.%u", i - VQ_REQUEST); virtio_fs_init_vq(&fs->vqs[i], vq_name, VQ_REQUEST); - callbacks[i] = virtio_fs_vq_done; - names[i] = fs->vqs[i].name; + vqs_info[i].callback = virtio_fs_vq_done; + vqs_info[i].name = fs->vqs[i].name; } - ret = virtio_find_vqs(vdev, fs->nvqs, vqs, callbacks, names, &desc); + ret = virtio_find_vqs(vdev, fs->nvqs, vqs, vqs_info, &desc); if (ret < 0) goto out; @@ -842,8 +845,7 @@ static int virtio_fs_setup_vqs(struct virtio_device *vdev, virtio_fs_start_all_queues(fs); out: - kfree(names); - kfree(callbacks); + kfree(vqs_info); kfree(vqs); if (ret) { kfree(fs->vqs); @@ -1367,7 +1369,7 @@ __releases(fiq->lock) fsvq = &fs->vqs[queue_id]; ret = virtio_fs_enqueue_req(fsvq, req, false); if (ret < 0) { - if (ret == -ENOMEM || ret == -ENOSPC) { + if (ret == -ENOSPC) { /* * Virtqueue full. Retry submission from worker * context as we might be holding fc->bg_lock. @@ -1375,8 +1377,6 @@ __releases(fiq->lock) spin_lock(&fsvq->lock); list_add_tail(&req->list, &fsvq->queued_reqs); inc_in_flight_req(fsvq); - schedule_delayed_work(&fsvq->dispatch_work, - msecs_to_jiffies(1)); spin_unlock(&fsvq->lock); return; } @@ -1386,7 +1386,7 @@ __releases(fiq->lock) /* Can't end request in submission context. Use a worker */ spin_lock(&fsvq->lock); list_add_tail(&req->list, &fsvq->end_reqs); - schedule_delayed_work(&fsvq->dispatch_work, 0); + schedule_work(&fsvq->dispatch_work); spin_unlock(&fsvq->lock); return; } diff --git a/include/linux/mlx5/mlx5_ifc_vdpa.h b/include/linux/mlx5/mlx5_ifc_vdpa.h index 40371c916cf9..58dfa2ee7c83 100644 --- a/include/linux/mlx5/mlx5_ifc_vdpa.h +++ b/include/linux/mlx5/mlx5_ifc_vdpa.h @@ -148,7 +148,9 @@ enum { MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_ADDRS = (u64)1 << 6, MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_AVAIL_IDX = (u64)1 << 7, MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_USED_IDX = (u64)1 << 8, + MLX5_VIRTQ_MODIFY_MASK_QUEUE_VIRTIO_VERSION = (u64)1 << 10, MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_MKEY = (u64)1 << 11, + MLX5_VIRTQ_MODIFY_MASK_QUEUE_FEATURES = (u64)1 << 12, MLX5_VIRTQ_MODIFY_MASK_DESC_GROUP_MKEY = (u64)1 << 14, }; diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h index da9b271b54db..ab4b9a3fef6b 100644 --- a/include/linux/virtio_config.h +++ b/include/linux/virtio_config.h @@ -19,6 +19,20 @@ struct virtio_shm_region { typedef void vq_callback_t(struct virtqueue *); /** + * struct virtqueue_info - Info for a virtqueue passed to find_vqs(). + * @name: virtqueue description. Used mainly for debugging, NULL for + * a virtqueue unused by the driver. + * @callback: A callback to invoke on a used buffer notification. + * NULL for a virtqueue that does not need a callback. + * @ctx: A flag to indicate to maintain an extra context per virtqueue. + */ +struct virtqueue_info { + const char *name; + vq_callback_t *callback; + bool ctx; +}; + +/** * struct virtio_config_ops - operations for configuring a virtio device * Note: Do not assume that a transport implements all of the operations * getting/setting a value as a simple read/write! Generally speaking, @@ -53,10 +67,7 @@ typedef void vq_callback_t(struct virtqueue *); * vdev: the virtio_device * nvqs: the number of virtqueues to find * vqs: on success, includes new virtqueues - * callbacks: array of callbacks, for each virtqueue - * include a NULL entry for vqs that do not need a callback - * names: array of virtqueue names (mainly for debugging) - * include a NULL entry for vqs unused by driver + * vqs_info: array of virtqueue info structures * Returns 0 on success or error status * @del_vqs: free virtqueues found by find_vqs(). * @synchronize_cbs: synchronize with the virtqueue callbacks (optional) @@ -105,9 +116,9 @@ struct virtio_config_ops { u8 (*get_status)(struct virtio_device *vdev); void (*set_status)(struct virtio_device *vdev, u8 status); void (*reset)(struct virtio_device *vdev); - int (*find_vqs)(struct virtio_device *, unsigned nvqs, - struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], const bool *ctx, + int (*find_vqs)(struct virtio_device *vdev, unsigned int nvqs, + struct virtqueue *vqs[], + struct virtqueue_info vqs_info[], struct irq_affinity *desc); void (*del_vqs)(struct virtio_device *); void (*synchronize_cbs)(struct virtio_device *); @@ -117,7 +128,7 @@ struct virtio_config_ops { int (*set_vq_affinity)(struct virtqueue *vq, const struct cpumask *cpu_mask); const struct cpumask *(*get_vq_affinity)(struct virtio_device *vdev, - int index); + int index); bool (*get_shm_region)(struct virtio_device *vdev, struct virtio_shm_region *region, u8 id); int (*disable_vq_and_reset)(struct virtqueue *vq); @@ -211,38 +222,29 @@ static inline bool virtio_has_dma_quirk(const struct virtio_device *vdev) } static inline +int virtio_find_vqs(struct virtio_device *vdev, unsigned int nvqs, + struct virtqueue *vqs[], + struct virtqueue_info vqs_info[], + struct irq_affinity *desc) +{ + return vdev->config->find_vqs(vdev, nvqs, vqs, vqs_info, desc); +} + +static inline struct virtqueue *virtio_find_single_vq(struct virtio_device *vdev, vq_callback_t *c, const char *n) { - vq_callback_t *callbacks[] = { c }; - const char *names[] = { n }; + struct virtqueue_info vqs_info[] = { + { n, c }, + }; struct virtqueue *vq; - int err = vdev->config->find_vqs(vdev, 1, &vq, callbacks, names, NULL, - NULL); + int err = virtio_find_vqs(vdev, 1, &vq, vqs_info, NULL); + if (err < 0) return ERR_PTR(err); return vq; } -static inline -int virtio_find_vqs(struct virtio_device *vdev, unsigned nvqs, - struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], - struct irq_affinity *desc) -{ - return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, NULL, desc); -} - -static inline -int virtio_find_vqs_ctx(struct virtio_device *vdev, unsigned nvqs, - struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], const bool *ctx, - struct irq_affinity *desc) -{ - return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, ctx, - desc); -} - /** * virtio_synchronize_cbs - synchronize with virtqueue callbacks * @dev: the virtio device diff --git a/include/linux/virtio_net.h b/include/linux/virtio_net.h index 4dfa9b69ca8d..d1d7825318c3 100644 --- a/include/linux/virtio_net.h +++ b/include/linux/virtio_net.h @@ -56,6 +56,7 @@ static inline int virtio_net_hdr_to_skb(struct sk_buff *skb, unsigned int thlen = 0; unsigned int p_off = 0; unsigned int ip_proto; + u64 ret, remainder, gso_size; if (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) { switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { @@ -98,6 +99,16 @@ static inline int virtio_net_hdr_to_skb(struct sk_buff *skb, u32 off = __virtio16_to_cpu(little_endian, hdr->csum_offset); u32 needed = start + max_t(u32, thlen, off + sizeof(__sum16)); + if (hdr->gso_size) { + gso_size = __virtio16_to_cpu(little_endian, hdr->gso_size); + ret = div64_u64_rem(skb->len, gso_size, &remainder); + if (!(ret && (hdr->gso_size > needed) && + ((remainder > needed) || (remainder == 0)))) { + return -EINVAL; + } + skb_shinfo(skb)->tx_flags |= SKBFL_SHARED_FRAG; + } + if (!pskb_may_pull(skb, needed)) return -EINVAL; diff --git a/net/vmw_vsock/virtio_transport.c b/net/vmw_vsock/virtio_transport.c index 43d405298857..64a07acfef12 100644 --- a/net/vmw_vsock/virtio_transport.c +++ b/net/vmw_vsock/virtio_transport.c @@ -617,20 +617,14 @@ out: static int virtio_vsock_vqs_init(struct virtio_vsock *vsock) { struct virtio_device *vdev = vsock->vdev; - static const char * const names[] = { - "rx", - "tx", - "event", - }; - vq_callback_t *callbacks[] = { - virtio_vsock_rx_done, - virtio_vsock_tx_done, - virtio_vsock_event_done, + struct virtqueue_info vqs_info[] = { + { "rx", virtio_vsock_rx_done }, + { "tx", virtio_vsock_tx_done }, + { "event", virtio_vsock_event_done }, }; int ret; - ret = virtio_find_vqs(vdev, VSOCK_VQ_MAX, vsock->vqs, callbacks, names, - NULL); + ret = virtio_find_vqs(vdev, VSOCK_VQ_MAX, vsock->vqs, vqs_info, NULL); if (ret < 0) return ret; diff --git a/sound/virtio/virtio_card.c b/sound/virtio/virtio_card.c index 7805daea0102..965209e1d872 100644 --- a/sound/virtio/virtio_card.c +++ b/sound/virtio/virtio_card.c @@ -110,25 +110,22 @@ static void virtsnd_event_notify_cb(struct virtqueue *vqueue) static int virtsnd_find_vqs(struct virtio_snd *snd) { struct virtio_device *vdev = snd->vdev; - static vq_callback_t *callbacks[VIRTIO_SND_VQ_MAX] = { - [VIRTIO_SND_VQ_CONTROL] = virtsnd_ctl_notify_cb, - [VIRTIO_SND_VQ_EVENT] = virtsnd_event_notify_cb, - [VIRTIO_SND_VQ_TX] = virtsnd_pcm_tx_notify_cb, - [VIRTIO_SND_VQ_RX] = virtsnd_pcm_rx_notify_cb - }; - static const char *names[VIRTIO_SND_VQ_MAX] = { - [VIRTIO_SND_VQ_CONTROL] = "virtsnd-ctl", - [VIRTIO_SND_VQ_EVENT] = "virtsnd-event", - [VIRTIO_SND_VQ_TX] = "virtsnd-tx", - [VIRTIO_SND_VQ_RX] = "virtsnd-rx" + struct virtqueue_info vqs_info[VIRTIO_SND_VQ_MAX] = { + [VIRTIO_SND_VQ_CONTROL] = { "virtsnd-ctl", + virtsnd_ctl_notify_cb }, + [VIRTIO_SND_VQ_EVENT] = { "virtsnd-event", + virtsnd_event_notify_cb }, + [VIRTIO_SND_VQ_TX] = { "virtsnd-tx", + virtsnd_pcm_tx_notify_cb }, + [VIRTIO_SND_VQ_RX] = { "virtsnd-rx", + virtsnd_pcm_rx_notify_cb }, }; struct virtqueue *vqs[VIRTIO_SND_VQ_MAX] = { 0 }; unsigned int i; unsigned int n; int rc; - rc = virtio_find_vqs(vdev, VIRTIO_SND_VQ_MAX, vqs, callbacks, names, - NULL); + rc = virtio_find_vqs(vdev, VIRTIO_SND_VQ_MAX, vqs, vqs_info, NULL); if (rc) { dev_err(&vdev->dev, "failed to initialize virtqueues\n"); return rc; diff --git a/tools/virtio/vringh_test.c b/tools/virtio/vringh_test.c index 98ff808d6f0c..43d3a6aa1dcf 100644 --- a/tools/virtio/vringh_test.c +++ b/tools/virtio/vringh_test.c @@ -139,7 +139,7 @@ static int parallel_test(u64 features, bool fast_vringh) { void *host_map, *guest_map; - int fd, mapsize, to_guest[2], to_host[2]; + int pipe_ret, fd, mapsize, to_guest[2], to_host[2]; unsigned long xfers = 0, notifies = 0, receives = 0; unsigned int first_cpu, last_cpu; cpu_set_t cpu_set; @@ -161,8 +161,11 @@ static int parallel_test(u64 features, host_map = mmap(NULL, mapsize, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); guest_map = mmap(NULL, mapsize, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); - pipe(to_guest); - pipe(to_host); + pipe_ret = pipe(to_guest); + assert(!pipe_ret); + + pipe_ret = pipe(to_host); + assert(!pipe_ret); CPU_ZERO(&cpu_set); find_cpus(&first_cpu, &last_cpu); |