diff options
Diffstat (limited to 'hw/virtio')
| -rw-r--r-- | hw/virtio/vhost-backend.c | 30 | ||||
| -rw-r--r-- | hw/virtio/vhost-user.c | 151 | ||||
| -rw-r--r-- | hw/virtio/vhost-vdpa.c | 39 | ||||
| -rw-r--r-- | hw/virtio/vhost.c | 31 | ||||
| -rw-r--r-- | hw/virtio/virtio-balloon.c | 41 | ||||
| -rw-r--r-- | hw/virtio/virtio-bus.c | 14 | ||||
| -rw-r--r-- | hw/virtio/virtio-pci.c | 14 | ||||
| -rw-r--r-- | hw/virtio/virtio.c | 7 |
8 files changed, 213 insertions, 114 deletions
diff --git a/hw/virtio/vhost-backend.c b/hw/virtio/vhost-backend.c index 594d770b75..b65f8f7e97 100644 --- a/hw/virtio/vhost-backend.c +++ b/hw/virtio/vhost-backend.c @@ -293,7 +293,7 @@ static void vhost_kernel_set_iotlb_callback(struct vhost_dev *dev, qemu_set_fd_handler((uintptr_t)dev->opaque, NULL, NULL, NULL); } -static const VhostOps kernel_ops = { +const VhostOps kernel_ops = { .backend_type = VHOST_BACKEND_TYPE_KERNEL, .vhost_backend_init = vhost_kernel_init, .vhost_backend_cleanup = vhost_kernel_cleanup, @@ -328,34 +328,6 @@ static const VhostOps kernel_ops = { }; #endif -int vhost_set_backend_type(struct vhost_dev *dev, VhostBackendType backend_type) -{ - int r = 0; - - switch (backend_type) { -#ifdef CONFIG_VHOST_KERNEL - case VHOST_BACKEND_TYPE_KERNEL: - dev->vhost_ops = &kernel_ops; - break; -#endif -#ifdef CONFIG_VHOST_USER - case VHOST_BACKEND_TYPE_USER: - dev->vhost_ops = &user_ops; - break; -#endif -#ifdef CONFIG_VHOST_VDPA - case VHOST_BACKEND_TYPE_VDPA: - dev->vhost_ops = &vdpa_ops; - break; -#endif - default: - error_report("Unknown vhost backend type"); - r = -1; - } - - return r; -} - int vhost_backend_update_device_iotlb(struct vhost_dev *dev, uint64_t iova, uint64_t uaddr, uint64_t len, diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c index 2407836fac..2c8556237f 100644 --- a/hw/virtio/vhost-user.c +++ b/hw/virtio/vhost-user.c @@ -429,7 +429,7 @@ static int process_message_reply(struct vhost_dev *dev, } if (msg_reply.hdr.request != msg->hdr.request) { - error_report("Received unexpected msg type." + error_report("Received unexpected msg type. " "Expected %d received %d", msg->hdr.request, msg_reply.hdr.request); return -1; @@ -1095,23 +1095,6 @@ static int vhost_user_set_mem_table(struct vhost_dev *dev, return 0; } -static int vhost_user_set_vring_addr(struct vhost_dev *dev, - struct vhost_vring_addr *addr) -{ - VhostUserMsg msg = { - .hdr.request = VHOST_USER_SET_VRING_ADDR, - .hdr.flags = VHOST_USER_VERSION, - .payload.addr = *addr, - .hdr.size = sizeof(msg.payload.addr), - }; - - if (vhost_user_write(dev, &msg, NULL, 0) < 0) { - return -1; - } - - return 0; -} - static int vhost_user_set_vring_endian(struct vhost_dev *dev, struct vhost_vring_state *ring) { @@ -1288,33 +1271,6 @@ static int vhost_user_set_vring_call(struct vhost_dev *dev, return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_CALL, file); } -static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64) -{ - VhostUserMsg msg = { - .hdr.request = request, - .hdr.flags = VHOST_USER_VERSION, - .payload.u64 = u64, - .hdr.size = sizeof(msg.payload.u64), - }; - - if (vhost_user_write(dev, &msg, NULL, 0) < 0) { - return -1; - } - - return 0; -} - -static int vhost_user_set_features(struct vhost_dev *dev, - uint64_t features) -{ - return vhost_user_set_u64(dev, VHOST_USER_SET_FEATURES, features); -} - -static int vhost_user_set_protocol_features(struct vhost_dev *dev, - uint64_t features) -{ - return vhost_user_set_u64(dev, VHOST_USER_SET_PROTOCOL_FEATURES, features); -} static int vhost_user_get_u64(struct vhost_dev *dev, int request, uint64_t *u64) { @@ -1360,6 +1316,107 @@ static int vhost_user_get_features(struct vhost_dev *dev, uint64_t *features) return 0; } +static int enforce_reply(struct vhost_dev *dev, + const VhostUserMsg *msg) +{ + uint64_t dummy; + + if (msg->hdr.flags & VHOST_USER_NEED_REPLY_MASK) { + return process_message_reply(dev, msg); + } + + /* + * We need to wait for a reply but the backend does not + * support replies for the command we just sent. + * Send VHOST_USER_GET_FEATURES which makes all backends + * send a reply. + */ + return vhost_user_get_features(dev, &dummy); +} + +static int vhost_user_set_vring_addr(struct vhost_dev *dev, + struct vhost_vring_addr *addr) +{ + VhostUserMsg msg = { + .hdr.request = VHOST_USER_SET_VRING_ADDR, + .hdr.flags = VHOST_USER_VERSION, + .payload.addr = *addr, + .hdr.size = sizeof(msg.payload.addr), + }; + + bool reply_supported = virtio_has_feature(dev->protocol_features, + VHOST_USER_PROTOCOL_F_REPLY_ACK); + + /* + * wait for a reply if logging is enabled to make sure + * backend is actually logging changes + */ + bool wait_for_reply = addr->flags & (1 << VHOST_VRING_F_LOG); + + if (reply_supported && wait_for_reply) { + msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK; + } + + if (vhost_user_write(dev, &msg, NULL, 0) < 0) { + return -1; + } + + if (wait_for_reply) { + return enforce_reply(dev, &msg); + } + + return 0; +} + +static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64, + bool wait_for_reply) +{ + VhostUserMsg msg = { + .hdr.request = request, + .hdr.flags = VHOST_USER_VERSION, + .payload.u64 = u64, + .hdr.size = sizeof(msg.payload.u64), + }; + + if (wait_for_reply) { + bool reply_supported = virtio_has_feature(dev->protocol_features, + VHOST_USER_PROTOCOL_F_REPLY_ACK); + if (reply_supported) { + msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK; + } + } + + if (vhost_user_write(dev, &msg, NULL, 0) < 0) { + return -1; + } + + if (wait_for_reply) { + return enforce_reply(dev, &msg); + } + + return 0; +} + +static int vhost_user_set_features(struct vhost_dev *dev, + uint64_t features) +{ + /* + * wait for a reply if logging is enabled to make sure + * backend is actually logging changes + */ + bool log_enabled = features & (0x1ULL << VHOST_F_LOG_ALL); + + return vhost_user_set_u64(dev, VHOST_USER_SET_FEATURES, features, + log_enabled); +} + +static int vhost_user_set_protocol_features(struct vhost_dev *dev, + uint64_t features) +{ + return vhost_user_set_u64(dev, VHOST_USER_SET_PROTOCOL_FEATURES, features, + false); +} + static int vhost_user_set_owner(struct vhost_dev *dev) { VhostUserMsg msg = { @@ -1474,6 +1531,7 @@ static int vhost_user_slave_handle_vring_host_notifier(struct vhost_dev *dev, g_free(name); if (virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true)) { + object_unparent(OBJECT(&n->mr)); munmap(addr, page_size); return -1; } @@ -2422,7 +2480,7 @@ void vhost_user_cleanup(VhostUserState *user) if (!user->chr) { return; } - + memory_region_transaction_begin(); for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { if (user->notifier[i].addr) { object_unparent(OBJECT(&user->notifier[i].mr)); @@ -2430,6 +2488,7 @@ void vhost_user_cleanup(VhostUserState *user) user->notifier[i].addr = NULL; } } + memory_region_transaction_commit(); user->chr = NULL; } diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c index 4fa414feea..7633ea66d1 100644 --- a/hw/virtio/vhost-vdpa.c +++ b/hw/virtio/vhost-vdpa.c @@ -89,19 +89,13 @@ static int vhost_vdpa_dma_unmap(struct vhost_vdpa *v, hwaddr iova, return ret; } -static void vhost_vdpa_listener_begin(MemoryListener *listener) +static void vhost_vdpa_listener_begin_batch(struct vhost_vdpa *v) { - struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); - struct vhost_dev *dev = v->dev; - struct vhost_msg_v2 msg = {}; int fd = v->device_fd; - - if (!(dev->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH))) { - return; - } - - msg.type = v->msg_type; - msg.iotlb.type = VHOST_IOTLB_BATCH_BEGIN; + struct vhost_msg_v2 msg = { + .type = v->msg_type, + .iotlb.type = VHOST_IOTLB_BATCH_BEGIN, + }; if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) { error_report("failed to write, fd=%d, errno=%d (%s)", @@ -109,6 +103,16 @@ static void vhost_vdpa_listener_begin(MemoryListener *listener) } } +static void vhost_vdpa_iotlb_batch_begin_once(struct vhost_vdpa *v) +{ + if (v->dev->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH) && + !v->iotlb_batch_begin_sent) { + vhost_vdpa_listener_begin_batch(v); + } + + v->iotlb_batch_begin_sent = true; +} + static void vhost_vdpa_listener_commit(MemoryListener *listener) { struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener); @@ -120,6 +124,10 @@ static void vhost_vdpa_listener_commit(MemoryListener *listener) return; } + if (!v->iotlb_batch_begin_sent) { + return; + } + msg.type = v->msg_type; msg.iotlb.type = VHOST_IOTLB_BATCH_END; @@ -127,6 +135,8 @@ static void vhost_vdpa_listener_commit(MemoryListener *listener) error_report("failed to write, fd=%d, errno=%d (%s)", fd, errno, strerror(errno)); } + + v->iotlb_batch_begin_sent = false; } static void vhost_vdpa_listener_region_add(MemoryListener *listener, @@ -170,6 +180,7 @@ static void vhost_vdpa_listener_region_add(MemoryListener *listener, llsize = int128_sub(llend, int128_make64(iova)); + vhost_vdpa_iotlb_batch_begin_once(v); ret = vhost_vdpa_dma_map(v, iova, int128_get64(llsize), vaddr, section->readonly); if (ret) { @@ -221,6 +232,7 @@ static void vhost_vdpa_listener_region_del(MemoryListener *listener, llsize = int128_sub(llend, int128_make64(iova)); + vhost_vdpa_iotlb_batch_begin_once(v); ret = vhost_vdpa_dma_unmap(v, iova, int128_get64(llsize)); if (ret) { error_report("vhost_vdpa dma unmap error!"); @@ -234,7 +246,6 @@ static void vhost_vdpa_listener_region_del(MemoryListener *listener, * depends on the addnop(). */ static const MemoryListener vhost_vdpa_memory_listener = { - .begin = vhost_vdpa_listener_begin, .commit = vhost_vdpa_listener_commit, .region_add = vhost_vdpa_listener_region_add, .region_del = vhost_vdpa_listener_region_del, @@ -432,13 +443,13 @@ static int vhost_vdpa_set_backend_cap(struct vhost_dev *dev) int r; if (vhost_vdpa_call(dev, VHOST_GET_BACKEND_FEATURES, &features)) { - return 0; + return -EFAULT; } features &= f; r = vhost_vdpa_call(dev, VHOST_SET_BACKEND_FEATURES, &features); if (r) { - return 0; + return -EFAULT; } dev->backend_cap = features; diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c index 3c0b537f89..b4b29413e6 100644 --- a/hw/virtio/vhost.c +++ b/hw/virtio/vhost.c @@ -174,6 +174,35 @@ static uint64_t vhost_get_log_size(struct vhost_dev *dev) return log_size; } +static int vhost_set_backend_type(struct vhost_dev *dev, + VhostBackendType backend_type) +{ + int r = 0; + + switch (backend_type) { +#ifdef CONFIG_VHOST_KERNEL + case VHOST_BACKEND_TYPE_KERNEL: + dev->vhost_ops = &kernel_ops; + break; +#endif +#ifdef CONFIG_VHOST_USER + case VHOST_BACKEND_TYPE_USER: + dev->vhost_ops = &user_ops; + break; +#endif +#ifdef CONFIG_VHOST_VDPA + case VHOST_BACKEND_TYPE_VDPA: + dev->vhost_ops = &vdpa_ops; + break; +#endif + default: + error_report("Unknown vhost backend type"); + r = -1; + } + + return r; +} + static struct vhost_log *vhost_log_alloc(uint64_t size, bool share) { Error *err = NULL; @@ -286,7 +315,7 @@ static int vhost_dev_has_iommu(struct vhost_dev *dev) * does not have IOMMU, there's no need to enable this feature * which may cause unnecessary IOTLB miss/update trnasactions. */ - return vdev->dma_as != &address_space_memory && + return virtio_bus_device_iommu_enabled(vdev) && virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM); } diff --git a/hw/virtio/virtio-balloon.c b/hw/virtio/virtio-balloon.c index 4b5d9e5e50..5a69dce35d 100644 --- a/hw/virtio/virtio-balloon.c +++ b/hw/virtio/virtio-balloon.c @@ -30,6 +30,7 @@ #include "trace.h" #include "qemu/error-report.h" #include "migration/misc.h" +#include "migration/migration.h" #include "hw/virtio/virtio-bus.h" #include "hw/virtio/virtio-access.h" @@ -533,22 +534,18 @@ static bool get_free_page_hints(VirtIOBalloon *dev) if (dev->free_page_hint_status == FREE_PAGE_HINT_S_REQUESTED && id == dev->free_page_hint_cmd_id) { dev->free_page_hint_status = FREE_PAGE_HINT_S_START; - } else { + } else if (dev->free_page_hint_status == FREE_PAGE_HINT_S_START) { /* * Stop the optimization only when it has started. This * avoids a stale stop sign for the previous command. */ - if (dev->free_page_hint_status == FREE_PAGE_HINT_S_START) { - dev->free_page_hint_status = FREE_PAGE_HINT_S_STOP; - } + dev->free_page_hint_status = FREE_PAGE_HINT_S_STOP; } } - if (elem->in_num) { - if (dev->free_page_hint_status == FREE_PAGE_HINT_S_START) { - qemu_guest_free_page_hint(elem->in_sg[0].iov_base, - elem->in_sg[0].iov_len); - } + if (elem->in_num && dev->free_page_hint_status == FREE_PAGE_HINT_S_START) { + qemu_guest_free_page_hint(elem->in_sg[0].iov_base, + elem->in_sg[0].iov_len); } out: @@ -591,16 +588,10 @@ static void virtio_balloon_free_page_start(VirtIOBalloon *s) { VirtIODevice *vdev = VIRTIO_DEVICE(s); - /* For the stop and copy phase, we don't need to start the optimization */ - if (!vdev->vm_running) { - return; - } - qemu_mutex_lock(&s->free_page_lock); if (s->free_page_hint_cmd_id == UINT_MAX) { - s->free_page_hint_cmd_id = - VIRTIO_BALLOON_FREE_PAGE_HINT_CMD_ID_MIN; + s->free_page_hint_cmd_id = VIRTIO_BALLOON_FREE_PAGE_HINT_CMD_ID_MIN; } else { s->free_page_hint_cmd_id++; } @@ -648,8 +639,7 @@ static void virtio_balloon_free_page_done(VirtIOBalloon *s) static int virtio_balloon_free_page_hint_notify(NotifierWithReturn *n, void *data) { - VirtIOBalloon *dev = container_of(n, VirtIOBalloon, - free_page_hint_notify); + VirtIOBalloon *dev = container_of(n, VirtIOBalloon, free_page_hint_notify); VirtIODevice *vdev = VIRTIO_DEVICE(dev); PrecopyNotifyData *pnd = data; @@ -662,6 +652,18 @@ virtio_balloon_free_page_hint_notify(NotifierWithReturn *n, void *data) return 0; } + /* + * Pages hinted via qemu_guest_free_page_hint() are cleared from the dirty + * bitmap and will not get migrated, especially also not when the postcopy + * destination starts using them and requests migration from the source; the + * faulting thread will stall until postcopy migration finishes and + * all threads are woken up. Let's not start free page hinting if postcopy + * is possible. + */ + if (migrate_postcopy_ram()) { + return 0; + } + switch (pnd->reason) { case PRECOPY_NOTIFY_BEFORE_BITMAP_SYNC: virtio_balloon_free_page_stop(dev); @@ -906,8 +908,7 @@ static void virtio_balloon_device_realize(DeviceState *dev, Error **errp) s->dvq = virtio_add_queue(vdev, 128, virtio_balloon_handle_output); s->svq = virtio_add_queue(vdev, 128, virtio_balloon_receive_stats); - if (virtio_has_feature(s->host_features, - VIRTIO_BALLOON_F_FREE_PAGE_HINT)) { + if (virtio_has_feature(s->host_features, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) { s->free_page_vq = virtio_add_queue(vdev, VIRTQUEUE_MAX_SIZE, virtio_balloon_handle_free_page_vq); precopy_add_notifier(&s->free_page_hint_notify); diff --git a/hw/virtio/virtio-bus.c b/hw/virtio/virtio-bus.c index 859978d248..d23db98c56 100644 --- a/hw/virtio/virtio-bus.c +++ b/hw/virtio/virtio-bus.c @@ -325,6 +325,20 @@ static char *virtio_bus_get_fw_dev_path(DeviceState *dev) return NULL; } +bool virtio_bus_device_iommu_enabled(VirtIODevice *vdev) +{ + DeviceState *qdev = DEVICE(vdev); + BusState *qbus = BUS(qdev_get_parent_bus(qdev)); + VirtioBusState *bus = VIRTIO_BUS(qbus); + VirtioBusClass *klass = VIRTIO_BUS_GET_CLASS(bus); + + if (!klass->iommu_enabled) { + return false; + } + + return klass->iommu_enabled(qbus->parent); +} + static void virtio_bus_class_init(ObjectClass *klass, void *data) { BusClass *bus_class = BUS_CLASS(klass); diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c index 433060ac02..6e16e2705c 100644 --- a/hw/virtio/virtio-pci.c +++ b/hw/virtio/virtio-pci.c @@ -1121,6 +1121,19 @@ static AddressSpace *virtio_pci_get_dma_as(DeviceState *d) return pci_get_address_space(dev); } +static bool virtio_pci_iommu_enabled(DeviceState *d) +{ + VirtIOPCIProxy *proxy = VIRTIO_PCI(d); + PCIDevice *dev = &proxy->pci_dev; + AddressSpace *dma_as = pci_device_iommu_address_space(dev); + + if (dma_as == &address_space_memory) { + return false; + } + + return true; +} + static bool virtio_pci_queue_enabled(DeviceState *d, int n) { VirtIOPCIProxy *proxy = VIRTIO_PCI(d); @@ -2202,6 +2215,7 @@ static void virtio_pci_bus_class_init(ObjectClass *klass, void *data) k->ioeventfd_enabled = virtio_pci_ioeventfd_enabled; k->ioeventfd_assign = virtio_pci_ioeventfd_assign; k->get_dma_as = virtio_pci_get_dma_as; + k->iommu_enabled = virtio_pci_iommu_enabled; k->queue_enabled = virtio_pci_queue_enabled; } diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c index 874377f37a..3a1f6c520c 100644 --- a/hw/virtio/virtio.c +++ b/hw/virtio/virtio.c @@ -133,12 +133,10 @@ struct VirtQueue QLIST_ENTRY(VirtQueue) node; }; +/* Called within call_rcu(). */ static void virtio_free_region_cache(VRingMemoryRegionCaches *caches) { - if (!caches) { - return; - } - + assert(caches != NULL); address_space_cache_destroy(&caches->desc); address_space_cache_destroy(&caches->avail); address_space_cache_destroy(&caches->used); @@ -634,6 +632,7 @@ static int virtio_queue_split_empty(VirtQueue *vq) return empty; } +/* Called within rcu_read_lock(). */ static int virtio_queue_packed_empty_rcu(VirtQueue *vq) { struct VRingPackedDesc desc; |