diff options
Diffstat (limited to 'hw/block')
| -rw-r--r-- | hw/block/dataplane/virtio-blk.c | 165 | ||||
| -rw-r--r-- | hw/block/dataplane/virtio-blk.h | 3 | ||||
| -rw-r--r-- | hw/block/dataplane/xen-block.c | 17 | ||||
| -rw-r--r-- | hw/block/virtio-blk.c | 209 |
4 files changed, 250 insertions, 144 deletions
diff --git a/hw/block/dataplane/virtio-blk.c b/hw/block/dataplane/virtio-blk.c index f83bb0f116..6debd4401e 100644 --- a/hw/block/dataplane/virtio-blk.c +++ b/hw/block/dataplane/virtio-blk.c @@ -32,13 +32,11 @@ struct VirtIOBlockDataPlane { VirtIOBlkConf *conf; VirtIODevice *vdev; - /* Note that these EventNotifiers are assigned by value. This is - * fine as long as you do not call event_notifier_cleanup on them - * (because you don't own the file descriptor or handle; you just - * use it). + /* + * The AioContext for each virtqueue. The BlockDriverState will use the + * first element as its AioContext. */ - IOThread *iothread; - AioContext *ctx; + AioContext **vq_aio_context; }; /* Raise an interrupt to signal guest, if necessary */ @@ -47,6 +45,45 @@ void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s, VirtQueue *vq) virtio_notify_irqfd(s->vdev, vq); } +/* Generate vq:AioContext mappings from a validated iothread-vq-mapping list */ +static void +apply_vq_mapping(IOThreadVirtQueueMappingList *iothread_vq_mapping_list, + AioContext **vq_aio_context, uint16_t num_queues) +{ + IOThreadVirtQueueMappingList *node; + size_t num_iothreads = 0; + size_t cur_iothread = 0; + + for (node = iothread_vq_mapping_list; node; node = node->next) { + num_iothreads++; + } + + for (node = iothread_vq_mapping_list; node; node = node->next) { + IOThread *iothread = iothread_by_id(node->value->iothread); + AioContext *ctx = iothread_get_aio_context(iothread); + + /* Released in virtio_blk_data_plane_destroy() */ + object_ref(OBJECT(iothread)); + + if (node->value->vqs) { + uint16List *vq; + + /* Explicit vq:IOThread assignment */ + for (vq = node->value->vqs; vq; vq = vq->next) { + vq_aio_context[vq->value] = ctx; + } + } else { + /* Round-robin vq:IOThread assignment */ + for (unsigned i = cur_iothread; i < num_queues; + i += num_iothreads) { + vq_aio_context[i] = ctx; + } + } + + cur_iothread++; + } +} + /* Context: QEMU global mutex held */ bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, VirtIOBlockDataPlane **dataplane, @@ -58,7 +95,7 @@ bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, *dataplane = NULL; - if (conf->iothread) { + if (conf->iothread || conf->iothread_vq_mapping_list) { if (!k->set_guest_notifiers || !k->ioeventfd_assign) { error_setg(errp, "device is incompatible with iothread " @@ -86,13 +123,24 @@ bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, s = g_new0(VirtIOBlockDataPlane, 1); s->vdev = vdev; s->conf = conf; + s->vq_aio_context = g_new(AioContext *, conf->num_queues); + + if (conf->iothread_vq_mapping_list) { + apply_vq_mapping(conf->iothread_vq_mapping_list, s->vq_aio_context, + conf->num_queues); + } else if (conf->iothread) { + AioContext *ctx = iothread_get_aio_context(conf->iothread); + for (unsigned i = 0; i < conf->num_queues; i++) { + s->vq_aio_context[i] = ctx; + } - if (conf->iothread) { - s->iothread = conf->iothread; - object_ref(OBJECT(s->iothread)); - s->ctx = iothread_get_aio_context(s->iothread); + /* Released in virtio_blk_data_plane_destroy() */ + object_ref(OBJECT(conf->iothread)); } else { - s->ctx = qemu_get_aio_context(); + AioContext *ctx = qemu_get_aio_context(); + for (unsigned i = 0; i < conf->num_queues; i++) { + s->vq_aio_context[i] = ctx; + } } *dataplane = s; @@ -104,6 +152,7 @@ bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s) { VirtIOBlock *vblk; + VirtIOBlkConf *conf = s->conf; if (!s) { return; @@ -111,9 +160,21 @@ void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s) vblk = VIRTIO_BLK(s->vdev); assert(!vblk->dataplane_started); - if (s->iothread) { - object_unref(OBJECT(s->iothread)); + + if (conf->iothread_vq_mapping_list) { + IOThreadVirtQueueMappingList *node; + + for (node = conf->iothread_vq_mapping_list; node; node = node->next) { + IOThread *iothread = iothread_by_id(node->value->iothread); + object_unref(OBJECT(iothread)); + } + } + + if (conf->iothread) { + object_unref(OBJECT(conf->iothread)); } + + g_free(s->vq_aio_context); g_free(s); } @@ -124,7 +185,6 @@ int virtio_blk_data_plane_start(VirtIODevice *vdev) VirtIOBlockDataPlane *s = vblk->dataplane; BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vblk))); VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); - AioContext *old_context; unsigned i; unsigned nvqs = s->conf->num_queues; Error *local_err = NULL; @@ -178,22 +238,13 @@ int virtio_blk_data_plane_start(VirtIODevice *vdev) trace_virtio_blk_data_plane_start(s); - old_context = blk_get_aio_context(s->conf->conf.blk); - aio_context_acquire(old_context); - r = blk_set_aio_context(s->conf->conf.blk, s->ctx, &local_err); - aio_context_release(old_context); + r = blk_set_aio_context(s->conf->conf.blk, s->vq_aio_context[0], + &local_err); if (r < 0) { error_report_err(local_err); goto fail_aio_context; } - /* Kick right away to begin processing requests already in vring */ - for (i = 0; i < nvqs; i++) { - VirtQueue *vq = virtio_get_queue(s->vdev, i); - - event_notifier_set(virtio_queue_get_host_notifier(vq)); - } - /* * These fields must be visible to the IOThread when it processes the * virtqueue, otherwise it will think dataplane has not started yet. @@ -208,13 +259,15 @@ int virtio_blk_data_plane_start(VirtIODevice *vdev) /* Get this show started by hooking up our callbacks */ if (!blk_in_drain(s->conf->conf.blk)) { - aio_context_acquire(s->ctx); for (i = 0; i < nvqs; i++) { VirtQueue *vq = virtio_get_queue(s->vdev, i); + AioContext *ctx = s->vq_aio_context[i]; + + /* Kick right away to begin processing requests already in vring */ + event_notifier_set(virtio_queue_get_host_notifier(vq)); - virtio_queue_aio_attach_host_notifier(vq, s->ctx); + virtio_queue_aio_attach_host_notifier(vq, ctx); } - aio_context_release(s->ctx); } return 0; @@ -242,23 +295,18 @@ int virtio_blk_data_plane_start(VirtIODevice *vdev) * * Context: BH in IOThread */ -static void virtio_blk_data_plane_stop_bh(void *opaque) +static void virtio_blk_data_plane_stop_vq_bh(void *opaque) { - VirtIOBlockDataPlane *s = opaque; - unsigned i; - - for (i = 0; i < s->conf->num_queues; i++) { - VirtQueue *vq = virtio_get_queue(s->vdev, i); - EventNotifier *host_notifier = virtio_queue_get_host_notifier(vq); + VirtQueue *vq = opaque; + EventNotifier *host_notifier = virtio_queue_get_host_notifier(vq); - virtio_queue_aio_detach_host_notifier(vq, s->ctx); + virtio_queue_aio_detach_host_notifier(vq, qemu_get_current_aio_context()); - /* - * Test and clear notifier after disabling event, in case poll callback - * didn't have time to run. - */ - virtio_queue_host_notifier_read(host_notifier); - } + /* + * Test and clear notifier after disabling event, in case poll callback + * didn't have time to run. + */ + virtio_queue_host_notifier_read(host_notifier); } /* Context: QEMU global mutex held */ @@ -285,7 +333,12 @@ void virtio_blk_data_plane_stop(VirtIODevice *vdev) trace_virtio_blk_data_plane_stop(s); if (!blk_in_drain(s->conf->conf.blk)) { - aio_wait_bh_oneshot(s->ctx, virtio_blk_data_plane_stop_bh, s); + for (i = 0; i < nvqs; i++) { + VirtQueue *vq = virtio_get_queue(s->vdev, i); + AioContext *ctx = s->vq_aio_context[i]; + + aio_wait_bh_oneshot(ctx, virtio_blk_data_plane_stop_vq_bh, vq); + } } /* @@ -314,8 +367,6 @@ void virtio_blk_data_plane_stop(VirtIODevice *vdev) */ vblk->dataplane_started = false; - aio_context_acquire(s->ctx); - /* Wait for virtio_blk_dma_restart_bh() and in flight I/O to complete */ blk_drain(s->conf->conf.blk); @@ -325,10 +376,28 @@ void virtio_blk_data_plane_stop(VirtIODevice *vdev) */ blk_set_aio_context(s->conf->conf.blk, qemu_get_aio_context(), NULL); - aio_context_release(s->ctx); - /* Clean up guest notifier (irq) */ k->set_guest_notifiers(qbus->parent, nvqs, false); s->stopping = false; } + +void virtio_blk_data_plane_detach(VirtIOBlockDataPlane *s) +{ + VirtIODevice *vdev = VIRTIO_DEVICE(s->vdev); + + for (uint16_t i = 0; i < s->conf->num_queues; i++) { + VirtQueue *vq = virtio_get_queue(vdev, i); + virtio_queue_aio_detach_host_notifier(vq, s->vq_aio_context[i]); + } +} + +void virtio_blk_data_plane_attach(VirtIOBlockDataPlane *s) +{ + VirtIODevice *vdev = VIRTIO_DEVICE(s->vdev); + + for (uint16_t i = 0; i < s->conf->num_queues; i++) { + VirtQueue *vq = virtio_get_queue(vdev, i); + virtio_queue_aio_attach_host_notifier(vq, s->vq_aio_context[i]); + } +} diff --git a/hw/block/dataplane/virtio-blk.h b/hw/block/dataplane/virtio-blk.h index 5e18bb99ae..1a806fe447 100644 --- a/hw/block/dataplane/virtio-blk.h +++ b/hw/block/dataplane/virtio-blk.h @@ -28,4 +28,7 @@ void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s, VirtQueue *vq); int virtio_blk_data_plane_start(VirtIODevice *vdev); void virtio_blk_data_plane_stop(VirtIODevice *vdev); +void virtio_blk_data_plane_detach(VirtIOBlockDataPlane *s); +void virtio_blk_data_plane_attach(VirtIOBlockDataPlane *s); + #endif /* HW_DATAPLANE_VIRTIO_BLK_H */ diff --git a/hw/block/dataplane/xen-block.c b/hw/block/dataplane/xen-block.c index c4bb28c66f..98501e6885 100644 --- a/hw/block/dataplane/xen-block.c +++ b/hw/block/dataplane/xen-block.c @@ -260,8 +260,6 @@ static void xen_block_complete_aio(void *opaque, int ret) XenBlockRequest *request = opaque; XenBlockDataPlane *dataplane = request->dataplane; - aio_context_acquire(dataplane->ctx); - if (ret != 0) { error_report("%s I/O error", request->req.operation == BLKIF_OP_READ ? @@ -273,10 +271,10 @@ static void xen_block_complete_aio(void *opaque, int ret) if (request->presync) { request->presync = 0; xen_block_do_aio(request); - goto done; + return; } if (request->aio_inflight > 0) { - goto done; + return; } switch (request->req.operation) { @@ -318,9 +316,6 @@ static void xen_block_complete_aio(void *opaque, int ret) if (dataplane->more_work) { qemu_bh_schedule(dataplane->bh); } - -done: - aio_context_release(dataplane->ctx); } static bool xen_block_split_discard(XenBlockRequest *request, @@ -601,9 +596,7 @@ static void xen_block_dataplane_bh(void *opaque) { XenBlockDataPlane *dataplane = opaque; - aio_context_acquire(dataplane->ctx); xen_block_handle_requests(dataplane); - aio_context_release(dataplane->ctx); } static bool xen_block_dataplane_event(void *opaque) @@ -703,10 +696,8 @@ void xen_block_dataplane_stop(XenBlockDataPlane *dataplane) xen_block_dataplane_detach(dataplane); } - aio_context_acquire(dataplane->ctx); /* Xen doesn't have multiple users for nodes, so this can't fail */ blk_set_aio_context(dataplane->blk, qemu_get_aio_context(), &error_abort); - aio_context_release(dataplane->ctx); /* * Now that the context has been moved onto the main thread, cancel @@ -752,7 +743,6 @@ void xen_block_dataplane_start(XenBlockDataPlane *dataplane, { ERRP_GUARD(); XenDevice *xendev = dataplane->xendev; - AioContext *old_context; unsigned int ring_size; unsigned int i; @@ -836,11 +826,8 @@ void xen_block_dataplane_start(XenBlockDataPlane *dataplane, goto stop; } - old_context = blk_get_aio_context(dataplane->blk); - aio_context_acquire(old_context); /* If other users keep the BlockBackend in the iothread, that's ok */ blk_set_aio_context(dataplane->blk, dataplane->ctx, NULL); - aio_context_release(old_context); if (!blk_in_drain(dataplane->blk)) { xen_block_dataplane_attach(dataplane); diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c index a1f8e15522..46e73b2c96 100644 --- a/hw/block/virtio-blk.c +++ b/hw/block/virtio-blk.c @@ -82,8 +82,11 @@ static int virtio_blk_handle_rw_error(VirtIOBlockReq *req, int error, /* Break the link as the next request is going to be parsed from the * ring again. Otherwise we may end up doing a double completion! */ req->mr_next = NULL; - req->next = s->rq; - s->rq = req; + + WITH_QEMU_LOCK_GUARD(&s->rq_lock) { + req->next = s->rq; + s->rq = req; + } } else if (action == BLOCK_ERROR_ACTION_REPORT) { virtio_blk_req_complete(req, VIRTIO_BLK_S_IOERR); if (acct_failed) { @@ -102,7 +105,6 @@ static void virtio_blk_rw_complete(void *opaque, int ret) VirtIOBlock *s = next->dev; VirtIODevice *vdev = VIRTIO_DEVICE(s); - aio_context_acquire(blk_get_aio_context(s->conf.conf.blk)); while (next) { VirtIOBlockReq *req = next; next = req->mr_next; @@ -135,7 +137,6 @@ static void virtio_blk_rw_complete(void *opaque, int ret) block_acct_done(blk_get_stats(s->blk), &req->acct); virtio_blk_free_request(req); } - aio_context_release(blk_get_aio_context(s->conf.conf.blk)); } static void virtio_blk_flush_complete(void *opaque, int ret) @@ -143,19 +144,13 @@ static void virtio_blk_flush_complete(void *opaque, int ret) VirtIOBlockReq *req = opaque; VirtIOBlock *s = req->dev; - aio_context_acquire(blk_get_aio_context(s->conf.conf.blk)); - if (ret) { - if (virtio_blk_handle_rw_error(req, -ret, 0, true)) { - goto out; - } + if (ret && virtio_blk_handle_rw_error(req, -ret, 0, true)) { + return; } virtio_blk_req_complete(req, VIRTIO_BLK_S_OK); block_acct_done(blk_get_stats(s->blk), &req->acct); virtio_blk_free_request(req); - -out: - aio_context_release(blk_get_aio_context(s->conf.conf.blk)); } static void virtio_blk_discard_write_zeroes_complete(void *opaque, int ret) @@ -165,11 +160,8 @@ static void virtio_blk_discard_write_zeroes_complete(void *opaque, int ret) bool is_write_zeroes = (virtio_ldl_p(VIRTIO_DEVICE(s), &req->out.type) & ~VIRTIO_BLK_T_BARRIER) == VIRTIO_BLK_T_WRITE_ZEROES; - aio_context_acquire(blk_get_aio_context(s->conf.conf.blk)); - if (ret) { - if (virtio_blk_handle_rw_error(req, -ret, false, is_write_zeroes)) { - goto out; - } + if (ret && virtio_blk_handle_rw_error(req, -ret, false, is_write_zeroes)) { + return; } virtio_blk_req_complete(req, VIRTIO_BLK_S_OK); @@ -177,9 +169,6 @@ static void virtio_blk_discard_write_zeroes_complete(void *opaque, int ret) block_acct_done(blk_get_stats(s->blk), &req->acct); } virtio_blk_free_request(req); - -out: - aio_context_release(blk_get_aio_context(s->conf.conf.blk)); } #ifdef __linux__ @@ -226,10 +215,8 @@ static void virtio_blk_ioctl_complete(void *opaque, int status) virtio_stl_p(vdev, &scsi->data_len, hdr->dxfer_len); out: - aio_context_acquire(blk_get_aio_context(s->conf.conf.blk)); virtio_blk_req_complete(req, status); virtio_blk_free_request(req); - aio_context_release(blk_get_aio_context(s->conf.conf.blk)); g_free(ioctl_req); } @@ -669,7 +656,6 @@ static void virtio_blk_zone_report_complete(void *opaque, int ret) { ZoneCmdData *data = opaque; VirtIOBlockReq *req = data->req; - VirtIOBlock *s = req->dev; VirtIODevice *vdev = VIRTIO_DEVICE(req->dev); struct iovec *in_iov = data->in_iov; unsigned in_num = data->in_num; @@ -760,10 +746,8 @@ static void virtio_blk_zone_report_complete(void *opaque, int ret) } out: - aio_context_acquire(blk_get_aio_context(s->conf.conf.blk)); virtio_blk_req_complete(req, err_status); virtio_blk_free_request(req); - aio_context_release(blk_get_aio_context(s->conf.conf.blk)); g_free(data->zone_report_data.zones); g_free(data); } @@ -826,10 +810,8 @@ static void virtio_blk_zone_mgmt_complete(void *opaque, int ret) err_status = VIRTIO_BLK_S_ZONE_INVALID_CMD; } - aio_context_acquire(blk_get_aio_context(s->conf.conf.blk)); virtio_blk_req_complete(req, err_status); virtio_blk_free_request(req); - aio_context_release(blk_get_aio_context(s->conf.conf.blk)); } static int virtio_blk_handle_zone_mgmt(VirtIOBlockReq *req, BlockZoneOp op) @@ -879,7 +861,6 @@ static void virtio_blk_zone_append_complete(void *opaque, int ret) { ZoneCmdData *data = opaque; VirtIOBlockReq *req = data->req; - VirtIOBlock *s = req->dev; VirtIODevice *vdev = VIRTIO_DEVICE(req->dev); int64_t append_sector, n; uint8_t err_status = VIRTIO_BLK_S_OK; @@ -902,10 +883,8 @@ static void virtio_blk_zone_append_complete(void *opaque, int ret) trace_virtio_blk_zone_append_complete(vdev, req, append_sector, ret); out: - aio_context_acquire(blk_get_aio_context(s->conf.conf.blk)); virtio_blk_req_complete(req, err_status); virtio_blk_free_request(req); - aio_context_release(blk_get_aio_context(s->conf.conf.blk)); g_free(data); } @@ -941,10 +920,8 @@ static int virtio_blk_handle_zone_append(VirtIOBlockReq *req, return 0; out: - aio_context_acquire(blk_get_aio_context(s->conf.conf.blk)); virtio_blk_req_complete(req, err_status); virtio_blk_free_request(req); - aio_context_release(blk_get_aio_context(s->conf.conf.blk)); return err_status; } @@ -1134,7 +1111,6 @@ void virtio_blk_handle_vq(VirtIOBlock *s, VirtQueue *vq) MultiReqBuffer mrb = {}; bool suppress_notifications = virtio_queue_get_notification(vq); - aio_context_acquire(blk_get_aio_context(s->blk)); defer_call_begin(); do { @@ -1160,7 +1136,6 @@ void virtio_blk_handle_vq(VirtIOBlock *s, VirtQueue *vq) } defer_call_end(); - aio_context_release(blk_get_aio_context(s->blk)); } static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq) @@ -1176,6 +1151,7 @@ static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq) return; } } + virtio_blk_handle_vq(s, vq); } @@ -1183,12 +1159,14 @@ static void virtio_blk_dma_restart_bh(void *opaque) { VirtIOBlock *s = opaque; - VirtIOBlockReq *req = s->rq; + VirtIOBlockReq *req; MultiReqBuffer mrb = {}; - s->rq = NULL; + WITH_QEMU_LOCK_GUARD(&s->rq_lock) { + req = s->rq; + s->rq = NULL; + } - aio_context_acquire(blk_get_aio_context(s->conf.conf.blk)); while (req) { VirtIOBlockReq *next = req->next; if (virtio_blk_handle_request(req, &mrb)) { @@ -1212,8 +1190,6 @@ static void virtio_blk_dma_restart_bh(void *opaque) /* Paired with inc in virtio_blk_dma_restart_cb() */ blk_dec_in_flight(s->conf.conf.blk); - - aio_context_release(blk_get_aio_context(s->conf.conf.blk)); } static void virtio_blk_dma_restart_cb(void *opaque, bool running, @@ -1235,25 +1211,28 @@ static void virtio_blk_dma_restart_cb(void *opaque, bool running, static void virtio_blk_reset(VirtIODevice *vdev) { VirtIOBlock *s = VIRTIO_BLK(vdev); - AioContext *ctx; VirtIOBlockReq *req; - ctx = blk_get_aio_context(s->blk); - aio_context_acquire(ctx); + /* Dataplane has stopped... */ + assert(!s->dataplane_started); + + /* ...but requests may still be in flight. */ blk_drain(s->blk); /* We drop queued requests after blk_drain() because blk_drain() itself can * produce them. */ - while (s->rq) { - req = s->rq; - s->rq = req->next; - virtqueue_detach_element(req->vq, &req->elem, 0); - virtio_blk_free_request(req); - } + WITH_QEMU_LOCK_GUARD(&s->rq_lock) { + while (s->rq) { + req = s->rq; + s->rq = req->next; - aio_context_release(ctx); + /* No other threads can access req->vq here */ + virtqueue_detach_element(req->vq, &req->elem, 0); + + virtio_blk_free_request(req); + } + } - assert(!s->dataplane_started); blk_set_enable_write_cache(s->blk, s->original_wce); } @@ -1268,10 +1247,6 @@ static void virtio_blk_update_config(VirtIODevice *vdev, uint8_t *config) uint64_t capacity; int64_t length; int blk_size = conf->logical_block_size; - AioContext *ctx; - - ctx = blk_get_aio_context(s->blk); - aio_context_acquire(ctx); blk_get_geometry(s->blk, &capacity); memset(&blkcfg, 0, sizeof(blkcfg)); @@ -1295,7 +1270,6 @@ static void virtio_blk_update_config(VirtIODevice *vdev, uint8_t *config) * per track (cylinder). */ length = blk_getlength(s->blk); - aio_context_release(ctx); if (length > 0 && length / conf->heads / conf->secs % blk_size) { blkcfg.geometry.sectors = conf->secs & ~s->sector_mask; } else { @@ -1362,9 +1336,7 @@ static void virtio_blk_set_config(VirtIODevice *vdev, const uint8_t *config) memcpy(&blkcfg, config, s->config_size); - aio_context_acquire(blk_get_aio_context(s->blk)); blk_set_enable_write_cache(s->blk, blkcfg.wce != 0); - aio_context_release(blk_get_aio_context(s->blk)); } static uint64_t virtio_blk_get_features(VirtIODevice *vdev, uint64_t features, @@ -1432,29 +1404,31 @@ static void virtio_blk_set_status(VirtIODevice *vdev, uint8_t status) * s->blk would erroneously be placed in writethrough mode. */ if (!virtio_vdev_has_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE)) { - aio_context_acquire(blk_get_aio_context(s->blk)); blk_set_enable_write_cache(s->blk, virtio_vdev_has_feature(vdev, VIRTIO_BLK_F_WCE)); - aio_context_release(blk_get_aio_context(s->blk)); } } static void virtio_blk_save_device(VirtIODevice *vdev, QEMUFile *f) { VirtIOBlock *s = VIRTIO_BLK(vdev); - VirtIOBlockReq *req = s->rq; - while (req) { - qemu_put_sbyte(f, 1); + WITH_QEMU_LOCK_GUARD(&s->rq_lock) { + VirtIOBlockReq *req = s->rq; - if (s->conf.num_queues > 1) { - qemu_put_be32(f, virtio_get_queue_index(req->vq)); - } + while (req) { + qemu_put_sbyte(f, 1); + + if (s->conf.num_queues > 1) { + qemu_put_be32(f, virtio_get_queue_index(req->vq)); + } - qemu_put_virtqueue_element(vdev, f, &req->elem); - req = req->next; + qemu_put_virtqueue_element(vdev, f, &req->elem); + req = req->next; + } } + qemu_put_sbyte(f, 0); } @@ -1480,13 +1454,78 @@ static int virtio_blk_load_device(VirtIODevice *vdev, QEMUFile *f, req = qemu_get_virtqueue_element(vdev, f, sizeof(VirtIOBlockReq)); virtio_blk_init_request(s, virtio_get_queue(vdev, vq_idx), req); - req->next = s->rq; - s->rq = req; + + WITH_QEMU_LOCK_GUARD(&s->rq_lock) { + req->next = s->rq; + s->rq = req; + } } return 0; } +static bool +validate_iothread_vq_mapping_list(IOThreadVirtQueueMappingList *list, + uint16_t num_queues, Error **errp) +{ + g_autofree unsigned long *vqs = bitmap_new(num_queues); + g_autoptr(GHashTable) iothreads = + g_hash_table_new(g_str_hash, g_str_equal); + + for (IOThreadVirtQueueMappingList *node = list; node; node = node->next) { + const char *name = node->value->iothread; + uint16List *vq; + + if (!iothread_by_id(name)) { + error_setg(errp, "IOThread \"%s\" object does not exist", name); + return false; + } + + if (!g_hash_table_add(iothreads, (gpointer)name)) { + error_setg(errp, + "duplicate IOThread name \"%s\" in iothread-vq-mapping", + name); + return false; + } + + if (node != list) { + if (!!node->value->vqs != !!list->value->vqs) { + error_setg(errp, "either all items in iothread-vq-mapping " + "must have vqs or none of them must have it"); + return false; + } + } + + for (vq = node->value->vqs; vq; vq = vq->next) { + if (vq->value >= num_queues) { + error_setg(errp, "vq index %u for IOThread \"%s\" must be " + "less than num_queues %u in iothread-vq-mapping", + vq->value, name, num_queues); + return false; + } + + if (test_and_set_bit(vq->value, vqs)) { + error_setg(errp, "cannot assign vq %u to IOThread \"%s\" " + "because it is already assigned", vq->value, name); + return false; + } + } + } + + if (list->value->vqs) { + for (uint16_t i = 0; i < num_queues; i++) { + if (!test_bit(i, vqs)) { + error_setg(errp, + "missing vq %u IOThread assignment in iothread-vq-mapping", + i); + return false; + } + } + } + + return true; +} + static void virtio_resize_cb(void *opaque) { VirtIODevice *vdev = opaque; @@ -1511,34 +1550,24 @@ static void virtio_blk_resize(void *opaque) static void virtio_blk_drained_begin(void *opaque) { VirtIOBlock *s = opaque; - VirtIODevice *vdev = VIRTIO_DEVICE(opaque); - AioContext *ctx = blk_get_aio_context(s->conf.conf.blk); if (!s->dataplane || !s->dataplane_started) { return; } - for (uint16_t i = 0; i < s->conf.num_queues; i++) { - VirtQueue *vq = virtio_get_queue(vdev, i); - virtio_queue_aio_detach_host_notifier(vq, ctx); - } + virtio_blk_data_plane_detach(s->dataplane); } /* Resume virtqueue ioeventfd processing after drain */ static void virtio_blk_drained_end(void *opaque) { VirtIOBlock *s = opaque; - VirtIODevice *vdev = VIRTIO_DEVICE(opaque); - AioContext *ctx = blk_get_aio_context(s->conf.conf.blk); if (!s->dataplane || !s->dataplane_started) { return; } - for (uint16_t i = 0; i < s->conf.num_queues; i++) { - VirtQueue *vq = virtio_get_queue(vdev, i); - virtio_queue_aio_attach_host_notifier(vq, ctx); - } + virtio_blk_data_plane_attach(s->dataplane); } static const BlockDevOps virtio_block_ops = { @@ -1624,10 +1653,25 @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp) return; } + if (conf->iothread_vq_mapping_list) { + if (conf->iothread) { + error_setg(errp, "iothread and iothread-vq-mapping properties " + "cannot be set at the same time"); + return; + } + + if (!validate_iothread_vq_mapping_list(conf->iothread_vq_mapping_list, + conf->num_queues, errp)) { + return; + } + } + s->config_size = virtio_get_config_size(&virtio_blk_cfg_size_params, s->host_features); virtio_init(vdev, VIRTIO_ID_BLOCK, s->config_size); + qemu_mutex_init(&s->rq_lock); + s->blk = conf->conf.blk; s->rq = NULL; s->sector_mask = (s->conf.conf.logical_block_size / BDRV_SECTOR_SIZE) - 1; @@ -1679,6 +1723,7 @@ static void virtio_blk_device_unrealize(DeviceState *dev) virtio_del_queue(vdev, i); } qemu_coroutine_dec_pool_size(conf->num_queues * conf->queue_size / 2); + qemu_mutex_destroy(&s->rq_lock); blk_ram_registrar_destroy(&s->blk_ram_registrar); qemu_del_vm_change_state_handler(s->change); blockdev_mark_auto_del(s->blk); @@ -1723,6 +1768,8 @@ static Property virtio_blk_properties[] = { DEFINE_PROP_BOOL("seg-max-adjust", VirtIOBlock, conf.seg_max_adjust, true), DEFINE_PROP_LINK("iothread", VirtIOBlock, conf.iothread, TYPE_IOTHREAD, IOThread *), + DEFINE_PROP_IOTHREAD_VQ_MAPPING_LIST("iothread-vq-mapping", VirtIOBlock, + conf.iothread_vq_mapping_list), DEFINE_PROP_BIT64("discard", VirtIOBlock, host_features, VIRTIO_BLK_F_DISCARD, true), DEFINE_PROP_BOOL("report-discard-granularity", VirtIOBlock, |