diff options
Diffstat (limited to 'hw')
| -rw-r--r-- | hw/block/dataplane/virtio-blk.c | 246 | ||||
| -rw-r--r-- | hw/block/virtio-blk.c | 139 |
2 files changed, 99 insertions, 286 deletions
diff --git a/hw/block/dataplane/virtio-blk.c b/hw/block/dataplane/virtio-blk.c index c10b7b70fb..09bd2c70ab 100644 --- a/hw/block/dataplane/virtio-blk.c +++ b/hw/block/dataplane/virtio-blk.c @@ -24,16 +24,6 @@ #include "hw/virtio/virtio-bus.h" #include "qom/object_interfaces.h" -typedef struct { - VirtIOBlockDataPlane *s; - QEMUIOVector *inhdr; /* iovecs for virtio_blk_inhdr */ - VirtQueueElement *elem; /* saved data from the virtqueue */ - QEMUIOVector qiov; /* original request iovecs */ - struct iovec bounce_iov; /* used if guest buffers are unaligned */ - QEMUIOVector bounce_qiov; /* bounce buffer iovecs */ - bool read; /* read or write? */ -} VirtIOBlockRequest; - struct VirtIOBlockDataPlane { bool started; bool starting; @@ -57,6 +47,8 @@ struct VirtIOBlockDataPlane { /* Operation blocker on BDS */ Error *blocker; + void (*saved_complete_request)(struct VirtIOBlockReq *req, + unsigned char status); }; /* Raise an interrupt to signal guest, if necessary */ @@ -69,215 +61,14 @@ static void notify_guest(VirtIOBlockDataPlane *s) event_notifier_set(s->guest_notifier); } -static void complete_rdwr(void *opaque, int ret) -{ - VirtIOBlockRequest *req = opaque; - struct virtio_blk_inhdr hdr; - int len; - - if (likely(ret == 0)) { - hdr.status = VIRTIO_BLK_S_OK; - len = req->qiov.size; - } else { - hdr.status = VIRTIO_BLK_S_IOERR; - len = 0; - } - - trace_virtio_blk_data_plane_complete_request(req->s, req->elem->index, ret); - - if (req->read && req->bounce_iov.iov_base) { - qemu_iovec_from_buf(&req->qiov, 0, req->bounce_iov.iov_base, len); - } - - if (req->bounce_iov.iov_base) { - qemu_vfree(req->bounce_iov.iov_base); - } - - qemu_iovec_from_buf(req->inhdr, 0, &hdr, sizeof(hdr)); - qemu_iovec_destroy(req->inhdr); - g_slice_free(QEMUIOVector, req->inhdr); - - /* According to the virtio specification len should be the number of bytes - * written to, but for virtio-blk it seems to be the number of bytes - * transferred plus the status bytes. - */ - vring_push(&req->s->vring, req->elem, len + sizeof(hdr)); - notify_guest(req->s); - g_slice_free(VirtIOBlockRequest, req); -} - -static void complete_request_early(VirtIOBlockDataPlane *s, VirtQueueElement *elem, - QEMUIOVector *inhdr, unsigned char status) -{ - struct virtio_blk_inhdr hdr = { - .status = status, - }; - - qemu_iovec_from_buf(inhdr, 0, &hdr, sizeof(hdr)); - qemu_iovec_destroy(inhdr); - g_slice_free(QEMUIOVector, inhdr); - - vring_push(&s->vring, elem, sizeof(hdr)); - notify_guest(s); -} - -/* Get disk serial number */ -static void do_get_id_cmd(VirtIOBlockDataPlane *s, - struct iovec *iov, unsigned int iov_cnt, - VirtQueueElement *elem, QEMUIOVector *inhdr) -{ - char id[VIRTIO_BLK_ID_BYTES]; - - /* Serial number not NUL-terminated when longer than buffer */ - strncpy(id, s->blk->serial ? s->blk->serial : "", sizeof(id)); - iov_from_buf(iov, iov_cnt, 0, id, sizeof(id)); - complete_request_early(s, elem, inhdr, VIRTIO_BLK_S_OK); -} - -static void do_rdwr_cmd(VirtIOBlockDataPlane *s, bool read, - struct iovec *iov, unsigned iov_cnt, - int64_t sector_num, VirtQueueElement *elem, - QEMUIOVector *inhdr) -{ - VirtIOBlockRequest *req = g_slice_new0(VirtIOBlockRequest); - QEMUIOVector *qiov; - int nb_sectors; - - /* Fill in virtio block metadata needed for completion */ - req->s = s; - req->elem = elem; - req->inhdr = inhdr; - req->read = read; - qemu_iovec_init_external(&req->qiov, iov, iov_cnt); - - qiov = &req->qiov; - - if (!bdrv_qiov_is_aligned(s->blk->conf.bs, qiov)) { - void *bounce_buffer = qemu_blockalign(s->blk->conf.bs, qiov->size); - - /* Populate bounce buffer with data for writes */ - if (!read) { - qemu_iovec_to_buf(qiov, 0, bounce_buffer, qiov->size); - } - - /* Redirect I/O to aligned bounce buffer */ - req->bounce_iov.iov_base = bounce_buffer; - req->bounce_iov.iov_len = qiov->size; - qemu_iovec_init_external(&req->bounce_qiov, &req->bounce_iov, 1); - qiov = &req->bounce_qiov; - } - - nb_sectors = qiov->size / BDRV_SECTOR_SIZE; - - if (read) { - bdrv_aio_readv(s->blk->conf.bs, sector_num, qiov, nb_sectors, - complete_rdwr, req); - } else { - bdrv_aio_writev(s->blk->conf.bs, sector_num, qiov, nb_sectors, - complete_rdwr, req); - } -} - -static void complete_flush(void *opaque, int ret) -{ - VirtIOBlockRequest *req = opaque; - unsigned char status; - - if (ret == 0) { - status = VIRTIO_BLK_S_OK; - } else { - status = VIRTIO_BLK_S_IOERR; - } - - complete_request_early(req->s, req->elem, req->inhdr, status); - g_slice_free(VirtIOBlockRequest, req); -} - -static void do_flush_cmd(VirtIOBlockDataPlane *s, VirtQueueElement *elem, - QEMUIOVector *inhdr) +static void complete_request_vring(VirtIOBlockReq *req, unsigned char status) { - VirtIOBlockRequest *req = g_slice_new(VirtIOBlockRequest); - req->s = s; - req->elem = elem; - req->inhdr = inhdr; + stb_p(&req->in->status, status); - bdrv_aio_flush(s->blk->conf.bs, complete_flush, req); -} - -static void do_scsi_cmd(VirtIOBlockDataPlane *s, VirtQueueElement *elem, - QEMUIOVector *inhdr) -{ - int status; - - status = virtio_blk_handle_scsi_req(VIRTIO_BLK(s->vdev), elem); - complete_request_early(s, elem, inhdr, status); -} - -static int process_request(VirtIOBlockDataPlane *s, VirtQueueElement *elem) -{ - struct iovec *iov = elem->out_sg; - struct iovec *in_iov = elem->in_sg; - unsigned out_num = elem->out_num; - unsigned in_num = elem->in_num; - struct virtio_blk_outhdr outhdr; - QEMUIOVector *inhdr; - size_t in_size; - - /* Copy in outhdr */ - if (unlikely(iov_to_buf(iov, out_num, 0, &outhdr, - sizeof(outhdr)) != sizeof(outhdr))) { - error_report("virtio-blk request outhdr too short"); - return -EFAULT; - } - iov_discard_front(&iov, &out_num, sizeof(outhdr)); - - /* Grab inhdr for later */ - in_size = iov_size(in_iov, in_num); - if (in_size < sizeof(struct virtio_blk_inhdr)) { - error_report("virtio_blk request inhdr too short"); - return -EFAULT; - } - inhdr = g_slice_new(QEMUIOVector); - qemu_iovec_init(inhdr, 1); - qemu_iovec_concat_iov(inhdr, in_iov, in_num, - in_size - sizeof(struct virtio_blk_inhdr), - sizeof(struct virtio_blk_inhdr)); - iov_discard_back(in_iov, &in_num, sizeof(struct virtio_blk_inhdr)); - - /* TODO Linux sets the barrier bit even when not advertised! */ - outhdr.type &= ~VIRTIO_BLK_T_BARRIER; - - switch (outhdr.type) { - case VIRTIO_BLK_T_IN: - do_rdwr_cmd(s, true, in_iov, in_num, - outhdr.sector * 512 / BDRV_SECTOR_SIZE, - elem, inhdr); - return 0; - - case VIRTIO_BLK_T_OUT: - do_rdwr_cmd(s, false, iov, out_num, - outhdr.sector * 512 / BDRV_SECTOR_SIZE, - elem, inhdr); - return 0; - - case VIRTIO_BLK_T_SCSI_CMD: - do_scsi_cmd(s, elem, inhdr); - return 0; - - case VIRTIO_BLK_T_FLUSH: - do_flush_cmd(s, elem, inhdr); - return 0; - - case VIRTIO_BLK_T_GET_ID: - do_get_id_cmd(s, in_iov, in_num, elem, inhdr); - return 0; - - default: - error_report("virtio-blk unsupported request type %#x", outhdr.type); - qemu_iovec_destroy(inhdr); - g_slice_free(QEMUIOVector, inhdr); - return -EFAULT; - } + vring_push(&req->dev->dataplane->vring, req->elem, + req->qiov.size + sizeof(*req->in)); + notify_guest(req->dev->dataplane); + g_slice_free(VirtIOBlockReq, req); } static void handle_notify(EventNotifier *e) @@ -286,7 +77,11 @@ static void handle_notify(EventNotifier *e) host_notifier); VirtQueueElement *elem; + VirtIOBlockReq *req; int ret; + MultiReqBuffer mrb = { + .num_writes = 0, + }; event_notifier_test_and_clear(&s->host_notifier); for (;;) { @@ -303,14 +98,14 @@ static void handle_notify(EventNotifier *e) trace_virtio_blk_data_plane_process_request(s, elem->out_num, elem->in_num, elem->index); - if (process_request(s, elem) < 0) { - vring_set_broken(&s->vring); - vring_free_element(elem); - ret = -EFAULT; - break; - } + req = g_slice_new(VirtIOBlockReq); + req->dev = VIRTIO_BLK(s->vdev); + req->elem = elem; + virtio_blk_handle_request(req, &mrb); } + virtio_submit_multiwrite(s->blk->conf.bs, &mrb); + if (likely(ret == -EAGAIN)) { /* vring emptied */ /* Re-enable guest->host notifies and stop processing the vring. * But if the guest has snuck in more descriptors, keep processing. @@ -330,6 +125,7 @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *blk, Error **errp) { VirtIOBlockDataPlane *s; + VirtIOBlock *vblk = VIRTIO_BLK(vdev); Error *local_err = NULL; *dataplane = NULL; @@ -372,6 +168,8 @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *blk, bdrv_op_block_all(blk->conf.bs, s->blocker); *dataplane = s; + s->saved_complete_request = vblk->complete_request; + vblk->complete_request = complete_request_vring; } /* Context: QEMU global mutex held */ @@ -446,10 +244,12 @@ void virtio_blk_data_plane_stop(VirtIOBlockDataPlane *s) { BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s->vdev))); VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); + VirtIOBlock *vblk = VIRTIO_BLK(s->vdev); if (!s->started || s->stopping) { return; } s->stopping = true; + vblk->complete_request = s->saved_complete_request; trace_virtio_blk_data_plane_stop(s); aio_context_acquire(s->ctx); diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c index 08562ea390..a222e3f9a4 100644 --- a/hw/block/virtio-blk.c +++ b/hw/block/virtio-blk.c @@ -12,6 +12,7 @@ */ #include "qemu-common.h" +#include "qemu/iov.h" #include "qemu/error-report.h" #include "trace.h" #include "hw/block/block.h" @@ -27,18 +28,24 @@ #endif #include "hw/virtio/virtio-bus.h" -typedef struct VirtIOBlockReq +static VirtIOBlockReq *virtio_blk_alloc_request(VirtIOBlock *s) +{ + VirtIOBlockReq *req = g_slice_new0(VirtIOBlockReq); + req->dev = s; + req->elem = g_slice_new0(VirtQueueElement); + return req; +} + +static void virtio_blk_free_request(VirtIOBlockReq *req) { - VirtIOBlock *dev; - VirtQueueElement elem; - struct virtio_blk_inhdr *in; - struct virtio_blk_outhdr *out; - QEMUIOVector qiov; - struct VirtIOBlockReq *next; - BlockAcctCookie acct; -} VirtIOBlockReq; + if (req) { + g_slice_free(VirtQueueElement, req->elem); + g_slice_free(VirtIOBlockReq, req); + } +} -static void virtio_blk_req_complete(VirtIOBlockReq *req, int status) +static void virtio_blk_complete_request(VirtIOBlockReq *req, + unsigned char status) { VirtIOBlock *s = req->dev; VirtIODevice *vdev = VIRTIO_DEVICE(s); @@ -46,10 +53,15 @@ static void virtio_blk_req_complete(VirtIOBlockReq *req, int status) trace_virtio_blk_req_complete(req, status); stb_p(&req->in->status, status); - virtqueue_push(s->vq, &req->elem, req->qiov.size + sizeof(*req->in)); + virtqueue_push(s->vq, req->elem, req->qiov.size + sizeof(*req->in)); virtio_notify(vdev, s->vq); } +static void virtio_blk_req_complete(VirtIOBlockReq *req, unsigned char status) +{ + req->dev->complete_request(req, status); +} + static int virtio_blk_handle_rw_error(VirtIOBlockReq *req, int error, bool is_read) { @@ -62,7 +74,7 @@ static int virtio_blk_handle_rw_error(VirtIOBlockReq *req, int error, } else if (action == BLOCK_ERROR_ACTION_REPORT) { virtio_blk_req_complete(req, VIRTIO_BLK_S_IOERR); bdrv_acct_done(s->bs, &req->acct); - g_free(req); + virtio_blk_free_request(req); } bdrv_error_action(s->bs, action, is_read, error); @@ -76,14 +88,14 @@ static void virtio_blk_rw_complete(void *opaque, int ret) trace_virtio_blk_rw_complete(req, ret); if (ret) { - bool is_read = !(ldl_p(&req->out->type) & VIRTIO_BLK_T_OUT); + bool is_read = !(ldl_p(&req->out.type) & VIRTIO_BLK_T_OUT); if (virtio_blk_handle_rw_error(req, -ret, is_read)) return; } virtio_blk_req_complete(req, VIRTIO_BLK_S_OK); bdrv_acct_done(req->dev->bs, &req->acct); - g_free(req); + virtio_blk_free_request(req); } static void virtio_blk_flush_complete(void *opaque, int ret) @@ -98,27 +110,16 @@ static void virtio_blk_flush_complete(void *opaque, int ret) virtio_blk_req_complete(req, VIRTIO_BLK_S_OK); bdrv_acct_done(req->dev->bs, &req->acct); - g_free(req); -} - -static VirtIOBlockReq *virtio_blk_alloc_request(VirtIOBlock *s) -{ - VirtIOBlockReq *req = g_malloc(sizeof(*req)); - req->dev = s; - req->qiov.size = 0; - req->next = NULL; - return req; + virtio_blk_free_request(req); } static VirtIOBlockReq *virtio_blk_get_request(VirtIOBlock *s) { VirtIOBlockReq *req = virtio_blk_alloc_request(s); - if (req != NULL) { - if (!virtqueue_pop(s->vq, &req->elem)) { - g_free(req); - return NULL; - } + if (!virtqueue_pop(s->vq, req->elem)) { + virtio_blk_free_request(req); + return NULL; } return req; @@ -247,17 +248,12 @@ static void virtio_blk_handle_scsi(VirtIOBlockReq *req) { int status; - status = virtio_blk_handle_scsi_req(req->dev, &req->elem); + status = virtio_blk_handle_scsi_req(req->dev, req->elem); virtio_blk_req_complete(req, status); - g_free(req); + virtio_blk_free_request(req); } -typedef struct MultiReqBuffer { - BlockRequest blkreq[32]; - unsigned int num_writes; -} MultiReqBuffer; - -static void virtio_submit_multiwrite(BlockDriverState *bs, MultiReqBuffer *mrb) +void virtio_submit_multiwrite(BlockDriverState *bs, MultiReqBuffer *mrb) { int i, ret; @@ -293,7 +289,7 @@ static void virtio_blk_handle_write(VirtIOBlockReq *req, MultiReqBuffer *mrb) BlockRequest *blkreq; uint64_t sector; - sector = ldq_p(&req->out->sector); + sector = ldq_p(&req->out.sector); bdrv_acct_start(req->dev->bs, &req->acct, req->qiov.size, BDRV_ACCT_WRITE); @@ -327,7 +323,7 @@ static void virtio_blk_handle_read(VirtIOBlockReq *req) { uint64_t sector; - sector = ldq_p(&req->out->sector); + sector = ldq_p(&req->out.sector); bdrv_acct_start(req->dev->bs, &req->acct, req->qiov.size, BDRV_ACCT_READ); @@ -346,26 +342,39 @@ static void virtio_blk_handle_read(VirtIOBlockReq *req) virtio_blk_rw_complete, req); } -static void virtio_blk_handle_request(VirtIOBlockReq *req, - MultiReqBuffer *mrb) +void virtio_blk_handle_request(VirtIOBlockReq *req, MultiReqBuffer *mrb) { uint32_t type; + struct iovec *in_iov = req->elem->in_sg; + struct iovec *iov = req->elem->out_sg; + unsigned in_num = req->elem->in_num; + unsigned out_num = req->elem->out_num; - if (req->elem.out_num < 1 || req->elem.in_num < 1) { + if (req->elem->out_num < 1 || req->elem->in_num < 1) { error_report("virtio-blk missing headers"); exit(1); } - if (req->elem.out_sg[0].iov_len < sizeof(*req->out) || - req->elem.in_sg[req->elem.in_num - 1].iov_len < sizeof(*req->in)) { - error_report("virtio-blk header not in correct element"); + if (unlikely(iov_to_buf(iov, out_num, 0, &req->out, + sizeof(req->out)) != sizeof(req->out))) { + error_report("virtio-blk request outhdr too short"); + exit(1); + } + + iov_discard_front(&iov, &out_num, sizeof(req->out)); + + if (in_num < 1 || + in_iov[in_num - 1].iov_len < sizeof(struct virtio_blk_inhdr)) { + error_report("virtio-blk request inhdr too short"); exit(1); } - req->out = (void *)req->elem.out_sg[0].iov_base; - req->in = (void *)req->elem.in_sg[req->elem.in_num - 1].iov_base; + req->in = (void *)in_iov[in_num - 1].iov_base + + in_iov[in_num - 1].iov_len + - sizeof(struct virtio_blk_inhdr); + iov_discard_back(in_iov, &in_num, sizeof(struct virtio_blk_inhdr)); - type = ldl_p(&req->out->type); + type = ldl_p(&req->out.type); if (type & VIRTIO_BLK_T_FLUSH) { virtio_blk_handle_flush(req, mrb); @@ -378,23 +387,23 @@ static void virtio_blk_handle_request(VirtIOBlockReq *req, * NB: per existing s/n string convention the string is * terminated by '\0' only when shorter than buffer. */ - strncpy(req->elem.in_sg[0].iov_base, + strncpy(req->elem->in_sg[0].iov_base, s->blk.serial ? s->blk.serial : "", - MIN(req->elem.in_sg[0].iov_len, VIRTIO_BLK_ID_BYTES)); + MIN(req->elem->in_sg[0].iov_len, VIRTIO_BLK_ID_BYTES)); virtio_blk_req_complete(req, VIRTIO_BLK_S_OK); - g_free(req); + virtio_blk_free_request(req); } else if (type & VIRTIO_BLK_T_OUT) { - qemu_iovec_init_external(&req->qiov, &req->elem.out_sg[1], - req->elem.out_num - 1); + qemu_iovec_init_external(&req->qiov, &req->elem->out_sg[1], + req->elem->out_num - 1); virtio_blk_handle_write(req, mrb); } else if (type == VIRTIO_BLK_T_IN || type == VIRTIO_BLK_T_BARRIER) { /* VIRTIO_BLK_T_IN is 0, so we can't just & it. */ - qemu_iovec_init_external(&req->qiov, &req->elem.in_sg[0], - req->elem.in_num - 1); + qemu_iovec_init_external(&req->qiov, &req->elem->in_sg[0], + req->elem->in_num - 1); virtio_blk_handle_read(req); } else { virtio_blk_req_complete(req, VIRTIO_BLK_S_UNSUPP); - g_free(req); + virtio_blk_free_request(req); } } @@ -460,7 +469,8 @@ static void virtio_blk_dma_restart_cb(void *opaque, int running, } if (!s->bh) { - s->bh = qemu_bh_new(virtio_blk_dma_restart_bh, s); + s->bh = aio_bh_new(bdrv_get_aio_context(s->blk.conf.bs), + virtio_blk_dma_restart_bh, s); qemu_bh_schedule(s->bh); } } @@ -609,7 +619,8 @@ static void virtio_blk_save(QEMUFile *f, void *opaque) while (req) { qemu_put_sbyte(f, 1); - qemu_put_buffer(f, (unsigned char*)&req->elem, sizeof(req->elem)); + qemu_put_buffer(f, (unsigned char *)req->elem, + sizeof(VirtQueueElement)); req = req->next; } qemu_put_sbyte(f, 0); @@ -631,14 +642,15 @@ static int virtio_blk_load(QEMUFile *f, void *opaque, int version_id) while (qemu_get_sbyte(f)) { VirtIOBlockReq *req = virtio_blk_alloc_request(s); - qemu_get_buffer(f, (unsigned char*)&req->elem, sizeof(req->elem)); + qemu_get_buffer(f, (unsigned char *)req->elem, + sizeof(VirtQueueElement)); req->next = s->rq; s->rq = req; - virtqueue_map_sg(req->elem.in_sg, req->elem.in_addr, - req->elem.in_num, 1); - virtqueue_map_sg(req->elem.out_sg, req->elem.out_addr, - req->elem.out_num, 0); + virtqueue_map_sg(req->elem->in_sg, req->elem->in_addr, + req->elem->in_num, 1); + virtqueue_map_sg(req->elem->out_sg, req->elem->out_addr, + req->elem->out_num, 0); } return 0; @@ -729,6 +741,7 @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp) s->sector_mask = (s->conf->logical_block_size / BDRV_SECTOR_SIZE) - 1; s->vq = virtio_add_queue(vdev, 128, virtio_blk_handle_output); + s->complete_request = virtio_blk_complete_request; #ifdef CONFIG_VIRTIO_BLK_DATA_PLANE virtio_blk_data_plane_create(vdev, blk, &s->dataplane, &err); if (err != NULL) { |