diff options
Diffstat (limited to 'hw/nvme/ctrl.c')
| -rw-r--r-- | hw/nvme/ctrl.c | 235 |
1 files changed, 190 insertions, 45 deletions
diff --git a/hw/nvme/ctrl.c b/hw/nvme/ctrl.c index 98aac98bef..03760ddeae 100644 --- a/hw/nvme/ctrl.c +++ b/hw/nvme/ctrl.c @@ -163,6 +163,7 @@ #include "migration/vmstate.h" #include "nvme.h" +#include "dif.h" #include "trace.h" #define NVME_MAX_IOQPAIRS 0xffff @@ -195,6 +196,7 @@ static const bool nvme_feature_support[NVME_FID_MAX] = { [NVME_WRITE_ATOMICITY] = true, [NVME_ASYNCHRONOUS_EVENT_CONF] = true, [NVME_TIMESTAMP] = true, + [NVME_HOST_BEHAVIOR_SUPPORT] = true, [NVME_COMMAND_SET_PROFILE] = true, }; @@ -205,6 +207,7 @@ static const uint32_t nvme_feature_cap[NVME_FID_MAX] = { [NVME_NUMBER_OF_QUEUES] = NVME_FEAT_CAP_CHANGE, [NVME_ASYNCHRONOUS_EVENT_CONF] = NVME_FEAT_CAP_CHANGE, [NVME_TIMESTAMP] = NVME_FEAT_CAP_CHANGE, + [NVME_HOST_BEHAVIOR_SUPPORT] = NVME_FEAT_CAP_CHANGE, [NVME_COMMAND_SET_PROFILE] = NVME_FEAT_CAP_CHANGE, }; @@ -1065,7 +1068,8 @@ static uint16_t nvme_map_data(NvmeCtrl *n, uint32_t nlb, NvmeRequest *req) size_t len = nvme_l2b(ns, nlb); uint16_t status; - if (nvme_ns_ext(ns) && !(pi && pract && ns->lbaf.ms == 8)) { + if (nvme_ns_ext(ns) && + !(pi && pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) { NvmeSg sg; len += nvme_m2b(ns, nlb); @@ -1244,7 +1248,8 @@ uint16_t nvme_bounce_data(NvmeCtrl *n, void *ptr, uint32_t len, bool pi = !!NVME_ID_NS_DPS_TYPE(ns->id_ns.dps); bool pract = !!(le16_to_cpu(rw->control) & NVME_RW_PRINFO_PRACT); - if (nvme_ns_ext(ns) && !(pi && pract && ns->lbaf.ms == 8)) { + if (nvme_ns_ext(ns) && + !(pi && pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) { return nvme_tx_interleaved(n, &req->sg, ptr, len, ns->lbasz, ns->lbaf.ms, 0, dir); } @@ -2045,9 +2050,12 @@ static void nvme_verify_cb(void *opaque, int ret) uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control)); uint16_t apptag = le16_to_cpu(rw->apptag); uint16_t appmask = le16_to_cpu(rw->appmask); - uint32_t reftag = le32_to_cpu(rw->reftag); + uint64_t reftag = le32_to_cpu(rw->reftag); + uint64_t cdw3 = le32_to_cpu(rw->cdw3); uint16_t status; + reftag |= cdw3 << 32; + trace_pci_nvme_verify_cb(nvme_cid(req), prinfo, apptag, appmask, reftag); if (ret) { @@ -2136,7 +2144,8 @@ static void nvme_compare_mdata_cb(void *opaque, int ret) uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control)); uint16_t apptag = le16_to_cpu(rw->apptag); uint16_t appmask = le16_to_cpu(rw->appmask); - uint32_t reftag = le32_to_cpu(rw->reftag); + uint64_t reftag = le32_to_cpu(rw->reftag); + uint64_t cdw3 = le32_to_cpu(rw->cdw3); struct nvme_compare_ctx *ctx = req->opaque; g_autofree uint8_t *buf = NULL; BlockBackend *blk = ns->blkconf.blk; @@ -2144,6 +2153,8 @@ static void nvme_compare_mdata_cb(void *opaque, int ret) BlockAcctStats *stats = blk_get_stats(blk); uint16_t status = NVME_SUCCESS; + reftag |= cdw3 << 32; + trace_pci_nvme_compare_mdata_cb(nvme_cid(req)); if (ret) { @@ -2181,7 +2192,7 @@ static void nvme_compare_mdata_cb(void *opaque, int ret) * tuple. */ if (!(ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT)) { - pil = ns->lbaf.ms - sizeof(NvmeDifTuple); + pil = ns->lbaf.ms - nvme_pi_tuple_size(ns); } for (bufp = buf; mbufp < end; bufp += ns->lbaf.ms, mbufp += ns->lbaf.ms) { @@ -2522,7 +2533,8 @@ typedef struct NvmeCopyAIOCB { QEMUBH *bh; int ret; - NvmeCopySourceRange *ranges; + void *ranges; + unsigned int format; int nr; int idx; @@ -2533,7 +2545,7 @@ typedef struct NvmeCopyAIOCB { BlockAcctCookie write; } acct; - uint32_t reftag; + uint64_t reftag; uint64_t slba; NvmeZone *zone; @@ -2587,13 +2599,101 @@ static void nvme_copy_bh(void *opaque) static void nvme_copy_cb(void *opaque, int ret); +static void nvme_copy_source_range_parse_format0(void *ranges, int idx, + uint64_t *slba, uint32_t *nlb, + uint16_t *apptag, + uint16_t *appmask, + uint64_t *reftag) +{ + NvmeCopySourceRangeFormat0 *_ranges = ranges; + + if (slba) { + *slba = le64_to_cpu(_ranges[idx].slba); + } + + if (nlb) { + *nlb = le16_to_cpu(_ranges[idx].nlb) + 1; + } + + if (apptag) { + *apptag = le16_to_cpu(_ranges[idx].apptag); + } + + if (appmask) { + *appmask = le16_to_cpu(_ranges[idx].appmask); + } + + if (reftag) { + *reftag = le32_to_cpu(_ranges[idx].reftag); + } +} + +static void nvme_copy_source_range_parse_format1(void *ranges, int idx, + uint64_t *slba, uint32_t *nlb, + uint16_t *apptag, + uint16_t *appmask, + uint64_t *reftag) +{ + NvmeCopySourceRangeFormat1 *_ranges = ranges; + + if (slba) { + *slba = le64_to_cpu(_ranges[idx].slba); + } + + if (nlb) { + *nlb = le16_to_cpu(_ranges[idx].nlb) + 1; + } + + if (apptag) { + *apptag = le16_to_cpu(_ranges[idx].apptag); + } + + if (appmask) { + *appmask = le16_to_cpu(_ranges[idx].appmask); + } + + if (reftag) { + *reftag = 0; + + *reftag |= (uint64_t)_ranges[idx].sr[4] << 40; + *reftag |= (uint64_t)_ranges[idx].sr[5] << 32; + *reftag |= (uint64_t)_ranges[idx].sr[6] << 24; + *reftag |= (uint64_t)_ranges[idx].sr[7] << 16; + *reftag |= (uint64_t)_ranges[idx].sr[8] << 8; + *reftag |= (uint64_t)_ranges[idx].sr[9]; + } +} + +static void nvme_copy_source_range_parse(void *ranges, int idx, uint8_t format, + uint64_t *slba, uint32_t *nlb, + uint16_t *apptag, uint16_t *appmask, + uint64_t *reftag) +{ + switch (format) { + case NVME_COPY_FORMAT_0: + nvme_copy_source_range_parse_format0(ranges, idx, slba, nlb, apptag, + appmask, reftag); + break; + + case NVME_COPY_FORMAT_1: + nvme_copy_source_range_parse_format1(ranges, idx, slba, nlb, apptag, + appmask, reftag); + break; + + default: + abort(); + } +} + static void nvme_copy_out_completed_cb(void *opaque, int ret) { NvmeCopyAIOCB *iocb = opaque; NvmeRequest *req = iocb->req; NvmeNamespace *ns = req->ns; - NvmeCopySourceRange *range = &iocb->ranges[iocb->idx]; - uint32_t nlb = le32_to_cpu(range->nlb) + 1; + uint32_t nlb; + + nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, NULL, + &nlb, NULL, NULL, NULL); if (ret < 0) { iocb->ret = ret; @@ -2617,7 +2717,6 @@ static void nvme_copy_out_cb(void *opaque, int ret) NvmeCopyAIOCB *iocb = opaque; NvmeRequest *req = iocb->req; NvmeNamespace *ns = req->ns; - NvmeCopySourceRange *range; uint32_t nlb; size_t mlen; uint8_t *mbounce; @@ -2634,8 +2733,8 @@ static void nvme_copy_out_cb(void *opaque, int ret) return; } - range = &iocb->ranges[iocb->idx]; - nlb = le32_to_cpu(range->nlb) + 1; + nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, NULL, + &nlb, NULL, NULL, NULL); mlen = nvme_m2b(ns, nlb); mbounce = iocb->bounce + nvme_l2b(ns, nlb); @@ -2658,8 +2757,10 @@ static void nvme_copy_in_completed_cb(void *opaque, int ret) NvmeCopyAIOCB *iocb = opaque; NvmeRequest *req = iocb->req; NvmeNamespace *ns = req->ns; - NvmeCopySourceRange *range; uint32_t nlb; + uint64_t slba; + uint16_t apptag, appmask; + uint64_t reftag; size_t len; uint16_t status; @@ -2670,8 +2771,8 @@ static void nvme_copy_in_completed_cb(void *opaque, int ret) goto out; } - range = &iocb->ranges[iocb->idx]; - nlb = le32_to_cpu(range->nlb) + 1; + nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba, + &nlb, &apptag, &appmask, &reftag); len = nvme_l2b(ns, nlb); trace_pci_nvme_copy_out(iocb->slba, nlb); @@ -2682,11 +2783,6 @@ static void nvme_copy_in_completed_cb(void *opaque, int ret) uint16_t prinfor = ((copy->control[0] >> 4) & 0xf); uint16_t prinfow = ((copy->control[2] >> 2) & 0xf); - uint16_t apptag = le16_to_cpu(range->apptag); - uint16_t appmask = le16_to_cpu(range->appmask); - uint32_t reftag = le32_to_cpu(range->reftag); - - uint64_t slba = le64_to_cpu(range->slba); size_t mlen = nvme_m2b(ns, nlb); uint8_t *mbounce = iocb->bounce + nvme_l2b(ns, nlb); @@ -2759,7 +2855,6 @@ static void nvme_copy_in_cb(void *opaque, int ret) NvmeCopyAIOCB *iocb = opaque; NvmeRequest *req = iocb->req; NvmeNamespace *ns = req->ns; - NvmeCopySourceRange *range; uint64_t slba; uint32_t nlb; @@ -2775,9 +2870,8 @@ static void nvme_copy_in_cb(void *opaque, int ret) return; } - range = &iocb->ranges[iocb->idx]; - slba = le64_to_cpu(range->slba); - nlb = le32_to_cpu(range->nlb) + 1; + nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba, + &nlb, NULL, NULL, NULL); qemu_iovec_reset(&iocb->iov); qemu_iovec_add(&iocb->iov, iocb->bounce + nvme_l2b(ns, nlb), @@ -2797,7 +2891,6 @@ static void nvme_copy_cb(void *opaque, int ret) NvmeCopyAIOCB *iocb = opaque; NvmeRequest *req = iocb->req; NvmeNamespace *ns = req->ns; - NvmeCopySourceRange *range; uint64_t slba; uint32_t nlb; size_t len; @@ -2814,9 +2907,8 @@ static void nvme_copy_cb(void *opaque, int ret) goto done; } - range = &iocb->ranges[iocb->idx]; - slba = le64_to_cpu(range->slba); - nlb = le32_to_cpu(range->nlb) + 1; + nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba, + &nlb, NULL, NULL, NULL); len = nvme_l2b(ns, nlb); trace_pci_nvme_copy_source_range(slba, nlb); @@ -2872,6 +2964,7 @@ static uint16_t nvme_copy(NvmeCtrl *n, NvmeRequest *req) uint8_t format = copy->control[0] & 0xf; uint16_t prinfor = ((copy->control[0] >> 4) & 0xf); uint16_t prinfow = ((copy->control[2] >> 2) & 0xf); + size_t len = sizeof(NvmeCopySourceRangeFormat0); uint16_t status; @@ -2897,10 +2990,18 @@ static uint16_t nvme_copy(NvmeCtrl *n, NvmeRequest *req) goto invalid; } - iocb->ranges = g_new(NvmeCopySourceRange, nr); + if (ns->pif && format != 0x1) { + status = NVME_INVALID_FORMAT | NVME_DNR; + goto invalid; + } - status = nvme_h2c(n, (uint8_t *)iocb->ranges, - sizeof(NvmeCopySourceRange) * nr, req); + if (ns->pif) { + len = sizeof(NvmeCopySourceRangeFormat1); + } + + iocb->format = format; + iocb->ranges = g_malloc_n(nr, len); + status = nvme_h2c(n, (uint8_t *)iocb->ranges, len * nr, req); if (status) { goto invalid; } @@ -2926,6 +3027,7 @@ static uint16_t nvme_copy(NvmeCtrl *n, NvmeRequest *req) iocb->nr = nr; iocb->idx = 0; iocb->reftag = le32_to_cpu(copy->reftag); + iocb->reftag |= (uint64_t)le32_to_cpu(copy->cdw3) << 32; iocb->bounce = g_malloc_n(le16_to_cpu(ns->id_ns.mssrl), ns->lbasz + ns->lbaf.ms); @@ -3164,7 +3266,7 @@ static uint16_t nvme_read(NvmeCtrl *n, NvmeRequest *req) if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) { bool pract = prinfo & NVME_PRINFO_PRACT; - if (pract && ns->lbaf.ms == 8) { + if (pract && ns->lbaf.ms == nvme_pi_tuple_size(ns)) { mapped_size = data_size; } } @@ -3241,7 +3343,7 @@ static uint16_t nvme_do_write(NvmeCtrl *n, NvmeRequest *req, bool append, if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) { bool pract = prinfo & NVME_PRINFO_PRACT; - if (pract && ns->lbaf.ms == 8) { + if (pract && ns->lbaf.ms == nvme_pi_tuple_size(ns)) { mapped_size -= nvme_m2b(ns, nlb); } } @@ -4712,7 +4814,8 @@ static uint16_t nvme_identify_ns_csi(NvmeCtrl *n, NvmeRequest *req, } if (c->csi == NVME_CSI_NVM) { - return nvme_rpt_empty_id_struct(n, req); + return nvme_c2h(n, (uint8_t *)&ns->id_ns_nvm, sizeof(NvmeIdNsNvm), + req); } else if (c->csi == NVME_CSI_ZONED && ns->csi == NVME_CSI_ZONED) { return nvme_c2h(n, (uint8_t *)ns->id_ns_zoned, sizeof(NvmeIdNsZoned), req); @@ -5090,6 +5193,9 @@ static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeRequest *req) goto out; case NVME_TIMESTAMP: return nvme_get_feature_timestamp(n, req); + case NVME_HOST_BEHAVIOR_SUPPORT: + return nvme_c2h(n, (uint8_t *)&n->features.hbs, + sizeof(n->features.hbs), req); default: break; } @@ -5159,6 +5265,7 @@ static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeRequest *req) uint32_t nsid = le32_to_cpu(cmd->nsid); uint8_t fid = NVME_GETSETFEAT_FID(dw10); uint8_t save = NVME_SETFEAT_SAVE(dw10); + uint16_t status; int i; trace_pci_nvme_setfeat(nvme_cid(req), nsid, fid, save, dw11); @@ -5280,6 +5387,27 @@ static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeRequest *req) break; case NVME_TIMESTAMP: return nvme_set_feature_timestamp(n, req); + case NVME_HOST_BEHAVIOR_SUPPORT: + status = nvme_h2c(n, (uint8_t *)&n->features.hbs, + sizeof(n->features.hbs), req); + if (status) { + return status; + } + + for (i = 1; i <= NVME_MAX_NAMESPACES; i++) { + ns = nvme_ns(n, i); + + if (!ns) { + continue; + } + + ns->id_ns.nlbaf = ns->nlbaf - 1; + if (!n->features.hbs.lbafee) { + ns->id_ns.nlbaf = MIN(ns->id_ns.nlbaf, 15); + } + } + + return status; case NVME_COMMAND_SET_PROFILE: if (dw11 & 0x1ff) { trace_pci_nvme_err_invalid_iocsci(dw11 & 0x1ff); @@ -5443,6 +5571,11 @@ typedef struct NvmeFormatAIOCB { uint32_t nsid; bool broadcast; int64_t offset; + + uint8_t lbaf; + uint8_t mset; + uint8_t pi; + uint8_t pil; } NvmeFormatAIOCB; static void nvme_format_bh(void *opaque); @@ -5462,18 +5595,16 @@ static const AIOCBInfo nvme_format_aiocb_info = { .get_aio_context = nvme_get_aio_context, }; -static void nvme_format_set(NvmeNamespace *ns, NvmeCmd *cmd) +static void nvme_format_set(NvmeNamespace *ns, uint8_t lbaf, uint8_t mset, + uint8_t pi, uint8_t pil) { - uint32_t dw10 = le32_to_cpu(cmd->cdw10); - uint8_t lbaf = dw10 & 0xf; - uint8_t pi = (dw10 >> 5) & 0x7; - uint8_t mset = (dw10 >> 4) & 0x1; - uint8_t pil = (dw10 >> 8) & 0x1; + uint8_t lbafl = lbaf & 0xf; + uint8_t lbafu = lbaf >> 4; trace_pci_nvme_format_set(ns->params.nsid, lbaf, mset, pi, pil); ns->id_ns.dps = (pil << 3) | pi; - ns->id_ns.flbas = lbaf | (mset << 4); + ns->id_ns.flbas = (lbafu << 5) | (mset << 4) | lbafl; nvme_ns_init_format(ns); } @@ -5481,7 +5612,6 @@ static void nvme_format_set(NvmeNamespace *ns, NvmeCmd *cmd) static void nvme_format_ns_cb(void *opaque, int ret) { NvmeFormatAIOCB *iocb = opaque; - NvmeRequest *req = iocb->req; NvmeNamespace *ns = iocb->ns; int bytes; @@ -5503,7 +5633,7 @@ static void nvme_format_ns_cb(void *opaque, int ret) return; } - nvme_format_set(ns, &req->cmd); + nvme_format_set(ns, iocb->lbaf, iocb->mset, iocb->pi, iocb->pil); ns->status = 0x0; iocb->ns = NULL; iocb->offset = 0; @@ -5523,7 +5653,7 @@ static uint16_t nvme_format_check(NvmeNamespace *ns, uint8_t lbaf, uint8_t pi) return NVME_INVALID_FORMAT | NVME_DNR; } - if (pi && (ns->id_ns.lbaf[lbaf].ms < sizeof(NvmeDifTuple))) { + if (pi && (ns->id_ns.lbaf[lbaf].ms < nvme_pi_tuple_size(ns))) { return NVME_INVALID_FORMAT | NVME_DNR; } @@ -5586,6 +5716,12 @@ static uint16_t nvme_format(NvmeCtrl *n, NvmeRequest *req) { NvmeFormatAIOCB *iocb; uint32_t nsid = le32_to_cpu(req->cmd.nsid); + uint32_t dw10 = le32_to_cpu(req->cmd.cdw10); + uint8_t lbaf = dw10 & 0xf; + uint8_t mset = (dw10 >> 4) & 0x1; + uint8_t pi = (dw10 >> 5) & 0x7; + uint8_t pil = (dw10 >> 8) & 0x1; + uint8_t lbafu = (dw10 >> 12) & 0x3; uint16_t status; iocb = qemu_aio_get(&nvme_format_aiocb_info, NULL, nvme_misc_cb, req); @@ -5595,9 +5731,17 @@ static uint16_t nvme_format(NvmeCtrl *n, NvmeRequest *req) iocb->ret = 0; iocb->ns = NULL; iocb->nsid = 0; + iocb->lbaf = lbaf; + iocb->mset = mset; + iocb->pi = pi; + iocb->pil = pil; iocb->broadcast = (nsid == NVME_NSID_BROADCAST); iocb->offset = 0; + if (n->features.hbs.lbafee) { + iocb->lbaf |= lbafu << 4; + } + if (!iocb->broadcast) { if (!nvme_nsid_valid(n, nsid)) { status = NVME_INVALID_NSID | NVME_DNR; @@ -6573,6 +6717,7 @@ static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev) id->cntlid = cpu_to_le16(n->cntlid); id->oaes = cpu_to_le32(NVME_OAES_NS_ATTR); + id->ctratt |= cpu_to_le32(NVME_CTRATT_ELBAS); id->rab = 6; @@ -6627,7 +6772,7 @@ static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev) */ id->vwc = NVME_VWC_NSID_BROADCAST_SUPPORT | NVME_VWC_PRESENT; - id->ocfs = cpu_to_le16(NVME_OCFS_COPY_FORMAT_0); + id->ocfs = cpu_to_le16(NVME_OCFS_COPY_FORMAT_0 | NVME_OCFS_COPY_FORMAT_1); id->sgls = cpu_to_le32(NVME_CTRL_SGLS_SUPPORT_NO_ALIGN | NVME_CTRL_SGLS_BITBUCKET); |