diff options
Diffstat (limited to 'hw/nvme/ctrl.c')
| -rw-r--r-- | hw/nvme/ctrl.c | 452 |
1 files changed, 359 insertions, 93 deletions
diff --git a/hw/nvme/ctrl.c b/hw/nvme/ctrl.c index 5b1b0cabcf..e86ea2e7ce 100644 --- a/hw/nvme/ctrl.c +++ b/hw/nvme/ctrl.c @@ -203,6 +203,7 @@ #include "sysemu/hostmem.h" #include "hw/pci/msix.h" #include "hw/pci/pcie_sriov.h" +#include "sysemu/spdm-socket.h" #include "migration/vmstate.h" #include "nvme.h" @@ -1758,6 +1759,10 @@ static void nvme_aio_err(NvmeRequest *req, int ret) break; } + if (ret == -ECANCELED) { + status = NVME_CMD_ABORT_REQ; + } + trace_pci_nvme_err_aio(nvme_cid(req), strerror(-ret), status); error_setg_errno(&local_err, -ret, "aio failed"); @@ -2591,6 +2596,7 @@ next: done: iocb->aiocb = NULL; iocb->common.cb(iocb->common.opaque, iocb->ret); + g_free(iocb->range); qemu_aio_unref(iocb); } @@ -2695,6 +2701,7 @@ typedef struct NvmeCopyAIOCB { BlockAIOCB common; BlockAIOCB *aiocb; NvmeRequest *req; + NvmeCtrl *n; int ret; void *ranges; @@ -2713,6 +2720,8 @@ typedef struct NvmeCopyAIOCB { uint64_t slba; NvmeZone *zone; + NvmeNamespace *sns; + uint32_t tcl; } NvmeCopyAIOCB; static void nvme_copy_cancel(BlockAIOCB *aiocb) @@ -2759,13 +2768,19 @@ static void nvme_copy_done(NvmeCopyAIOCB *iocb) static void nvme_do_copy(NvmeCopyAIOCB *iocb); -static void nvme_copy_source_range_parse_format0(void *ranges, int idx, - uint64_t *slba, uint32_t *nlb, - uint16_t *apptag, - uint16_t *appmask, - uint64_t *reftag) +static void nvme_copy_source_range_parse_format0_2(void *ranges, + int idx, uint64_t *slba, + uint32_t *nlb, + uint32_t *snsid, + uint16_t *apptag, + uint16_t *appmask, + uint64_t *reftag) { - NvmeCopySourceRangeFormat0 *_ranges = ranges; + NvmeCopySourceRangeFormat0_2 *_ranges = ranges; + + if (snsid) { + *snsid = le32_to_cpu(_ranges[idx].sparams); + } if (slba) { *slba = le64_to_cpu(_ranges[idx].slba); @@ -2788,13 +2803,19 @@ static void nvme_copy_source_range_parse_format0(void *ranges, int idx, } } -static void nvme_copy_source_range_parse_format1(void *ranges, int idx, - uint64_t *slba, uint32_t *nlb, - uint16_t *apptag, - uint16_t *appmask, - uint64_t *reftag) +static void nvme_copy_source_range_parse_format1_3(void *ranges, int idx, + uint64_t *slba, + uint32_t *nlb, + uint32_t *snsid, + uint16_t *apptag, + uint16_t *appmask, + uint64_t *reftag) { - NvmeCopySourceRangeFormat1 *_ranges = ranges; + NvmeCopySourceRangeFormat1_3 *_ranges = ranges; + + if (snsid) { + *snsid = le32_to_cpu(_ranges[idx].sparams); + } if (slba) { *slba = le64_to_cpu(_ranges[idx].slba); @@ -2826,18 +2847,20 @@ static void nvme_copy_source_range_parse_format1(void *ranges, int idx, static void nvme_copy_source_range_parse(void *ranges, int idx, uint8_t format, uint64_t *slba, uint32_t *nlb, - uint16_t *apptag, uint16_t *appmask, - uint64_t *reftag) + uint32_t *snsid, uint16_t *apptag, + uint16_t *appmask, uint64_t *reftag) { switch (format) { case NVME_COPY_FORMAT_0: - nvme_copy_source_range_parse_format0(ranges, idx, slba, nlb, apptag, - appmask, reftag); + case NVME_COPY_FORMAT_2: + nvme_copy_source_range_parse_format0_2(ranges, idx, slba, nlb, snsid, + apptag, appmask, reftag); break; case NVME_COPY_FORMAT_1: - nvme_copy_source_range_parse_format1(ranges, idx, slba, nlb, apptag, - appmask, reftag); + case NVME_COPY_FORMAT_3: + nvme_copy_source_range_parse_format1_3(ranges, idx, slba, nlb, snsid, + apptag, appmask, reftag); break; default: @@ -2853,10 +2876,10 @@ static inline uint16_t nvme_check_copy_mcl(NvmeNamespace *ns, for (int idx = 0; idx < nr; idx++) { uint32_t nlb; nvme_copy_source_range_parse(iocb->ranges, idx, iocb->format, NULL, - &nlb, NULL, NULL, NULL); + &nlb, NULL, NULL, NULL, NULL); copy_len += nlb; } - + iocb->tcl = copy_len; if (copy_len > ns->id_ns.mcl) { return NVME_CMD_SIZE_LIMIT | NVME_DNR; } @@ -2868,11 +2891,11 @@ static void nvme_copy_out_completed_cb(void *opaque, int ret) { NvmeCopyAIOCB *iocb = opaque; NvmeRequest *req = iocb->req; - NvmeNamespace *ns = req->ns; + NvmeNamespace *dns = req->ns; uint32_t nlb; nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, NULL, - &nlb, NULL, NULL, NULL); + &nlb, NULL, NULL, NULL, NULL); if (ret < 0) { iocb->ret = ret; @@ -2881,8 +2904,8 @@ static void nvme_copy_out_completed_cb(void *opaque, int ret) goto out; } - if (ns->params.zoned) { - nvme_advance_zone_wp(ns, iocb->zone, nlb); + if (dns->params.zoned) { + nvme_advance_zone_wp(dns, iocb->zone, nlb); } iocb->idx++; @@ -2895,25 +2918,25 @@ static void nvme_copy_out_cb(void *opaque, int ret) { NvmeCopyAIOCB *iocb = opaque; NvmeRequest *req = iocb->req; - NvmeNamespace *ns = req->ns; + NvmeNamespace *dns = req->ns; uint32_t nlb; size_t mlen; uint8_t *mbounce; - if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) { + if (ret < 0 || iocb->ret < 0 || !dns->lbaf.ms) { goto out; } nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, NULL, - &nlb, NULL, NULL, NULL); + &nlb, NULL, NULL, NULL, NULL); - mlen = nvme_m2b(ns, nlb); - mbounce = iocb->bounce + nvme_l2b(ns, nlb); + mlen = nvme_m2b(dns, nlb); + mbounce = iocb->bounce + nvme_l2b(dns, nlb); qemu_iovec_reset(&iocb->iov); qemu_iovec_add(&iocb->iov, mbounce, mlen); - iocb->aiocb = blk_aio_pwritev(ns->blkconf.blk, nvme_moff(ns, iocb->slba), + iocb->aiocb = blk_aio_pwritev(dns->blkconf.blk, nvme_moff(dns, iocb->slba), &iocb->iov, 0, nvme_copy_out_completed_cb, iocb); @@ -2927,12 +2950,15 @@ static void nvme_copy_in_completed_cb(void *opaque, int ret) { NvmeCopyAIOCB *iocb = opaque; NvmeRequest *req = iocb->req; - NvmeNamespace *ns = req->ns; + NvmeNamespace *sns = iocb->sns; + NvmeNamespace *dns = req->ns; + NvmeCopyCmd *copy = NULL; + uint8_t *mbounce = NULL; uint32_t nlb; uint64_t slba; uint16_t apptag, appmask; uint64_t reftag; - size_t len; + size_t len, mlen; uint16_t status; if (ret < 0) { @@ -2943,43 +2969,51 @@ static void nvme_copy_in_completed_cb(void *opaque, int ret) } nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba, - &nlb, &apptag, &appmask, &reftag); - len = nvme_l2b(ns, nlb); + &nlb, NULL, &apptag, &appmask, &reftag); trace_pci_nvme_copy_out(iocb->slba, nlb); - if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) { - NvmeCopyCmd *copy = (NvmeCopyCmd *)&req->cmd; + len = nvme_l2b(sns, nlb); + + if (NVME_ID_NS_DPS_TYPE(sns->id_ns.dps)) { + copy = (NvmeCopyCmd *)&req->cmd; uint16_t prinfor = ((copy->control[0] >> 4) & 0xf); - uint16_t prinfow = ((copy->control[2] >> 2) & 0xf); - size_t mlen = nvme_m2b(ns, nlb); - uint8_t *mbounce = iocb->bounce + nvme_l2b(ns, nlb); + mlen = nvme_m2b(sns, nlb); + mbounce = iocb->bounce + nvme_l2b(sns, nlb); - status = nvme_dif_mangle_mdata(ns, mbounce, mlen, slba); + status = nvme_dif_mangle_mdata(sns, mbounce, mlen, slba); if (status) { goto invalid; } - status = nvme_dif_check(ns, iocb->bounce, len, mbounce, mlen, prinfor, + status = nvme_dif_check(sns, iocb->bounce, len, mbounce, mlen, prinfor, slba, apptag, appmask, &reftag); if (status) { goto invalid; } + } + + if (NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) { + copy = (NvmeCopyCmd *)&req->cmd; + uint16_t prinfow = ((copy->control[2] >> 2) & 0xf); + + mlen = nvme_m2b(dns, nlb); + mbounce = iocb->bounce + nvme_l2b(dns, nlb); apptag = le16_to_cpu(copy->apptag); appmask = le16_to_cpu(copy->appmask); if (prinfow & NVME_PRINFO_PRACT) { - status = nvme_check_prinfo(ns, prinfow, iocb->slba, iocb->reftag); + status = nvme_check_prinfo(dns, prinfow, iocb->slba, iocb->reftag); if (status) { goto invalid; } - nvme_dif_pract_generate_dif(ns, iocb->bounce, len, mbounce, mlen, + nvme_dif_pract_generate_dif(dns, iocb->bounce, len, mbounce, mlen, apptag, &iocb->reftag); } else { - status = nvme_dif_check(ns, iocb->bounce, len, mbounce, mlen, + status = nvme_dif_check(dns, iocb->bounce, len, mbounce, mlen, prinfow, iocb->slba, apptag, appmask, &iocb->reftag); if (status) { @@ -2988,13 +3022,13 @@ static void nvme_copy_in_completed_cb(void *opaque, int ret) } } - status = nvme_check_bounds(ns, iocb->slba, nlb); + status = nvme_check_bounds(dns, iocb->slba, nlb); if (status) { goto invalid; } - if (ns->params.zoned) { - status = nvme_check_zone_write(ns, iocb->zone, iocb->slba, nlb); + if (dns->params.zoned) { + status = nvme_check_zone_write(dns, iocb->zone, iocb->slba, nlb); if (status) { goto invalid; } @@ -3007,7 +3041,10 @@ static void nvme_copy_in_completed_cb(void *opaque, int ret) qemu_iovec_reset(&iocb->iov); qemu_iovec_add(&iocb->iov, iocb->bounce, len); - iocb->aiocb = blk_aio_pwritev(ns->blkconf.blk, nvme_l2b(ns, iocb->slba), + block_acct_start(blk_get_stats(dns->blkconf.blk), &iocb->acct.write, 0, + BLOCK_ACCT_WRITE); + + iocb->aiocb = blk_aio_pwritev(dns->blkconf.blk, nvme_l2b(dns, iocb->slba), &iocb->iov, 0, nvme_copy_out_cb, iocb); return; @@ -3022,23 +3059,22 @@ out: static void nvme_copy_in_cb(void *opaque, int ret) { NvmeCopyAIOCB *iocb = opaque; - NvmeRequest *req = iocb->req; - NvmeNamespace *ns = req->ns; + NvmeNamespace *sns = iocb->sns; uint64_t slba; uint32_t nlb; - if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) { + if (ret < 0 || iocb->ret < 0 || !sns->lbaf.ms) { goto out; } nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba, - &nlb, NULL, NULL, NULL); + &nlb, NULL, NULL, NULL, NULL); qemu_iovec_reset(&iocb->iov); - qemu_iovec_add(&iocb->iov, iocb->bounce + nvme_l2b(ns, nlb), - nvme_m2b(ns, nlb)); + qemu_iovec_add(&iocb->iov, iocb->bounce + nvme_l2b(sns, nlb), + nvme_m2b(sns, nlb)); - iocb->aiocb = blk_aio_preadv(ns->blkconf.blk, nvme_moff(ns, slba), + iocb->aiocb = blk_aio_preadv(sns->blkconf.blk, nvme_moff(sns, slba), &iocb->iov, 0, nvme_copy_in_completed_cb, iocb); return; @@ -3047,14 +3083,78 @@ out: nvme_copy_in_completed_cb(iocb, ret); } +static inline bool nvme_csi_supports_copy(uint8_t csi) +{ + return csi == NVME_CSI_NVM || csi == NVME_CSI_ZONED; +} + +static inline bool nvme_copy_ns_format_match(NvmeNamespace *sns, + NvmeNamespace *dns) +{ + return sns->lbaf.ds == dns->lbaf.ds && sns->lbaf.ms == dns->lbaf.ms; +} + +static bool nvme_copy_matching_ns_format(NvmeNamespace *sns, NvmeNamespace *dns, + bool pi_enable) +{ + if (!nvme_csi_supports_copy(sns->csi) || + !nvme_csi_supports_copy(dns->csi)) { + return false; + } + + if (!pi_enable && !nvme_copy_ns_format_match(sns, dns)) { + return false; + } + + if (pi_enable && (!nvme_copy_ns_format_match(sns, dns) || + sns->id_ns.dps != dns->id_ns.dps)) { + return false; + } + + return true; +} + +static inline bool nvme_copy_corresp_pi_match(NvmeNamespace *sns, + NvmeNamespace *dns) +{ + return sns->lbaf.ms == 0 && + ((dns->lbaf.ms == 8 && dns->pif == 0) || + (dns->lbaf.ms == 16 && dns->pif == 1)); +} + +static bool nvme_copy_corresp_pi_format(NvmeNamespace *sns, NvmeNamespace *dns, + bool sns_pi_en) +{ + if (!nvme_csi_supports_copy(sns->csi) || + !nvme_csi_supports_copy(dns->csi)) { + return false; + } + + if (!sns_pi_en && !nvme_copy_corresp_pi_match(sns, dns)) { + return false; + } + + if (sns_pi_en && !nvme_copy_corresp_pi_match(dns, sns)) { + return false; + } + + return true; +} + static void nvme_do_copy(NvmeCopyAIOCB *iocb) { NvmeRequest *req = iocb->req; - NvmeNamespace *ns = req->ns; + NvmeNamespace *sns; + NvmeNamespace *dns = req->ns; + NvmeCopyCmd *copy = (NvmeCopyCmd *)&req->cmd; + uint16_t prinfor = ((copy->control[0] >> 4) & 0xf); + uint16_t prinfow = ((copy->control[2] >> 2) & 0xf); uint64_t slba; uint32_t nlb; size_t len; uint16_t status; + uint32_t dnsid = le32_to_cpu(req->cmd.nsid); + uint32_t snsid = dnsid; if (iocb->ret < 0) { goto done; @@ -3064,40 +3164,124 @@ static void nvme_do_copy(NvmeCopyAIOCB *iocb) goto done; } - nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba, - &nlb, NULL, NULL, NULL); - len = nvme_l2b(ns, nlb); + if (iocb->format == 2 || iocb->format == 3) { + nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, + &slba, &nlb, &snsid, NULL, NULL, NULL); + if (snsid != dnsid) { + if (snsid == NVME_NSID_BROADCAST || + !nvme_nsid_valid(iocb->n, snsid)) { + status = NVME_INVALID_NSID | NVME_DNR; + goto invalid; + } + iocb->sns = nvme_ns(iocb->n, snsid); + if (unlikely(!iocb->sns)) { + status = NVME_INVALID_FIELD | NVME_DNR; + goto invalid; + } + } else { + if (((slba + nlb) > iocb->slba) && + ((slba + nlb) < (iocb->slba + iocb->tcl))) { + status = NVME_CMD_OVERLAP_IO_RANGE | NVME_DNR; + goto invalid; + } + } + } else { + nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, + &slba, &nlb, NULL, NULL, NULL, NULL); + } + + sns = iocb->sns; + if ((snsid == dnsid) && NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) && + ((prinfor & NVME_PRINFO_PRACT) != (prinfow & NVME_PRINFO_PRACT))) { + status = NVME_INVALID_FIELD | NVME_DNR; + goto invalid; + } else if (snsid != dnsid) { + if (!NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) && + !NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) { + if (!nvme_copy_matching_ns_format(sns, dns, false)) { + status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR; + goto invalid; + } + } + if (NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) && + NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) { + if ((prinfor & NVME_PRINFO_PRACT) != + (prinfow & NVME_PRINFO_PRACT)) { + status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR; + goto invalid; + } else { + if (!nvme_copy_matching_ns_format(sns, dns, true)) { + status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR; + goto invalid; + } + } + } + + if (!NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) && + NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) { + if (!(prinfow & NVME_PRINFO_PRACT)) { + status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR; + goto invalid; + } else { + if (!nvme_copy_corresp_pi_format(sns, dns, false)) { + status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR; + goto invalid; + } + } + } + + if (NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) && + !NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) { + if (!(prinfor & NVME_PRINFO_PRACT)) { + status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR; + goto invalid; + } else { + if (!nvme_copy_corresp_pi_format(sns, dns, true)) { + status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR; + goto invalid; + } + } + } + } + len = nvme_l2b(sns, nlb); trace_pci_nvme_copy_source_range(slba, nlb); - if (nlb > le16_to_cpu(ns->id_ns.mssrl)) { + if (nlb > le16_to_cpu(sns->id_ns.mssrl)) { status = NVME_CMD_SIZE_LIMIT | NVME_DNR; goto invalid; } - status = nvme_check_bounds(ns, slba, nlb); + status = nvme_check_bounds(sns, slba, nlb); if (status) { goto invalid; } - if (NVME_ERR_REC_DULBE(ns->features.err_rec)) { - status = nvme_check_dulbe(ns, slba, nlb); + if (NVME_ERR_REC_DULBE(sns->features.err_rec)) { + status = nvme_check_dulbe(sns, slba, nlb); if (status) { goto invalid; } } - if (ns->params.zoned) { - status = nvme_check_zone_read(ns, slba, nlb); + if (sns->params.zoned) { + status = nvme_check_zone_read(sns, slba, nlb); if (status) { goto invalid; } } + g_free(iocb->bounce); + iocb->bounce = g_malloc_n(le16_to_cpu(sns->id_ns.mssrl), + sns->lbasz + sns->lbaf.ms); + qemu_iovec_reset(&iocb->iov); qemu_iovec_add(&iocb->iov, iocb->bounce, len); - iocb->aiocb = blk_aio_preadv(ns->blkconf.blk, nvme_l2b(ns, slba), + block_acct_start(blk_get_stats(sns->blkconf.blk), &iocb->acct.read, 0, + BLOCK_ACCT_READ); + + iocb->aiocb = blk_aio_preadv(sns->blkconf.blk, nvme_l2b(sns, slba), &iocb->iov, 0, nvme_copy_in_cb, iocb); return; @@ -3116,9 +3300,7 @@ static uint16_t nvme_copy(NvmeCtrl *n, NvmeRequest *req) nvme_misc_cb, req); uint16_t nr = copy->nr + 1; uint8_t format = copy->control[0] & 0xf; - uint16_t prinfor = ((copy->control[0] >> 4) & 0xf); - uint16_t prinfow = ((copy->control[2] >> 2) & 0xf); - size_t len = sizeof(NvmeCopySourceRangeFormat0); + size_t len = sizeof(NvmeCopySourceRangeFormat0_2); uint16_t status; @@ -3127,13 +3309,9 @@ static uint16_t nvme_copy(NvmeCtrl *n, NvmeRequest *req) iocb->ranges = NULL; iocb->zone = NULL; - if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) && - ((prinfor & NVME_PRINFO_PRACT) != (prinfow & NVME_PRINFO_PRACT))) { - status = NVME_INVALID_FIELD | NVME_DNR; - goto invalid; - } - - if (!(n->id_ctrl.ocfs & (1 << format))) { + if (!(n->id_ctrl.ocfs & (1 << format)) || + ((format == 2 || format == 3) && + !(n->features.hbs.cdfe & (1 << format)))) { trace_pci_nvme_err_copy_invalid_format(format); status = NVME_INVALID_FIELD | NVME_DNR; goto invalid; @@ -3144,14 +3322,14 @@ static uint16_t nvme_copy(NvmeCtrl *n, NvmeRequest *req) goto invalid; } - if ((ns->pif == 0x0 && format != 0x0) || - (ns->pif != 0x0 && format != 0x1)) { + if ((ns->pif == 0x0 && (format != 0x0 && format != 0x2)) || + (ns->pif != 0x0 && (format != 0x1 && format != 0x3))) { status = NVME_INVALID_FORMAT | NVME_DNR; goto invalid; } if (ns->pif) { - len = sizeof(NvmeCopySourceRangeFormat1); + len = sizeof(NvmeCopySourceRangeFormat1_3); } iocb->format = format; @@ -3187,17 +3365,13 @@ static uint16_t nvme_copy(NvmeCtrl *n, NvmeRequest *req) iocb->idx = 0; iocb->reftag = le32_to_cpu(copy->reftag); iocb->reftag |= (uint64_t)le32_to_cpu(copy->cdw3) << 32; - iocb->bounce = g_malloc_n(le16_to_cpu(ns->id_ns.mssrl), - ns->lbasz + ns->lbaf.ms); qemu_iovec_init(&iocb->iov, 1); - block_acct_start(blk_get_stats(ns->blkconf.blk), &iocb->acct.read, 0, - BLOCK_ACCT_READ); - block_acct_start(blk_get_stats(ns->blkconf.blk), &iocb->acct.write, 0, - BLOCK_ACCT_WRITE); - req->aiocb = &iocb->common; + iocb->sns = req->ns; + iocb->n = n; + iocb->bounce = NULL; nvme_do_copy(iocb); return NVME_NO_COMPLETE; @@ -4167,7 +4341,7 @@ static bool nvme_zone_matches_filter(uint32_t zafs, NvmeZone *zl) static uint16_t nvme_zone_mgmt_recv(NvmeCtrl *n, NvmeRequest *req) { - NvmeCmd *cmd = (NvmeCmd *)&req->cmd; + NvmeCmd *cmd = &req->cmd; NvmeNamespace *ns = req->ns; /* cdw12 is zero-based number of dwords to return. Convert to bytes */ uint32_t data_size = (le32_to_cpu(cmd->cdw12) + 1) << 2; @@ -4406,10 +4580,6 @@ static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeRequest *req) trace_pci_nvme_io_cmd(nvme_cid(req), nsid, nvme_sqid(req), req->cmd.opcode, nvme_io_opc_str(req->cmd.opcode)); - if (!nvme_nsid_valid(n, nsid)) { - return NVME_INVALID_NSID | NVME_DNR; - } - /* * In the base NVM command set, Flush may apply to all namespaces * (indicated by NSID being set to FFFFFFFFh). But if that feature is used @@ -4429,10 +4599,15 @@ static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeRequest *req) * device only supports namespace types that includes the NVM Flush command * (NVM and Zoned), so always do an NVM Flush. */ + if (req->cmd.opcode == NVME_CMD_FLUSH) { return nvme_flush(n, req); } + if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) { + return NVME_INVALID_NSID | NVME_DNR; + } + ns = nvme_ns(n, nsid); if (unlikely(!ns)) { return NVME_INVALID_FIELD | NVME_DNR; @@ -5780,12 +5955,40 @@ static uint16_t nvme_identify(NvmeCtrl *n, NvmeRequest *req) static uint16_t nvme_abort(NvmeCtrl *n, NvmeRequest *req) { uint16_t sqid = le32_to_cpu(req->cmd.cdw10) & 0xffff; + uint16_t cid = (le32_to_cpu(req->cmd.cdw10) >> 16) & 0xffff; + NvmeSQueue *sq = n->sq[sqid]; + NvmeRequest *r, *next; + int i; req->cqe.result = 1; if (nvme_check_sqid(n, sqid)) { return NVME_INVALID_FIELD | NVME_DNR; } + if (sqid == 0) { + for (i = 0; i < n->outstanding_aers; i++) { + NvmeRequest *re = n->aer_reqs[i]; + if (re->cqe.cid == cid) { + memmove(n->aer_reqs + i, n->aer_reqs + i + 1, + (n->outstanding_aers - i - 1) * sizeof(NvmeRequest *)); + n->outstanding_aers--; + re->status = NVME_CMD_ABORT_REQ; + req->cqe.result = 0; + nvme_enqueue_req_completion(&n->admin_cq, re); + return NVME_SUCCESS; + } + } + } + + QTAILQ_FOREACH_SAFE(r, &sq->out_req_list, entry, next) { + if (r->cqe.cid == cid) { + if (r->aiocb) { + blk_aio_cancel_async(r->aiocb); + } + break; + } + } + return NVME_SUCCESS; } @@ -8113,6 +8316,27 @@ static int nvme_add_pm_capability(PCIDevice *pci_dev, uint8_t offset) return 0; } +static bool pcie_doe_spdm_rsp(DOECap *doe_cap) +{ + void *req = pcie_doe_get_write_mbox_ptr(doe_cap); + uint32_t req_len = pcie_doe_get_obj_len(req) * 4; + void *rsp = doe_cap->read_mbox; + uint32_t rsp_len = SPDM_SOCKET_MAX_MESSAGE_BUFFER_SIZE; + + uint32_t recvd = spdm_socket_rsp(doe_cap->spdm_socket, + SPDM_SOCKET_TRANSPORT_TYPE_PCI_DOE, + req, req_len, rsp, rsp_len); + doe_cap->read_mbox_len += DIV_ROUND_UP(recvd, 4); + + return recvd != 0; +} + +static DOEProtocol doe_spdm_prot[] = { + { PCI_VENDOR_ID_PCI_SIG, PCI_SIG_DOE_CMA, pcie_doe_spdm_rsp }, + { PCI_VENDOR_ID_PCI_SIG, PCI_SIG_DOE_SECURED_CMA, pcie_doe_spdm_rsp }, + { } +}; + static bool nvme_init_pci(NvmeCtrl *n, PCIDevice *pci_dev, Error **errp) { ERRP_GUARD(); @@ -8200,6 +8424,25 @@ static bool nvme_init_pci(NvmeCtrl *n, PCIDevice *pci_dev, Error **errp) nvme_update_msixcap_ts(pci_dev, n->conf_msix_qsize); + pcie_cap_deverr_init(pci_dev); + + /* DOE Initialisation */ + if (pci_dev->spdm_port) { + uint16_t doe_offset = n->params.sriov_max_vfs ? + PCI_CONFIG_SPACE_SIZE + PCI_ARI_SIZEOF + : PCI_CONFIG_SPACE_SIZE; + + pcie_doe_init(pci_dev, &pci_dev->doe_spdm, doe_offset, + doe_spdm_prot, true, 0); + + pci_dev->doe_spdm.spdm_socket = spdm_socket_connect(pci_dev->spdm_port, + errp); + + if (pci_dev->doe_spdm.spdm_socket < 0) { + return false; + } + } + if (n->params.cmb_size_mb) { nvme_init_cmb(n, pci_dev); } @@ -8287,7 +8530,8 @@ static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev) id->nn = cpu_to_le32(NVME_MAX_NAMESPACES); id->oncs = cpu_to_le16(NVME_ONCS_WRITE_ZEROES | NVME_ONCS_TIMESTAMP | NVME_ONCS_FEATURES | NVME_ONCS_DSM | - NVME_ONCS_COMPARE | NVME_ONCS_COPY); + NVME_ONCS_COMPARE | NVME_ONCS_COPY | + NVME_ONCS_NVMCSA | NVME_ONCS_NVMAFC); /* * NOTE: If this device ever supports a command set that does NOT use 0x0 @@ -8298,7 +8542,8 @@ static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev) */ id->vwc = NVME_VWC_NSID_BROADCAST_SUPPORT | NVME_VWC_PRESENT; - id->ocfs = cpu_to_le16(NVME_OCFS_COPY_FORMAT_0 | NVME_OCFS_COPY_FORMAT_1); + id->ocfs = cpu_to_le16(NVME_OCFS_COPY_FORMAT_0 | NVME_OCFS_COPY_FORMAT_1 | + NVME_OCFS_COPY_FORMAT_2 | NVME_OCFS_COPY_FORMAT_3); id->sgls = cpu_to_le32(NVME_CTRL_SGLS_SUPPORT_NO_ALIGN); nvme_init_subnqn(n); @@ -8446,6 +8691,11 @@ static void nvme_exit(PCIDevice *pci_dev) g_free(n->cmb.buf); } + if (pci_dev->doe_spdm.spdm_socket > 0) { + spdm_socket_close(pci_dev->doe_spdm.spdm_socket, + SPDM_SOCKET_TRANSPORT_TYPE_PCI_DOE); + } + if (n->pmr.dev) { host_memory_backend_set_mapped(n->pmr.dev, false); } @@ -8491,6 +8741,7 @@ static Property nvme_props[] = { DEFINE_PROP_BOOL("msix-exclusive-bar", NvmeCtrl, params.msix_exclusive_bar, false), DEFINE_PROP_UINT16("mqes", NvmeCtrl, params.mqes, 0x7ff), + DEFINE_PROP_UINT16("spdm_port", PCIDevice, spdm_port, 0), DEFINE_PROP_END_OF_LIST(), }; @@ -8562,11 +8813,25 @@ static void nvme_pci_write_config(PCIDevice *dev, uint32_t address, { uint16_t old_num_vfs = pcie_sriov_num_vfs(dev); + if (pcie_find_capability(dev, PCI_EXT_CAP_ID_DOE)) { + pcie_doe_write_config(&dev->doe_spdm, address, val, len); + } pci_default_write_config(dev, address, val, len); pcie_cap_flr_write_config(dev, address, val, len); nvme_sriov_post_write_config(dev, old_num_vfs); } +static uint32_t nvme_pci_read_config(PCIDevice *dev, uint32_t address, int len) +{ + uint32_t val; + if (dev->spdm_port && pcie_find_capability(dev, PCI_EXT_CAP_ID_DOE)) { + if (pcie_doe_read_config(&dev->doe_spdm, address, len, &val)) { + return val; + } + } + return pci_default_read_config(dev, address, len); +} + static const VMStateDescription nvme_vmstate = { .name = "nvme", .unmigratable = 1, @@ -8579,6 +8844,7 @@ static void nvme_class_init(ObjectClass *oc, void *data) pc->realize = nvme_realize; pc->config_write = nvme_pci_write_config; + pc->config_read = nvme_pci_read_config; pc->exit = nvme_exit; pc->class_id = PCI_CLASS_STORAGE_EXPRESS; pc->revision = 2; |