diff options
Diffstat (limited to 'hw/arm/smmuv3.c')
| -rw-r--r-- | hw/arm/smmuv3.c | 467 |
1 files changed, 307 insertions, 160 deletions
diff --git a/hw/arm/smmuv3.c b/hw/arm/smmuv3.c index 445e04ddf7..3971976389 100644 --- a/hw/arm/smmuv3.c +++ b/hw/arm/smmuv3.c @@ -34,8 +34,10 @@ #include "smmuv3-internal.h" #include "smmu-internal.h" -#define PTW_RECORD_FAULT(cfg) (((cfg)->stage == 1) ? (cfg)->record_faults : \ - (cfg)->s2cfg.record_faults) +#define PTW_RECORD_FAULT(ptw_info, cfg) (((ptw_info).stage == SMMU_STAGE_1 && \ + (cfg)->record_faults) || \ + ((ptw_info).stage == SMMU_STAGE_2 && \ + (cfg)->s2cfg.record_faults)) /** * smmuv3_trigger_irq - pulse @irq if enabled and update @@ -259,6 +261,9 @@ static void smmuv3_init_regs(SMMUv3State *s) /* Based on sys property, the stages supported in smmu will be advertised.*/ if (s->stage && !strcmp("2", s->stage)) { s->idr[0] = FIELD_DP32(s->idr[0], IDR0, S2P, 1); + } else if (s->stage && !strcmp("nested", s->stage)) { + s->idr[0] = FIELD_DP32(s->idr[0], IDR0, S1P, 1); + s->idr[0] = FIELD_DP32(s->idr[0], IDR0, S2P, 1); } else { s->idr[0] = FIELD_DP32(s->idr[0], IDR0, S1P, 1); } @@ -336,14 +341,35 @@ static int smmu_get_ste(SMMUv3State *s, dma_addr_t addr, STE *buf, } +static SMMUTranslationStatus smmuv3_do_translate(SMMUv3State *s, hwaddr addr, + SMMUTransCfg *cfg, + SMMUEventInfo *event, + IOMMUAccessFlags flag, + SMMUTLBEntry **out_entry, + SMMUTranslationClass class); /* @ssid > 0 not supported yet */ -static int smmu_get_cd(SMMUv3State *s, STE *ste, uint32_t ssid, - CD *buf, SMMUEventInfo *event) +static int smmu_get_cd(SMMUv3State *s, STE *ste, SMMUTransCfg *cfg, + uint32_t ssid, CD *buf, SMMUEventInfo *event) { dma_addr_t addr = STE_CTXPTR(ste); int ret, i; + SMMUTranslationStatus status; + SMMUTLBEntry *entry; trace_smmuv3_get_cd(addr); + + if (cfg->stage == SMMU_NESTED) { + status = smmuv3_do_translate(s, addr, cfg, event, + IOMMU_RO, &entry, SMMU_CLASS_CD); + + /* Same PTW faults are reported but with CLASS = CD. */ + if (status != SMMU_TRANS_SUCCESS) { + return -EINVAL; + } + + addr = CACHED_ENTRY_TO_ADDR(entry, addr); + } + /* TODO: guarantee 64-bit single-copy atomicity */ ret = dma_memory_read(&address_space_memory, addr, buf, sizeof(*buf), MEMTXATTRS_UNSPECIFIED); @@ -376,10 +402,10 @@ static bool s2t0sz_valid(SMMUTransCfg *cfg) } if (cfg->s2cfg.granule_sz == 16) { - return (cfg->s2cfg.tsz >= 64 - oas2bits(SMMU_IDR5_OAS)); + return (cfg->s2cfg.tsz >= 64 - cfg->s2cfg.eff_ps); } - return (cfg->s2cfg.tsz >= MAX(64 - oas2bits(SMMU_IDR5_OAS), 16)); + return (cfg->s2cfg.tsz >= MAX(64 - cfg->s2cfg.eff_ps, 16)); } /* @@ -400,9 +426,10 @@ static bool s2_pgtable_config_valid(uint8_t sl0, uint8_t t0sz, uint8_t gran) return nr_concat <= VMSA_MAX_S2_CONCAT; } -static int decode_ste_s2_cfg(SMMUTransCfg *cfg, STE *ste) +static int decode_ste_s2_cfg(SMMUv3State *s, SMMUTransCfg *cfg, + STE *ste) { - cfg->stage = 2; + uint8_t oas = FIELD_EX32(s->idr[5], IDR5, OAS); if (STE_S2AA64(ste) == 0x0) { qemu_log_mask(LOG_UNIMP, @@ -436,7 +463,15 @@ static int decode_ste_s2_cfg(SMMUTransCfg *cfg, STE *ste) } /* For AA64, The effective S2PS size is capped to the OAS. */ - cfg->s2cfg.eff_ps = oas2bits(MIN(STE_S2PS(ste), SMMU_IDR5_OAS)); + cfg->s2cfg.eff_ps = oas2bits(MIN(STE_S2PS(ste), oas)); + /* + * For SMMUv3.1 and later, when OAS == IAS == 52, the stage 2 input + * range is further limited to 48 bits unless STE.S2TG indicates a + * 64KB granule. + */ + if (cfg->s2cfg.granule_sz != 16) { + cfg->s2cfg.eff_ps = MIN(cfg->s2cfg.eff_ps, 48); + } /* * It is ILLEGAL for the address in S2TTB to be outside the range * described by the effective S2PS value. @@ -486,11 +521,33 @@ bad_ste: return -EINVAL; } +static void decode_ste_config(SMMUTransCfg *cfg, uint32_t config) +{ + + if (STE_CFG_ABORT(config)) { + cfg->aborted = true; + return; + } + if (STE_CFG_BYPASS(config)) { + cfg->bypassed = true; + return; + } + + if (STE_CFG_S1_ENABLED(config)) { + cfg->stage = SMMU_STAGE_1; + } + + if (STE_CFG_S2_ENABLED(config)) { + cfg->stage |= SMMU_STAGE_2; + } +} + /* Returns < 0 in case of invalid STE, 0 otherwise */ static int decode_ste(SMMUv3State *s, SMMUTransCfg *cfg, STE *ste, SMMUEventInfo *event) { uint32_t config; + uint8_t oas = FIELD_EX32(s->idr[5], IDR5, OAS); int ret; if (!STE_VALID(ste)) { @@ -502,13 +559,9 @@ static int decode_ste(SMMUv3State *s, SMMUTransCfg *cfg, config = STE_CONFIG(ste); - if (STE_CFG_ABORT(config)) { - cfg->aborted = true; - return 0; - } + decode_ste_config(cfg, config); - if (STE_CFG_BYPASS(config)) { - cfg->bypassed = true; + if (cfg->aborted || cfg->bypassed) { return 0; } @@ -538,8 +591,8 @@ static int decode_ste(SMMUv3State *s, SMMUTransCfg *cfg, * Stage-1 OAS defaults to OAS even if not enabled as it would be used * in input address check for stage-2. */ - cfg->oas = oas2bits(SMMU_IDR5_OAS); - ret = decode_ste_s2_cfg(cfg, ste); + cfg->oas = oas2bits(oas); + ret = decode_ste_s2_cfg(s, cfg, ste); if (ret) { goto bad_ste; } @@ -658,10 +711,14 @@ static int smmu_find_ste(SMMUv3State *s, uint32_t sid, STE *ste, return 0; } -static int decode_cd(SMMUTransCfg *cfg, CD *cd, SMMUEventInfo *event) +static int decode_cd(SMMUv3State *s, SMMUTransCfg *cfg, + CD *cd, SMMUEventInfo *event) { int ret = -EINVAL; int i; + SMMUTranslationStatus status; + SMMUTLBEntry *entry; + uint8_t oas = FIELD_EX32(s->idr[5], IDR5, OAS); if (!CD_VALID(cd) || !CD_AARCH64(cd)) { goto bad_cd; @@ -678,10 +735,9 @@ static int decode_cd(SMMUTransCfg *cfg, CD *cd, SMMUEventInfo *event) /* we support only those at the moment */ cfg->aa64 = true; - cfg->stage = 1; cfg->oas = oas2bits(CD_IPS(cd)); - cfg->oas = MIN(oas2bits(SMMU_IDR5_OAS), cfg->oas); + cfg->oas = MIN(oas2bits(oas), cfg->oas); cfg->tbi = CD_TBI(cd); cfg->asid = CD_ASID(cd); cfg->affd = CD_AFFD(cd); @@ -710,11 +766,36 @@ static int decode_cd(SMMUTransCfg *cfg, CD *cd, SMMUEventInfo *event) goto bad_cd; } + /* + * An address greater than 48 bits in size can only be output from a + * TTD when, in SMMUv3.1 and later, the effective IPS is 52 and a 64KB + * granule is in use for that translation table + */ + if (tt->granule_sz != 16) { + cfg->oas = MIN(cfg->oas, 48); + } tt->tsz = tsz; tt->ttb = CD_TTB(cd, i); + if (tt->ttb & ~(MAKE_64BIT_MASK(0, cfg->oas))) { goto bad_cd; } + + /* Translate the TTBx, from IPA to PA if nesting is enabled. */ + if (cfg->stage == SMMU_NESTED) { + status = smmuv3_do_translate(s, tt->ttb, cfg, event, IOMMU_RO, + &entry, SMMU_CLASS_TT); + /* + * Same PTW faults are reported but with CLASS = TT. + * If TTBx is larger than the effective stage 1 output addres + * size, it reports C_BAD_CD, which is handled by the above case. + */ + if (status != SMMU_TRANS_SUCCESS) { + return -EINVAL; + } + tt->ttb = CACHED_ENTRY_TO_ADDR(entry, tt->ttb); + } + tt->had = CD_HAD(cd, i); trace_smmuv3_decode_cd_tt(i, tt->tsz, tt->ttb, tt->granule_sz, tt->had); } @@ -762,16 +843,16 @@ static int smmuv3_decode_config(IOMMUMemoryRegion *mr, SMMUTransCfg *cfg, return ret; } - if (cfg->aborted || cfg->bypassed || (cfg->stage == 2)) { + if (cfg->aborted || cfg->bypassed || (cfg->stage == SMMU_STAGE_2)) { return 0; } - ret = smmu_get_cd(s, &ste, 0 /* ssid */, &cd, event); + ret = smmu_get_cd(s, &ste, cfg, 0 /* ssid */, &cd, event); if (ret) { return ret; } - return decode_cd(cfg, &cd, event); + return decode_cd(s, cfg, &cd, event); } /** @@ -826,6 +907,133 @@ static void smmuv3_flush_config(SMMUDevice *sdev) g_hash_table_remove(bc->configs, sdev); } +/* Do translation with TLB lookup. */ +static SMMUTranslationStatus smmuv3_do_translate(SMMUv3State *s, hwaddr addr, + SMMUTransCfg *cfg, + SMMUEventInfo *event, + IOMMUAccessFlags flag, + SMMUTLBEntry **out_entry, + SMMUTranslationClass class) +{ + SMMUPTWEventInfo ptw_info = {}; + SMMUState *bs = ARM_SMMU(s); + SMMUTLBEntry *cached_entry = NULL; + int asid, stage; + bool desc_s2_translation = class != SMMU_CLASS_IN; + + /* + * The function uses the argument class to identify which stage is used: + * - CLASS = IN: Means an input translation, determine the stage from STE. + * - CLASS = CD: Means the addr is an IPA of the CD, and it would be + * translated using the stage-2. + * - CLASS = TT: Means the addr is an IPA of the stage-1 translation table + * and it would be translated using the stage-2. + * For the last 2 cases instead of having intrusive changes in the common + * logic, we modify the cfg to be a stage-2 translation only in case of + * nested, and then restore it after. + */ + if (desc_s2_translation) { + asid = cfg->asid; + stage = cfg->stage; + cfg->asid = -1; + cfg->stage = SMMU_STAGE_2; + } + + cached_entry = smmu_translate(bs, cfg, addr, flag, &ptw_info); + + if (desc_s2_translation) { + cfg->asid = asid; + cfg->stage = stage; + } + + if (!cached_entry) { + /* All faults from PTW has S2 field. */ + event->u.f_walk_eabt.s2 = (ptw_info.stage == SMMU_STAGE_2); + /* + * Fault class is set as follows based on "class" input to + * the function and to "ptw_info" from "smmu_translate()" + * For stage-1: + * - EABT => CLASS_TT (hardcoded) + * - other events => CLASS_IN (input to function) + * For stage-2 => CLASS_IN (input to function) + * For nested, for all events: + * - CD fetch => CLASS_CD (input to function) + * - walking stage 1 translation table => CLASS_TT (from + * is_ipa_descriptor or input in case of TTBx) + * - s2 translation => CLASS_IN (input to function) + */ + class = ptw_info.is_ipa_descriptor ? SMMU_CLASS_TT : class; + switch (ptw_info.type) { + case SMMU_PTW_ERR_WALK_EABT: + event->type = SMMU_EVT_F_WALK_EABT; + event->u.f_walk_eabt.rnw = flag & 0x1; + event->u.f_walk_eabt.class = (ptw_info.stage == SMMU_STAGE_2) ? + class : SMMU_CLASS_TT; + event->u.f_walk_eabt.addr2 = ptw_info.addr; + break; + case SMMU_PTW_ERR_TRANSLATION: + if (PTW_RECORD_FAULT(ptw_info, cfg)) { + event->type = SMMU_EVT_F_TRANSLATION; + event->u.f_translation.addr2 = ptw_info.addr; + event->u.f_translation.class = class; + event->u.f_translation.rnw = flag & 0x1; + } + break; + case SMMU_PTW_ERR_ADDR_SIZE: + if (PTW_RECORD_FAULT(ptw_info, cfg)) { + event->type = SMMU_EVT_F_ADDR_SIZE; + event->u.f_addr_size.addr2 = ptw_info.addr; + event->u.f_addr_size.class = class; + event->u.f_addr_size.rnw = flag & 0x1; + } + break; + case SMMU_PTW_ERR_ACCESS: + if (PTW_RECORD_FAULT(ptw_info, cfg)) { + event->type = SMMU_EVT_F_ACCESS; + event->u.f_access.addr2 = ptw_info.addr; + event->u.f_access.class = class; + event->u.f_access.rnw = flag & 0x1; + } + break; + case SMMU_PTW_ERR_PERMISSION: + if (PTW_RECORD_FAULT(ptw_info, cfg)) { + event->type = SMMU_EVT_F_PERMISSION; + event->u.f_permission.addr2 = ptw_info.addr; + event->u.f_permission.class = class; + event->u.f_permission.rnw = flag & 0x1; + } + break; + default: + g_assert_not_reached(); + } + return SMMU_TRANS_ERROR; + } + *out_entry = cached_entry; + return SMMU_TRANS_SUCCESS; +} + +/* + * Sets the InputAddr for an SMMU_TRANS_ERROR, as it can't be + * set from all contexts, as smmuv3_get_config() can return + * translation faults in case of nested translation (for CD + * and TTBx). But in that case the iova is not known. + */ +static void smmuv3_fixup_event(SMMUEventInfo *event, hwaddr iova) +{ + switch (event->type) { + case SMMU_EVT_F_WALK_EABT: + case SMMU_EVT_F_TRANSLATION: + case SMMU_EVT_F_ADDR_SIZE: + case SMMU_EVT_F_ACCESS: + case SMMU_EVT_F_PERMISSION: + event->u.f_walk_eabt.addr = iova; + break; + default: + break; + } +} + +/* Entry point to SMMU, does everything. */ static IOMMUTLBEntry smmuv3_translate(IOMMUMemoryRegion *mr, hwaddr addr, IOMMUAccessFlags flag, int iommu_idx) { @@ -835,12 +1043,7 @@ static IOMMUTLBEntry smmuv3_translate(IOMMUMemoryRegion *mr, hwaddr addr, SMMUEventInfo event = {.type = SMMU_EVT_NONE, .sid = sid, .inval_ste_allowed = false}; - SMMUPTWEventInfo ptw_info = {}; SMMUTranslationStatus status; - SMMUState *bs = ARM_SMMU(s); - uint64_t page_mask, aligned_addr; - SMMUTLBEntry *cached_entry = NULL; - SMMUTransTableInfo *tt; SMMUTransCfg *cfg = NULL; IOMMUTLBEntry entry = { .target_as = &address_space_memory, @@ -849,11 +1052,7 @@ static IOMMUTLBEntry smmuv3_translate(IOMMUMemoryRegion *mr, hwaddr addr, .addr_mask = ~(hwaddr)0, .perm = IOMMU_NONE, }; - /* - * Combined attributes used for TLB lookup, as only one stage is supported, - * it will hold attributes based on the enabled stage. - */ - SMMUTransTableInfo tt_combined; + SMMUTLBEntry *cached_entry = NULL; qemu_mutex_lock(&s->mutex); @@ -882,116 +1081,19 @@ static IOMMUTLBEntry smmuv3_translate(IOMMUMemoryRegion *mr, hwaddr addr, goto epilogue; } - if (cfg->stage == 1) { - /* Select stage1 translation table. */ - tt = select_tt(cfg, addr); - if (!tt) { - if (cfg->record_faults) { - event.type = SMMU_EVT_F_TRANSLATION; - event.u.f_translation.addr = addr; - event.u.f_translation.rnw = flag & 0x1; - } - status = SMMU_TRANS_ERROR; - goto epilogue; - } - tt_combined.granule_sz = tt->granule_sz; - tt_combined.tsz = tt->tsz; - - } else { - /* Stage2. */ - tt_combined.granule_sz = cfg->s2cfg.granule_sz; - tt_combined.tsz = cfg->s2cfg.tsz; - } - /* - * TLB lookup looks for granule and input size for a translation stage, - * as only one stage is supported right now, choose the right values - * from the configuration. - */ - page_mask = (1ULL << tt_combined.granule_sz) - 1; - aligned_addr = addr & ~page_mask; - - cached_entry = smmu_iotlb_lookup(bs, cfg, &tt_combined, aligned_addr); - if (cached_entry) { - if ((flag & IOMMU_WO) && !(cached_entry->entry.perm & IOMMU_WO)) { - status = SMMU_TRANS_ERROR; - /* - * We know that the TLB only contains either stage-1 or stage-2 as - * nesting is not supported. So it is sufficient to check the - * translation stage to know the TLB stage for now. - */ - event.u.f_walk_eabt.s2 = (cfg->stage == 2); - if (PTW_RECORD_FAULT(cfg)) { - event.type = SMMU_EVT_F_PERMISSION; - event.u.f_permission.addr = addr; - event.u.f_permission.rnw = flag & 0x1; - } - } else { - status = SMMU_TRANS_SUCCESS; - } - goto epilogue; - } - - cached_entry = g_new0(SMMUTLBEntry, 1); - - if (smmu_ptw(cfg, aligned_addr, flag, cached_entry, &ptw_info)) { - /* All faults from PTW has S2 field. */ - event.u.f_walk_eabt.s2 = (ptw_info.stage == 2); - g_free(cached_entry); - switch (ptw_info.type) { - case SMMU_PTW_ERR_WALK_EABT: - event.type = SMMU_EVT_F_WALK_EABT; - event.u.f_walk_eabt.addr = addr; - event.u.f_walk_eabt.rnw = flag & 0x1; - event.u.f_walk_eabt.class = 0x1; - event.u.f_walk_eabt.addr2 = ptw_info.addr; - break; - case SMMU_PTW_ERR_TRANSLATION: - if (PTW_RECORD_FAULT(cfg)) { - event.type = SMMU_EVT_F_TRANSLATION; - event.u.f_translation.addr = addr; - event.u.f_translation.rnw = flag & 0x1; - } - break; - case SMMU_PTW_ERR_ADDR_SIZE: - if (PTW_RECORD_FAULT(cfg)) { - event.type = SMMU_EVT_F_ADDR_SIZE; - event.u.f_addr_size.addr = addr; - event.u.f_addr_size.rnw = flag & 0x1; - } - break; - case SMMU_PTW_ERR_ACCESS: - if (PTW_RECORD_FAULT(cfg)) { - event.type = SMMU_EVT_F_ACCESS; - event.u.f_access.addr = addr; - event.u.f_access.rnw = flag & 0x1; - } - break; - case SMMU_PTW_ERR_PERMISSION: - if (PTW_RECORD_FAULT(cfg)) { - event.type = SMMU_EVT_F_PERMISSION; - event.u.f_permission.addr = addr; - event.u.f_permission.rnw = flag & 0x1; - } - break; - default: - g_assert_not_reached(); - } - status = SMMU_TRANS_ERROR; - } else { - smmu_iotlb_insert(bs, cfg, cached_entry); - status = SMMU_TRANS_SUCCESS; - } + status = smmuv3_do_translate(s, addr, cfg, &event, flag, + &cached_entry, SMMU_CLASS_IN); epilogue: qemu_mutex_unlock(&s->mutex); switch (status) { case SMMU_TRANS_SUCCESS: entry.perm = cached_entry->entry.perm; - entry.translated_addr = cached_entry->entry.translated_addr + - (addr & cached_entry->entry.addr_mask); + entry.translated_addr = CACHED_ENTRY_TO_ADDR(cached_entry, addr); entry.addr_mask = cached_entry->entry.addr_mask; trace_smmuv3_translate_success(mr->parent_obj.name, sid, addr, - entry.translated_addr, entry.perm); + entry.translated_addr, entry.perm, + cfg->stage); break; case SMMU_TRANS_DISABLE: entry.perm = flag; @@ -1011,6 +1113,7 @@ epilogue: entry.perm); break; case SMMU_TRANS_ERROR: + smmuv3_fixup_event(&event, addr); qemu_log_mask(LOG_GUEST_ERROR, "%s translation failed for iova=0x%"PRIx64" (%s)\n", mr->parent_obj.name, addr, smmu_event_string(event.type)); @@ -1032,27 +1135,38 @@ epilogue: * @iova: iova * @tg: translation granule (if communicated through range invalidation) * @num_pages: number of @granule sized pages (if tg != 0), otherwise 1 + * @stage: Which stage(1 or 2) is used */ static void smmuv3_notify_iova(IOMMUMemoryRegion *mr, IOMMUNotifier *n, int asid, int vmid, dma_addr_t iova, uint8_t tg, - uint64_t num_pages) + uint64_t num_pages, int stage) { SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu); + SMMUEventInfo eventinfo = {.inval_ste_allowed = true}; + SMMUTransCfg *cfg = smmuv3_get_config(sdev, &eventinfo); IOMMUTLBEvent event; uint8_t granule; - SMMUv3State *s = sdev->smmu; + + if (!cfg) { + return; + } + + /* + * stage is passed from TLB invalidation commands which can be either + * stage-1 or stage-2. + * However, IOMMUTLBEvent only understands IOVA, for stage-1 or stage-2 + * SMMU instances we consider the input address as the IOVA, but when + * nesting is used, we can't mix stage-1 and stage-2 addresses, so for + * nesting only stage-1 is considered the IOVA and would be notified. + */ + if ((stage == SMMU_STAGE_2) && (cfg->stage == SMMU_NESTED)) + return; if (!tg) { - SMMUEventInfo eventinfo = {.inval_ste_allowed = true}; - SMMUTransCfg *cfg = smmuv3_get_config(sdev, &eventinfo); SMMUTransTableInfo *tt; - if (!cfg) { - return; - } - if (asid >= 0 && cfg->asid != asid) { return; } @@ -1061,7 +1175,7 @@ static void smmuv3_notify_iova(IOMMUMemoryRegion *mr, return; } - if (STAGE1_SUPPORTED(s)) { + if (stage == SMMU_STAGE_1) { tt = select_tt(cfg, iova); if (!tt) { return; @@ -1087,7 +1201,7 @@ static void smmuv3_notify_iova(IOMMUMemoryRegion *mr, /* invalidate an asid/vmid/iova range tuple in all mr's */ static void smmuv3_inv_notifiers_iova(SMMUState *s, int asid, int vmid, dma_addr_t iova, uint8_t tg, - uint64_t num_pages) + uint64_t num_pages, int stage) { SMMUDevice *sdev; @@ -1096,15 +1210,15 @@ static void smmuv3_inv_notifiers_iova(SMMUState *s, int asid, int vmid, IOMMUNotifier *n; trace_smmuv3_inv_notifiers_iova(mr->parent_obj.name, asid, vmid, - iova, tg, num_pages); + iova, tg, num_pages, stage); IOMMU_NOTIFIER_FOREACH(n, mr) { - smmuv3_notify_iova(mr, n, asid, vmid, iova, tg, num_pages); + smmuv3_notify_iova(mr, n, asid, vmid, iova, tg, num_pages, stage); } } } -static void smmuv3_range_inval(SMMUState *s, Cmd *cmd) +static void smmuv3_range_inval(SMMUState *s, Cmd *cmd, SMMUStage stage) { dma_addr_t end, addr = CMD_ADDR(cmd); uint8_t type = CMD_TYPE(cmd); @@ -1129,9 +1243,13 @@ static void smmuv3_range_inval(SMMUState *s, Cmd *cmd) } if (!tg) { - trace_smmuv3_range_inval(vmid, asid, addr, tg, 1, ttl, leaf); - smmuv3_inv_notifiers_iova(s, asid, vmid, addr, tg, 1); - smmu_iotlb_inv_iova(s, asid, vmid, addr, tg, 1, ttl); + trace_smmuv3_range_inval(vmid, asid, addr, tg, 1, ttl, leaf, stage); + smmuv3_inv_notifiers_iova(s, asid, vmid, addr, tg, 1, stage); + if (stage == SMMU_STAGE_1) { + smmu_iotlb_inv_iova(s, asid, vmid, addr, tg, 1, ttl); + } else { + smmu_iotlb_inv_ipa(s, vmid, addr, tg, 1, ttl); + } return; } @@ -1147,9 +1265,14 @@ static void smmuv3_range_inval(SMMUState *s, Cmd *cmd) uint64_t mask = dma_aligned_pow2_mask(addr, end, 64); num_pages = (mask + 1) >> granule; - trace_smmuv3_range_inval(vmid, asid, addr, tg, num_pages, ttl, leaf); - smmuv3_inv_notifiers_iova(s, asid, vmid, addr, tg, num_pages); - smmu_iotlb_inv_iova(s, asid, vmid, addr, tg, num_pages, ttl); + trace_smmuv3_range_inval(vmid, asid, addr, tg, num_pages, + ttl, leaf, stage); + smmuv3_inv_notifiers_iova(s, asid, vmid, addr, tg, num_pages, stage); + if (stage == SMMU_STAGE_1) { + smmu_iotlb_inv_iova(s, asid, vmid, addr, tg, num_pages, ttl); + } else { + smmu_iotlb_inv_ipa(s, vmid, addr, tg, num_pages, ttl); + } addr += mask + 1; } } @@ -1275,26 +1398,50 @@ static int smmuv3_cmdq_consume(SMMUv3State *s) } case SMMU_CMD_TLBI_NH_ASID: { - uint16_t asid = CMD_ASID(&cmd); + int asid = CMD_ASID(&cmd); + int vmid = -1; if (!STAGE1_SUPPORTED(s)) { cmd_error = SMMU_CERROR_ILL; break; } + /* + * VMID is only matched when stage 2 is supported, otherwise set it + * to -1 as the value used for stage-1 only VMIDs. + */ + if (STAGE2_SUPPORTED(s)) { + vmid = CMD_VMID(&cmd); + } + trace_smmuv3_cmdq_tlbi_nh_asid(asid); smmu_inv_notifiers_all(&s->smmu_state); - smmu_iotlb_inv_asid(bs, asid); + smmu_iotlb_inv_asid_vmid(bs, asid, vmid); break; } case SMMU_CMD_TLBI_NH_ALL: + { + int vmid = -1; + if (!STAGE1_SUPPORTED(s)) { cmd_error = SMMU_CERROR_ILL; break; } + + /* + * If stage-2 is supported, invalidate for this VMID only, otherwise + * invalidate the whole thing. + */ + if (STAGE2_SUPPORTED(s)) { + vmid = CMD_VMID(&cmd); + trace_smmuv3_cmdq_tlbi_nh(vmid); + smmu_iotlb_inv_vmid_s1(bs, vmid); + break; + } QEMU_FALLTHROUGH; + } case SMMU_CMD_TLBI_NSNH_ALL: - trace_smmuv3_cmdq_tlbi_nh(); + trace_smmuv3_cmdq_tlbi_nsnh(); smmu_inv_notifiers_all(&s->smmu_state); smmu_iotlb_inv_all(bs); break; @@ -1304,11 +1451,11 @@ static int smmuv3_cmdq_consume(SMMUv3State *s) cmd_error = SMMU_CERROR_ILL; break; } - smmuv3_range_inval(bs, &cmd); + smmuv3_range_inval(bs, &cmd, SMMU_STAGE_1); break; case SMMU_CMD_TLBI_S12_VMALL: { - uint16_t vmid = CMD_VMID(&cmd); + int vmid = CMD_VMID(&cmd); if (!STAGE2_SUPPORTED(s)) { cmd_error = SMMU_CERROR_ILL; @@ -1329,7 +1476,7 @@ static int smmuv3_cmdq_consume(SMMUv3State *s) * As currently only either s1 or s2 are supported * we can reuse same function for s2. */ - smmuv3_range_inval(bs, &cmd); + smmuv3_range_inval(bs, &cmd, SMMU_STAGE_2); break; case SMMU_CMD_TLBI_EL3_ALL: case SMMU_CMD_TLBI_EL3_VA: |