diff options
Diffstat (limited to 'hw/arm/smmu-common.c')
| -rw-r--r-- | hw/arm/smmu-common.c | 214 |
1 files changed, 145 insertions, 69 deletions
diff --git a/hw/arm/smmu-common.c b/hw/arm/smmu-common.c index e13a5f4a7c..3838db1395 100644 --- a/hw/arm/smmu-common.c +++ b/hw/arm/smmu-common.c @@ -32,6 +32,91 @@ /* IOTLB Management */ +static guint smmu_iotlb_key_hash(gconstpointer v) +{ + SMMUIOTLBKey *key = (SMMUIOTLBKey *)v; + uint32_t a, b, c; + + /* Jenkins hash */ + a = b = c = JHASH_INITVAL + sizeof(*key); + a += key->asid + key->level + key->tg; + b += extract64(key->iova, 0, 32); + c += extract64(key->iova, 32, 32); + + __jhash_mix(a, b, c); + __jhash_final(a, b, c); + + return c; +} + +static gboolean smmu_iotlb_key_equal(gconstpointer v1, gconstpointer v2) +{ + SMMUIOTLBKey *k1 = (SMMUIOTLBKey *)v1, *k2 = (SMMUIOTLBKey *)v2; + + return (k1->asid == k2->asid) && (k1->iova == k2->iova) && + (k1->level == k2->level) && (k1->tg == k2->tg); +} + +SMMUIOTLBKey smmu_get_iotlb_key(uint16_t asid, uint64_t iova, + uint8_t tg, uint8_t level) +{ + SMMUIOTLBKey key = {.asid = asid, .iova = iova, .tg = tg, .level = level}; + + return key; +} + +SMMUTLBEntry *smmu_iotlb_lookup(SMMUState *bs, SMMUTransCfg *cfg, + SMMUTransTableInfo *tt, hwaddr iova) +{ + uint8_t tg = (tt->granule_sz - 10) / 2; + uint8_t inputsize = 64 - tt->tsz; + uint8_t stride = tt->granule_sz - 3; + uint8_t level = 4 - (inputsize - 4) / stride; + SMMUTLBEntry *entry = NULL; + + while (level <= 3) { + uint64_t subpage_size = 1ULL << level_shift(level, tt->granule_sz); + uint64_t mask = subpage_size - 1; + SMMUIOTLBKey key; + + key = smmu_get_iotlb_key(cfg->asid, iova & ~mask, tg, level); + entry = g_hash_table_lookup(bs->iotlb, &key); + if (entry) { + break; + } + level++; + } + + if (entry) { + cfg->iotlb_hits++; + trace_smmu_iotlb_lookup_hit(cfg->asid, iova, + cfg->iotlb_hits, cfg->iotlb_misses, + 100 * cfg->iotlb_hits / + (cfg->iotlb_hits + cfg->iotlb_misses)); + } else { + cfg->iotlb_misses++; + trace_smmu_iotlb_lookup_miss(cfg->asid, iova, + cfg->iotlb_hits, cfg->iotlb_misses, + 100 * cfg->iotlb_hits / + (cfg->iotlb_hits + cfg->iotlb_misses)); + } + return entry; +} + +void smmu_iotlb_insert(SMMUState *bs, SMMUTransCfg *cfg, SMMUTLBEntry *new) +{ + SMMUIOTLBKey *key = g_new0(SMMUIOTLBKey, 1); + uint8_t tg = (new->granule - 10) / 2; + + if (g_hash_table_size(bs->iotlb) >= SMMU_IOTLB_MAX_SIZE) { + smmu_iotlb_inv_all(bs); + } + + *key = smmu_get_iotlb_key(cfg->asid, new->entry.iova, tg, new->level); + trace_smmu_iotlb_insert(cfg->asid, new->entry.iova, tg, new->level); + g_hash_table_insert(bs->iotlb, key, new); +} + inline void smmu_iotlb_inv_all(SMMUState *s) { trace_smmu_iotlb_inv_all(); @@ -44,15 +129,44 @@ static gboolean smmu_hash_remove_by_asid(gpointer key, gpointer value, uint16_t asid = *(uint16_t *)user_data; SMMUIOTLBKey *iotlb_key = (SMMUIOTLBKey *)key; - return iotlb_key->asid == asid; + return SMMU_IOTLB_ASID(*iotlb_key) == asid; } -inline void smmu_iotlb_inv_iova(SMMUState *s, uint16_t asid, dma_addr_t iova) +static gboolean smmu_hash_remove_by_asid_iova(gpointer key, gpointer value, + gpointer user_data) { - SMMUIOTLBKey key = {.asid = asid, .iova = iova}; + SMMUTLBEntry *iter = (SMMUTLBEntry *)value; + IOMMUTLBEntry *entry = &iter->entry; + SMMUIOTLBPageInvInfo *info = (SMMUIOTLBPageInvInfo *)user_data; + SMMUIOTLBKey iotlb_key = *(SMMUIOTLBKey *)key; - trace_smmu_iotlb_inv_iova(asid, iova); - g_hash_table_remove(s->iotlb, &key); + if (info->asid >= 0 && info->asid != SMMU_IOTLB_ASID(iotlb_key)) { + return false; + } + return ((info->iova & ~entry->addr_mask) == entry->iova) || + ((entry->iova & ~info->mask) == info->iova); +} + +inline void +smmu_iotlb_inv_iova(SMMUState *s, int asid, dma_addr_t iova, + uint8_t tg, uint64_t num_pages, uint8_t ttl) +{ + if (ttl && (num_pages == 1)) { + SMMUIOTLBKey key = smmu_get_iotlb_key(asid, iova, tg, ttl); + + g_hash_table_remove(s->iotlb, &key); + } else { + /* if tg is not set we use 4KB range invalidation */ + uint8_t granule = tg ? tg * 2 + 10 : 12; + + SMMUIOTLBPageInvInfo info = { + .asid = asid, .iova = iova, + .mask = (num_pages * 1 << granule) - 1}; + + g_hash_table_foreach_remove(s->iotlb, + smmu_hash_remove_by_asid_iova, + &info); + } } inline void smmu_iotlb_inv_asid(SMMUState *s, uint16_t asid) @@ -149,7 +263,7 @@ SMMUTransTableInfo *select_tt(SMMUTransCfg *cfg, dma_addr_t iova) * @cfg: translation config * @iova: iova to translate * @perm: access type - * @tlbe: IOMMUTLBEntry (out) + * @tlbe: SMMUTLBEntry (out) * @info: handle to an error info * * Return 0 on success, < 0 on error. In case of error, @info is filled @@ -159,7 +273,7 @@ SMMUTransTableInfo *select_tt(SMMUTransCfg *cfg, dma_addr_t iova) */ static int smmu_ptw_64(SMMUTransCfg *cfg, dma_addr_t iova, IOMMUAccessFlags perm, - IOMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) + SMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) { dma_addr_t baseaddr, indexmask; int stage = cfg->stage; @@ -179,14 +293,11 @@ static int smmu_ptw_64(SMMUTransCfg *cfg, baseaddr = extract64(tt->ttb, 0, 48); baseaddr &= ~indexmask; - tlbe->iova = iova; - tlbe->addr_mask = (1 << granule_sz) - 1; - while (level <= 3) { uint64_t subpage_size = 1ULL << level_shift(level, granule_sz); uint64_t mask = subpage_size - 1; uint32_t offset = iova_level_offset(iova, inputsize, level, granule_sz); - uint64_t pte; + uint64_t pte, gpa; dma_addr_t pte_addr = baseaddr + offset * sizeof(pte); uint8_t ap; @@ -199,60 +310,50 @@ static int smmu_ptw_64(SMMUTransCfg *cfg, if (is_invalid_pte(pte) || is_reserved_pte(pte, level)) { trace_smmu_ptw_invalid_pte(stage, level, baseaddr, pte_addr, offset, pte); - info->type = SMMU_PTW_ERR_TRANSLATION; - goto error; + break; } - if (is_page_pte(pte, level)) { - uint64_t gpa = get_page_pte_address(pte, granule_sz); + if (is_table_pte(pte, level)) { + ap = PTE_APTABLE(pte); - ap = PTE_AP(pte); - if (is_permission_fault(ap, perm)) { + if (is_permission_fault(ap, perm) && !tt->had) { info->type = SMMU_PTW_ERR_PERMISSION; goto error; } - - tlbe->translated_addr = gpa + (iova & mask); - tlbe->perm = PTE_AP_TO_PERM(ap); + baseaddr = get_table_pte_address(pte, granule_sz); + level++; + continue; + } else if (is_page_pte(pte, level)) { + gpa = get_page_pte_address(pte, granule_sz); trace_smmu_ptw_page_pte(stage, level, iova, baseaddr, pte_addr, pte, gpa); - return 0; - } - if (is_block_pte(pte, level)) { + } else { uint64_t block_size; - hwaddr gpa = get_block_pte_address(pte, level, granule_sz, - &block_size); - - ap = PTE_AP(pte); - if (is_permission_fault(ap, perm)) { - info->type = SMMU_PTW_ERR_PERMISSION; - goto error; - } + gpa = get_block_pte_address(pte, level, granule_sz, + &block_size); trace_smmu_ptw_block_pte(stage, level, baseaddr, pte_addr, pte, iova, gpa, block_size >> 20); - - tlbe->translated_addr = gpa + (iova & mask); - tlbe->perm = PTE_AP_TO_PERM(ap); - return 0; } - - /* table pte */ - ap = PTE_APTABLE(pte); - + ap = PTE_AP(pte); if (is_permission_fault(ap, perm)) { info->type = SMMU_PTW_ERR_PERMISSION; goto error; } - baseaddr = get_table_pte_address(pte, granule_sz); - level++; - } + tlbe->entry.translated_addr = gpa; + tlbe->entry.iova = iova & ~mask; + tlbe->entry.addr_mask = mask; + tlbe->entry.perm = PTE_AP_TO_PERM(ap); + tlbe->level = level; + tlbe->granule = granule_sz; + return 0; + } info->type = SMMU_PTW_ERR_TRANSLATION; error: - tlbe->perm = IOMMU_NONE; + tlbe->entry.perm = IOMMU_NONE; return -EINVAL; } @@ -268,7 +369,7 @@ error: * return 0 on success */ inline int smmu_ptw(SMMUTransCfg *cfg, dma_addr_t iova, IOMMUAccessFlags perm, - IOMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) + SMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) { if (!cfg->aa64) { /* @@ -361,31 +462,6 @@ IOMMUMemoryRegion *smmu_iommu_mr(SMMUState *s, uint32_t sid) return NULL; } -static guint smmu_iotlb_key_hash(gconstpointer v) -{ - SMMUIOTLBKey *key = (SMMUIOTLBKey *)v; - uint32_t a, b, c; - - /* Jenkins hash */ - a = b = c = JHASH_INITVAL + sizeof(*key); - a += key->asid; - b += extract64(key->iova, 0, 32); - c += extract64(key->iova, 32, 32); - - __jhash_mix(a, b, c); - __jhash_final(a, b, c); - - return c; -} - -static gboolean smmu_iotlb_key_equal(gconstpointer v1, gconstpointer v2) -{ - const SMMUIOTLBKey *k1 = v1; - const SMMUIOTLBKey *k2 = v2; - - return (k1->asid == k2->asid) && (k1->iova == k2->iova); -} - /* Unmap the whole notifier's range */ static void smmu_unmap_notifier_range(IOMMUNotifier *n) { |