path: root/hw/i386/amd_iommu.c

/*
 * QEMU emulation of AMD IOMMU (AMD-Vi)
 *
 * Copyright (C) 2011 Eduard - Gabriel Munteanu
 * Copyright (C) 2015, 2016 David Kiarie Kahurani
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.

 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.

 * You should have received a copy of the GNU General Public License along
 * with this program; if not, see <http://www.gnu.org/licenses/>.
 *
 * Cache implementation inspired by hw/i386/intel_iommu.c
 */

#include "qemu/osdep.h"
#include "hw/i386/pc.h"
#include "hw/pci/msi.h"
#include "hw/pci/pci_bus.h"
#include "migration/vmstate.h"
#include "amd_iommu.h"
#include "qapi/error.h"
#include "qemu/error-report.h"
#include "hw/i386/apic_internal.h"
#include "trace.h"
#include "hw/i386/apic-msidef.h"
#include "hw/qdev-properties.h"
#include "kvm/kvm_i386.h"
#include "qemu/iova-tree.h"

/* used AMD-Vi MMIO registers */
const char *amdvi_mmio_low[] = {
    "AMDVI_MMIO_DEVTAB_BASE",
    "AMDVI_MMIO_CMDBUF_BASE",
    "AMDVI_MMIO_EVTLOG_BASE",
    "AMDVI_MMIO_CONTROL",
    "AMDVI_MMIO_EXCL_BASE",
    "AMDVI_MMIO_EXCL_LIMIT",
    "AMDVI_MMIO_EXT_FEATURES",
    "AMDVI_MMIO_PPR_BASE",
    "UNHANDLED"
};
const char *amdvi_mmio_high[] = {
    "AMDVI_MMIO_COMMAND_HEAD",
    "AMDVI_MMIO_COMMAND_TAIL",
    "AMDVI_MMIO_EVTLOG_HEAD",
    "AMDVI_MMIO_EVTLOG_TAIL",
    "AMDVI_MMIO_STATUS",
    "AMDVI_MMIO_PPR_HEAD",
    "AMDVI_MMIO_PPR_TAIL",
    "UNHANDLED"
};

struct AMDVIAddressSpace {
    uint8_t bus_num;            /* bus number                           */
    uint8_t devfn;              /* device function                      */
    AMDVIState *iommu_state;    /* AMDVI - one per machine              */
    MemoryRegion root;          /* AMDVI Root memory map region         */
    IOMMUMemoryRegion iommu;    /* Device's address translation region  */
    MemoryRegion iommu_nodma;   /* Alias of shared nodma memory region  */
    MemoryRegion iommu_ir;      /* Device's interrupt remapping region  */
    AddressSpace as;            /* device's corresponding address space */

    /* DMA address translation support */
    IOMMUNotifierFlag notifier_flags;
    /* entry in list of Address spaces with registered notifiers */
    QLIST_ENTRY(AMDVIAddressSpace) next;
    /* Record DMA translation ranges */
    IOVATree *iova_tree;
    /* DMA address translation active */
    bool addr_translation;
};

/* AMDVI cache entry */
typedef struct AMDVIIOTLBEntry {
    uint16_t domid;             /* assigned domain id  */
    uint16_t devid;             /* device owning entry */
    uint64_t perms;             /* access permissions  */
    uint64_t translated_addr;   /* translated address  */
    uint64_t page_mask;         /* physical page size  */
} AMDVIIOTLBEntry;

/*
 * These 'fault' reasons have an overloaded meaning since they are not only
 * intended for describing reasons that generate an IO_PAGE_FAULT as per the AMD
 * IOMMU specification, but are also used to signal internal errors in the
 * emulation code.
 */
typedef enum AMDVIFaultReason {
    AMDVI_FR_DTE_RTR_ERR = 1,   /* Failure to retrieve DTE */
    AMDVI_FR_DTE_V,             /* DTE[V] = 0 */
    AMDVI_FR_DTE_TV,            /* DTE[TV] = 0 */
    AMDVI_FR_PT_ROOT_INV,       /* Page Table Root ptr invalid */
    AMDVI_FR_PT_ENTRY_INV,      /* Failure to read PTE from guest memory */
} AMDVIFaultReason;

uint64_t amdvi_extended_feature_register(AMDVIState *s)
{
    uint64_t feature = AMDVI_DEFAULT_EXT_FEATURES;
    if (s->xtsup) {
        feature |= AMDVI_FEATURE_XT;
    }
    if (!s->iommu.dma_translation) {
        feature |= AMDVI_HATS_MODE_RESERVED;
    }

    return feature;
}

/* configure MMIO registers at startup/reset */
static void amdvi_set_quad(AMDVIState *s, hwaddr addr, uint64_t val,
                           uint64_t romask, uint64_t w1cmask)
{
    stq_le_p(&s->mmior[addr], val);
    stq_le_p(&s->romask[addr], romask);
    stq_le_p(&s->w1cmask[addr], w1cmask);
}

static uint16_t amdvi_readw(AMDVIState *s, hwaddr addr)
{
    return lduw_le_p(&s->mmior[addr]);
}

static uint32_t amdvi_readl(AMDVIState *s, hwaddr addr)
{
    return ldl_le_p(&s->mmior[addr]);
}

static uint64_t amdvi_readq(AMDVIState *s, hwaddr addr)
{
    return ldq_le_p(&s->mmior[addr]);
}

/* internal write */
static void amdvi_writeq_raw(AMDVIState *s, hwaddr addr, uint64_t val)
{
    stq_le_p(&s->mmior[addr], val);
}

/* external write */
static void amdvi_writew(AMDVIState *s, hwaddr addr, uint16_t val)
{
    uint16_t romask = lduw_le_p(&s->romask[addr]);
    uint16_t w1cmask = lduw_le_p(&s->w1cmask[addr]);
    uint16_t oldval = lduw_le_p(&s->mmior[addr]);

    uint16_t oldval_preserved = oldval & (romask | w1cmask);
    uint16_t newval_write = val & ~romask;
    uint16_t newval_w1c_set = val & w1cmask;

    stw_le_p(&s->mmior[addr],
             (oldval_preserved | newval_write) & ~newval_w1c_set);
}

static void amdvi_writel(AMDVIState *s, hwaddr addr, uint32_t val)
{
    uint32_t romask = ldl_le_p(&s->romask[addr]);
    uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]);
    uint32_t oldval = ldl_le_p(&s->mmior[addr]);

    uint32_t oldval_preserved = oldval & (romask | w1cmask);
    uint32_t newval_write = val & ~romask;
    uint32_t newval_w1c_set = val & w1cmask;

    stl_le_p(&s->mmior[addr],
             (oldval_preserved | newval_write) & ~newval_w1c_set);
}

static void amdvi_writeq(AMDVIState *s, hwaddr addr, uint64_t val)
{
    uint64_t romask = ldq_le_p(&s->romask[addr]);
    uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]);
    uint64_t oldval = ldq_le_p(&s->mmior[addr]);

    uint64_t oldval_preserved = oldval & (romask | w1cmask);
    uint64_t newval_write = val & ~romask;
    uint64_t newval_w1c_set = val & w1cmask;

    stq_le_p(&s->mmior[addr],
             (oldval_preserved | newval_write) & ~newval_w1c_set);
}

/* AND a 64-bit register with a 64-bit value */
static bool amdvi_test_mask(AMDVIState *s, hwaddr addr, uint64_t val)
{
    return amdvi_readq(s, addr) & val;
}

/* OR a 64-bit register with a 64-bit value storing result in the register */
static void amdvi_assign_orq(AMDVIState *s, hwaddr addr, uint64_t val)
{
    amdvi_writeq_raw(s, addr, amdvi_readq(s, addr) | val);
}

/* AND a 64-bit register with a 64-bit value storing result in the register */
static void amdvi_assign_andq(AMDVIState *s, hwaddr addr, uint64_t val)
{
   amdvi_writeq_raw(s, addr, amdvi_readq(s, addr) & val);
}

static void amdvi_generate_msi_interrupt(AMDVIState *s)
{
    MSIMessage msg = {};
    MemTxAttrs attrs = {
        .requester_id = pci_requester_id(&s->pci->dev)
    };

    if (msi_enabled(&s->pci->dev)) {
        msg = msi_get_message(&s->pci->dev, 0);
        address_space_stl_le(&address_space_memory, msg.address, msg.data,
                             attrs, NULL);
    }
}

static uint32_t get_next_eventlog_entry(AMDVIState *s)
{
    uint32_t evtlog_size = s->evtlog_len * AMDVI_EVENT_LEN;
    return (s->evtlog_tail + AMDVI_EVENT_LEN) % evtlog_size;
}

static void amdvi_log_event(AMDVIState *s, uint64_t *evt)
{
    uint32_t evtlog_tail_next;

    /* event logging not enabled */
    if (!s->evtlog_enabled || amdvi_test_mask(s, AMDVI_MMIO_STATUS,
        AMDVI_MMIO_STATUS_EVT_OVF)) {
        return;
    }

    evtlog_tail_next = get_next_eventlog_entry(s);

    /* event log buffer full */
    if (evtlog_tail_next == s->evtlog_head) {
        /* generate overflow interrupt */
        if (s->evtlog_intr) {
            amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_EVT_OVF);
            amdvi_generate_msi_interrupt(s);
        }
        return;
    }

    if (dma_memory_write(&address_space_memory, s->evtlog + s->evtlog_tail,
                         evt, AMDVI_EVENT_LEN, MEMTXATTRS_UNSPECIFIED)) {
        trace_amdvi_evntlog_fail(s->evtlog, s->evtlog_tail);
    }

    s->evtlog_tail = evtlog_tail_next;
    amdvi_writeq_raw(s, AMDVI_MMIO_EVENT_TAIL, s->evtlog_tail);

    if (s->evtlog_intr) {
        amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_EVENT_INT);
        amdvi_generate_msi_interrupt(s);
    }
}

static void amdvi_setevent_bits(uint64_t *buffer, uint64_t value, int start,
                                int length)
{
    int index = start / 64, bitpos = start % 64;
    uint64_t mask = MAKE_64BIT_MASK(start, length);
    buffer[index] &= ~mask;
    buffer[index] |= (value << bitpos) & mask;
}
/*
 * AMDVi event structure
 *    0:15   -> DeviceID
 *    48:63  -> event type + miscellaneous info
 *    64:127 -> related address
 */
static void amdvi_encode_event(uint64_t *evt, uint16_t devid, uint64_t addr,
                               uint16_t info)
{
    evt[0] = 0;
    evt[1] = 0;

    amdvi_setevent_bits(evt, devid, 0, 16);
    amdvi_setevent_bits(evt, info, 48, 16);
    amdvi_setevent_bits(evt, addr, 64, 64);
}
/* log an error encountered during a page walk
 *
 * @addr: virtual address in translation request
 */
static void amdvi_page_fault(AMDVIState *s, uint16_t devid,
                             hwaddr addr, uint16_t info)
{
    uint64_t evt[2];

    info |= AMDVI_EVENT_IOPF_I | AMDVI_EVENT_IOPF;
    amdvi_encode_event(evt, devid, addr, info);
    amdvi_log_event(s, evt);
    pci_word_test_and_set_mask(s->pci->dev.config + PCI_STATUS,
            PCI_STATUS_SIG_TARGET_ABORT);
}
/*
 * log a master abort accessing device table
 *  @devtab : address of device table entry
 *  @info : error flags
 */
static void amdvi_log_devtab_error(AMDVIState *s, uint16_t devid,
                                   hwaddr devtab, uint16_t info)
{
    uint64_t evt[2];

    info |= AMDVI_EVENT_DEV_TAB_HW_ERROR;

    amdvi_encode_event(evt, devid, devtab, info);
    amdvi_log_event(s, evt);
    pci_word_test_and_set_mask(s->pci->dev.config + PCI_STATUS,
            PCI_STATUS_SIG_TARGET_ABORT);
}
/* log an event trying to access command buffer
 *   @addr : address that couldn't be accessed
 */
static void amdvi_log_command_error(AMDVIState *s, hwaddr addr)
{
    uint64_t evt[2];
    uint16_t info = AMDVI_EVENT_COMMAND_HW_ERROR;

    amdvi_encode_event(evt, 0, addr, info);
    amdvi_log_event(s, evt);
    pci_word_test_and_set_mask(s->pci->dev.config + PCI_STATUS,
            PCI_STATUS_SIG_TARGET_ABORT);
}
/* log an illegal command event
 *   @addr : address of illegal command
 */
static void amdvi_log_illegalcom_error(AMDVIState *s, uint16_t info,
                                       hwaddr addr)
{
    uint64_t evt[2];

    info |= AMDVI_EVENT_ILLEGAL_COMMAND_ERROR;
    amdvi_encode_event(evt, 0, addr, info);
    amdvi_log_event(s, evt);
}
/* log an error accessing device table
 *
 *  @devid : device owning the table entry
 *  @devtab : address of device table entry
 *  @info : error flags
 */
static void amdvi_log_illegaldevtab_error(AMDVIState *s, uint16_t devid,
                                          hwaddr addr, uint16_t info)
{
    uint64_t evt[2];

    info |= AMDVI_EVENT_ILLEGAL_DEVTAB_ENTRY;
    amdvi_encode_event(evt, devid, addr, info);
    amdvi_log_event(s, evt);
}
/* log an error accessing a PTE entry
 * @addr : address that couldn't be accessed
 */
static void amdvi_log_pagetab_error(AMDVIState *s, uint16_t devid,
                                    hwaddr addr, uint16_t info)
{
    uint64_t evt[2];

    info |= AMDVI_EVENT_PAGE_TAB_HW_ERROR;
    amdvi_encode_event(evt, devid, addr, info);
    amdvi_log_event(s, evt);
    pci_word_test_and_set_mask(s->pci->dev.config + PCI_STATUS,
             PCI_STATUS_SIG_TARGET_ABORT);
}

static gboolean amdvi_uint64_equal(gconstpointer v1, gconstpointer v2)
{
    return *((const uint64_t *)v1) == *((const uint64_t *)v2);
}

static guint amdvi_uint64_hash(gconstpointer v)
{
    return (guint)*(const uint64_t *)v;
}

static AMDVIIOTLBEntry *amdvi_iotlb_lookup(AMDVIState *s, hwaddr addr,
                                           uint64_t devid)
{
    uint64_t key = (addr >> AMDVI_PAGE_SHIFT_4K) |
                   ((uint64_t)(devid) << AMDVI_DEVID_SHIFT);
    return g_hash_table_lookup(s->iotlb, &key);
}

static void amdvi_iotlb_reset(AMDVIState *s)
{
    assert(s->iotlb);
    trace_amdvi_iotlb_reset();
    g_hash_table_remove_all(s->iotlb);
}

static gboolean amdvi_iotlb_remove_by_devid(gpointer key, gpointer value,
                                            gpointer user_data)
{
    AMDVIIOTLBEntry *entry = (AMDVIIOTLBEntry *)value;
    uint16_t devid = *(uint16_t *)user_data;
    return entry->devid == devid;
}

static void amdvi_iotlb_remove_page(AMDVIState *s, hwaddr addr,
                                    uint64_t devid)
{
    uint64_t key = (addr >> AMDVI_PAGE_SHIFT_4K) |
                   ((uint64_t)(devid) << AMDVI_DEVID_SHIFT);
    g_hash_table_remove(s->iotlb, &key);
}

static void amdvi_update_iotlb(AMDVIState *s, uint16_t devid,
                               uint64_t gpa, IOMMUTLBEntry to_cache,
                               uint16_t domid)
{
    /* don't cache erroneous translations */
    if (to_cache.perm != IOMMU_NONE) {
        AMDVIIOTLBEntry *entry = g_new(AMDVIIOTLBEntry, 1);
        uint64_t *key = g_new(uint64_t, 1);
        uint64_t gfn = gpa >> AMDVI_PAGE_SHIFT_4K;

        trace_amdvi_cache_update(domid, PCI_BUS_NUM(devid), PCI_SLOT(devid),
                PCI_FUNC(devid), gpa, to_cache.translated_addr);

        if (g_hash_table_size(s->iotlb) >= AMDVI_IOTLB_MAX_SIZE) {
            amdvi_iotlb_reset(s);
        }

        entry->domid = domid;
        entry->perms = to_cache.perm;
        entry->translated_addr = to_cache.translated_addr;
        entry->page_mask = to_cache.addr_mask;
        *key = gfn | ((uint64_t)(devid) << AMDVI_DEVID_SHIFT);
        g_hash_table_replace(s->iotlb, key, entry);
    }
}

static void amdvi_completion_wait(AMDVIState *s, uint64_t *cmd)
{
    /* pad the last 3 bits */
    hwaddr addr = cpu_to_le64(extract64(cmd[0], 3, 49)) << 3;
    uint64_t data = cpu_to_le64(cmd[1]);

    if (extract64(cmd[0], 52, 8)) {
        amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
                                   s->cmdbuf + s->cmdbuf_head);
    }
    if (extract64(cmd[0], 0, 1)) {
        if (dma_memory_write(&address_space_memory, addr, &data,
                             AMDVI_COMPLETION_DATA_SIZE,
                             MEMTXATTRS_UNSPECIFIED)) {
            trace_amdvi_completion_wait_fail(addr);
        }
    }
    /* set completion interrupt */
    if (extract64(cmd[0], 1, 1)) {
        amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_COMP_INT);
        /* generate interrupt */
        amdvi_generate_msi_interrupt(s);
    }
    trace_amdvi_completion_wait(addr, data);
}

static inline uint64_t amdvi_get_perms(uint64_t entry)
{
    return (entry & (AMDVI_DEV_PERM_READ | AMDVI_DEV_PERM_WRITE)) >>
           AMDVI_DEV_PERM_SHIFT;
}

/* validate that reserved bits are honoured */
static bool amdvi_validate_dte(AMDVIState *s, uint16_t devid,
                               uint64_t *dte)
{

    uint64_t root;

    if ((dte[0] & AMDVI_DTE_QUAD0_RESERVED) ||
        (dte[1] & AMDVI_DTE_QUAD1_RESERVED) ||
        (dte[2] & AMDVI_DTE_QUAD2_RESERVED) ||
        (dte[3] & AMDVI_DTE_QUAD3_RESERVED)) {
        amdvi_log_illegaldevtab_error(s, devid,
                                      s->devtab +
                                      devid * AMDVI_DEVTAB_ENTRY_SIZE, 0);
        return false;
    }

    /*
     * 1 = Host Address Translation is not supported. Value in MMIO Offset
     * 0030h[HATS] is not meaningful. A non-zero host page table root pointer
     * in the DTE would result in an ILLEGAL_DEV_TABLE_ENTRY event.
     */
    root = (dte[0] & AMDVI_DEV_PT_ROOT_MASK) >> 12;
    if (root && !s->iommu.dma_translation) {
        amdvi_log_illegaldevtab_error(s, devid,
                                      s->devtab +
                                      devid * AMDVI_DEVTAB_ENTRY_SIZE, 0);
        return false;
    }

    return true;
}

/* get a device table entry given the devid */
static bool amdvi_get_dte(AMDVIState *s, int devid, uint64_t *entry)
{
    uint32_t offset = devid * AMDVI_DEVTAB_ENTRY_SIZE;

    if (dma_memory_read(&address_space_memory, s->devtab + offset, entry,
                        AMDVI_DEVTAB_ENTRY_SIZE, MEMTXATTRS_UNSPECIFIED)) {
        trace_amdvi_dte_get_fail(s->devtab, offset);
        /* log error accessing dte */
        amdvi_log_devtab_error(s, devid, s->devtab + offset, 0);
        return false;
    }

    *entry = le64_to_cpu(*entry);
    if (!amdvi_validate_dte(s, devid, entry)) {
        trace_amdvi_invalid_dte(entry[0]);
        return false;
    }

    return true;
}

/* get pte translation mode */
static inline uint8_t get_pte_translation_mode(uint64_t pte)
{
    return (pte >> AMDVI_DEV_MODE_RSHIFT) & AMDVI_DEV_MODE_MASK;
}

static inline uint64_t amdvi_get_pte_entry(AMDVIState *s, uint64_t pte_addr,
                                          uint16_t devid)
{
    uint64_t pte;

    if (dma_memory_read(&address_space_memory, pte_addr,
                        &pte, sizeof(pte), MEMTXATTRS_UNSPECIFIED)) {
        trace_amdvi_get_pte_hwerror(pte_addr);
        amdvi_log_pagetab_error(s, devid, pte_addr, 0);
        pte = (uint64_t)-1;
        return pte;
    }

    pte = le64_to_cpu(pte);
    return pte;
}

static int amdvi_as_to_dte(AMDVIAddressSpace *as, uint64_t *dte)
{
    uint16_t devid = PCI_BUILD_BDF(as->bus_num, as->devfn);
    AMDVIState *s = as->iommu_state;

    if (!amdvi_get_dte(s, devid, dte)) {
        /* Unable to retrieve DTE for devid */
        return -AMDVI_FR_DTE_RTR_ERR;
    }

    if (!(dte[0] & AMDVI_DEV_VALID)) {
        /* DTE[V] not set, address is passed untranslated for devid */
        return -AMDVI_FR_DTE_V;
    }

    if (!(dte[0] & AMDVI_DEV_TRANSLATION_VALID)) {
        /* DTE[TV] not set, host page table not valid for devid */
        return -AMDVI_FR_DTE_TV;
    }
    return 0;
}

/*
 * For a PTE encoding a large page, return the page size it encodes as described
 * by the AMD IOMMU Specification Table 14: Example Page Size Encodings.
 * No need to adjust the value of the PTE to point to the first PTE in the large
 * page since the encoding guarantees all "base" PTEs in the large page are the
 * same.
 */
static uint64_t large_pte_page_size(uint64_t pte)
{
    assert(PTE_NEXT_LEVEL(pte) == 7);

    /* Determine size of the large/contiguous page encoded in the PTE */
    return PTE_LARGE_PAGE_SIZE(pte);
}

/*
 * Helper function to fetch a PTE using AMD v1 pgtable format.
 * On successful page walk, returns 0 and pte parameter points to a valid PTE.
 * On failure, returns:
 * -AMDVI_FR_PT_ROOT_INV: A page walk is not possible due to conditions like DTE
 *      with invalid permissions, Page Table Root can not be read from DTE, or a
 *      larger IOVA than supported by page table level encoded in DTE[Mode].
 * -AMDVI_FR_PT_ENTRY_INV: A PTE could not be read from guest memory during a
 *      page table walk. This means that the DTE has valid data, but one of the
 *      lower level entries in the Page Table could not be read.
 */
static uint64_t fetch_pte(AMDVIAddressSpace *as, hwaddr address, uint64_t dte,
                          uint64_t *pte, hwaddr *page_size)
{
    IOMMUAccessFlags perms = amdvi_get_perms(dte);

    uint8_t level, mode;
    uint64_t pte_addr;

    *pte = dte;
    *page_size = 0;

    if (perms == IOMMU_NONE) {
        return -AMDVI_FR_PT_ROOT_INV;
    }

    /*
     * The Linux kernel driver initializes the default mode to 3, corresponding
     * to a 39-bit GPA space, where each entry in the pagetable translates to a
     * 1GB (2^30) page size.
     */
    level = mode = get_pte_translation_mode(dte);
    assert(mode > 0 && mode < 7);

    /*
     * If IOVA is larger than the max supported by the current pgtable level,
     * there is nothing to do.
     */
    if (address > PT_LEVEL_MAX_ADDR(mode - 1)) {
        /* IOVA too large for the current DTE */
        return -AMDVI_FR_PT_ROOT_INV;
    }

    do {
        level -= 1;

        /* Update the page_size */
        *page_size = PTE_LEVEL_PAGE_SIZE(level);

        /* Permission bits are ANDed at every level, including the DTE */
        perms &= amdvi_get_perms(*pte);
        if (perms == IOMMU_NONE) {
            return 0;
        }

        /* Not Present */
        if (!IOMMU_PTE_PRESENT(*pte)) {
            return 0;
        }

        /* Large or Leaf PTE found */
        if (PTE_NEXT_LEVEL(*pte) == 7 || PTE_NEXT_LEVEL(*pte) == 0) {
            /* Leaf PTE found */
            break;
        }

        /*
         * Index the pgtable using the IOVA bits corresponding to current level
         * and walk down to the lower level.
         */
        pte_addr = NEXT_PTE_ADDR(*pte, level, address);
        *pte = amdvi_get_pte_entry(as->iommu_state, pte_addr, as->devfn);

        if (*pte == (uint64_t)-1) {
            /*
             * A returned PTE of -1 indicates a failure to read the page table
             * entry from guest memory.
             */
            if (level == mode - 1) {
                /* Failure to retrieve the Page Table from Root Pointer */
                *page_size = 0;
                return -AMDVI_FR_PT_ROOT_INV;
            } else {
                /* Failure to read PTE. Page walk skips a page_size chunk */
                return -AMDVI_FR_PT_ENTRY_INV;
            }
        }
    } while (level > 0);

    assert(PTE_NEXT_LEVEL(*pte) == 0 || PTE_NEXT_LEVEL(*pte) == 7 ||
           level == 0);
    /*
     * Page walk ends when Next Level field on PTE shows that either a leaf PTE
     * or a series of large PTEs have been reached. In the latter case, even if
     * the range starts in the middle of a contiguous page, the returned PTE
     * must be the first PTE of the series.
     */
    if (PTE_NEXT_LEVEL(*pte) == 7) {
        /* Update page_size with the large PTE page size */
        *page_size = large_pte_page_size(*pte);
    }

    return 0;
}

/*
 * Invoke notifiers registered for the address space. Update record of mapped
 * ranges in IOVA Tree.
 */
static void amdvi_notify_iommu(AMDVIAddressSpace *as, IOMMUTLBEvent *event)
{
    IOMMUTLBEntry *entry = &event->entry;

    DMAMap target = {
        .iova = entry->iova,
        .size = entry->addr_mask,
        .translated_addr = entry->translated_addr,
        .perm = entry->perm,
    };

    /*
     * Search the IOVA Tree for an existing translation for the target, and skip
     * the notification if the mapping is already recorded.
     * When the guest uses large pages, comparing against the record makes it
     * possible to determine the size of the original MAP and adjust the UNMAP
     * request to match it. This avoids failed checks against the mappings kept
     * by the VFIO kernel driver.
     */
    const DMAMap *mapped = iova_tree_find(as->iova_tree, &target);

    if (event->type == IOMMU_NOTIFIER_UNMAP) {
        if (!mapped) {
            /* No record exists of this mapping, nothing to do */
            return;
        }
        /*
         * Adjust the size based on the original record. This is essential to
         * determine when large/contiguous pages are used, since the guest has
         * already cleared the PTE (erasing the pagesize encoded on it) before
         * issuing the invalidation command.
         */
        if (mapped->size != target.size) {
            assert(mapped->size > target.size);
            target.size = mapped->size;
            /* Adjust event to invoke notifier with correct range */
            entry->addr_mask = mapped->size;
        }
        iova_tree_remove(as->iova_tree, target);
    } else { /* IOMMU_NOTIFIER_MAP */
        if (mapped) {
            /*
             * If a mapping is present and matches the request, skip the
             * notification.
             */
            if (!memcmp(mapped, &target, sizeof(DMAMap))) {
                return;
            } else {
                /*
                 * This should never happen unless a buggy guest OS omits or
                 * sends incorrect invalidation(s). Report an error in the event
                 * it does happen.
                 */
                error_report("Found conflicting translation. This could be due "
                             "to an incorrect or missing invalidation command");
            }
        }
        /* Record the new mapping */
        iova_tree_insert(as->iova_tree, &target);
    }

    /* Invoke the notifiers registered for this address space */
    memory_region_notify_iommu(&as->iommu, 0, *event);
}

/*
 * Walk the guest page table for an IOVA and range and signal the registered
 * notifiers to sync the shadow page tables in the host.
 * Must be called with a valid DTE for DMA remapping i.e. V=1,TV=1
 */
static void amdvi_sync_shadow_page_table_range(AMDVIAddressSpace *as,
                                               uint64_t *dte, hwaddr addr,
                                               uint64_t size, bool send_unmap)
{
    IOMMUTLBEvent event;

    hwaddr page_mask, pagesize;
    hwaddr iova = addr;
    hwaddr end = iova + size - 1;

    uint64_t pte;
    int ret;

    while (iova < end) {

        ret = fetch_pte(as, iova, dte[0], &pte, &pagesize);

        if (ret == -AMDVI_FR_PT_ROOT_INV) {
            /*
             * Invalid conditions such as the IOVA being larger than supported
             * by current page table mode as configured in the DTE, or a failure
             * to fetch the Page Table from the Page Table Root Pointer in DTE.
             */
            assert(pagesize == 0);
            return;
        }
        /* PTE has been validated for major errors and pagesize is set */
        assert(pagesize);
        page_mask = ~(pagesize - 1);

        if (ret == -AMDVI_FR_PT_ENTRY_INV) {
            /*
             * Failure to read PTE from memory, the pagesize matches the current
             * level. Unable to determine the region type, so a safe strategy is
             * to skip the range and continue the page walk.
             */
            goto next;
        }

        event.entry.target_as = &address_space_memory;
        event.entry.iova = iova & page_mask;
        /* translated_addr is irrelevant for the unmap case */
        event.entry.translated_addr = (pte & AMDVI_DEV_PT_ROOT_MASK) &
                                      page_mask;
        event.entry.addr_mask = ~page_mask;
        event.entry.perm = amdvi_get_perms(pte);

        /*
         * In cases where the leaf PTE is not found, or it has invalid
         * permissions, an UNMAP type notification is sent, but only if the
         * caller requested it.
         */
        if (!IOMMU_PTE_PRESENT(pte) || (event.entry.perm == IOMMU_NONE)) {
            if (!send_unmap) {
                goto next;
            }
            event.type = IOMMU_NOTIFIER_UNMAP;
        } else {
            event.type = IOMMU_NOTIFIER_MAP;
        }

        /*
         * The following call might need to adjust event.entry.size in cases
         * where the guest unmapped a series of large pages.
         */
        amdvi_notify_iommu(as, &event);
        /*
         * In the special scenario where the guest is unmapping a large page,
         * addr_mask has been adjusted before sending the notification. Update
         * pagesize accordingly in order to correctly compute the next IOVA.
         */
        pagesize = event.entry.addr_mask + 1;

next:
        iova &= ~(pagesize - 1);

        /* Check for 64-bit overflow and terminate walk in such cases */
        if ((iova + pagesize) < iova) {
            break;
        } else {
            iova += pagesize;
        }
    }
}

/*
 * Unmap entire range that the notifier registered for i.e. the full AS.
 *
 * This is seemingly technically equivalent to directly calling
 * memory_region_unmap_iommu_notifier_range(), but it allows to check for
 * notifier boundaries and issue notifications with ranges within those bounds.
 */
static void amdvi_address_space_unmap(AMDVIAddressSpace *as, IOMMUNotifier *n)
{

    hwaddr start = n->start;
    hwaddr end = n->end;
    hwaddr remain;
    DMAMap map;

    assert(start <= end);
    remain = end - start + 1;

    /*
     * Divide the notifier range into chunks that are aligned and do not exceed
     * the notifier boundaries.
     */
    while (remain >= AMDVI_PAGE_SIZE) {

        IOMMUTLBEvent event;

        uint64_t mask = dma_aligned_pow2_mask(start, end, 64);

        event.type = IOMMU_NOTIFIER_UNMAP;

        IOMMUTLBEntry entry = {
            .target_as = &address_space_memory,
            .iova = start,
            .translated_addr = 0,   /* irrelevant for unmap case */
            .addr_mask = mask,
            .perm = IOMMU_NONE,
        };
        event.entry = entry;

        /* Call notifier registered for updates on this address space */
        memory_region_notify_iommu_one(n, &event);

        start += mask + 1;
        remain -= mask + 1;
    }

    assert(!remain);

    map.iova = n->start;
    map.size = n->end - n->start;

    iova_tree_remove(as->iova_tree, map);
}

/*
 * For all the address spaces with notifiers registered, unmap the entire range
 * the notifier registered for i.e. clear all the address spaces managed by the
 * IOMMU.
 */
static void amdvi_address_space_unmap_all(AMDVIState *s)
{
    AMDVIAddressSpace *as;
    IOMMUNotifier *n;

    QLIST_FOREACH(as, &s->amdvi_as_with_notifiers, next) {
        IOMMU_NOTIFIER_FOREACH(n, &as->iommu) {
            amdvi_address_space_unmap(as, n);
        }
    }
}

/*
 * For every translation present in the IOMMU, construct IOMMUTLBEntry data
 * and pass it as parameter to notifier callback.
 */
static void amdvi_iommu_replay(IOMMUMemoryRegion *iommu_mr, IOMMUNotifier *n)
{
    AMDVIAddressSpace *as = container_of(iommu_mr, AMDVIAddressSpace, iommu);
    uint64_t dte[4] = { 0 };

    if (!(n->notifier_flags & IOMMU_NOTIFIER_MAP)) {
        return;
    }

    if (amdvi_as_to_dte(as, dte)) {
        return;
    }

    /* Dropping all mappings for the address space. Also clears the IOVA tree */
    amdvi_address_space_unmap(as, n);

    amdvi_sync_shadow_page_table_range(as, &dte[0], 0, UINT64_MAX, false);
}

static void amdvi_address_space_sync(AMDVIAddressSpace *as)
{
    IOMMUNotifier *n;
    uint64_t dte[4] = { 0 };

    /* If only UNMAP notifiers are registered, drop all existing mappings */
    if (!(as->notifier_flags & IOMMU_NOTIFIER_MAP)) {
        IOMMU_NOTIFIER_FOREACH(n, &as->iommu) {
            /*
             * Directly calling memory_region_unmap_iommu_notifier_range() does
             * not guarantee that the addr_mask eventually passed as parameter
             * to the notifier is valid. Use amdvi_address_space_unmap() which
             * ensures the notifier range is divided into properly aligned
             * regions, and issues notifications for each one.
             */
            amdvi_address_space_unmap(as, n);
        }
        return;
    }

    if (amdvi_as_to_dte(as, dte)) {
        return;
    }

    amdvi_sync_shadow_page_table_range(as, &dte[0], 0, UINT64_MAX, true);
}

/*
 * This differs from the replay() method in that it issues both MAP and UNMAP
 * notifications since it is called after global invalidation events in order to
 * re-sync all address spaces.
 */
static void amdvi_iommu_address_space_sync_all(AMDVIState *s)
{
    AMDVIAddressSpace *as;

    QLIST_FOREACH(as, &s->amdvi_as_with_notifiers, next) {
        amdvi_address_space_sync(as);
    }
}

/*
 * Toggle between address translation and passthrough modes by enabling the
 * corresponding memory regions.
 */
static void amdvi_switch_address_space(AMDVIAddressSpace *amdvi_as)
{
    AMDVIState *s = amdvi_as->iommu_state;

    if (s->dma_remap && amdvi_as->addr_translation) {
        /* Enabling DMA region */
        memory_region_set_enabled(&amdvi_as->iommu_nodma, false);
        memory_region_set_enabled(MEMORY_REGION(&amdvi_as->iommu), true);
    } else {
        /* Disabling DMA region, using passthrough */
        memory_region_set_enabled(MEMORY_REGION(&amdvi_as->iommu), false);
        memory_region_set_enabled(&amdvi_as->iommu_nodma, true);
    }
}

/*
 * For all existing address spaces managed by the IOMMU, enable/disable the
 * corresponding memory regions to reset the address translation mode and
 * use passthrough by default.
 */
static void amdvi_reset_address_translation_all(AMDVIState *s)
{
    AMDVIAddressSpace **iommu_as;

    for (int bus_num = 0; bus_num < PCI_BUS_MAX; bus_num++) {

        /* Nothing to do if there are no devices on the current bus */
        if (!s->address_spaces[bus_num]) {
            continue;
        }
        iommu_as = s->address_spaces[bus_num];

        for (int devfn = 0; devfn < PCI_DEVFN_MAX; devfn++) {

            if (!iommu_as[devfn]) {
                continue;
            }
            /* Use passthrough as default mode after reset */
            iommu_as[devfn]->addr_translation = false;
            amdvi_switch_address_space(iommu_as[devfn]);
        }
    }
}

static void enable_dma_mode(AMDVIAddressSpace *as, bool inval_current)
{
    /*
     * When enabling DMA mode for the purpose of isolating guest devices on
     * a failure to retrieve or invalid DTE, all existing mappings must be
     * dropped.
     */
    if (inval_current) {
        IOMMUNotifier *n;
        IOMMU_NOTIFIER_FOREACH(n, &as->iommu) {
            amdvi_address_space_unmap(as, n);
        }
    }

    if (as->addr_translation) {
        return;
    }

    /* Installing DTE enabling translation, activate region */
    as->addr_translation = true;
    amdvi_switch_address_space(as);
    /* Sync shadow page tables */
    amdvi_address_space_sync(as);
}

/*
 * If paging was previously in use in the address space
 * - invalidate all existing mappings
 * - switch to no_dma memory region
 */
static void enable_nodma_mode(AMDVIAddressSpace *as)
{
    IOMMUNotifier *n;

    if (!as->addr_translation) {
        /* passthrough is already active, nothing to do */
        return;
    }

    as->addr_translation = false;
    IOMMU_NOTIFIER_FOREACH(n, &as->iommu) {
        /* Drop all mappings for the address space */
        amdvi_address_space_unmap(as, n);
    }
    amdvi_switch_address_space(as);
}

/*
 * A guest driver must issue the INVALIDATE_DEVTAB_ENTRY command to the IOMMU
 * after changing a Device Table entry. We can use this fact to detect when a
 * Device Table entry is created for a device attached to a paging domain and
 * enable the corresponding IOMMU memory region to allow for DMA translation if
 * appropriate.
 */
static void amdvi_update_addr_translation_mode(AMDVIState *s, uint16_t devid)
{
    uint8_t bus_num, devfn, dte_mode;
    AMDVIAddressSpace *as;
    uint64_t dte[4] = { 0 };
    int ret;

    /*
     * Convert the devid encoded in the command to a bus and devfn in
     * order to retrieve the corresponding address space.
     */
    bus_num = PCI_BUS_NUM(devid);
    devfn = devid & 0xff;

    /*
     * The main buffer of size (AMDVIAddressSpace *) * (PCI_BUS_MAX) has already
     * been allocated within AMDVIState, but must be careful to not access
     * unallocated devfn.
     */
    if (!s->address_spaces[bus_num] || !s->address_spaces[bus_num][devfn]) {
        return;
    }
    as = s->address_spaces[bus_num][devfn];

    ret = amdvi_as_to_dte(as, dte);

    if (!ret) {
        dte_mode = (dte[0] >> AMDVI_DEV_MODE_RSHIFT) & AMDVI_DEV_MODE_MASK;
    }

    switch (ret) {
    case 0:
        /* DTE was successfully retrieved */
        if (!dte_mode) {
            enable_nodma_mode(as); /* DTE[V]=1 && DTE[Mode]=0 => passthrough */
        } else {
            enable_dma_mode(as, false); /* Enable DMA translation */
        }
        break;
    case -AMDVI_FR_DTE_V:
        /* DTE[V]=0, address is passed untranslated */
        enable_nodma_mode(as);
        break;
    case -AMDVI_FR_DTE_RTR_ERR:
    case -AMDVI_FR_DTE_TV:
        /*
         * Enforce isolation by using DMA in rare scenarios where the DTE cannot
         * be retrieved or DTE[TV]=0. Existing mappings are dropped.
         */
        enable_dma_mode(as, true);
        break;
    }
}

/* log error without aborting since linux seems to be using reserved bits */
static void amdvi_inval_devtab_entry(AMDVIState *s, uint64_t *cmd)
{
    uint16_t devid = cpu_to_le16((uint16_t)extract64(cmd[0], 0, 16));

    trace_amdvi_devtab_inval(PCI_BUS_NUM(devid), PCI_SLOT(devid),
                             PCI_FUNC(devid));

    /* This command should invalidate internal caches of which there isn't */
    if (extract64(cmd[0], 16, 44) || cmd[1]) {
        amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
                                   s->cmdbuf + s->cmdbuf_head);
        return;
    }

    /*
     * When DMA remapping capability is enabled, check if updated DTE is setup
     * for paging or not, and configure the corresponding memory regions.
     */
    if (s->dma_remap) {
        amdvi_update_addr_translation_mode(s, devid);
    }
}

static void amdvi_complete_ppr(AMDVIState *s, uint64_t *cmd)
{
    if (extract64(cmd[0], 16, 16) ||  extract64(cmd[0], 52, 8) ||
        extract64(cmd[1], 0, 2) || extract64(cmd[1], 3, 29)
        || extract64(cmd[1], 48, 16)) {
        amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
                                   s->cmdbuf + s->cmdbuf_head);
    }
    trace_amdvi_ppr_exec();
}

static void amdvi_intremap_inval_notify_all(AMDVIState *s, bool global,
                               uint32_t index, uint32_t mask)
{
    x86_iommu_iec_notify_all(X86_IOMMU_DEVICE(s), global, index, mask);
}

static void amdvi_inval_all(AMDVIState *s, uint64_t *cmd)
{
    if (extract64(cmd[0], 0, 60) || cmd[1]) {
        amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
                                   s->cmdbuf + s->cmdbuf_head);
    }

    /* Notify global invalidation */
    amdvi_intremap_inval_notify_all(s, true, 0, 0);

    amdvi_iotlb_reset(s);

    /*
     * Fully replay the address space i.e. send both UNMAP and MAP events in
     * order to synchronize guest and host IO page tables tables.
     */
    amdvi_iommu_address_space_sync_all(s);

    trace_amdvi_all_inval();
}

static gboolean amdvi_iotlb_remove_by_domid(gpointer key, gpointer value,
                                            gpointer user_data)
{
    AMDVIIOTLBEntry *entry = (AMDVIIOTLBEntry *)value;
    uint16_t domid = *(uint16_t *)user_data;
    return entry->domid == domid;
}

/*
 * Helper to decode the size of the range to invalidate encoded in the
 * INVALIDATE_IOMMU_PAGES Command format.
 * The size of the region to invalidate depends on the S bit and address.
 * S bit value:
 * 0 :  Invalidation size is 4 Kbytes.
 * 1 :  Invalidation size is determined by first zero bit in the address
 *      starting from Address[12].
 *
 * In the AMD IOMMU Linux driver, an invalidation command with address
 * ((1 << 63) - 1) is sent when intending to clear the entire cache.
 * However, Table 14: Example Page Size Encodings shows that an address of
 * ((1ULL << 51) - 1) encodes the entire cache, so effectively any address with
 * first zero at bit 51 or larger is a request to invalidate the entire address
 * space.
 */
static uint64_t amdvi_decode_invalidation_size(hwaddr addr, uint16_t flags)
{
    uint64_t size = AMDVI_PAGE_SIZE;
    uint8_t fzbit = 0;

    if (flags & AMDVI_CMD_INVAL_IOMMU_PAGES_S) {
        fzbit = cto64(addr | 0xFFF);

        if (fzbit >= 51) {
            size = AMDVI_INV_ALL_PAGES;
        } else {
            size = 1ULL << (fzbit + 1);
        }
    }
    return size;
}

/*
 * Synchronize the guest page tables with the shadow page tables kept in the
 * host for the specified range.
 * The invalidation command issued by the guest and intercepted by the VMM
 * does not specify a device, but a domain, since all devices in the same domain
 * share the same page tables. However, vIOMMU emulation creates separate
 * address spaces per device, so it is necessary to traverse the list of all of
 * address spaces (i.e. devices) that have notifiers registered in order to
 * propagate the changes to the host page tables.
 * We cannot return early from this function once a matching domain has been
 * identified and its page tables synced (based on the fact that all devices in
 * the same domain share the page tables). The reason is that different devices
 * (i.e. address spaces) could have different notifiers registered, and by
 * skipping address spaces that appear later on the amdvi_as_with_notifiers list
 * their notifiers (which could differ from the ones registered for the first
 * device/address space) would not be invoked.
 */
static void amdvi_sync_domain(AMDVIState *s, uint16_t domid, uint64_t addr,
                              uint16_t flags)
{
    AMDVIAddressSpace *as;

    uint64_t size = amdvi_decode_invalidation_size(addr, flags);

    if (size == AMDVI_INV_ALL_PAGES) {
        addr = 0;       /* Set start address to 0 and invalidate entire AS */
    } else {
        addr &= ~(size - 1);
    }

    /*
     * Call notifiers that have registered for each address space matching the
     * domain ID, in order to sync the guest pagetable state with the host.
     */
    QLIST_FOREACH(as, &s->amdvi_as_with_notifiers, next) {

        uint64_t dte[4] = { 0 };

        /*
         * Retrieve the Device Table entry for the devid corresponding to the
         * current address space, and verify the DomainID matches i.e. the page
         * tables to be synced belong to devices in the domain.
         */
        if (amdvi_as_to_dte(as, dte)) {
            continue;
        }

        /* Only need to sync the Page Tables for a matching domain */
        if (domid != (dte[1] & AMDVI_DEV_DOMID_ID_MASK)) {
            continue;
        }

        /*
         * We have determined that there is a valid Device Table Entry for a
         * device matching the DomainID in the INV_IOMMU_PAGES command issued by
         * the guest. Walk the guest page table to sync shadow page table.
         */
        if (as->notifier_flags & IOMMU_NOTIFIER_MAP) {
            /* Sync guest IOMMU mappings with host */
            amdvi_sync_shadow_page_table_range(as, &dte[0], addr, size, true);
        }
    }
}

/* we don't have devid - we can't remove pages by address */
static void amdvi_inval_pages(AMDVIState *s, uint64_t *cmd)
{
    uint16_t domid = cpu_to_le16((uint16_t)extract64(cmd[0], 32, 16));
    uint64_t addr = cpu_to_le64(extract64(cmd[1], 12, 52)) << 12;
    uint16_t flags = cpu_to_le16((uint16_t)extract64(cmd[1], 0, 3));

    if (extract64(cmd[0], 20, 12) || extract64(cmd[0], 48, 12) ||
        extract64(cmd[1], 3, 9)) {
        amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
                                   s->cmdbuf + s->cmdbuf_head);
    }

    g_hash_table_foreach_remove(s->iotlb, amdvi_iotlb_remove_by_domid,
                                &domid);

    amdvi_sync_domain(s, domid, addr, flags);
    trace_amdvi_pages_inval(domid);
}

static void amdvi_prefetch_pages(AMDVIState *s, uint64_t *cmd)
{
    if (extract64(cmd[0], 16, 8) || extract64(cmd[0], 52, 8) ||
        extract64(cmd[1], 1, 1) || extract64(cmd[1], 3, 1) ||
        extract64(cmd[1], 5, 7)) {
        amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
                                   s->cmdbuf + s->cmdbuf_head);
    }

    trace_amdvi_prefetch_pages();
}

static void amdvi_inval_inttable(AMDVIState *s, uint64_t *cmd)
{
    if (extract64(cmd[0], 16, 44) || cmd[1]) {
        amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
                                   s->cmdbuf + s->cmdbuf_head);
        return;
    }

    /* Notify global invalidation */
    amdvi_intremap_inval_notify_all(s, true, 0, 0);

    trace_amdvi_intr_inval();
}

/* FIXME: Try to work with the specified size instead of all the pages
 * when the S bit is on
 */
static void iommu_inval_iotlb(AMDVIState *s, uint64_t *cmd)
{

    uint16_t devid = cpu_to_le16(extract64(cmd[0], 0, 16));
    if (extract64(cmd[1], 1, 1) || extract64(cmd[1], 3, 1) ||
        extract64(cmd[1], 6, 6)) {
        amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
                                   s->cmdbuf + s->cmdbuf_head);
        return;
    }

    if (extract64(cmd[1], 0, 1)) {
        g_hash_table_foreach_remove(s->iotlb, amdvi_iotlb_remove_by_devid,
                                    &devid);
    } else {
        amdvi_iotlb_remove_page(s, cpu_to_le64(extract64(cmd[1], 12, 52)) << 12,
                                devid);
    }
    trace_amdvi_iotlb_inval();
}

/* not honouring reserved bits is regarded as an illegal command */
static void amdvi_cmdbuf_exec(AMDVIState *s)
{
    uint64_t cmd[2];

    if (dma_memory_read(&address_space_memory, s->cmdbuf + s->cmdbuf_head,
                        cmd, AMDVI_COMMAND_SIZE, MEMTXATTRS_UNSPECIFIED)) {
        trace_amdvi_command_read_fail(s->cmdbuf, s->cmdbuf_head);
        amdvi_log_command_error(s, s->cmdbuf + s->cmdbuf_head);
        return;
    }

    switch (extract64(cmd[0], 60, 4)) {
    case AMDVI_CMD_COMPLETION_WAIT:
        amdvi_completion_wait(s, cmd);
        break;
    case AMDVI_CMD_INVAL_DEVTAB_ENTRY:
        amdvi_inval_devtab_entry(s, cmd);
        break;
    case AMDVI_CMD_INVAL_AMDVI_PAGES:
        amdvi_inval_pages(s, cmd);
        break;
    case AMDVI_CMD_INVAL_IOTLB_PAGES:
        iommu_inval_iotlb(s, cmd);
        break;
    case AMDVI_CMD_INVAL_INTR_TABLE:
        amdvi_inval_inttable(s, cmd);
        break;
    case AMDVI_CMD_PREFETCH_AMDVI_PAGES:
        amdvi_prefetch_pages(s, cmd);
        break;
    case AMDVI_CMD_COMPLETE_PPR_REQUEST:
        amdvi_complete_ppr(s, cmd);
        break;
    case AMDVI_CMD_INVAL_AMDVI_ALL:
        amdvi_inval_all(s, cmd);
        break;
    default:
        trace_amdvi_unhandled_command(extract64(cmd[1], 60, 4));
        /* log illegal command */
        amdvi_log_illegalcom_error(s, extract64(cmd[1], 60, 4),
                                   s->cmdbuf + s->cmdbuf_head);
    }
}

static void amdvi_cmdbuf_run(AMDVIState *s)
{
    if (!s->cmdbuf_enabled) {
        trace_amdvi_command_error(amdvi_readq(s, AMDVI_MMIO_CONTROL));
        return;
    }

    /* check if there is work to do. */
    while (s->cmdbuf_head != s->cmdbuf_tail) {
        trace_amdvi_command_exec(s->cmdbuf_head, s->cmdbuf_tail, s->cmdbuf);
        amdvi_cmdbuf_exec(s);
        s->cmdbuf_head += AMDVI_COMMAND_SIZE;
        amdvi_writeq_raw(s, AMDVI_MMIO_COMMAND_HEAD, s->cmdbuf_head);

        /* wrap head pointer */
        if (s->cmdbuf_head >= s->cmdbuf_len * AMDVI_COMMAND_SIZE) {
            s->cmdbuf_head = 0;
        }
    }
}

static inline uint8_t amdvi_mmio_get_index(hwaddr addr)
{
    uint8_t index = (addr & ~0x2000) / 8;

    if ((addr & 0x2000)) {
        /* high table */
        index = index >= AMDVI_MMIO_REGS_HIGH ? AMDVI_MMIO_REGS_HIGH : index;
    } else {
        index = index >= AMDVI_MMIO_REGS_LOW ? AMDVI_MMIO_REGS_LOW : index;
    }

    return index;
}

static void amdvi_mmio_trace_read(hwaddr addr, unsigned size)
{
    uint8_t index = amdvi_mmio_get_index(addr);
    trace_amdvi_mmio_read(amdvi_mmio_low[index], addr, size, addr & ~0x07);
}

static void amdvi_mmio_trace_write(hwaddr addr, unsigned size, uint64_t val)
{
    uint8_t index = amdvi_mmio_get_index(addr);
    trace_amdvi_mmio_write(amdvi_mmio_low[index], addr, size, val,
                           addr & ~0x07);
}

static uint64_t amdvi_mmio_read(void *opaque, hwaddr addr, unsigned size)
{
    AMDVIState *s = opaque;

    uint64_t val = -1;
    if (addr + size > AMDVI_MMIO_SIZE) {
        trace_amdvi_mmio_read_invalid(AMDVI_MMIO_SIZE, addr, size);
        return (uint64_t)-1;
    }

    if (size == 2) {
        val = amdvi_readw(s, addr);
    } else if (size == 4) {
        val = amdvi_readl(s, addr);
    } else if (size == 8) {
        val = amdvi_readq(s, addr);
    }
    amdvi_mmio_trace_read(addr, size);

    return val;
}

static void amdvi_handle_control_write(AMDVIState *s)
{
    unsigned long control = amdvi_readq(s, AMDVI_MMIO_CONTROL);
    s->enabled = !!(control & AMDVI_MMIO_CONTROL_AMDVIEN);

    s->evtlog_enabled = s->enabled && !!(control &
                        AMDVI_MMIO_CONTROL_EVENTLOGEN);

    s->evtlog_intr = !!(control & AMDVI_MMIO_CONTROL_EVENTINTEN);
    s->completion_wait_intr = !!(control & AMDVI_MMIO_CONTROL_COMWAITINTEN);
    s->cmdbuf_enabled = s->enabled && !!(control &
                        AMDVI_MMIO_CONTROL_CMDBUFLEN);
    s->ga_enabled = !!(control & AMDVI_MMIO_CONTROL_GAEN);

    /* update the flags depending on the control register */
    if (s->cmdbuf_enabled) {
        amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_CMDBUF_RUN);
    } else {
        amdvi_assign_andq(s, AMDVI_MMIO_STATUS, ~AMDVI_MMIO_STATUS_CMDBUF_RUN);
    }
    if (s->evtlog_enabled) {
        amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_EVT_RUN);
    } else {
        amdvi_assign_andq(s, AMDVI_MMIO_STATUS, ~AMDVI_MMIO_STATUS_EVT_RUN);
    }

    trace_amdvi_control_status(control);
    amdvi_cmdbuf_run(s);
}

static inline void amdvi_handle_devtab_write(AMDVIState *s)

{
    uint64_t val = amdvi_readq(s, AMDVI_MMIO_DEVICE_TABLE);
    s->devtab = (val & AMDVI_MMIO_DEVTAB_BASE_MASK);

    /* set device table length (i.e. number of entries table can hold) */
    s->devtab_len = (((val & AMDVI_MMIO_DEVTAB_SIZE_MASK) + 1) *
                    (AMDVI_MMIO_DEVTAB_SIZE_UNIT /
                     AMDVI_MMIO_DEVTAB_ENTRY_SIZE));
}

static inline void amdvi_handle_cmdhead_write(AMDVIState *s)
{
    s->cmdbuf_head = amdvi_readq(s, AMDVI_MMIO_COMMAND_HEAD)
                     & AMDVI_MMIO_CMDBUF_HEAD_MASK;
    amdvi_cmdbuf_run(s);
}

static inline void amdvi_handle_cmdbase_write(AMDVIState *s)
{
    s->cmdbuf = amdvi_readq(s, AMDVI_MMIO_COMMAND_BASE)
                & AMDVI_MMIO_CMDBUF_BASE_MASK;
    s->cmdbuf_len = 1UL << (amdvi_readq(s, AMDVI_MMIO_CMDBUF_SIZE_BYTE)
                    & AMDVI_MMIO_CMDBUF_SIZE_MASK);
    s->cmdbuf_head = s->cmdbuf_tail = 0;
}

static inline void amdvi_handle_cmdtail_write(AMDVIState *s)
{
    s->cmdbuf_tail = amdvi_readq(s, AMDVI_MMIO_COMMAND_TAIL)
                     & AMDVI_MMIO_CMDBUF_TAIL_MASK;
    amdvi_cmdbuf_run(s);
}

static inline void amdvi_handle_excllim_write(AMDVIState *s)
{
    uint64_t val = amdvi_readq(s, AMDVI_MMIO_EXCL_LIMIT);
    s->excl_limit = (val & AMDVI_MMIO_EXCL_LIMIT_MASK) |
                    AMDVI_MMIO_EXCL_LIMIT_LOW;
}

static inline void amdvi_handle_evtbase_write(AMDVIState *s)
{
    uint64_t val = amdvi_readq(s, AMDVI_MMIO_EVENT_BASE);

    if (amdvi_readq(s, AMDVI_MMIO_STATUS) & AMDVI_MMIO_STATUS_EVENT_INT)
        /* Do not reset if eventlog interrupt bit is set*/
        return;

    s->evtlog = val & AMDVI_MMIO_EVTLOG_BASE_MASK;
    s->evtlog_len = 1UL << (amdvi_readq(s, AMDVI_MMIO_EVTLOG_SIZE_BYTE)
                    & AMDVI_MMIO_EVTLOG_SIZE_MASK);

    /* clear tail and head pointer to 0 when event base is updated */
    s->evtlog_tail = s->evtlog_head = 0;
    amdvi_writeq_raw(s, AMDVI_MMIO_EVENT_HEAD, s->evtlog_head);
    amdvi_writeq_raw(s, AMDVI_MMIO_EVENT_TAIL, s->evtlog_tail);
}

static inline void amdvi_handle_evttail_write(AMDVIState *s)
{
    uint64_t val = amdvi_readq(s, AMDVI_MMIO_EVENT_TAIL);
    s->evtlog_tail = val & AMDVI_MMIO_EVTLOG_TAIL_MASK;
}

static inline void amdvi_handle_evthead_write(AMDVIState *s)
{
    uint64_t val = amdvi_readq(s, AMDVI_MMIO_EVENT_HEAD);
    s->evtlog_head = val & AMDVI_MMIO_EVTLOG_HEAD_MASK;
}

static inline void amdvi_handle_pprbase_write(AMDVIState *s)
{
    uint64_t val = amdvi_readq(s, AMDVI_MMIO_PPR_BASE);
    s->ppr_log = val & AMDVI_MMIO_PPRLOG_BASE_MASK;
    s->pprlog_len = 1UL << (amdvi_readq(s, AMDVI_MMIO_PPRLOG_SIZE_BYTE)
                    & AMDVI_MMIO_PPRLOG_SIZE_MASK);
}

static inline void amdvi_handle_pprhead_write(AMDVIState *s)
{
    uint64_t val = amdvi_readq(s, AMDVI_MMIO_PPR_HEAD);
    s->pprlog_head = val & AMDVI_MMIO_PPRLOG_HEAD_MASK;
}

static inline void amdvi_handle_pprtail_write(AMDVIState *s)
{
    uint64_t val = amdvi_readq(s, AMDVI_MMIO_PPR_TAIL);
    s->pprlog_tail = val & AMDVI_MMIO_PPRLOG_TAIL_MASK;
}

/* FIXME: something might go wrong if System Software writes in chunks
 * of one byte but linux writes in chunks of 4 bytes so currently it
 * works correctly with linux but will definitely be busted if software
 * reads/writes 8 bytes
 */
static void amdvi_mmio_reg_write(AMDVIState *s, unsigned size, uint64_t val,
                                 hwaddr addr)
{
    if (size == 2) {
        amdvi_writew(s, addr, val);
    } else if (size == 4) {
        amdvi_writel(s, addr, val);
    } else if (size == 8) {
        amdvi_writeq(s, addr, val);
    }
}

static void amdvi_mmio_write(void *opaque, hwaddr addr, uint64_t val,
                             unsigned size)
{
    AMDVIState *s = opaque;
    unsigned long offset = addr & 0x07;

    if (addr + size > AMDVI_MMIO_SIZE) {
        trace_amdvi_mmio_write("error: addr outside region: max ",
                (uint64_t)AMDVI_MMIO_SIZE, size, val, offset);
        return;
    }

    amdvi_mmio_trace_write(addr, size, val);
    switch (addr & ~0x07) {
    case AMDVI_MMIO_CONTROL:
        amdvi_mmio_reg_write(s, size, val, addr);
        amdvi_handle_control_write(s);
        break;
    case AMDVI_MMIO_DEVICE_TABLE:
        amdvi_mmio_reg_write(s, size, val, addr);
       /*  set device table address
        *   This also suffers from inability to tell whether software
        *   is done writing
        */
        if (offset || (size == 8)) {
            amdvi_handle_devtab_write(s);
        }
        break;
    case AMDVI_MMIO_COMMAND_HEAD:
        amdvi_mmio_reg_write(s, size, val, addr);
        amdvi_handle_cmdhead_write(s);
        break;
    case AMDVI_MMIO_COMMAND_BASE:
        amdvi_mmio_reg_write(s, size, val, addr);
        /* FIXME - make sure System Software has finished writing in case
         * it writes in chucks less than 8 bytes in a robust way.As for
         * now, this hacks works for the linux driver
         */
        if (offset || (size == 8)) {
            amdvi_handle_cmdbase_write(s);
        }
        break;
    case AMDVI_MMIO_COMMAND_TAIL:
        amdvi_mmio_reg_write(s, size, val, addr);
        amdvi_handle_cmdtail_write(s);
        break;
    case AMDVI_MMIO_EVENT_BASE:
        amdvi_mmio_reg_write(s, size, val, addr);
        amdvi_handle_evtbase_write(s);
        break;
    case AMDVI_MMIO_EVENT_HEAD:
        amdvi_mmio_reg_write(s, size, val, addr);
        amdvi_handle_evthead_write(s);
        break;
    case AMDVI_MMIO_EVENT_TAIL:
        amdvi_mmio_reg_write(s, size, val, addr);
        amdvi_handle_evttail_write(s);
        break;
    case AMDVI_MMIO_EXCL_LIMIT:
        amdvi_mmio_reg_write(s, size, val, addr);
        amdvi_handle_excllim_write(s);
        break;
        /* PPR log base - unused for now */
    case AMDVI_MMIO_PPR_BASE:
        amdvi_mmio_reg_write(s, size, val, addr);
        amdvi_handle_pprbase_write(s);
        break;
        /* PPR log head - also unused for now */
    case AMDVI_MMIO_PPR_HEAD:
        amdvi_mmio_reg_write(s, size, val, addr);
        amdvi_handle_pprhead_write(s);
        break;
        /* PPR log tail - unused for now */
    case AMDVI_MMIO_PPR_TAIL:
        amdvi_mmio_reg_write(s, size, val, addr);
        amdvi_handle_pprtail_write(s);
        break;
    case AMDVI_MMIO_STATUS:
        amdvi_mmio_reg_write(s, size, val, addr);
        break;
    }
}

static void amdvi_page_walk(AMDVIAddressSpace *as, uint64_t *dte,
                            IOMMUTLBEntry *ret, unsigned perms,
                            hwaddr addr)
{
    hwaddr page_mask, pagesize = 0;
    uint8_t mode;
    uint64_t pte;
    int fetch_ret;

    /* make sure the DTE has TV = 1 */
    if (!(dte[0] & AMDVI_DEV_TRANSLATION_VALID)) {
        /*
         * A DTE with V=1, TV=0 does not have a valid Page Table Root Pointer.
         * An IOMMU processing a request that requires a table walk terminates
         * the walk when it encounters this condition. Do the same and return
         * instead of assuming that the address is forwarded without translation
         * i.e. the passthrough case, as it is done for the case where DTE[V]=0.
         */
        return;
    }

    mode = get_pte_translation_mode(dte[0]);
    if (mode >= 7) {
        trace_amdvi_mode_invalid(mode, addr);
        return;
    }
    if (mode == 0) {
        goto no_remap;
    }

    /* Attempt to fetch the PTE to determine if a valid mapping exists */
    fetch_ret = fetch_pte(as, addr, dte[0], &pte, &pagesize);

    /*
     * If walking the page table results in an error of any type, returns an
     * empty PTE i.e. no mapping, or the permissions do not match, return since
     * there is no translation available.
     */
    if (fetch_ret < 0 || !IOMMU_PTE_PRESENT(pte) ||
        perms != (perms & amdvi_get_perms(pte))) {

        amdvi_page_fault(as->iommu_state, as->devfn, addr, perms);
        trace_amdvi_page_fault(addr);
        return;
    }

    /* A valid PTE and page size has been retrieved */
    assert(pagesize);
    page_mask = ~(pagesize - 1);

    /* get access permissions from pte */
    ret->iova = addr & page_mask;
    ret->translated_addr = (pte & AMDVI_DEV_PT_ROOT_MASK) & page_mask;
    ret->addr_mask = ~page_mask;
    ret->perm = amdvi_get_perms(pte);
    return;

no_remap:
    ret->iova = addr & AMDVI_PAGE_MASK_4K;
    ret->translated_addr = addr & AMDVI_PAGE_MASK_4K;
    ret->addr_mask = ~AMDVI_PAGE_MASK_4K;
    ret->perm = amdvi_get_perms(dte[0]);
}

static void amdvi_do_translate(AMDVIAddressSpace *as, hwaddr addr,
                               bool is_write, IOMMUTLBEntry *ret)
{
    AMDVIState *s = as->iommu_state;
    uint16_t devid = PCI_BUILD_BDF(as->bus_num, as->devfn);
    AMDVIIOTLBEntry *iotlb_entry = amdvi_iotlb_lookup(s, addr, devid);
    uint64_t entry[4];
    int dte_ret;

    if (iotlb_entry) {
        trace_amdvi_iotlb_hit(PCI_BUS_NUM(devid), PCI_SLOT(devid),
                PCI_FUNC(devid), addr, iotlb_entry->translated_addr);
        ret->iova = addr & ~iotlb_entry->page_mask;
        ret->translated_addr = iotlb_entry->translated_addr;
        ret->addr_mask = iotlb_entry->page_mask;
        ret->perm = iotlb_entry->perms;
        return;
    }

    dte_ret = amdvi_as_to_dte(as, entry);

    if (dte_ret < 0) {
        if (dte_ret == -AMDVI_FR_DTE_V) {
            /* DTE[V]=0, address is passed untranslated */
            goto out;
        }
        return;
    }

    amdvi_page_walk(as, entry, ret,
                    is_write ? AMDVI_PERM_WRITE : AMDVI_PERM_READ, addr);

    amdvi_update_iotlb(s, devid, addr, *ret,
                       entry[1] & AMDVI_DEV_DOMID_ID_MASK);
    return;

out:
    ret->iova = addr & AMDVI_PAGE_MASK_4K;
    ret->translated_addr = addr & AMDVI_PAGE_MASK_4K;
    ret->addr_mask = ~AMDVI_PAGE_MASK_4K;
    ret->perm = IOMMU_RW;
}

static inline bool amdvi_is_interrupt_addr(hwaddr addr)
{
    return addr >= AMDVI_INT_ADDR_FIRST && addr <= AMDVI_INT_ADDR_LAST;
}

static IOMMUTLBEntry amdvi_translate(IOMMUMemoryRegion *iommu, hwaddr addr,
                                     IOMMUAccessFlags flag, int iommu_idx)
{
    AMDVIAddressSpace *as = container_of(iommu, AMDVIAddressSpace, iommu);
    AMDVIState *s = as->iommu_state;
    IOMMUTLBEntry ret = {
        .target_as = &address_space_memory,
        .iova = addr,
        .translated_addr = 0,
        .addr_mask = ~(hwaddr)0,
        .perm = IOMMU_NONE
    };

    if (!s->enabled) {
        /* AMDVI disabled - corresponds to iommu=off not
         * failure to provide any parameter
         */
        ret.iova = addr & AMDVI_PAGE_MASK_4K;
        ret.translated_addr = addr & AMDVI_PAGE_MASK_4K;
        ret.addr_mask = ~AMDVI_PAGE_MASK_4K;
        ret.perm = IOMMU_RW;
        return ret;
    } else if (amdvi_is_interrupt_addr(addr)) {
        ret.iova = addr & AMDVI_PAGE_MASK_4K;
        ret.translated_addr = addr & AMDVI_PAGE_MASK_4K;
        ret.addr_mask = ~AMDVI_PAGE_MASK_4K;
        ret.perm = IOMMU_WO;
        return ret;
    }

    amdvi_do_translate(as, addr, flag & IOMMU_WO, &ret);
    trace_amdvi_translation_result(as->bus_num, PCI_SLOT(as->devfn),
            PCI_FUNC(as->devfn), addr, ret.translated_addr);
    return ret;
}

static int amdvi_get_irte(AMDVIState *s, MSIMessage *origin, uint64_t *dte,
                          union irte *irte, uint16_t devid)
{
    uint64_t irte_root, offset;

    irte_root = dte[2] & AMDVI_IR_PHYS_ADDR_MASK;
    offset = (origin->data & AMDVI_IRTE_OFFSET) << 2;

    trace_amdvi_ir_irte(irte_root, offset);

    if (dma_memory_read(&address_space_memory, irte_root + offset,
                        irte, sizeof(*irte), MEMTXATTRS_UNSPECIFIED)) {
        trace_amdvi_ir_err("failed to get irte");
        return -AMDVI_IR_GET_IRTE;
    }

    trace_amdvi_ir_irte_val(irte->val);

    return 0;
}

static int amdvi_int_remap_legacy(AMDVIState *iommu,
                                  MSIMessage *origin,
                                  MSIMessage *translated,
                                  uint64_t *dte,
                                  X86IOMMUIrq *irq,
                                  uint16_t sid)
{
    int ret;
    union irte irte;

    /* get interrupt remapping table */
    ret = amdvi_get_irte(iommu, origin, dte, &irte, sid);
    if (ret < 0) {
        return ret;
    }

    if (!irte.fields.valid) {
        trace_amdvi_ir_target_abort("RemapEn is disabled");
        return -AMDVI_IR_TARGET_ABORT;
    }

    if (irte.fields.guest_mode) {
        error_report_once("guest mode is not zero");
        return -AMDVI_IR_ERR;
    }

    if (irte.fields.int_type > AMDVI_IOAPIC_INT_TYPE_ARBITRATED) {
        error_report_once("reserved int_type");
        return -AMDVI_IR_ERR;
    }

    irq->delivery_mode = irte.fields.int_type;
    irq->vector = irte.fields.vector;
    irq->dest_mode = irte.fields.dm;
    irq->redir_hint = irte.fields.rq_eoi;
    irq->dest = irte.fields.destination;

    return 0;
}

static int amdvi_get_irte_ga(AMDVIState *s, MSIMessage *origin, uint64_t *dte,
                             struct irte_ga *irte, uint16_t devid)
{
    uint64_t irte_root, offset;

    irte_root = dte[2] & AMDVI_IR_PHYS_ADDR_MASK;
    offset = (origin->data & AMDVI_IRTE_OFFSET) << 4;
    trace_amdvi_ir_irte(irte_root, offset);

    if (dma_memory_read(&address_space_memory, irte_root + offset,
                        irte, sizeof(*irte), MEMTXATTRS_UNSPECIFIED)) {
        trace_amdvi_ir_err("failed to get irte_ga");
        return -AMDVI_IR_GET_IRTE;
    }

    trace_amdvi_ir_irte_ga_val(irte->hi.val, irte->lo.val);
    return 0;
}

static int amdvi_int_remap_ga(AMDVIState *iommu,
                              MSIMessage *origin,
                              MSIMessage *translated,
                              uint64_t *dte,
                              X86IOMMUIrq *irq,
                              uint16_t sid)
{
    int ret;
    struct irte_ga irte;

    /* get interrupt remapping table */
    ret = amdvi_get_irte_ga(iommu, origin, dte, &irte, sid);
    if (ret < 0) {
        return ret;
    }

    if (!irte.lo.fields_remap.valid) {
        trace_amdvi_ir_target_abort("RemapEn is disabled");
        return -AMDVI_IR_TARGET_ABORT;
    }

    if (irte.lo.fields_remap.guest_mode) {
        error_report_once("guest mode is not zero");
        return -AMDVI_IR_ERR;
    }

    if (irte.lo.fields_remap.int_type > AMDVI_IOAPIC_INT_TYPE_ARBITRATED) {
        error_report_once("reserved int_type is set");
        return -AMDVI_IR_ERR;
    }

    irq->delivery_mode = irte.lo.fields_remap.int_type;
    irq->vector = irte.hi.fields.vector;
    irq->dest_mode = irte.lo.fields_remap.dm;
    irq->redir_hint = irte.lo.fields_remap.rq_eoi;
    if (iommu->xtsup) {
        irq->dest = irte.lo.fields_remap.destination |
                    (irte.hi.fields.destination_hi << 24);
    } else {
        irq->dest = irte.lo.fields_remap.destination & 0xff;
    }

    return 0;
}

static int __amdvi_int_remap_msi(AMDVIState *iommu,
                                 MSIMessage *origin,
                                 MSIMessage *translated,
                                 uint64_t *dte,
                                 X86IOMMUIrq *irq,
                                 uint16_t sid)
{
    int ret;
    uint8_t int_ctl;

    int_ctl = (dte[2] >> AMDVI_IR_INTCTL_SHIFT) & 3;
    trace_amdvi_ir_intctl(int_ctl);

    switch (int_ctl) {
    case AMDVI_IR_INTCTL_PASS:
        memcpy(translated, origin, sizeof(*origin));
        return 0;
    case AMDVI_IR_INTCTL_REMAP:
        break;
    case AMDVI_IR_INTCTL_ABORT:
        trace_amdvi_ir_target_abort("int_ctl abort");
        return -AMDVI_IR_TARGET_ABORT;
    default:
        trace_amdvi_ir_err("int_ctl reserved");
        return -AMDVI_IR_ERR;
    }

    if (iommu->ga_enabled) {
        ret = amdvi_int_remap_ga(iommu, origin, translated, dte, irq, sid);
    } else {
        ret = amdvi_int_remap_legacy(iommu, origin, translated, dte, irq, sid);
    }

    return ret;
}

/* Interrupt remapping for MSI/MSI-X entry */
static int amdvi_int_remap_msi(AMDVIState *iommu,
                               MSIMessage *origin,
                               MSIMessage *translated,
                               uint16_t sid)
{
    int ret = 0;
    uint64_t pass = 0;
    uint64_t dte[4] = { 0 };
    X86IOMMUIrq irq = { 0 };
    uint8_t dest_mode, delivery_mode;

    assert(origin && translated);

    /*
     * When IOMMU is enabled, interrupt remap request will come either from
     * IO-APIC or PCI device. If interrupt is from PCI device then it will
     * have a valid requester id but if the interrupt is from IO-APIC
     * then requester id will be invalid.
     */
    if (sid == X86_IOMMU_SID_INVALID) {
        sid = AMDVI_IOAPIC_SB_DEVID;
    }

    trace_amdvi_ir_remap_msi_req(origin->address, origin->data, sid);

    /* check if device table entry is set before we go further. */
    if (!iommu || !iommu->devtab_len) {
        memcpy(translated, origin, sizeof(*origin));
        goto out;
    }

    if (!amdvi_get_dte(iommu, sid, dte)) {
        return -AMDVI_IR_ERR;
    }

    /* Check if IR is enabled in DTE */
    if (!(dte[2] & AMDVI_IR_REMAP_ENABLE)) {
        memcpy(translated, origin, sizeof(*origin));
        goto out;
    }

    /* validate that we are configure with intremap=on */
    if (!x86_iommu_ir_supported(X86_IOMMU_DEVICE(iommu))) {
        trace_amdvi_err("Interrupt remapping is enabled in the guest but "
                        "not in the host. Use intremap=on to enable interrupt "
                        "remapping in amd-iommu.");
        return -AMDVI_IR_ERR;
    }

    if (origin->address < AMDVI_INT_ADDR_FIRST ||
        origin->address + sizeof(origin->data) > AMDVI_INT_ADDR_LAST + 1) {
        trace_amdvi_err("MSI is not from IOAPIC.");
        return -AMDVI_IR_ERR;
    }

    /*
     * The MSI data register [10:8] are used to get the upstream interrupt type.
     *
     * See MSI/MSI-X format:
     * https://pdfs.semanticscholar.org/presentation/9420/c279e942eca568157711ef5c92b800c40a79.pdf
     * (page 5)
     */
    delivery_mode = (origin->data >> MSI_DATA_DELIVERY_MODE_SHIFT) & 7;

    switch (delivery_mode) {
    case AMDVI_IOAPIC_INT_TYPE_FIXED:
    case AMDVI_IOAPIC_INT_TYPE_ARBITRATED:
        trace_amdvi_ir_delivery_mode("fixed/arbitrated");
        ret = __amdvi_int_remap_msi(iommu, origin, translated, dte, &irq, sid);
        if (ret < 0) {
            goto remap_fail;
        } else {
            /* Translate IRQ to MSI messages */
            x86_iommu_irq_to_msi_message(&irq, translated);
            goto out;
        }
        break;
    case AMDVI_IOAPIC_INT_TYPE_SMI:
        error_report("SMI is not supported!");
        ret = -AMDVI_IR_ERR;
        break;
    case AMDVI_IOAPIC_INT_TYPE_NMI:
        pass = dte[2] & AMDVI_DEV_NMI_PASS_MASK;
        trace_amdvi_ir_delivery_mode("nmi");
        break;
    case AMDVI_IOAPIC_INT_TYPE_INIT:
        pass = dte[2] & AMDVI_DEV_INT_PASS_MASK;
        trace_amdvi_ir_delivery_mode("init");
        break;
    case AMDVI_IOAPIC_INT_TYPE_EINT:
        pass = dte[2] & AMDVI_DEV_EINT_PASS_MASK;
        trace_amdvi_ir_delivery_mode("eint");
        break;
    default:
        trace_amdvi_ir_delivery_mode("unsupported delivery_mode");
        ret = -AMDVI_IR_ERR;
        break;
    }

    if (ret < 0) {
        goto remap_fail;
    }

    /*
     * The MSI address register bit[2] is used to get the destination
     * mode. The dest_mode 1 is valid for fixed and arbitrated interrupts
     * only.
     */
    dest_mode = (origin->address >> MSI_ADDR_DEST_MODE_SHIFT) & 1;
    if (dest_mode) {
        trace_amdvi_ir_err("invalid dest_mode");
        ret = -AMDVI_IR_ERR;
        goto remap_fail;
    }

    if (pass) {
        memcpy(translated, origin, sizeof(*origin));
    } else {
        trace_amdvi_ir_err("passthrough is not enabled");
        ret = -AMDVI_IR_ERR;
        goto remap_fail;
    }

out:
    trace_amdvi_ir_remap_msi(origin->address, origin->data,
                             translated->address, translated->data);
    return 0;

remap_fail:
    return ret;
}

static int amdvi_int_remap(X86IOMMUState *iommu,
                           MSIMessage *origin,
                           MSIMessage *translated,
                           uint16_t sid)
{
    return amdvi_int_remap_msi(AMD_IOMMU_DEVICE(iommu), origin,
                               translated, sid);
}

static MemTxResult amdvi_mem_ir_write(void *opaque, hwaddr addr,
                                      uint64_t value, unsigned size,
                                      MemTxAttrs attrs)
{
    int ret;
    MSIMessage from = { 0, 0 }, to = { 0, 0 };
    uint16_t sid = AMDVI_IOAPIC_SB_DEVID;

    from.address = (uint64_t) addr + AMDVI_INT_ADDR_FIRST;
    from.data = (uint32_t) value;

    trace_amdvi_mem_ir_write_req(addr, value, size);

    if (!attrs.unspecified) {
        /* We have explicit Source ID */
        sid = attrs.requester_id;
    }

    ret = amdvi_int_remap_msi(opaque, &from, &to, sid);
    if (ret < 0) {
        /* TODO: log the event using IOMMU log event interface */
        error_report_once("failed to remap interrupt from devid 0x%x", sid);
        return MEMTX_ERROR;
    }

    apic_get_class(NULL)->send_msi(&to);

    trace_amdvi_mem_ir_write(to.address, to.data);
    return MEMTX_OK;
}

static MemTxResult amdvi_mem_ir_read(void *opaque, hwaddr addr,
                                     uint64_t *data, unsigned size,
                                     MemTxAttrs attrs)
{
    return MEMTX_OK;
}

static const MemoryRegionOps amdvi_ir_ops = {
    .read_with_attrs = amdvi_mem_ir_read,
    .write_with_attrs = amdvi_mem_ir_write,
    .endianness = DEVICE_LITTLE_ENDIAN,
    .impl = {
        .min_access_size = 4,
        .max_access_size = 4,
    },
    .valid = {
        .min_access_size = 4,
        .max_access_size = 4,
    }
};

static AddressSpace *amdvi_host_dma_iommu(PCIBus *bus, void *opaque, int devfn)
{
    char name[128];
    AMDVIState *s = opaque;
    AMDVIAddressSpace **iommu_as, *amdvi_dev_as;
    int bus_num = pci_bus_num(bus);

    iommu_as = s->address_spaces[bus_num];

    /* allocate memory during the first run */
    if (!iommu_as) {
        iommu_as = g_new0(AMDVIAddressSpace *, PCI_DEVFN_MAX);
        s->address_spaces[bus_num] = iommu_as;
    }

    /* set up AMD-Vi region */
    if (!iommu_as[devfn]) {
        snprintf(name, sizeof(name), "amd_iommu_devfn_%d", devfn);

        iommu_as[devfn] = g_new0(AMDVIAddressSpace, 1);
        iommu_as[devfn]->bus_num = (uint8_t)bus_num;
        iommu_as[devfn]->devfn = (uint8_t)devfn;
        iommu_as[devfn]->iommu_state = s;
        iommu_as[devfn]->notifier_flags = IOMMU_NOTIFIER_NONE;
        iommu_as[devfn]->iova_tree = iova_tree_new();
        iommu_as[devfn]->addr_translation = false;

        amdvi_dev_as = iommu_as[devfn];

        /*
         * Memory region relationships looks like (Address range shows
         * only lower 32 bits to make it short in length...):
         *
         * |--------------------+-------------------+----------|
         * | Name               | Address range     | Priority |
         * |--------------------+-------------------+----------+
         * | amdvi-root         | 00000000-ffffffff |        0 |
         * |  amdvi-iommu_nodma  | 00000000-ffffffff |       0 |
         * |  amdvi-iommu_ir     | fee00000-feefffff |       1 |
         * |--------------------+-------------------+----------|
         */
        memory_region_init_iommu(&amdvi_dev_as->iommu,
                                 sizeof(amdvi_dev_as->iommu),
                                 TYPE_AMD_IOMMU_MEMORY_REGION,
                                 OBJECT(s),
                                 "amd_iommu", UINT64_MAX);
        memory_region_init(&amdvi_dev_as->root, OBJECT(s),
                           "amdvi_root", UINT64_MAX);
        address_space_init(&amdvi_dev_as->as, &amdvi_dev_as->root, name);
        memory_region_add_subregion_overlap(&amdvi_dev_as->root, 0,
                                            MEMORY_REGION(&amdvi_dev_as->iommu),
                                            0);

        /* Build the DMA Disabled alias to shared memory */
        memory_region_init_alias(&amdvi_dev_as->iommu_nodma, OBJECT(s),
                                 "amdvi-sys", &s->mr_sys, 0,
                                 memory_region_size(&s->mr_sys));
        memory_region_add_subregion_overlap(&amdvi_dev_as->root, 0,
                                            &amdvi_dev_as->iommu_nodma,
                                            0);
        /* Build the Interrupt Remapping alias to shared memory */
        memory_region_init_alias(&amdvi_dev_as->iommu_ir, OBJECT(s),
                                 "amdvi-ir", &s->mr_ir, 0,
                                 memory_region_size(&s->mr_ir));
        memory_region_add_subregion_overlap(MEMORY_REGION(&amdvi_dev_as->iommu),
                                            AMDVI_INT_ADDR_FIRST,
                                            &amdvi_dev_as->iommu_ir, 1);

        amdvi_switch_address_space(amdvi_dev_as);
    }
    return &iommu_as[devfn]->as;
}

static const PCIIOMMUOps amdvi_iommu_ops = {
    .get_address_space = amdvi_host_dma_iommu,
};

static const MemoryRegionOps mmio_mem_ops = {
    .read = amdvi_mmio_read,
    .write = amdvi_mmio_write,
    .endianness = DEVICE_LITTLE_ENDIAN,
    .impl = {
        .min_access_size = 1,
        .max_access_size = 8,
        .unaligned = false,
    },
    .valid = {
        .min_access_size = 1,
        .max_access_size = 8,
    }
};

static int amdvi_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu,
                                           IOMMUNotifierFlag old,
                                           IOMMUNotifierFlag new,
                                           Error **errp)
{
    AMDVIAddressSpace *as = container_of(iommu, AMDVIAddressSpace, iommu);
    AMDVIState *s = as->iommu_state;

    /*
     * Accurate synchronization of the vIOMMU page tables required to support
     * MAP notifiers is provided by the dma-remap feature. In addition, this
     * also requires that the vIOMMU presents the NpCache capability, so a guest
     * driver issues invalidations for both map() and unmap() operations. The
     * capability is already set by default as part of AMDVI_CAPAB_FEATURES and
     * written to the configuration in amdvi_pci_realize().
     */
    if (!s->dma_remap && (new & IOMMU_NOTIFIER_MAP)) {
        error_setg_errno(errp, ENOTSUP,
            "device %02x.%02x.%x requires dma-remap=1",
            as->bus_num, PCI_SLOT(as->devfn), PCI_FUNC(as->devfn));
        return -ENOTSUP;
    }

    /*
     * Update notifier flags for address space and the list of address spaces
     * with registered notifiers.
     */
    as->notifier_flags = new;

    if (old == IOMMU_NOTIFIER_NONE) {
        QLIST_INSERT_HEAD(&s->amdvi_as_with_notifiers, as, next);
    } else if (new == IOMMU_NOTIFIER_NONE) {
        QLIST_REMOVE(as, next);
    }

    return 0;
}

static void amdvi_init(AMDVIState *s)
{
    amdvi_iotlb_reset(s);

    s->devtab_len = 0;
    s->cmdbuf_len = 0;
    s->cmdbuf_head = 0;
    s->cmdbuf_tail = 0;
    s->evtlog_head = 0;
    s->evtlog_tail = 0;
    s->excl_enabled = false;
    s->excl_allow = false;
    s->mmio_enabled = false;
    s->enabled = false;
    s->cmdbuf_enabled = false;

    /* reset MMIO */
    memset(s->mmior, 0, AMDVI_MMIO_SIZE);
    amdvi_set_quad(s, AMDVI_MMIO_EXT_FEATURES,
                   amdvi_extended_feature_register(s),
                   0xffffffffffffffef, 0);
    amdvi_set_quad(s, AMDVI_MMIO_STATUS, 0, 0x98, 0x67);
}

static void amdvi_pci_realize(PCIDevice *pdev, Error **errp)
{
    AMDVIPCIState *s = AMD_IOMMU_PCI(pdev);
    int ret;

    ret = pci_add_capability(pdev, AMDVI_CAPAB_ID_SEC, 0,
                             AMDVI_CAPAB_SIZE, errp);
    if (ret < 0) {
        return;
    }
    s->capab_offset = ret;

    ret = pci_add_capability(pdev, PCI_CAP_ID_MSI, 0,
                             AMDVI_CAPAB_REG_SIZE, errp);
    if (ret < 0) {
        return;
    }
    ret = pci_add_capability(pdev, PCI_CAP_ID_HT, 0,
                             AMDVI_CAPAB_REG_SIZE, errp);
    if (ret < 0) {
        return;
    }

    if (msi_init(pdev, 0, 1, true, false, errp) < 0) {
        return;
    }

    /* reset device ident */
    pci_config_set_prog_interface(pdev->config, 0);

    /* reset AMDVI specific capabilities, all r/o */
    pci_set_long(pdev->config + s->capab_offset, AMDVI_CAPAB_FEATURES);
    pci_set_long(pdev->config + s->capab_offset + AMDVI_CAPAB_BAR_LOW,
                 AMDVI_BASE_ADDR & MAKE_64BIT_MASK(14, 18));
    pci_set_long(pdev->config + s->capab_offset + AMDVI_CAPAB_BAR_HIGH,
                AMDVI_BASE_ADDR >> 32);
    pci_set_long(pdev->config + s->capab_offset + AMDVI_CAPAB_RANGE,
                 0xff000000);
    pci_set_long(pdev->config + s->capab_offset + AMDVI_CAPAB_MISC, 0);
    pci_set_long(pdev->config + s->capab_offset + AMDVI_CAPAB_MISC,
            AMDVI_MAX_PH_ADDR | AMDVI_MAX_GVA_ADDR | AMDVI_MAX_VA_ADDR);
}

static void amdvi_sysbus_reset(DeviceState *dev)
{
    AMDVIState *s = AMD_IOMMU_DEVICE(dev);

    msi_reset(&s->pci->dev);
    amdvi_init(s);

    /* Discard all mappings on device reset */
    amdvi_address_space_unmap_all(s);
    amdvi_reset_address_translation_all(s);
}

static const VMStateDescription vmstate_amdvi_sysbus_migratable = {
    .name = "amd-iommu",
    .version_id = 1,
    .minimum_version_id = 1,
    .priority = MIG_PRI_IOMMU,
    .fields = (VMStateField[]) {
      /* Updated in  amdvi_handle_control_write() */
      VMSTATE_BOOL(enabled, AMDVIState),
      VMSTATE_BOOL(ga_enabled, AMDVIState),
      /* bool ats_enabled is obsolete */
      VMSTATE_UNUSED(1), /* was ats_enabled */
      VMSTATE_BOOL(cmdbuf_enabled, AMDVIState),
      VMSTATE_BOOL(completion_wait_intr, AMDVIState),
      VMSTATE_BOOL(evtlog_enabled, AMDVIState),
      VMSTATE_BOOL(evtlog_intr, AMDVIState),
      /* Updated in amdvi_handle_devtab_write() */
      VMSTATE_UINT64(devtab, AMDVIState),
      VMSTATE_UINT64(devtab_len, AMDVIState),
      /* Updated in amdvi_handle_cmdbase_write() */
      VMSTATE_UINT64(cmdbuf, AMDVIState),
      VMSTATE_UINT64(cmdbuf_len, AMDVIState),
      /* Updated in amdvi_handle_cmdhead_write() */
      VMSTATE_UINT32(cmdbuf_head, AMDVIState),
      /* Updated in amdvi_handle_cmdtail_write() */
      VMSTATE_UINT32(cmdbuf_tail, AMDVIState),
      /* Updated in amdvi_handle_evtbase_write() */
      VMSTATE_UINT64(evtlog, AMDVIState),
      VMSTATE_UINT32(evtlog_len, AMDVIState),
      /* Updated in amdvi_handle_evthead_write() */
      VMSTATE_UINT32(evtlog_head, AMDVIState),
      /* Updated in amdvi_handle_evttail_write() */
      VMSTATE_UINT32(evtlog_tail, AMDVIState),
      /* Updated in amdvi_handle_pprbase_write() */
      VMSTATE_UINT64(ppr_log, AMDVIState),
      VMSTATE_UINT32(pprlog_len, AMDVIState),
      /* Updated in amdvi_handle_pprhead_write() */
      VMSTATE_UINT32(pprlog_head, AMDVIState),
      /* Updated in amdvi_handle_tailhead_write() */
      VMSTATE_UINT32(pprlog_tail, AMDVIState),
      /* MMIO registers */
      VMSTATE_UINT8_ARRAY(mmior, AMDVIState, AMDVI_MMIO_SIZE),
      VMSTATE_UINT8_ARRAY(romask, AMDVIState, AMDVI_MMIO_SIZE),
      VMSTATE_UINT8_ARRAY(w1cmask, AMDVIState, AMDVI_MMIO_SIZE),
      VMSTATE_END_OF_LIST()
    }
};

static void amdvi_sysbus_realize(DeviceState *dev, Error **errp)
{
    DeviceClass *dc = (DeviceClass *) object_get_class(OBJECT(dev));
    AMDVIState *s = AMD_IOMMU_DEVICE(dev);
    MachineState *ms = MACHINE(qdev_get_machine());
    PCMachineState *pcms = PC_MACHINE(ms);
    X86MachineState *x86ms = X86_MACHINE(ms);
    PCIBus *bus = pcms->pcibus;

    if (s->pci_id) {
        PCIDevice *pdev = NULL;
        int ret = pci_qdev_find_device(s->pci_id, &pdev);

        if (ret) {
            error_report("Cannot find PCI device '%s'", s->pci_id);
            return;
        }

        if (!object_dynamic_cast(OBJECT(pdev), TYPE_AMD_IOMMU_PCI)) {
            error_report("Device '%s' must be an AMDVI-PCI device type", s->pci_id);
            return;
        }

        s->pci = AMD_IOMMU_PCI(pdev);
        dc->vmsd = &vmstate_amdvi_sysbus_migratable;
    } else {
        s->pci = AMD_IOMMU_PCI(object_new(TYPE_AMD_IOMMU_PCI));
        /* This device should take care of IOMMU PCI properties */
        if (!qdev_realize(DEVICE(s->pci), &bus->qbus, errp)) {
            return;
        }
    }

    s->iotlb = g_hash_table_new_full(amdvi_uint64_hash,
                                     amdvi_uint64_equal, g_free, g_free);

    /* set up MMIO */
    memory_region_init_io(&s->mr_mmio, OBJECT(s), &mmio_mem_ops, s,
                          "amdvi-mmio", AMDVI_MMIO_SIZE);
    memory_region_add_subregion(get_system_memory(), AMDVI_BASE_ADDR,
                                &s->mr_mmio);

    /* Create the share memory regions by all devices */
    memory_region_init(&s->mr_sys, OBJECT(s), "amdvi-sys", UINT64_MAX);

    /* set up the DMA disabled memory region */
    memory_region_init_alias(&s->mr_nodma, OBJECT(s),
                             "amdvi-nodma", get_system_memory(), 0,
                             memory_region_size(get_system_memory()));
    memory_region_add_subregion_overlap(&s->mr_sys, 0,
                                        &s->mr_nodma, 0);

    /* set up the Interrupt Remapping memory region */
    memory_region_init_io(&s->mr_ir, OBJECT(s), &amdvi_ir_ops,
                          s, "amdvi-ir", AMDVI_INT_ADDR_SIZE);
    memory_region_add_subregion_overlap(&s->mr_sys, AMDVI_INT_ADDR_FIRST,
                                        &s->mr_ir, 1);

    /* Pseudo address space under root PCI bus. */
    x86ms->ioapic_as = amdvi_host_dma_iommu(bus, s, AMDVI_IOAPIC_SB_DEVID);

    if (kvm_enabled() && x86ms->apic_id_limit > 255 && !s->xtsup) {
        error_report("AMD IOMMU with x2APIC configuration requires xtsup=on");
        exit(EXIT_FAILURE);
    }

    if (s->xtsup) {
        if (kvm_irqchip_is_split() && !kvm_enable_x2apic()) {
            error_report("AMD IOMMU xtsup=on requires x2APIC support on "
                          "the KVM side");
            exit(EXIT_FAILURE);
        }
    }

    pci_setup_iommu(bus, &amdvi_iommu_ops, s);
    amdvi_init(s);
}

static const Property amdvi_properties[] = {
    DEFINE_PROP_BOOL("xtsup", AMDVIState, xtsup, false),
    DEFINE_PROP_STRING("pci-id", AMDVIState, pci_id),
    DEFINE_PROP_BOOL("dma-remap", AMDVIState, dma_remap, false),
};

static const VMStateDescription vmstate_amdvi_sysbus = {
    .name = "amd-iommu",
    .unmigratable = 1
};

static void amdvi_sysbus_class_init(ObjectClass *klass, const void *data)
{
    DeviceClass *dc = DEVICE_CLASS(klass);
    X86IOMMUClass *dc_class = X86_IOMMU_DEVICE_CLASS(klass);

    device_class_set_legacy_reset(dc, amdvi_sysbus_reset);
    dc->vmsd = &vmstate_amdvi_sysbus;
    dc->hotpluggable = false;
    dc_class->realize = amdvi_sysbus_realize;
    dc_class->int_remap = amdvi_int_remap;
    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
    dc->desc = "AMD IOMMU (AMD-Vi) DMA Remapping device";
    device_class_set_props(dc, amdvi_properties);
}

static const TypeInfo amdvi_sysbus = {
    .name = TYPE_AMD_IOMMU_DEVICE,
    .parent = TYPE_X86_IOMMU_DEVICE,
    .instance_size = sizeof(AMDVIState),
    .class_init = amdvi_sysbus_class_init
};

static void amdvi_pci_class_init(ObjectClass *klass, const void *data)
{
    DeviceClass *dc = DEVICE_CLASS(klass);
    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);

    k->vendor_id = PCI_VENDOR_ID_AMD;
    k->device_id = 0x1419;
    k->class_id = 0x0806;
    k->realize = amdvi_pci_realize;

    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
    dc->desc = "AMD IOMMU (AMD-Vi) DMA Remapping device";
}

static const TypeInfo amdvi_pci = {
    .name = TYPE_AMD_IOMMU_PCI,
    .parent = TYPE_PCI_DEVICE,
    .instance_size = sizeof(AMDVIPCIState),
    .class_init = amdvi_pci_class_init,
    .interfaces = (const InterfaceInfo[]) {
        { INTERFACE_CONVENTIONAL_PCI_DEVICE },
        { },
    },
};

static void amdvi_iommu_memory_region_class_init(ObjectClass *klass,
                                                 const void *data)
{
    IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass);

    imrc->translate = amdvi_translate;
    imrc->notify_flag_changed = amdvi_iommu_notify_flag_changed;
    imrc->replay = amdvi_iommu_replay;
}

static const TypeInfo amdvi_iommu_memory_region_info = {
    .parent = TYPE_IOMMU_MEMORY_REGION,
    .name = TYPE_AMD_IOMMU_MEMORY_REGION,
    .class_init = amdvi_iommu_memory_region_class_init,
};

static void amdvi_register_types(void)
{
    type_register_static(&amdvi_pci);
    type_register_static(&amdvi_sysbus);
    type_register_static(&amdvi_iommu_memory_region_info);
}

type_init(amdvi_register_types);