28 files changed, 2380 insertions, 544 deletions

diff --git a/.gitignore b/.gitignore
index c72955aad1..81b1510297 100644
--- a/.gitignore
+++ b/.gitignore
@@ -77,6 +77,7 @@ pc-bios/vgabios-pq/status
 pc-bios/optionrom/linuxboot.bin
 pc-bios/optionrom/multiboot.bin
 pc-bios/optionrom/multiboot.raw
+pc-bios/optionrom/kvmvapic.bin
 .stgit-*
 cscope.*
 tags
diff --git a/Makefile b/Makefile
index d49b84e0e1..49c775b82d 100644
--- a/Makefile
+++ b/Makefile
@@ -256,7 +256,7 @@ pxe-e1000.rom pxe-eepro100.rom pxe-ne2k_pci.rom \
 pxe-pcnet.rom pxe-rtl8139.rom pxe-virtio.rom \
 bamboo.dtb petalogix-s3adsp1800.dtb petalogix-ml605.dtb \
 mpc8544ds.dtb \
-multiboot.bin linuxboot.bin \
+multiboot.bin linuxboot.bin kvmvapic.bin \
 s390-zipl.rom \
 spapr-rtas.bin slof.bin \
 palcode-clipper
diff --git a/Makefile.target b/Makefile.target
index 68a5641183..692c9d78a6 100644
--- a/Makefile.target
+++ b/Makefile.target
@@ -237,7 +237,8 @@ obj-y += device-hotplug.o
 
 # Hardware support
 obj-i386-y += mc146818rtc.o pc.o
-obj-i386-y += sga.o apic_common.o apic.o ioapic_common.o ioapic.o piix_pci.o
+obj-i386-y += apic_common.o apic.o kvmvapic.o
+obj-i386-y += sga.o ioapic_common.o ioapic.o piix_pci.o
 obj-i386-y += vmport.o
 obj-i386-y += pci-hotplug.o smbios.o wdt_ib700.o
 obj-i386-y += debugcon.o multiboot.o
diff --git a/cpu-all.h b/cpu-all.h
index e2c3c49283..80e6d4234f 100644
--- a/cpu-all.h
+++ b/cpu-all.h
@@ -375,8 +375,9 @@ DECLARE_TLS(CPUState *,cpu_single_env);
 #define CPU_INTERRUPT_TGT_INT_0   0x0100
 #define CPU_INTERRUPT_TGT_INT_1   0x0400
 #define CPU_INTERRUPT_TGT_INT_2   0x0800
+#define CPU_INTERRUPT_TGT_INT_3   0x2000
 
-/* First unused bit: 0x2000.  */
+/* First unused bit: 0x4000.  */
 
 /* The set of all bits that should be masked when single-stepping.  */
 #define CPU_INTERRUPT_SSTEP_MASK \
diff --git a/cpus.c b/cpus.c
index f45a438b29..17b055fba0 100644
--- a/cpus.c
+++ b/cpus.c
@@ -714,6 +714,7 @@ static void *qemu_kvm_cpu_thread_fn(void *arg)
     qemu_mutex_lock(&qemu_global_mutex);
     qemu_thread_get_self(env->thread);
     env->thread_id = qemu_get_thread_id();
+    cpu_single_env = env;
 
     r = kvm_init_vcpu(env);
     if (r < 0) {
@@ -760,6 +761,11 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
     /* wait for initial kick-off after machine start */
     while (first_cpu->stopped) {
         qemu_cond_wait(tcg_halt_cond, &qemu_global_mutex);
+
+        /* process any pending work */
+        for (env = first_cpu; env != NULL; env = env->next_cpu) {
+            qemu_wait_io_event_common(env);
+        }
     }
 
     while (1) {
@@ -852,7 +858,7 @@ static int all_vcpus_paused(void)
         if (!penv->stopped) {
             return 0;
         }
-        penv = (CPUState *)penv->next_cpu;
+        penv = penv->next_cpu;
     }
 
     return 1;
@@ -866,7 +872,19 @@ void pause_all_vcpus(void)
     while (penv) {
         penv->stop = 1;
         qemu_cpu_kick(penv);
-        penv = (CPUState *)penv->next_cpu;
+        penv = penv->next_cpu;
+    }
+
+    if (!qemu_thread_is_self(&io_thread)) {
+        cpu_stop_current();
+        if (!kvm_enabled()) {
+            while (penv) {
+                penv->stop = 0;
+                penv->stopped = 1;
+                penv = penv->next_cpu;
+            }
+            return;
+        }
     }
 
     while (!all_vcpus_paused()) {
@@ -874,7 +892,7 @@ void pause_all_vcpus(void)
         penv = first_cpu;
         while (penv) {
             qemu_cpu_kick(penv);
-            penv = (CPUState *)penv->next_cpu;
+            penv = penv->next_cpu;
         }
     }
 }
@@ -888,7 +906,7 @@ void resume_all_vcpus(void)
         penv->stop = 0;
         penv->stopped = 0;
         qemu_cpu_kick(penv);
-        penv = (CPUState *)penv->next_cpu;
+        penv = penv->next_cpu;
     }
 }
 
diff --git a/exec-obsolete.h b/exec-obsolete.h
index 94c23d0951..4dbe4768aa 100644
--- a/exec-obsolete.h
+++ b/exec-obsolete.h
@@ -37,7 +37,7 @@ void cpu_unregister_io_memory(int table_address);
 
 struct MemoryRegionSection;
 void cpu_register_physical_memory_log(struct MemoryRegionSection *section,
-                                      bool readable, bool readonly);
+                                      bool readonly);
 
 void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size);
 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size);
@@ -121,6 +121,9 @@ static inline void cpu_physical_memory_mask_dirty_range(ram_addr_t start,
 
 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
                                      int dirty_flags);
+
+extern const IORangeOps memory_region_iorange_ops;
+
 #endif
 
 #endif
diff --git a/exec.c b/exec.c
index b81677ade9..0c93b2691f 100644
--- a/exec.c
+++ b/exec.c
@@ -160,29 +160,21 @@ typedef struct PageDesc {
 #define L2_BITS 10
 #define L2_SIZE (1 << L2_BITS)
 
+#define P_L2_LEVELS \
+    (((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / L2_BITS) + 1)
+
 /* The bits remaining after N lower levels of page tables.  */
-#define P_L1_BITS_REM \
-    ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
 #define V_L1_BITS_REM \
     ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
 
-/* Size of the L1 page table.  Avoid silly small sizes.  */
-#if P_L1_BITS_REM < 4
-#define P_L1_BITS  (P_L1_BITS_REM + L2_BITS)
-#else
-#define P_L1_BITS  P_L1_BITS_REM
-#endif
-
 #if V_L1_BITS_REM < 4
 #define V_L1_BITS  (V_L1_BITS_REM + L2_BITS)
 #else
 #define V_L1_BITS  V_L1_BITS_REM
 #endif
 
-#define P_L1_SIZE  ((target_phys_addr_t)1 << P_L1_BITS)
 #define V_L1_SIZE  ((target_ulong)1 << V_L1_BITS)
 
-#define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
 
 unsigned long qemu_real_host_page_size;
@@ -194,15 +186,27 @@ unsigned long qemu_host_page_mask;
 static void *l1_map[V_L1_SIZE];
 
 #if !defined(CONFIG_USER_ONLY)
-typedef struct PhysPageDesc {
-    /* offset in host memory of the page + io_index in the low bits */
-    ram_addr_t phys_offset;
-    ram_addr_t region_offset;
-} PhysPageDesc;
+typedef struct PhysPageEntry PhysPageEntry;
+
+static MemoryRegionSection *phys_sections;
+static unsigned phys_sections_nb, phys_sections_nb_alloc;
+static uint16_t phys_section_unassigned;
+
+struct PhysPageEntry {
+    uint16_t is_leaf : 1;
+     /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
+    uint16_t ptr : 15;
+};
+
+/* Simple allocator for PhysPageEntry nodes */
+static PhysPageEntry (*phys_map_nodes)[L2_SIZE];
+static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc;
+
+#define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
 
 /* This is a multi-level map on the physical address space.
-   The bottom level has pointers to PhysPageDesc.  */
-static void *l1_phys_map[P_L1_SIZE];
+   The bottom level has pointers to MemoryRegionSections.  */
+static PhysPageEntry phys_map = { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
 
 static void io_mem_init(void);
 static void memory_map_init(void);
@@ -398,59 +402,112 @@ static inline PageDesc *page_find(tb_page_addr_t index)
 }
 
 #if !defined(CONFIG_USER_ONLY)
-static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
+
+static void phys_map_node_reserve(unsigned nodes)
 {
-    PhysPageDesc *pd;
-    void **lp;
-    int i;
+    if (phys_map_nodes_nb + nodes > phys_map_nodes_nb_alloc) {
+        typedef PhysPageEntry Node[L2_SIZE];
+        phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc * 2, 16);
+        phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc,
+                                      phys_map_nodes_nb + nodes);
+        phys_map_nodes = g_renew(Node, phys_map_nodes,
+                                 phys_map_nodes_nb_alloc);
+    }
+}
 
-    /* Level 1.  Always allocated.  */
-    lp = l1_phys_map + ((index >> P_L1_SHIFT) & (P_L1_SIZE - 1));
+static uint16_t phys_map_node_alloc(void)
+{
+    unsigned i;
+    uint16_t ret;
 
-    /* Level 2..N-1.  */
-    for (i = P_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
-        void **p = *lp;
-        if (p == NULL) {
-            if (!alloc) {
-                return NULL;
-            }
-            *lp = p = g_malloc0(sizeof(void *) * L2_SIZE);
-        }
-        lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
+    ret = phys_map_nodes_nb++;
+    assert(ret != PHYS_MAP_NODE_NIL);
+    assert(ret != phys_map_nodes_nb_alloc);
+    for (i = 0; i < L2_SIZE; ++i) {
+        phys_map_nodes[ret][i].is_leaf = 0;
+        phys_map_nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
     }
+    return ret;
+}
 
-    pd = *lp;
-    if (pd == NULL) {
-        int i;
-        int first_index = index & ~(L2_SIZE - 1);
+static void phys_map_nodes_reset(void)
+{
+    phys_map_nodes_nb = 0;
+}
 
-        if (!alloc) {
-            return NULL;
-        }
 
-        *lp = pd = g_malloc(sizeof(PhysPageDesc) * L2_SIZE);
+static void phys_page_set_level(PhysPageEntry *lp, target_phys_addr_t *index,
+                                target_phys_addr_t *nb, uint16_t leaf,
+                                int level)
+{
+    PhysPageEntry *p;
+    int i;
+    target_phys_addr_t step = (target_phys_addr_t)1 << (level * L2_BITS);
+
+    if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
+        lp->ptr = phys_map_node_alloc();
+        p = phys_map_nodes[lp->ptr];
+        if (level == 0) {
+            for (i = 0; i < L2_SIZE; i++) {
+                p[i].is_leaf = 1;
+                p[i].ptr = phys_section_unassigned;
+            }
+        }
+    } else {
+        p = phys_map_nodes[lp->ptr];
+    }
+    lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
 
-        for (i = 0; i < L2_SIZE; i++) {
-            pd[i].phys_offset = io_mem_unassigned.ram_addr;
-            pd[i].region_offset = (first_index + i) << TARGET_PAGE_BITS;
+    while (*nb && lp < &p[L2_SIZE]) {
+        if ((*index & (step - 1)) == 0 && *nb >= step) {
+            lp->is_leaf = true;
+            lp->ptr = leaf;
+            *index += step;
+            *nb -= step;
+        } else {
+            phys_page_set_level(lp, index, nb, leaf, level - 1);
         }
+        ++lp;
     }
+}
 
-    return pd + (index & (L2_SIZE - 1));
+static void phys_page_set(target_phys_addr_t index, target_phys_addr_t nb,
+                          uint16_t leaf)
+{
+    /* Wildly overreserve - it doesn't matter much. */
+    phys_map_node_reserve(3 * P_L2_LEVELS);
+
+    phys_page_set_level(&phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
 }
 
-static inline PhysPageDesc phys_page_find(target_phys_addr_t index)
+static MemoryRegionSection phys_page_find(target_phys_addr_t index)
 {
-    PhysPageDesc *p = phys_page_find_alloc(index, 0);
+    PhysPageEntry lp = phys_map;
+    PhysPageEntry *p;
+    int i;
+    MemoryRegionSection section;
+    target_phys_addr_t delta;
+    uint16_t s_index = phys_section_unassigned;
 
-    if (p) {
-        return *p;
-    } else {
-        return (PhysPageDesc) {
-            .phys_offset = io_mem_unassigned.ram_addr,
-            .region_offset = index << TARGET_PAGE_BITS,
-        };
+    for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
+        if (lp.ptr == PHYS_MAP_NODE_NIL) {
+            goto not_found;
+        }
+        p = phys_map_nodes[lp.ptr];
+        lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
     }
+
+    s_index = lp.ptr;
+not_found:
+    section = phys_sections[s_index];
+    index <<= TARGET_PAGE_BITS;
+    assert(section.offset_within_address_space <= index
+           && index <= section.offset_within_address_space + section.size-1);
+    delta = index - section.offset_within_address_space;
+    section.offset_within_address_space += delta;
+    section.offset_within_region += delta;
+    section.size -= delta;
+    return section;
 }
 
 static void tlb_protect_code(ram_addr_t ram_addr);
@@ -1410,14 +1467,18 @@ static void breakpoint_invalidate(CPUState *env, target_ulong pc)
 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
 {
     target_phys_addr_t addr;
-    target_ulong pd;
     ram_addr_t ram_addr;
-    PhysPageDesc p;
+    MemoryRegionSection section;
 
     addr = cpu_get_phys_page_debug(env, pc);
-    p = phys_page_find(addr >> TARGET_PAGE_BITS);
-    pd = p.phys_offset;
-    ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
+    section = phys_page_find(addr >> TARGET_PAGE_BITS);
+    if (!(memory_region_is_ram(section.mr)
+          || (section.mr->rom_device && section.mr->readable))) {
+        return;
+    }
+    ram_addr = (memory_region_get_ram_addr(section.mr)
+                + section.offset_within_region) & TARGET_PAGE_MASK;
+    ram_addr |= (pc & ~TARGET_PAGE_MASK);
     tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
 }
 #endif
@@ -2095,24 +2156,21 @@ static void tlb_add_large_page(CPUState *env, target_ulong vaddr,
     env->tlb_flush_mask = mask;
 }
 
-static bool is_ram_rom(ram_addr_t pd)
+static bool is_ram_rom(MemoryRegionSection *s)
 {
-    pd &= ~TARGET_PAGE_MASK;
-    return pd == io_mem_ram.ram_addr || pd == io_mem_rom.ram_addr;
+    return memory_region_is_ram(s->mr);
 }
 
-static bool is_romd(ram_addr_t pd)
+static bool is_romd(MemoryRegionSection *s)
 {
-    MemoryRegion *mr;
+    MemoryRegion *mr = s->mr;
 
-    pd &= ~TARGET_PAGE_MASK;
-    mr = io_mem_region[pd];
     return mr->rom_device && mr->readable;
 }
 
-static bool is_ram_rom_romd(ram_addr_t pd)
+static bool is_ram_rom_romd(MemoryRegionSection *s)
 {
-    return is_ram_rom(pd) || is_romd(pd);
+    return is_ram_rom(s) || is_romd(s);
 }
 
 /* Add a new TLB entry. At most one entry for a given virtual address
@@ -2122,8 +2180,7 @@ void tlb_set_page(CPUState *env, target_ulong vaddr,
                   target_phys_addr_t paddr, int prot,
                   int mmu_idx, target_ulong size)
 {
-    PhysPageDesc p;
-    unsigned long pd;
+    MemoryRegionSection section;
     unsigned int index;
     target_ulong address;
     target_ulong code_address;
@@ -2136,8 +2193,7 @@ void tlb_set_page(CPUState *env, target_ulong vaddr,
     if (size != TARGET_PAGE_SIZE) {
         tlb_add_large_page(env, vaddr, size);
     }
-    p = phys_page_find(paddr >> TARGET_PAGE_BITS);
-    pd = p.phys_offset;
+    section = phys_page_find(paddr >> TARGET_PAGE_BITS);
 #if defined(DEBUG_TLB)
     printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
            " prot=%x idx=%d pd=0x%08lx\n",
@@ -2145,15 +2201,21 @@ void tlb_set_page(CPUState *env, target_ulong vaddr,
 #endif
 
     address = vaddr;
-    if (!is_ram_rom_romd(pd)) {
+    if (!is_ram_rom_romd(&section)) {
         /* IO memory case (romd handled later) */
         address |= TLB_MMIO;
     }
-    addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
-    if (is_ram_rom(pd)) {
+    if (is_ram_rom_romd(&section)) {
+        addend = (unsigned long)(memory_region_get_ram_ptr(section.mr)
+                                 + section.offset_within_region);
+    } else {
+        addend = 0;
+    }
+    if (is_ram_rom(&section)) {
         /* Normal RAM.  */
-        iotlb = pd & TARGET_PAGE_MASK;
-        if ((pd & ~TARGET_PAGE_MASK) == io_mem_ram.ram_addr)
+        iotlb = (memory_region_get_ram_addr(section.mr)
+                 + section.offset_within_region) & TARGET_PAGE_MASK;
+        if (!section.readonly)
             iotlb |= io_mem_notdirty.ram_addr;
         else
             iotlb |= io_mem_rom.ram_addr;
@@ -2164,8 +2226,8 @@ void tlb_set_page(CPUState *env, target_ulong vaddr,
            and avoid full address decoding in every device.
            We can't use the high bits of pd for this because
            IO_MEM_ROMD uses these as a ram address.  */
-        iotlb = (pd & ~TARGET_PAGE_MASK);
-        iotlb += p.region_offset;
+        iotlb = memory_region_get_ram_addr(section.mr) & ~TARGET_PAGE_MASK;
+        iotlb += section.offset_within_region;
     }
 
     code_address = address;
@@ -2198,11 +2260,14 @@ void tlb_set_page(CPUState *env, target_ulong vaddr,
         te->addr_code = -1;
     }
     if (prot & PAGE_WRITE) {
-        if ((pd & ~TARGET_PAGE_MASK) == io_mem_rom.ram_addr || is_romd(pd)) {
+        if ((memory_region_is_ram(section.mr) && section.readonly)
+            || is_romd(&section)) {
             /* Write access calls the I/O callback.  */
             te->addr_write = address | TLB_MMIO;
-        } else if ((pd & ~TARGET_PAGE_MASK) == io_mem_ram.ram_addr &&
-                   !cpu_physical_memory_is_dirty(pd)) {
+        } else if (memory_region_is_ram(section.mr)
+                   && !cpu_physical_memory_is_dirty(
+                           section.mr->ram_addr
+                           + section.offset_within_region)) {
             te->addr_write = address | TLB_NOTDIRTY;
         } else {
             te->addr_write = address;
@@ -2491,34 +2556,66 @@ static inline void tlb_set_dirty(CPUState *env,
 typedef struct subpage_t {
     MemoryRegion iomem;
     target_phys_addr_t base;
-    ram_addr_t sub_io_index[TARGET_PAGE_SIZE];
-    ram_addr_t region_offset[TARGET_PAGE_SIZE];
+    uint16_t sub_section[TARGET_PAGE_SIZE];
 } subpage_t;
 
 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
-                             ram_addr_t memory, ram_addr_t region_offset);
-static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
-                                ram_addr_t orig_memory,
-                                ram_addr_t region_offset);
-#define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
-                      need_subpage)                                     \
-    do {                                                                \
-        if (addr > start_addr)                                          \
-            start_addr2 = 0;                                            \
-        else {                                                          \
-            start_addr2 = start_addr & ~TARGET_PAGE_MASK;               \
-            if (start_addr2 > 0)                                        \
-                need_subpage = 1;                                       \
-        }                                                               \
-                                                                        \
-        if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE)        \
-            end_addr2 = TARGET_PAGE_SIZE - 1;                           \
-        else {                                                          \
-            end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
-            if (end_addr2 < TARGET_PAGE_SIZE - 1)                       \
-                need_subpage = 1;                                       \
-        }                                                               \
-    } while (0)
+                             uint16_t section);
+static subpage_t *subpage_init(target_phys_addr_t base);
+static void destroy_page_desc(uint16_t section_index)
+{
+    MemoryRegionSection *section = &phys_sections[section_index];
+    MemoryRegion *mr = section->mr;
+
+    if (mr->subpage) {
+        subpage_t *subpage = container_of(mr, subpage_t, iomem);
+        memory_region_destroy(&subpage->iomem);
+        g_free(subpage);
+    }
+}
+
+static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level)
+{
+    unsigned i;
+    PhysPageEntry *p;
+
+    if (lp->ptr == PHYS_MAP_NODE_NIL) {
+        return;
+    }
+
+    p = phys_map_nodes[lp->ptr];
+    for (i = 0; i < L2_SIZE; ++i) {
+        if (!p[i].is_leaf) {
+            destroy_l2_mapping(&p[i], level - 1);
+        } else {
+            destroy_page_desc(p[i].ptr);
+        }
+    }
+    lp->is_leaf = 0;
+    lp->ptr = PHYS_MAP_NODE_NIL;
+}
+
+static void destroy_all_mappings(void)
+{
+    destroy_l2_mapping(&phys_map, P_L2_LEVELS - 1);
+    phys_map_nodes_reset();
+}
+
+static uint16_t phys_section_add(MemoryRegionSection *section)
+{
+    if (phys_sections_nb == phys_sections_nb_alloc) {
+        phys_sections_nb_alloc = MAX(phys_sections_nb_alloc * 2, 16);
+        phys_sections = g_renew(MemoryRegionSection, phys_sections,
+                                phys_sections_nb_alloc);
+    }
+    phys_sections[phys_sections_nb] = *section;
+    return phys_sections_nb++;
+}
+
+static void phys_sections_clear(void)
+{
+    phys_sections_nb = 0;
+}
 
 /* register physical memory.
    For RAM, 'size' must be a multiple of the target page size.
@@ -2528,101 +2625,78 @@ static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
    start_addr and region_offset are rounded down to a page boundary
    before calculating this offset.  This should not be a problem unless
    the low bits of start_addr and region_offset differ.  */
-void cpu_register_physical_memory_log(MemoryRegionSection *section,
-                                      bool readable, bool readonly)
+static void register_subpage(MemoryRegionSection *section)
 {
-    target_phys_addr_t start_addr = section->offset_within_address_space;
-    ram_addr_t size = section->size;
-    ram_addr_t phys_offset = section->mr->ram_addr;
-    ram_addr_t region_offset = section->offset_within_region;
-    target_phys_addr_t addr, end_addr;
-    PhysPageDesc *p;
-    CPUState *env;
-    ram_addr_t orig_size = size;
     subpage_t *subpage;
-
-    if (memory_region_is_ram(section->mr)) {
-        phys_offset += region_offset;
-        region_offset = 0;
+    target_phys_addr_t base = section->offset_within_address_space
+        & TARGET_PAGE_MASK;
+    MemoryRegionSection existing = phys_page_find(base >> TARGET_PAGE_BITS);
+    MemoryRegionSection subsection = {
+        .offset_within_address_space = base,
+        .size = TARGET_PAGE_SIZE,
+    };
+    target_phys_addr_t start, end;
+
+    assert(existing.mr->subpage || existing.mr == &io_mem_unassigned);
+
+    if (!(existing.mr->subpage)) {
+        subpage = subpage_init(base);
+        subsection.mr = &subpage->iomem;
+        phys_page_set(base >> TARGET_PAGE_BITS, 1,
+                      phys_section_add(&subsection));
+    } else {
+        subpage = container_of(existing.mr, subpage_t, iomem);
     }
+    start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
+    end = start + section->size;
+    subpage_register(subpage, start, end, phys_section_add(section));
+}
 
-    if (readonly) {
-        phys_offset |= io_mem_rom.ram_addr;
-    }
 
-    assert(size);
+static void register_multipage(MemoryRegionSection *section)
+{
+    target_phys_addr_t start_addr = section->offset_within_address_space;
+    ram_addr_t size = section->size;
+    target_phys_addr_t addr;
+    uint16_t section_index = phys_section_add(section);
 
-    if (phys_offset == io_mem_unassigned.ram_addr) {
-        region_offset = start_addr;
-    }
-    region_offset &= TARGET_PAGE_MASK;
-    size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
-    end_addr = start_addr + (target_phys_addr_t)size;
+    assert(size);
 
     addr = start_addr;
-    do {
-        p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 0);
-        if (p && p->phys_offset != io_mem_unassigned.ram_addr) {
-            ram_addr_t orig_memory = p->phys_offset;
-            target_phys_addr_t start_addr2, end_addr2;
-            int need_subpage = 0;
-            MemoryRegion *mr = io_mem_region[orig_memory & ~TARGET_PAGE_MASK];
-
-            CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
-                          need_subpage);
-            if (need_subpage) {
-                if (!(mr->subpage)) {
-                    subpage = subpage_init((addr & TARGET_PAGE_MASK),
-                                           &p->phys_offset, orig_memory,
-                                           p->region_offset);
-                } else {
-                    subpage = container_of(mr, subpage_t, iomem);
-                }
-                subpage_register(subpage, start_addr2, end_addr2, phys_offset,
-                                 region_offset);
-                p->region_offset = 0;
-            } else {
-                p->phys_offset = phys_offset;
-                p->region_offset = region_offset;
-                if (is_ram_rom_romd(phys_offset))
-                    phys_offset += TARGET_PAGE_SIZE;
-            }
-        } else {
-            p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
-            p->phys_offset = phys_offset;
-            p->region_offset = region_offset;
-            if (is_ram_rom_romd(phys_offset)) {
-                phys_offset += TARGET_PAGE_SIZE;
-            } else {
-                target_phys_addr_t start_addr2, end_addr2;
-                int need_subpage = 0;
-
-                CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
-                              end_addr2, need_subpage);
-
-                if (need_subpage) {
-                    subpage = subpage_init((addr & TARGET_PAGE_MASK),
-                                           &p->phys_offset,
-                                           io_mem_unassigned.ram_addr,
-                                           addr & TARGET_PAGE_MASK);
-                    subpage_register(subpage, start_addr2, end_addr2,
-                                     phys_offset, region_offset);
-                    p->region_offset = 0;
-                }
-            }
-        }
-        region_offset += TARGET_PAGE_SIZE;
-        addr += TARGET_PAGE_SIZE;
-    } while (addr != end_addr);
+    phys_page_set(addr >> TARGET_PAGE_BITS, size >> TARGET_PAGE_BITS,
+                  section_index);
+}
 
-    /* since each CPU stores ram addresses in its TLB cache, we must
-       reset the modified entries */
-    /* XXX: slow ! */
-    for(env = first_cpu; env != NULL; env = env->next_cpu) {
-        tlb_flush(env, 1);
+void cpu_register_physical_memory_log(MemoryRegionSection *section,
+                                      bool readonly)
+{
+    MemoryRegionSection now = *section, remain = *section;
+
+    if ((now.offset_within_address_space & ~TARGET_PAGE_MASK)
+        || (now.size < TARGET_PAGE_SIZE)) {
+        now.size = MIN(TARGET_PAGE_ALIGN(now.offset_within_address_space)
+                       - now.offset_within_address_space,
+                       now.size);
+        register_subpage(&now);
+        remain.size -= now.size;
+        remain.offset_within_address_space += now.size;
+        remain.offset_within_region += now.size;
+    }
+    now = remain;
+    now.size &= TARGET_PAGE_MASK;
+    if (now.size) {
+        register_multipage(&now);
+        remain.size -= now.size;
+        remain.offset_within_address_space += now.size;
+        remain.offset_within_region += now.size;
+    }
+    now = remain;
+    if (now.size) {
+        register_subpage(&now);
     }
 }
 
+
 void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
 {
     if (kvm_enabled())
@@ -3307,14 +3381,17 @@ static uint64_t subpage_read(void *opaque, target_phys_addr_t addr,
 {
     subpage_t *mmio = opaque;
     unsigned int idx = SUBPAGE_IDX(addr);
+    MemoryRegionSection *section;
 #if defined(DEBUG_SUBPAGE)
     printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
            mmio, len, addr, idx);
 #endif
 
-    addr += mmio->region_offset[idx];
-    idx = mmio->sub_io_index[idx];
-    return io_mem_read(idx, addr, len);
+    section = &phys_sections[mmio->sub_section[idx]];
+    addr += mmio->base;
+    addr -= section->offset_within_address_space;
+    addr += section->offset_within_region;
+    return io_mem_read(section->mr->ram_addr, addr, len);
 }
 
 static void subpage_write(void *opaque, target_phys_addr_t addr,
@@ -3322,15 +3399,18 @@ static void subpage_write(void *opaque, target_phys_addr_t addr,
 {
     subpage_t *mmio = opaque;
     unsigned int idx = SUBPAGE_IDX(addr);
+    MemoryRegionSection *section;
 #if defined(DEBUG_SUBPAGE)
     printf("%s: subpage %p len %d addr " TARGET_FMT_plx
            " idx %d value %"PRIx64"\n",
            __func__, mmio, len, addr, idx, value);
 #endif
 
-    addr += mmio->region_offset[idx];
-    idx = mmio->sub_io_index[idx];
-    io_mem_write(idx, addr, value, len);
+    section = &phys_sections[mmio->sub_section[idx]];
+    addr += mmio->base;
+    addr -= section->offset_within_address_space;
+    addr += section->offset_within_region;
+    io_mem_write(section->mr->ram_addr, addr, value, len);
 }
 
 static const MemoryRegionOps subpage_ops = {
@@ -3372,7 +3452,7 @@ static const MemoryRegionOps subpage_ram_ops = {
 };
 
 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
-                             ram_addr_t memory, ram_addr_t region_offset)
+                             uint16_t section)
 {
     int idx, eidx;
 
@@ -3384,24 +3464,21 @@ static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
     printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
            mmio, start, end, idx, eidx, memory);
 #endif
-    if ((memory & ~TARGET_PAGE_MASK) == io_mem_ram.ram_addr) {
-        memory = io_mem_subpage_ram.ram_addr;
+    if (memory_region_is_ram(phys_sections[section].mr)) {
+        MemoryRegionSection new_section = phys_sections[section];
+        new_section.mr = &io_mem_subpage_ram;
+        section = phys_section_add(&new_section);
     }
-    memory &= IO_MEM_NB_ENTRIES - 1;
     for (; idx <= eidx; idx++) {
-        mmio->sub_io_index[idx] = memory;
-        mmio->region_offset[idx] = region_offset;
+        mmio->sub_section[idx] = section;
     }
 
     return 0;
 }
 
-static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
-                                ram_addr_t orig_memory,
-                                ram_addr_t region_offset)
+static subpage_t *subpage_init(target_phys_addr_t base)
 {
     subpage_t *mmio;
-    int subpage_memory;
 
     mmio = g_malloc0(sizeof(subpage_t));
 
@@ -3409,13 +3486,11 @@ static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
     memory_region_init_io(&mmio->iomem, &subpage_ops, mmio,
                           "subpage", TARGET_PAGE_SIZE);
     mmio->iomem.subpage = true;
-    subpage_memory = mmio->iomem.ram_addr;
 #if defined(DEBUG_SUBPAGE)
     printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
            mmio, base, TARGET_PAGE_SIZE, subpage_memory);
 #endif
-    *phys = subpage_memory;
-    subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, orig_memory, region_offset);
+    subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, phys_section_unassigned);
 
     return mmio;
 }
@@ -3467,6 +3542,18 @@ void cpu_unregister_io_memory(int io_index)
     io_mem_used[io_index] = 0;
 }
 
+static uint16_t dummy_section(MemoryRegion *mr)
+{
+    MemoryRegionSection section = {
+        .mr = mr,
+        .offset_within_address_space = 0,
+        .offset_within_region = 0,
+        .size = UINT64_MAX,
+    };
+
+    return phys_section_add(&section);
+}
+
 static void io_mem_init(void)
 {
     int i;
@@ -3488,6 +3575,174 @@ static void io_mem_init(void)
                           "watch", UINT64_MAX);
 }
 
+static void core_begin(MemoryListener *listener)
+{
+    destroy_all_mappings();
+    phys_sections_clear();
+    phys_map.ptr = PHYS_MAP_NODE_NIL;
+    phys_section_unassigned = dummy_section(&io_mem_unassigned);
+}
+
+static void core_commit(MemoryListener *listener)
+{
+    CPUState *env;
+
+    /* since each CPU stores ram addresses in its TLB cache, we must
+       reset the modified entries */
+    /* XXX: slow ! */
+    for(env = first_cpu; env != NULL; env = env->next_cpu) {
+        tlb_flush(env, 1);
+    }
+}
+
+static void core_region_add(MemoryListener *listener,
+                            MemoryRegionSection *section)
+{
+    cpu_register_physical_memory_log(section, section->readonly);
+}
+
+static void core_region_del(MemoryListener *listener,
+                            MemoryRegionSection *section)
+{
+}
+
+static void core_region_nop(MemoryListener *listener,
+                            MemoryRegionSection *section)
+{
+    cpu_register_physical_memory_log(section, section->readonly);
+}
+
+static void core_log_start(MemoryListener *listener,
+                           MemoryRegionSection *section)
+{
+}
+
+static void core_log_stop(MemoryListener *listener,
+                          MemoryRegionSection *section)
+{
+}
+
+static void core_log_sync(MemoryListener *listener,
+                          MemoryRegionSection *section)
+{
+}
+
+static void core_log_global_start(MemoryListener *listener)
+{
+    cpu_physical_memory_set_dirty_tracking(1);
+}
+
+static void core_log_global_stop(MemoryListener *listener)
+{
+    cpu_physical_memory_set_dirty_tracking(0);
+}
+
+static void core_eventfd_add(MemoryListener *listener,
+                             MemoryRegionSection *section,
+                             bool match_data, uint64_t data, int fd)
+{
+}
+
+static void core_eventfd_del(MemoryListener *listener,
+                             MemoryRegionSection *section,
+                             bool match_data, uint64_t data, int fd)
+{
+}
+
+static void io_begin(MemoryListener *listener)
+{
+}
+
+static void io_commit(MemoryListener *listener)
+{
+}
+
+static void io_region_add(MemoryListener *listener,
+                          MemoryRegionSection *section)
+{
+    iorange_init(&section->mr->iorange, &memory_region_iorange_ops,
+                 section->offset_within_address_space, section->size);
+    ioport_register(&section->mr->iorange);
+}
+
+static void io_region_del(MemoryListener *listener,
+                          MemoryRegionSection *section)
+{
+    isa_unassign_ioport(section->offset_within_address_space, section->size);
+}
+
+static void io_region_nop(MemoryListener *listener,
+                          MemoryRegionSection *section)
+{
+}
+
+static void io_log_start(MemoryListener *listener,
+                         MemoryRegionSection *section)
+{
+}
+
+static void io_log_stop(MemoryListener *listener,
+                        MemoryRegionSection *section)
+{
+}
+
+static void io_log_sync(MemoryListener *listener,
+                        MemoryRegionSection *section)
+{
+}
+
+static void io_log_global_start(MemoryListener *listener)
+{
+}
+
+static void io_log_global_stop(MemoryListener *listener)
+{
+}
+
+static void io_eventfd_add(MemoryListener *listener,
+                           MemoryRegionSection *section,
+                           bool match_data, uint64_t data, int fd)
+{
+}
+
+static void io_eventfd_del(MemoryListener *listener,
+                           MemoryRegionSection *section,
+                           bool match_data, uint64_t data, int fd)
+{
+}
+
+static MemoryListener core_memory_listener = {
+    .begin = core_begin,
+    .commit = core_commit,
+    .region_add = core_region_add,
+    .region_del = core_region_del,
+    .region_nop = core_region_nop,
+    .log_start = core_log_start,
+    .log_stop = core_log_stop,
+    .log_sync = core_log_sync,
+    .log_global_start = core_log_global_start,
+    .log_global_stop = core_log_global_stop,
+    .eventfd_add = core_eventfd_add,
+    .eventfd_del = core_eventfd_del,
+    .priority = 0,
+};
+
+static MemoryListener io_memory_listener = {
+    .begin = io_begin,
+    .commit = io_commit,
+    .region_add = io_region_add,
+    .region_del = io_region_del,
+    .region_nop = io_region_nop,
+    .log_start = io_log_start,
+    .log_stop = io_log_stop,
+    .log_sync = io_log_sync,
+    .log_global_start = io_log_global_start,
+    .log_global_stop = io_log_global_stop,
+    .eventfd_add = io_eventfd_add,
+    .eventfd_del = io_eventfd_del,
+    .priority = 0,
+};
+
 static void memory_map_init(void)
 {
     system_memory = g_malloc(sizeof(*system_memory));
@@ -3497,6 +3752,9 @@ static void memory_map_init(void)
     system_io = g_malloc(sizeof(*system_io));
     memory_region_init(system_io, "io", 65536);
     set_system_io_map(system_io);
+
+    memory_listener_register(&core_memory_listener, system_memory);
+    memory_listener_register(&io_memory_listener, system_io);
 }
 
 MemoryRegion *get_system_memory(void)
@@ -3560,22 +3818,22 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
     uint8_t *ptr;
     uint32_t val;
     target_phys_addr_t page;
-    ram_addr_t pd;
-    PhysPageDesc p;
+    MemoryRegionSection section;
 
     while (len > 0) {
         page = addr & TARGET_PAGE_MASK;
         l = (page + TARGET_PAGE_SIZE) - addr;
         if (l > len)
             l = len;
-        p = phys_page_find(page >> TARGET_PAGE_BITS);
-        pd = p.phys_offset;
+        section = phys_page_find(page >> TARGET_PAGE_BITS);
 
         if (is_write) {
-            if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) {
+            if (!memory_region_is_ram(section.mr)) {
                 target_phys_addr_t addr1;
-                io_index = pd & (IO_MEM_NB_ENTRIES - 1);
-                addr1 = (addr & ~TARGET_PAGE_MASK) + p.region_offset;
+                io_index = memory_region_get_ram_addr(section.mr)
+                    & (IO_MEM_NB_ENTRIES - 1);
+                addr1 = (addr & ~TARGET_PAGE_MASK)
+                    + section.offset_within_region;
                 /* XXX: could force cpu_single_env to NULL to avoid
                    potential bugs */
                 if (l >= 4 && ((addr1 & 3) == 0)) {
@@ -3594,9 +3852,11 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
                     io_mem_write(io_index, addr1, val, 1);
                     l = 1;
                 }
-            } else {
+            } else if (!section.readonly) {
                 ram_addr_t addr1;
-                addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
+                addr1 = (memory_region_get_ram_addr(section.mr)
+                         + section.offset_within_region)
+                    | (addr & ~TARGET_PAGE_MASK);
                 /* RAM case */
                 ptr = qemu_get_ram_ptr(addr1);
                 memcpy(ptr, buf, l);
@@ -3610,11 +3870,13 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
                 qemu_put_ram_ptr(ptr);
             }
         } else {
-            if (!is_ram_rom_romd(pd)) {
+            if (!is_ram_rom_romd(&section)) {
                 target_phys_addr_t addr1;
                 /* I/O case */
-                io_index = pd & (IO_MEM_NB_ENTRIES - 1);
-                addr1 = (addr & ~TARGET_PAGE_MASK) + p.region_offset;
+                io_index = memory_region_get_ram_addr(section.mr)
+                    & (IO_MEM_NB_ENTRIES - 1);
+                addr1 = (addr & ~TARGET_PAGE_MASK)
+                    + section.offset_within_region;
                 if (l >= 4 && ((addr1 & 3) == 0)) {
                     /* 32 bit read access */
                     val = io_mem_read(io_index, addr1, 4);
@@ -3633,7 +3895,8 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
                 }
             } else {
                 /* RAM case */
-                ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
+                ptr = qemu_get_ram_ptr(section.mr->ram_addr
+                                       + section.offset_within_region);
                 memcpy(buf, ptr + (addr & ~TARGET_PAGE_MASK), l);
                 qemu_put_ram_ptr(ptr);
             }
@@ -3651,22 +3914,22 @@ void cpu_physical_memory_write_rom(target_phys_addr_t addr,
     int l;
     uint8_t *ptr;
     target_phys_addr_t page;
-    unsigned long pd;
-    PhysPageDesc p;
+    MemoryRegionSection section;
 
     while (len > 0) {
         page = addr & TARGET_PAGE_MASK;
         l = (page + TARGET_PAGE_SIZE) - addr;
         if (l > len)
             l = len;
-        p = phys_page_find(page >> TARGET_PAGE_BITS);
-        pd = p.phys_offset;
+        section = phys_page_find(page >> TARGET_PAGE_BITS);
 
-        if (!is_ram_rom_romd(pd)) {
+        if (!is_ram_rom_romd(&section)) {
             /* do nothing */
         } else {
             unsigned long addr1;
-            addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
+            addr1 = (memory_region_get_ram_addr(section.mr)
+                     + section.offset_within_region)
+                + (addr & ~TARGET_PAGE_MASK);
             /* ROM/RAM case */
             ptr = qemu_get_ram_ptr(addr1);
             memcpy(ptr, buf, l);
@@ -3739,8 +4002,7 @@ void *cpu_physical_memory_map(target_phys_addr_t addr,
     target_phys_addr_t todo = 0;
     int l;
     target_phys_addr_t page;
-    unsigned long pd;
-    PhysPageDesc p;
+    MemoryRegionSection section;
     ram_addr_t raddr = RAM_ADDR_MAX;
     ram_addr_t rlen;
     void *ret;
@@ -3750,10 +4012,9 @@ void *cpu_physical_memory_map(target_phys_addr_t addr,
         l = (page + TARGET_PAGE_SIZE) - addr;
         if (l > len)
             l = len;
-        p = phys_page_find(page >> TARGET_PAGE_BITS);
-        pd = p.phys_offset;
+        section = phys_page_find(page >> TARGET_PAGE_BITS);
 
-        if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) {
+        if (!(memory_region_is_ram(section.mr) && !section.readonly)) {
             if (todo || bounce.buffer) {
                 break;
             }
@@ -3768,7 +4029,9 @@ void *cpu_physical_memory_map(target_phys_addr_t addr,
             return bounce.buffer;
         }
         if (!todo) {
-            raddr = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
+            raddr = memory_region_get_ram_addr(section.mr)
+                + section.offset_within_region
+                + (addr & ~TARGET_PAGE_MASK);
         }
 
         len -= l;
@@ -3827,16 +4090,15 @@ static inline uint32_t ldl_phys_internal(target_phys_addr_t addr,
     int io_index;
     uint8_t *ptr;
     uint32_t val;
-    unsigned long pd;
-    PhysPageDesc p;
+    MemoryRegionSection section;
 
-    p = phys_page_find(addr >> TARGET_PAGE_BITS);
-    pd = p.phys_offset;
+    section = phys_page_find(addr >> TARGET_PAGE_BITS);
 
-    if (!is_ram_rom_romd(pd)) {
+    if (!is_ram_rom_romd(&section)) {
         /* I/O case */
-        io_index = pd & (IO_MEM_NB_ENTRIES - 1);
-        addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset;
+        io_index = memory_region_get_ram_addr(section.mr)
+            & (IO_MEM_NB_ENTRIES - 1);
+        addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region;
         val = io_mem_read(io_index, addr, 4);
 #if defined(TARGET_WORDS_BIGENDIAN)
         if (endian == DEVICE_LITTLE_ENDIAN) {
@@ -3849,7 +4111,9 @@ static inline uint32_t ldl_phys_internal(target_phys_addr_t addr,
 #endif
     } else {
         /* RAM case */
-        ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
+        ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section.mr)
+                                & TARGET_PAGE_MASK)
+                               + section.offset_within_region) +
             (addr & ~TARGET_PAGE_MASK);
         switch (endian) {
         case DEVICE_LITTLE_ENDIAN:
@@ -3888,16 +4152,15 @@ static inline uint64_t ldq_phys_internal(target_phys_addr_t addr,
     int io_index;
     uint8_t *ptr;
     uint64_t val;
-    unsigned long pd;
-    PhysPageDesc p;
+    MemoryRegionSection section;
 
-    p = phys_page_find(addr >> TARGET_PAGE_BITS);
-    pd = p.phys_offset;
+    section = phys_page_find(addr >> TARGET_PAGE_BITS);
 
-    if (!is_ram_rom_romd(pd)) {
+    if (!is_ram_rom_romd(&section)) {
         /* I/O case */
-        io_index = pd & (IO_MEM_NB_ENTRIES - 1);
-        addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset;
+        io_index = memory_region_get_ram_addr(section.mr)
+            & (IO_MEM_NB_ENTRIES - 1);
+        addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region;
 
         /* XXX This is broken when device endian != cpu endian.
                Fix and add "endian" variable check */
@@ -3910,8 +4173,10 @@ static inline uint64_t ldq_phys_internal(target_phys_addr_t addr,
 #endif
     } else {
         /* RAM case */
-        ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
-            (addr & ~TARGET_PAGE_MASK);
+        ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section.mr)
+                                & TARGET_PAGE_MASK)
+                               + section.offset_within_region)
+            + (addr & ~TARGET_PAGE_MASK);
         switch (endian) {
         case DEVICE_LITTLE_ENDIAN:
             val = ldq_le_p(ptr);
@@ -3957,16 +4222,15 @@ static inline uint32_t lduw_phys_internal(target_phys_addr_t addr,
     int io_index;
     uint8_t *ptr;
     uint64_t val;
-    unsigned long pd;
-    PhysPageDesc p;
+    MemoryRegionSection section;
 
-    p = phys_page_find(addr >> TARGET_PAGE_BITS);
-    pd = p.phys_offset;
+    section = phys_page_find(addr >> TARGET_PAGE_BITS);
 
-    if (!is_ram_rom_romd(pd)) {
+    if (!is_ram_rom_romd(&section)) {
         /* I/O case */
-        io_index = pd & (IO_MEM_NB_ENTRIES - 1);
-        addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset;
+        io_index = memory_region_get_ram_addr(section.mr)
+            & (IO_MEM_NB_ENTRIES - 1);
+        addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region;
         val = io_mem_read(io_index, addr, 2);
 #if defined(TARGET_WORDS_BIGENDIAN)
         if (endian == DEVICE_LITTLE_ENDIAN) {
@@ -3979,8 +4243,10 @@ static inline uint32_t lduw_phys_internal(target_phys_addr_t addr,
 #endif
     } else {
         /* RAM case */
-        ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
-            (addr & ~TARGET_PAGE_MASK);
+        ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section.mr)
+                                & TARGET_PAGE_MASK)
+                               + section.offset_within_region)
+            + (addr & ~TARGET_PAGE_MASK);
         switch (endian) {
         case DEVICE_LITTLE_ENDIAN:
             val = lduw_le_p(ptr);
@@ -4018,18 +4284,23 @@ void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
 {
     int io_index;
     uint8_t *ptr;
-    unsigned long pd;
-    PhysPageDesc p;
+    MemoryRegionSection section;
 
-    p = phys_page_find(addr >> TARGET_PAGE_BITS);
-    pd = p.phys_offset;
+    section = phys_page_find(addr >> TARGET_PAGE_BITS);
 
-    if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) {
-        io_index = pd & (IO_MEM_NB_ENTRIES - 1);
-        addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset;
+    if (!memory_region_is_ram(section.mr) || section.readonly) {
+        if (memory_region_is_ram(section.mr)) {
+            io_index = io_mem_rom.ram_addr;
+        } else {
+            io_index = memory_region_get_ram_addr(section.mr);
+        }
+        addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region;
         io_mem_write(io_index, addr, val, 4);
     } else {
-        unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
+        unsigned long addr1 = (memory_region_get_ram_addr(section.mr)
+                               & TARGET_PAGE_MASK)
+            + section.offset_within_region
+            + (addr & ~TARGET_PAGE_MASK);
         ptr = qemu_get_ram_ptr(addr1);
         stl_p(ptr, val);
 
@@ -4049,15 +4320,18 @@ void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
 {
     int io_index;
     uint8_t *ptr;
-    unsigned long pd;
-    PhysPageDesc p;
+    MemoryRegionSection section;
 
-    p = phys_page_find(addr >> TARGET_PAGE_BITS);
-    pd = p.phys_offset;
+    section = phys_page_find(addr >> TARGET_PAGE_BITS);
 
-    if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) {
-        io_index = pd & (IO_MEM_NB_ENTRIES - 1);
-        addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset;
+    if (!memory_region_is_ram(section.mr) || section.readonly) {
+        if (memory_region_is_ram(section.mr)) {
+                io_index = io_mem_rom.ram_addr;
+        } else {
+            io_index = memory_region_get_ram_addr(section.mr)
+                & (IO_MEM_NB_ENTRIES - 1);
+        }
+        addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region;
 #ifdef TARGET_WORDS_BIGENDIAN
         io_mem_write(io_index, addr, val >> 32, 4);
         io_mem_write(io_index, addr + 4, (uint32_t)val, 4);
@@ -4066,8 +4340,10 @@ void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
         io_mem_write(io_index, addr + 4, val >> 32, 4);
 #endif
     } else {
-        ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
-            (addr & ~TARGET_PAGE_MASK);
+        ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section.mr)
+                                & TARGET_PAGE_MASK)
+                               + section.offset_within_region)
+            + (addr & ~TARGET_PAGE_MASK);
         stq_p(ptr, val);
     }
 }
@@ -4078,15 +4354,18 @@ static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val,
 {
     int io_index;
     uint8_t *ptr;
-    unsigned long pd;
-    PhysPageDesc p;
+    MemoryRegionSection section;
 
-    p = phys_page_find(addr >> TARGET_PAGE_BITS);
-    pd = p.phys_offset;
+    section = phys_page_find(addr >> TARGET_PAGE_BITS);
 
-    if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) {
-        io_index = pd & (IO_MEM_NB_ENTRIES - 1);
-        addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset;
+    if (!memory_region_is_ram(section.mr) || section.readonly) {
+        if (memory_region_is_ram(section.mr)) {
+            io_index = io_mem_rom.ram_addr;
+        } else {
+            io_index = memory_region_get_ram_addr(section.mr)
+                & (IO_MEM_NB_ENTRIES - 1);
+        }
+        addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region;
 #if defined(TARGET_WORDS_BIGENDIAN)
         if (endian == DEVICE_LITTLE_ENDIAN) {
             val = bswap32(val);
@@ -4099,7 +4378,9 @@ static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val,
         io_mem_write(io_index, addr, val, 4);
     } else {
         unsigned long addr1;
-        addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
+        addr1 = (memory_region_get_ram_addr(section.mr) & TARGET_PAGE_MASK)
+            + section.offset_within_region
+            + (addr & ~TARGET_PAGE_MASK);
         /* RAM case */
         ptr = qemu_get_ram_ptr(addr1);
         switch (endian) {
@@ -4151,15 +4432,18 @@ static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val,
 {
     int io_index;
     uint8_t *ptr;
-    unsigned long pd;
-    PhysPageDesc p;
+    MemoryRegionSection section;
 
-    p = phys_page_find(addr >> TARGET_PAGE_BITS);
-    pd = p.phys_offset;
+    section = phys_page_find(addr >> TARGET_PAGE_BITS);
 
-    if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) {
-        io_index = pd & (IO_MEM_NB_ENTRIES - 1);
-        addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset;
+    if (!memory_region_is_ram(section.mr) || section.readonly) {
+        if (memory_region_is_ram(section.mr)) {
+            io_index = io_mem_rom.ram_addr;
+        } else {
+            io_index = memory_region_get_ram_addr(section.mr)
+                & (IO_MEM_NB_ENTRIES - 1);
+        }
+        addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region;
 #if defined(TARGET_WORDS_BIGENDIAN)
         if (endian == DEVICE_LITTLE_ENDIAN) {
             val = bswap16(val);
@@ -4172,7 +4456,8 @@ static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val,
         io_mem_write(io_index, addr, val, 2);
     } else {
         unsigned long addr1;
-        addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
+        addr1 = (memory_region_get_ram_addr(section.mr) & TARGET_PAGE_MASK)
+            + section.offset_within_region + (addr & ~TARGET_PAGE_MASK);
         /* RAM case */
         ptr = qemu_get_ram_ptr(addr1);
         switch (endian) {
@@ -4389,7 +4674,7 @@ tb_page_addr_t get_page_addr_code(CPUState *env1, target_ulong addr)
     }
     pd = env1->tlb_table[mmu_idx][page_index].addr_code & ~TARGET_PAGE_MASK;
     if (pd != io_mem_ram.ram_addr && pd != io_mem_rom.ram_addr
-        && !is_romd(pd)) {
+        && !io_mem_region[pd]->rom_device) {
 #if defined(TARGET_ALPHA) || defined(TARGET_MIPS) || defined(TARGET_SPARC)
         cpu_unassigned_access(env1, addr, 0, 1, 0, 4);
 #else
diff --git a/hw/apic.c b/hw/apic.c
index ff9d24e914..4eeaf8801c 100644
--- a/hw/apic.c
+++ b/hw/apic.c
@@ -35,6 +35,10 @@
 #define MSI_ADDR_DEST_ID_SHIFT		12
 #define	MSI_ADDR_DEST_ID_MASK		0x00ffff0
 
+#define SYNC_FROM_VAPIC                 0x1
+#define SYNC_TO_VAPIC                   0x2
+#define SYNC_ISR_IRR_TO_VAPIC           0x4
+
 static APICCommonState *local_apics[MAX_APICS + 1];
 
 static void apic_set_irq(APICCommonState *s, int vector_num, int trigger_mode);
@@ -78,6 +82,70 @@ static inline int get_bit(uint32_t *tab, int index)
     return !!(tab[i] & mask);
 }
 
+/* return -1 if no bit is set */
+static int get_highest_priority_int(uint32_t *tab)
+{
+    int i;
+    for (i = 7; i >= 0; i--) {
+        if (tab[i] != 0) {
+            return i * 32 + fls_bit(tab[i]);
+        }
+    }
+    return -1;
+}
+
+static void apic_sync_vapic(APICCommonState *s, int sync_type)
+{
+    VAPICState vapic_state;
+    size_t length;
+    off_t start;
+    int vector;
+
+    if (!s->vapic_paddr) {
+        return;
+    }
+    if (sync_type & SYNC_FROM_VAPIC) {
+        cpu_physical_memory_rw(s->vapic_paddr, (void *)&vapic_state,
+                               sizeof(vapic_state), 0);
+        s->tpr = vapic_state.tpr;
+    }
+    if (sync_type & (SYNC_TO_VAPIC | SYNC_ISR_IRR_TO_VAPIC)) {
+        start = offsetof(VAPICState, isr);
+        length = offsetof(VAPICState, enabled) - offsetof(VAPICState, isr);
+
+        if (sync_type & SYNC_TO_VAPIC) {
+            assert(qemu_cpu_is_self(s->cpu_env));
+
+            vapic_state.tpr = s->tpr;
+            vapic_state.enabled = 1;
+            start = 0;
+            length = sizeof(VAPICState);
+        }
+
+        vector = get_highest_priority_int(s->isr);
+        if (vector < 0) {
+            vector = 0;
+        }
+        vapic_state.isr = vector & 0xf0;
+
+        vapic_state.zero = 0;
+
+        vector = get_highest_priority_int(s->irr);
+        if (vector < 0) {
+            vector = 0;
+        }
+        vapic_state.irr = vector & 0xff;
+
+        cpu_physical_memory_write_rom(s->vapic_paddr + start,
+                                      ((void *)&vapic_state) + start, length);
+    }
+}
+
+static void apic_vapic_base_update(APICCommonState *s)
+{
+    apic_sync_vapic(s, SYNC_TO_VAPIC);
+}
+
 static void apic_local_deliver(APICCommonState *s, int vector)
 {
     uint32_t lvt = s->lvt[vector];
@@ -239,20 +307,17 @@ static void apic_set_base(APICCommonState *s, uint64_t val)
 
 static void apic_set_tpr(APICCommonState *s, uint8_t val)
 {
-    s->tpr = (val & 0x0f) << 4;
-    apic_update_irq(s);
+    /* Updates from cr8 are ignored while the VAPIC is active */
+    if (!s->vapic_paddr) {
+        s->tpr = val << 4;
+        apic_update_irq(s);
+    }
 }
 
-/* return -1 if no bit is set */
-static int get_highest_priority_int(uint32_t *tab)
+static uint8_t apic_get_tpr(APICCommonState *s)
 {
-    int i;
-    for(i = 7; i >= 0; i--) {
-        if (tab[i] != 0) {
-            return i * 32 + fls_bit(tab[i]);
-        }
-    }
-    return -1;
+    apic_sync_vapic(s, SYNC_FROM_VAPIC);
+    return s->tpr >> 4;
 }
 
 static int apic_get_ppr(APICCommonState *s)
@@ -312,6 +377,14 @@ static void apic_update_irq(APICCommonState *s)
     }
 }
 
+void apic_poll_irq(DeviceState *d)
+{
+    APICCommonState *s = APIC_COMMON(d);
+
+    apic_sync_vapic(s, SYNC_FROM_VAPIC);
+    apic_update_irq(s);
+}
+
 static void apic_set_irq(APICCommonState *s, int vector_num, int trigger_mode)
 {
     apic_report_irq_delivered(!get_bit(s->irr, vector_num));
@@ -321,6 +394,16 @@ static void apic_set_irq(APICCommonState *s, int vector_num, int trigger_mode)
         set_bit(s->tmr, vector_num);
     else
         reset_bit(s->tmr, vector_num);
+    if (s->vapic_paddr) {
+        apic_sync_vapic(s, SYNC_ISR_IRR_TO_VAPIC);
+        /*
+         * The vcpu thread needs to see the new IRR before we pull its current
+         * TPR value. That way, if we miss a lowering of the TRP, the guest
+         * has the chance to notice the new IRR and poll for IRQs on its own.
+         */
+        smp_wmb();
+        apic_sync_vapic(s, SYNC_FROM_VAPIC);
+    }
     apic_update_irq(s);
 }
 
@@ -334,6 +417,7 @@ static void apic_eoi(APICCommonState *s)
     if (!(s->spurious_vec & APIC_SV_DIRECTED_IO) && get_bit(s->tmr, isrv)) {
         ioapic_eoi_broadcast(isrv);
     }
+    apic_sync_vapic(s, SYNC_FROM_VAPIC | SYNC_TO_VAPIC);
     apic_update_irq(s);
 }
 
@@ -471,15 +555,19 @@ int apic_get_interrupt(DeviceState *d)
     if (!(s->spurious_vec & APIC_SV_ENABLE))
         return -1;
 
+    apic_sync_vapic(s, SYNC_FROM_VAPIC);
     intno = apic_irq_pending(s);
 
     if (intno == 0) {
+        apic_sync_vapic(s, SYNC_TO_VAPIC);
         return -1;
     } else if (intno < 0) {
+        apic_sync_vapic(s, SYNC_TO_VAPIC);
         return s->spurious_vec & 0xff;
     }
     reset_bit(s->irr, intno);
     set_bit(s->isr, intno);
+    apic_sync_vapic(s, SYNC_TO_VAPIC);
     apic_update_irq(s);
     return intno;
 }
@@ -576,6 +664,10 @@ static uint32_t apic_mem_readl(void *opaque, target_phys_addr_t addr)
         val = 0x11 | ((APIC_LVT_NB - 1) << 16); /* version 0x11 */
         break;
     case 0x08:
+        apic_sync_vapic(s, SYNC_FROM_VAPIC);
+        if (apic_report_tpr_access) {
+            cpu_report_tpr_access(s->cpu_env, TPR_ACCESS_READ);
+        }
         val = s->tpr;
         break;
     case 0x09:
@@ -675,7 +767,11 @@ static void apic_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
     case 0x03:
         break;
     case 0x08:
+        if (apic_report_tpr_access) {
+            cpu_report_tpr_access(s->cpu_env, TPR_ACCESS_WRITE);
+        }
         s->tpr = val;
+        apic_sync_vapic(s, SYNC_TO_VAPIC);
         apic_update_irq(s);
         break;
     case 0x09:
@@ -737,6 +833,11 @@ static void apic_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
     }
 }
 
+static void apic_pre_save(APICCommonState *s)
+{
+    apic_sync_vapic(s, SYNC_FROM_VAPIC);
+}
+
 static void apic_post_load(APICCommonState *s)
 {
     if (s->timer_expiry != -1) {
@@ -770,7 +871,10 @@ static void apic_class_init(ObjectClass *klass, void *data)
     k->init = apic_init;
     k->set_base = apic_set_base;
     k->set_tpr = apic_set_tpr;
+    k->get_tpr = apic_get_tpr;
+    k->vapic_base_update = apic_vapic_base_update;
     k->external_nmi = apic_external_nmi;
+    k->pre_save = apic_pre_save;
     k->post_load = apic_post_load;
 }
 
diff --git a/hw/apic.h b/hw/apic.h
index a62d83ba9f..d6d6d440ee 100644
--- a/hw/apic.h
+++ b/hw/apic.h
@@ -18,6 +18,8 @@ void cpu_set_apic_tpr(DeviceState *s, uint8_t val);
 uint8_t cpu_get_apic_tpr(DeviceState *s);
 void apic_init_reset(DeviceState *s);
 void apic_sipi(DeviceState *s);
+void apic_handle_tpr_access_report(DeviceState *d, target_ulong ip,
+                                   TPRAccess access);
 
 /* pc.c */
 int cpu_is_bsp(CPUState *env);
diff --git a/hw/apic_common.c b/hw/apic_common.c
index c91f7d5391..60b82596e7 100644
--- a/hw/apic_common.c
+++ b/hw/apic_common.c
@@ -20,8 +20,10 @@
 #include "apic.h"
 #include "apic_internal.h"
 #include "trace.h"
+#include "kvm.h"
 
 static int apic_irq_delivered;
+bool apic_report_tpr_access;
 
 void cpu_set_apic_base(DeviceState *d, uint64_t val)
 {
@@ -63,9 +65,45 @@ void cpu_set_apic_tpr(DeviceState *d, uint8_t val)
 
 uint8_t cpu_get_apic_tpr(DeviceState *d)
 {
+    APICCommonState *s;
+    APICCommonClass *info;
+
+    if (!d) {
+        return 0;
+    }
+
+    s = APIC_COMMON(d);
+    info = APIC_COMMON_GET_CLASS(s);
+
+    return info->get_tpr(s);
+}
+
+void apic_enable_tpr_access_reporting(DeviceState *d, bool enable)
+{
     APICCommonState *s = DO_UPCAST(APICCommonState, busdev.qdev, d);
+    APICCommonClass *info = APIC_COMMON_GET_CLASS(s);
 
-    return s ? s->tpr >> 4 : 0;
+    apic_report_tpr_access = enable;
+    if (info->enable_tpr_reporting) {
+        info->enable_tpr_reporting(s, enable);
+    }
+}
+
+void apic_enable_vapic(DeviceState *d, target_phys_addr_t paddr)
+{
+    APICCommonState *s = DO_UPCAST(APICCommonState, busdev.qdev, d);
+    APICCommonClass *info = APIC_COMMON_GET_CLASS(s);
+
+    s->vapic_paddr = paddr;
+    info->vapic_base_update(s);
+}
+
+void apic_handle_tpr_access_report(DeviceState *d, target_ulong ip,
+                                   TPRAccess access)
+{
+    APICCommonState *s = DO_UPCAST(APICCommonState, busdev.qdev, d);
+
+    vapic_report_tpr_access(s->vapic, s->cpu_env, ip, access);
 }
 
 void apic_report_irq_delivered(int delivered)
@@ -166,12 +204,16 @@ void apic_init_reset(DeviceState *d)
 static void apic_reset_common(DeviceState *d)
 {
     APICCommonState *s = DO_UPCAST(APICCommonState, busdev.qdev, d);
+    APICCommonClass *info = APIC_COMMON_GET_CLASS(s);
     bool bsp;
 
     bsp = cpu_is_bsp(s->cpu_env);
     s->apicbase = 0xfee00000 |
         (bsp ? MSR_IA32_APICBASE_BSP : 0) | MSR_IA32_APICBASE_ENABLE;
 
+    s->vapic_paddr = 0;
+    info->vapic_base_update(s);
+
     apic_init_reset(d);
 
     if (bsp) {
@@ -234,6 +276,7 @@ static int apic_init_common(SysBusDevice *dev)
 {
     APICCommonState *s = APIC_COMMON(dev);
     APICCommonClass *info;
+    static DeviceState *vapic;
     static int apic_no;
 
     if (apic_no >= MAX_APICS) {
@@ -244,10 +287,29 @@ static int apic_init_common(SysBusDevice *dev)
     info = APIC_COMMON_GET_CLASS(s);
     info->init(s);
 
-    sysbus_init_mmio(&s->busdev, &s->io_memory);
+    sysbus_init_mmio(dev, &s->io_memory);
+
+    if (!vapic && s->vapic_control & VAPIC_ENABLE_MASK) {
+        vapic = sysbus_create_simple("kvmvapic", -1, NULL);
+    }
+    s->vapic = vapic;
+    if (apic_report_tpr_access && info->enable_tpr_reporting) {
+        info->enable_tpr_reporting(s, true);
+    }
+
     return 0;
 }
 
+static void apic_dispatch_pre_save(void *opaque)
+{
+    APICCommonState *s = APIC_COMMON(opaque);
+    APICCommonClass *info = APIC_COMMON_GET_CLASS(s);
+
+    if (info->pre_save) {
+        info->pre_save(s);
+    }
+}
+
 static int apic_dispatch_post_load(void *opaque, int version_id)
 {
     APICCommonState *s = APIC_COMMON(opaque);
@@ -265,6 +327,7 @@ static const VMStateDescription vmstate_apic_common = {
     .minimum_version_id = 3,
     .minimum_version_id_old = 1,
     .load_state_old = apic_load_old,
+    .pre_save = apic_dispatch_pre_save,
     .post_load = apic_dispatch_post_load,
     .fields = (VMStateField[]) {
         VMSTATE_UINT32(apicbase, APICCommonState),
@@ -294,6 +357,8 @@ static const VMStateDescription vmstate_apic_common = {
 static Property apic_properties_common[] = {
     DEFINE_PROP_UINT8("id", APICCommonState, id, -1),
     DEFINE_PROP_PTR("cpu_env", APICCommonState, cpu_env),
+    DEFINE_PROP_BIT("vapic", APICCommonState, vapic_control, VAPIC_ENABLE_BIT,
+                    true),
     DEFINE_PROP_END_OF_LIST(),
 };
 
diff --git a/hw/apic_internal.h b/hw/apic_internal.h
index 0cab010717..60a6a8bdae 100644
--- a/hw/apic_internal.h
+++ b/hw/apic_internal.h
@@ -61,6 +61,9 @@
 #define APIC_SV_DIRECTED_IO             (1<<12)
 #define APIC_SV_ENABLE                  (1<<8)
 
+#define VAPIC_ENABLE_BIT                0
+#define VAPIC_ENABLE_MASK               (1 << VAPIC_ENABLE_BIT)
+
 #define MAX_APICS 255
 
 #define MSI_SPACE_SIZE                  0x100000
@@ -82,7 +85,11 @@ typedef struct APICCommonClass
     void (*init)(APICCommonState *s);
     void (*set_base)(APICCommonState *s, uint64_t val);
     void (*set_tpr)(APICCommonState *s, uint8_t val);
+    uint8_t (*get_tpr)(APICCommonState *s);
+    void (*enable_tpr_reporting)(APICCommonState *s, bool enable);
+    void (*vapic_base_update)(APICCommonState *s);
     void (*external_nmi)(APICCommonState *s);
+    void (*pre_save)(APICCommonState *s);
     void (*post_load)(APICCommonState *s);
 } APICCommonClass;
 
@@ -114,9 +121,29 @@ struct APICCommonState {
     int64_t timer_expiry;
     int sipi_vector;
     int wait_for_sipi;
+
+    uint32_t vapic_control;
+    DeviceState *vapic;
+    target_phys_addr_t vapic_paddr; /* note: persistence via kvmvapic */
 };
 
+typedef struct VAPICState {
+    uint8_t tpr;
+    uint8_t isr;
+    uint8_t zero;
+    uint8_t irr;
+    uint8_t enabled;
+} QEMU_PACKED VAPICState;
+
+extern bool apic_report_tpr_access;
+
 void apic_report_irq_delivered(int delivered);
 bool apic_next_timer(APICCommonState *s, int64_t current_time);
+void apic_enable_tpr_access_reporting(DeviceState *d, bool enable);
+void apic_enable_vapic(DeviceState *d, target_phys_addr_t paddr);
+void apic_poll_irq(DeviceState *d);
+
+void vapic_report_tpr_access(DeviceState *dev, void *cpu, target_ulong ip,
+                             TPRAccess access);
 
 #endif /* !QEMU_APIC_INTERNAL_H */
diff --git a/hw/kvm/apic.c b/hw/kvm/apic.c
index 5bb0a4b9fd..9ca68f81aa 100644
--- a/hw/kvm/apic.c
+++ b/hw/kvm/apic.c
@@ -92,6 +92,35 @@ static void kvm_apic_set_tpr(APICCommonState *s, uint8_t val)
     s->tpr = (val & 0x0f) << 4;
 }
 
+static uint8_t kvm_apic_get_tpr(APICCommonState *s)
+{
+    return s->tpr >> 4;
+}
+
+static void kvm_apic_enable_tpr_reporting(APICCommonState *s, bool enable)
+{
+    struct kvm_tpr_access_ctl ctl = {
+        .enabled = enable
+    };
+
+    kvm_vcpu_ioctl(s->cpu_env, KVM_TPR_ACCESS_REPORTING, &ctl);
+}
+
+static void kvm_apic_vapic_base_update(APICCommonState *s)
+{
+    struct kvm_vapic_addr vapid_addr = {
+        .vapic_addr = s->vapic_paddr,
+    };
+    int ret;
+
+    ret = kvm_vcpu_ioctl(s->cpu_env, KVM_SET_VAPIC_ADDR, &vapid_addr);
+    if (ret < 0) {
+        fprintf(stderr, "KVM: setting VAPIC address failed (%s)\n",
+                strerror(-ret));
+        abort();
+    }
+}
+
 static void do_inject_external_nmi(void *data)
 {
     APICCommonState *s = data;
@@ -129,6 +158,9 @@ static void kvm_apic_class_init(ObjectClass *klass, void *data)
     k->init = kvm_apic_init;
     k->set_base = kvm_apic_set_base;
     k->set_tpr = kvm_apic_set_tpr;
+    k->get_tpr = kvm_apic_get_tpr;
+    k->enable_tpr_reporting = kvm_apic_enable_tpr_reporting;
+    k->vapic_base_update = kvm_apic_vapic_base_update;
     k->external_nmi = kvm_apic_external_nmi;
 }
 
diff --git a/hw/kvmvapic.c b/hw/kvmvapic.c
new file mode 100644
index 0000000000..36ccfbcdbd
--- /dev/null
+++ b/hw/kvmvapic.c
@@ -0,0 +1,805 @@
+/*
+ * TPR optimization for 32-bit Windows guests (XP and Server 2003)
+ *
+ * Copyright (C) 2007-2008 Qumranet Technologies
+ * Copyright (C) 2012      Jan Kiszka, Siemens AG
+ *
+ * This work is licensed under the terms of the GNU GPL version 2, or
+ * (at your option) any later version. See the COPYING file in the
+ * top-level directory.
+ */
+#include "sysemu.h"
+#include "cpus.h"
+#include "kvm.h"
+#include "apic_internal.h"
+
+#define APIC_DEFAULT_ADDRESS    0xfee00000
+
+#define VAPIC_IO_PORT           0x7e
+
+#define VAPIC_CPU_SHIFT         7
+
+#define ROM_BLOCK_SIZE          512
+#define ROM_BLOCK_MASK          (~(ROM_BLOCK_SIZE - 1))
+
+typedef enum VAPICMode {
+    VAPIC_INACTIVE = 0,
+    VAPIC_ACTIVE   = 1,
+    VAPIC_STANDBY  = 2,
+} VAPICMode;
+
+typedef struct VAPICHandlers {
+    uint32_t set_tpr;
+    uint32_t set_tpr_eax;
+    uint32_t get_tpr[8];
+    uint32_t get_tpr_stack;
+} QEMU_PACKED VAPICHandlers;
+
+typedef struct GuestROMState {
+    char signature[8];
+    uint32_t vaddr;
+    uint32_t fixup_start;
+    uint32_t fixup_end;
+    uint32_t vapic_vaddr;
+    uint32_t vapic_size;
+    uint32_t vcpu_shift;
+    uint32_t real_tpr_addr;
+    VAPICHandlers up;
+    VAPICHandlers mp;
+} QEMU_PACKED GuestROMState;
+
+typedef struct VAPICROMState {
+    SysBusDevice busdev;
+    MemoryRegion io;
+    MemoryRegion rom;
+    uint32_t state;
+    uint32_t rom_state_paddr;
+    uint32_t rom_state_vaddr;
+    uint32_t vapic_paddr;
+    uint32_t real_tpr_addr;
+    GuestROMState rom_state;
+    size_t rom_size;
+    bool rom_mapped_writable;
+} VAPICROMState;
+
+#define TPR_INSTR_ABS_MODRM             0x1
+#define TPR_INSTR_MATCH_MODRM_REG       0x2
+
+typedef struct TPRInstruction {
+    uint8_t opcode;
+    uint8_t modrm_reg;
+    unsigned int flags;
+    TPRAccess access;
+    size_t length;
+    off_t addr_offset;
+} TPRInstruction;
+
+/* must be sorted by length, shortest first */
+static const TPRInstruction tpr_instr[] = {
+    { /* mov abs to eax */
+        .opcode = 0xa1,
+        .access = TPR_ACCESS_READ,
+        .length = 5,
+        .addr_offset = 1,
+    },
+    { /* mov eax to abs */
+        .opcode = 0xa3,
+        .access = TPR_ACCESS_WRITE,
+        .length = 5,
+        .addr_offset = 1,
+    },
+    { /* mov r32 to r/m32 */
+        .opcode = 0x89,
+        .flags = TPR_INSTR_ABS_MODRM,
+        .access = TPR_ACCESS_WRITE,
+        .length = 6,
+        .addr_offset = 2,
+    },
+    { /* mov r/m32 to r32 */
+        .opcode = 0x8b,
+        .flags = TPR_INSTR_ABS_MODRM,
+        .access = TPR_ACCESS_READ,
+        .length = 6,
+        .addr_offset = 2,
+    },
+    { /* push r/m32 */
+        .opcode = 0xff,
+        .modrm_reg = 6,
+        .flags = TPR_INSTR_ABS_MODRM | TPR_INSTR_MATCH_MODRM_REG,
+        .access = TPR_ACCESS_READ,
+        .length = 6,
+        .addr_offset = 2,
+    },
+    { /* mov imm32, r/m32 (c7/0) */
+        .opcode = 0xc7,
+        .modrm_reg = 0,
+        .flags = TPR_INSTR_ABS_MODRM | TPR_INSTR_MATCH_MODRM_REG,
+        .access = TPR_ACCESS_WRITE,
+        .length = 10,
+        .addr_offset = 2,
+    },
+};
+
+static void read_guest_rom_state(VAPICROMState *s)
+{
+    cpu_physical_memory_rw(s->rom_state_paddr, (void *)&s->rom_state,
+                           sizeof(GuestROMState), 0);
+}
+
+static void write_guest_rom_state(VAPICROMState *s)
+{
+    cpu_physical_memory_rw(s->rom_state_paddr, (void *)&s->rom_state,
+                           sizeof(GuestROMState), 1);
+}
+
+static void update_guest_rom_state(VAPICROMState *s)
+{
+    read_guest_rom_state(s);
+
+    s->rom_state.real_tpr_addr = cpu_to_le32(s->real_tpr_addr);
+    s->rom_state.vcpu_shift = cpu_to_le32(VAPIC_CPU_SHIFT);
+
+    write_guest_rom_state(s);
+}
+
+static int find_real_tpr_addr(VAPICROMState *s, CPUState *env)
+{
+    target_phys_addr_t paddr;
+    target_ulong addr;
+
+    if (s->state == VAPIC_ACTIVE) {
+        return 0;
+    }
+    /*
+     * If there is no prior TPR access instruction we could analyze (which is
+     * the case after resume from hibernation), we need to scan the possible
+     * virtual address space for the APIC mapping.
+     */
+    for (addr = 0xfffff000; addr >= 0x80000000; addr -= TARGET_PAGE_SIZE) {
+        paddr = cpu_get_phys_page_debug(env, addr);
+        if (paddr != APIC_DEFAULT_ADDRESS) {
+            continue;
+        }
+        s->real_tpr_addr = addr + 0x80;
+        update_guest_rom_state(s);
+        return 0;
+    }
+    return -1;
+}
+
+static uint8_t modrm_reg(uint8_t modrm)
+{
+    return (modrm >> 3) & 7;
+}
+
+static bool is_abs_modrm(uint8_t modrm)
+{
+    return (modrm & 0xc7) == 0x05;
+}
+
+static bool opcode_matches(uint8_t *opcode, const TPRInstruction *instr)
+{
+    return opcode[0] == instr->opcode &&
+        (!(instr->flags & TPR_INSTR_ABS_MODRM) || is_abs_modrm(opcode[1])) &&
+        (!(instr->flags & TPR_INSTR_MATCH_MODRM_REG) ||
+         modrm_reg(opcode[1]) == instr->modrm_reg);
+}
+
+static int evaluate_tpr_instruction(VAPICROMState *s, CPUState *env,
+                                    target_ulong *pip, TPRAccess access)
+{
+    const TPRInstruction *instr;
+    target_ulong ip = *pip;
+    uint8_t opcode[2];
+    uint32_t real_tpr_addr;
+    int i;
+
+    if ((ip & 0xf0000000ULL) != 0x80000000ULL &&
+        (ip & 0xf0000000ULL) != 0xe0000000ULL) {
+        return -1;
+    }
+
+    /*
+     * Early Windows 2003 SMP initialization contains a
+     *
+     *   mov imm32, r/m32
+     *
+     * instruction that is patched by TPR optimization. The problem is that
+     * RSP, used by the patched instruction, is zero, so the guest gets a
+     * double fault and dies.
+     */
+    if (env->regs[R_ESP] == 0) {
+        return -1;
+    }
+
+    if (kvm_enabled() && !kvm_irqchip_in_kernel()) {
+        /*
+         * KVM without kernel-based TPR access reporting will pass an IP that
+         * points after the accessing instruction. So we need to look backward
+         * to find the reason.
+         */
+        for (i = 0; i < ARRAY_SIZE(tpr_instr); i++) {
+            instr = &tpr_instr[i];
+            if (instr->access != access) {
+                continue;
+            }
+            if (cpu_memory_rw_debug(env, ip - instr->length, opcode,
+                                    sizeof(opcode), 0) < 0) {
+                return -1;
+            }
+            if (opcode_matches(opcode, instr)) {
+                ip -= instr->length;
+                goto instruction_ok;
+            }
+        }
+        return -1;
+    } else {
+        if (cpu_memory_rw_debug(env, ip, opcode, sizeof(opcode), 0) < 0) {
+            return -1;
+        }
+        for (i = 0; i < ARRAY_SIZE(tpr_instr); i++) {
+            instr = &tpr_instr[i];
+            if (opcode_matches(opcode, instr)) {
+                goto instruction_ok;
+            }
+        }
+        return -1;
+    }
+
+instruction_ok:
+    /*
+     * Grab the virtual TPR address from the instruction
+     * and update the cached values.
+     */
+    if (cpu_memory_rw_debug(env, ip + instr->addr_offset,
+                            (void *)&real_tpr_addr,
+                            sizeof(real_tpr_addr), 0) < 0) {
+        return -1;
+    }
+    real_tpr_addr = le32_to_cpu(real_tpr_addr);
+    if ((real_tpr_addr & 0xfff) != 0x80) {
+        return -1;
+    }
+    s->real_tpr_addr = real_tpr_addr;
+    update_guest_rom_state(s);
+
+    *pip = ip;
+    return 0;
+}
+
+static int update_rom_mapping(VAPICROMState *s, CPUState *env, target_ulong ip)
+{
+    target_phys_addr_t paddr;
+    uint32_t rom_state_vaddr;
+    uint32_t pos, patch, offset;
+
+    /* nothing to do if already activated */
+    if (s->state == VAPIC_ACTIVE) {
+        return 0;
+    }
+
+    /* bail out if ROM init code was not executed (missing ROM?) */
+    if (s->state == VAPIC_INACTIVE) {
+        return -1;
+    }
+
+    /* find out virtual address of the ROM */
+    rom_state_vaddr = s->rom_state_paddr + (ip & 0xf0000000);
+    paddr = cpu_get_phys_page_debug(env, rom_state_vaddr);
+    if (paddr == -1) {
+        return -1;
+    }
+    paddr += rom_state_vaddr & ~TARGET_PAGE_MASK;
+    if (paddr != s->rom_state_paddr) {
+        return -1;
+    }
+    read_guest_rom_state(s);
+    if (memcmp(s->rom_state.signature, "kvm aPiC", 8) != 0) {
+        return -1;
+    }
+    s->rom_state_vaddr = rom_state_vaddr;
+
+    /* fixup addresses in ROM if needed */
+    if (rom_state_vaddr == le32_to_cpu(s->rom_state.vaddr)) {
+        return 0;
+    }
+    for (pos = le32_to_cpu(s->rom_state.fixup_start);
+         pos < le32_to_cpu(s->rom_state.fixup_end);
+         pos += 4) {
+        cpu_physical_memory_rw(paddr + pos - s->rom_state.vaddr,
+                               (void *)&offset, sizeof(offset), 0);
+        offset = le32_to_cpu(offset);
+        cpu_physical_memory_rw(paddr + offset, (void *)&patch,
+                               sizeof(patch), 0);
+        patch = le32_to_cpu(patch);
+        patch += rom_state_vaddr - le32_to_cpu(s->rom_state.vaddr);
+        patch = cpu_to_le32(patch);
+        cpu_physical_memory_rw(paddr + offset, (void *)&patch,
+                               sizeof(patch), 1);
+    }
+    read_guest_rom_state(s);
+    s->vapic_paddr = paddr + le32_to_cpu(s->rom_state.vapic_vaddr) -
+        le32_to_cpu(s->rom_state.vaddr);
+
+    return 0;
+}
+
+/*
+ * Tries to read the unique processor number from the Kernel Processor Control
+ * Region (KPCR) of 32-bit Windows XP and Server 2003. Returns -1 if the KPCR
+ * cannot be accessed or is considered invalid. This also ensures that we are
+ * not patching the wrong guest.
+ */
+static int get_kpcr_number(CPUState *env)
+{
+    struct kpcr {
+        uint8_t  fill1[0x1c];
+        uint32_t self;
+        uint8_t  fill2[0x31];
+        uint8_t  number;
+    } QEMU_PACKED kpcr;
+
+    if (cpu_memory_rw_debug(env, env->segs[R_FS].base,
+                            (void *)&kpcr, sizeof(kpcr), 0) < 0 ||
+        kpcr.self != env->segs[R_FS].base) {
+        return -1;
+    }
+    return kpcr.number;
+}
+
+static int vapic_enable(VAPICROMState *s, CPUState *env)
+{
+    int cpu_number = get_kpcr_number(env);
+    target_phys_addr_t vapic_paddr;
+    static const uint8_t enabled = 1;
+
+    if (cpu_number < 0) {
+        return -1;
+    }
+    vapic_paddr = s->vapic_paddr +
+        (((target_phys_addr_t)cpu_number) << VAPIC_CPU_SHIFT);
+    cpu_physical_memory_rw(vapic_paddr + offsetof(VAPICState, enabled),
+                           (void *)&enabled, sizeof(enabled), 1);
+    apic_enable_vapic(env->apic_state, vapic_paddr);
+
+    s->state = VAPIC_ACTIVE;
+
+    return 0;
+}
+
+static void patch_byte(CPUState *env, target_ulong addr, uint8_t byte)
+{
+    cpu_memory_rw_debug(env, addr, &byte, 1, 1);
+}
+
+static void patch_call(VAPICROMState *s, CPUState *env, target_ulong ip,
+                       uint32_t target)
+{
+    uint32_t offset;
+
+    offset = cpu_to_le32(target - ip - 5);
+    patch_byte(env, ip, 0xe8); /* call near */
+    cpu_memory_rw_debug(env, ip + 1, (void *)&offset, sizeof(offset), 1);
+}
+
+static void patch_instruction(VAPICROMState *s, CPUState *env, target_ulong ip)
+{
+    target_phys_addr_t paddr;
+    VAPICHandlers *handlers;
+    uint8_t opcode[2];
+    uint32_t imm32;
+
+    if (smp_cpus == 1) {
+        handlers = &s->rom_state.up;
+    } else {
+        handlers = &s->rom_state.mp;
+    }
+
+    pause_all_vcpus();
+
+    cpu_memory_rw_debug(env, ip, opcode, sizeof(opcode), 0);
+
+    switch (opcode[0]) {
+    case 0x89: /* mov r32 to r/m32 */
+        patch_byte(env, ip, 0x50 + modrm_reg(opcode[1]));  /* push reg */
+        patch_call(s, env, ip + 1, handlers->set_tpr);
+        break;
+    case 0x8b: /* mov r/m32 to r32 */
+        patch_byte(env, ip, 0x90);
+        patch_call(s, env, ip + 1, handlers->get_tpr[modrm_reg(opcode[1])]);
+        break;
+    case 0xa1: /* mov abs to eax */
+        patch_call(s, env, ip, handlers->get_tpr[0]);
+        break;
+    case 0xa3: /* mov eax to abs */
+        patch_call(s, env, ip, handlers->set_tpr_eax);
+        break;
+    case 0xc7: /* mov imm32, r/m32 (c7/0) */
+        patch_byte(env, ip, 0x68);  /* push imm32 */
+        cpu_memory_rw_debug(env, ip + 6, (void *)&imm32, sizeof(imm32), 0);
+        cpu_memory_rw_debug(env, ip + 1, (void *)&imm32, sizeof(imm32), 1);
+        patch_call(s, env, ip + 5, handlers->set_tpr);
+        break;
+    case 0xff: /* push r/m32 */
+        patch_byte(env, ip, 0x50); /* push eax */
+        patch_call(s, env, ip + 1, handlers->get_tpr_stack);
+        break;
+    default:
+        abort();
+    }
+
+    resume_all_vcpus();
+
+    paddr = cpu_get_phys_page_debug(env, ip);
+    paddr += ip & ~TARGET_PAGE_MASK;
+    tb_invalidate_phys_page_range(paddr, paddr + 1, 1);
+}
+
+void vapic_report_tpr_access(DeviceState *dev, void *cpu, target_ulong ip,
+                             TPRAccess access)
+{
+    VAPICROMState *s = DO_UPCAST(VAPICROMState, busdev.qdev, dev);
+    CPUState *env = cpu;
+
+    cpu_synchronize_state(env);
+
+    if (evaluate_tpr_instruction(s, env, &ip, access) < 0) {
+        if (s->state == VAPIC_ACTIVE) {
+            vapic_enable(s, env);
+        }
+        return;
+    }
+    if (update_rom_mapping(s, env, ip) < 0) {
+        return;
+    }
+    if (vapic_enable(s, env) < 0) {
+        return;
+    }
+    patch_instruction(s, env, ip);
+}
+
+typedef struct VAPICEnableTPRReporting {
+    DeviceState *apic;
+    bool enable;
+} VAPICEnableTPRReporting;
+
+static void vapic_do_enable_tpr_reporting(void *data)
+{
+    VAPICEnableTPRReporting *info = data;
+
+    apic_enable_tpr_access_reporting(info->apic, info->enable);
+}
+
+static void vapic_enable_tpr_reporting(bool enable)
+{
+    VAPICEnableTPRReporting info = {
+        .enable = enable,
+    };
+    CPUState *env;
+
+    for (env = first_cpu; env != NULL; env = env->next_cpu) {
+        info.apic = env->apic_state;
+        run_on_cpu(env, vapic_do_enable_tpr_reporting, &info);
+    }
+}
+
+static void vapic_reset(DeviceState *dev)
+{
+    VAPICROMState *s = DO_UPCAST(VAPICROMState, busdev.qdev, dev);
+
+    if (s->state == VAPIC_ACTIVE) {
+        s->state = VAPIC_STANDBY;
+    }
+    vapic_enable_tpr_reporting(false);
+}
+
+/*
+ * Set the IRQ polling hypercalls to the supported variant:
+ *  - vmcall if using KVM in-kernel irqchip
+ *  - 32-bit VAPIC port write otherwise
+ */
+static int patch_hypercalls(VAPICROMState *s)
+{
+    target_phys_addr_t rom_paddr = s->rom_state_paddr & ROM_BLOCK_MASK;
+    static const uint8_t vmcall_pattern[] = { /* vmcall */
+        0xb8, 0x1, 0, 0, 0, 0xf, 0x1, 0xc1
+    };
+    static const uint8_t outl_pattern[] = { /* nop; outl %eax,0x7e */
+        0xb8, 0x1, 0, 0, 0, 0x90, 0xe7, 0x7e
+    };
+    uint8_t alternates[2];
+    const uint8_t *pattern;
+    const uint8_t *patch;
+    int patches = 0;
+    off_t pos;
+    uint8_t *rom;
+
+    rom = g_malloc(s->rom_size);
+    cpu_physical_memory_rw(rom_paddr, rom, s->rom_size, 0);
+
+    for (pos = 0; pos < s->rom_size - sizeof(vmcall_pattern); pos++) {
+        if (kvm_irqchip_in_kernel()) {
+            pattern = outl_pattern;
+            alternates[0] = outl_pattern[7];
+            alternates[1] = outl_pattern[7];
+            patch = &vmcall_pattern[5];
+        } else {
+            pattern = vmcall_pattern;
+            alternates[0] = vmcall_pattern[7];
+            alternates[1] = 0xd9; /* AMD's VMMCALL */
+            patch = &outl_pattern[5];
+        }
+        if (memcmp(rom + pos, pattern, 7) == 0 &&
+            (rom[pos + 7] == alternates[0] || rom[pos + 7] == alternates[1])) {
+            cpu_physical_memory_rw(rom_paddr + pos + 5, (uint8_t *)patch,
+                                   3, 1);
+            /*
+             * Don't flush the tb here. Under ordinary conditions, the patched
+             * calls are miles away from the current IP. Under malicious
+             * conditions, the guest could trick us to crash.
+             */
+        }
+    }
+
+    g_free(rom);
+
+    if (patches != 0 && patches != 2) {
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * For TCG mode or the time KVM honors read-only memory regions, we need to
+ * enable write access to the option ROM so that variables can be updated by
+ * the guest.
+ */
+static void vapic_map_rom_writable(VAPICROMState *s)
+{
+    target_phys_addr_t rom_paddr = s->rom_state_paddr & ROM_BLOCK_MASK;
+    MemoryRegionSection section;
+    MemoryRegion *as;
+    size_t rom_size;
+    uint8_t *ram;
+
+    as = sysbus_address_space(&s->busdev);
+
+    if (s->rom_mapped_writable) {
+        memory_region_del_subregion(as, &s->rom);
+        memory_region_destroy(&s->rom);
+    }
+
+    /* grab RAM memory region (region @rom_paddr may still be pc.rom) */
+    section = memory_region_find(as, 0, 1);
+
+    /* read ROM size from RAM region */
+    ram = memory_region_get_ram_ptr(section.mr);
+    rom_size = ram[rom_paddr + 2] * ROM_BLOCK_SIZE;
+    s->rom_size = rom_size;
+
+    /* We need to round up to avoid creating subpages
+     * from which we cannot run code. */
+    rom_size = TARGET_PAGE_ALIGN(rom_size);
+
+    memory_region_init_alias(&s->rom, "kvmvapic-rom", section.mr, rom_paddr,
+                             rom_size);
+    memory_region_add_subregion_overlap(as, rom_paddr, &s->rom, 1000);
+    s->rom_mapped_writable = true;
+}
+
+static int vapic_prepare(VAPICROMState *s)
+{
+    vapic_map_rom_writable(s);
+
+    if (patch_hypercalls(s) < 0) {
+        return -1;
+    }
+
+    vapic_enable_tpr_reporting(true);
+
+    return 0;
+}
+
+static void vapic_write(void *opaque, target_phys_addr_t addr, uint64_t data,
+                        unsigned int size)
+{
+    CPUState *env = cpu_single_env;
+    target_phys_addr_t rom_paddr;
+    VAPICROMState *s = opaque;
+
+    cpu_synchronize_state(env);
+
+    /*
+     * The VAPIC supports two PIO-based hypercalls, both via port 0x7E.
+     *  o 16-bit write access:
+     *    Reports the option ROM initialization to the hypervisor. Written
+     *    value is the offset of the state structure in the ROM.
+     *  o 8-bit write access:
+     *    Reactivates the VAPIC after a guest hibernation, i.e. after the
+     *    option ROM content has been re-initialized by a guest power cycle.
+     *  o 32-bit write access:
+     *    Poll for pending IRQs, considering the current VAPIC state.
+     */
+    switch (size) {
+    case 2:
+        if (s->state == VAPIC_INACTIVE) {
+            rom_paddr = (env->segs[R_CS].base + env->eip) & ROM_BLOCK_MASK;
+            s->rom_state_paddr = rom_paddr + data;
+
+            s->state = VAPIC_STANDBY;
+        }
+        if (vapic_prepare(s) < 0) {
+            s->state = VAPIC_INACTIVE;
+            break;
+        }
+        break;
+    case 1:
+        if (kvm_enabled()) {
+            /*
+             * Disable triggering instruction in ROM by writing a NOP.
+             *
+             * We cannot do this in TCG mode as the reported IP is not
+             * accurate.
+             */
+            pause_all_vcpus();
+            patch_byte(env, env->eip - 2, 0x66);
+            patch_byte(env, env->eip - 1, 0x90);
+            resume_all_vcpus();
+        }
+
+        if (s->state == VAPIC_ACTIVE) {
+            break;
+        }
+        if (update_rom_mapping(s, env, env->eip) < 0) {
+            break;
+        }
+        if (find_real_tpr_addr(s, env) < 0) {
+            break;
+        }
+        vapic_enable(s, env);
+        break;
+    default:
+    case 4:
+        if (!kvm_irqchip_in_kernel()) {
+            apic_poll_irq(env->apic_state);
+        }
+        break;
+    }
+}
+
+static const MemoryRegionOps vapic_ops = {
+    .write = vapic_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static int vapic_init(SysBusDevice *dev)
+{
+    VAPICROMState *s = FROM_SYSBUS(VAPICROMState, dev);
+
+    memory_region_init_io(&s->io, &vapic_ops, s, "kvmvapic", 2);
+    sysbus_add_io(dev, VAPIC_IO_PORT, &s->io);
+    sysbus_init_ioports(dev, VAPIC_IO_PORT, 2);
+
+    option_rom[nb_option_roms].name = "kvmvapic.bin";
+    option_rom[nb_option_roms].bootindex = -1;
+    nb_option_roms++;
+
+    return 0;
+}
+
+static void do_vapic_enable(void *data)
+{
+    VAPICROMState *s = data;
+
+    vapic_enable(s, first_cpu);
+}
+
+static int vapic_post_load(void *opaque, int version_id)
+{
+    VAPICROMState *s = opaque;
+    uint8_t *zero;
+
+    /*
+     * The old implementation of qemu-kvm did not provide the state
+     * VAPIC_STANDBY. Reconstruct it.
+     */
+    if (s->state == VAPIC_INACTIVE && s->rom_state_paddr != 0) {
+        s->state = VAPIC_STANDBY;
+    }
+
+    if (s->state != VAPIC_INACTIVE) {
+        if (vapic_prepare(s) < 0) {
+            return -1;
+        }
+    }
+    if (s->state == VAPIC_ACTIVE) {
+        if (smp_cpus == 1) {
+            run_on_cpu(first_cpu, do_vapic_enable, s);
+        } else {
+            zero = g_malloc0(s->rom_state.vapic_size);
+            cpu_physical_memory_rw(s->vapic_paddr, zero,
+                                   s->rom_state.vapic_size, 1);
+            g_free(zero);
+        }
+    }
+
+    return 0;
+}
+
+static const VMStateDescription vmstate_handlers = {
+    .name = "kvmvapic-handlers",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(set_tpr, VAPICHandlers),
+        VMSTATE_UINT32(set_tpr_eax, VAPICHandlers),
+        VMSTATE_UINT32_ARRAY(get_tpr, VAPICHandlers, 8),
+        VMSTATE_UINT32(get_tpr_stack, VAPICHandlers),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_guest_rom = {
+    .name = "kvmvapic-guest-rom",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UNUSED(8),     /* signature */
+        VMSTATE_UINT32(vaddr, GuestROMState),
+        VMSTATE_UINT32(fixup_start, GuestROMState),
+        VMSTATE_UINT32(fixup_end, GuestROMState),
+        VMSTATE_UINT32(vapic_vaddr, GuestROMState),
+        VMSTATE_UINT32(vapic_size, GuestROMState),
+        VMSTATE_UINT32(vcpu_shift, GuestROMState),
+        VMSTATE_UINT32(real_tpr_addr, GuestROMState),
+        VMSTATE_STRUCT(up, GuestROMState, 0, vmstate_handlers, VAPICHandlers),
+        VMSTATE_STRUCT(mp, GuestROMState, 0, vmstate_handlers, VAPICHandlers),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_vapic = {
+    .name = "kvm-tpr-opt",      /* compatible with qemu-kvm VAPIC */
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .post_load = vapic_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_STRUCT(rom_state, VAPICROMState, 0, vmstate_guest_rom,
+                       GuestROMState),
+        VMSTATE_UINT32(state, VAPICROMState),
+        VMSTATE_UINT32(real_tpr_addr, VAPICROMState),
+        VMSTATE_UINT32(rom_state_vaddr, VAPICROMState),
+        VMSTATE_UINT32(vapic_paddr, VAPICROMState),
+        VMSTATE_UINT32(rom_state_paddr, VAPICROMState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void vapic_class_init(ObjectClass *klass, void *data)
+{
+    SysBusDeviceClass *sc = SYS_BUS_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->no_user = 1;
+    dc->reset   = vapic_reset;
+    dc->vmsd    = &vmstate_vapic;
+    sc->init    = vapic_init;
+}
+
+static TypeInfo vapic_type = {
+    .name          = "kvmvapic",
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(VAPICROMState),
+    .class_init    = vapic_class_init,
+};
+
+static void vapic_register(void)
+{
+    type_register_static(&vapic_type);
+}
+
+type_init(vapic_register);
diff --git a/hw/mc146818rtc.c b/hw/mc146818rtc.c
index a46fdfc487..8b5cf8c81f 100644
--- a/hw/mc146818rtc.c
+++ b/hw/mc146818rtc.c
@@ -25,10 +25,13 @@
 #include "qemu-timer.h"
 #include "sysemu.h"
 #include "pc.h"
-#include "apic.h"
 #include "isa.h"
 #include "mc146818rtc.h"
 
+#ifdef TARGET_I386
+#include "apic.h"
+#endif
+
 //#define DEBUG_CMOS
 //#define DEBUG_COALESCED
 
diff --git a/hw/vhost.c b/hw/vhost.c
index 5ece659f4a..8d3ba5b608 100644
--- a/hw/vhost.c
+++ b/hw/vhost.c
@@ -436,6 +436,14 @@ static bool vhost_section(MemoryRegionSection *section)
         && memory_region_is_ram(section->mr);
 }
 
+static void vhost_begin(MemoryListener *listener)
+{
+}
+
+static void vhost_commit(MemoryListener *listener)
+{
+}
+
 static void vhost_region_add(MemoryListener *listener,
                              MemoryRegionSection *section)
 {
@@ -476,6 +484,11 @@ static void vhost_region_del(MemoryListener *listener,
     }
 }
 
+static void vhost_region_nop(MemoryListener *listener,
+                             MemoryRegionSection *section)
+{
+}
+
 static int vhost_virtqueue_set_addr(struct vhost_dev *dev,
                                     struct vhost_virtqueue *vq,
                                     unsigned idx, bool enable_log)
@@ -720,6 +733,18 @@ static void vhost_virtqueue_cleanup(struct vhost_dev *dev,
                               0, virtio_queue_get_desc_size(vdev, idx));
 }
 
+static void vhost_eventfd_add(MemoryListener *listener,
+                              MemoryRegionSection *section,
+                              bool match_data, uint64_t data, int fd)
+{
+}
+
+static void vhost_eventfd_del(MemoryListener *listener,
+                              MemoryRegionSection *section,
+                              bool match_data, uint64_t data, int fd)
+{
+}
+
 int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force)
 {
     uint64_t features;
@@ -744,13 +769,19 @@ int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force)
     hdev->features = features;
 
     hdev->memory_listener = (MemoryListener) {
+        .begin = vhost_begin,
+        .commit = vhost_commit,
         .region_add = vhost_region_add,
         .region_del = vhost_region_del,
+        .region_nop = vhost_region_nop,
         .log_start = vhost_log_start,
         .log_stop = vhost_log_stop,
         .log_sync = vhost_log_sync,
         .log_global_start = vhost_log_global_start,
         .log_global_stop = vhost_log_global_stop,
+        .eventfd_add = vhost_eventfd_add,
+        .eventfd_del = vhost_eventfd_del,
+        .priority = 10
     };
     hdev->mem = g_malloc0(offsetof(struct vhost_memory, regions));
     hdev->n_mem_sections = 0;
@@ -759,7 +790,7 @@ int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force)
     hdev->log_size = 0;
     hdev->log_enabled = false;
     hdev->started = false;
-    memory_listener_register(&hdev->memory_listener);
+    memory_listener_register(&hdev->memory_listener, NULL);
     hdev->force = force;
     return 0;
 fail:
diff --git a/ioport.c b/ioport.c
index 36fa3a477e..8a474d3492 100644
--- a/ioport.c
+++ b/ioport.c
@@ -328,6 +328,7 @@ void portio_list_init(PortioList *piolist,
     piolist->ports = callbacks;
     piolist->nr = 0;
     piolist->regions = g_new0(MemoryRegion *, n);
+    piolist->aliases = g_new0(MemoryRegion *, n);
     piolist->address_space = NULL;
     piolist->opaque = opaque;
     piolist->name = name;
@@ -336,6 +337,7 @@ void portio_list_init(PortioList *piolist,
 void portio_list_destroy(PortioList *piolist)
 {
     g_free(piolist->regions);
+    g_free(piolist->aliases);
 }
 
 static void portio_list_add_1(PortioList *piolist,
@@ -345,7 +347,7 @@ static void portio_list_add_1(PortioList *piolist,
 {
     MemoryRegionPortio *pio;
     MemoryRegionOps *ops;
-    MemoryRegion *region;
+    MemoryRegion *region, *alias;
     unsigned i;
 
     /* Copy the sub-list and null-terminate it.  */
@@ -362,12 +364,20 @@ static void portio_list_add_1(PortioList *piolist,
     ops->old_portio = pio;
 
     region = g_new(MemoryRegion, 1);
+    alias = g_new(MemoryRegion, 1);
+    /*
+     * Use an alias so that the callback is called with an absolute address,
+     * rather than an offset relative to to start + off_low.
+     */
     memory_region_init_io(region, ops, piolist->opaque, piolist->name,
-                          off_high - off_low);
-    memory_region_set_offset(region, start + off_low);
+                          UINT64_MAX);
+    memory_region_init_alias(alias, piolist->name,
+                             region, start + off_low, off_high - off_low);
     memory_region_add_subregion(piolist->address_space,
-                                start + off_low, region);
-    piolist->regions[piolist->nr++] = region;
+                                start + off_low, alias);
+    piolist->regions[piolist->nr] = region;
+    piolist->aliases[piolist->nr] = alias;
+    ++piolist->nr;
 }
 
 void portio_list_add(PortioList *piolist,
@@ -409,15 +419,19 @@ void portio_list_add(PortioList *piolist,
 
 void portio_list_del(PortioList *piolist)
 {
-    MemoryRegion *mr;
+    MemoryRegion *mr, *alias;
     unsigned i;
 
     for (i = 0; i < piolist->nr; ++i) {
         mr = piolist->regions[i];
-        memory_region_del_subregion(piolist->address_space, mr);
+        alias = piolist->aliases[i];
+        memory_region_del_subregion(piolist->address_space, alias);
+        memory_region_destroy(alias);
         memory_region_destroy(mr);
         g_free((MemoryRegionOps *)mr->ops);
         g_free(mr);
+        g_free(alias);
         piolist->regions[i] = NULL;
+        piolist->aliases[i] = NULL;
     }
 }
diff --git a/ioport.h b/ioport.h
index ae3e9da0b5..ab29c89fb3 100644
--- a/ioport.h
+++ b/ioport.h
@@ -60,6 +60,7 @@ typedef struct PortioList {
     struct MemoryRegion *address_space;
     unsigned nr;
     struct MemoryRegion **regions;
+    struct MemoryRegion **aliases;
     void *opaque;
     const char *name;
 } PortioList;
diff --git a/kvm-all.c b/kvm-all.c
index 4b7a4ae5dd..77eadf608f 100644
--- a/kvm-all.c
+++ b/kvm-all.c
@@ -28,6 +28,7 @@
 #include "kvm.h"
 #include "bswap.h"
 #include "memory.h"
+#include "exec-memory.h"
 
 /* This check must be after config-host.h is included */
 #ifdef CONFIG_EVENTFD
@@ -683,6 +684,14 @@ static void kvm_set_phys_mem(MemoryRegionSection *section, bool add)
     }
 }
 
+static void kvm_begin(MemoryListener *listener)
+{
+}
+
+static void kvm_commit(MemoryListener *listener)
+{
+}
+
 static void kvm_region_add(MemoryListener *listener,
                            MemoryRegionSection *section)
 {
@@ -695,6 +704,11 @@ static void kvm_region_del(MemoryListener *listener,
     kvm_set_phys_mem(section, false);
 }
 
+static void kvm_region_nop(MemoryListener *listener,
+                           MemoryRegionSection *section)
+{
+}
+
 static void kvm_log_sync(MemoryListener *listener,
                          MemoryRegionSection *section)
 {
@@ -722,14 +736,95 @@ static void kvm_log_global_stop(struct MemoryListener *listener)
     assert(r >= 0);
 }
 
+static void kvm_mem_ioeventfd_add(MemoryRegionSection *section,
+                                  bool match_data, uint64_t data, int fd)
+{
+    int r;
+
+    assert(match_data && section->size == 4);
+
+    r = kvm_set_ioeventfd_mmio_long(fd, section->offset_within_address_space,
+                                    data, true);
+    if (r < 0) {
+        abort();
+    }
+}
+
+static void kvm_mem_ioeventfd_del(MemoryRegionSection *section,
+                                  bool match_data, uint64_t data, int fd)
+{
+    int r;
+
+    r = kvm_set_ioeventfd_mmio_long(fd, section->offset_within_address_space,
+                                    data, false);
+    if (r < 0) {
+        abort();
+    }
+}
+
+static void kvm_io_ioeventfd_add(MemoryRegionSection *section,
+                                 bool match_data, uint64_t data, int fd)
+{
+    int r;
+
+    assert(match_data && section->size == 2);
+
+    r = kvm_set_ioeventfd_pio_word(fd, section->offset_within_address_space,
+                                   data, true);
+    if (r < 0) {
+        abort();
+    }
+}
+
+static void kvm_io_ioeventfd_del(MemoryRegionSection *section,
+                                 bool match_data, uint64_t data, int fd)
+
+{
+    int r;
+
+    r = kvm_set_ioeventfd_pio_word(fd, section->offset_within_address_space,
+                                   data, false);
+    if (r < 0) {
+        abort();
+    }
+}
+
+static void kvm_eventfd_add(MemoryListener *listener,
+                            MemoryRegionSection *section,
+                            bool match_data, uint64_t data, int fd)
+{
+    if (section->address_space == get_system_memory()) {
+        kvm_mem_ioeventfd_add(section, match_data, data, fd);
+    } else {
+        kvm_io_ioeventfd_add(section, match_data, data, fd);
+    }
+}
+
+static void kvm_eventfd_del(MemoryListener *listener,
+                            MemoryRegionSection *section,
+                            bool match_data, uint64_t data, int fd)
+{
+    if (section->address_space == get_system_memory()) {
+        kvm_mem_ioeventfd_del(section, match_data, data, fd);
+    } else {
+        kvm_io_ioeventfd_del(section, match_data, data, fd);
+    }
+}
+
 static MemoryListener kvm_memory_listener = {
+    .begin = kvm_begin,
+    .commit = kvm_commit,
     .region_add = kvm_region_add,
     .region_del = kvm_region_del,
+    .region_nop = kvm_region_nop,
     .log_start = kvm_log_start,
     .log_stop = kvm_log_stop,
     .log_sync = kvm_log_sync,
     .log_global_start = kvm_log_global_start,
     .log_global_stop = kvm_log_global_stop,
+    .eventfd_add = kvm_eventfd_add,
+    .eventfd_del = kvm_eventfd_del,
+    .priority = 10,
 };
 
 static void kvm_handle_interrupt(CPUState *env, int mask)
@@ -974,7 +1069,7 @@ int kvm_init(void)
     }
 
     kvm_state = s;
-    memory_listener_register(&kvm_memory_listener);
+    memory_listener_register(&kvm_memory_listener, NULL);
 
     s->many_ioeventfds = kvm_check_many_ioeventfds();
 
@@ -1127,8 +1222,6 @@ int kvm_cpu_exec(CPUState *env)
         return EXCP_HLT;
     }
 
-    cpu_single_env = env;
-
     do {
         if (env->kvm_vcpu_dirty) {
             kvm_arch_put_registers(env, KVM_PUT_RUNTIME_STATE);
@@ -1145,13 +1238,11 @@ int kvm_cpu_exec(CPUState *env)
              */
             qemu_cpu_kick_self();
         }
-        cpu_single_env = NULL;
         qemu_mutex_unlock_iothread();
 
         run_ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
 
         qemu_mutex_lock_iothread();
-        cpu_single_env = env;
         kvm_arch_post_run(env, run);
 
         kvm_flush_coalesced_mmio_buffer();
@@ -1215,7 +1306,6 @@ int kvm_cpu_exec(CPUState *env)
     }
 
     env->exit_request = 0;
-    cpu_single_env = NULL;
     return ret;
 }
 
diff --git a/memory.c b/memory.c
index 22816e20f9..6565e2e696 100644
--- a/memory.c
+++ b/memory.c
@@ -27,8 +27,8 @@ unsigned memory_region_transaction_depth = 0;
 static bool memory_region_update_pending = false;
 static bool global_dirty_log = false;
 
-static QLIST_HEAD(, MemoryListener) memory_listeners
-    = QLIST_HEAD_INITIALIZER(memory_listeners);
+static QTAILQ_HEAD(memory_listeners, MemoryListener) memory_listeners
+    = QTAILQ_HEAD_INITIALIZER(memory_listeners);
 
 typedef struct AddrRange AddrRange;
 
@@ -82,6 +82,71 @@ static AddrRange addrrange_intersection(AddrRange r1, AddrRange r2)
     return addrrange_make(start, int128_sub(end, start));
 }
 
+enum ListenerDirection { Forward, Reverse };
+
+static bool memory_listener_match(MemoryListener *listener,
+                                  MemoryRegionSection *section)
+{
+    return !listener->address_space_filter
+        || listener->address_space_filter == section->address_space;
+}
+
+#define MEMORY_LISTENER_CALL_GLOBAL(_callback, _direction, _args...)    \
+    do {                                                                \
+        MemoryListener *_listener;                                      \
+                                                                        \
+        switch (_direction) {                                           \
+        case Forward:                                                   \
+            QTAILQ_FOREACH(_listener, &memory_listeners, link) {        \
+                _listener->_callback(_listener, ##_args);               \
+            }                                                           \
+            break;                                                      \
+        case Reverse:                                                   \
+            QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners,        \
+                                   memory_listeners, link) {            \
+                _listener->_callback(_listener, ##_args);               \
+            }                                                           \
+            break;                                                      \
+        default:                                                        \
+            abort();                                                    \
+        }                                                               \
+    } while (0)
+
+#define MEMORY_LISTENER_CALL(_callback, _direction, _section, _args...) \
+    do {                                                                \
+        MemoryListener *_listener;                                      \
+                                                                        \
+        switch (_direction) {                                           \
+        case Forward:                                                   \
+            QTAILQ_FOREACH(_listener, &memory_listeners, link) {        \
+                if (memory_listener_match(_listener, _section)) {       \
+                    _listener->_callback(_listener, _section, ##_args); \
+                }                                                       \
+            }                                                           \
+            break;                                                      \
+        case Reverse:                                                   \
+            QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners,        \
+                                   memory_listeners, link) {            \
+                if (memory_listener_match(_listener, _section)) {       \
+                    _listener->_callback(_listener, _section, ##_args); \
+                }                                                       \
+            }                                                           \
+            break;                                                      \
+        default:                                                        \
+            abort();                                                    \
+        }                                                               \
+    } while (0)
+
+#define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback)            \
+    MEMORY_LISTENER_CALL(callback, dir, (&(MemoryRegionSection) {       \
+        .mr = (fr)->mr,                                                 \
+        .address_space = (as)->root,                                    \
+        .offset_within_region = (fr)->offset_in_region,                 \
+        .size = int128_get64((fr)->addr.size),                          \
+        .offset_within_address_space = int128_get64((fr)->addr.start),  \
+        .readonly = (fr)->readonly,                                     \
+              }))
+
 struct CoalescedMemoryRange {
     AddrRange addr;
     QTAILQ_ENTRY(CoalescedMemoryRange) link;
@@ -158,22 +223,12 @@ typedef struct AddressSpaceOps AddressSpaceOps;
 
 /* A system address space - I/O, memory, etc. */
 struct AddressSpace {
-    const AddressSpaceOps *ops;
     MemoryRegion *root;
     FlatView current_map;
     int ioeventfd_nb;
     MemoryRegionIoeventfd *ioeventfds;
 };
 
-struct AddressSpaceOps {
-    void (*range_add)(AddressSpace *as, FlatRange *fr);
-    void (*range_del)(AddressSpace *as, FlatRange *fr);
-    void (*log_start)(AddressSpace *as, FlatRange *fr);
-    void (*log_stop)(AddressSpace *as, FlatRange *fr);
-    void (*ioeventfd_add)(AddressSpace *as, MemoryRegionIoeventfd *fd);
-    void (*ioeventfd_del)(AddressSpace *as, MemoryRegionIoeventfd *fd);
-};
-
 #define FOR_EACH_FLAT_RANGE(var, view)          \
     for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var)
 
@@ -305,74 +360,7 @@ static void access_with_adjusted_size(target_phys_addr_t addr,
     }
 }
 
-static void as_memory_range_add(AddressSpace *as, FlatRange *fr)
-{
-    MemoryRegionSection section = {
-        .mr = fr->mr,
-        .offset_within_address_space = int128_get64(fr->addr.start),
-        .offset_within_region = fr->offset_in_region,
-        .size = int128_get64(fr->addr.size),
-    };
-
-    cpu_register_physical_memory_log(&section, fr->readable, fr->readonly);
-}
-
-static void as_memory_range_del(AddressSpace *as, FlatRange *fr)
-{
-    MemoryRegionSection section = {
-        .mr = &io_mem_unassigned,
-        .offset_within_address_space = int128_get64(fr->addr.start),
-        .offset_within_region = int128_get64(fr->addr.start),
-        .size = int128_get64(fr->addr.size),
-    };
-
-    cpu_register_physical_memory_log(&section, true, false);
-}
-
-static void as_memory_log_start(AddressSpace *as, FlatRange *fr)
-{
-}
-
-static void as_memory_log_stop(AddressSpace *as, FlatRange *fr)
-{
-}
-
-static void as_memory_ioeventfd_add(AddressSpace *as, MemoryRegionIoeventfd *fd)
-{
-    int r;
-
-    assert(fd->match_data && int128_get64(fd->addr.size) == 4);
-
-    r = kvm_set_ioeventfd_mmio_long(fd->fd, int128_get64(fd->addr.start),
-                                    fd->data, true);
-    if (r < 0) {
-        abort();
-    }
-}
-
-static void as_memory_ioeventfd_del(AddressSpace *as, MemoryRegionIoeventfd *fd)
-{
-    int r;
-
-    r = kvm_set_ioeventfd_mmio_long(fd->fd, int128_get64(fd->addr.start),
-                                    fd->data, false);
-    if (r < 0) {
-        abort();
-    }
-}
-
-static const AddressSpaceOps address_space_ops_memory = {
-    .range_add = as_memory_range_add,
-    .range_del = as_memory_range_del,
-    .log_start = as_memory_log_start,
-    .log_stop = as_memory_log_stop,
-    .ioeventfd_add = as_memory_ioeventfd_add,
-    .ioeventfd_del = as_memory_ioeventfd_del,
-};
-
-static AddressSpace address_space_memory = {
-    .ops = &address_space_ops_memory,
-};
+static AddressSpace address_space_memory;
 
 static const MemoryRegionPortio *find_portio(MemoryRegion *mr, uint64_t offset,
                                              unsigned width, bool write)
@@ -401,17 +389,17 @@ static void memory_region_iorange_read(IORange *iorange,
 
         *data = ((uint64_t)1 << (width * 8)) - 1;
         if (mrp) {
-            *data = mrp->read(mr->opaque, offset + mr->offset);
+            *data = mrp->read(mr->opaque, offset);
         } else if (width == 2) {
             mrp = find_portio(mr, offset, 1, false);
             assert(mrp);
-            *data = mrp->read(mr->opaque, offset + mr->offset) |
-                    (mrp->read(mr->opaque, offset + mr->offset + 1) << 8);
+            *data = mrp->read(mr->opaque, offset) |
+                    (mrp->read(mr->opaque, offset + 1) << 8);
         }
         return;
     }
     *data = 0;
-    access_with_adjusted_size(offset + mr->offset, data, width,
+    access_with_adjusted_size(offset, data, width,
                               mr->ops->impl.min_access_size,
                               mr->ops->impl.max_access_size,
                               memory_region_read_accessor, mr);
@@ -428,73 +416,27 @@ static void memory_region_iorange_write(IORange *iorange,
         const MemoryRegionPortio *mrp = find_portio(mr, offset, width, true);
 
         if (mrp) {
-            mrp->write(mr->opaque, offset + mr->offset, data);
+            mrp->write(mr->opaque, offset, data);
         } else if (width == 2) {
             mrp = find_portio(mr, offset, 1, false);
             assert(mrp);
-            mrp->write(mr->opaque, offset + mr->offset, data & 0xff);
-            mrp->write(mr->opaque, offset + mr->offset + 1, data >> 8);
+            mrp->write(mr->opaque, offset, data & 0xff);
+            mrp->write(mr->opaque, offset + 1, data >> 8);
         }
         return;
     }
-    access_with_adjusted_size(offset + mr->offset, &data, width,
+    access_with_adjusted_size(offset, &data, width,
                               mr->ops->impl.min_access_size,
                               mr->ops->impl.max_access_size,
                               memory_region_write_accessor, mr);
 }
 
-static const IORangeOps memory_region_iorange_ops = {
+const IORangeOps memory_region_iorange_ops = {
     .read = memory_region_iorange_read,
     .write = memory_region_iorange_write,
 };
 
-static void as_io_range_add(AddressSpace *as, FlatRange *fr)
-{
-    iorange_init(&fr->mr->iorange, &memory_region_iorange_ops,
-                 int128_get64(fr->addr.start), int128_get64(fr->addr.size));
-    ioport_register(&fr->mr->iorange);
-}
-
-static void as_io_range_del(AddressSpace *as, FlatRange *fr)
-{
-    isa_unassign_ioport(int128_get64(fr->addr.start),
-                        int128_get64(fr->addr.size));
-}
-
-static void as_io_ioeventfd_add(AddressSpace *as, MemoryRegionIoeventfd *fd)
-{
-    int r;
-
-    assert(fd->match_data && int128_get64(fd->addr.size) == 2);
-
-    r = kvm_set_ioeventfd_pio_word(fd->fd, int128_get64(fd->addr.start),
-                                   fd->data, true);
-    if (r < 0) {
-        abort();
-    }
-}
-
-static void as_io_ioeventfd_del(AddressSpace *as, MemoryRegionIoeventfd *fd)
-{
-    int r;
-
-    r = kvm_set_ioeventfd_pio_word(fd->fd, int128_get64(fd->addr.start),
-                                   fd->data, false);
-    if (r < 0) {
-        abort();
-    }
-}
-
-static const AddressSpaceOps address_space_ops_io = {
-    .range_add = as_io_range_add,
-    .range_del = as_io_range_del,
-    .ioeventfd_add = as_io_ioeventfd_add,
-    .ioeventfd_del = as_io_ioeventfd_del,
-};
-
-static AddressSpace address_space_io = {
-    .ops = &address_space_ops_io,
-};
+static AddressSpace address_space_io;
 
 static AddressSpace *memory_region_to_address_space(MemoryRegion *mr)
 {
@@ -621,6 +563,8 @@ static void address_space_add_del_ioeventfds(AddressSpace *as,
                                              unsigned fds_old_nb)
 {
     unsigned iold, inew;
+    MemoryRegionIoeventfd *fd;
+    MemoryRegionSection section;
 
     /* Generate a symmetric difference of the old and new fd sets, adding
      * and deleting as necessary.
@@ -632,13 +576,27 @@ static void address_space_add_del_ioeventfds(AddressSpace *as,
             && (inew == fds_new_nb
                 || memory_region_ioeventfd_before(fds_old[iold],
                                                   fds_new[inew]))) {
-            as->ops->ioeventfd_del(as, &fds_old[iold]);
+            fd = &fds_old[iold];
+            section = (MemoryRegionSection) {
+                .address_space = as->root,
+                .offset_within_address_space = int128_get64(fd->addr.start),
+                .size = int128_get64(fd->addr.size),
+            };
+            MEMORY_LISTENER_CALL(eventfd_del, Forward, &section,
+                                 fd->match_data, fd->data, fd->fd);
             ++iold;
         } else if (inew < fds_new_nb
                    && (iold == fds_old_nb
                        || memory_region_ioeventfd_before(fds_new[inew],
                                                          fds_old[iold]))) {
-            as->ops->ioeventfd_add(as, &fds_new[inew]);
+            fd = &fds_new[inew];
+            section = (MemoryRegionSection) {
+                .address_space = as->root,
+                .offset_within_address_space = int128_get64(fd->addr.start),
+                .size = int128_get64(fd->addr.size),
+            };
+            MEMORY_LISTENER_CALL(eventfd_add, Reverse, &section,
+                                 fd->match_data, fd->data, fd->fd);
             ++inew;
         } else {
             ++iold;
@@ -678,32 +636,6 @@ static void address_space_update_ioeventfds(AddressSpace *as)
     as->ioeventfd_nb = ioeventfd_nb;
 }
 
-typedef void ListenerCallback(MemoryListener *listener,
-                              MemoryRegionSection *mrs);
-
-/* Want "void (&MemoryListener::*callback)(const MemoryRegionSection& s)" */
-static void memory_listener_update_region(FlatRange *fr, AddressSpace *as,
-                                          size_t callback_offset)
-{
-    MemoryRegionSection section = {
-        .mr = fr->mr,
-        .address_space = as->root,
-        .offset_within_region = fr->offset_in_region,
-        .size = int128_get64(fr->addr.size),
-        .offset_within_address_space = int128_get64(fr->addr.start),
-    };
-    MemoryListener *listener;
-
-    QLIST_FOREACH(listener, &memory_listeners, link) {
-        ListenerCallback *callback
-            = *(ListenerCallback **)((void *)listener + callback_offset);
-        callback(listener, &section);
-    }
-}
-
-#define MEMORY_LISTENER_UPDATE_REGION(fr, as, callback) \
-    memory_listener_update_region(fr, as, offsetof(MemoryListener, callback))
-
 static void address_space_update_topology_pass(AddressSpace *as,
                                                FlatView old_view,
                                                FlatView new_view,
@@ -736,8 +668,7 @@ static void address_space_update_topology_pass(AddressSpace *as,
             /* In old, but (not in new, or in new but attributes changed). */
 
             if (!adding) {
-                MEMORY_LISTENER_UPDATE_REGION(frold, as, region_del);
-                as->ops->range_del(as, frold);
+                MEMORY_LISTENER_UPDATE_REGION(frold, as, Reverse, region_del);
             }
 
             ++iold;
@@ -745,12 +676,11 @@ static void address_space_update_topology_pass(AddressSpace *as,
             /* In both (logging may have changed) */
 
             if (adding) {
+                MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_nop);
                 if (frold->dirty_log_mask && !frnew->dirty_log_mask) {
-                    MEMORY_LISTENER_UPDATE_REGION(frnew, as, log_stop);
-                    as->ops->log_stop(as, frnew);
+                    MEMORY_LISTENER_UPDATE_REGION(frnew, as, Reverse, log_stop);
                 } else if (frnew->dirty_log_mask && !frold->dirty_log_mask) {
-                    as->ops->log_start(as, frnew);
-                    MEMORY_LISTENER_UPDATE_REGION(frnew, as, log_start);
+                    MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, log_start);
                 }
             }
 
@@ -760,8 +690,7 @@ static void address_space_update_topology_pass(AddressSpace *as,
             /* In new */
 
             if (adding) {
-                as->ops->range_add(as, frnew);
-                MEMORY_LISTENER_UPDATE_REGION(frnew, as, region_add);
+                MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_add);
             }
 
             ++inew;
@@ -794,6 +723,8 @@ static void memory_region_update_topology(MemoryRegion *mr)
         return;
     }
 
+    MEMORY_LISTENER_CALL_GLOBAL(begin, Forward);
+
     if (address_space_memory.root) {
         address_space_update_topology(&address_space_memory);
     }
@@ -801,6 +732,8 @@ static void memory_region_update_topology(MemoryRegion *mr)
         address_space_update_topology(&address_space_io);
     }
 
+    MEMORY_LISTENER_CALL_GLOBAL(commit, Forward);
+
     memory_region_update_pending = false;
 }
 
@@ -863,7 +796,6 @@ void memory_region_init(MemoryRegion *mr,
         mr->size = int128_2_64();
     }
     mr->addr = 0;
-    mr->offset = 0;
     mr->subpage = false;
     mr->enabled = true;
     mr->terminates = false;
@@ -925,7 +857,7 @@ static uint64_t memory_region_dispatch_read1(MemoryRegion *mr,
     }
 
     /* FIXME: support unaligned access */
-    access_with_adjusted_size(addr + mr->offset, &data, size,
+    access_with_adjusted_size(addr, &data, size,
                               mr->ops->impl.min_access_size,
                               mr->ops->impl.max_access_size,
                               memory_region_read_accessor, mr);
@@ -979,7 +911,7 @@ static void memory_region_dispatch_write(MemoryRegion *mr,
     }
 
     /* FIXME: support unaligned access */
-    access_with_adjusted_size(addr + mr->offset, &data, size,
+    access_with_adjusted_size(addr, &data, size,
                               mr->ops->impl.min_access_size,
                               mr->ops->impl.max_access_size,
                               memory_region_write_accessor, mr);
@@ -1122,11 +1054,6 @@ bool memory_region_is_rom(MemoryRegion *mr)
     return mr->ram && mr->readonly;
 }
 
-void memory_region_set_offset(MemoryRegion *mr, target_phys_addr_t offset)
-{
-    mr->offset = offset;
-}
-
 void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client)
 {
     uint8_t mask = 1 << client;
@@ -1156,7 +1083,8 @@ void memory_region_sync_dirty_bitmap(MemoryRegion *mr)
 
     FOR_EACH_FLAT_RANGE(fr, &address_space_memory.current_map) {
         if (fr->mr == mr) {
-            MEMORY_LISTENER_UPDATE_REGION(fr, &address_space_memory, log_sync);
+            MEMORY_LISTENER_UPDATE_REGION(fr, &address_space_memory,
+                                          Forward, log_sync);
         }
     }
 }
@@ -1474,6 +1402,7 @@ MemoryRegionSection memory_region_find(MemoryRegion *address_space,
                                                         fr->addr.start));
     ret.size = int128_get64(range.size);
     ret.offset_within_address_space = int128_get64(range.start);
+    ret.readonly = fr->readonly;
     return ret;
 }
 
@@ -1483,30 +1412,20 @@ void memory_global_sync_dirty_bitmap(MemoryRegion *address_space)
     FlatRange *fr;
 
     FOR_EACH_FLAT_RANGE(fr, &as->current_map) {
-        MEMORY_LISTENER_UPDATE_REGION(fr, as, log_sync);
+        MEMORY_LISTENER_UPDATE_REGION(fr, as, Forward, log_sync);
     }
 }
 
 void memory_global_dirty_log_start(void)
 {
-    MemoryListener *listener;
-
-    cpu_physical_memory_set_dirty_tracking(1);
     global_dirty_log = true;
-    QLIST_FOREACH(listener, &memory_listeners, link) {
-        listener->log_global_start(listener);
-    }
+    MEMORY_LISTENER_CALL_GLOBAL(log_global_start, Forward);
 }
 
 void memory_global_dirty_log_stop(void)
 {
-    MemoryListener *listener;
-
     global_dirty_log = false;
-    QLIST_FOREACH(listener, &memory_listeners, link) {
-        listener->log_global_stop(listener);
-    }
-    cpu_physical_memory_set_dirty_tracking(0);
+    MEMORY_LISTENER_CALL_GLOBAL(log_global_stop, Reverse);
 }
 
 static void listener_add_address_space(MemoryListener *listener,
@@ -1524,21 +1443,36 @@ static void listener_add_address_space(MemoryListener *listener,
             .offset_within_region = fr->offset_in_region,
             .size = int128_get64(fr->addr.size),
             .offset_within_address_space = int128_get64(fr->addr.start),
+            .readonly = fr->readonly,
         };
         listener->region_add(listener, &section);
     }
 }
 
-void memory_listener_register(MemoryListener *listener)
+void memory_listener_register(MemoryListener *listener, MemoryRegion *filter)
 {
-    QLIST_INSERT_HEAD(&memory_listeners, listener, link);
+    MemoryListener *other = NULL;
+
+    listener->address_space_filter = filter;
+    if (QTAILQ_EMPTY(&memory_listeners)
+        || listener->priority >= QTAILQ_LAST(&memory_listeners,
+                                             memory_listeners)->priority) {
+        QTAILQ_INSERT_TAIL(&memory_listeners, listener, link);
+    } else {
+        QTAILQ_FOREACH(other, &memory_listeners, link) {
+            if (listener->priority < other->priority) {
+                break;
+            }
+        }
+        QTAILQ_INSERT_BEFORE(other, listener, link);
+    }
     listener_add_address_space(listener, &address_space_memory);
     listener_add_address_space(listener, &address_space_io);
 }
 
 void memory_listener_unregister(MemoryListener *listener)
 {
-    QLIST_REMOVE(listener, link);
+    QTAILQ_REMOVE(&memory_listeners, listener, link);
 }
 
 void set_system_memory_map(MemoryRegion *mr)
diff --git a/memory.h b/memory.h
index 4cf8d2f27a..b7bccd1968 100644
--- a/memory.h
+++ b/memory.h
@@ -115,7 +115,6 @@ struct MemoryRegion {
     MemoryRegion *parent;
     Int128 size;
     target_phys_addr_t addr;
-    target_phys_addr_t offset;
     void (*destructor)(MemoryRegion *mr);
     ram_addr_t ram_addr;
     IORange iorange;
@@ -161,6 +160,7 @@ typedef struct MemoryRegionSection MemoryRegionSection;
  * @size: the size of the section; will not exceed @mr's boundaries
  * @offset_within_address_space: the address of the first byte of the section
  *     relative to the region's address space
+ * @readonly: writes to this section are ignored
  */
 struct MemoryRegionSection {
     MemoryRegion *mr;
@@ -168,6 +168,7 @@ struct MemoryRegionSection {
     target_phys_addr_t offset_within_region;
     uint64_t size;
     target_phys_addr_t offset_within_address_space;
+    bool readonly;
 };
 
 typedef struct MemoryListener MemoryListener;
@@ -179,14 +180,24 @@ typedef struct MemoryListener MemoryListener;
  * Use with memory_listener_register() and memory_listener_unregister().
  */
 struct MemoryListener {
+    void (*begin)(MemoryListener *listener);
+    void (*commit)(MemoryListener *listener);
     void (*region_add)(MemoryListener *listener, MemoryRegionSection *section);
     void (*region_del)(MemoryListener *listener, MemoryRegionSection *section);
+    void (*region_nop)(MemoryListener *listener, MemoryRegionSection *section);
     void (*log_start)(MemoryListener *listener, MemoryRegionSection *section);
     void (*log_stop)(MemoryListener *listener, MemoryRegionSection *section);
     void (*log_sync)(MemoryListener *listener, MemoryRegionSection *section);
     void (*log_global_start)(MemoryListener *listener);
     void (*log_global_stop)(MemoryListener *listener);
-    QLIST_ENTRY(MemoryListener) link;
+    void (*eventfd_add)(MemoryListener *listener, MemoryRegionSection *section,
+                        bool match_data, uint64_t data, int fd);
+    void (*eventfd_del)(MemoryListener *listener, MemoryRegionSection *section,
+                        bool match_data, uint64_t data, int fd);
+    /* Lower = earlier (during add), later (during del) */
+    unsigned priority;
+    MemoryRegion *address_space_filter;
+    QTAILQ_ENTRY(MemoryListener) link;
 };
 
 /**
@@ -359,14 +370,6 @@ bool memory_region_is_rom(MemoryRegion *mr);
 void *memory_region_get_ram_ptr(MemoryRegion *mr);
 
 /**
- * memory_region_set_offset: Sets an offset to be added to MemoryRegionOps
- *                           callbacks.
- *
- * This function is deprecated and should not be used in new code.
- */
-void memory_region_set_offset(MemoryRegion *mr, target_phys_addr_t offset);
-
-/**
  * memory_region_set_log: Turn dirty logging on or off for a region.
  *
  * Turns dirty logging on or off for a specified client (display, migration).
@@ -686,8 +689,9 @@ void memory_region_transaction_commit(void);
  *                           space
  *
  * @listener: an object containing the callbacks to be called
+ * @filter: if non-%NULL, only regions in this address space will be observed
  */
-void memory_listener_register(MemoryListener *listener);
+void memory_listener_register(MemoryListener *listener, MemoryRegion *filter);
 
 /**
  * memory_listener_unregister: undo the effect of memory_listener_register()
diff --git a/pc-bios/kvmvapic.bin b/pc-bios/kvmvapic.bin
new file mode 100755
index 0000000000..045f5c2884
--- /dev/null
+++ b/pc-bios/kvmvapic.bin
Binary files differdiff --git a/pc-bios/optionrom/Makefile b/pc-bios/optionrom/Makefile
index 2caf7e6b69..f6b402713b 100644
--- a/pc-bios/optionrom/Makefile
+++ b/pc-bios/optionrom/Makefile
@@ -14,7 +14,7 @@ CFLAGS += -I$(SRC_PATH)
 CFLAGS += $(call cc-option, $(CFLAGS), -fno-stack-protector)
 QEMU_CFLAGS = $(CFLAGS)
 
-build-all: multiboot.bin linuxboot.bin
+build-all: multiboot.bin linuxboot.bin kvmvapic.bin
 
 # suppress auto-removal of intermediate files
 .SECONDARY:
diff --git a/pc-bios/optionrom/kvmvapic.S b/pc-bios/optionrom/kvmvapic.S
new file mode 100644
index 0000000000..aa17a402df
--- /dev/null
+++ b/pc-bios/optionrom/kvmvapic.S
@@ -0,0 +1,335 @@
+#
+# Local APIC acceleration for Windows XP and related guests
+#
+# Copyright 2011 Red Hat, Inc. and/or its affiliates
+#
+# Author: Avi Kivity <avi@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2, or (at your
+# option) any later version.  See the COPYING file in the top-level directory.
+#
+
+#include "optionrom.h"
+
+OPTION_ROM_START
+
+	# clear vapic area: firmware load using rep insb may cause
+	# stale tpr/isr/irr data to corrupt the vapic area.
+	push %es
+	push %cs
+	pop %es
+	xor %ax, %ax
+	mov $vapic_size/2, %cx
+	lea vapic, %di
+	cld
+	rep stosw
+	pop %es
+
+	# announce presence to the hypervisor
+	mov $vapic_base, %ax
+	out %ax, $0x7e
+
+	lret
+
+	.code32
+vapic_size = 2*4096
+
+.macro fixup delta=-4
+777:
+	.text 1
+	.long 777b + \delta  - vapic_base
+	.text 0
+.endm
+
+.macro reenable_vtpr
+	out %al, $0x7e
+.endm
+
+.text 1
+	fixup_start = .
+.text 0
+
+.align 16
+
+vapic_base:
+	.ascii "kvm aPiC"
+
+	/* relocation data */
+	.long vapic_base	; fixup
+	.long fixup_start	; fixup
+	.long fixup_end		; fixup
+
+	.long vapic		; fixup
+	.long vapic_size
+vcpu_shift:
+	.long 0
+real_tpr:
+	.long 0
+	.long up_set_tpr	; fixup
+	.long up_set_tpr_eax	; fixup
+	.long up_get_tpr_eax	; fixup
+	.long up_get_tpr_ecx	; fixup
+	.long up_get_tpr_edx	; fixup
+	.long up_get_tpr_ebx	; fixup
+	.long 0 /* esp. won't work. */
+	.long up_get_tpr_ebp	; fixup
+	.long up_get_tpr_esi	; fixup
+	.long up_get_tpr_edi	; fixup
+	.long up_get_tpr_stack  ; fixup
+	.long mp_set_tpr	; fixup
+	.long mp_set_tpr_eax	; fixup
+	.long mp_get_tpr_eax	; fixup
+	.long mp_get_tpr_ecx	; fixup
+	.long mp_get_tpr_edx	; fixup
+	.long mp_get_tpr_ebx	; fixup
+	.long 0 /* esp. won't work. */
+	.long mp_get_tpr_ebp	; fixup
+	.long mp_get_tpr_esi	; fixup
+	.long mp_get_tpr_edi	; fixup
+	.long mp_get_tpr_stack  ; fixup
+
+.macro kvm_hypercall
+	.byte 0x0f, 0x01, 0xc1
+.endm
+
+kvm_hypercall_vapic_poll_irq = 1
+
+pcr_cpu = 0x51
+
+.align 64
+
+mp_get_tpr_eax:
+	pushf
+	cli
+	reenable_vtpr
+	push %ecx
+
+	fs/movzbl pcr_cpu, %eax
+
+	mov vcpu_shift, %ecx	; fixup
+	shl %cl, %eax
+	testb $1, vapic+4(%eax)	; fixup delta=-5
+	jz mp_get_tpr_bad
+	movzbl vapic(%eax), %eax ; fixup
+
+mp_get_tpr_out:
+	pop %ecx
+	popf
+	ret
+
+mp_get_tpr_bad:
+	mov real_tpr, %eax	; fixup
+	mov (%eax), %eax
+	jmp mp_get_tpr_out
+
+mp_get_tpr_ebx:
+	mov %eax, %ebx
+	call mp_get_tpr_eax
+	xchg %eax, %ebx
+	ret
+
+mp_get_tpr_ecx:
+	mov %eax, %ecx
+	call mp_get_tpr_eax
+	xchg %eax, %ecx
+	ret
+
+mp_get_tpr_edx:
+	mov %eax, %edx
+	call mp_get_tpr_eax
+	xchg %eax, %edx
+	ret
+
+mp_get_tpr_esi:
+	mov %eax, %esi
+	call mp_get_tpr_eax
+	xchg %eax, %esi
+	ret
+
+mp_get_tpr_edi:
+	mov %eax, %edi
+	call mp_get_tpr_edi
+	xchg %eax, %edi
+	ret
+
+mp_get_tpr_ebp:
+	mov %eax, %ebp
+	call mp_get_tpr_eax
+	xchg %eax, %ebp
+	ret
+
+mp_get_tpr_stack:
+	call mp_get_tpr_eax
+	xchg %eax, 4(%esp)
+	ret
+
+mp_set_tpr_eax:
+	push %eax
+	call mp_set_tpr
+	ret
+
+mp_set_tpr:
+	pushf
+	push %eax
+	push %ecx
+	push %edx
+	push %ebx
+	cli
+	reenable_vtpr
+
+mp_set_tpr_failed:
+	fs/movzbl pcr_cpu, %edx
+
+	mov vcpu_shift, %ecx	; fixup
+	shl %cl, %edx
+
+	testb $1, vapic+4(%edx)	; fixup delta=-5
+	jz mp_set_tpr_bad
+
+	mov vapic(%edx), %eax	; fixup
+
+	mov %eax, %ebx
+	mov 24(%esp), %bl
+
+	/* %ebx = new vapic (%bl = tpr, %bh = isr, %b3 = irr) */
+
+	lock cmpxchg %ebx, vapic(%edx) ; fixup
+	jnz mp_set_tpr_failed
+
+	/* compute ppr */
+	cmp %bh, %bl
+	jae mp_tpr_is_bigger
+mp_isr_is_bigger:
+	mov %bh, %bl
+mp_tpr_is_bigger:
+	/* %bl = ppr */
+	rol $8, %ebx
+	/* now: %bl = irr, %bh = ppr */
+	cmp %bh, %bl
+	ja mp_set_tpr_poll_irq
+
+mp_set_tpr_out:
+	pop %ebx
+	pop %edx
+	pop %ecx
+	pop %eax
+	popf
+	ret $4
+
+mp_set_tpr_poll_irq:
+	mov $kvm_hypercall_vapic_poll_irq, %eax
+	kvm_hypercall
+	jmp mp_set_tpr_out
+
+mp_set_tpr_bad:
+	mov 24(%esp), %ecx
+	mov real_tpr, %eax	; fixup
+	mov %ecx, (%eax)
+	jmp mp_set_tpr_out
+
+up_get_tpr_eax:
+	reenable_vtpr
+	movzbl vapic, %eax ; fixup
+	ret
+
+up_get_tpr_ebx:
+	reenable_vtpr
+	movzbl vapic, %ebx ; fixup
+	ret
+
+up_get_tpr_ecx:
+	reenable_vtpr
+	movzbl vapic, %ecx ; fixup
+	ret
+
+up_get_tpr_edx:
+	reenable_vtpr
+	movzbl vapic, %edx ; fixup
+	ret
+
+up_get_tpr_esi:
+	reenable_vtpr
+	movzbl vapic, %esi ; fixup
+	ret
+
+up_get_tpr_edi:
+	reenable_vtpr
+	movzbl vapic, %edi ; fixup
+	ret
+
+up_get_tpr_ebp:
+	reenable_vtpr
+	movzbl vapic, %ebp ; fixup
+	ret
+
+up_get_tpr_stack:
+	reenable_vtpr
+	movzbl vapic, %eax ; fixup
+	xchg %eax, 4(%esp)
+	ret
+
+up_set_tpr_eax:
+	push %eax
+	call up_set_tpr
+	ret
+
+up_set_tpr:
+	pushf
+	push %eax
+	push %ebx
+	reenable_vtpr
+
+up_set_tpr_failed:
+	mov vapic, %eax	; fixup
+
+	mov %eax, %ebx
+	mov 16(%esp), %bl
+
+	/* %ebx = new vapic (%bl = tpr, %bh = isr, %b3 = irr) */
+
+	lock cmpxchg %ebx, vapic ; fixup
+	jnz up_set_tpr_failed
+
+	/* compute ppr */
+	cmp %bh, %bl
+	jae up_tpr_is_bigger
+up_isr_is_bigger:
+	mov %bh, %bl
+up_tpr_is_bigger:
+	/* %bl = ppr */
+	rol $8, %ebx
+	/* now: %bl = irr, %bh = ppr */
+	cmp %bh, %bl
+	ja up_set_tpr_poll_irq
+
+up_set_tpr_out:
+	pop %ebx
+	pop %eax
+	popf
+	ret $4
+
+up_set_tpr_poll_irq:
+	mov $kvm_hypercall_vapic_poll_irq, %eax
+	kvm_hypercall
+	jmp up_set_tpr_out
+
+.text 1
+	fixup_end = .
+.text 0
+
+/*
+ * vapic format:
+ *  per-vcpu records of size 2^vcpu shift.
+ *     byte 0: tpr (r/w)
+ *     byte 1: highest in-service interrupt (isr) (r/o); bits 3:0 are zero
+ *     byte 2: zero (r/o)
+ *     byte 3: highest pending interrupt (irr) (r/o)
+ */
+.text 2
+
+.align 128
+
+vapic:
+. = . + vapic_size
+
+OPTION_ROM_END
diff --git a/pc-bios/optionrom/optionrom.h b/pc-bios/optionrom/optionrom.h
index aa783deed1..3daf7da495 100644
--- a/pc-bios/optionrom/optionrom.h
+++ b/pc-bios/optionrom/optionrom.h
@@ -124,7 +124,8 @@
 	movw		%ax, %ds;
 
 #define OPTION_ROM_END					\
-    .align 512, 0;					\
+	.byte		0;				\
+	.align		512, 0;				\
     _end:
 
 #define BOOT_ROM_END					\
diff --git a/target-i386/cpu.h b/target-i386/cpu.h
index 37dde79581..196b0c5c40 100644
--- a/target-i386/cpu.h
+++ b/target-i386/cpu.h
@@ -482,6 +482,7 @@
 #define CPU_INTERRUPT_VIRQ      CPU_INTERRUPT_TGT_INT_0
 #define CPU_INTERRUPT_INIT      CPU_INTERRUPT_TGT_INT_1
 #define CPU_INTERRUPT_SIPI      CPU_INTERRUPT_TGT_INT_2
+#define CPU_INTERRUPT_TPR       CPU_INTERRUPT_TGT_INT_3
 
 
 enum {
@@ -613,6 +614,11 @@ typedef struct {
 
 #define NB_MMU_MODES 2
 
+typedef enum TPRAccess {
+    TPR_ACCESS_READ,
+    TPR_ACCESS_WRITE,
+} TPRAccess;
+
 typedef struct CPUX86State {
     /* standard registers */
     target_ulong regs[CPU_NB_REGS];
@@ -772,6 +778,8 @@ typedef struct CPUX86State {
     XMMReg ymmh_regs[CPU_NB_REGS];
 
     uint64_t xcr0;
+
+    TPRAccess tpr_access_type;
 } CPUX86State;
 
 CPUX86State *cpu_x86_init(const char *cpu_model);
@@ -1064,4 +1072,6 @@ void svm_check_intercept(CPUState *env1, uint32_t type);
 
 uint32_t cpu_cc_compute_all(CPUState *env1, int op);
 
+void cpu_report_tpr_access(CPUState *env, TPRAccess access);
+
 #endif /* CPU_I386_H */
diff --git a/target-i386/helper.c b/target-i386/helper.c
index 2586aff700..d12c9621bd 100644
--- a/target-i386/helper.c
+++ b/target-i386/helper.c
@@ -1189,6 +1189,22 @@ void cpu_x86_inject_mce(Monitor *mon, CPUState *cenv, int bank,
         }
     }
 }
+
+void cpu_report_tpr_access(CPUState *env, TPRAccess access)
+{
+    TranslationBlock *tb;
+
+    if (kvm_enabled()) {
+        env->tpr_access_type = access;
+
+        cpu_interrupt(env, CPU_INTERRUPT_TPR);
+    } else {
+        tb = tb_find_pc(env->mem_io_pc);
+        cpu_restore_state(tb, env, env->mem_io_pc);
+
+        apic_handle_tpr_access_report(env->apic_state, env->eip, access);
+    }
+}
 #endif /* !CONFIG_USER_ONLY */
 
 static void mce_init(CPUX86State *cenv)
diff --git a/target-i386/kvm.c b/target-i386/kvm.c
index 981192ddf8..9a732078f3 100644
--- a/target-i386/kvm.c
+++ b/target-i386/kvm.c
@@ -1635,8 +1635,10 @@ void kvm_arch_pre_run(CPUState *env, struct kvm_run *run)
     }
 
     if (!kvm_irqchip_in_kernel()) {
-        /* Force the VCPU out of its inner loop to process the INIT request */
-        if (env->interrupt_request & CPU_INTERRUPT_INIT) {
+        /* Force the VCPU out of its inner loop to process any INIT requests
+         * or pending TPR access reports. */
+        if (env->interrupt_request &
+            (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) {
             env->exit_request = 1;
         }
 
@@ -1730,6 +1732,12 @@ int kvm_arch_process_async_events(CPUState *env)
         kvm_cpu_synchronize_state(env);
         do_cpu_sipi(env);
     }
+    if (env->interrupt_request & CPU_INTERRUPT_TPR) {
+        env->interrupt_request &= ~CPU_INTERRUPT_TPR;
+        kvm_cpu_synchronize_state(env);
+        apic_handle_tpr_access_report(env->apic_state, env->eip,
+                                      env->tpr_access_type);
+    }
 
     return env->halted;
 }
@@ -1746,6 +1754,16 @@ static int kvm_handle_halt(CPUState *env)
     return 0;
 }
 
+static int kvm_handle_tpr_access(CPUState *env)
+{
+    struct kvm_run *run = env->kvm_run;
+
+    apic_handle_tpr_access_report(env->apic_state, run->tpr_access.rip,
+                                  run->tpr_access.is_write ? TPR_ACCESS_WRITE
+                                                           : TPR_ACCESS_READ);
+    return 1;
+}
+
 int kvm_arch_insert_sw_breakpoint(CPUState *env, struct kvm_sw_breakpoint *bp)
 {
     static const uint8_t int3 = 0xcc;
@@ -1950,6 +1968,9 @@ int kvm_arch_handle_exit(CPUState *env, struct kvm_run *run)
     case KVM_EXIT_SET_TPR:
         ret = 0;
         break;
+    case KVM_EXIT_TPR_ACCESS:
+        ret = kvm_handle_tpr_access(env);
+        break;
     case KVM_EXIT_FAIL_ENTRY:
         code = run->fail_entry.hardware_entry_failure_reason;
         fprintf(stderr, "KVM: entry failed, hardware error 0x%" PRIx64 "\n",
@@ -1987,6 +2008,7 @@ int kvm_arch_handle_exit(CPUState *env, struct kvm_run *run)
 
 bool kvm_arch_stop_on_emulation_error(CPUState *env)
 {
+    kvm_cpu_synchronize_state(env);
     return !(env->cr[0] & CR0_PE_MASK) ||
            ((env->segs[R_CS].selector  & 3) != 3);
 }
diff --git a/xen-all.c b/xen-all.c
index b0ed1ed193..6cef5062bc 100644
--- a/xen-all.c
+++ b/xen-all.c
@@ -392,6 +392,14 @@ static void xen_set_memory(struct MemoryListener *listener,
     }
 }
 
+static void xen_begin(MemoryListener *listener)
+{
+}
+
+static void xen_commit(MemoryListener *listener)
+{
+}
+
 static void xen_region_add(MemoryListener *listener,
                            MemoryRegionSection *section)
 {
@@ -404,6 +412,11 @@ static void xen_region_del(MemoryListener *listener,
     xen_set_memory(listener, section, false);
 }
 
+static void xen_region_nop(MemoryListener *listener,
+                           MemoryRegionSection *section)
+{
+}
+
 static void xen_sync_dirty_bitmap(XenIOState *state,
                                   target_phys_addr_t start_addr,
                                   ram_addr_t size)
@@ -485,14 +498,32 @@ static void xen_log_global_stop(MemoryListener *listener)
 {
 }
 
+static void xen_eventfd_add(MemoryListener *listener,
+                            MemoryRegionSection *section,
+                            bool match_data, uint64_t data, int fd)
+{
+}
+
+static void xen_eventfd_del(MemoryListener *listener,
+                            MemoryRegionSection *section,
+                            bool match_data, uint64_t data, int fd)
+{
+}
+
 static MemoryListener xen_memory_listener = {
+    .begin = xen_begin,
+    .commit = xen_commit,
     .region_add = xen_region_add,
     .region_del = xen_region_del,
+    .region_nop = xen_region_nop,
     .log_start = xen_log_start,
     .log_stop = xen_log_stop,
     .log_sync = xen_log_sync,
     .log_global_start = xen_log_global_start,
     .log_global_stop = xen_log_global_stop,
+    .eventfd_add = xen_eventfd_add,
+    .eventfd_del = xen_eventfd_del,
+    .priority = 10,
 };
 
 /* VCPU Operations, MMIO, IO ring ... */
@@ -975,7 +1006,7 @@ int xen_hvm_init(void)
 
     state->memory_listener = xen_memory_listener;
     QLIST_INIT(&state->physmap);
-    memory_listener_register(&state->memory_listener);
+    memory_listener_register(&state->memory_listener, get_system_memory());
     state->log_for_dirtybit = NULL;
 
     /* Initialize backend core & drivers */