diff options
Diffstat (limited to 'target-i386/helper.c')
| -rw-r--r-- | target-i386/helper.c | 396 |
1 files changed, 148 insertions, 248 deletions
diff --git a/target-i386/helper.c b/target-i386/helper.c index 46d20e4b89..11ca8649b5 100644 --- a/target-i386/helper.c +++ b/target-i386/helper.c @@ -510,14 +510,6 @@ int x86_cpu_handle_mmu_fault(CPUState *cs, vaddr addr, #else -/* XXX: This value should match the one returned by CPUID - * and in exec.c */ -# if defined(TARGET_X86_64) -# define PHYS_ADDR_MASK 0xfffffff000LL -# else -# define PHYS_ADDR_MASK 0xffffff000LL -# endif - /* return value: * -1 = cannot handle fault * 0 = nothing more to do @@ -530,10 +522,12 @@ int x86_cpu_handle_mmu_fault(CPUState *cs, vaddr addr, CPUX86State *env = &cpu->env; uint64_t ptep, pte; target_ulong pde_addr, pte_addr; - int error_code, is_dirty, prot, page_size, is_write, is_user; + int error_code = 0; + int is_dirty, prot, page_size, is_write, is_user; hwaddr paddr; + uint64_t rsvd_mask = PG_HI_RSVD_MASK; uint32_t page_offset; - target_ulong vaddr, virt_addr; + target_ulong vaddr; is_user = mmu_idx == MMU_USER_IDX; #if defined(DEBUG_MMU) @@ -550,12 +544,15 @@ int x86_cpu_handle_mmu_fault(CPUState *cs, vaddr addr, pte = (uint32_t)pte; } #endif - virt_addr = addr & TARGET_PAGE_MASK; prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; page_size = 4096; goto do_mapping; } + if (!(env->efer & MSR_EFER_NXE)) { + rsvd_mask |= PG_NX_MASK; + } + if (env->cr[4] & CR4_PAE_MASK) { uint64_t pde, pdpe; target_ulong pdpe_addr; @@ -577,34 +574,37 @@ int x86_cpu_handle_mmu_fault(CPUState *cs, vaddr addr, env->a20_mask; pml4e = ldq_phys(cs->as, pml4e_addr); if (!(pml4e & PG_PRESENT_MASK)) { - error_code = 0; goto do_fault; } - if (!(env->efer & MSR_EFER_NXE) && (pml4e & PG_NX_MASK)) { - error_code = PG_ERROR_RSVD_MASK; - goto do_fault; + if (pml4e & (rsvd_mask | PG_PSE_MASK)) { + goto do_fault_rsvd; } if (!(pml4e & PG_ACCESSED_MASK)) { pml4e |= PG_ACCESSED_MASK; stl_phys_notdirty(cs->as, pml4e_addr, pml4e); } ptep = pml4e ^ PG_NX_MASK; - pdpe_addr = ((pml4e & PHYS_ADDR_MASK) + (((addr >> 30) & 0x1ff) << 3)) & + pdpe_addr = ((pml4e & PG_ADDRESS_MASK) + (((addr >> 30) & 0x1ff) << 3)) & env->a20_mask; pdpe = ldq_phys(cs->as, pdpe_addr); if (!(pdpe & PG_PRESENT_MASK)) { - error_code = 0; goto do_fault; } - if (!(env->efer & MSR_EFER_NXE) && (pdpe & PG_NX_MASK)) { - error_code = PG_ERROR_RSVD_MASK; - goto do_fault; + if (pdpe & rsvd_mask) { + goto do_fault_rsvd; } ptep &= pdpe ^ PG_NX_MASK; if (!(pdpe & PG_ACCESSED_MASK)) { pdpe |= PG_ACCESSED_MASK; stl_phys_notdirty(cs->as, pdpe_addr, pdpe); } + if (pdpe & PG_PSE_MASK) { + /* 1 GB page */ + page_size = 1024 * 1024 * 1024; + pte_addr = pdpe_addr; + pte = pdpe; + goto do_check_protect; + } } else #endif { @@ -613,134 +613,49 @@ int x86_cpu_handle_mmu_fault(CPUState *cs, vaddr addr, env->a20_mask; pdpe = ldq_phys(cs->as, pdpe_addr); if (!(pdpe & PG_PRESENT_MASK)) { - error_code = 0; goto do_fault; } + rsvd_mask |= PG_HI_USER_MASK | PG_NX_MASK; + if (pdpe & rsvd_mask) { + goto do_fault_rsvd; + } ptep = PG_NX_MASK | PG_USER_MASK | PG_RW_MASK; } - pde_addr = ((pdpe & PHYS_ADDR_MASK) + (((addr >> 21) & 0x1ff) << 3)) & + pde_addr = ((pdpe & PG_ADDRESS_MASK) + (((addr >> 21) & 0x1ff) << 3)) & env->a20_mask; pde = ldq_phys(cs->as, pde_addr); if (!(pde & PG_PRESENT_MASK)) { - error_code = 0; goto do_fault; } - if (!(env->efer & MSR_EFER_NXE) && (pde & PG_NX_MASK)) { - error_code = PG_ERROR_RSVD_MASK; - goto do_fault; + if (pde & rsvd_mask) { + goto do_fault_rsvd; } ptep &= pde ^ PG_NX_MASK; if (pde & PG_PSE_MASK) { /* 2 MB page */ page_size = 2048 * 1024; - ptep ^= PG_NX_MASK; - if ((ptep & PG_NX_MASK) && is_write1 == 2) { - goto do_fault_protect; - } - switch (mmu_idx) { - case MMU_USER_IDX: - if (!(ptep & PG_USER_MASK)) { - goto do_fault_protect; - } - if (is_write && !(ptep & PG_RW_MASK)) { - goto do_fault_protect; - } - break; - - case MMU_KERNEL_IDX: - if (is_write1 != 2 && (env->cr[4] & CR4_SMAP_MASK) && - (ptep & PG_USER_MASK)) { - goto do_fault_protect; - } - /* fall through */ - case MMU_KSMAP_IDX: - if (is_write1 == 2 && (env->cr[4] & CR4_SMEP_MASK) && - (ptep & PG_USER_MASK)) { - goto do_fault_protect; - } - if ((env->cr[0] & CR0_WP_MASK) && - is_write && !(ptep & PG_RW_MASK)) { - goto do_fault_protect; - } - break; - - default: /* cannot happen */ - break; - } - is_dirty = is_write && !(pde & PG_DIRTY_MASK); - if (!(pde & PG_ACCESSED_MASK) || is_dirty) { - pde |= PG_ACCESSED_MASK; - if (is_dirty) - pde |= PG_DIRTY_MASK; - stl_phys_notdirty(cs->as, pde_addr, pde); - } - /* align to page_size */ - pte = pde & ((PHYS_ADDR_MASK & ~(page_size - 1)) | 0xfff); - virt_addr = addr & ~(page_size - 1); - } else { - /* 4 KB page */ - if (!(pde & PG_ACCESSED_MASK)) { - pde |= PG_ACCESSED_MASK; - stl_phys_notdirty(cs->as, pde_addr, pde); - } - pte_addr = ((pde & PHYS_ADDR_MASK) + (((addr >> 12) & 0x1ff) << 3)) & - env->a20_mask; - pte = ldq_phys(cs->as, pte_addr); - if (!(pte & PG_PRESENT_MASK)) { - error_code = 0; - goto do_fault; - } - if (!(env->efer & MSR_EFER_NXE) && (pte & PG_NX_MASK)) { - error_code = PG_ERROR_RSVD_MASK; - goto do_fault; - } - /* combine pde and pte nx, user and rw protections */ - ptep &= pte ^ PG_NX_MASK; - ptep ^= PG_NX_MASK; - if ((ptep & PG_NX_MASK) && is_write1 == 2) - goto do_fault_protect; - switch (mmu_idx) { - case MMU_USER_IDX: - if (!(ptep & PG_USER_MASK)) { - goto do_fault_protect; - } - if (is_write && !(ptep & PG_RW_MASK)) { - goto do_fault_protect; - } - break; - - case MMU_KERNEL_IDX: - if (is_write1 != 2 && (env->cr[4] & CR4_SMAP_MASK) && - (ptep & PG_USER_MASK)) { - goto do_fault_protect; - } - /* fall through */ - case MMU_KSMAP_IDX: - if (is_write1 == 2 && (env->cr[4] & CR4_SMEP_MASK) && - (ptep & PG_USER_MASK)) { - goto do_fault_protect; - } - if ((env->cr[0] & CR0_WP_MASK) && - is_write && !(ptep & PG_RW_MASK)) { - goto do_fault_protect; - } - break; - - default: /* cannot happen */ - break; - } - is_dirty = is_write && !(pte & PG_DIRTY_MASK); - if (!(pte & PG_ACCESSED_MASK) || is_dirty) { - pte |= PG_ACCESSED_MASK; - if (is_dirty) - pte |= PG_DIRTY_MASK; - stl_phys_notdirty(cs->as, pte_addr, pte); - } - page_size = 4096; - virt_addr = addr & ~0xfff; - pte = pte & (PHYS_ADDR_MASK | 0xfff); + pte_addr = pde_addr; + pte = pde; + goto do_check_protect; + } + /* 4 KB page */ + if (!(pde & PG_ACCESSED_MASK)) { + pde |= PG_ACCESSED_MASK; + stl_phys_notdirty(cs->as, pde_addr, pde); + } + pte_addr = ((pde & PG_ADDRESS_MASK) + (((addr >> 12) & 0x1ff) << 3)) & + env->a20_mask; + pte = ldq_phys(cs->as, pte_addr); + if (!(pte & PG_PRESENT_MASK)) { + goto do_fault; + } + if (pte & rsvd_mask) { + goto do_fault_rsvd; } + /* combine pde and pte nx, user and rw protections */ + ptep &= pte ^ PG_NX_MASK; + page_size = 4096; } else { uint32_t pde; @@ -749,114 +664,95 @@ int x86_cpu_handle_mmu_fault(CPUState *cs, vaddr addr, env->a20_mask; pde = ldl_phys(cs->as, pde_addr); if (!(pde & PG_PRESENT_MASK)) { - error_code = 0; goto do_fault; } + ptep = pde | PG_NX_MASK; + /* if PSE bit is set, then we use a 4MB page */ if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) { page_size = 4096 * 1024; - switch (mmu_idx) { - case MMU_USER_IDX: - if (!(pde & PG_USER_MASK)) { - goto do_fault_protect; - } - if (is_write && !(pde & PG_RW_MASK)) { - goto do_fault_protect; - } - break; - - case MMU_KERNEL_IDX: - if (is_write1 != 2 && (env->cr[4] & CR4_SMAP_MASK) && - (pde & PG_USER_MASK)) { - goto do_fault_protect; - } - /* fall through */ - case MMU_KSMAP_IDX: - if (is_write1 == 2 && (env->cr[4] & CR4_SMEP_MASK) && - (pde & PG_USER_MASK)) { - goto do_fault_protect; - } - if ((env->cr[0] & CR0_WP_MASK) && - is_write && !(pde & PG_RW_MASK)) { - goto do_fault_protect; - } - break; + pte_addr = pde_addr; + + /* Bits 20-13 provide bits 39-32 of the address, bit 21 is reserved. + * Leave bits 20-13 in place for setting accessed/dirty bits below. + */ + pte = pde | ((pde & 0x1fe000) << (32 - 13)); + rsvd_mask = 0x200000; + goto do_check_protect_pse36; + } - default: /* cannot happen */ - break; - } - is_dirty = is_write && !(pde & PG_DIRTY_MASK); - if (!(pde & PG_ACCESSED_MASK) || is_dirty) { - pde |= PG_ACCESSED_MASK; - if (is_dirty) - pde |= PG_DIRTY_MASK; - stl_phys_notdirty(cs->as, pde_addr, pde); - } + if (!(pde & PG_ACCESSED_MASK)) { + pde |= PG_ACCESSED_MASK; + stl_phys_notdirty(cs->as, pde_addr, pde); + } - pte = pde & ~( (page_size - 1) & ~0xfff); /* align to page_size */ - ptep = pte; - virt_addr = addr & ~(page_size - 1); - } else { - if (!(pde & PG_ACCESSED_MASK)) { - pde |= PG_ACCESSED_MASK; - stl_phys_notdirty(cs->as, pde_addr, pde); - } + /* page directory entry */ + pte_addr = ((pde & ~0xfff) + ((addr >> 10) & 0xffc)) & + env->a20_mask; + pte = ldl_phys(cs->as, pte_addr); + if (!(pte & PG_PRESENT_MASK)) { + goto do_fault; + } + /* combine pde and pte user and rw protections */ + ptep &= pte | PG_NX_MASK; + page_size = 4096; + rsvd_mask = 0; + } - /* page directory entry */ - pte_addr = ((pde & ~0xfff) + ((addr >> 10) & 0xffc)) & - env->a20_mask; - pte = ldl_phys(cs->as, pte_addr); - if (!(pte & PG_PRESENT_MASK)) { - error_code = 0; - goto do_fault; - } - /* combine pde and pte user and rw protections */ - ptep = pte & pde; - switch (mmu_idx) { - case MMU_USER_IDX: - if (!(ptep & PG_USER_MASK)) { - goto do_fault_protect; - } - if (is_write && !(ptep & PG_RW_MASK)) { - goto do_fault_protect; - } - break; +do_check_protect: + rsvd_mask |= (page_size - 1) & PG_ADDRESS_MASK & ~PG_PSE_PAT_MASK; +do_check_protect_pse36: + if (pte & rsvd_mask) { + goto do_fault_rsvd; + } + ptep ^= PG_NX_MASK; + if ((ptep & PG_NX_MASK) && is_write1 == 2) { + goto do_fault_protect; + } + switch (mmu_idx) { + case MMU_USER_IDX: + if (!(ptep & PG_USER_MASK)) { + goto do_fault_protect; + } + if (is_write && !(ptep & PG_RW_MASK)) { + goto do_fault_protect; + } + break; - case MMU_KERNEL_IDX: - if (is_write1 != 2 && (env->cr[4] & CR4_SMAP_MASK) && - (ptep & PG_USER_MASK)) { - goto do_fault_protect; - } - /* fall through */ - case MMU_KSMAP_IDX: - if (is_write1 == 2 && (env->cr[4] & CR4_SMEP_MASK) && - (ptep & PG_USER_MASK)) { - goto do_fault_protect; - } - if ((env->cr[0] & CR0_WP_MASK) && - is_write && !(ptep & PG_RW_MASK)) { - goto do_fault_protect; - } - break; + case MMU_KSMAP_IDX: + if (is_write1 != 2 && (ptep & PG_USER_MASK)) { + goto do_fault_protect; + } + /* fall through */ + case MMU_KNOSMAP_IDX: + if (is_write1 == 2 && (env->cr[4] & CR4_SMEP_MASK) && + (ptep & PG_USER_MASK)) { + goto do_fault_protect; + } + if ((env->cr[0] & CR0_WP_MASK) && + is_write && !(ptep & PG_RW_MASK)) { + goto do_fault_protect; + } + break; - default: /* cannot happen */ - break; - } - is_dirty = is_write && !(pte & PG_DIRTY_MASK); - if (!(pte & PG_ACCESSED_MASK) || is_dirty) { - pte |= PG_ACCESSED_MASK; - if (is_dirty) - pte |= PG_DIRTY_MASK; - stl_phys_notdirty(cs->as, pte_addr, pte); - } - page_size = 4096; - virt_addr = addr & ~0xfff; + default: /* cannot happen */ + break; + } + is_dirty = is_write && !(pte & PG_DIRTY_MASK); + if (!(pte & PG_ACCESSED_MASK) || is_dirty) { + pte |= PG_ACCESSED_MASK; + if (is_dirty) { + pte |= PG_DIRTY_MASK; } + stl_phys_notdirty(cs->as, pte_addr, pte); } + /* the page can be put in the TLB */ prot = PAGE_READ; - if (!(ptep & PG_NX_MASK)) + if (!(ptep & PG_NX_MASK) && + !((env->cr[4] & CR4_SMEP_MASK) && (ptep & PG_USER_MASK))) { prot |= PAGE_EXEC; + } if (pte & PG_DIRTY_MASK) { /* only set write access if already dirty... otherwise wait for dirty access */ @@ -872,16 +768,21 @@ int x86_cpu_handle_mmu_fault(CPUState *cs, vaddr addr, do_mapping: pte = pte & env->a20_mask; + /* align to page_size */ + pte &= PG_ADDRESS_MASK & ~(page_size - 1); + /* Even if 4MB pages, we map only one 4KB page in the cache to avoid filling it too fast */ - page_offset = (addr & TARGET_PAGE_MASK) & (page_size - 1); - paddr = (pte & TARGET_PAGE_MASK) + page_offset; - vaddr = virt_addr + page_offset; + vaddr = addr & TARGET_PAGE_MASK; + page_offset = vaddr & (page_size - 1); + paddr = pte + page_offset; tlb_set_page(cs, vaddr, paddr, prot, mmu_idx, page_size); return 0; + do_fault_rsvd: + error_code |= PG_ERROR_RSVD_MASK; do_fault_protect: - error_code = PG_ERROR_P_MASK; + error_code |= PG_ERROR_P_MASK; do_fault: error_code |= (is_write << PG_ERROR_W_BIT); if (is_user) @@ -910,7 +811,6 @@ hwaddr x86_cpu_get_phys_page_debug(CPUState *cs, vaddr addr) CPUX86State *env = &cpu->env; target_ulong pde_addr, pte_addr; uint64_t pte; - hwaddr paddr; uint32_t page_offset; int page_size; @@ -928,25 +828,24 @@ hwaddr x86_cpu_get_phys_page_debug(CPUState *cs, vaddr addr) /* test virtual address sign extension */ sext = (int64_t)addr >> 47; - if (sext != 0 && sext != -1) + if (sext != 0 && sext != -1) { return -1; - + } pml4e_addr = ((env->cr[3] & ~0xfff) + (((addr >> 39) & 0x1ff) << 3)) & env->a20_mask; pml4e = ldq_phys(cs->as, pml4e_addr); - if (!(pml4e & PG_PRESENT_MASK)) + if (!(pml4e & PG_PRESENT_MASK)) { return -1; - - pdpe_addr = ((pml4e & ~0xfff & ~(PG_NX_MASK | PG_HI_USER_MASK)) + + } + pdpe_addr = ((pml4e & PG_ADDRESS_MASK) + (((addr >> 30) & 0x1ff) << 3)) & env->a20_mask; pdpe = ldq_phys(cs->as, pdpe_addr); - if (!(pdpe & PG_PRESENT_MASK)) + if (!(pdpe & PG_PRESENT_MASK)) { return -1; - + } if (pdpe & PG_PSE_MASK) { page_size = 1024 * 1024 * 1024; - pte = pdpe & ~( (page_size - 1) & ~0xfff); - pte &= ~(PG_NX_MASK | PG_HI_USER_MASK); + pte = pdpe; goto out; } @@ -960,7 +859,7 @@ hwaddr x86_cpu_get_phys_page_debug(CPUState *cs, vaddr addr) return -1; } - pde_addr = ((pdpe & ~0xfff & ~(PG_NX_MASK | PG_HI_USER_MASK)) + + pde_addr = ((pdpe & PG_ADDRESS_MASK) + (((addr >> 21) & 0x1ff) << 3)) & env->a20_mask; pde = ldq_phys(cs->as, pde_addr); if (!(pde & PG_PRESENT_MASK)) { @@ -969,17 +868,17 @@ hwaddr x86_cpu_get_phys_page_debug(CPUState *cs, vaddr addr) if (pde & PG_PSE_MASK) { /* 2 MB page */ page_size = 2048 * 1024; - pte = pde & ~( (page_size - 1) & ~0xfff); /* align to page_size */ + pte = pde; } else { /* 4 KB page */ - pte_addr = ((pde & ~0xfff & ~(PG_NX_MASK | PG_HI_USER_MASK)) + + pte_addr = ((pde & PG_ADDRESS_MASK) + (((addr >> 12) & 0x1ff) << 3)) & env->a20_mask; page_size = 4096; pte = ldq_phys(cs->as, pte_addr); } - pte &= ~(PG_NX_MASK | PG_HI_USER_MASK); - if (!(pte & PG_PRESENT_MASK)) + if (!(pte & PG_PRESENT_MASK)) { return -1; + } } else { uint32_t pde; @@ -989,14 +888,15 @@ hwaddr x86_cpu_get_phys_page_debug(CPUState *cs, vaddr addr) if (!(pde & PG_PRESENT_MASK)) return -1; if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) { - pte = pde & ~0x003ff000; /* align to 4MB */ + pte = pde | ((pde & 0x1fe000) << (32 - 13)); page_size = 4096 * 1024; } else { /* page directory entry */ pte_addr = ((pde & ~0xfff) + ((addr >> 10) & 0xffc)) & env->a20_mask; pte = ldl_phys(cs->as, pte_addr); - if (!(pte & PG_PRESENT_MASK)) + if (!(pte & PG_PRESENT_MASK)) { return -1; + } page_size = 4096; } pte = pte & env->a20_mask; @@ -1005,9 +905,9 @@ hwaddr x86_cpu_get_phys_page_debug(CPUState *cs, vaddr addr) #ifdef TARGET_X86_64 out: #endif + pte &= PG_ADDRESS_MASK & ~(page_size - 1); page_offset = (addr & TARGET_PAGE_MASK) & (page_size - 1); - paddr = (pte & TARGET_PAGE_MASK) + page_offset; - return paddr; + return pte | page_offset; } void hw_breakpoint_insert(CPUX86State *env, int index) |