diff options
Diffstat (limited to 'linux-user/mmap.c')
| -rw-r--r-- | linux-user/mmap.c | 879 |
1 files changed, 580 insertions, 299 deletions
diff --git a/linux-user/mmap.c b/linux-user/mmap.c index 96c9433e27..4505fd7376 100644 --- a/linux-user/mmap.c +++ b/linux-user/mmap.c @@ -165,6 +165,7 @@ static int target_to_host_prot(int prot) /* NOTE: all the constants are the HOST ones, but addresses are target. */ int target_mprotect(abi_ulong start, abi_ulong len, int target_prot) { + int host_page_size = qemu_real_host_page_size(); abi_ulong starts[3]; abi_ulong lens[3]; int prots[3]; @@ -189,13 +190,13 @@ int target_mprotect(abi_ulong start, abi_ulong len, int target_prot) } last = start + len - 1; - host_start = start & qemu_host_page_mask; - host_last = HOST_PAGE_ALIGN(last) - 1; + host_start = start & -host_page_size; + host_last = ROUND_UP(last, host_page_size) - 1; nranges = 0; mmap_lock(); - if (host_last - host_start < qemu_host_page_size) { + if (host_last - host_start < host_page_size) { /* Single host page contains all guest pages: sum the prot. */ prot1 = target_prot; for (abi_ulong a = host_start; a < start; a += TARGET_PAGE_SIZE) { @@ -205,7 +206,7 @@ int target_mprotect(abi_ulong start, abi_ulong len, int target_prot) prot1 |= page_get_flags(a + 1); } starts[nranges] = host_start; - lens[nranges] = qemu_host_page_size; + lens[nranges] = host_page_size; prots[nranges] = prot1; nranges++; } else { @@ -218,10 +219,10 @@ int target_mprotect(abi_ulong start, abi_ulong len, int target_prot) /* If the resulting sum differs, create a new range. */ if (prot1 != target_prot) { starts[nranges] = host_start; - lens[nranges] = qemu_host_page_size; + lens[nranges] = host_page_size; prots[nranges] = prot1; nranges++; - host_start += qemu_host_page_size; + host_start += host_page_size; } } @@ -233,9 +234,9 @@ int target_mprotect(abi_ulong start, abi_ulong len, int target_prot) } /* If the resulting sum differs, create a new range. */ if (prot1 != target_prot) { - host_last -= qemu_host_page_size; + host_last -= host_page_size; starts[nranges] = host_last + 1; - lens[nranges] = qemu_host_page_size; + lens[nranges] = host_page_size; prots[nranges] = prot1; nranges++; } @@ -266,10 +267,35 @@ int target_mprotect(abi_ulong start, abi_ulong len, int target_prot) return ret; } -/* map an incomplete host page */ +/* + * Perform munmap on behalf of the target, with host parameters. + * If reserved_va, we must replace the memory reservation. + */ +static int do_munmap(void *addr, size_t len) +{ + if (reserved_va) { + void *ptr = mmap(addr, len, PROT_NONE, + MAP_FIXED | MAP_ANONYMOUS + | MAP_PRIVATE | MAP_NORESERVE, -1, 0); + return ptr == addr ? 0 : -1; + } + return munmap(addr, len); +} + +/* + * Map an incomplete host page. + * + * Here be dragons. This case will not work if there is an existing + * overlapping host page, which is file mapped, and for which the mapping + * is beyond the end of the file. In that case, we will see SIGBUS when + * trying to write a portion of this page. + * + * FIXME: Work around this with a temporary signal handler and longjmp. + */ static bool mmap_frag(abi_ulong real_start, abi_ulong start, abi_ulong last, int prot, int flags, int fd, off_t offset) { + int host_page_size = qemu_real_host_page_size(); abi_ulong real_last; void *host_start; int prot_old, prot_new; @@ -286,7 +312,7 @@ static bool mmap_frag(abi_ulong real_start, abi_ulong start, abi_ulong last, return false; } - real_last = real_start + qemu_host_page_size - 1; + real_last = real_start + host_page_size - 1; host_start = g2h_untagged(real_start); /* Get the protection of the target pages outside the mapping. */ @@ -304,12 +330,12 @@ static bool mmap_frag(abi_ulong real_start, abi_ulong start, abi_ulong last, * outside of the fragment we need to map. Allocate a new host * page to cover, discarding whatever else may have been present. */ - void *p = mmap(host_start, qemu_host_page_size, + void *p = mmap(host_start, host_page_size, target_to_host_prot(prot), flags | MAP_ANONYMOUS, -1, 0); if (p != host_start) { if (p != MAP_FAILED) { - munmap(p, qemu_host_page_size); + do_munmap(p, host_page_size); errno = EEXIST; } return false; @@ -324,7 +350,7 @@ static bool mmap_frag(abi_ulong real_start, abi_ulong start, abi_ulong last, /* Adjust protection to be able to write. */ if (!(host_prot_old & PROT_WRITE)) { host_prot_old |= PROT_WRITE; - mprotect(host_start, qemu_host_page_size, host_prot_old); + mprotect(host_start, host_page_size, host_prot_old); } /* Read or zero the new guest pages. */ @@ -338,7 +364,7 @@ static bool mmap_frag(abi_ulong real_start, abi_ulong start, abi_ulong last, /* Put final protection */ if (host_prot_new != host_prot_old) { - mprotect(host_start, qemu_host_page_size, host_prot_new); + mprotect(host_start, host_page_size, host_prot_new); } return true; } @@ -373,21 +399,21 @@ static abi_ulong mmap_find_vma_reserved(abi_ulong start, abi_ulong size, */ abi_ulong mmap_find_vma(abi_ulong start, abi_ulong size, abi_ulong align) { + int host_page_size = qemu_real_host_page_size(); void *ptr, *prev; abi_ulong addr; int wrapped, repeat; - align = MAX(align, qemu_host_page_size); + align = MAX(align, host_page_size); /* If 'start' == 0, then a default start address is used. */ if (start == 0) { start = mmap_next_start; } else { - start &= qemu_host_page_mask; + start &= -host_page_size; } start = ROUND_UP(start, align); - - size = HOST_PAGE_ALIGN(size); + size = ROUND_UP(size, host_page_size); if (reserved_va) { return mmap_find_vma_reserved(start, size, align); @@ -488,302 +514,463 @@ abi_ulong mmap_find_vma(abi_ulong start, abi_ulong size, abi_ulong align) } } -/* NOTE: all the constants are the HOST ones */ -abi_long target_mmap(abi_ulong start, abi_ulong len, int target_prot, - int flags, int fd, off_t offset) +/* + * Record a successful mmap within the user-exec interval tree. + */ +static abi_long mmap_end(abi_ulong start, abi_ulong last, + abi_ulong passthrough_start, + abi_ulong passthrough_last, + int flags, int page_flags) { - abi_ulong ret, last, real_start, real_last, retaddr, host_len; - abi_ulong passthrough_start = -1, passthrough_last = 0; - int page_flags; - off_t host_offset; - - mmap_lock(); - trace_target_mmap(start, len, target_prot, flags, fd, offset); - - if (!len) { - errno = EINVAL; - goto fail; + if (flags & MAP_ANONYMOUS) { + page_flags |= PAGE_ANON; } - - page_flags = validate_prot_to_pageflags(target_prot); - if (!page_flags) { - errno = EINVAL; - goto fail; + page_flags |= PAGE_RESET; + if (passthrough_start > passthrough_last) { + page_set_flags(start, last, page_flags); + } else { + if (start < passthrough_start) { + page_set_flags(start, passthrough_start - 1, page_flags); + } + page_set_flags(passthrough_start, passthrough_last, + page_flags | PAGE_PASSTHROUGH); + if (passthrough_last < last) { + page_set_flags(passthrough_last + 1, last, page_flags); + } } - - /* Also check for overflows... */ - len = TARGET_PAGE_ALIGN(len); - if (!len) { - errno = ENOMEM; - goto fail; + shm_region_rm_complete(start, last); + trace_target_mmap_complete(start); + if (qemu_loglevel_mask(CPU_LOG_PAGE)) { + FILE *f = qemu_log_trylock(); + if (f) { + fprintf(f, "page layout changed following mmap\n"); + page_dump(f); + qemu_log_unlock(f); + } } + return start; +} - if (offset & ~TARGET_PAGE_MASK) { - errno = EINVAL; - goto fail; - } +/* + * Special case host page size == target page size, + * where there are no edge conditions. + */ +static abi_long mmap_h_eq_g(abi_ulong start, abi_ulong len, + int host_prot, int flags, int page_flags, + int fd, off_t offset) +{ + void *p, *want_p = g2h_untagged(start); + abi_ulong last; - /* - * If we're mapping shared memory, ensure we generate code for parallel - * execution and flush old translations. This will work up to the level - * supported by the host -- anything that requires EXCP_ATOMIC will not - * be atomic with respect to an external process. - */ - if (flags & MAP_SHARED) { - CPUState *cpu = thread_cpu; - if (!(cpu->tcg_cflags & CF_PARALLEL)) { - cpu->tcg_cflags |= CF_PARALLEL; - tb_flush(cpu); - } + p = mmap(want_p, len, host_prot, flags, fd, offset); + if (p == MAP_FAILED) { + return -1; + } + /* If the host kernel does not support MAP_FIXED_NOREPLACE, emulate. */ + if ((flags & MAP_FIXED_NOREPLACE) && p != want_p) { + do_munmap(p, len); + errno = EEXIST; + return -1; } - real_start = start & qemu_host_page_mask; - host_offset = offset & qemu_host_page_mask; + start = h2g(p); + last = start + len - 1; + return mmap_end(start, last, start, last, flags, page_flags); +} - /* - * If the user is asking for the kernel to find a location, do that - * before we truncate the length for mapping files below. - */ - if (!(flags & (MAP_FIXED | MAP_FIXED_NOREPLACE))) { - host_len = len + offset - host_offset; - host_len = HOST_PAGE_ALIGN(host_len); - start = mmap_find_vma(real_start, host_len, TARGET_PAGE_SIZE); - if (start == (abi_ulong)-1) { - errno = ENOMEM; - goto fail; - } - } +/* + * Special case host page size < target page size. + * + * The two special cases are increased guest alignment, and mapping + * past the end of a file. + * + * When mapping files into a memory area larger than the file, + * accesses to pages beyond the file size will cause a SIGBUS. + * + * For example, if mmaping a file of 100 bytes on a host with 4K + * pages emulating a target with 8K pages, the target expects to + * be able to access the first 8K. But the host will trap us on + * any access beyond 4K. + * + * When emulating a target with a larger page-size than the hosts, + * we may need to truncate file maps at EOF and add extra anonymous + * pages up to the targets page boundary. + * + * This workaround only works for files that do not change. + * If the file is later extended (e.g. ftruncate), the SIGBUS + * vanishes and the proper behaviour is that changes within the + * anon page should be reflected in the file. + * + * However, this case is rather common with executable images, + * so the workaround is important for even trivial tests, whereas + * the mmap of of a file being extended is less common. + */ +static abi_long mmap_h_lt_g(abi_ulong start, abi_ulong len, int host_prot, + int mmap_flags, int page_flags, int fd, + off_t offset, int host_page_size) +{ + void *p, *want_p = g2h_untagged(start); + off_t fileend_adj = 0; + int flags = mmap_flags; + abi_ulong last, pass_last; - /* - * When mapping files into a memory area larger than the file, accesses - * to pages beyond the file size will cause a SIGBUS. - * - * For example, if mmaping a file of 100 bytes on a host with 4K pages - * emulating a target with 8K pages, the target expects to be able to - * access the first 8K. But the host will trap us on any access beyond - * 4K. - * - * When emulating a target with a larger page-size than the hosts, we - * may need to truncate file maps at EOF and add extra anonymous pages - * up to the targets page boundary. - */ - if ((qemu_real_host_page_size() < qemu_host_page_size) && - !(flags & MAP_ANONYMOUS)) { + if (!(flags & MAP_ANONYMOUS)) { struct stat sb; if (fstat(fd, &sb) == -1) { - goto fail; + return -1; } - - /* Are we trying to create a map beyond EOF?. */ - if (offset + len > sb.st_size) { + if (offset >= sb.st_size) { /* - * If so, truncate the file map at eof aligned with - * the hosts real pagesize. Additional anonymous maps - * will be created beyond EOF. + * The entire map is beyond the end of the file. + * Transform it to an anonymous mapping. */ - len = REAL_HOST_PAGE_ALIGN(sb.st_size - offset); + flags |= MAP_ANONYMOUS; + fd = -1; + offset = 0; + } else if (offset + len > sb.st_size) { + /* + * A portion of the map is beyond the end of the file. + * Truncate the file portion of the allocation. + */ + fileend_adj = offset + len - sb.st_size; } } - if (!(flags & (MAP_FIXED | MAP_FIXED_NOREPLACE))) { - uintptr_t host_start; - int host_prot; - void *p; + if (flags & (MAP_FIXED | MAP_FIXED_NOREPLACE)) { + if (fileend_adj) { + p = mmap(want_p, len, host_prot, flags | MAP_ANONYMOUS, -1, 0); + } else { + p = mmap(want_p, len, host_prot, flags, fd, offset); + } + if (p != want_p) { + if (p != MAP_FAILED) { + /* Host does not support MAP_FIXED_NOREPLACE: emulate. */ + do_munmap(p, len); + errno = EEXIST; + } + return -1; + } - host_len = len + offset - host_offset; - host_len = HOST_PAGE_ALIGN(host_len); - host_prot = target_to_host_prot(target_prot); + if (fileend_adj) { + void *t = mmap(p, len - fileend_adj, host_prot, + (flags & ~MAP_FIXED_NOREPLACE) | MAP_FIXED, + fd, offset); + + if (t == MAP_FAILED) { + int save_errno = errno; + + /* + * We failed a map over the top of the successful anonymous + * mapping above. The only failure mode is running out of VMAs, + * and there's nothing that we can do to detect that earlier. + * If we have replaced an existing mapping with MAP_FIXED, + * then we cannot properly recover. It's a coin toss whether + * it would be better to exit or continue here. + */ + if (!(flags & MAP_FIXED_NOREPLACE) && + !page_check_range_empty(start, start + len - 1)) { + qemu_log("QEMU target_mmap late failure: %s", + strerror(save_errno)); + } + + do_munmap(want_p, len); + errno = save_errno; + return -1; + } + } + } else { + size_t host_len, part_len; /* - * Note: we prefer to control the mapping address. It is - * especially important if qemu_host_page_size > - * qemu_real_host_page_size. + * Take care to align the host memory. Perform a larger anonymous + * allocation and extract the aligned portion. Remap the file on + * top of that. */ - p = mmap(g2h_untagged(start), host_len, host_prot, - flags | MAP_FIXED | MAP_ANONYMOUS, -1, 0); + host_len = len + TARGET_PAGE_SIZE - host_page_size; + p = mmap(want_p, host_len, host_prot, flags | MAP_ANONYMOUS, -1, 0); if (p == MAP_FAILED) { - goto fail; + return -1; + } + + part_len = (uintptr_t)p & (TARGET_PAGE_SIZE - 1); + if (part_len) { + part_len = TARGET_PAGE_SIZE - part_len; + do_munmap(p, part_len); + p += part_len; + host_len -= part_len; } - /* update start so that it points to the file position at 'offset' */ - host_start = (uintptr_t)p; + if (len < host_len) { + do_munmap(p + len, host_len - len); + } + if (!(flags & MAP_ANONYMOUS)) { - p = mmap(g2h_untagged(start), len, host_prot, - flags | MAP_FIXED, fd, host_offset); - if (p == MAP_FAILED) { - munmap(g2h_untagged(start), host_len); - goto fail; + void *t = mmap(p, len - fileend_adj, host_prot, + flags | MAP_FIXED, fd, offset); + + if (t == MAP_FAILED) { + int save_errno = errno; + do_munmap(p, len); + errno = save_errno; + return -1; } - host_start += offset - host_offset; } - start = h2g(host_start); - last = start + len - 1; - passthrough_start = start; - passthrough_last = last; + + start = h2g(p); + } + + last = start + len - 1; + if (fileend_adj) { + pass_last = ROUND_UP(last - fileend_adj, host_page_size) - 1; } else { - if (start & ~TARGET_PAGE_MASK) { - errno = EINVAL; - goto fail; + pass_last = last; + } + return mmap_end(start, last, start, pass_last, mmap_flags, page_flags); +} + +/* + * Special case host page size > target page size. + * + * The two special cases are address and file offsets that are valid + * for the guest that cannot be directly represented by the host. + */ +static abi_long mmap_h_gt_g(abi_ulong start, abi_ulong len, + int target_prot, int host_prot, + int flags, int page_flags, int fd, + off_t offset, int host_page_size) +{ + void *p, *want_p = g2h_untagged(start); + off_t host_offset = offset & -host_page_size; + abi_ulong last, real_start, real_last; + bool misaligned_offset = false; + size_t host_len; + + if (!(flags & (MAP_FIXED | MAP_FIXED_NOREPLACE))) { + /* + * Adjust the offset to something representable on the host. + */ + host_len = len + offset - host_offset; + p = mmap(want_p, host_len, host_prot, flags, fd, host_offset); + if (p == MAP_FAILED) { + return -1; } + + /* Update start to the file position at offset. */ + p += offset - host_offset; + + start = h2g(p); last = start + len - 1; - real_last = HOST_PAGE_ALIGN(last) - 1; + return mmap_end(start, last, start, last, flags, page_flags); + } + + if (!(flags & MAP_ANONYMOUS)) { + misaligned_offset = (start ^ offset) & (host_page_size - 1); /* - * Test if requested memory area fits target address space - * It can fail only on 64-bit host with 32-bit target. - * On any other target/host host mmap() handles this error correctly. + * The fallback for misalignment is a private mapping + read. + * This carries none of semantics required of MAP_SHARED. */ - if (last < start || !guest_range_valid_untagged(start, len)) { - errno = ENOMEM; - goto fail; + if (misaligned_offset && (flags & MAP_TYPE) != MAP_PRIVATE) { + errno = EINVAL; + return -1; } + } - if (flags & MAP_FIXED_NOREPLACE) { - /* Validate that the chosen range is empty. */ - if (!page_check_range_empty(start, last)) { - errno = EEXIST; - goto fail; - } + last = start + len - 1; + real_start = start & -host_page_size; + real_last = ROUND_UP(last, host_page_size) - 1; - /* - * With reserved_va, the entire address space is mmaped in the - * host to ensure it isn't accidentally used for something else. - * We have just checked that the guest address is not mapped - * within the guest, but need to replace the host reservation. - * - * Without reserved_va, despite the guest address check above, - * keep MAP_FIXED_NOREPLACE so that the guest does not overwrite - * any host address mappings. - */ - if (reserved_va) { - flags = (flags & ~MAP_FIXED_NOREPLACE) | MAP_FIXED; + /* + * Handle the start and end of the mapping. + */ + if (real_start < start) { + abi_ulong real_page_last = real_start + host_page_size - 1; + if (last <= real_page_last) { + /* Entire allocation a subset of one host page. */ + if (!mmap_frag(real_start, start, last, target_prot, + flags, fd, offset)) { + return -1; } + return mmap_end(start, last, -1, 0, flags, page_flags); } - /* - * worst case: we cannot map the file because the offset is not - * aligned, so we read it - */ - if (!(flags & MAP_ANONYMOUS) && - (offset & ~qemu_host_page_mask) != (start & ~qemu_host_page_mask)) { - /* - * msync() won't work here, so we return an error if write is - * possible while it is a shared mapping - */ - if ((flags & MAP_TYPE) == MAP_SHARED - && (target_prot & PROT_WRITE)) { - errno = EINVAL; - goto fail; - } - retaddr = target_mmap(start, len, target_prot | PROT_WRITE, - (flags & (MAP_FIXED | MAP_FIXED_NOREPLACE)) - | MAP_PRIVATE | MAP_ANONYMOUS, - -1, 0); - if (retaddr == -1) { - goto fail; - } - if (pread(fd, g2h_untagged(start), len, offset) == -1) { - goto fail; - } - if (!(target_prot & PROT_WRITE)) { - ret = target_mprotect(start, len, target_prot); - assert(ret == 0); - } - goto the_end; + if (!mmap_frag(real_start, start, real_page_last, target_prot, + flags, fd, offset)) { + return -1; } + real_start = real_page_last + 1; + } - /* handle the start of the mapping */ - if (start > real_start) { - if (real_last == real_start + qemu_host_page_size - 1) { - /* one single host page */ - if (!mmap_frag(real_start, start, last, - target_prot, flags, fd, offset)) { - goto fail; - } - goto the_end1; - } - if (!mmap_frag(real_start, start, - real_start + qemu_host_page_size - 1, - target_prot, flags, fd, offset)) { - goto fail; - } - real_start += qemu_host_page_size; + if (last < real_last) { + abi_ulong real_page_start = real_last - host_page_size + 1; + if (!mmap_frag(real_page_start, real_page_start, last, + target_prot, flags, fd, + offset + real_page_start - start)) { + return -1; } - /* handle the end of the mapping */ - if (last < real_last) { - abi_ulong real_page = real_last - qemu_host_page_size + 1; - if (!mmap_frag(real_page, real_page, last, - target_prot, flags, fd, - offset + real_page - start)) { - goto fail; - } - real_last -= qemu_host_page_size; + real_last = real_page_start - 1; + } + + if (real_start > real_last) { + return mmap_end(start, last, -1, 0, flags, page_flags); + } + + /* + * Handle the middle of the mapping. + */ + + host_len = real_last - real_start + 1; + want_p += real_start - start; + + if (flags & MAP_ANONYMOUS) { + p = mmap(want_p, host_len, host_prot, flags, -1, 0); + } else if (!misaligned_offset) { + p = mmap(want_p, host_len, host_prot, flags, fd, + offset + real_start - start); + } else { + p = mmap(want_p, host_len, host_prot | PROT_WRITE, + flags | MAP_ANONYMOUS, -1, 0); + } + if (p != want_p) { + if (p != MAP_FAILED) { + do_munmap(p, host_len); + errno = EEXIST; } + return -1; + } - /* map the middle (easier) */ - if (real_start < real_last) { - void *p, *want_p; - off_t offset1; - size_t len1; + if (misaligned_offset) { + /* TODO: The read could be short. */ + if (pread(fd, p, host_len, offset + real_start - start) != host_len) { + do_munmap(p, host_len); + return -1; + } + if (!(host_prot & PROT_WRITE)) { + mprotect(p, host_len, host_prot); + } + } - if (flags & MAP_ANONYMOUS) { - offset1 = 0; - } else { - offset1 = offset + real_start - start; + return mmap_end(start, last, -1, 0, flags, page_flags); +} + +static abi_long target_mmap__locked(abi_ulong start, abi_ulong len, + int target_prot, int flags, int page_flags, + int fd, off_t offset) +{ + int host_page_size = qemu_real_host_page_size(); + int host_prot; + + /* + * For reserved_va, we are in full control of the allocation. + * Find a suitable hole and convert to MAP_FIXED. + */ + if (reserved_va) { + if (flags & MAP_FIXED_NOREPLACE) { + /* Validate that the chosen range is empty. */ + if (!page_check_range_empty(start, start + len - 1)) { + errno = EEXIST; + return -1; } - len1 = real_last - real_start + 1; - want_p = g2h_untagged(real_start); - - p = mmap(want_p, len1, target_to_host_prot(target_prot), - flags, fd, offset1); - if (p != want_p) { - if (p != MAP_FAILED) { - munmap(p, len1); - errno = EEXIST; - } - goto fail; + flags = (flags & ~MAP_FIXED_NOREPLACE) | MAP_FIXED; + } else if (!(flags & MAP_FIXED)) { + abi_ulong real_start = start & -host_page_size; + off_t host_offset = offset & -host_page_size; + size_t real_len = len + offset - host_offset; + abi_ulong align = MAX(host_page_size, TARGET_PAGE_SIZE); + + start = mmap_find_vma(real_start, real_len, align); + if (start == (abi_ulong)-1) { + errno = ENOMEM; + return -1; } - passthrough_start = real_start; - passthrough_last = real_last; + start += offset - host_offset; + flags |= MAP_FIXED; } } - the_end1: - if (flags & MAP_ANONYMOUS) { - page_flags |= PAGE_ANON; - } - page_flags |= PAGE_RESET; - if (passthrough_start > passthrough_last) { - page_set_flags(start, last, page_flags); + + host_prot = target_to_host_prot(target_prot); + + if (host_page_size == TARGET_PAGE_SIZE) { + return mmap_h_eq_g(start, len, host_prot, flags, + page_flags, fd, offset); + } else if (host_page_size < TARGET_PAGE_SIZE) { + return mmap_h_lt_g(start, len, host_prot, flags, + page_flags, fd, offset, host_page_size); } else { - if (start < passthrough_start) { - page_set_flags(start, passthrough_start - 1, page_flags); + return mmap_h_gt_g(start, len, target_prot, host_prot, flags, + page_flags, fd, offset, host_page_size); + } +} + +/* NOTE: all the constants are the HOST ones */ +abi_long target_mmap(abi_ulong start, abi_ulong len, int target_prot, + int flags, int fd, off_t offset) +{ + abi_long ret; + int page_flags; + + trace_target_mmap(start, len, target_prot, flags, fd, offset); + + if (!len) { + errno = EINVAL; + return -1; + } + + page_flags = validate_prot_to_pageflags(target_prot); + if (!page_flags) { + errno = EINVAL; + return -1; + } + + /* Also check for overflows... */ + len = TARGET_PAGE_ALIGN(len); + if (!len || len != (size_t)len) { + errno = ENOMEM; + return -1; + } + + if (offset & ~TARGET_PAGE_MASK) { + errno = EINVAL; + return -1; + } + if (flags & (MAP_FIXED | MAP_FIXED_NOREPLACE)) { + if (start & ~TARGET_PAGE_MASK) { + errno = EINVAL; + return -1; } - page_set_flags(passthrough_start, passthrough_last, - page_flags | PAGE_PASSTHROUGH); - if (passthrough_last < last) { - page_set_flags(passthrough_last + 1, last, page_flags); + if (!guest_range_valid_untagged(start, len)) { + errno = ENOMEM; + return -1; } } - shm_region_rm_complete(start, last); - the_end: - trace_target_mmap_complete(start); - if (qemu_loglevel_mask(CPU_LOG_PAGE)) { - FILE *f = qemu_log_trylock(); - if (f) { - fprintf(f, "page layout changed following mmap\n"); - page_dump(f); - qemu_log_unlock(f); + + mmap_lock(); + + ret = target_mmap__locked(start, len, target_prot, flags, + page_flags, fd, offset); + + mmap_unlock(); + + /* + * If we're mapping shared memory, ensure we generate code for parallel + * execution and flush old translations. This will work up to the level + * supported by the host -- anything that requires EXCP_ATOMIC will not + * be atomic with respect to an external process. + */ + if (ret != -1 && (flags & MAP_TYPE) != MAP_PRIVATE) { + CPUState *cpu = thread_cpu; + if (!(cpu->tcg_cflags & CF_PARALLEL)) { + cpu->tcg_cflags |= CF_PARALLEL; + tb_flush(cpu); } } - mmap_unlock(); - return start; -fail: - mmap_unlock(); - return -1; + + return ret; } static int mmap_reserve_or_unmap(abi_ulong start, abi_ulong len) { + int host_page_size = qemu_real_host_page_size(); abi_ulong real_start; abi_ulong real_last; abi_ulong real_len; @@ -793,8 +980,8 @@ static int mmap_reserve_or_unmap(abi_ulong start, abi_ulong len) int prot; last = start + len - 1; - real_start = start & qemu_host_page_mask; - real_last = HOST_PAGE_ALIGN(last) - 1; + real_start = start & -host_page_size; + real_last = ROUND_UP(last, host_page_size) - 1; /* * If guest pages remain on the first or last host pages, @@ -802,7 +989,7 @@ static int mmap_reserve_or_unmap(abi_ulong start, abi_ulong len) * The single page special case is required for the last page, * lest real_start overflow to zero. */ - if (real_last - real_start < qemu_host_page_size) { + if (real_last - real_start < host_page_size) { prot = 0; for (a = real_start; a < start; a += TARGET_PAGE_SIZE) { prot |= page_get_flags(a); @@ -818,14 +1005,14 @@ static int mmap_reserve_or_unmap(abi_ulong start, abi_ulong len) prot |= page_get_flags(a); } if (prot != 0) { - real_start += qemu_host_page_size; + real_start += host_page_size; } for (prot = 0, a = last; a < real_last; a += TARGET_PAGE_SIZE) { prot |= page_get_flags(a + 1); } if (prot != 0) { - real_last -= qemu_host_page_size; + real_last -= host_page_size; } if (real_last < real_start) { @@ -836,13 +1023,7 @@ static int mmap_reserve_or_unmap(abi_ulong start, abi_ulong len) real_len = real_last - real_start + 1; host_start = g2h_untagged(real_start); - if (reserved_va) { - void *ptr = mmap(host_start, real_len, PROT_NONE, - MAP_FIXED | MAP_ANONYMOUS - | MAP_PRIVATE | MAP_NORESERVE, -1, 0); - return ptr == host_start ? 0 : -1; - } - return munmap(host_start, real_len); + return do_munmap(host_start, real_len); } int target_munmap(abi_ulong start, abi_ulong len) @@ -1055,69 +1236,161 @@ static inline abi_ulong target_shmlba(CPUArchState *cpu_env) } #endif +#if defined(__arm__) || defined(__mips__) || defined(__sparc__) +#define HOST_FORCE_SHMLBA 1 +#else +#define HOST_FORCE_SHMLBA 0 +#endif + abi_ulong target_shmat(CPUArchState *cpu_env, int shmid, abi_ulong shmaddr, int shmflg) { CPUState *cpu = env_cpu(cpu_env); - abi_ulong raddr; struct shmid_ds shm_info; int ret; - abi_ulong shmlba; + int h_pagesize; + int t_shmlba, h_shmlba, m_shmlba; + size_t t_len, h_len, m_len; /* shmat pointers are always untagged */ - /* find out the length of the shared memory segment */ + /* + * Because we can't use host shmat() unless the address is sufficiently + * aligned for the host, we'll need to check both. + * TODO: Could be fixed with softmmu. + */ + t_shmlba = target_shmlba(cpu_env); + h_pagesize = qemu_real_host_page_size(); + h_shmlba = (HOST_FORCE_SHMLBA ? SHMLBA : h_pagesize); + m_shmlba = MAX(t_shmlba, h_shmlba); + + if (shmaddr) { + if (shmaddr & (m_shmlba - 1)) { + if (shmflg & SHM_RND) { + /* + * The guest is allowing the kernel to round the address. + * Assume that the guest is ok with us rounding to the + * host required alignment too. Anyway if we don't, we'll + * get an error from the kernel. + */ + shmaddr &= ~(m_shmlba - 1); + if (shmaddr == 0 && (shmflg & SHM_REMAP)) { + return -TARGET_EINVAL; + } + } else { + int require = TARGET_PAGE_SIZE; +#ifdef TARGET_FORCE_SHMLBA + require = t_shmlba; +#endif + /* + * Include host required alignment, as otherwise we cannot + * use host shmat at all. + */ + require = MAX(require, h_shmlba); + if (shmaddr & (require - 1)) { + return -TARGET_EINVAL; + } + } + } + } else { + if (shmflg & SHM_REMAP) { + return -TARGET_EINVAL; + } + } + /* All rounding now manually concluded. */ + shmflg &= ~SHM_RND; + + /* Find out the length of the shared memory segment. */ ret = get_errno(shmctl(shmid, IPC_STAT, &shm_info)); if (is_error(ret)) { /* can't get length, bail out */ return ret; } + t_len = TARGET_PAGE_ALIGN(shm_info.shm_segsz); + h_len = ROUND_UP(shm_info.shm_segsz, h_pagesize); + m_len = MAX(t_len, h_len); - shmlba = target_shmlba(cpu_env); - - if (shmaddr & (shmlba - 1)) { - if (shmflg & SHM_RND) { - shmaddr &= ~(shmlba - 1); - } else { - return -TARGET_EINVAL; - } - } - if (!guest_range_valid_untagged(shmaddr, shm_info.shm_segsz)) { + if (!guest_range_valid_untagged(shmaddr, m_len)) { return -TARGET_EINVAL; } WITH_MMAP_LOCK_GUARD() { - void *host_raddr; + bool mapped = false; + void *want, *test; abi_ulong last; - if (shmaddr) { - host_raddr = shmat(shmid, (void *)g2h_untagged(shmaddr), shmflg); + if (!shmaddr) { + shmaddr = mmap_find_vma(0, m_len, m_shmlba); + if (shmaddr == -1) { + return -TARGET_ENOMEM; + } + mapped = !reserved_va; + } else if (shmflg & SHM_REMAP) { + /* + * If host page size > target page size, the host shmat may map + * more memory than the guest expects. Reject a mapping that + * would replace memory in the unexpected gap. + * TODO: Could be fixed with softmmu. + */ + if (t_len < h_len && + !page_check_range_empty(shmaddr + t_len, + shmaddr + h_len - 1)) { + return -TARGET_EINVAL; + } } else { - abi_ulong mmap_start; + if (!page_check_range_empty(shmaddr, shmaddr + m_len - 1)) { + return -TARGET_EINVAL; + } + } - /* In order to use the host shmat, we need to honor host SHMLBA. */ - mmap_start = mmap_find_vma(0, shm_info.shm_segsz, - MAX(SHMLBA, shmlba)); + /* All placement is now complete. */ + want = (void *)g2h_untagged(shmaddr); - if (mmap_start == -1) { - return -TARGET_ENOMEM; + /* + * Map anonymous pages across the entire range, then remap with + * the shared memory. This is required for a number of corner + * cases for which host and guest page sizes differ. + */ + if (h_len != t_len) { + int mmap_p = PROT_READ | (shmflg & SHM_RDONLY ? 0 : PROT_WRITE); + int mmap_f = MAP_PRIVATE | MAP_ANONYMOUS + | (reserved_va || (shmflg & SHM_REMAP) + ? MAP_FIXED : MAP_FIXED_NOREPLACE); + + test = mmap(want, m_len, mmap_p, mmap_f, -1, 0); + if (unlikely(test != want)) { + /* shmat returns EINVAL not EEXIST like mmap. */ + ret = (test == MAP_FAILED && errno != EEXIST + ? get_errno(-1) : -TARGET_EINVAL); + if (mapped) { + do_munmap(want, m_len); + } + return ret; } - host_raddr = shmat(shmid, g2h_untagged(mmap_start), - shmflg | SHM_REMAP); + mapped = true; } - if (host_raddr == (void *)-1) { - return get_errno(-1); + if (reserved_va || mapped) { + shmflg |= SHM_REMAP; + } + test = shmat(shmid, want, shmflg); + if (test == MAP_FAILED) { + ret = get_errno(-1); + if (mapped) { + do_munmap(want, m_len); + } + return ret; } - raddr = h2g(host_raddr); - last = raddr + shm_info.shm_segsz - 1; + assert(test == want); - page_set_flags(raddr, last, + last = shmaddr + m_len - 1; + page_set_flags(shmaddr, last, PAGE_VALID | PAGE_RESET | PAGE_READ | - (shmflg & SHM_RDONLY ? 0 : PAGE_WRITE)); + (shmflg & SHM_RDONLY ? 0 : PAGE_WRITE) | + (shmflg & SHM_EXEC ? PAGE_EXEC : 0)); - shm_region_rm_complete(raddr, last); - shm_region_add(raddr, last); + shm_region_rm_complete(shmaddr, last); + shm_region_add(shmaddr, last); } /* @@ -1131,7 +1404,15 @@ abi_ulong target_shmat(CPUArchState *cpu_env, int shmid, tb_flush(cpu); } - return raddr; + if (qemu_loglevel_mask(CPU_LOG_PAGE)) { + FILE *f = qemu_log_trylock(); + if (f) { + fprintf(f, "page layout changed following shmat\n"); + page_dump(f); + qemu_log_unlock(f); + } + } + return shmaddr; } abi_long target_shmdt(abi_ulong shmaddr) |