summary refs log tree commit diff stats
diff options
context:
space:
mode:
-rw-r--r--.gitmodules14
-rw-r--r--Changelog2
-rw-r--r--MAINTAINERS56
-rw-r--r--Makefile11
-rw-r--r--Makefile.target10
-rw-r--r--QMP/README88
-rw-r--r--README2
-rw-r--r--arch_init.c33
-rw-r--r--audio/mixeng.c6
-rw-r--r--block.c278
-rw-r--r--block/backup.c6
-rw-r--r--block/blkdebug.c11
-rw-r--r--block/blkverify.c16
-rw-r--r--block/bochs.c3
-rw-r--r--block/cloop.c3
-rw-r--r--block/commit.c6
-rw-r--r--block/cow.c15
-rw-r--r--block/curl.c3
-rw-r--r--block/dmg.c3
-rw-r--r--block/gluster.c8
-rw-r--r--block/iscsi.c407
-rw-r--r--block/mirror.c11
-rw-r--r--block/nbd.c3
-rw-r--r--block/parallels.c3
-rw-r--r--block/qapi.c124
-rw-r--r--block/qcow.c15
-rw-r--r--block/qcow2-cache.c26
-rw-r--r--block/qcow2-cluster.c313
-rw-r--r--block/qcow2-refcount.c75
-rw-r--r--block/qcow2-snapshot.c92
-rw-r--r--block/qcow2.c475
-rw-r--r--block/qcow2.h52
-rw-r--r--block/qed.c18
-rw-r--r--block/qed.h2
-rw-r--r--block/raw-posix.c95
-rw-r--r--block/raw-win32.c29
-rw-r--r--block/raw_bsd.c19
-rw-r--r--block/rbd.c31
-rw-r--r--block/sheepdog.c24
-rw-r--r--block/snapshot.c133
-rw-r--r--block/ssh.c6
-rw-r--r--block/stream.c11
-rw-r--r--block/vdi.c8
-rw-r--r--block/vhdx.c13
-rw-r--r--block/vmdk.c142
-rw-r--r--block/vpc.c34
-rw-r--r--block/vvfat.c13
-rw-r--r--blockdev.c899
-rw-r--r--blockjob.c22
-rw-r--r--bsd-user/qemu.h6
-rwxr-xr-xconfigure81
-rw-r--r--cpu-exec.c4
-rw-r--r--cpus.c144
-rw-r--r--cputlb.c15
-rw-r--r--default-configs/arm-linux-user.mak2
-rw-r--r--default-configs/arm-softmmu.mak2
-rw-r--r--default-configs/armeb-linux-user.mak2
-rw-r--r--default-configs/m68k-linux-user.mak2
-rw-r--r--default-configs/m68k-softmmu.mak1
-rw-r--r--default-configs/ppc-linux-user.mak2
-rw-r--r--default-configs/ppc-softmmu.mak1
-rw-r--r--default-configs/ppc64-linux-user.mak2
-rw-r--r--default-configs/ppc64-softmmu.mak1
-rw-r--r--default-configs/ppc64abi32-linux-user.mak2
-rw-r--r--default-configs/ppcemb-softmmu.mak1
-rw-r--r--docs/qapi-code-gen.txt17
-rw-r--r--docs/qmp/README87
-rw-r--r--docs/qmp/qmp-events.txt (renamed from QMP/qmp-events.txt)34
-rw-r--r--docs/qmp/qmp-spec.txt (renamed from QMP/qmp-spec.txt)65
-rw-r--r--docs/rdma.txt2
-rw-r--r--docs/specs/acpi_cpu_hotplug.txt2
-rw-r--r--docs/specs/qcow2.txt3
-rw-r--r--exec.c256
-rw-r--r--gdbstub.c6
-rw-r--r--hmp-commands.hx37
-rw-r--r--hmp.c26
-rw-r--r--hmp.h2
-rw-r--r--hw/9pfs/virtio-9p-xattr.c6
-rw-r--r--hw/acpi/core.c3
-rw-r--r--hw/alpha/typhoon.c2
-rw-r--r--hw/arm/boot.c6
-rw-r--r--hw/arm/integratorcp.c2
-rw-r--r--hw/arm/omap_sx1.c10
-rw-r--r--hw/arm/palm.c10
-rw-r--r--hw/arm/z2.c12
-rw-r--r--hw/audio/hda-codec-common.h456
-rw-r--r--hw/audio/hda-codec.c454
-rw-r--r--hw/block/m25p80.c5
-rw-r--r--hw/block/virtio-blk.c25
-rw-r--r--hw/block/xen_disk.c13
-rw-r--r--hw/char/Makefile.objs2
-rw-r--r--hw/char/sclpconsole-lm.c398
-rw-r--r--hw/char/sclpconsole.c88
-rw-r--r--hw/char/sh_serial.c2
-rw-r--r--hw/display/Makefile.objs3
-rw-r--r--hw/display/cirrus_vga.c3
-rw-r--r--hw/display/qxl.c32
-rw-r--r--hw/display/qxl.h3
-rw-r--r--hw/display/vga.c5
-rw-r--r--hw/i386/kvm/clock.c2
-rw-r--r--hw/i386/kvmvapic.c20
-rw-r--r--hw/i386/pc_piix.c1
-rw-r--r--hw/i386/pc_sysfw.c5
-rw-r--r--hw/ide/ahci.c10
-rw-r--r--hw/microblaze/boot.c50
-rw-r--r--hw/microblaze/boot.h4
-rw-r--r--hw/microblaze/petalogix_ml605_mmu.c6
-rw-r--r--hw/microblaze/petalogix_s3adsp1800_mmu.c4
-rw-r--r--hw/misc/Makefile.objs1
-rw-r--r--hw/misc/arm_integrator_debug.c99
-rw-r--r--hw/misc/vfio.c627
-rw-r--r--hw/net/e1000.c79
-rw-r--r--hw/net/pcnet-pci.c4
-rw-r--r--hw/net/rtl8139.c7
-rw-r--r--hw/pci-host/q35.c2
-rw-r--r--hw/pci/Makefile.objs2
-rw-r--r--hw/s390x/event-facility.c17
-rw-r--r--hw/s390x/sclpquiesce.c29
-rw-r--r--hw/scsi/lsi53c895a.c51
-rw-r--r--hw/scsi/scsi-bus.c47
-rw-r--r--hw/scsi/spapr_vscsi.c195
-rw-r--r--hw/scsi/srp.h7
-rw-r--r--hw/scsi/virtio-scsi.c1
-rw-r--r--hw/sd/milkymist-memcard.c4
-rw-r--r--hw/sd/omap_mmc.c6
-rw-r--r--hw/sd/pl181.c4
-rw-r--r--hw/sd/pxa2xx_mmci.c3
-rw-r--r--hw/sd/sd.c5
-rw-r--r--hw/sd/sdhci.c3
-rw-r--r--hw/sd/ssi-sd.c3
-rw-r--r--hw/usb/combined-packet.c1
-rw-r--r--hw/usb/core.c3
-rw-r--r--hw/usb/hcd-ohci.c2
-rw-r--r--hw/usb/hcd-xhci.c45
-rw-r--r--hw/usb/host-bsd.c639
-rw-r--r--hw/usb/host-linux.c1911
-rw-r--r--hw/xen/xen_backend.c19
-rw-r--r--include/block/block.h32
-rw-r--r--include/block/block_int.h30
-rw-r--r--include/block/blockjob.h14
-rw-r--r--include/block/qapi.h2
-rw-r--r--include/block/snapshot.h14
-rw-r--r--include/exec/cpu-all.h2
-rw-r--r--include/exec/cpu-defs.h2
-rw-r--r--include/exec/def-helper.h3
-rw-r--r--include/exec/exec-all.h48
-rw-r--r--include/exec/ioport.h2
-rw-r--r--include/exec/softmmu_template.h286
-rw-r--r--include/hw/i386/pc.h8
-rw-r--r--include/hw/misc/arm_integrator_debug.h18
-rw-r--r--include/hw/pci-host/q35.h2
-rw-r--r--include/hw/pci/pci_ids.h1
-rw-r--r--include/hw/qdev-core.h16
-rw-r--r--include/hw/s390x/ebcdic.h104
-rw-r--r--include/hw/s390x/event-facility.h88
-rw-r--r--include/hw/scsi/scsi.h2
-rw-r--r--include/hw/virtio/virtio-blk.h1
-rw-r--r--include/migration/vmstate.h17
-rw-r--r--include/qapi/qmp/dispatch.h7
-rw-r--r--include/qapi/qmp/qdict.h2
-rw-r--r--include/qemu-common.h3
-rw-r--r--include/qemu/bitops.h80
-rw-r--r--include/qemu/option.h1
-rw-r--r--include/qemu/seqlock.h72
-rw-r--r--include/qemu/sockets.h1
-rw-r--r--include/qemu/thread-posix.h8
-rw-r--r--include/qemu/thread-win32.h4
-rw-r--r--include/qemu/thread.h7
-rw-r--r--include/qemu/timer.h51
-rw-r--r--include/sysemu/blockdev.h1
-rw-r--r--include/sysemu/char.h1
-rw-r--r--include/sysemu/kvm.h34
-rw-r--r--include/sysemu/sysemu.h6
-rw-r--r--include/ui/qemu-spice.h5
-rw-r--r--ioport.c9
-rw-r--r--kvm-all.c95
-rw-r--r--kvm-stub.c1
-rw-r--r--libcacard/Makefile3
-rw-r--r--libcacard/vscclient.c3
-rw-r--r--linux-headers/asm-arm64/kvm.h168
-rw-r--r--linux-headers/asm-arm64/kvm_para.h1
-rw-r--r--linux-headers/asm-mips/kvm.h81
-rw-r--r--linux-headers/asm-x86/kvm_para.h1
-rw-r--r--linux-headers/linux/kvm.h4
-rw-r--r--linux-headers/linux/kvm_para.h1
-rw-r--r--linux-headers/linux/vfio.h80
-rw-r--r--linux-headers/linux/virtio_config.h3
-rw-r--r--linux-user/alpha/syscall_nr.h4
-rw-r--r--linux-user/ioctls.h1
-rw-r--r--linux-user/linuxload.c8
-rw-r--r--linux-user/main.c125
-rw-r--r--linux-user/qemu.h8
-rw-r--r--linux-user/strace.list9
-rw-r--r--linux-user/syscall.c210
-rw-r--r--linux-user/syscall_defs.h12
-rw-r--r--main-loop.c3
-rw-r--r--memory.c4
-rw-r--r--migration-rdma.c17
-rw-r--r--migration.c4
-rw-r--r--net/socket.c19
-rw-r--r--pc-bios/README4
-rw-r--r--pc-bios/acpi-dsdt.amlbin4407 -> 4407 bytes
-rw-r--r--pc-bios/bios.binbin131072 -> 131072 bytes
-rw-r--r--pc-bios/openbios-ppcbin733976 -> 729880 bytes
-rw-r--r--pc-bios/openbios-sparc32bin381484 -> 381488 bytes
-rw-r--r--pc-bios/openbios-sparc64bin1598328 -> 1598328 bytes
-rw-r--r--pc-bios/q35-acpi-dsdt.amlbin7344 -> 7344 bytes
-rw-r--r--pc-bios/s390-ccw.imgbin9432 -> 9336 bytes
-rw-r--r--pc-bios/s390-ccw/virtio.c7
-rw-r--r--pc-bios/s390-ccw/virtio.h1
-rw-r--r--qapi-schema.json354
-rw-r--r--qapi/qmp-registry.c33
-rw-r--r--qdev-monitor.c85
-rw-r--r--qemu-char.c105
-rw-r--r--qemu-coroutine.c34
-rw-r--r--qemu-img-cmds.hx6
-rw-r--r--qemu-img.c136
-rw-r--r--qemu-img.texi5
-rw-r--r--qemu-io-cmds.c9
-rw-r--r--qemu-io.c49
-rw-r--r--qemu-nbd.c6
-rw-r--r--qemu-seccomp.c1
-rw-r--r--qemu-timer.c171
-rw-r--r--qemu.nsi2
-rw-r--r--qga/commands-posix.c4
-rw-r--r--qga/commands.c39
-rw-r--r--qga/main.c75
-rw-r--r--qga/qapi-schema.json5
-rw-r--r--qmp-commands.hx159
-rw-r--r--qobject/qdict.c21
-rw-r--r--qom/cpu.c1
-rw-r--r--roms/Makefile99
-rw-r--r--roms/config.vga-cirrus (renamed from roms/config.vga.cirrus)0
-rw-r--r--roms/config.vga-isavga (renamed from roms/config.vga.isavga)0
-rw-r--r--roms/config.vga-qxl (renamed from roms/config.vga.qxl)0
-rw-r--r--roms/config.vga-stdvga (renamed from roms/config.vga.stdvga)0
-rw-r--r--roms/config.vga-vmware (renamed from roms/config.vga.vmware)0
m---------roms/openbios0
m---------roms/seabios0
-rw-r--r--rules.mak28
-rw-r--r--savevm.c41
-rwxr-xr-xscripts/get_maintainer.pl2
-rw-r--r--scripts/qapi-types.py15
-rw-r--r--scripts/qapi-visit.py26
-rwxr-xr-xscripts/qmp/qemu-ga-client (renamed from QMP/qemu-ga-client)2
-rwxr-xr-xscripts/qmp/qmp (renamed from QMP/qmp)0
-rwxr-xr-xscripts/qmp/qmp-shell (renamed from QMP/qmp-shell)2
-rw-r--r--scripts/qmp/qmp.py (renamed from QMP/qmp.py)8
-rwxr-xr-xscripts/qmp/qom-fuse (renamed from QMP/qom-fuse)0
-rwxr-xr-xscripts/qmp/qom-get (renamed from QMP/qom-get)0
-rwxr-xr-xscripts/qmp/qom-list (renamed from QMP/qom-list)0
-rwxr-xr-xscripts/qmp/qom-set (renamed from QMP/qom-set)0
-rwxr-xr-xscripts/refresh-pxe-roms.sh80
-rw-r--r--slirp/libslirp.h3
-rw-r--r--slirp/misc.c3
-rw-r--r--slirp/slirp.c61
-rw-r--r--slirp/slirp.h3
-rw-r--r--slirp/socket.c4
-rw-r--r--slirp/tcp_subr.c6
-rw-r--r--slirp/udp.c4
-rw-r--r--stubs/Makefile.objs1
-rw-r--r--stubs/slirp.c6
-rw-r--r--stubs/uuid.c12
-rw-r--r--target-alpha/cpu.c4
-rw-r--r--target-alpha/helper.h2
-rw-r--r--target-alpha/translate.c53
-rw-r--r--target-arm/Makefile.objs2
-rw-r--r--target-arm/cpu.h1
-rw-r--r--target-arm/helper.c36
-rw-r--r--target-arm/helper.h8
-rw-r--r--target-arm/iwmmxt_helper.c2
-rw-r--r--target-arm/kvm.c8
-rw-r--r--target-arm/translate.c3
-rw-r--r--target-cris/helper.h8
-rw-r--r--target-cris/translate.c3
-rw-r--r--target-i386/Makefile.objs2
-rw-r--r--target-i386/arch_memory_mapping.c2
-rw-r--r--target-i386/cpu-qom.h3
-rw-r--r--target-i386/cpu.c94
-rw-r--r--target-i386/cpu.h4
-rw-r--r--target-i386/helper.c40
-rw-r--r--target-i386/machine.c6
-rw-r--r--target-i386/translate.c23
-rw-r--r--target-m68k/helper.c3
-rw-r--r--target-m68k/helper.h (renamed from target-m68k/helpers.h)0
-rw-r--r--target-m68k/op_helper.c2
-rw-r--r--target-m68k/translate.c7
-rw-r--r--target-microblaze/cpu.h1
-rw-r--r--target-microblaze/translate.c77
-rw-r--r--target-mips/helper.h12
-rw-r--r--target-mips/translate.c5
-rw-r--r--target-moxie/cpu.c1
-rw-r--r--target-openrisc/cpu.c1
-rw-r--r--target-openrisc/mmu.c9
-rw-r--r--target-openrisc/translate.c2
-rw-r--r--target-ppc/Makefile.objs2
-rw-r--r--target-ppc/helper.h10
-rw-r--r--target-ppc/translate.c6
-rw-r--r--target-ppc/translate_init.c3
-rw-r--r--target-s390x/arch_dump.c1
-rw-r--r--target-s390x/cpu.h11
-rw-r--r--target-s390x/helper.c3
-rw-r--r--target-s390x/ioinst.c110
-rw-r--r--target-s390x/ioinst.h26
-rw-r--r--target-s390x/kvm.c79
-rw-r--r--target-s390x/misc_helper.c107
-rw-r--r--target-s390x/translate.c4
-rw-r--r--target-sh4/cpu.c3
-rw-r--r--target-sh4/translate.c4
-rw-r--r--target-sparc/cpu.c1
-rw-r--r--target-sparc/helper.h18
-rw-r--r--target-sparc/translate.c5
-rw-r--r--target-unicore32/helper.c1
-rw-r--r--target-unicore32/translate.c3
-rw-r--r--target-xtensa/translate.c10
-rw-r--r--tcg/README43
-rw-r--r--tcg/aarch64/tcg-target.c51
-rw-r--r--tcg/aarch64/tcg-target.h2
-rw-r--r--tcg/arm/tcg-target.c802
-rw-r--r--tcg/arm/tcg-target.h2
-rw-r--r--tcg/hppa/tcg-target.c1831
-rw-r--r--tcg/hppa/tcg-target.h123
-rw-r--r--tcg/i386/tcg-target.c671
-rw-r--r--tcg/i386/tcg-target.h2
-rw-r--r--tcg/ia64/tcg-target.c2
-rw-r--r--tcg/ia64/tcg-target.h2
-rw-r--r--tcg/mips/tcg-target.c2
-rw-r--r--tcg/mips/tcg-target.h2
-rw-r--r--tcg/optimize.c12
-rw-r--r--tcg/ppc/tcg-target.c684
-rw-r--r--tcg/ppc/tcg-target.h2
-rw-r--r--tcg/ppc64/tcg-target.c1162
-rw-r--r--tcg/ppc64/tcg-target.h2
-rw-r--r--tcg/s390/tcg-target.c2
-rw-r--r--tcg/s390/tcg-target.h2
-rw-r--r--tcg/sparc/tcg-target.c2
-rw-r--r--tcg/sparc/tcg-target.h4
-rw-r--r--tcg/tcg-be-ldst.h90
-rw-r--r--tcg/tcg-be-null.h43
-rw-r--r--tcg/tcg-op.h239
-rw-r--r--tcg/tcg-opc.h96
-rw-r--r--tcg/tcg.c338
-rw-r--r--tcg/tcg.h166
-rw-r--r--tcg/tci/tcg-target.c3
-rw-r--r--tcg/tci/tcg-target.h2
-rw-r--r--tci.c26
-rw-r--r--tests/.gitignore4
-rw-r--r--tests/Makefile25
-rw-r--r--tests/qemu-iotests/.gitignore1
-rwxr-xr-xtests/qemu-iotests/0016
-rwxr-xr-xtests/qemu-iotests/00236
-rwxr-xr-xtests/qemu-iotests/00310
-rwxr-xr-xtests/qemu-iotests/00424
-rwxr-xr-xtests/qemu-iotests/0054
-rwxr-xr-xtests/qemu-iotests/0074
-rwxr-xr-xtests/qemu-iotests/0086
-rwxr-xr-xtests/qemu-iotests/0092
-rwxr-xr-xtests/qemu-iotests/0102
-rwxr-xr-xtests/qemu-iotests/0112
-rwxr-xr-xtests/qemu-iotests/0124
-rwxr-xr-xtests/qemu-iotests/0134
-rwxr-xr-xtests/qemu-iotests/0142
-rwxr-xr-xtests/qemu-iotests/01516
-rwxr-xr-xtests/qemu-iotests/01612
-rwxr-xr-xtests/qemu-iotests/0186
-rwxr-xr-xtests/qemu-iotests/01912
-rwxr-xr-xtests/qemu-iotests/02012
-rwxr-xr-xtests/qemu-iotests/0212
-rwxr-xr-xtests/qemu-iotests/0234
-rwxr-xr-xtests/qemu-iotests/02412
-rwxr-xr-xtests/qemu-iotests/0254
-rwxr-xr-xtests/qemu-iotests/02620
-rw-r--r--tests/qemu-iotests/026.out32
-rw-r--r--tests/qemu-iotests/026.out.nocache32
-rwxr-xr-xtests/qemu-iotests/02710
-rwxr-xr-xtests/qemu-iotests/0286
-rwxr-xr-xtests/qemu-iotests/02912
-rwxr-xr-xtests/qemu-iotests/03112
-rwxr-xr-xtests/qemu-iotests/0324
-rwxr-xr-xtests/qemu-iotests/03318
-rwxr-xr-xtests/qemu-iotests/03464
-rwxr-xr-xtests/qemu-iotests/0352
-rwxr-xr-xtests/qemu-iotests/0366
-rwxr-xr-xtests/qemu-iotests/03762
-rwxr-xr-xtests/qemu-iotests/03813
-rw-r--r--tests/qemu-iotests/038.out10
-rwxr-xr-xtests/qemu-iotests/03928
-rwxr-xr-xtests/qemu-iotests/04210
-rwxr-xr-xtests/qemu-iotests/04332
-rwxr-xr-xtests/qemu-iotests/04551
-rw-r--r--tests/qemu-iotests/045.out4
-rwxr-xr-xtests/qemu-iotests/04610
-rwxr-xr-xtests/qemu-iotests/0472
-rwxr-xr-xtests/qemu-iotests/0488
-rwxr-xr-xtests/qemu-iotests/04936
-rw-r--r--tests/qemu-iotests/049.out18
-rwxr-xr-xtests/qemu-iotests/05020
-rwxr-xr-xtests/qemu-iotests/05167
-rw-r--r--tests/qemu-iotests/051.out51
-rwxr-xr-xtests/qemu-iotests/0527
-rwxr-xr-xtests/qemu-iotests/05310
-rwxr-xr-xtests/qemu-iotests/0542
-rw-r--r--tests/qemu-iotests/054.out4
-rwxr-xr-xtests/qemu-iotests/057259
-rw-r--r--tests/qemu-iotests/057.out5
-rwxr-xr-xtests/qemu-iotests/05921
-rw-r--r--tests/qemu-iotests/059.out25
-rwxr-xr-xtests/qemu-iotests/06047
-rw-r--r--tests/qemu-iotests/060.out43
-rwxr-xr-xtests/qemu-iotests/061215
-rw-r--r--tests/qemu-iotests/061.out387
-rwxr-xr-xtests/qemu-iotests/06328
-rwxr-xr-xtests/qemu-iotests/06462
-rw-r--r--tests/qemu-iotests/064.out14
-rwxr-xr-xtests/qemu-iotests/065125
-rw-r--r--tests/qemu-iotests/065.out5
-rwxr-xr-xtests/qemu-iotests/06663
-rw-r--r--tests/qemu-iotests/066.out13
-rwxr-xr-xtests/qemu-iotests/067133
-rw-r--r--tests/qemu-iotests/067.out80
-rwxr-xr-xtests/qemu-iotests/check1
-rw-r--r--tests/qemu-iotests/common8
-rw-r--r--tests/qemu-iotests/common.config11
-rw-r--r--tests/qemu-iotests/common.filter8
-rw-r--r--tests/qemu-iotests/common.rc56
-rw-r--r--tests/qemu-iotests/group6
-rw-r--r--tests/qemu-iotests/iotests.py29
-rw-r--r--tests/qemu-iotests/sample_images/README8
-rw-r--r--tests/qemu-iotests/sample_images/iotest-dynamic-1G.vhdx.bz2bin0 -> 874 bytes
-rw-r--r--tests/qemu-iotests/socket_scm_helper.c135
-rw-r--r--tests/test-coroutine.c45
-rw-r--r--trace-events9
-rw-r--r--translate-all.c17
-rw-r--r--ui/Makefile.objs2
-rw-r--r--ui/spice-core.c40
-rw-r--r--ui/spice-display.c60
-rw-r--r--util/compatfd.c16
-rw-r--r--util/iov.c10
-rw-r--r--util/osdep.c7
-rw-r--r--util/oslib-posix.c16
-rw-r--r--util/oslib-win32.c15
-rw-r--r--util/path.c4
-rw-r--r--util/qemu-option.c6
-rw-r--r--util/qemu-sockets.c6
-rw-r--r--util/qemu-thread-posix.c116
-rw-r--r--util/qemu-thread-win32.c26
-rw-r--r--version.rc2
-rw-r--r--vl.c25
-rw-r--r--xen-all.c19
449 files changed, 13287 insertions, 10560 deletions
diff --git a/.gitmodules b/.gitmodules
index d7e3f3c7cd..45e51e79be 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,27 +1,27 @@
 [submodule "roms/vgabios"]
 	path = roms/vgabios
-	url = git://git.qemu.org/vgabios.git/
+	url = git://git.qemu-project.org/vgabios.git/
 [submodule "roms/seabios"]
 	path = roms/seabios
-	url = git://git.qemu.org/seabios.git/
+	url = git://git.qemu-project.org/seabios.git/
 [submodule "roms/SLOF"]
 	path = roms/SLOF
-	url = git://git.qemu.org/SLOF.git
+	url = git://git.qemu-project.org/SLOF.git
 [submodule "roms/ipxe"]
 	path = roms/ipxe
-	url = git://git.qemu.org/ipxe.git
+	url = git://git.qemu-project.org/ipxe.git
 [submodule "roms/openbios"]
 	path = roms/openbios
-	url = git://git.qemu.org/openbios.git
+	url = git://git.qemu-project.org/openbios.git
 [submodule "roms/qemu-palcode"]
 	path = roms/qemu-palcode
 	url = git://github.com/rth7680/qemu-palcode.git
 [submodule "roms/sgabios"]
 	path = roms/sgabios
-	url = git://git.qemu.org/sgabios.git
+	url = git://git.qemu-project.org/sgabios.git
 [submodule "pixman"]
 	path = pixman
 	url = git://anongit.freedesktop.org/pixman
 [submodule "dtc"]
 	path = dtc
-	url = git://git.qemu.org/dtc.git
+	url = git://git.qemu-project.org/dtc.git
diff --git a/Changelog b/Changelog
index 13eebefb74..1249b8aac5 100644
--- a/Changelog
+++ b/Changelog
@@ -1,6 +1,6 @@
 This file documents changes for QEMU releases 0.12 and earlier.
 For changelog information for later releases, see
-http://wiki.qemu.org/ChangeLog or look at the git history for
+http://wiki.qemu-project.org/ChangeLog or look at the git history for
 more detailed information.
 
 
diff --git a/MAINTAINERS b/MAINTAINERS
index d128ed035a..77edacf271 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -50,8 +50,7 @@ Descriptions of section entries:
 
 General Project Administration
 ------------------------------
-M: Anthony Liguori <anthony@codemonkey.ws>
-M: Paul Brook <paul@codesourcery.com>
+M: Anthony Liguori <aliguori@amazon.com>
 
 Guest CPU cores (TCG):
 ----------------------
@@ -62,7 +61,6 @@ F: target-alpha/
 F: hw/alpha/
 
 ARM
-M: Paul Brook <paul@codesourcery.com>
 M: Peter Maydell <peter.maydell@linaro.org>
 S: Maintained
 F: target-arm/
@@ -83,8 +81,7 @@ F: hw/lm32/
 F: hw/char/lm32_*
 
 M68K
-M: Paul Brook <paul@codesourcery.com>
-S: Odd Fixes
+S: Orphan
 F: target-m68k/
 F: hw/m68k/
 
@@ -248,7 +245,6 @@ F: hw/*/imx*
 F: hw/arm/kzm.c
 
 Integrator CP
-M: Paul Brook <paul@codesourcery.com>
 M: Peter Maydell <peter.maydell@linaro.org>
 S: Maintained
 F: hw/arm/integratorcp.c
@@ -274,7 +270,6 @@ S: Maintained
 F: hw/arm/palm.c
 
 Real View
-M: Paul Brook <paul@codesourcery.com>
 M: Peter Maydell <peter.maydell@linaro.org>
 S: Maintained
 F: hw/arm/realview*
@@ -285,13 +280,11 @@ S: Maintained
 F: hw/arm/spitz.c
 
 Stellaris
-M: Paul Brook <paul@codesourcery.com>
 M: Peter Maydell <peter.maydell@linaro.org>
 S: Maintained
 F: hw/*/stellaris*
 
 Versatile PB
-M: Paul Brook <paul@codesourcery.com>
 M: Peter Maydell <peter.maydell@linaro.org>
 S: Maintained
 F: hw/*/versatile*
@@ -327,18 +320,15 @@ F: hw/lm32/milkymist.c
 M68K Machines
 -------------
 an5206
-M: Paul Brook <paul@codesourcery.com>
-S: Maintained
+S: Orphan
 F: hw/m68k/an5206.c
 
 dummy_m68k
-M: Paul Brook <paul@codesourcery.com>
-S: Maintained
+S: Orphan
 F: hw/m68k/dummy_m68k.c
 
 mcf5208
-M: Paul Brook <paul@codesourcery.com>
-S: Maintained
+S: Orphan
 F: hw/m68k/mcf5208.c
 
 MicroBlaze Machines
@@ -509,7 +499,7 @@ F: hw/unicore32/
 X86 Machines
 ------------
 PC
-M: Anthony Liguori <anthony@codemonkey.ws>
+M: Anthony Liguori <aliguori@amazon.com>
 S: Supported
 F: hw/i386/pc.[ch]
 F: hw/i386/pc_piix.c
@@ -567,8 +557,7 @@ F: hw/scsi/*
 T: git git://github.com/bonzini/qemu.git scsi-next
 
 LSI53C895A
-M: Paul Brook <paul@codesourcery.com>
-S: Odd Fixes
+S: Orphan
 F: hw/scsi/lsi53c895a.c
 
 SSI
@@ -593,7 +582,7 @@ S: Supported
 F: hw/*/*vhost*
 
 virtio
-M: Anthony Liguori <anthony@codemonkey.ws>
+M: Anthony Liguori <aliguori@amazon.com>
 S: Supported
 F: hw/*/virtio*
 
@@ -638,6 +627,7 @@ Subsystems
 ----------
 Audio
 M: Vassili Karpov (malc) <av1474@comtv.ru>
+M: Gerd Hoffmann <kraxel@redhat.com>
 S: Maintained
 F: audio/
 F: hw/audio/
@@ -651,7 +641,7 @@ F: block/
 F: hw/block/
 
 Character Devices
-M: Anthony Liguori <anthony@codemonkey.ws>
+M: Anthony Liguori <aliguori@amazon.com>
 S: Maintained
 F: qemu-char.c
 
@@ -689,7 +679,7 @@ F: audio/spiceaudio.c
 F: hw/display/qxl*
 
 Graphics
-M: Anthony Liguori <anthony@codemonkey.ws>
+M: Anthony Liguori <aliguori@amazon.com>
 S: Maintained
 F: ui/
 
@@ -699,7 +689,7 @@ S: Odd Fixes
 F: ui/cocoa.m
 
 Main loop
-M: Anthony Liguori <anthony@codemonkey.ws>
+M: Anthony Liguori <aliguori@amazon.com>
 S: Supported
 F: vl.c
 
@@ -711,7 +701,7 @@ F: hmp.c
 F: hmp-commands.hx
 
 Network device layer
-M: Anthony Liguori <anthony@codemonkey.ws>
+M: Anthony Liguori <aliguori@amazon.com>
 M: Stefan Hajnoczi <stefanha@redhat.com>
 S: Maintained
 F: net/
@@ -766,6 +756,12 @@ M: Blue Swirl <blauwirbel@gmail.com>
 S: Odd Fixes
 F: scripts/checkpatch.pl
 
+Seccomp
+M: Eduardo Otubo <otubo@linux.vnet.ibm.com>
+S: Supported
+F: qemu-seccomp.c
+F: include/sysemu/seccomp.h
+
 Usermode Emulation
 ------------------
 BSD user
@@ -797,11 +793,6 @@ M: Andrzej Zaborowski <balrogg@gmail.com>
 S: Maintained
 F: tcg/arm/
 
-HPPA target
-M: Richard Henderson <rth@twiddle.net>
-S: Maintained
-F: tcg/hppa/
-
 i386 target
 M: qemu-devel@nongnu.org
 S: Maintained
@@ -842,26 +833,27 @@ TCI target
 M: Stefan Weil <sw@weilnetz.de>
 S: Maintained
 F: tcg/tci/
+F: tci.c
 
 Stable branches
 ---------------
 Stable 1.0
 L: qemu-stable@nongnu.org
-T: git git://git.qemu.org/qemu-stable-1.0.git
+T: git git://git.qemu-project.org/qemu-stable-1.0.git
 S: Orphan
 
 Stable 0.15
 L: qemu-stable@nongnu.org
 M: Andreas Färber <afaerber@suse.de>
-T: git git://git.qemu.org/qemu-stable-0.15.git
+T: git git://git.qemu-project.org/qemu-stable-0.15.git
 S: Supported
 
 Stable 0.14
 L: qemu-stable@nongnu.org
-T: git git://git.qemu.org/qemu-stable-0.14.git
+T: git git://git.qemu-project.org/qemu-stable-0.14.git
 S: Orphan
 
 Stable 0.10
 L: qemu-stable@nongnu.org
-T: git git://git.qemu.org/qemu-stable-0.10.git
+T: git git://git.qemu-project.org/qemu-stable-0.10.git
 S: Orphan
diff --git a/Makefile b/Makefile
index 362fe3e66e..b15003f9d2 100644
--- a/Makefile
+++ b/Makefile
@@ -65,7 +65,7 @@ LIBS+=-lz $(LIBS_TOOLS)
 HELPERS-$(CONFIG_LINUX) = qemu-bridge-helper$(EXESUF)
 
 ifdef BUILD_DOCS
-DOCS=qemu-doc.html qemu-tech.html qemu.1 qemu-img.1 qemu-nbd.8 QMP/qmp-commands.txt
+DOCS=qemu-doc.html qemu-tech.html qemu.1 qemu-img.1 qemu-nbd.8 qmp-commands.txt
 ifdef CONFIG_VIRTFS
 DOCS+=fsdev/virtfs-proxy-helper.1
 endif
@@ -236,7 +236,8 @@ clean:
 	find . -name '*.[oda]' -type f -exec rm -f {} +
 	find . -name '*.l[oa]' -type f -exec rm -f {} +
 	rm -f $(filter-out %.tlb,$(TOOLS)) $(HELPERS-y) qemu-ga TAGS cscope.* *.pod *~ */*~
-	rm -Rf .libs
+	rm -f fsdev/*.pod
+	rm -rf .libs */.libs
 	rm -f qemu-img-cmds.h
 	@# May not be present in GENERATED_HEADERS
 	rm -f trace/generated-tracers-dtrace.dtrace*
@@ -245,7 +246,6 @@ clean:
 	rm -f $(foreach f,$(GENERATED_SOURCES),$(f) $(f)-timestamp)
 	rm -rf qapi-generated
 	rm -rf qga/qapi-generated
-	$(MAKE) -C tests/tcg clean
 	for d in $(ALL_SUBDIRS); do \
 	if test -d $$d; then $(MAKE) -C $$d $@ || exit 1; fi; \
 	rm -f $$d/qemu-options.def; \
@@ -261,6 +261,7 @@ qemu-%.tar.bz2:
 distclean: clean
 	rm -f config-host.mak config-host.h* config-host.ld $(DOCS) qemu-options.texi qemu-img-cmds.texi qemu-monitor.texi
 	rm -f config-all-devices.mak config-all-disas.mak
+	rm -f po/*.mo
 	rm -f roms/seabios/config.mak roms/vgabios/config.mak
 	rm -f qemu-doc.info qemu-doc.aux qemu-doc.cp qemu-doc.cps qemu-doc.dvi
 	rm -f qemu-doc.fn qemu-doc.fns qemu-doc.info qemu-doc.ky qemu-doc.kys
@@ -304,7 +305,7 @@ endif
 install-doc: $(DOCS)
 	$(INSTALL_DIR) "$(DESTDIR)$(qemu_docdir)"
 	$(INSTALL_DATA) qemu-doc.html  qemu-tech.html "$(DESTDIR)$(qemu_docdir)"
-	$(INSTALL_DATA) QMP/qmp-commands.txt "$(DESTDIR)$(qemu_docdir)"
+	$(INSTALL_DATA) qmp-commands.txt "$(DESTDIR)$(qemu_docdir)"
 ifdef CONFIG_POSIX
 	$(INSTALL_DIR) "$(DESTDIR)$(mandir)/man1"
 	$(INSTALL_DATA) qemu.1 "$(DESTDIR)$(mandir)/man1"
@@ -398,7 +399,7 @@ qemu-options.texi: $(SRC_PATH)/qemu-options.hx
 qemu-monitor.texi: $(SRC_PATH)/hmp-commands.hx
 	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -t < $< > $@,"  GEN   $@")
 
-QMP/qmp-commands.txt: $(SRC_PATH)/qmp-commands.hx
+qmp-commands.txt: $(SRC_PATH)/qmp-commands.hx
 	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -q < $< > $@,"  GEN   $@")
 
 qemu-img-cmds.texi: $(SRC_PATH)/qemu-img-cmds.hx
diff --git a/Makefile.target b/Makefile.target
index 9a4985213b..af6ac7eaa1 100644
--- a/Makefile.target
+++ b/Makefile.target
@@ -70,10 +70,6 @@ all: $(PROGS) stap
 # Dummy command so that make thinks it has done something
 	@true
 
-CONFIG_NO_PCI = $(if $(subst n,,$(CONFIG_PCI)),n,y)
-CONFIG_NO_KVM = $(if $(subst n,,$(CONFIG_KVM)),n,y)
-CONFIG_NO_XEN = $(if $(subst n,,$(CONFIG_XEN)),n,y)
-
 #########################################################
 # cpu emulator library
 obj-y = exec.o translate-all.o cpu-exec.o
@@ -83,8 +79,8 @@ obj-$(CONFIG_TCG_INTERPRETER) += disas/tci.o
 obj-y += fpu/softfloat.o
 obj-y += target-$(TARGET_BASE_ARCH)/
 obj-y += disas.o
-obj-$(CONFIG_GDBSTUB_XML) += gdbstub-xml.o
-obj-$(CONFIG_NO_KVM) += kvm-stub.o
+obj-$(call notempty,$(TARGET_XML_FILES)) += gdbstub-xml.o
+obj-$(call lnot,$(CONFIG_KVM)) += kvm-stub.o
 
 #########################################################
 # Linux user emulator target
@@ -125,7 +121,7 @@ LIBS+=$(libs_softmmu)
 
 # xen support
 obj-$(CONFIG_XEN) += xen-all.o xen-mapcache.o
-obj-$(CONFIG_NO_XEN) += xen-stub.o
+obj-$(call lnot,$(CONFIG_XEN)) += xen-stub.o
 
 # Hardware support
 ifeq ($(TARGET_NAME), sparc64)
diff --git a/QMP/README b/QMP/README
deleted file mode 100644
index c95a08c234..0000000000
--- a/QMP/README
+++ /dev/null
@@ -1,88 +0,0 @@
-                          QEMU Monitor Protocol
-                          =====================
-
-Introduction
--------------
-
-The QEMU Monitor Protocol (QMP) allows applications to communicate with
-QEMU's Monitor.
-
-QMP is JSON[1] based and currently has the following features:
-
-- Lightweight, text-based, easy to parse data format
-- Asynchronous messages support (ie. events)
-- Capabilities Negotiation
-
-For detailed information on QMP's usage, please, refer to the following files:
-
-o qmp-spec.txt      QEMU Monitor Protocol current specification
-o qmp-commands.txt  QMP supported commands (auto-generated at build-time)
-o qmp-events.txt    List of available asynchronous events
-
-There is also a simple Python script called 'qmp-shell' available.
-
-IMPORTANT: It's strongly recommended to read the 'Stability Considerations'
-section in the qmp-commands.txt file before making any serious use of QMP.
-
-
-[1] http://www.json.org
-
-Usage
------
-
-To enable QMP, you need a QEMU monitor instance in "control mode". There are
-two ways of doing this.
-
-The simplest one is using the '-qmp' command-line option. The following
-example makes QMP available on localhost port 4444:
-
-  $ qemu [...] -qmp tcp:localhost:4444,server
-
-However, in order to have more complex combinations, like multiple monitors,
-the '-mon' command-line option should be used along with the '-chardev' one.
-For instance, the following example creates one user monitor on stdio and one
-QMP monitor on localhost port 4444.
-
-   $ qemu [...] -chardev stdio,id=mon0 -mon chardev=mon0,mode=readline \
-                -chardev socket,id=mon1,host=localhost,port=4444,server \
-                -mon chardev=mon1,mode=control
-
-Please, refer to QEMU's manpage for more information.
-
-Simple Testing
---------------
-
-To manually test QMP one can connect with telnet and issue commands by hand:
-
-$ telnet localhost 4444
-Trying 127.0.0.1...
-Connected to localhost.
-Escape character is '^]'.
-{"QMP": {"version": {"qemu": {"micro": 50, "minor": 13, "major": 0}, "package": ""}, "capabilities": []}}
-{ "execute": "qmp_capabilities" }
-{"return": {}}
-{ "execute": "query-version" }
-{"return": {"qemu": {"micro": 50, "minor": 13, "major": 0}, "package": ""}}
-
-Development Process
--------------------
-
-When changing QMP's interface (by adding new commands, events or modifying
-existing ones) it's mandatory to update the relevant documentation, which is
-one (or more) of the files listed in the 'Introduction' section*.
-
-Also, it's strongly recommended to send the documentation patch first, before
-doing any code change. This is so because:
-
-  1. Avoids the code dictating the interface
-
-  2. Review can improve your interface.  Letting that happen before
-     you implement it can save you work.
-
-* The qmp-commands.txt file is generated from the qmp-commands.hx one, which
-  is the file that should be edited.
-
-Homepage
---------
-
-http://wiki.qemu.org/QMP
diff --git a/README b/README
index c77d12642d..c7c990d895 100644
--- a/README
+++ b/README
@@ -1,3 +1,3 @@
-Read the documentation in qemu-doc.html or on http://wiki.qemu.org
+Read the documentation in qemu-doc.html or on http://wiki.qemu-project.org
 
 - QEMU team
diff --git a/arch_init.c b/arch_init.c
index 150647b6bb..7545d96739 100644
--- a/arch_init.c
+++ b/arch_init.c
@@ -150,10 +150,9 @@ int qemu_read_default_config_files(bool userconfig)
     return 0;
 }
 
-static inline bool is_zero_page(uint8_t *p)
+static inline bool is_zero_range(uint8_t *p, uint64_t size)
 {
-    return buffer_find_nonzero_offset(p, TARGET_PAGE_SIZE) ==
-        TARGET_PAGE_SIZE;
+    return buffer_find_nonzero_offset(p, size) == size;
 }
 
 /* struct contains XBZRLE cache and a static page
@@ -497,7 +496,7 @@ static int ram_save_block(QEMUFile *f, bool last_stage)
                         acct_info.dup_pages++;
                     }
                 }
-            } else if (is_zero_page(p)) {
+            } else if (is_zero_range(p, TARGET_PAGE_SIZE)) {
                 acct_info.dup_pages++;
                 bytes_sent = save_block_hdr(f, block, offset, cont,
                                             RAM_SAVE_FLAG_COMPRESS);
@@ -710,15 +709,20 @@ static int ram_save_iterate(QEMUFile *f, void *opaque)
      */
     ram_control_after_iterate(f, RAM_CONTROL_ROUND);
 
+    bytes_transferred += total_sent;
+
+    /*
+     * Do not count these 8 bytes into total_sent, so that we can
+     * return 0 if no page had been dirtied.
+     */
+    qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
+    bytes_transferred += 8;
+
+    ret = qemu_file_get_error(f);
     if (ret < 0) {
-        bytes_transferred += total_sent;
         return ret;
     }
 
-    qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
-    total_sent += 8;
-    bytes_transferred += total_sent;
-
     return total_sent;
 }
 
@@ -844,13 +848,14 @@ static inline void *host_from_stream_offset(QEMUFile *f,
  */
 void ram_handle_compressed(void *host, uint8_t ch, uint64_t size)
 {
-    if (ch != 0 || !is_zero_page(host)) {
+    if (ch != 0 || !is_zero_range(host, size)) {
         memset(host, ch, size);
 #ifndef _WIN32
-        if (ch == 0 &&
-            (!kvm_enabled() || kvm_has_sync_mmu()) &&
-            getpagesize() <= TARGET_PAGE_SIZE) {
-            qemu_madvise(host, TARGET_PAGE_SIZE, QEMU_MADV_DONTNEED);
+        if (ch == 0 && (!kvm_enabled() || kvm_has_sync_mmu())) {
+            size = size & ~(getpagesize() - 1);
+            if (size > 0) {
+                qemu_madvise(host, size, QEMU_MADV_DONTNEED);
+            }
         }
 #endif
     }
diff --git a/audio/mixeng.c b/audio/mixeng.c
index 02a9d9fb92..0e4976f271 100644
--- a/audio/mixeng.c
+++ b/audio/mixeng.c
@@ -348,7 +348,6 @@ void mixeng_clear (struct st_sample *buf, int len)
 
 void mixeng_volume (struct st_sample *buf, int len, struct mixeng_volume *vol)
 {
-#ifdef CONFIG_MIXEMU
     if (vol->mute) {
         mixeng_clear (buf, len);
         return;
@@ -364,9 +363,4 @@ void mixeng_volume (struct st_sample *buf, int len, struct mixeng_volume *vol)
 #endif
         buf += 1;
     }
-#else
-    (void) buf;
-    (void) len;
-    (void) vol;
-#endif
 }
diff --git a/block.c b/block.c
index a325efcb21..fd05a8008a 100644
--- a/block.c
+++ b/block.c
@@ -394,18 +394,26 @@ typedef struct CreateCo {
     char *filename;
     QEMUOptionParameter *options;
     int ret;
+    Error *err;
 } CreateCo;
 
 static void coroutine_fn bdrv_create_co_entry(void *opaque)
 {
+    Error *local_err = NULL;
+    int ret;
+
     CreateCo *cco = opaque;
     assert(cco->drv);
 
-    cco->ret = cco->drv->bdrv_create(cco->filename, cco->options);
+    ret = cco->drv->bdrv_create(cco->filename, cco->options, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(&cco->err, local_err);
+    }
+    cco->ret = ret;
 }
 
 int bdrv_create(BlockDriver *drv, const char* filename,
-    QEMUOptionParameter *options)
+    QEMUOptionParameter *options, Error **errp)
 {
     int ret;
 
@@ -415,9 +423,11 @@ int bdrv_create(BlockDriver *drv, const char* filename,
         .filename = g_strdup(filename),
         .options = options,
         .ret = NOT_DONE,
+        .err = NULL,
     };
 
     if (!drv->bdrv_create) {
+        error_setg(errp, "Driver '%s' does not support image creation", drv->format_name);
         ret = -ENOTSUP;
         goto out;
     }
@@ -434,22 +444,37 @@ int bdrv_create(BlockDriver *drv, const char* filename,
     }
 
     ret = cco.ret;
+    if (ret < 0) {
+        if (error_is_set(&cco.err)) {
+            error_propagate(errp, cco.err);
+        } else {
+            error_setg_errno(errp, -ret, "Could not create image");
+        }
+    }
 
 out:
     g_free(cco.filename);
     return ret;
 }
 
-int bdrv_create_file(const char* filename, QEMUOptionParameter *options)
+int bdrv_create_file(const char* filename, QEMUOptionParameter *options,
+                     Error **errp)
 {
     BlockDriver *drv;
+    Error *local_err = NULL;
+    int ret;
 
     drv = bdrv_find_protocol(filename, true);
     if (drv == NULL) {
+        error_setg(errp, "Could not find protocol for file '%s'", filename);
         return -ENOENT;
     }
 
-    return bdrv_create(drv, filename, options);
+    ret = bdrv_create(drv, filename, options, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(errp, local_err);
+    }
+    return ret;
 }
 
 /*
@@ -552,7 +577,7 @@ BlockDriver *bdrv_find_protocol(const char *filename,
 }
 
 static int find_image_format(BlockDriverState *bs, const char *filename,
-                             BlockDriver **pdrv)
+                             BlockDriver **pdrv, Error **errp)
 {
     int score, score_max;
     BlockDriver *drv1, *drv;
@@ -563,6 +588,7 @@ static int find_image_format(BlockDriverState *bs, const char *filename,
     if (bs->sg || !bdrv_is_inserted(bs) || bdrv_getlength(bs) == 0) {
         drv = bdrv_find_format("raw");
         if (!drv) {
+            error_setg(errp, "Could not find raw image format");
             ret = -ENOENT;
         }
         *pdrv = drv;
@@ -571,6 +597,8 @@ static int find_image_format(BlockDriverState *bs, const char *filename,
 
     ret = bdrv_pread(bs, 0, buf, sizeof(buf));
     if (ret < 0) {
+        error_setg_errno(errp, -ret, "Could not read image for determining its "
+                         "format");
         *pdrv = NULL;
         return ret;
     }
@@ -587,6 +615,8 @@ static int find_image_format(BlockDriverState *bs, const char *filename,
         }
     }
     if (!drv) {
+        error_setg(errp, "Could not determine image format: No compatible "
+                   "driver found");
         ret = -ENOENT;
     }
     *pdrv = drv;
@@ -706,10 +736,11 @@ static int bdrv_open_flags(BlockDriverState *bs, int flags)
  * Removes all processed options from *options.
  */
 static int bdrv_open_common(BlockDriverState *bs, BlockDriverState *file,
-    QDict *options, int flags, BlockDriver *drv)
+    QDict *options, int flags, BlockDriver *drv, Error **errp)
 {
     int ret, open_flags;
     const char *filename;
+    Error *local_err = NULL;
 
     assert(drv != NULL);
     assert(bs->file == NULL);
@@ -738,12 +769,22 @@ static int bdrv_open_common(BlockDriverState *bs, BlockDriverState *file,
     bs->read_only = !(open_flags & BDRV_O_RDWR);
 
     if (use_bdrv_whitelist && !bdrv_is_whitelisted(drv, bs->read_only)) {
+        error_setg(errp,
+                   !bs->read_only && bdrv_is_whitelisted(drv, true)
+                        ? "Driver '%s' can only be used for read-only devices"
+                        : "Driver '%s' is not whitelisted",
+                   drv->format_name);
         return -ENOTSUP;
     }
 
     assert(bs->copy_on_read == 0); /* bdrv_new() and bdrv_close() make it so */
-    if (!bs->read_only && (flags & BDRV_O_COPY_ON_READ)) {
-        bdrv_enable_copy_on_read(bs);
+    if (flags & BDRV_O_COPY_ON_READ) {
+        if (!bs->read_only) {
+            bdrv_enable_copy_on_read(bs);
+        } else {
+            error_setg(errp, "Can't use copy-on-read on read-only device");
+            return -EINVAL;
+        }
     }
 
     if (filename != NULL) {
@@ -760,33 +801,40 @@ static int bdrv_open_common(BlockDriverState *bs, BlockDriverState *file,
     /* Open the image, either directly or using a protocol */
     if (drv->bdrv_file_open) {
         assert(file == NULL);
-        assert(drv->bdrv_parse_filename || filename != NULL);
-        ret = drv->bdrv_file_open(bs, options, open_flags);
+        assert(!drv->bdrv_needs_filename || filename != NULL);
+        ret = drv->bdrv_file_open(bs, options, open_flags, &local_err);
     } else {
         if (file == NULL) {
-            qerror_report(ERROR_CLASS_GENERIC_ERROR, "Can't use '%s' as a "
-                          "block driver for the protocol level",
-                          drv->format_name);
+            error_setg(errp, "Can't use '%s' as a block driver for the "
+                       "protocol level", drv->format_name);
             ret = -EINVAL;
             goto free_and_fail;
         }
         bs->file = file;
-        ret = drv->bdrv_open(bs, options, open_flags);
+        ret = drv->bdrv_open(bs, options, open_flags, &local_err);
     }
 
     if (ret < 0) {
+        if (error_is_set(&local_err)) {
+            error_propagate(errp, local_err);
+        } else if (bs->filename[0]) {
+            error_setg_errno(errp, -ret, "Could not open '%s'", bs->filename);
+        } else {
+            error_setg_errno(errp, -ret, "Could not open image");
+        }
         goto free_and_fail;
     }
 
     ret = refresh_total_sectors(bs, bs->total_sectors);
     if (ret < 0) {
+        error_setg_errno(errp, -ret, "Could not refresh total sector count");
         goto free_and_fail;
     }
 
 #ifndef _WIN32
     if (bs->is_temporary) {
-        assert(filename != NULL);
-        unlink(filename);
+        assert(bs->filename[0] != '\0');
+        unlink(bs->filename);
     }
 #endif
     return 0;
@@ -808,12 +856,13 @@ free_and_fail:
  * dictionary, it needs to use QINCREF() before calling bdrv_file_open.
  */
 int bdrv_file_open(BlockDriverState **pbs, const char *filename,
-                   QDict *options, int flags)
+                   QDict *options, int flags, Error **errp)
 {
     BlockDriverState *bs;
     BlockDriver *drv;
     const char *drvname;
     bool allow_protocol_prefix = false;
+    Error *local_err = NULL;
     int ret;
 
     /* NULL means an empty set of options */
@@ -832,8 +881,8 @@ int bdrv_file_open(BlockDriverState **pbs, const char *filename,
         qdict_put(options, "filename", qstring_from_str(filename));
         allow_protocol_prefix = true;
     } else {
-        qerror_report(ERROR_CLASS_GENERIC_ERROR, "Can't specify 'file' and "
-                      "'filename' options at the same time");
+        error_setg(errp, "Can't specify 'file' and 'filename' options at the "
+                   "same time");
         ret = -EINVAL;
         goto fail;
     }
@@ -841,54 +890,54 @@ int bdrv_file_open(BlockDriverState **pbs, const char *filename,
     /* Find the right block driver */
     drvname = qdict_get_try_str(options, "driver");
     if (drvname) {
-        drv = bdrv_find_whitelisted_format(drvname, !(flags & BDRV_O_RDWR));
+        drv = bdrv_find_format(drvname);
+        if (!drv) {
+            error_setg(errp, "Unknown driver '%s'", drvname);
+        }
         qdict_del(options, "driver");
     } else if (filename) {
         drv = bdrv_find_protocol(filename, allow_protocol_prefix);
         if (!drv) {
-            qerror_report(ERROR_CLASS_GENERIC_ERROR, "Unknown protocol");
+            error_setg(errp, "Unknown protocol");
         }
     } else {
-        qerror_report(ERROR_CLASS_GENERIC_ERROR,
-                      "Must specify either driver or file");
+        error_setg(errp, "Must specify either driver or file");
         drv = NULL;
     }
 
     if (!drv) {
+        /* errp has been set already */
         ret = -ENOENT;
         goto fail;
     }
 
     /* Parse the filename and open it */
     if (drv->bdrv_parse_filename && filename) {
-        Error *local_err = NULL;
         drv->bdrv_parse_filename(filename, options, &local_err);
         if (error_is_set(&local_err)) {
-            qerror_report_err(local_err);
-            error_free(local_err);
+            error_propagate(errp, local_err);
             ret = -EINVAL;
             goto fail;
         }
         qdict_del(options, "filename");
-    } else if (!drv->bdrv_parse_filename && !filename) {
-        qerror_report(ERROR_CLASS_GENERIC_ERROR,
-                      "The '%s' block driver requires a file name",
-                      drv->format_name);
+    } else if (drv->bdrv_needs_filename && !filename) {
+        error_setg(errp, "The '%s' block driver requires a file name",
+                   drv->format_name);
         ret = -EINVAL;
         goto fail;
     }
 
-    ret = bdrv_open_common(bs, NULL, options, flags, drv);
+    ret = bdrv_open_common(bs, NULL, options, flags, drv, &local_err);
     if (ret < 0) {
+        error_propagate(errp, local_err);
         goto fail;
     }
 
     /* Check if any unknown options were used */
     if (qdict_size(options) != 0) {
         const QDictEntry *entry = qdict_first(options);
-        qerror_report(ERROR_CLASS_GENERIC_ERROR, "Block protocol '%s' doesn't "
-                      "support the option '%s'",
-                      drv->format_name, entry->key);
+        error_setg(errp, "Block protocol '%s' doesn't support the option '%s'",
+                   drv->format_name, entry->key);
         ret = -EINVAL;
         goto fail;
     }
@@ -915,11 +964,12 @@ fail:
  * function (even on failure), so if the caller intends to reuse the dictionary,
  * it needs to use QINCREF() before calling bdrv_file_open.
  */
-int bdrv_open_backing_file(BlockDriverState *bs, QDict *options)
+int bdrv_open_backing_file(BlockDriverState *bs, QDict *options, Error **errp)
 {
     char backing_filename[PATH_MAX];
     int back_flags, ret;
     BlockDriver *back_drv = NULL;
+    Error *local_err = NULL;
 
     if (bs->backing_hd != NULL) {
         QDECREF(options);
@@ -937,11 +987,12 @@ int bdrv_open_backing_file(BlockDriverState *bs, QDict *options)
     } else if (bs->backing_file[0] == '\0' && qdict_size(options) == 0) {
         QDECREF(options);
         return 0;
+    } else {
+        bdrv_get_full_backing_filename(bs, backing_filename,
+                                       sizeof(backing_filename));
     }
 
     bs->backing_hd = bdrv_new("");
-    bdrv_get_full_backing_filename(bs, backing_filename,
-                                   sizeof(backing_filename));
 
     if (bs->backing_format[0] != '\0') {
         back_drv = bdrv_find_format(bs->backing_format);
@@ -952,35 +1003,19 @@ int bdrv_open_backing_file(BlockDriverState *bs, QDict *options)
 
     ret = bdrv_open(bs->backing_hd,
                     *backing_filename ? backing_filename : NULL, options,
-                    back_flags, back_drv);
+                    back_flags, back_drv, &local_err);
+    pstrcpy(bs->backing_file, sizeof(bs->backing_file),
+            bs->backing_hd->file->filename);
     if (ret < 0) {
         bdrv_unref(bs->backing_hd);
         bs->backing_hd = NULL;
         bs->open_flags |= BDRV_O_NO_BACKING;
+        error_propagate(errp, local_err);
         return ret;
     }
     return 0;
 }
 
-static void extract_subqdict(QDict *src, QDict **dst, const char *start)
-{
-    const QDictEntry *entry, *next;
-    const char *p;
-
-    *dst = qdict_new();
-    entry = qdict_first(src);
-
-    while (entry != NULL) {
-        next = qdict_next(src, entry);
-        if (strstart(entry->key, start, &p)) {
-            qobject_incref(entry->value);
-            qdict_put_obj(*dst, p, entry->value);
-            qdict_del(src, entry->key);
-        }
-        entry = next;
-    }
-}
-
 /*
  * Opens a disk image (raw, qcow2, vmdk, ...)
  *
@@ -990,7 +1025,7 @@ static void extract_subqdict(QDict *src, QDict **dst, const char *start)
  * dictionary, it needs to use QINCREF() before calling bdrv_open.
  */
 int bdrv_open(BlockDriverState *bs, const char *filename, QDict *options,
-              int flags, BlockDriver *drv)
+              int flags, BlockDriver *drv, Error **errp)
 {
     int ret;
     /* TODO: extra byte is a hack to ensure MAX_PATH space on Windows. */
@@ -998,6 +1033,7 @@ int bdrv_open(BlockDriverState *bs, const char *filename, QDict *options,
     BlockDriverState *file = NULL;
     QDict *file_options = NULL;
     const char *drvname;
+    Error *local_err = NULL;
 
     /* NULL means an empty set of options */
     if (options == NULL) {
@@ -1016,7 +1052,7 @@ int bdrv_open(BlockDriverState *bs, const char *filename, QDict *options,
         char backing_filename[PATH_MAX];
 
         if (qdict_size(options) != 0) {
-            error_report("Can't use snapshot=on with driver-specific options");
+            error_setg(errp, "Can't use snapshot=on with driver-specific options");
             ret = -EINVAL;
             goto fail;
         }
@@ -1027,7 +1063,7 @@ int bdrv_open(BlockDriverState *bs, const char *filename, QDict *options,
 
         /* if there is a backing file, use it */
         bs1 = bdrv_new("");
-        ret = bdrv_open(bs1, filename, NULL, 0, drv);
+        ret = bdrv_open(bs1, filename, NULL, 0, drv, &local_err);
         if (ret < 0) {
             bdrv_unref(bs1);
             goto fail;
@@ -1038,6 +1074,7 @@ int bdrv_open(BlockDriverState *bs, const char *filename, QDict *options,
 
         ret = get_tmp_filename(tmp_filename, sizeof(tmp_filename));
         if (ret < 0) {
+            error_setg_errno(errp, -ret, "Could not get temporary filename");
             goto fail;
         }
 
@@ -1046,6 +1083,7 @@ int bdrv_open(BlockDriverState *bs, const char *filename, QDict *options,
             snprintf(backing_filename, sizeof(backing_filename),
                      "%s", filename);
         } else if (!realpath(filename, backing_filename)) {
+            error_setg_errno(errp, errno, "Could not resolve path '%s'", filename);
             ret = -errno;
             goto fail;
         }
@@ -1062,9 +1100,14 @@ int bdrv_open(BlockDriverState *bs, const char *filename, QDict *options,
                 drv->format_name);
         }
 
-        ret = bdrv_create(bdrv_qcow2, tmp_filename, create_options);
+        ret = bdrv_create(bdrv_qcow2, tmp_filename, create_options, &local_err);
         free_option_parameters(create_options);
         if (ret < 0) {
+            error_setg_errno(errp, -ret, "Could not create temporary overlay "
+                             "'%s': %s", tmp_filename,
+                             error_get_pretty(local_err));
+            error_free(local_err);
+            local_err = NULL;
             goto fail;
         }
 
@@ -1078,10 +1121,10 @@ int bdrv_open(BlockDriverState *bs, const char *filename, QDict *options,
         flags |= BDRV_O_ALLOW_RDWR;
     }
 
-    extract_subqdict(options, &file_options, "file.");
+    qdict_extract_subqdict(options, &file_options, "file.");
 
     ret = bdrv_file_open(&file, filename, file_options,
-                         bdrv_open_flags(bs, flags | BDRV_O_UNMAP));
+                         bdrv_open_flags(bs, flags | BDRV_O_UNMAP), &local_err);
     if (ret < 0) {
         goto fail;
     }
@@ -1089,12 +1132,12 @@ int bdrv_open(BlockDriverState *bs, const char *filename, QDict *options,
     /* Find the right image format driver */
     drvname = qdict_get_try_str(options, "driver");
     if (drvname) {
-        drv = bdrv_find_whitelisted_format(drvname, !(flags & BDRV_O_RDWR));
+        drv = bdrv_find_format(drvname);
         qdict_del(options, "driver");
     }
 
     if (!drv) {
-        ret = find_image_format(file, filename, &drv);
+        ret = find_image_format(file, filename, &drv, &local_err);
     }
 
     if (!drv) {
@@ -1102,7 +1145,7 @@ int bdrv_open(BlockDriverState *bs, const char *filename, QDict *options,
     }
 
     /* Open the image */
-    ret = bdrv_open_common(bs, file, options, flags, drv);
+    ret = bdrv_open_common(bs, file, options, flags, drv, &local_err);
     if (ret < 0) {
         goto unlink_and_fail;
     }
@@ -1116,8 +1159,8 @@ int bdrv_open(BlockDriverState *bs, const char *filename, QDict *options,
     if ((flags & BDRV_O_NO_BACKING) == 0) {
         QDict *backing_options;
 
-        extract_subqdict(options, &backing_options, "backing.");
-        ret = bdrv_open_backing_file(bs, backing_options);
+        qdict_extract_subqdict(options, &backing_options, "backing.");
+        ret = bdrv_open_backing_file(bs, backing_options, &local_err);
         if (ret < 0) {
             goto close_and_fail;
         }
@@ -1126,9 +1169,9 @@ int bdrv_open(BlockDriverState *bs, const char *filename, QDict *options,
     /* Check if any unknown options were used */
     if (qdict_size(options) != 0) {
         const QDictEntry *entry = qdict_first(options);
-        qerror_report(ERROR_CLASS_GENERIC_ERROR, "Block format '%s' used by "
-            "device '%s' doesn't support the option '%s'",
-            drv->format_name, bs->device_name, entry->key);
+        error_setg(errp, "Block format '%s' used by device '%s' doesn't "
+                   "support the option '%s'", drv->format_name, bs->device_name,
+                   entry->key);
 
         ret = -EINVAL;
         goto close_and_fail;
@@ -1152,11 +1195,17 @@ fail:
     QDECREF(bs->options);
     QDECREF(options);
     bs->options = NULL;
+    if (error_is_set(&local_err)) {
+        error_propagate(errp, local_err);
+    }
     return ret;
 
 close_and_fail:
     bdrv_close(bs);
     QDECREF(options);
+    if (error_is_set(&local_err)) {
+        error_propagate(errp, local_err);
+    }
     return ret;
 }
 
@@ -2613,7 +2662,7 @@ static int coroutine_fn bdrv_co_do_readv(BlockDriverState *bs,
             goto out;
         }
 
-        total_sectors = len >> BDRV_SECTOR_BITS;
+        total_sectors = DIV_ROUND_UP(len, BDRV_SECTOR_SIZE);
         max_nb_sectors = MAX(0, total_sectors - sector_num);
         if (max_nb_sectors > 0) {
             ret = drv->bdrv_co_readv(bs, sector_num,
@@ -3103,13 +3152,20 @@ static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
 
     ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum);
     if (ret < 0) {
+        *pnum = 0;
         return ret;
     }
 
+    if (ret & BDRV_BLOCK_RAW) {
+        assert(ret & BDRV_BLOCK_OFFSET_VALID);
+        return bdrv_get_block_status(bs->file, ret >> BDRV_SECTOR_BITS,
+                                     *pnum, pnum);
+    }
+
     if (!(ret & BDRV_BLOCK_DATA)) {
         if (bdrv_has_zero_init(bs)) {
             ret |= BDRV_BLOCK_ZERO;
-        } else {
+        } else if (bs->backing_hd) {
             BlockDriverState *bs2 = bs->backing_hd;
             int64_t length2 = bdrv_getlength(bs2);
             if (length2 >= 0 && sector_num >= (length2 >> BDRV_SECTOR_BITS)) {
@@ -3281,6 +3337,15 @@ int bdrv_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
     return drv->bdrv_get_info(bs, bdi);
 }
 
+ImageInfoSpecific *bdrv_get_specific_info(BlockDriverState *bs)
+{
+    BlockDriver *drv = bs->drv;
+    if (drv && drv->bdrv_get_specific_info) {
+        return drv->bdrv_get_specific_info(bs);
+    }
+    return NULL;
+}
+
 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
                       int64_t pos, int size)
 {
@@ -4433,6 +4498,7 @@ void bdrv_img_create(const char *filename, const char *fmt,
     BlockDriverState *bs = NULL;
     BlockDriver *drv, *proto_drv;
     BlockDriver *backing_drv = NULL;
+    Error *local_err = NULL;
     int ret = 0;
 
     /* Find driver and parse its options */
@@ -4519,10 +4585,13 @@ void bdrv_img_create(const char *filename, const char *fmt,
             bs = bdrv_new("");
 
             ret = bdrv_open(bs, backing_file->value.s, NULL, back_flags,
-                            backing_drv);
+                            backing_drv, &local_err);
             if (ret < 0) {
-                error_setg_errno(errp, -ret, "Could not open '%s'",
-                                 backing_file->value.s);
+                error_setg_errno(errp, -ret, "Could not open '%s': %s",
+                                 backing_file->value.s,
+                                 error_get_pretty(local_err));
+                error_free(local_err);
+                local_err = NULL;
                 goto out;
             }
             bdrv_get_geometry(bs, &size);
@@ -4541,22 +4610,19 @@ void bdrv_img_create(const char *filename, const char *fmt,
         print_option_parameters(param);
         puts("");
     }
-    ret = bdrv_create(drv, filename, param);
-    if (ret < 0) {
-        if (ret == -ENOTSUP) {
-            error_setg(errp,"Formatting or formatting option not supported for "
-                            "file format '%s'", fmt);
-        } else if (ret == -EFBIG) {
-            const char *cluster_size_hint = "";
-            if (get_option_parameter(create_options, BLOCK_OPT_CLUSTER_SIZE)) {
-                cluster_size_hint = " (try using a larger cluster size)";
-            }
-            error_setg(errp, "The image size is too large for file format '%s'%s",
-                       fmt, cluster_size_hint);
-        } else {
-            error_setg(errp, "%s: error while creating %s: %s", filename, fmt,
-                       strerror(-ret));
+    ret = bdrv_create(drv, filename, param, &local_err);
+    if (ret == -EFBIG) {
+        /* This is generally a better message than whatever the driver would
+         * deliver (especially because of the cluster_size_hint), since that
+         * is most probably not much different from "image too large". */
+        const char *cluster_size_hint = "";
+        if (get_option_parameter(create_options, BLOCK_OPT_CLUSTER_SIZE)) {
+            cluster_size_hint = " (try using a larger cluster size)";
         }
+        error_setg(errp, "The image size is too large for file format '%s'"
+                   "%s", fmt, cluster_size_hint);
+        error_free(local_err);
+        local_err = NULL;
     }
 
 out:
@@ -4566,6 +4632,9 @@ out:
     if (bs) {
         bdrv_unref(bs);
     }
+    if (error_is_set(&local_err)) {
+        error_propagate(errp, local_err);
+    }
 }
 
 AioContext *bdrv_get_aio_context(BlockDriverState *bs)
@@ -4579,3 +4648,30 @@ void bdrv_add_before_write_notifier(BlockDriverState *bs,
 {
     notifier_with_return_list_add(&bs->before_write_notifiers, notifier);
 }
+
+int bdrv_amend_options(BlockDriverState *bs, QEMUOptionParameter *options)
+{
+    if (bs->drv->bdrv_amend_options == NULL) {
+        return -ENOTSUP;
+    }
+    return bs->drv->bdrv_amend_options(bs, options);
+}
+
+ExtSnapshotPerm bdrv_check_ext_snapshot(BlockDriverState *bs)
+{
+    if (bs->drv->bdrv_check_ext_snapshot) {
+        return bs->drv->bdrv_check_ext_snapshot(bs);
+    }
+
+    if (bs->file && bs->file->drv && bs->file->drv->bdrv_check_ext_snapshot) {
+        return bs->file->drv->bdrv_check_ext_snapshot(bs);
+    }
+
+    /* external snapshots are allowed by default */
+    return EXT_SNAPSHOT_ALLOWED;
+}
+
+ExtSnapshotPerm bdrv_check_ext_snapshot_forbidden(BlockDriverState *bs)
+{
+    return EXT_SNAPSHOT_FORBIDDEN;
+}
diff --git a/block/backup.c b/block/backup.c
index 04c4b5c263..cad14c90b2 100644
--- a/block/backup.c
+++ b/block/backup.c
@@ -202,9 +202,9 @@ static void backup_iostatus_reset(BlockJob *job)
     bdrv_iostatus_reset(s->target);
 }
 
-static const BlockJobType backup_job_type = {
+static const BlockJobDriver backup_job_driver = {
     .instance_size  = sizeof(BackupBlockJob),
-    .job_type       = "backup",
+    .job_type       = BLOCK_JOB_TYPE_BACKUP,
     .set_speed      = backup_set_speed,
     .iostatus_reset = backup_iostatus_reset,
 };
@@ -370,7 +370,7 @@ void backup_start(BlockDriverState *bs, BlockDriverState *target,
         return;
     }
 
-    BackupBlockJob *job = block_job_create(&backup_job_type, bs, speed,
+    BackupBlockJob *job = block_job_create(&backup_job_driver, bs, speed,
                                            cb, opaque, errp);
     if (!job) {
         return;
diff --git a/block/blkdebug.c b/block/blkdebug.c
index 5d33e03608..16d2b91ac9 100644
--- a/block/blkdebug.c
+++ b/block/blkdebug.c
@@ -350,7 +350,8 @@ static QemuOptsList runtime_opts = {
     },
 };
 
-static int blkdebug_open(BlockDriverState *bs, QDict *options, int flags)
+static int blkdebug_open(BlockDriverState *bs, QDict *options, int flags,
+                         Error **errp)
 {
     BDRVBlkdebugState *s = bs->opaque;
     QemuOpts *opts;
@@ -361,8 +362,7 @@ static int blkdebug_open(BlockDriverState *bs, QDict *options, int flags)
     opts = qemu_opts_create_nofail(&runtime_opts);
     qemu_opts_absorb_qdict(opts, options, &local_err);
     if (error_is_set(&local_err)) {
-        qerror_report_err(local_err);
-        error_free(local_err);
+        error_propagate(errp, local_err);
         ret = -EINVAL;
         goto fail;
     }
@@ -372,6 +372,7 @@ static int blkdebug_open(BlockDriverState *bs, QDict *options, int flags)
     if (config) {
         ret = read_config(s, config);
         if (ret < 0) {
+            error_setg_errno(errp, -ret, "Could not read blkdebug config file");
             goto fail;
         }
     }
@@ -382,12 +383,14 @@ static int blkdebug_open(BlockDriverState *bs, QDict *options, int flags)
     /* Open the backing file */
     filename = qemu_opt_get(opts, "x-image");
     if (filename == NULL) {
+        error_setg(errp, "Could not retrieve image file name");
         ret = -EINVAL;
         goto fail;
     }
 
-    ret = bdrv_file_open(&bs->file, filename, NULL, flags);
+    ret = bdrv_file_open(&bs->file, filename, NULL, flags, &local_err);
     if (ret < 0) {
+        error_propagate(errp, local_err);
         goto fail;
     }
 
diff --git a/block/blkverify.c b/block/blkverify.c
index c4e961eeb1..3c6352898f 100644
--- a/block/blkverify.c
+++ b/block/blkverify.c
@@ -116,7 +116,8 @@ static QemuOptsList runtime_opts = {
     },
 };
 
-static int blkverify_open(BlockDriverState *bs, QDict *options, int flags)
+static int blkverify_open(BlockDriverState *bs, QDict *options, int flags,
+                          Error **errp)
 {
     BDRVBlkverifyState *s = bs->opaque;
     QemuOpts *opts;
@@ -127,8 +128,7 @@ static int blkverify_open(BlockDriverState *bs, QDict *options, int flags)
     opts = qemu_opts_create_nofail(&runtime_opts);
     qemu_opts_absorb_qdict(opts, options, &local_err);
     if (error_is_set(&local_err)) {
-        qerror_report_err(local_err);
-        error_free(local_err);
+        error_propagate(errp, local_err);
         ret = -EINVAL;
         goto fail;
     }
@@ -136,25 +136,29 @@ static int blkverify_open(BlockDriverState *bs, QDict *options, int flags)
     /* Parse the raw image filename */
     raw = qemu_opt_get(opts, "x-raw");
     if (raw == NULL) {
+        error_setg(errp, "Could not retrieve raw image filename");
         ret = -EINVAL;
         goto fail;
     }
 
-    ret = bdrv_file_open(&bs->file, raw, NULL, flags);
+    ret = bdrv_file_open(&bs->file, raw, NULL, flags, &local_err);
     if (ret < 0) {
+        error_propagate(errp, local_err);
         goto fail;
     }
 
     /* Open the test file */
     filename = qemu_opt_get(opts, "x-image");
     if (filename == NULL) {
+        error_setg(errp, "Could not retrieve test image filename");
         ret = -EINVAL;
         goto fail;
     }
 
     s->test_file = bdrv_new("");
-    ret = bdrv_open(s->test_file, filename, NULL, flags, NULL);
+    ret = bdrv_open(s->test_file, filename, NULL, flags, NULL, &local_err);
     if (ret < 0) {
+        error_propagate(errp, local_err);
         bdrv_unref(s->test_file);
         s->test_file = NULL;
         goto fail;
@@ -412,6 +416,8 @@ static BlockDriver bdrv_blkverify = {
     .bdrv_aio_readv         = blkverify_aio_readv,
     .bdrv_aio_writev        = blkverify_aio_writev,
     .bdrv_aio_flush         = blkverify_aio_flush,
+
+    .bdrv_check_ext_snapshot = bdrv_check_ext_snapshot_forbidden,
 };
 
 static void bdrv_blkverify_init(void)
diff --git a/block/bochs.c b/block/bochs.c
index d7078c0775..51d9a90577 100644
--- a/block/bochs.c
+++ b/block/bochs.c
@@ -108,7 +108,8 @@ static int bochs_probe(const uint8_t *buf, int buf_size, const char *filename)
     return 0;
 }
 
-static int bochs_open(BlockDriverState *bs, QDict *options, int flags)
+static int bochs_open(BlockDriverState *bs, QDict *options, int flags,
+                      Error **errp)
 {
     BDRVBochsState *s = bs->opaque;
     int i;
diff --git a/block/cloop.c b/block/cloop.c
index 6ea7cf4046..b907023e10 100644
--- a/block/cloop.c
+++ b/block/cloop.c
@@ -53,7 +53,8 @@ static int cloop_probe(const uint8_t *buf, int buf_size, const char *filename)
     return 0;
 }
 
-static int cloop_open(BlockDriverState *bs, QDict *options, int flags)
+static int cloop_open(BlockDriverState *bs, QDict *options, int flags,
+                      Error **errp)
 {
     BDRVCloopState *s = bs->opaque;
     uint32_t offsets_size, max_compressed_block_size = 1, i;
diff --git a/block/commit.c b/block/commit.c
index ac4b7ccbc9..d4090cbf7d 100644
--- a/block/commit.c
+++ b/block/commit.c
@@ -173,9 +173,9 @@ static void commit_set_speed(BlockJob *job, int64_t speed, Error **errp)
     ratelimit_set_speed(&s->limit, speed / BDRV_SECTOR_SIZE, SLICE_TIME);
 }
 
-static const BlockJobType commit_job_type = {
+static const BlockJobDriver commit_job_driver = {
     .instance_size = sizeof(CommitBlockJob),
-    .job_type      = "commit",
+    .job_type      = BLOCK_JOB_TYPE_COMMIT,
     .set_speed     = commit_set_speed,
 };
 
@@ -238,7 +238,7 @@ void commit_start(BlockDriverState *bs, BlockDriverState *base,
     }
 
 
-    s = block_job_create(&commit_job_type, bs, speed, cb, opaque, errp);
+    s = block_job_create(&commit_job_driver, bs, speed, cb, opaque, errp);
     if (!s) {
         return;
     }
diff --git a/block/cow.c b/block/cow.c
index 764b93fae0..909c3e7182 100644
--- a/block/cow.c
+++ b/block/cow.c
@@ -58,7 +58,8 @@ static int cow_probe(const uint8_t *buf, int buf_size, const char *filename)
         return 0;
 }
 
-static int cow_open(BlockDriverState *bs, QDict *options, int flags)
+static int cow_open(BlockDriverState *bs, QDict *options, int flags,
+                    Error **errp)
 {
     BDRVCowState *s = bs->opaque;
     struct cow_header_v2 cow_header;
@@ -294,12 +295,14 @@ static void cow_close(BlockDriverState *bs)
 {
 }
 
-static int cow_create(const char *filename, QEMUOptionParameter *options)
+static int cow_create(const char *filename, QEMUOptionParameter *options,
+                      Error **errp)
 {
     struct cow_header_v2 cow_header;
     struct stat st;
     int64_t image_sectors = 0;
     const char *image_filename = NULL;
+    Error *local_err = NULL;
     int ret;
     BlockDriverState *cow_bs;
 
@@ -313,13 +316,17 @@ static int cow_create(const char *filename, QEMUOptionParameter *options)
         options++;
     }
 
-    ret = bdrv_create_file(filename, options);
+    ret = bdrv_create_file(filename, options, &local_err);
     if (ret < 0) {
+        qerror_report_err(local_err);
+        error_free(local_err);
         return ret;
     }
 
-    ret = bdrv_file_open(&cow_bs, filename, NULL, BDRV_O_RDWR);
+    ret = bdrv_file_open(&cow_bs, filename, NULL, BDRV_O_RDWR, &local_err);
     if (ret < 0) {
+        qerror_report_err(local_err);
+        error_free(local_err);
         return ret;
     }
 
diff --git a/block/curl.c b/block/curl.c
index ca2cedcec1..5a46f9707c 100644
--- a/block/curl.c
+++ b/block/curl.c
@@ -395,7 +395,8 @@ static QemuOptsList runtime_opts = {
     },
 };
 
-static int curl_open(BlockDriverState *bs, QDict *options, int flags)
+static int curl_open(BlockDriverState *bs, QDict *options, int flags,
+                     Error **errp)
 {
     BDRVCURLState *s = bs->opaque;
     CURLState *state = NULL;
diff --git a/block/dmg.c b/block/dmg.c
index 3141cb5b88..d5e9b1ff01 100644
--- a/block/dmg.c
+++ b/block/dmg.c
@@ -92,7 +92,8 @@ static int read_uint32(BlockDriverState *bs, int64_t offset, uint32_t *result)
     return 0;
 }
 
-static int dmg_open(BlockDriverState *bs, QDict *options, int flags)
+static int dmg_open(BlockDriverState *bs, QDict *options, int flags,
+                    Error **errp)
 {
     BDRVDMGState *s = bs->opaque;
     uint64_t info_begin,info_end,last_in_offset,last_out_offset;
diff --git a/block/gluster.c b/block/gluster.c
index dbb03f4de5..877686a7fe 100644
--- a/block/gluster.c
+++ b/block/gluster.c
@@ -288,7 +288,7 @@ static QemuOptsList runtime_opts = {
 };
 
 static int qemu_gluster_open(BlockDriverState *bs,  QDict *options,
-                             int bdrv_flags)
+                             int bdrv_flags, Error **errp)
 {
     BDRVGlusterState *s = bs->opaque;
     int open_flags = O_BINARY;
@@ -357,7 +357,7 @@ out:
 }
 
 static int qemu_gluster_create(const char *filename,
-        QEMUOptionParameter *options)
+        QEMUOptionParameter *options, Error **errp)
 {
     struct glfs *glfs;
     struct glfs_fd *fd;
@@ -611,6 +611,7 @@ static BlockDriver bdrv_gluster = {
     .format_name                  = "gluster",
     .protocol_name                = "gluster",
     .instance_size                = sizeof(BDRVGlusterState),
+    .bdrv_needs_filename          = true,
     .bdrv_file_open               = qemu_gluster_open,
     .bdrv_close                   = qemu_gluster_close,
     .bdrv_create                  = qemu_gluster_create,
@@ -631,6 +632,7 @@ static BlockDriver bdrv_gluster_tcp = {
     .format_name                  = "gluster",
     .protocol_name                = "gluster+tcp",
     .instance_size                = sizeof(BDRVGlusterState),
+    .bdrv_needs_filename          = true,
     .bdrv_file_open               = qemu_gluster_open,
     .bdrv_close                   = qemu_gluster_close,
     .bdrv_create                  = qemu_gluster_create,
@@ -651,6 +653,7 @@ static BlockDriver bdrv_gluster_unix = {
     .format_name                  = "gluster",
     .protocol_name                = "gluster+unix",
     .instance_size                = sizeof(BDRVGlusterState),
+    .bdrv_needs_filename          = true,
     .bdrv_file_open               = qemu_gluster_open,
     .bdrv_close                   = qemu_gluster_close,
     .bdrv_create                  = qemu_gluster_create,
@@ -671,6 +674,7 @@ static BlockDriver bdrv_gluster_rdma = {
     .format_name                  = "gluster",
     .protocol_name                = "gluster+rdma",
     .instance_size                = sizeof(BDRVGlusterState),
+    .bdrv_needs_filename          = true,
     .bdrv_file_open               = qemu_gluster_open,
     .bdrv_close                   = qemu_gluster_close,
     .bdrv_create                  = qemu_gluster_create,
diff --git a/block/iscsi.c b/block/iscsi.c
index 813abd8fef..a2a961e163 100644
--- a/block/iscsi.c
+++ b/block/iscsi.c
@@ -33,6 +33,8 @@
 #include "trace.h"
 #include "block/scsi.h"
 #include "qemu/iov.h"
+#include "sysemu/sysemu.h"
+#include "qmp-commands.h"
 
 #include <iscsi/iscsi.h>
 #include <iscsi/scsi-lowlevel.h>
@@ -50,8 +52,21 @@ typedef struct IscsiLun {
     uint64_t num_blocks;
     int events;
     QEMUTimer *nop_timer;
+    uint8_t lbpme;
+    uint8_t lbprz;
+    struct scsi_inquiry_logical_block_provisioning lbp;
+    struct scsi_inquiry_block_limits bl;
 } IscsiLun;
 
+typedef struct IscsiTask {
+    int status;
+    int complete;
+    int retries;
+    int do_retry;
+    struct scsi_task *task;
+    Coroutine *co;
+} IscsiTask;
+
 typedef struct IscsiAIOCB {
     BlockDriverAIOCB common;
     QEMUIOVector *qiov;
@@ -72,6 +87,7 @@ typedef struct IscsiAIOCB {
 #define NOP_INTERVAL 5000
 #define MAX_NOP_FAILURES 3
 #define ISCSI_CMD_RETRIES 5
+#define ISCSI_MAX_UNMAP 131072
 
 static void
 iscsi_bh_cb(void *p)
@@ -105,6 +121,41 @@ iscsi_schedule_bh(IscsiAIOCB *acb)
     qemu_bh_schedule(acb->bh);
 }
 
+static void
+iscsi_co_generic_cb(struct iscsi_context *iscsi, int status,
+                        void *command_data, void *opaque)
+{
+    struct IscsiTask *iTask = opaque;
+    struct scsi_task *task = command_data;
+
+    iTask->complete = 1;
+    iTask->status = status;
+    iTask->do_retry = 0;
+    iTask->task = task;
+
+    if (iTask->retries-- > 0 && status == SCSI_STATUS_CHECK_CONDITION
+        && task->sense.key == SCSI_SENSE_UNIT_ATTENTION) {
+        iTask->do_retry = 1;
+        goto out;
+    }
+
+    if (status != SCSI_STATUS_GOOD) {
+        error_report("iSCSI: Failure. %s", iscsi_get_error(iscsi));
+    }
+
+out:
+    if (iTask->co) {
+        qemu_coroutine_enter(iTask->co, NULL);
+    }
+}
+
+static void iscsi_co_init_iscsitask(IscsiLun *iscsilun, struct IscsiTask *iTask)
+{
+    *iTask = (struct IscsiTask) {
+        .co         = qemu_coroutine_self(),
+        .retries    = ISCSI_CMD_RETRIES,
+    };
+}
 
 static void
 iscsi_abort_task_cb(struct iscsi_context *iscsi, int status, void *command_data,
@@ -568,88 +619,6 @@ iscsi_aio_flush(BlockDriverState *bs,
     return &acb->common;
 }
 
-static int iscsi_aio_discard_acb(IscsiAIOCB *acb);
-
-static void
-iscsi_unmap_cb(struct iscsi_context *iscsi, int status,
-                     void *command_data, void *opaque)
-{
-    IscsiAIOCB *acb = opaque;
-
-    if (acb->canceled != 0) {
-        return;
-    }
-
-    acb->status = 0;
-    if (status != 0) {
-        if (status == SCSI_STATUS_CHECK_CONDITION
-            && acb->task->sense.key == SCSI_SENSE_UNIT_ATTENTION
-            && acb->retries-- > 0) {
-            scsi_free_scsi_task(acb->task);
-            acb->task = NULL;
-            if (iscsi_aio_discard_acb(acb) == 0) {
-                iscsi_set_events(acb->iscsilun);
-                return;
-            }
-        }
-        error_report("Failed to unmap data on iSCSI lun. %s",
-                     iscsi_get_error(iscsi));
-        acb->status = -EIO;
-    }
-
-    iscsi_schedule_bh(acb);
-}
-
-static int iscsi_aio_discard_acb(IscsiAIOCB *acb) {
-    struct iscsi_context *iscsi = acb->iscsilun->iscsi;
-    struct unmap_list list[1];
-
-    acb->canceled   = 0;
-    acb->bh         = NULL;
-    acb->status     = -EINPROGRESS;
-    acb->buf        = NULL;
-
-    list[0].lba = sector_qemu2lun(acb->sector_num, acb->iscsilun);
-    list[0].num = acb->nb_sectors * BDRV_SECTOR_SIZE / acb->iscsilun->block_size;
-
-    acb->task = iscsi_unmap_task(iscsi, acb->iscsilun->lun,
-                                 0, 0, &list[0], 1,
-                                 iscsi_unmap_cb,
-                                 acb);
-    if (acb->task == NULL) {
-        error_report("iSCSI: Failed to send unmap command. %s",
-                     iscsi_get_error(iscsi));
-        return -1;
-    }
-
-    return 0;
-}
-
-static BlockDriverAIOCB *
-iscsi_aio_discard(BlockDriverState *bs,
-                  int64_t sector_num, int nb_sectors,
-                  BlockDriverCompletionFunc *cb, void *opaque)
-{
-    IscsiLun *iscsilun = bs->opaque;
-    IscsiAIOCB *acb;
-
-    acb = qemu_aio_get(&iscsi_aiocb_info, bs, cb, opaque);
-
-    acb->iscsilun    = iscsilun;
-    acb->nb_sectors  = nb_sectors;
-    acb->sector_num  = sector_num;
-    acb->retries     = ISCSI_CMD_RETRIES;
-
-    if (iscsi_aio_discard_acb(acb) != 0) {
-        qemu_aio_release(acb);
-        return NULL;
-    }
-
-    iscsi_set_events(iscsilun);
-
-    return &acb->common;
-}
-
 #ifdef __linux__
 static void
 iscsi_aio_ioctl_cb(struct iscsi_context *iscsi, int status,
@@ -842,6 +811,171 @@ iscsi_getlength(BlockDriverState *bs)
     return len;
 }
 
+#if defined(LIBISCSI_FEATURE_IOVECTOR)
+
+static int64_t coroutine_fn iscsi_co_get_block_status(BlockDriverState *bs,
+                                                  int64_t sector_num,
+                                                  int nb_sectors, int *pnum)
+{
+    IscsiLun *iscsilun = bs->opaque;
+    struct scsi_get_lba_status *lbas = NULL;
+    struct scsi_lba_status_descriptor *lbasd = NULL;
+    struct IscsiTask iTask;
+    int64_t ret;
+
+    iscsi_co_init_iscsitask(iscsilun, &iTask);
+
+    if (!is_request_lun_aligned(sector_num, nb_sectors, iscsilun)) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    /* default to all sectors allocated */
+    ret = BDRV_BLOCK_DATA;
+    ret |= (sector_num << BDRV_SECTOR_BITS) | BDRV_BLOCK_OFFSET_VALID;
+    *pnum = nb_sectors;
+
+    /* LUN does not support logical block provisioning */
+    if (iscsilun->lbpme == 0) {
+        goto out;
+    }
+
+retry:
+    if (iscsi_get_lba_status_task(iscsilun->iscsi, iscsilun->lun,
+                                  sector_qemu2lun(sector_num, iscsilun),
+                                  8 + 16, iscsi_co_generic_cb,
+                                  &iTask) == NULL) {
+        ret = -EIO;
+        goto out;
+    }
+
+    while (!iTask.complete) {
+        iscsi_set_events(iscsilun);
+        qemu_coroutine_yield();
+    }
+
+    if (iTask.do_retry) {
+        if (iTask.task != NULL) {
+            scsi_free_scsi_task(iTask.task);
+            iTask.task = NULL;
+        }
+        goto retry;
+    }
+
+    if (iTask.status != SCSI_STATUS_GOOD) {
+        /* in case the get_lba_status_callout fails (i.e.
+         * because the device is busy or the cmd is not
+         * supported) we pretend all blocks are allocated
+         * for backwards compatiblity */
+        goto out;
+    }
+
+    lbas = scsi_datain_unmarshall(iTask.task);
+    if (lbas == NULL) {
+        ret = -EIO;
+        goto out;
+    }
+
+    lbasd = &lbas->descriptors[0];
+
+    if (sector_qemu2lun(sector_num, iscsilun) != lbasd->lba) {
+        ret = -EIO;
+        goto out;
+    }
+
+    *pnum = sector_lun2qemu(lbasd->num_blocks, iscsilun);
+    if (*pnum > nb_sectors) {
+        *pnum = nb_sectors;
+    }
+
+    if (lbasd->provisioning == SCSI_PROVISIONING_TYPE_DEALLOCATED ||
+        lbasd->provisioning == SCSI_PROVISIONING_TYPE_ANCHORED) {
+        ret &= ~BDRV_BLOCK_DATA;
+        if (iscsilun->lbprz) {
+            ret |= BDRV_BLOCK_ZERO;
+        }
+    }
+
+out:
+    if (iTask.task != NULL) {
+        scsi_free_scsi_task(iTask.task);
+    }
+    return ret;
+}
+
+#endif /* LIBISCSI_FEATURE_IOVECTOR */
+
+static int
+coroutine_fn iscsi_co_discard(BlockDriverState *bs, int64_t sector_num,
+                                   int nb_sectors)
+{
+    IscsiLun *iscsilun = bs->opaque;
+    struct IscsiTask iTask;
+    struct unmap_list list;
+    uint32_t nb_blocks;
+    uint32_t max_unmap;
+
+    if (!is_request_lun_aligned(sector_num, nb_sectors, iscsilun)) {
+        return -EINVAL;
+    }
+
+    if (!iscsilun->lbp.lbpu) {
+        /* UNMAP is not supported by the target */
+        return 0;
+    }
+
+    list.lba = sector_qemu2lun(sector_num, iscsilun);
+    nb_blocks = sector_qemu2lun(nb_sectors, iscsilun);
+
+    max_unmap = iscsilun->bl.max_unmap;
+    if (max_unmap == 0xffffffff) {
+        max_unmap = ISCSI_MAX_UNMAP;
+    }
+
+    while (nb_blocks > 0) {
+        iscsi_co_init_iscsitask(iscsilun, &iTask);
+        list.num = nb_blocks;
+        if (list.num > max_unmap) {
+            list.num = max_unmap;
+        }
+retry:
+        if (iscsi_unmap_task(iscsilun->iscsi, iscsilun->lun, 0, 0, &list, 1,
+                         iscsi_co_generic_cb, &iTask) == NULL) {
+            return -EIO;
+        }
+
+        while (!iTask.complete) {
+            iscsi_set_events(iscsilun);
+            qemu_coroutine_yield();
+        }
+
+        if (iTask.task != NULL) {
+            scsi_free_scsi_task(iTask.task);
+            iTask.task = NULL;
+        }
+
+        if (iTask.do_retry) {
+            goto retry;
+        }
+
+        if (iTask.status == SCSI_STATUS_CHECK_CONDITION) {
+            /* the target might fail with a check condition if it
+               is not happy with the alignment of the UNMAP request
+               we silently fail in this case */
+            return 0;
+        }
+
+        if (iTask.status != SCSI_STATUS_GOOD) {
+            return -EIO;
+        }
+
+        list.lba += list.num;
+        nb_blocks -= list.num;
+    }
+
+    return 0;
+}
+
 static int parse_chap(struct iscsi_context *iscsi, const char *target)
 {
     QemuOptsList *list;
@@ -922,8 +1056,9 @@ static char *parse_initiator_name(const char *target)
 {
     QemuOptsList *list;
     QemuOpts *opts;
-    const char *name = NULL;
-    const char *iscsi_name = qemu_get_vm_name();
+    const char *name;
+    char *iscsi_name;
+    UuidInfo *uuid_info;
 
     list = qemu_find_opts("iscsi");
     if (list) {
@@ -933,16 +1068,22 @@ static char *parse_initiator_name(const char *target)
         }
         if (opts) {
             name = qemu_opt_get(opts, "initiator-name");
+            if (name) {
+                return g_strdup(name);
+            }
         }
     }
 
-    if (name) {
-        return g_strdup(name);
+    uuid_info = qmp_query_uuid(NULL);
+    if (strcmp(uuid_info->UUID, UUID_NONE) == 0) {
+        name = qemu_get_vm_name();
     } else {
-        return g_strdup_printf("iqn.2008-11.org.linux-kvm%s%s",
-                               iscsi_name ? ":" : "",
-                               iscsi_name ? iscsi_name : "");
+        name = uuid_info->UUID;
     }
+    iscsi_name = g_strdup_printf("iqn.2008-11.org.linux-kvm%s%s",
+                                 name ? ":" : "", name ? name : "");
+    qapi_free_UuidInfo(uuid_info);
+    return iscsi_name;
 }
 
 #if defined(LIBISCSI_FEATURE_NOP_COUNTER)
@@ -990,6 +1131,8 @@ static int iscsi_readcapacity_sync(IscsiLun *iscsilun)
                 } else {
                     iscsilun->block_size = rc16->block_length;
                     iscsilun->num_blocks = rc16->returned_lba + 1;
+                    iscsilun->lbpme = rc16->lbpme;
+                    iscsilun->lbprz = rc16->lbprz;
                 }
             }
             break;
@@ -1042,11 +1185,43 @@ static QemuOptsList runtime_opts = {
     },
 };
 
+static struct scsi_task *iscsi_do_inquiry(struct iscsi_context *iscsi,
+                                          int lun, int evpd, int pc) {
+        int full_size;
+        struct scsi_task *task = NULL;
+        task = iscsi_inquiry_sync(iscsi, lun, evpd, pc, 64);
+        if (task == NULL || task->status != SCSI_STATUS_GOOD) {
+            goto fail;
+        }
+        full_size = scsi_datain_getfullsize(task);
+        if (full_size > task->datain.size) {
+            scsi_free_scsi_task(task);
+
+            /* we need more data for the full list */
+            task = iscsi_inquiry_sync(iscsi, lun, evpd, pc, full_size);
+            if (task == NULL || task->status != SCSI_STATUS_GOOD) {
+                goto fail;
+            }
+        }
+
+        return task;
+
+fail:
+        error_report("iSCSI: Inquiry command failed : %s",
+                     iscsi_get_error(iscsi));
+        if (task) {
+            scsi_free_scsi_task(task);
+            return NULL;
+        }
+        return NULL;
+}
+
 /*
  * We support iscsi url's on the form
  * iscsi://[<username>%<password>@]<host>[:<port>]/<targetname>/<lun>
  */
-static int iscsi_open(BlockDriverState *bs, QDict *options, int flags)
+static int iscsi_open(BlockDriverState *bs, QDict *options, int flags,
+                      Error **errp)
 {
     IscsiLun *iscsilun = bs->opaque;
     struct iscsi_context *iscsi = NULL;
@@ -1171,6 +1346,46 @@ static int iscsi_open(BlockDriverState *bs, QDict *options, int flags)
         bs->sg = 1;
     }
 
+    if (iscsilun->lbpme) {
+        struct scsi_inquiry_logical_block_provisioning *inq_lbp;
+        task = iscsi_do_inquiry(iscsilun->iscsi, iscsilun->lun, 1,
+                                SCSI_INQUIRY_PAGECODE_LOGICAL_BLOCK_PROVISIONING);
+        if (task == NULL) {
+            ret = -EINVAL;
+            goto out;
+        }
+        inq_lbp = scsi_datain_unmarshall(task);
+        if (inq_lbp == NULL) {
+            error_report("iSCSI: failed to unmarshall inquiry datain blob");
+            ret = -EINVAL;
+            goto out;
+        }
+        memcpy(&iscsilun->lbp, inq_lbp,
+               sizeof(struct scsi_inquiry_logical_block_provisioning));
+        scsi_free_scsi_task(task);
+        task = NULL;
+    }
+
+    if (iscsilun->lbp.lbpu || iscsilun->lbp.lbpws) {
+        struct scsi_inquiry_block_limits *inq_bl;
+        task = iscsi_do_inquiry(iscsilun->iscsi, iscsilun->lun, 1,
+                                SCSI_INQUIRY_PAGECODE_BLOCK_LIMITS);
+        if (task == NULL) {
+            ret = -EINVAL;
+            goto out;
+        }
+        inq_bl = scsi_datain_unmarshall(task);
+        if (inq_bl == NULL) {
+            error_report("iSCSI: failed to unmarshall inquiry datain blob");
+            ret = -EINVAL;
+            goto out;
+        }
+        memcpy(&iscsilun->bl, inq_bl,
+               sizeof(struct scsi_inquiry_block_limits));
+        scsi_free_scsi_task(task);
+        task = NULL;
+    }
+
 #if defined(LIBISCSI_FEATURE_NOP_COUNTER)
     /* Set up a timer for sending out iSCSI NOPs */
     iscsilun->nop_timer = timer_new_ms(QEMU_CLOCK_REALTIME, iscsi_nop_timed_event, iscsilun);
@@ -1237,7 +1452,8 @@ static int iscsi_has_zero_init(BlockDriverState *bs)
     return 0;
 }
 
-static int iscsi_create(const char *filename, QEMUOptionParameter *options)
+static int iscsi_create(const char *filename, QEMUOptionParameter *options,
+                        Error **errp)
 {
     int ret = 0;
     int64_t total_size = 0;
@@ -1260,7 +1476,7 @@ static int iscsi_create(const char *filename, QEMUOptionParameter *options)
 
     bs_options = qdict_new();
     qdict_put(bs_options, "filename", qstring_from_str(filename));
-    ret = iscsi_open(bs, bs_options, 0);
+    ret = iscsi_open(bs, bs_options, 0, NULL);
     QDECREF(bs_options);
 
     if (ret != 0) {
@@ -1304,6 +1520,7 @@ static BlockDriver bdrv_iscsi = {
     .protocol_name   = "iscsi",
 
     .instance_size   = sizeof(IscsiLun),
+    .bdrv_needs_filename = true,
     .bdrv_file_open  = iscsi_open,
     .bdrv_close      = iscsi_close,
     .bdrv_create     = iscsi_create,
@@ -1312,11 +1529,15 @@ static BlockDriver bdrv_iscsi = {
     .bdrv_getlength  = iscsi_getlength,
     .bdrv_truncate   = iscsi_truncate,
 
+#if defined(LIBISCSI_FEATURE_IOVECTOR)
+    .bdrv_co_get_block_status = iscsi_co_get_block_status,
+#endif
+    .bdrv_co_discard      = iscsi_co_discard,
+
     .bdrv_aio_readv  = iscsi_aio_readv,
     .bdrv_aio_writev = iscsi_aio_writev,
     .bdrv_aio_flush  = iscsi_aio_flush,
 
-    .bdrv_aio_discard = iscsi_aio_discard,
     .bdrv_has_zero_init = iscsi_has_zero_init,
 
 #ifdef __linux__
diff --git a/block/mirror.c b/block/mirror.c
index f61a7799de..7b95acf88c 100644
--- a/block/mirror.c
+++ b/block/mirror.c
@@ -505,14 +505,15 @@ static void mirror_iostatus_reset(BlockJob *job)
 static void mirror_complete(BlockJob *job, Error **errp)
 {
     MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
+    Error *local_err = NULL;
     int ret;
 
-    ret = bdrv_open_backing_file(s->target, NULL);
+    ret = bdrv_open_backing_file(s->target, NULL, &local_err);
     if (ret < 0) {
         char backing_filename[PATH_MAX];
         bdrv_get_full_backing_filename(s->target, backing_filename,
                                        sizeof(backing_filename));
-        error_setg_file_open(errp, -ret, backing_filename);
+        error_propagate(errp, local_err);
         return;
     }
     if (!s->synced) {
@@ -524,9 +525,9 @@ static void mirror_complete(BlockJob *job, Error **errp)
     block_job_resume(job);
 }
 
-static const BlockJobType mirror_job_type = {
+static const BlockJobDriver mirror_job_driver = {
     .instance_size = sizeof(MirrorBlockJob),
-    .job_type      = "mirror",
+    .job_type      = BLOCK_JOB_TYPE_MIRROR,
     .set_speed     = mirror_set_speed,
     .iostatus_reset= mirror_iostatus_reset,
     .complete      = mirror_complete,
@@ -562,7 +563,7 @@ void mirror_start(BlockDriverState *bs, BlockDriverState *target,
         return;
     }
 
-    s = block_job_create(&mirror_job_type, bs, speed, cb, opaque, errp);
+    s = block_job_create(&mirror_job_driver, bs, speed, cb, opaque, errp);
     if (!s) {
         return;
     }
diff --git a/block/nbd.c b/block/nbd.c
index 691066f726..c8deeee67f 100644
--- a/block/nbd.c
+++ b/block/nbd.c
@@ -453,7 +453,8 @@ static void nbd_teardown_connection(BlockDriverState *bs)
     closesocket(s->sock);
 }
 
-static int nbd_open(BlockDriverState *bs, QDict *options, int flags)
+static int nbd_open(BlockDriverState *bs, QDict *options, int flags,
+                    Error **errp)
 {
     BDRVNBDState *s = bs->opaque;
     int result;
diff --git a/block/parallels.c b/block/parallels.c
index 18b3ac0b28..2121e43204 100644
--- a/block/parallels.c
+++ b/block/parallels.c
@@ -68,7 +68,8 @@ static int parallels_probe(const uint8_t *buf, int buf_size, const char *filenam
     return 0;
 }
 
-static int parallels_open(BlockDriverState *bs, QDict *options, int flags)
+static int parallels_open(BlockDriverState *bs, QDict *options, int flags,
+                          Error **errp)
 {
     BDRVParallelsState *s = bs->opaque;
     int i;
diff --git a/block/qapi.c b/block/qapi.c
index 782051c65d..5880b3e42b 100644
--- a/block/qapi.c
+++ b/block/qapi.c
@@ -25,6 +25,9 @@
 #include "block/qapi.h"
 #include "block/block_int.h"
 #include "qmp-commands.h"
+#include "qapi-visit.h"
+#include "qapi/qmp-output-visitor.h"
+#include "qapi/qmp/types.h"
 
 /*
  * Returns 0 on success, with *p_list either set to describe snapshot
@@ -134,6 +137,9 @@ void bdrv_query_image_info(BlockDriverState *bs,
         info->dirty_flag = bdi.is_dirty;
         info->has_dirty_flag = true;
     }
+    info->format_specific     = bdrv_get_specific_info(bs);
+    info->has_format_specific = info->format_specific != NULL;
+
     backing_filename = bs->backing_file;
     if (backing_filename[0] != '\0') {
         info->backing_filename = g_strdup(backing_filename);
@@ -423,6 +429,119 @@ void bdrv_snapshot_dump(fprintf_function func_fprintf, void *f,
     }
 }
 
+static void dump_qdict(fprintf_function func_fprintf, void *f, int indentation,
+                       QDict *dict);
+static void dump_qlist(fprintf_function func_fprintf, void *f, int indentation,
+                       QList *list);
+
+static void dump_qobject(fprintf_function func_fprintf, void *f,
+                         int comp_indent, QObject *obj)
+{
+    switch (qobject_type(obj)) {
+        case QTYPE_QINT: {
+            QInt *value = qobject_to_qint(obj);
+            func_fprintf(f, "%" PRId64, qint_get_int(value));
+            break;
+        }
+        case QTYPE_QSTRING: {
+            QString *value = qobject_to_qstring(obj);
+            func_fprintf(f, "%s", qstring_get_str(value));
+            break;
+        }
+        case QTYPE_QDICT: {
+            QDict *value = qobject_to_qdict(obj);
+            dump_qdict(func_fprintf, f, comp_indent, value);
+            break;
+        }
+        case QTYPE_QLIST: {
+            QList *value = qobject_to_qlist(obj);
+            dump_qlist(func_fprintf, f, comp_indent, value);
+            break;
+        }
+        case QTYPE_QFLOAT: {
+            QFloat *value = qobject_to_qfloat(obj);
+            func_fprintf(f, "%g", qfloat_get_double(value));
+            break;
+        }
+        case QTYPE_QBOOL: {
+            QBool *value = qobject_to_qbool(obj);
+            func_fprintf(f, "%s", qbool_get_int(value) ? "true" : "false");
+            break;
+        }
+        case QTYPE_QERROR: {
+            QString *value = qerror_human((QError *)obj);
+            func_fprintf(f, "%s", qstring_get_str(value));
+            break;
+        }
+        case QTYPE_NONE:
+            break;
+        case QTYPE_MAX:
+        default:
+            abort();
+    }
+}
+
+static void dump_qlist(fprintf_function func_fprintf, void *f, int indentation,
+                       QList *list)
+{
+    const QListEntry *entry;
+    int i = 0;
+
+    for (entry = qlist_first(list); entry; entry = qlist_next(entry), i++) {
+        qtype_code type = qobject_type(entry->value);
+        bool composite = (type == QTYPE_QDICT || type == QTYPE_QLIST);
+        const char *format = composite ? "%*s[%i]:\n" : "%*s[%i]: ";
+
+        func_fprintf(f, format, indentation * 4, "", i);
+        dump_qobject(func_fprintf, f, indentation + 1, entry->value);
+        if (!composite) {
+            func_fprintf(f, "\n");
+        }
+    }
+}
+
+static void dump_qdict(fprintf_function func_fprintf, void *f, int indentation,
+                       QDict *dict)
+{
+    const QDictEntry *entry;
+
+    for (entry = qdict_first(dict); entry; entry = qdict_next(dict, entry)) {
+        qtype_code type = qobject_type(entry->value);
+        bool composite = (type == QTYPE_QDICT || type == QTYPE_QLIST);
+        const char *format = composite ? "%*s%s:\n" : "%*s%s: ";
+        char key[strlen(entry->key) + 1];
+        int i;
+
+        /* replace dashes with spaces in key (variable) names */
+        for (i = 0; entry->key[i]; i++) {
+            key[i] = entry->key[i] == '-' ? ' ' : entry->key[i];
+        }
+        key[i] = 0;
+
+        func_fprintf(f, format, indentation * 4, "", key);
+        dump_qobject(func_fprintf, f, indentation + 1, entry->value);
+        if (!composite) {
+            func_fprintf(f, "\n");
+        }
+    }
+}
+
+void bdrv_image_info_specific_dump(fprintf_function func_fprintf, void *f,
+                                   ImageInfoSpecific *info_spec)
+{
+    Error *local_err = NULL;
+    QmpOutputVisitor *ov = qmp_output_visitor_new();
+    QObject *obj, *data;
+
+    visit_type_ImageInfoSpecific(qmp_output_get_visitor(ov), &info_spec, NULL,
+                                 &local_err);
+    obj = qmp_output_get_qobject(ov);
+    assert(qobject_type(obj) == QTYPE_QDICT);
+    data = qdict_get(qobject_to_qdict(obj), "data");
+    dump_qobject(func_fprintf, f, 1, data);
+    qmp_output_visitor_cleanup(ov);
+}
+
 void bdrv_image_info_dump(fprintf_function func_fprintf, void *f,
                           ImageInfo *info)
 {
@@ -493,4 +612,9 @@ void bdrv_image_info_dump(fprintf_function func_fprintf, void *f,
             func_fprintf(f, "\n");
         }
     }
+
+    if (info->has_format_specific) {
+        func_fprintf(f, "Format specific information:\n");
+        bdrv_image_info_specific_dump(func_fprintf, f, info->format_specific);
+    }
 }
diff --git a/block/qcow.c b/block/qcow.c
index 93a993bb44..c470e05f60 100644
--- a/block/qcow.c
+++ b/block/qcow.c
@@ -92,7 +92,8 @@ static int qcow_probe(const uint8_t *buf, int buf_size, const char *filename)
         return 0;
 }
 
-static int qcow_open(BlockDriverState *bs, QDict *options, int flags)
+static int qcow_open(BlockDriverState *bs, QDict *options, int flags,
+                     Error **errp)
 {
     BDRVQcowState *s = bs->opaque;
     int len, i, shift, ret;
@@ -658,7 +659,8 @@ static void qcow_close(BlockDriverState *bs)
     error_free(s->migration_blocker);
 }
 
-static int qcow_create(const char *filename, QEMUOptionParameter *options)
+static int qcow_create(const char *filename, QEMUOptionParameter *options,
+                       Error **errp)
 {
     int header_size, backing_filename_len, l1_size, shift, i;
     QCowHeader header;
@@ -666,6 +668,7 @@ static int qcow_create(const char *filename, QEMUOptionParameter *options)
     int64_t total_size = 0;
     const char *backing_file = NULL;
     int flags = 0;
+    Error *local_err = NULL;
     int ret;
     BlockDriverState *qcow_bs;
 
@@ -681,13 +684,17 @@ static int qcow_create(const char *filename, QEMUOptionParameter *options)
         options++;
     }
 
-    ret = bdrv_create_file(filename, options);
+    ret = bdrv_create_file(filename, options, &local_err);
     if (ret < 0) {
+        qerror_report_err(local_err);
+        error_free(local_err);
         return ret;
     }
 
-    ret = bdrv_file_open(&qcow_bs, filename, NULL, BDRV_O_RDWR);
+    ret = bdrv_file_open(&qcow_bs, filename, NULL, BDRV_O_RDWR, &local_err);
     if (ret < 0) {
+        qerror_report_err(local_err);
+        error_free(local_err);
         return ret;
     }
 
diff --git a/block/qcow2-cache.c b/block/qcow2-cache.c
index 7bcae09a69..8ecbb5bc00 100644
--- a/block/qcow2-cache.c
+++ b/block/qcow2-cache.c
@@ -115,15 +115,13 @@ static int qcow2_cache_entry_flush(BlockDriverState *bs, Qcow2Cache *c, int i)
     }
 
     if (c == s->refcount_block_cache) {
-        ret = qcow2_pre_write_overlap_check(bs,
-                QCOW2_OL_DEFAULT & ~QCOW2_OL_REFCOUNT_BLOCK,
+        ret = qcow2_pre_write_overlap_check(bs, QCOW2_OL_REFCOUNT_BLOCK,
                 c->entries[i].offset, s->cluster_size);
     } else if (c == s->l2_table_cache) {
-        ret = qcow2_pre_write_overlap_check(bs,
-                QCOW2_OL_DEFAULT & ~QCOW2_OL_ACTIVE_L2,
+        ret = qcow2_pre_write_overlap_check(bs, QCOW2_OL_ACTIVE_L2,
                 c->entries[i].offset, s->cluster_size);
     } else {
-        ret = qcow2_pre_write_overlap_check(bs, QCOW2_OL_DEFAULT,
+        ret = qcow2_pre_write_overlap_check(bs, 0,
                 c->entries[i].offset, s->cluster_size);
     }
 
@@ -202,6 +200,24 @@ void qcow2_cache_depends_on_flush(Qcow2Cache *c)
     c->depends_on_flush = true;
 }
 
+int qcow2_cache_empty(BlockDriverState *bs, Qcow2Cache *c)
+{
+    int ret, i;
+
+    ret = qcow2_cache_flush(bs, c);
+    if (ret < 0) {
+        return ret;
+    }
+
+    for (i = 0; i < c->size; i++) {
+        assert(c->entries[i].ref == 0);
+        c->entries[i].offset = 0;
+        c->entries[i].cache_hits = 0;
+    }
+
+    return 0;
+}
+
 static int qcow2_cache_find_entry_to_replace(Qcow2Cache *c)
 {
     int i;
diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c
index 2d5aa92962..0348b971b1 100644
--- a/block/qcow2-cluster.c
+++ b/block/qcow2-cluster.c
@@ -35,6 +35,7 @@ int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
     BDRVQcowState *s = bs->opaque;
     int new_l1_size2, ret, i;
     uint64_t *new_l1_table;
+    int64_t old_l1_table_offset, old_l1_size;
     int64_t new_l1_table_offset, new_l1_size;
     uint8_t data[12];
 
@@ -82,8 +83,8 @@ int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
 
     /* the L1 position has not yet been updated, so these clusters must
      * indeed be completely free */
-    ret = qcow2_pre_write_overlap_check(bs, QCOW2_OL_DEFAULT,
-                                        new_l1_table_offset, new_l1_size2);
+    ret = qcow2_pre_write_overlap_check(bs, 0, new_l1_table_offset,
+                                        new_l1_size2);
     if (ret < 0) {
         goto fail;
     }
@@ -106,11 +107,13 @@ int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
         goto fail;
     }
     g_free(s->l1_table);
-    qcow2_free_clusters(bs, s->l1_table_offset, s->l1_size * sizeof(uint64_t),
-                        QCOW2_DISCARD_OTHER);
+    old_l1_table_offset = s->l1_table_offset;
     s->l1_table_offset = new_l1_table_offset;
     s->l1_table = new_l1_table;
+    old_l1_size = s->l1_size;
     s->l1_size = new_l1_size;
+    qcow2_free_clusters(bs, old_l1_table_offset, old_l1_size * sizeof(uint64_t),
+                        QCOW2_DISCARD_OTHER);
     return 0;
  fail:
     g_free(new_l1_table);
@@ -157,8 +160,7 @@ int qcow2_write_l1_entry(BlockDriverState *bs, int l1_index)
         buf[i] = cpu_to_be64(s->l1_table[l1_start_index + i]);
     }
 
-    ret = qcow2_pre_write_overlap_check(bs,
-            QCOW2_OL_DEFAULT & ~QCOW2_OL_ACTIVE_L1,
+    ret = qcow2_pre_write_overlap_check(bs, QCOW2_OL_ACTIVE_L1,
             s->l1_table_offset + 8 * l1_start_index, sizeof(buf));
     if (ret < 0) {
         return ret;
@@ -188,7 +190,7 @@ static int l2_allocate(BlockDriverState *bs, int l1_index, uint64_t **table)
 {
     BDRVQcowState *s = bs->opaque;
     uint64_t old_l2_offset;
-    uint64_t *l2_table;
+    uint64_t *l2_table = NULL;
     int64_t l2_offset;
     int ret;
 
@@ -200,7 +202,8 @@ static int l2_allocate(BlockDriverState *bs, int l1_index, uint64_t **table)
 
     l2_offset = qcow2_alloc_clusters(bs, s->l2_size * sizeof(uint64_t));
     if (l2_offset < 0) {
-        return l2_offset;
+        ret = l2_offset;
+        goto fail;
     }
 
     ret = qcow2_cache_flush(bs, s->refcount_block_cache);
@@ -213,7 +216,7 @@ static int l2_allocate(BlockDriverState *bs, int l1_index, uint64_t **table)
     trace_qcow2_l2_allocate_get_empty(bs, l1_index);
     ret = qcow2_cache_get_empty(bs, s->l2_table_cache, l2_offset, (void**) table);
     if (ret < 0) {
-        return ret;
+        goto fail;
     }
 
     l2_table = *table;
@@ -265,8 +268,14 @@ static int l2_allocate(BlockDriverState *bs, int l1_index, uint64_t **table)
 
 fail:
     trace_qcow2_l2_allocate_done(bs, l1_index, ret);
-    qcow2_cache_put(bs, s->l2_table_cache, (void**) table);
+    if (l2_table != NULL) {
+        qcow2_cache_put(bs, s->l2_table_cache, (void**) table);
+    }
     s->l1_table[l1_index] = old_l2_offset;
+    if (l2_offset > 0) {
+        qcow2_free_clusters(bs, l2_offset, s->l2_size * sizeof(uint64_t),
+                            QCOW2_DISCARD_ALWAYS);
+    }
     return ret;
 }
 
@@ -278,23 +287,26 @@ fail:
  * cluster which may require a different handling)
  */
 static int count_contiguous_clusters(uint64_t nb_clusters, int cluster_size,
-        uint64_t *l2_table, uint64_t start, uint64_t stop_flags)
+        uint64_t *l2_table, uint64_t stop_flags)
 {
     int i;
-    uint64_t mask = stop_flags | L2E_OFFSET_MASK;
-    uint64_t offset = be64_to_cpu(l2_table[0]) & mask;
+    uint64_t mask = stop_flags | L2E_OFFSET_MASK | QCOW2_CLUSTER_COMPRESSED;
+    uint64_t first_entry = be64_to_cpu(l2_table[0]);
+    uint64_t offset = first_entry & mask;
 
     if (!offset)
         return 0;
 
-    for (i = start; i < start + nb_clusters; i++) {
+    assert(qcow2_get_cluster_type(first_entry) != QCOW2_CLUSTER_COMPRESSED);
+
+    for (i = 0; i < nb_clusters; i++) {
         uint64_t l2_entry = be64_to_cpu(l2_table[i]) & mask;
         if (offset + (uint64_t) i * cluster_size != l2_entry) {
             break;
         }
     }
 
-	return (i - start);
+	return i;
 }
 
 static int count_contiguous_free_clusters(uint64_t nb_clusters, uint64_t *l2_table)
@@ -383,7 +395,7 @@ static int coroutine_fn copy_sectors(BlockDriverState *bs,
                         &s->aes_encrypt_key);
     }
 
-    ret = qcow2_pre_write_overlap_check(bs, QCOW2_OL_DEFAULT,
+    ret = qcow2_pre_write_overlap_check(bs, 0,
             cluster_offset + n_start * BDRV_SECTOR_SIZE, n * BDRV_SECTOR_SIZE);
     if (ret < 0) {
         goto out;
@@ -487,8 +499,7 @@ int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,
             return -EIO;
         }
         c = count_contiguous_clusters(nb_clusters, s->cluster_size,
-                &l2_table[l2_index], 0,
-                QCOW_OFLAG_COMPRESSED | QCOW_OFLAG_ZERO);
+                &l2_table[l2_index], QCOW_OFLAG_ZERO);
         *cluster_offset = 0;
         break;
     case QCOW2_CLUSTER_UNALLOCATED:
@@ -499,8 +510,7 @@ int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,
     case QCOW2_CLUSTER_NORMAL:
         /* how many allocated clusters ? */
         c = count_contiguous_clusters(nb_clusters, s->cluster_size,
-                &l2_table[l2_index], 0,
-                QCOW_OFLAG_COMPRESSED | QCOW_OFLAG_ZERO);
+                &l2_table[l2_index], QCOW_OFLAG_ZERO);
         *cluster_offset &= L2E_OFFSET_MASK;
         break;
     default:
@@ -716,6 +726,7 @@ int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m)
     }
     qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);
 
+    assert(l2_index + m->nb_clusters <= s->l2_size);
     for (i = 0; i < m->nb_clusters; i++) {
         /* if two concurrent writes happen to the same unallocated cluster
 	 * each write allocates separate cluster and writes data concurrently.
@@ -929,7 +940,7 @@ static int handle_copied(BlockDriverState *bs, uint64_t guest_offset,
         /* We keep all QCOW_OFLAG_COPIED clusters */
         keep_clusters =
             count_contiguous_clusters(nb_clusters, s->cluster_size,
-                                      &l2_table[l2_index], 0,
+                                      &l2_table[l2_index],
                                       QCOW_OFLAG_COPIED | QCOW_OFLAG_ZERO);
         assert(keep_clusters <= nb_clusters);
 
@@ -1338,7 +1349,7 @@ int qcow2_decompress_cluster(BlockDriverState *bs, uint64_t cluster_offset)
  * clusters.
  */
 static int discard_single_l2(BlockDriverState *bs, uint64_t offset,
-    unsigned int nb_clusters)
+    unsigned int nb_clusters, enum qcow2_discard_type type)
 {
     BDRVQcowState *s = bs->opaque;
     uint64_t *l2_table;
@@ -1367,7 +1378,7 @@ static int discard_single_l2(BlockDriverState *bs, uint64_t offset,
         l2_table[l2_index + i] = cpu_to_be64(0);
 
         /* Then decrease the refcount */
-        qcow2_free_any_clusters(bs, old_offset, 1, QCOW2_DISCARD_REQUEST);
+        qcow2_free_any_clusters(bs, old_offset, 1, type);
     }
 
     ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
@@ -1379,7 +1390,7 @@ static int discard_single_l2(BlockDriverState *bs, uint64_t offset,
 }
 
 int qcow2_discard_clusters(BlockDriverState *bs, uint64_t offset,
-    int nb_sectors)
+    int nb_sectors, enum qcow2_discard_type type)
 {
     BDRVQcowState *s = bs->opaque;
     uint64_t end_offset;
@@ -1402,7 +1413,7 @@ int qcow2_discard_clusters(BlockDriverState *bs, uint64_t offset,
 
     /* Each L2 table is handled by its own loop iteration */
     while (nb_clusters > 0) {
-        ret = discard_single_l2(bs, offset, nb_clusters);
+        ret = discard_single_l2(bs, offset, nb_clusters, type);
         if (ret < 0) {
             goto fail;
         }
@@ -1497,3 +1508,255 @@ fail:
 
     return ret;
 }
+
+/*
+ * Expands all zero clusters in a specific L1 table (or deallocates them, for
+ * non-backed non-pre-allocated zero clusters).
+ *
+ * expanded_clusters is a bitmap where every bit corresponds to one cluster in
+ * the image file; a bit gets set if the corresponding cluster has been used for
+ * zero expansion (i.e., has been filled with zeroes and is referenced from an
+ * L2 table). nb_clusters contains the total cluster count of the image file,
+ * i.e., the number of bits in expanded_clusters.
+ */
+static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
+                                      int l1_size, uint8_t **expanded_clusters,
+                                      uint64_t *nb_clusters)
+{
+    BDRVQcowState *s = bs->opaque;
+    bool is_active_l1 = (l1_table == s->l1_table);
+    uint64_t *l2_table = NULL;
+    int ret;
+    int i, j;
+
+    if (!is_active_l1) {
+        /* inactive L2 tables require a buffer to be stored in when loading
+         * them from disk */
+        l2_table = qemu_blockalign(bs, s->cluster_size);
+    }
+
+    for (i = 0; i < l1_size; i++) {
+        uint64_t l2_offset = l1_table[i] & L1E_OFFSET_MASK;
+        bool l2_dirty = false;
+
+        if (!l2_offset) {
+            /* unallocated */
+            continue;
+        }
+
+        if (is_active_l1) {
+            /* get active L2 tables from cache */
+            ret = qcow2_cache_get(bs, s->l2_table_cache, l2_offset,
+                    (void **)&l2_table);
+        } else {
+            /* load inactive L2 tables from disk */
+            ret = bdrv_read(bs->file, l2_offset / BDRV_SECTOR_SIZE,
+                    (void *)l2_table, s->cluster_sectors);
+        }
+        if (ret < 0) {
+            goto fail;
+        }
+
+        for (j = 0; j < s->l2_size; j++) {
+            uint64_t l2_entry = be64_to_cpu(l2_table[j]);
+            int64_t offset = l2_entry & L2E_OFFSET_MASK, cluster_index;
+            int cluster_type = qcow2_get_cluster_type(l2_entry);
+            bool preallocated = offset != 0;
+
+            if (cluster_type == QCOW2_CLUSTER_NORMAL) {
+                cluster_index = offset >> s->cluster_bits;
+                assert((cluster_index >= 0) && (cluster_index < *nb_clusters));
+                if ((*expanded_clusters)[cluster_index / 8] &
+                    (1 << (cluster_index % 8))) {
+                    /* Probably a shared L2 table; this cluster was a zero
+                     * cluster which has been expanded, its refcount
+                     * therefore most likely requires an update. */
+                    ret = qcow2_update_cluster_refcount(bs, cluster_index, 1,
+                                                        QCOW2_DISCARD_NEVER);
+                    if (ret < 0) {
+                        goto fail;
+                    }
+                    /* Since we just increased the refcount, the COPIED flag may
+                     * no longer be set. */
+                    l2_table[j] = cpu_to_be64(l2_entry & ~QCOW_OFLAG_COPIED);
+                    l2_dirty = true;
+                }
+                continue;
+            }
+            else if (qcow2_get_cluster_type(l2_entry) != QCOW2_CLUSTER_ZERO) {
+                continue;
+            }
+
+            if (!preallocated) {
+                if (!bs->backing_hd) {
+                    /* not backed; therefore we can simply deallocate the
+                     * cluster */
+                    l2_table[j] = 0;
+                    l2_dirty = true;
+                    continue;
+                }
+
+                offset = qcow2_alloc_clusters(bs, s->cluster_size);
+                if (offset < 0) {
+                    ret = offset;
+                    goto fail;
+                }
+            }
+
+            ret = qcow2_pre_write_overlap_check(bs, 0, offset, s->cluster_size);
+            if (ret < 0) {
+                if (!preallocated) {
+                    qcow2_free_clusters(bs, offset, s->cluster_size,
+                                        QCOW2_DISCARD_ALWAYS);
+                }
+                goto fail;
+            }
+
+            ret = bdrv_write_zeroes(bs->file, offset / BDRV_SECTOR_SIZE,
+                                    s->cluster_sectors);
+            if (ret < 0) {
+                if (!preallocated) {
+                    qcow2_free_clusters(bs, offset, s->cluster_size,
+                                        QCOW2_DISCARD_ALWAYS);
+                }
+                goto fail;
+            }
+
+            l2_table[j] = cpu_to_be64(offset | QCOW_OFLAG_COPIED);
+            l2_dirty = true;
+
+            cluster_index = offset >> s->cluster_bits;
+
+            if (cluster_index >= *nb_clusters) {
+                uint64_t old_bitmap_size = (*nb_clusters + 7) / 8;
+                uint64_t new_bitmap_size;
+                /* The offset may lie beyond the old end of the underlying image
+                 * file for growable files only */
+                assert(bs->file->growable);
+                *nb_clusters = size_to_clusters(s, bs->file->total_sectors *
+                                                BDRV_SECTOR_SIZE);
+                new_bitmap_size = (*nb_clusters + 7) / 8;
+                *expanded_clusters = g_realloc(*expanded_clusters,
+                                               new_bitmap_size);
+                /* clear the newly allocated space */
+                memset(&(*expanded_clusters)[old_bitmap_size], 0,
+                       new_bitmap_size - old_bitmap_size);
+            }
+
+            assert((cluster_index >= 0) && (cluster_index < *nb_clusters));
+            (*expanded_clusters)[cluster_index / 8] |= 1 << (cluster_index % 8);
+        }
+
+        if (is_active_l1) {
+            if (l2_dirty) {
+                qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);
+                qcow2_cache_depends_on_flush(s->l2_table_cache);
+            }
+            ret = qcow2_cache_put(bs, s->l2_table_cache, (void **)&l2_table);
+            if (ret < 0) {
+                l2_table = NULL;
+                goto fail;
+            }
+        } else {
+            if (l2_dirty) {
+                ret = qcow2_pre_write_overlap_check(bs,
+                        QCOW2_OL_INACTIVE_L2 | QCOW2_OL_ACTIVE_L2, l2_offset,
+                        s->cluster_size);
+                if (ret < 0) {
+                    goto fail;
+                }
+
+                ret = bdrv_write(bs->file, l2_offset / BDRV_SECTOR_SIZE,
+                        (void *)l2_table, s->cluster_sectors);
+                if (ret < 0) {
+                    goto fail;
+                }
+            }
+        }
+    }
+
+    ret = 0;
+
+fail:
+    if (l2_table) {
+        if (!is_active_l1) {
+            qemu_vfree(l2_table);
+        } else {
+            if (ret < 0) {
+                qcow2_cache_put(bs, s->l2_table_cache, (void **)&l2_table);
+            } else {
+                ret = qcow2_cache_put(bs, s->l2_table_cache,
+                        (void **)&l2_table);
+            }
+        }
+    }
+    return ret;
+}
+
+/*
+ * For backed images, expands all zero clusters on the image. For non-backed
+ * images, deallocates all non-pre-allocated zero clusters (and claims the
+ * allocation for pre-allocated ones). This is important for downgrading to a
+ * qcow2 version which doesn't yet support metadata zero clusters.
+ */
+int qcow2_expand_zero_clusters(BlockDriverState *bs)
+{
+    BDRVQcowState *s = bs->opaque;
+    uint64_t *l1_table = NULL;
+    uint64_t nb_clusters;
+    uint8_t *expanded_clusters;
+    int ret;
+    int i, j;
+
+    nb_clusters = size_to_clusters(s, bs->file->total_sectors *
+                                   BDRV_SECTOR_SIZE);
+    expanded_clusters = g_malloc0((nb_clusters + 7) / 8);
+
+    ret = expand_zero_clusters_in_l1(bs, s->l1_table, s->l1_size,
+                                     &expanded_clusters, &nb_clusters);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /* Inactive L1 tables may point to active L2 tables - therefore it is
+     * necessary to flush the L2 table cache before trying to access the L2
+     * tables pointed to by inactive L1 entries (else we might try to expand
+     * zero clusters that have already been expanded); furthermore, it is also
+     * necessary to empty the L2 table cache, since it may contain tables which
+     * are now going to be modified directly on disk, bypassing the cache.
+     * qcow2_cache_empty() does both for us. */
+    ret = qcow2_cache_empty(bs, s->l2_table_cache);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    for (i = 0; i < s->nb_snapshots; i++) {
+        int l1_sectors = (s->snapshots[i].l1_size * sizeof(uint64_t) +
+                BDRV_SECTOR_SIZE - 1) / BDRV_SECTOR_SIZE;
+
+        l1_table = g_realloc(l1_table, l1_sectors * BDRV_SECTOR_SIZE);
+
+        ret = bdrv_read(bs->file, s->snapshots[i].l1_table_offset /
+                BDRV_SECTOR_SIZE, (void *)l1_table, l1_sectors);
+        if (ret < 0) {
+            goto fail;
+        }
+
+        for (j = 0; j < s->snapshots[i].l1_size; j++) {
+            be64_to_cpus(&l1_table[j]);
+        }
+
+        ret = expand_zero_clusters_in_l1(bs, l1_table, s->snapshots[i].l1_size,
+                                         &expanded_clusters, &nb_clusters);
+        if (ret < 0) {
+            goto fail;
+        }
+    }
+
+    ret = 0;
+
+fail:
+    g_free(expanded_clusters);
+    g_free(l1_table);
+    return ret;
+}
diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c
index ba129de478..1ff43d0906 100644
--- a/block/qcow2-refcount.c
+++ b/block/qcow2-refcount.c
@@ -601,10 +601,10 @@ fail:
  * If the return value is non-negative, it is the new refcount of the cluster.
  * If it is negative, it is -errno and indicates an error.
  */
-static int update_cluster_refcount(BlockDriverState *bs,
-                                   int64_t cluster_index,
-                                   int addend,
-                                   enum qcow2_discard_type type)
+int qcow2_update_cluster_refcount(BlockDriverState *bs,
+                                  int64_t cluster_index,
+                                  int addend,
+                                  enum qcow2_discard_type type)
 {
     BDRVQcowState *s = bs->opaque;
     int ret;
@@ -733,8 +733,8 @@ int64_t qcow2_alloc_bytes(BlockDriverState *bs, int size)
         if (free_in_cluster == 0)
             s->free_byte_offset = 0;
         if ((offset & (s->cluster_size - 1)) != 0)
-            update_cluster_refcount(bs, offset >> s->cluster_bits, 1,
-                                    QCOW2_DISCARD_NEVER);
+            qcow2_update_cluster_refcount(bs, offset >> s->cluster_bits, 1,
+                                          QCOW2_DISCARD_NEVER);
     } else {
         offset = qcow2_alloc_clusters(bs, s->cluster_size);
         if (offset < 0) {
@@ -744,8 +744,8 @@ int64_t qcow2_alloc_bytes(BlockDriverState *bs, int size)
         if ((cluster_offset + s->cluster_size) == offset) {
             /* we are lucky: contiguous data */
             offset = s->free_byte_offset;
-            update_cluster_refcount(bs, offset >> s->cluster_bits, 1,
-                                    QCOW2_DISCARD_NEVER);
+            qcow2_update_cluster_refcount(bs, offset >> s->cluster_bits, 1,
+                                          QCOW2_DISCARD_NEVER);
             s->free_byte_offset += size;
         } else {
             s->free_byte_offset = offset;
@@ -754,8 +754,8 @@ int64_t qcow2_alloc_bytes(BlockDriverState *bs, int size)
     }
 
     /* The cluster refcount was incremented, either by qcow2_alloc_clusters()
-     * or explicitly by update_cluster_refcount().  Refcount blocks must be
-     * flushed before the caller's L2 table updates.
+     * or explicitly by qcow2_update_cluster_refcount().  Refcount blocks must
+     * be flushed before the caller's L2 table updates.
      */
     qcow2_cache_set_dependency(bs, s->l2_table_cache, s->refcount_block_cache);
     return offset;
@@ -796,11 +796,13 @@ void qcow2_free_any_clusters(BlockDriverState *bs, uint64_t l2_entry,
         }
         break;
     case QCOW2_CLUSTER_NORMAL:
-        qcow2_free_clusters(bs, l2_entry & L2E_OFFSET_MASK,
-                            nb_clusters << s->cluster_bits, type);
+    case QCOW2_CLUSTER_ZERO:
+        if (l2_entry & L2E_OFFSET_MASK) {
+            qcow2_free_clusters(bs, l2_entry & L2E_OFFSET_MASK,
+                                nb_clusters << s->cluster_bits, type);
+        }
         break;
     case QCOW2_CLUSTER_UNALLOCATED:
-    case QCOW2_CLUSTER_ZERO:
         break;
     default:
         abort();
@@ -874,7 +876,6 @@ int qcow2_update_snapshot_refcount(BlockDriverState *bs,
                         nb_csectors = ((offset >> s->csize_shift) &
                                        s->csize_mask) + 1;
                         if (addend != 0) {
-                            int ret;
                             ret = update_refcount(bs,
                                 (offset & s->cluster_offset_mask) & ~511,
                                 nb_csectors * 512, addend,
@@ -896,8 +897,9 @@ int qcow2_update_snapshot_refcount(BlockDriverState *bs,
                             break;
                         }
                         if (addend != 0) {
-                            refcount = update_cluster_refcount(bs, cluster_index, addend,
-                                                               QCOW2_DISCARD_SNAPSHOT);
+                            refcount = qcow2_update_cluster_refcount(bs,
+                                    cluster_index, addend,
+                                    QCOW2_DISCARD_SNAPSHOT);
                         } else {
                             refcount = get_refcount(bs, cluster_index);
                         }
@@ -936,8 +938,8 @@ int qcow2_update_snapshot_refcount(BlockDriverState *bs,
 
 
             if (addend != 0) {
-                refcount = update_cluster_refcount(bs, l2_offset >> s->cluster_bits, addend,
-                                                   QCOW2_DISCARD_SNAPSHOT);
+                refcount = qcow2_update_cluster_refcount(bs, l2_offset >>
+                        s->cluster_bits, addend, QCOW2_DISCARD_SNAPSHOT);
             } else {
                 refcount = get_refcount(bs, l2_offset >> s->cluster_bits);
             }
@@ -1034,7 +1036,6 @@ static void inc_refcounts(BlockDriverState *bs,
 
 /* Flags for check_refcounts_l1() and check_refcounts_l2() */
 enum {
-    CHECK_OFLAG_COPIED = 0x1,   /* check QCOW_OFLAG_COPIED matches refcount */
     CHECK_FRAG_INFO = 0x2,      /* update BlockFragInfo counters */
 };
 
@@ -1310,9 +1311,8 @@ static int check_oflag_copied(BlockDriverState *bs, BdrvCheckResult *res,
         }
 
         if (l2_dirty) {
-            ret = qcow2_pre_write_overlap_check(bs,
-                    QCOW2_OL_DEFAULT & ~QCOW2_OL_ACTIVE_L2, l2_offset,
-                    s->cluster_size);
+            ret = qcow2_pre_write_overlap_check(bs, QCOW2_OL_ACTIVE_L2,
+                                                l2_offset, s->cluster_size);
             if (ret < 0) {
                 fprintf(stderr, "ERROR: Could not write L2 table; metadata "
                         "overlap check failed: %s\n", strerror(-ret));
@@ -1353,8 +1353,7 @@ static int write_reftable_entry(BlockDriverState *bs, int rt_index)
         buf[i] = cpu_to_be64(s->refcount_table[rt_start_index + i]);
     }
 
-    ret = qcow2_pre_write_overlap_check(bs,
-            QCOW2_OL_DEFAULT & ~QCOW2_OL_REFCOUNT_TABLE,
+    ret = qcow2_pre_write_overlap_check(bs, QCOW2_OL_REFCOUNT_TABLE,
             s->refcount_table_offset + rt_start_index * sizeof(uint64_t),
             sizeof(buf));
     if (ret < 0) {
@@ -1405,8 +1404,7 @@ static int64_t realloc_refcount_block(BlockDriverState *bs, int reftable_index,
 
     /* new block has not yet been entered into refcount table, therefore it is
      * no refcount block yet (regarding this check) */
-    ret = qcow2_pre_write_overlap_check(bs, QCOW2_OL_DEFAULT, new_offset,
-            s->cluster_size);
+    ret = qcow2_pre_write_overlap_check(bs, 0, new_offset, s->cluster_size);
     if (ret < 0) {
         fprintf(stderr, "Could not write refcount block; metadata overlap "
                 "check failed: %s\n", strerror(-ret));
@@ -1481,8 +1479,7 @@ int qcow2_check_refcounts(BlockDriverState *bs, BdrvCheckResult *res,
 
     /* current L1 table */
     ret = check_refcounts_l1(bs, res, refcount_table, nb_clusters,
-                             s->l1_table_offset, s->l1_size,
-                             CHECK_OFLAG_COPIED | CHECK_FRAG_INFO);
+                             s->l1_table_offset, s->l1_size, CHECK_FRAG_INFO);
     if (ret < 0) {
         goto fail;
     }
@@ -1639,8 +1636,8 @@ fail:
  * looking for overlaps with important metadata sections (L1/L2 tables etc.),
  * i.e. a sanity check without relying on the refcount tables.
  *
- * The chk parameter specifies exactly what checks to perform (being a bitmask
- * of QCow2MetadataOverlap values).
+ * The ign parameter specifies what checks not to perform (being a bitmask of
+ * QCow2MetadataOverlap values), i.e., what sections to ignore.
  *
  * Returns:
  * - 0 if writing to this offset will not affect the mentioned metadata
@@ -1648,10 +1645,11 @@ fail:
  * - a negative value (-errno) indicating an error while performing a check,
  *   e.g. when bdrv_read failed on QCOW2_OL_INACTIVE_L2
  */
-int qcow2_check_metadata_overlap(BlockDriverState *bs, int chk, int64_t offset,
+int qcow2_check_metadata_overlap(BlockDriverState *bs, int ign, int64_t offset,
                                  int64_t size)
 {
     BDRVQcowState *s = bs->opaque;
+    int chk = s->overlap_check & ~ign;
     int i, j;
 
     if (!size) {
@@ -1721,20 +1719,19 @@ int qcow2_check_metadata_overlap(BlockDriverState *bs, int chk, int64_t offset,
         for (i = 0; i < s->nb_snapshots; i++) {
             uint64_t l1_ofs = s->snapshots[i].l1_table_offset;
             uint32_t l1_sz  = s->snapshots[i].l1_size;
-            uint64_t *l1 = g_malloc(l1_sz * sizeof(uint64_t));
+            uint64_t l1_sz2 = l1_sz * sizeof(uint64_t);
+            uint64_t *l1 = g_malloc(l1_sz2);
             int ret;
 
-            ret = bdrv_read(bs->file, l1_ofs / BDRV_SECTOR_SIZE, (uint8_t *)l1,
-                            l1_sz * sizeof(uint64_t) / BDRV_SECTOR_SIZE);
-
+            ret = bdrv_pread(bs->file, l1_ofs, l1, l1_sz2);
             if (ret < 0) {
                 g_free(l1);
                 return ret;
             }
 
             for (j = 0; j < l1_sz; j++) {
-                if ((l1[j] & L1E_OFFSET_MASK) &&
-                    overlaps_with(l1[j] & L1E_OFFSET_MASK, s->cluster_size)) {
+                uint64_t l2_ofs = be64_to_cpu(l1[j]) & L1E_OFFSET_MASK;
+                if (l2_ofs && overlaps_with(l2_ofs, s->cluster_size)) {
                     g_free(l1);
                     return QCOW2_OL_INACTIVE_L2;
                 }
@@ -1768,10 +1765,10 @@ static const char *metadata_ol_names[] = {
  * Returns 0 if there were neither overlaps nor errors while checking for
  * overlaps; or a negative value (-errno) on error.
  */
-int qcow2_pre_write_overlap_check(BlockDriverState *bs, int chk, int64_t offset,
+int qcow2_pre_write_overlap_check(BlockDriverState *bs, int ign, int64_t offset,
                                   int64_t size)
 {
-    int ret = qcow2_check_metadata_overlap(bs, chk, offset, size);
+    int ret = qcow2_check_metadata_overlap(bs, ign, offset, size);
 
     if (ret < 0) {
         return ret;
diff --git a/block/qcow2-snapshot.c b/block/qcow2-snapshot.c
index e7e601301a..3529c683c6 100644
--- a/block/qcow2-snapshot.c
+++ b/block/qcow2-snapshot.c
@@ -182,19 +182,19 @@ static int qcow2_write_snapshots(BlockDriverState *bs)
     snapshots_offset = qcow2_alloc_clusters(bs, snapshots_size);
     offset = snapshots_offset;
     if (offset < 0) {
-        return offset;
+        ret = offset;
+        goto fail;
     }
     ret = bdrv_flush(bs);
     if (ret < 0) {
-        return ret;
+        goto fail;
     }
 
     /* The snapshot list position has not yet been updated, so these clusters
      * must indeed be completely free */
-    ret = qcow2_pre_write_overlap_check(bs, QCOW2_OL_DEFAULT, offset,
-                                        s->snapshots_size);
+    ret = qcow2_pre_write_overlap_check(bs, 0, offset, snapshots_size);
     if (ret < 0) {
-        return ret;
+        goto fail;
     }
 
 
@@ -220,6 +220,7 @@ static int qcow2_write_snapshots(BlockDriverState *bs)
 
         id_str_size = strlen(sn->id_str);
         name_size = strlen(sn->name);
+        assert(id_str_size <= UINT16_MAX && name_size <= UINT16_MAX);
         h.id_str_size = cpu_to_be16(id_str_size);
         h.name_size = cpu_to_be16(name_size);
         offset = align_offset(offset, 8);
@@ -278,6 +279,10 @@ static int qcow2_write_snapshots(BlockDriverState *bs)
     return 0;
 
 fail:
+    if (snapshots_offset > 0) {
+        qcow2_free_clusters(bs, snapshots_offset, snapshots_size,
+                            QCOW2_DISCARD_ALWAYS);
+    }
     return ret;
 }
 
@@ -286,7 +291,8 @@ static void find_new_snapshot_id(BlockDriverState *bs,
 {
     BDRVQcowState *s = bs->opaque;
     QCowSnapshot *sn;
-    int i, id, id_max = 0;
+    int i;
+    unsigned long id, id_max = 0;
 
     for(i = 0; i < s->nb_snapshots; i++) {
         sn = s->snapshots + i;
@@ -294,34 +300,50 @@ static void find_new_snapshot_id(BlockDriverState *bs,
         if (id > id_max)
             id_max = id;
     }
-    snprintf(id_str, id_str_size, "%d", id_max + 1);
+    snprintf(id_str, id_str_size, "%lu", id_max + 1);
 }
 
-static int find_snapshot_by_id(BlockDriverState *bs, const char *id_str)
+static int find_snapshot_by_id_and_name(BlockDriverState *bs,
+                                        const char *id,
+                                        const char *name)
 {
     BDRVQcowState *s = bs->opaque;
     int i;
 
-    for(i = 0; i < s->nb_snapshots; i++) {
-        if (!strcmp(s->snapshots[i].id_str, id_str))
-            return i;
+    if (id && name) {
+        for (i = 0; i < s->nb_snapshots; i++) {
+            if (!strcmp(s->snapshots[i].id_str, id) &&
+                !strcmp(s->snapshots[i].name, name)) {
+                return i;
+            }
+        }
+    } else if (id) {
+        for (i = 0; i < s->nb_snapshots; i++) {
+            if (!strcmp(s->snapshots[i].id_str, id)) {
+                return i;
+            }
+        }
+    } else if (name) {
+        for (i = 0; i < s->nb_snapshots; i++) {
+            if (!strcmp(s->snapshots[i].name, name)) {
+                return i;
+            }
+        }
     }
+
     return -1;
 }
 
-static int find_snapshot_by_id_or_name(BlockDriverState *bs, const char *name)
+static int find_snapshot_by_id_or_name(BlockDriverState *bs,
+                                       const char *id_or_name)
 {
-    BDRVQcowState *s = bs->opaque;
-    int i, ret;
+    int ret;
 
-    ret = find_snapshot_by_id(bs, name);
-    if (ret >= 0)
+    ret = find_snapshot_by_id_and_name(bs, id_or_name, NULL);
+    if (ret >= 0) {
         return ret;
-    for(i = 0; i < s->nb_snapshots; i++) {
-        if (!strcmp(s->snapshots[i].name, name))
-            return i;
     }
-    return -1;
+    return find_snapshot_by_id_and_name(bs, NULL, id_or_name);
 }
 
 /* if no id is provided, a new one is constructed */
@@ -343,7 +365,7 @@ int qcow2_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
     }
 
     /* Check that the ID is unique */
-    if (find_snapshot_by_id(bs, sn_info->id_str) >= 0) {
+    if (find_snapshot_by_id_and_name(bs, sn_info->id_str, NULL) >= 0) {
         return -EEXIST;
     }
 
@@ -372,8 +394,8 @@ int qcow2_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
         l1_table[i] = cpu_to_be64(s->l1_table[i]);
     }
 
-    ret = qcow2_pre_write_overlap_check(bs, QCOW2_OL_DEFAULT,
-            sn->l1_table_offset, s->l1_size * sizeof(uint64_t));
+    ret = qcow2_pre_write_overlap_check(bs, 0, sn->l1_table_offset,
+                                        s->l1_size * sizeof(uint64_t));
     if (ret < 0) {
         goto fail;
     }
@@ -411,11 +433,19 @@ int qcow2_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
     if (ret < 0) {
         g_free(s->snapshots);
         s->snapshots = old_snapshot_list;
+        s->nb_snapshots--;
         goto fail;
     }
 
     g_free(old_snapshot_list);
 
+    /* The VM state isn't needed any more in the active L1 table; in fact, it
+     * hurts by causing expensive COW for the next snapshot. */
+    qcow2_discard_clusters(bs, qcow2_vm_state_offset(s),
+                           align_offset(sn->vm_state_size, s->cluster_size)
+                                >> BDRV_SECTOR_BITS,
+                           QCOW2_DISCARD_NEVER);
+
 #ifdef DEBUG_ALLOC
     {
       BdrvCheckResult result = {0};
@@ -490,9 +520,8 @@ int qcow2_snapshot_goto(BlockDriverState *bs, const char *snapshot_id)
         goto fail;
     }
 
-    ret = qcow2_pre_write_overlap_check(bs,
-            QCOW2_OL_DEFAULT & ~QCOW2_OL_ACTIVE_L1,
-            s->l1_table_offset, cur_l1_bytes);
+    ret = qcow2_pre_write_overlap_check(bs, QCOW2_OL_ACTIVE_L1,
+                                        s->l1_table_offset, cur_l1_bytes);
     if (ret < 0) {
         goto fail;
     }
@@ -553,15 +582,19 @@ fail:
     return ret;
 }
 
-int qcow2_snapshot_delete(BlockDriverState *bs, const char *snapshot_id)
+int qcow2_snapshot_delete(BlockDriverState *bs,
+                          const char *snapshot_id,
+                          const char *name,
+                          Error **errp)
 {
     BDRVQcowState *s = bs->opaque;
     QCowSnapshot sn;
     int snapshot_index, ret;
 
     /* Search the snapshot */
-    snapshot_index = find_snapshot_by_id_or_name(bs, snapshot_id);
+    snapshot_index = find_snapshot_by_id_and_name(bs, snapshot_id, name);
     if (snapshot_index < 0) {
+        error_setg(errp, "Can't find the snapshot");
         return -ENOENT;
     }
     sn = s->snapshots[snapshot_index];
@@ -573,6 +606,7 @@ int qcow2_snapshot_delete(BlockDriverState *bs, const char *snapshot_id)
     s->nb_snapshots--;
     ret = qcow2_write_snapshots(bs);
     if (ret < 0) {
+        error_setg(errp, "Failed to remove snapshot from snapshot list");
         return ret;
     }
 
@@ -590,6 +624,7 @@ int qcow2_snapshot_delete(BlockDriverState *bs, const char *snapshot_id)
     ret = qcow2_update_snapshot_refcount(bs, sn.l1_table_offset,
                                          sn.l1_size, -1);
     if (ret < 0) {
+        error_setg(errp, "Failed to free the cluster and L1 table");
         return ret;
     }
     qcow2_free_clusters(bs, sn.l1_table_offset, sn.l1_size * sizeof(uint64_t),
@@ -598,6 +633,7 @@ int qcow2_snapshot_delete(BlockDriverState *bs, const char *snapshot_id)
     /* must update the copied flag on the current cluster offsets */
     ret = qcow2_update_snapshot_refcount(bs, s->l1_table_offset, s->l1_size, 0);
     if (ret < 0) {
+        error_setg(errp, "Failed to update snapshot status in disk");
         return ret;
     }
 
diff --git a/block/qcow2.c b/block/qcow2.c
index 578792f0a3..c1abaffa19 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -52,7 +52,7 @@
 typedef struct {
     uint32_t magic;
     uint32_t len;
-} QCowExtension;
+} QEMU_PACKED QCowExtension;
 
 #define  QCOW2_EXT_MAGIC_END 0
 #define  QCOW2_EXT_MAGIC_BACKING_FORMAT 0xE2792ACA
@@ -79,7 +79,8 @@ static int qcow2_probe(const uint8_t *buf, int buf_size, const char *filename)
  * return 0 upon success, non-0 otherwise
  */
 static int qcow2_read_extensions(BlockDriverState *bs, uint64_t start_offset,
-                                 uint64_t end_offset, void **p_feature_table)
+                                 uint64_t end_offset, void **p_feature_table,
+                                 Error **errp)
 {
     BDRVQcowState *s = bs->opaque;
     QCowExtension ext;
@@ -100,10 +101,10 @@ static int qcow2_read_extensions(BlockDriverState *bs, uint64_t start_offset,
         printf("attempting to read extended header in offset %lu\n", offset);
 #endif
 
-        if (bdrv_pread(bs->file, offset, &ext, sizeof(ext)) != sizeof(ext)) {
-            fprintf(stderr, "qcow2_read_extension: ERROR: "
-                    "pread fail from offset %" PRIu64 "\n",
-                    offset);
+        ret = bdrv_pread(bs->file, offset, &ext, sizeof(ext));
+        if (ret < 0) {
+            error_setg_errno(errp, -ret, "qcow2_read_extension: ERROR: "
+                             "pread fail from offset %" PRIu64, offset);
             return 1;
         }
         be32_to_cpus(&ext.magic);
@@ -113,7 +114,7 @@ static int qcow2_read_extensions(BlockDriverState *bs, uint64_t start_offset,
         printf("ext.magic = 0x%x\n", ext.magic);
 #endif
         if (ext.len > end_offset - offset) {
-            error_report("Header extension too large");
+            error_setg(errp, "Header extension too large");
             return -EINVAL;
         }
 
@@ -123,14 +124,16 @@ static int qcow2_read_extensions(BlockDriverState *bs, uint64_t start_offset,
 
         case QCOW2_EXT_MAGIC_BACKING_FORMAT:
             if (ext.len >= sizeof(bs->backing_format)) {
-                fprintf(stderr, "ERROR: ext_backing_format: len=%u too large"
-                        " (>=%zu)\n",
-                        ext.len, sizeof(bs->backing_format));
+                error_setg(errp, "ERROR: ext_backing_format: len=%u too large"
+                           " (>=%zu)", ext.len, sizeof(bs->backing_format));
                 return 2;
             }
-            if (bdrv_pread(bs->file, offset , bs->backing_format,
-                           ext.len) != ext.len)
+            ret = bdrv_pread(bs->file, offset, bs->backing_format, ext.len);
+            if (ret < 0) {
+                error_setg_errno(errp, -ret, "ERROR: ext_backing_format: "
+                                 "Could not read format name");
                 return 3;
+            }
             bs->backing_format[ext.len] = '\0';
 #ifdef DEBUG_EXT
             printf("Qcow2: Got format extension %s\n", bs->backing_format);
@@ -142,6 +145,8 @@ static int qcow2_read_extensions(BlockDriverState *bs, uint64_t start_offset,
                 void* feature_table = g_malloc0(ext.len + 2 * sizeof(Qcow2Feature));
                 ret = bdrv_pread(bs->file, offset , feature_table, ext.len);
                 if (ret < 0) {
+                    error_setg_errno(errp, -ret, "ERROR: ext_feature_table: "
+                                     "Could not read table");
                     return ret;
                 }
 
@@ -161,6 +166,8 @@ static int qcow2_read_extensions(BlockDriverState *bs, uint64_t start_offset,
 
                 ret = bdrv_pread(bs->file, offset , uext->data, uext->len);
                 if (ret < 0) {
+                    error_setg_errno(errp, -ret, "ERROR: unknown extension: "
+                                     "Could not read data");
                     return ret;
                 }
             }
@@ -184,8 +191,8 @@ static void cleanup_unknown_header_ext(BlockDriverState *bs)
     }
 }
 
-static void GCC_FMT_ATTR(2, 3) report_unsupported(BlockDriverState *bs,
-    const char *fmt, ...)
+static void GCC_FMT_ATTR(3, 4) report_unsupported(BlockDriverState *bs,
+    Error **errp, const char *fmt, ...)
 {
     char msg[64];
     va_list ap;
@@ -194,17 +201,17 @@ static void GCC_FMT_ATTR(2, 3) report_unsupported(BlockDriverState *bs,
     vsnprintf(msg, sizeof(msg), fmt, ap);
     va_end(ap);
 
-    qerror_report(QERR_UNKNOWN_BLOCK_FORMAT_FEATURE,
-        bs->device_name, "qcow2", msg);
+    error_set(errp, QERR_UNKNOWN_BLOCK_FORMAT_FEATURE, bs->device_name, "qcow2",
+              msg);
 }
 
 static void report_unsupported_feature(BlockDriverState *bs,
-    Qcow2Feature *table, uint64_t mask)
+    Error **errp, Qcow2Feature *table, uint64_t mask)
 {
     while (table && table->name[0] != '\0') {
         if (table->type == QCOW2_FEAT_TYPE_INCOMPATIBLE) {
             if (mask & (1 << table->bit)) {
-                report_unsupported(bs, "%.46s",table->name);
+                report_unsupported(bs, errp, "%.46s", table->name);
                 mask &= ~(1 << table->bit);
             }
         }
@@ -212,7 +219,8 @@ static void report_unsupported_feature(BlockDriverState *bs,
     }
 
     if (mask) {
-        report_unsupported(bs, "Unknown incompatible feature: %" PRIx64, mask);
+        report_unsupported(bs, errp, "Unknown incompatible feature: %" PRIx64,
+                           mask);
     }
 }
 
@@ -346,11 +354,69 @@ static QemuOptsList qcow2_runtime_opts = {
             .type = QEMU_OPT_BOOL,
             .help = "Generate discard requests when other clusters are freed",
         },
+        {
+            .name = QCOW2_OPT_OVERLAP,
+            .type = QEMU_OPT_STRING,
+            .help = "Selects which overlap checks to perform from a range of "
+                    "templates (none, constant, cached, all)",
+        },
+        {
+            .name = QCOW2_OPT_OVERLAP_MAIN_HEADER,
+            .type = QEMU_OPT_BOOL,
+            .help = "Check for unintended writes into the main qcow2 header",
+        },
+        {
+            .name = QCOW2_OPT_OVERLAP_ACTIVE_L1,
+            .type = QEMU_OPT_BOOL,
+            .help = "Check for unintended writes into the active L1 table",
+        },
+        {
+            .name = QCOW2_OPT_OVERLAP_ACTIVE_L2,
+            .type = QEMU_OPT_BOOL,
+            .help = "Check for unintended writes into an active L2 table",
+        },
+        {
+            .name = QCOW2_OPT_OVERLAP_REFCOUNT_TABLE,
+            .type = QEMU_OPT_BOOL,
+            .help = "Check for unintended writes into the refcount table",
+        },
+        {
+            .name = QCOW2_OPT_OVERLAP_REFCOUNT_BLOCK,
+            .type = QEMU_OPT_BOOL,
+            .help = "Check for unintended writes into a refcount block",
+        },
+        {
+            .name = QCOW2_OPT_OVERLAP_SNAPSHOT_TABLE,
+            .type = QEMU_OPT_BOOL,
+            .help = "Check for unintended writes into the snapshot table",
+        },
+        {
+            .name = QCOW2_OPT_OVERLAP_INACTIVE_L1,
+            .type = QEMU_OPT_BOOL,
+            .help = "Check for unintended writes into an inactive L1 table",
+        },
+        {
+            .name = QCOW2_OPT_OVERLAP_INACTIVE_L2,
+            .type = QEMU_OPT_BOOL,
+            .help = "Check for unintended writes into an inactive L2 table",
+        },
         { /* end of list */ }
     },
 };
 
-static int qcow2_open(BlockDriverState *bs, QDict *options, int flags)
+static const char *overlap_bool_option_names[QCOW2_OL_MAX_BITNR] = {
+    [QCOW2_OL_MAIN_HEADER_BITNR]    = QCOW2_OPT_OVERLAP_MAIN_HEADER,
+    [QCOW2_OL_ACTIVE_L1_BITNR]      = QCOW2_OPT_OVERLAP_ACTIVE_L1,
+    [QCOW2_OL_ACTIVE_L2_BITNR]      = QCOW2_OPT_OVERLAP_ACTIVE_L2,
+    [QCOW2_OL_REFCOUNT_TABLE_BITNR] = QCOW2_OPT_OVERLAP_REFCOUNT_TABLE,
+    [QCOW2_OL_REFCOUNT_BLOCK_BITNR] = QCOW2_OPT_OVERLAP_REFCOUNT_BLOCK,
+    [QCOW2_OL_SNAPSHOT_TABLE_BITNR] = QCOW2_OPT_OVERLAP_SNAPSHOT_TABLE,
+    [QCOW2_OL_INACTIVE_L1_BITNR]    = QCOW2_OPT_OVERLAP_INACTIVE_L1,
+    [QCOW2_OL_INACTIVE_L2_BITNR]    = QCOW2_OPT_OVERLAP_INACTIVE_L2,
+};
+
+static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
+                      Error **errp)
 {
     BDRVQcowState *s = bs->opaque;
     int len, i, ret = 0;
@@ -359,9 +425,12 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags)
     Error *local_err = NULL;
     uint64_t ext_end;
     uint64_t l1_vm_state_index;
+    const char *opt_overlap_check;
+    int overlap_check_template = 0;
 
     ret = bdrv_pread(bs->file, 0, &header, sizeof(header));
     if (ret < 0) {
+        error_setg_errno(errp, -ret, "Could not read qcow2 header");
         goto fail;
     }
     be32_to_cpus(&header.magic);
@@ -379,11 +448,12 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags)
     be32_to_cpus(&header.nb_snapshots);
 
     if (header.magic != QCOW_MAGIC) {
+        error_setg(errp, "Image is not in qcow2 format");
         ret = -EMEDIUMTYPE;
         goto fail;
     }
     if (header.version < 2 || header.version > 3) {
-        report_unsupported(bs, "QCOW version %d", header.version);
+        report_unsupported(bs, errp, "QCOW version %d", header.version);
         ret = -ENOTSUP;
         goto fail;
     }
@@ -411,6 +481,8 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags)
         ret = bdrv_pread(bs->file, sizeof(header), s->unknown_header_fields,
                          s->unknown_header_fields_size);
         if (ret < 0) {
+            error_setg_errno(errp, -ret, "Could not read unknown qcow2 header "
+                             "fields");
             goto fail;
         }
     }
@@ -429,8 +501,8 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags)
     if (s->incompatible_features & ~QCOW2_INCOMPAT_MASK) {
         void *feature_table = NULL;
         qcow2_read_extensions(bs, header.header_length, ext_end,
-                              &feature_table);
-        report_unsupported_feature(bs, feature_table,
+                              &feature_table, NULL);
+        report_unsupported_feature(bs, errp, feature_table,
                                    s->incompatible_features &
                                    ~QCOW2_INCOMPAT_MASK);
         ret = -ENOTSUP;
@@ -441,8 +513,8 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags)
         /* Corrupt images may not be written to unless they are being repaired
          */
         if ((flags & BDRV_O_RDWR) && !(flags & BDRV_O_CHECK)) {
-            error_report("qcow2: Image is corrupt; cannot be opened "
-                    "read/write.");
+            error_setg(errp, "qcow2: Image is corrupt; cannot be opened "
+                       "read/write");
             ret = -EACCES;
             goto fail;
         }
@@ -450,18 +522,22 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags)
 
     /* Check support for various header values */
     if (header.refcount_order != 4) {
-        report_unsupported(bs, "%d bit reference counts",
+        report_unsupported(bs, errp, "%d bit reference counts",
                            1 << header.refcount_order);
         ret = -ENOTSUP;
         goto fail;
     }
+    s->refcount_order = header.refcount_order;
 
     if (header.cluster_bits < MIN_CLUSTER_BITS ||
         header.cluster_bits > MAX_CLUSTER_BITS) {
+        error_setg(errp, "Unsupported cluster size: 2^%i", header.cluster_bits);
         ret = -EINVAL;
         goto fail;
     }
     if (header.crypt_method > QCOW_CRYPT_AES) {
+        error_setg(errp, "Unsupported encryption method: %i",
+                   header.crypt_method);
         ret = -EINVAL;
         goto fail;
     }
@@ -490,6 +566,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags)
 
     l1_vm_state_index = size_to_l1(s, header.size);
     if (l1_vm_state_index > INT_MAX) {
+        error_setg(errp, "Image is too big");
         ret = -EFBIG;
         goto fail;
     }
@@ -498,6 +575,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags)
     /* the L1 table must contain at least enough entries to put
        header.size bytes */
     if (s->l1_size < s->l1_vm_state_index) {
+        error_setg(errp, "L1 table is too small");
         ret = -EINVAL;
         goto fail;
     }
@@ -508,6 +586,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags)
         ret = bdrv_pread(bs->file, s->l1_table_offset, s->l1_table,
                          s->l1_size * sizeof(uint64_t));
         if (ret < 0) {
+            error_setg_errno(errp, -ret, "Could not read L1 table");
             goto fail;
         }
         for(i = 0;i < s->l1_size; i++) {
@@ -528,6 +607,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags)
 
     ret = qcow2_refcount_init(bs);
     if (ret != 0) {
+        error_setg_errno(errp, -ret, "Could not initialize refcount handling");
         goto fail;
     }
 
@@ -535,7 +615,9 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags)
     QTAILQ_INIT(&s->discards);
 
     /* read qcow2 extensions */
-    if (qcow2_read_extensions(bs, header.header_length, ext_end, NULL)) {
+    if (qcow2_read_extensions(bs, header.header_length, ext_end, NULL,
+        &local_err)) {
+        error_propagate(errp, local_err);
         ret = -EINVAL;
         goto fail;
     }
@@ -549,6 +631,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags)
         ret = bdrv_pread(bs->file, header.backing_file_offset,
                          bs->backing_file, len);
         if (ret < 0) {
+            error_setg_errno(errp, -ret, "Could not read backing file name");
             goto fail;
         }
         bs->backing_file[len] = '\0';
@@ -556,6 +639,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags)
 
     ret = qcow2_read_snapshots(bs);
     if (ret < 0) {
+        error_setg_errno(errp, -ret, "Could not read snapshots");
         goto fail;
     }
 
@@ -564,6 +648,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags)
         s->autoclear_features = 0;
         ret = qcow2_update_header(bs);
         if (ret < 0) {
+            error_setg_errno(errp, -ret, "Could not update qcow2 header");
             goto fail;
         }
     }
@@ -578,6 +663,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags)
 
         ret = qcow2_check(bs, &result, BDRV_FIX_ERRORS);
         if (ret < 0) {
+            error_setg_errno(errp, -ret, "Could not repair dirty image");
             goto fail;
         }
     }
@@ -586,8 +672,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags)
     opts = qemu_opts_create_nofail(&qcow2_runtime_opts);
     qemu_opts_absorb_qdict(opts, options, &local_err);
     if (error_is_set(&local_err)) {
-        qerror_report_err(local_err);
-        error_free(local_err);
+        error_propagate(errp, local_err);
         ret = -EINVAL;
         goto fail;
     }
@@ -605,11 +690,38 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags)
     s->discard_passthrough[QCOW2_DISCARD_OTHER] =
         qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_OTHER, false);
 
+    opt_overlap_check = qemu_opt_get(opts, "overlap-check") ?: "cached";
+    if (!strcmp(opt_overlap_check, "none")) {
+        overlap_check_template = 0;
+    } else if (!strcmp(opt_overlap_check, "constant")) {
+        overlap_check_template = QCOW2_OL_CONSTANT;
+    } else if (!strcmp(opt_overlap_check, "cached")) {
+        overlap_check_template = QCOW2_OL_CACHED;
+    } else if (!strcmp(opt_overlap_check, "all")) {
+        overlap_check_template = QCOW2_OL_ALL;
+    } else {
+        error_setg(errp, "Unsupported value '%s' for qcow2 option "
+                   "'overlap-check'. Allowed are either of the following: "
+                   "none, constant, cached, all", opt_overlap_check);
+        qemu_opts_del(opts);
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    s->overlap_check = 0;
+    for (i = 0; i < QCOW2_OL_MAX_BITNR; i++) {
+        /* overlap-check defines a template bitmask, but every flag may be
+         * overwritten through the associated boolean option */
+        s->overlap_check |=
+            qemu_opt_get_bool(opts, overlap_bool_option_names[i],
+                              overlap_check_template & (1 << i)) << i;
+    }
+
     qemu_opts_del(opts);
 
     if (s->use_lazy_refcounts && s->qcow_version < 3) {
-        qerror_report(ERROR_CLASS_GENERIC_ERROR, "Lazy refcounts require "
-            "a qcow2 image with at least qemu 1.1 compatibility level");
+        error_setg(errp, "Lazy refcounts require a qcow2 image with at least "
+                   "qemu 1.1 compatibility level");
         ret = -EINVAL;
         goto fail;
     }
@@ -939,7 +1051,7 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
                 cur_nr_sectors * 512);
         }
 
-        ret = qcow2_pre_write_overlap_check(bs, QCOW2_OL_DEFAULT,
+        ret = qcow2_pre_write_overlap_check(bs, 0,
                 cluster_offset + index_in_cluster * BDRV_SECTOR_SIZE,
                 cur_nr_sectors * BDRV_SECTOR_SIZE);
         if (ret < 0) {
@@ -1059,7 +1171,7 @@ static void qcow2_invalidate_cache(BlockDriverState *bs)
               qbool_from_int(s->use_lazy_refcounts));
 
     memset(s, 0, sizeof(BDRVQcowState));
-    qcow2_open(bs, options, flags);
+    qcow2_open(bs, options, flags, NULL);
 
     QDECREF(options);
 
@@ -1143,7 +1255,7 @@ int qcow2_update_header(BlockDriverState *bs)
         .incompatible_features  = cpu_to_be64(s->incompatible_features),
         .compatible_features    = cpu_to_be64(s->compatible_features),
         .autoclear_features     = cpu_to_be64(s->autoclear_features),
-        .refcount_order         = cpu_to_be32(3 + REFCOUNT_SHIFT),
+        .refcount_order         = cpu_to_be32(s->refcount_order),
         .header_length          = cpu_to_be32(header_length),
     };
 
@@ -1332,7 +1444,8 @@ static int preallocate(BlockDriverState *bs)
 static int qcow2_create2(const char *filename, int64_t total_size,
                          const char *backing_file, const char *backing_format,
                          int flags, size_t cluster_size, int prealloc,
-                         QEMUOptionParameter *options, int version)
+                         QEMUOptionParameter *options, int version,
+                         Error **errp)
 {
     /* Calculate cluster_bits */
     int cluster_bits;
@@ -1340,9 +1453,8 @@ static int qcow2_create2(const char *filename, int64_t total_size,
     if (cluster_bits < MIN_CLUSTER_BITS || cluster_bits > MAX_CLUSTER_BITS ||
         (1 << cluster_bits) != cluster_size)
     {
-        error_report(
-            "Cluster size must be a power of two between %d and %dk",
-            1 << MIN_CLUSTER_BITS, 1 << (MAX_CLUSTER_BITS - 10));
+        error_setg(errp, "Cluster size must be a power of two between %d and "
+                   "%dk", 1 << MIN_CLUSTER_BITS, 1 << (MAX_CLUSTER_BITS - 10));
         return -EINVAL;
     }
 
@@ -1361,15 +1473,18 @@ static int qcow2_create2(const char *filename, int64_t total_size,
     BlockDriverState* bs;
     QCowHeader header;
     uint8_t* refcount_table;
+    Error *local_err = NULL;
     int ret;
 
-    ret = bdrv_create_file(filename, options);
+    ret = bdrv_create_file(filename, options, &local_err);
     if (ret < 0) {
+        error_propagate(errp, local_err);
         return ret;
     }
 
-    ret = bdrv_file_open(&bs, filename, NULL, BDRV_O_RDWR);
+    ret = bdrv_file_open(&bs, filename, NULL, BDRV_O_RDWR, &local_err);
     if (ret < 0) {
+        error_propagate(errp, local_err);
         return ret;
     }
 
@@ -1399,6 +1514,7 @@ static int qcow2_create2(const char *filename, int64_t total_size,
 
     ret = bdrv_pwrite(bs, 0, &header, sizeof(header));
     if (ret < 0) {
+        error_setg_errno(errp, -ret, "Could not write qcow2 header");
         goto out;
     }
 
@@ -1408,6 +1524,7 @@ static int qcow2_create2(const char *filename, int64_t total_size,
     g_free(refcount_table);
 
     if (ret < 0) {
+        error_setg_errno(errp, -ret, "Could not write refcount table");
         goto out;
     }
 
@@ -1421,13 +1538,16 @@ static int qcow2_create2(const char *filename, int64_t total_size,
     BlockDriver* drv = bdrv_find_format("qcow2");
     assert(drv != NULL);
     ret = bdrv_open(bs, filename, NULL,
-        BDRV_O_RDWR | BDRV_O_CACHE_WB | BDRV_O_NO_FLUSH, drv);
+        BDRV_O_RDWR | BDRV_O_CACHE_WB | BDRV_O_NO_FLUSH, drv, &local_err);
     if (ret < 0) {
+        error_propagate(errp, local_err);
         goto out;
     }
 
     ret = qcow2_alloc_clusters(bs, 2 * cluster_size);
     if (ret < 0) {
+        error_setg_errno(errp, -ret, "Could not allocate clusters for qcow2 "
+                         "header and refcount table");
         goto out;
 
     } else if (ret != 0) {
@@ -1438,6 +1558,7 @@ static int qcow2_create2(const char *filename, int64_t total_size,
     /* Okay, now that we have a valid image, let's give it the right size */
     ret = bdrv_truncate(bs, total_size * BDRV_SECTOR_SIZE);
     if (ret < 0) {
+        error_setg_errno(errp, -ret, "Could not resize image");
         goto out;
     }
 
@@ -1445,6 +1566,8 @@ static int qcow2_create2(const char *filename, int64_t total_size,
     if (backing_file) {
         ret = bdrv_change_backing_file(bs, backing_file, backing_format);
         if (ret < 0) {
+            error_setg_errno(errp, -ret, "Could not assign backing file '%s' "
+                             "with format '%s'", backing_file, backing_format);
             goto out;
         }
     }
@@ -1456,6 +1579,7 @@ static int qcow2_create2(const char *filename, int64_t total_size,
         ret = preallocate(bs);
         qemu_co_mutex_unlock(&s->lock);
         if (ret < 0) {
+            error_setg_errno(errp, -ret, "Could not preallocate metadata");
             goto out;
         }
     }
@@ -1466,7 +1590,8 @@ out:
     return ret;
 }
 
-static int qcow2_create(const char *filename, QEMUOptionParameter *options)
+static int qcow2_create(const char *filename, QEMUOptionParameter *options,
+                        Error **errp)
 {
     const char *backing_file = NULL;
     const char *backing_fmt = NULL;
@@ -1475,6 +1600,8 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options)
     size_t cluster_size = DEFAULT_CLUSTER_SIZE;
     int prealloc = 0;
     int version = 3;
+    Error *local_err = NULL;
+    int ret;
 
     /* Read out options */
     while (options && options->name) {
@@ -1496,8 +1623,8 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options)
             } else if (!strcmp(options->value.s, "metadata")) {
                 prealloc = 1;
             } else {
-                fprintf(stderr, "Invalid preallocation mode: '%s'\n",
-                    options->value.s);
+                error_setg(errp, "Invalid preallocation mode: '%s'",
+                           options->value.s);
                 return -EINVAL;
             }
         } else if (!strcmp(options->name, BLOCK_OPT_COMPAT_LEVEL)) {
@@ -1508,8 +1635,8 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options)
             } else if (!strcmp(options->value.s, "1.1")) {
                 version = 3;
             } else {
-                fprintf(stderr, "Invalid compatibility level: '%s'\n",
-                    options->value.s);
+                error_setg(errp, "Invalid compatibility level: '%s'",
+                           options->value.s);
                 return -EINVAL;
             }
         } else if (!strcmp(options->name, BLOCK_OPT_LAZY_REFCOUNTS)) {
@@ -1519,19 +1646,23 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options)
     }
 
     if (backing_file && prealloc) {
-        fprintf(stderr, "Backing file and preallocation cannot be used at "
-            "the same time\n");
+        error_setg(errp, "Backing file and preallocation cannot be used at "
+                   "the same time");
         return -EINVAL;
     }
 
     if (version < 3 && (flags & BLOCK_FLAG_LAZY_REFCOUNTS)) {
-        fprintf(stderr, "Lazy refcounts only supported with compatibility "
-                "level 1.1 and above (use compat=1.1 or greater)\n");
+        error_setg(errp, "Lazy refcounts only supported with compatibility "
+                   "level 1.1 and above (use compat=1.1 or greater)");
         return -EINVAL;
     }
 
-    return qcow2_create2(filename, sectors, backing_file, backing_fmt, flags,
-                         cluster_size, prealloc, options, version);
+    ret = qcow2_create2(filename, sectors, backing_file, backing_fmt, flags,
+                        cluster_size, prealloc, options, version, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(errp, local_err);
+    }
+    return ret;
 }
 
 static int qcow2_make_empty(BlockDriverState *bs)
@@ -1582,7 +1713,7 @@ static coroutine_fn int qcow2_co_discard(BlockDriverState *bs,
 
     qemu_co_mutex_lock(&s->lock);
     ret = qcow2_discard_clusters(bs, sector_num << BDRV_SECTOR_BITS,
-        nb_sectors);
+        nb_sectors, QCOW2_DISCARD_REQUEST);
     qemu_co_mutex_unlock(&s->lock);
     return ret;
 }
@@ -1693,14 +1824,6 @@ static int qcow2_write_compressed(BlockDriverState *bs, int64_t sector_num,
 
     if (ret != Z_STREAM_END || out_len >= s->cluster_size) {
         /* could not compress: write normal cluster */
-
-        ret = qcow2_pre_write_overlap_check(bs, QCOW2_OL_DEFAULT,
-                sector_num * BDRV_SECTOR_SIZE,
-                s->cluster_sectors * BDRV_SECTOR_SIZE);
-        if (ret < 0) {
-            goto fail;
-        }
-
         ret = bdrv_write(bs, sector_num, buf, s->cluster_sectors);
         if (ret < 0) {
             goto fail;
@@ -1714,8 +1837,7 @@ static int qcow2_write_compressed(BlockDriverState *bs, int64_t sector_num,
         }
         cluster_offset &= s->cluster_offset_mask;
 
-        ret = qcow2_pre_write_overlap_check(bs, QCOW2_OL_DEFAULT,
-                cluster_offset, out_len);
+        ret = qcow2_pre_write_overlap_check(bs, 0, cluster_offset, out_len);
         if (ret < 0) {
             goto fail;
         }
@@ -1757,11 +1879,6 @@ static coroutine_fn int qcow2_co_flush_to_os(BlockDriverState *bs)
     return 0;
 }
 
-static int64_t qcow2_vm_state_offset(BDRVQcowState *s)
-{
-	return (int64_t)s->l1_vm_state_index << (s->cluster_bits + s->l2_bits);
-}
-
 static int qcow2_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
 {
     BDRVQcowState *s = bs->opaque;
@@ -1770,6 +1887,33 @@ static int qcow2_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
     return 0;
 }
 
+static ImageInfoSpecific *qcow2_get_specific_info(BlockDriverState *bs)
+{
+    BDRVQcowState *s = bs->opaque;
+    ImageInfoSpecific *spec_info = g_new(ImageInfoSpecific, 1);
+
+    *spec_info = (ImageInfoSpecific){
+        .kind  = IMAGE_INFO_SPECIFIC_KIND_QCOW2,
+        {
+            .qcow2 = g_new(ImageInfoSpecificQCow2, 1),
+        },
+    };
+    if (s->qcow_version == 2) {
+        *spec_info->qcow2 = (ImageInfoSpecificQCow2){
+            .compat = g_strdup("0.10"),
+        };
+    } else if (s->qcow_version == 3) {
+        *spec_info->qcow2 = (ImageInfoSpecificQCow2){
+            .compat             = g_strdup("1.1"),
+            .lazy_refcounts     = s->compatible_features &
+                                  QCOW2_COMPAT_LAZY_REFCOUNTS,
+            .has_lazy_refcounts = true,
+        };
+    }
+
+    return spec_info;
+}
+
 #if 0
 static void dump_refcounts(BlockDriverState *bs)
 {
@@ -1824,6 +1968,199 @@ static int qcow2_load_vmstate(BlockDriverState *bs, uint8_t *buf,
     return ret;
 }
 
+/*
+ * Downgrades an image's version. To achieve this, any incompatible features
+ * have to be removed.
+ */
+static int qcow2_downgrade(BlockDriverState *bs, int target_version)
+{
+    BDRVQcowState *s = bs->opaque;
+    int current_version = s->qcow_version;
+    int ret;
+
+    if (target_version == current_version) {
+        return 0;
+    } else if (target_version > current_version) {
+        return -EINVAL;
+    } else if (target_version != 2) {
+        return -EINVAL;
+    }
+
+    if (s->refcount_order != 4) {
+        /* we would have to convert the image to a refcount_order == 4 image
+         * here; however, since qemu (at the time of writing this) does not
+         * support anything different than 4 anyway, there is no point in doing
+         * so right now; however, we should error out (if qemu supports this in
+         * the future and this code has not been adapted) */
+        error_report("qcow2_downgrade: Image refcount orders other than 4 are "
+                     "currently not supported.");
+        return -ENOTSUP;
+    }
+
+    /* clear incompatible features */
+    if (s->incompatible_features & QCOW2_INCOMPAT_DIRTY) {
+        ret = qcow2_mark_clean(bs);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    /* with QCOW2_INCOMPAT_CORRUPT, it is pretty much impossible to get here in
+     * the first place; if that happens nonetheless, returning -ENOTSUP is the
+     * best thing to do anyway */
+
+    if (s->incompatible_features) {
+        return -ENOTSUP;
+    }
+
+    /* since we can ignore compatible features, we can set them to 0 as well */
+    s->compatible_features = 0;
+    /* if lazy refcounts have been used, they have already been fixed through
+     * clearing the dirty flag */
+
+    /* clearing autoclear features is trivial */
+    s->autoclear_features = 0;
+
+    ret = qcow2_expand_zero_clusters(bs);
+    if (ret < 0) {
+        return ret;
+    }
+
+    s->qcow_version = target_version;
+    ret = qcow2_update_header(bs);
+    if (ret < 0) {
+        s->qcow_version = current_version;
+        return ret;
+    }
+    return 0;
+}
+
+static int qcow2_amend_options(BlockDriverState *bs,
+                               QEMUOptionParameter *options)
+{
+    BDRVQcowState *s = bs->opaque;
+    int old_version = s->qcow_version, new_version = old_version;
+    uint64_t new_size = 0;
+    const char *backing_file = NULL, *backing_format = NULL;
+    bool lazy_refcounts = s->use_lazy_refcounts;
+    int ret;
+    int i;
+
+    for (i = 0; options[i].name; i++)
+    {
+        if (!options[i].assigned) {
+            /* only change explicitly defined options */
+            continue;
+        }
+
+        if (!strcmp(options[i].name, "compat")) {
+            if (!options[i].value.s) {
+                /* preserve default */
+            } else if (!strcmp(options[i].value.s, "0.10")) {
+                new_version = 2;
+            } else if (!strcmp(options[i].value.s, "1.1")) {
+                new_version = 3;
+            } else {
+                fprintf(stderr, "Unknown compatibility level %s.\n",
+                        options[i].value.s);
+                return -EINVAL;
+            }
+        } else if (!strcmp(options[i].name, "preallocation")) {
+            fprintf(stderr, "Cannot change preallocation mode.\n");
+            return -ENOTSUP;
+        } else if (!strcmp(options[i].name, "size")) {
+            new_size = options[i].value.n;
+        } else if (!strcmp(options[i].name, "backing_file")) {
+            backing_file = options[i].value.s;
+        } else if (!strcmp(options[i].name, "backing_fmt")) {
+            backing_format = options[i].value.s;
+        } else if (!strcmp(options[i].name, "encryption")) {
+            if ((options[i].value.n != !!s->crypt_method)) {
+                fprintf(stderr, "Changing the encryption flag is not "
+                        "supported.\n");
+                return -ENOTSUP;
+            }
+        } else if (!strcmp(options[i].name, "cluster_size")) {
+            if (options[i].value.n != s->cluster_size) {
+                fprintf(stderr, "Changing the cluster size is not "
+                        "supported.\n");
+                return -ENOTSUP;
+            }
+        } else if (!strcmp(options[i].name, "lazy_refcounts")) {
+            lazy_refcounts = options[i].value.n;
+        } else {
+            /* if this assertion fails, this probably means a new option was
+             * added without having it covered here */
+            assert(false);
+        }
+    }
+
+    if (new_version != old_version) {
+        if (new_version > old_version) {
+            /* Upgrade */
+            s->qcow_version = new_version;
+            ret = qcow2_update_header(bs);
+            if (ret < 0) {
+                s->qcow_version = old_version;
+                return ret;
+            }
+        } else {
+            ret = qcow2_downgrade(bs, new_version);
+            if (ret < 0) {
+                return ret;
+            }
+        }
+    }
+
+    if (backing_file || backing_format) {
+        ret = qcow2_change_backing_file(bs, backing_file ?: bs->backing_file,
+                                        backing_format ?: bs->backing_format);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    if (s->use_lazy_refcounts != lazy_refcounts) {
+        if (lazy_refcounts) {
+            if (s->qcow_version < 3) {
+                fprintf(stderr, "Lazy refcounts only supported with compatibility "
+                        "level 1.1 and above (use compat=1.1 or greater)\n");
+                return -EINVAL;
+            }
+            s->compatible_features |= QCOW2_COMPAT_LAZY_REFCOUNTS;
+            ret = qcow2_update_header(bs);
+            if (ret < 0) {
+                s->compatible_features &= ~QCOW2_COMPAT_LAZY_REFCOUNTS;
+                return ret;
+            }
+            s->use_lazy_refcounts = true;
+        } else {
+            /* make image clean first */
+            ret = qcow2_mark_clean(bs);
+            if (ret < 0) {
+                return ret;
+            }
+            /* now disallow lazy refcounts */
+            s->compatible_features &= ~QCOW2_COMPAT_LAZY_REFCOUNTS;
+            ret = qcow2_update_header(bs);
+            if (ret < 0) {
+                s->compatible_features |= QCOW2_COMPAT_LAZY_REFCOUNTS;
+                return ret;
+            }
+            s->use_lazy_refcounts = false;
+        }
+    }
+
+    if (new_size) {
+        ret = bdrv_truncate(bs, new_size);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    return 0;
+}
+
 static QEMUOptionParameter qcow2_create_options[] = {
     {
         .name = BLOCK_OPT_SIZE,
@@ -1897,6 +2234,7 @@ static BlockDriver bdrv_qcow2 = {
     .bdrv_snapshot_list     = qcow2_snapshot_list,
     .bdrv_snapshot_load_tmp     = qcow2_snapshot_load_tmp,
     .bdrv_get_info      = qcow2_get_info,
+    .bdrv_get_specific_info = qcow2_get_specific_info,
 
     .bdrv_save_vmstate    = qcow2_save_vmstate,
     .bdrv_load_vmstate    = qcow2_load_vmstate,
@@ -1907,6 +2245,7 @@ static BlockDriver bdrv_qcow2 = {
 
     .create_options = qcow2_create_options,
     .bdrv_check = qcow2_check,
+    .bdrv_amend_options = qcow2_amend_options,
 };
 
 static void bdrv_qcow2_init(void)
diff --git a/block/qcow2.h b/block/qcow2.h
index 1000239e4c..922e19062a 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -63,6 +63,15 @@
 #define QCOW2_OPT_DISCARD_REQUEST "pass-discard-request"
 #define QCOW2_OPT_DISCARD_SNAPSHOT "pass-discard-snapshot"
 #define QCOW2_OPT_DISCARD_OTHER "pass-discard-other"
+#define QCOW2_OPT_OVERLAP "overlap-check"
+#define QCOW2_OPT_OVERLAP_MAIN_HEADER "overlap-check.main-header"
+#define QCOW2_OPT_OVERLAP_ACTIVE_L1 "overlap-check.active-l1"
+#define QCOW2_OPT_OVERLAP_ACTIVE_L2 "overlap-check.active-l2"
+#define QCOW2_OPT_OVERLAP_REFCOUNT_TABLE "overlap-check.refcount-table"
+#define QCOW2_OPT_OVERLAP_REFCOUNT_BLOCK "overlap-check.refcount-block"
+#define QCOW2_OPT_OVERLAP_SNAPSHOT_TABLE "overlap-check.snapshot-table"
+#define QCOW2_OPT_OVERLAP_INACTIVE_L1 "overlap-check.inactive-l1"
+#define QCOW2_OPT_OVERLAP_INACTIVE_L2 "overlap-check.inactive-l2"
 
 typedef struct QCowHeader {
     uint32_t magic;
@@ -86,7 +95,7 @@ typedef struct QCowHeader {
 
     uint32_t refcount_order;
     uint32_t header_length;
-} QCowHeader;
+} QEMU_PACKED QCowHeader;
 
 typedef struct QCowSnapshot {
     uint64_t l1_table_offset;
@@ -199,9 +208,12 @@ typedef struct BDRVQcowState {
     int flags;
     int qcow_version;
     bool use_lazy_refcounts;
+    int refcount_order;
 
     bool discard_passthrough[QCOW2_DISCARD_MAX];
 
+    int overlap_check; /* bitmask of Qcow2MetadataOverlap values */
+
     uint64_t incompatible_features;
     uint64_t compatible_features;
     uint64_t autoclear_features;
@@ -314,14 +326,19 @@ typedef enum QCow2MetadataOverlap {
     QCOW2_OL_INACTIVE_L2    = (1 << QCOW2_OL_INACTIVE_L2_BITNR),
 } QCow2MetadataOverlap;
 
+/* Perform all overlap checks which can be done in constant time */
+#define QCOW2_OL_CONSTANT \
+    (QCOW2_OL_MAIN_HEADER | QCOW2_OL_ACTIVE_L1 | QCOW2_OL_REFCOUNT_TABLE | \
+     QCOW2_OL_SNAPSHOT_TABLE)
+
 /* Perform all overlap checks which don't require disk access */
 #define QCOW2_OL_CACHED \
-    (QCOW2_OL_MAIN_HEADER | QCOW2_OL_ACTIVE_L1 | QCOW2_OL_ACTIVE_L2 | \
-     QCOW2_OL_REFCOUNT_TABLE | QCOW2_OL_REFCOUNT_BLOCK | \
-     QCOW2_OL_SNAPSHOT_TABLE | QCOW2_OL_INACTIVE_L1)
+    (QCOW2_OL_CONSTANT | QCOW2_OL_ACTIVE_L2 | QCOW2_OL_REFCOUNT_BLOCK | \
+     QCOW2_OL_INACTIVE_L1)
 
-/* The default checks to perform */
-#define QCOW2_OL_DEFAULT QCOW2_OL_CACHED
+/* Perform all overlap checks */
+#define QCOW2_OL_ALL \
+    (QCOW2_OL_CACHED | QCOW2_OL_INACTIVE_L2)
 
 #define L1E_OFFSET_MASK 0x00ffffffffffff00ULL
 #define L2E_OFFSET_MASK 0x00ffffffffffff00ULL
@@ -361,6 +378,11 @@ static inline int64_t align_offset(int64_t offset, int n)
     return offset;
 }
 
+static inline int64_t qcow2_vm_state_offset(BDRVQcowState *s)
+{
+    return (int64_t)s->l1_vm_state_index << (s->cluster_bits + s->l2_bits);
+}
+
 static inline int qcow2_get_cluster_type(uint64_t l2_entry)
 {
     if (l2_entry & QCOW_OFLAG_COMPRESSED) {
@@ -406,6 +428,9 @@ int qcow2_update_header(BlockDriverState *bs);
 int qcow2_refcount_init(BlockDriverState *bs);
 void qcow2_refcount_close(BlockDriverState *bs);
 
+int qcow2_update_cluster_refcount(BlockDriverState *bs, int64_t cluster_index,
+                                  int addend, enum qcow2_discard_type type);
+
 int64_t qcow2_alloc_clusters(BlockDriverState *bs, int64_t size);
 int qcow2_alloc_clusters_at(BlockDriverState *bs, uint64_t offset,
     int nb_clusters);
@@ -424,9 +449,9 @@ int qcow2_check_refcounts(BlockDriverState *bs, BdrvCheckResult *res,
 
 void qcow2_process_discards(BlockDriverState *bs, int ret);
 
-int qcow2_check_metadata_overlap(BlockDriverState *bs, int chk, int64_t offset,
+int qcow2_check_metadata_overlap(BlockDriverState *bs, int ign, int64_t offset,
                                  int64_t size);
-int qcow2_pre_write_overlap_check(BlockDriverState *bs, int chk, int64_t offset,
+int qcow2_pre_write_overlap_check(BlockDriverState *bs, int ign, int64_t offset,
                                   int64_t size);
 
 /* qcow2-cluster.c functions */
@@ -450,13 +475,18 @@ uint64_t qcow2_alloc_compressed_cluster_offset(BlockDriverState *bs,
 
 int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m);
 int qcow2_discard_clusters(BlockDriverState *bs, uint64_t offset,
-    int nb_sectors);
+    int nb_sectors, enum qcow2_discard_type type);
 int qcow2_zero_clusters(BlockDriverState *bs, uint64_t offset, int nb_sectors);
 
+int qcow2_expand_zero_clusters(BlockDriverState *bs);
+
 /* qcow2-snapshot.c functions */
 int qcow2_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info);
 int qcow2_snapshot_goto(BlockDriverState *bs, const char *snapshot_id);
-int qcow2_snapshot_delete(BlockDriverState *bs, const char *snapshot_id);
+int qcow2_snapshot_delete(BlockDriverState *bs,
+                          const char *snapshot_id,
+                          const char *name,
+                          Error **errp);
 int qcow2_snapshot_list(BlockDriverState *bs, QEMUSnapshotInfo **psn_tab);
 int qcow2_snapshot_load_tmp(BlockDriverState *bs, const char *snapshot_name);
 
@@ -473,6 +503,8 @@ int qcow2_cache_set_dependency(BlockDriverState *bs, Qcow2Cache *c,
     Qcow2Cache *dependency);
 void qcow2_cache_depends_on_flush(Qcow2Cache *c);
 
+int qcow2_cache_empty(BlockDriverState *bs, Qcow2Cache *c);
+
 int qcow2_cache_get(BlockDriverState *bs, Qcow2Cache *c, uint64_t offset,
     void **table);
 int qcow2_cache_get_empty(BlockDriverState *bs, Qcow2Cache *c, uint64_t offset,
diff --git a/block/qed.c b/block/qed.c
index 49b3a37ed5..6c0cba04f3 100644
--- a/block/qed.c
+++ b/block/qed.c
@@ -373,7 +373,8 @@ static void bdrv_qed_rebind(BlockDriverState *bs)
     s->bs = bs;
 }
 
-static int bdrv_qed_open(BlockDriverState *bs, QDict *options, int flags)
+static int bdrv_qed_open(BlockDriverState *bs, QDict *options, int flags,
+                         Error **errp)
 {
     BDRVQEDState *s = bs->opaque;
     QEDHeader le_header;
@@ -550,16 +551,22 @@ static int qed_create(const char *filename, uint32_t cluster_size,
     QEDHeader le_header;
     uint8_t *l1_table = NULL;
     size_t l1_size = header.cluster_size * header.table_size;
+    Error *local_err = NULL;
     int ret = 0;
     BlockDriverState *bs = NULL;
 
-    ret = bdrv_create_file(filename, NULL);
+    ret = bdrv_create_file(filename, NULL, &local_err);
     if (ret < 0) {
+        qerror_report_err(local_err);
+        error_free(local_err);
         return ret;
     }
 
-    ret = bdrv_file_open(&bs, filename, NULL, BDRV_O_RDWR | BDRV_O_CACHE_WB);
+    ret = bdrv_file_open(&bs, filename, NULL, BDRV_O_RDWR | BDRV_O_CACHE_WB,
+                         &local_err);
     if (ret < 0) {
+        qerror_report_err(local_err);
+        error_free(local_err);
         return ret;
     }
 
@@ -603,7 +610,8 @@ out:
     return ret;
 }
 
-static int bdrv_qed_create(const char *filename, QEMUOptionParameter *options)
+static int bdrv_qed_create(const char *filename, QEMUOptionParameter *options,
+                           Error **errp)
 {
     uint64_t image_size = 0;
     uint32_t cluster_size = QED_DEFAULT_CLUSTER_SIZE;
@@ -1547,7 +1555,7 @@ static void bdrv_qed_invalidate_cache(BlockDriverState *bs)
 
     bdrv_qed_close(bs);
     memset(s, 0, sizeof(BDRVQEDState));
-    bdrv_qed_open(bs, NULL, bs->open_flags);
+    bdrv_qed_open(bs, NULL, bs->open_flags, NULL);
 }
 
 static int bdrv_qed_check(BlockDriverState *bs, BdrvCheckResult *result,
diff --git a/block/qed.h b/block/qed.h
index 2b4ddedf31..5d65bea075 100644
--- a/block/qed.h
+++ b/block/qed.h
@@ -100,7 +100,7 @@ typedef struct {
     /* if (features & QED_F_BACKING_FILE) */
     uint32_t backing_filename_offset; /* in bytes from start of header */
     uint32_t backing_filename_size;   /* in bytes */
-} QEDHeader;
+} QEMU_PACKED QEDHeader;
 
 typedef struct {
     uint64_t offsets[0];            /* in bytes */
diff --git a/block/raw-posix.c b/block/raw-posix.c
index 1b41ea3356..6f03fbf793 100644
--- a/block/raw-posix.c
+++ b/block/raw-posix.c
@@ -276,7 +276,7 @@ static QemuOptsList raw_runtime_opts = {
 };
 
 static int raw_open_common(BlockDriverState *bs, QDict *options,
-                           int bdrv_flags, int open_flags)
+                           int bdrv_flags, int open_flags, Error **errp)
 {
     BDRVRawState *s = bs->opaque;
     QemuOpts *opts;
@@ -287,8 +287,7 @@ static int raw_open_common(BlockDriverState *bs, QDict *options,
     opts = qemu_opts_create_nofail(&raw_runtime_opts);
     qemu_opts_absorb_qdict(opts, options, &local_err);
     if (error_is_set(&local_err)) {
-        qerror_report_err(local_err);
-        error_free(local_err);
+        error_propagate(errp, local_err);
         ret = -EINVAL;
         goto fail;
     }
@@ -297,6 +296,7 @@ static int raw_open_common(BlockDriverState *bs, QDict *options,
 
     ret = raw_normalize_devicepath(&filename);
     if (ret != 0) {
+        error_setg_errno(errp, -ret, "Could not normalize device path");
         goto fail;
     }
 
@@ -310,6 +310,7 @@ static int raw_open_common(BlockDriverState *bs, QDict *options,
         if (ret == -EROFS) {
             ret = -EACCES;
         }
+        error_setg_errno(errp, -ret, "Could not open file");
         goto fail;
     }
     s->fd = fd;
@@ -318,6 +319,7 @@ static int raw_open_common(BlockDriverState *bs, QDict *options,
     if (raw_set_aio(&s->aio_ctx, &s->use_aio, bdrv_flags)) {
         qemu_close(fd);
         ret = -errno;
+        error_setg_errno(errp, -ret, "Could not set AIO state");
         goto fail;
     }
 #endif
@@ -335,12 +337,19 @@ fail:
     return ret;
 }
 
-static int raw_open(BlockDriverState *bs, QDict *options, int flags)
+static int raw_open(BlockDriverState *bs, QDict *options, int flags,
+                    Error **errp)
 {
     BDRVRawState *s = bs->opaque;
+    Error *local_err = NULL;
+    int ret;
 
     s->type = FTYPE_FILE;
-    return raw_open_common(bs, options, flags, 0);
+    ret = raw_open_common(bs, options, flags, 0, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(errp, local_err);
+    }
+    return ret;
 }
 
 static int raw_reopen_prepare(BDRVReopenState *state,
@@ -365,6 +374,7 @@ static int raw_reopen_prepare(BDRVReopenState *state,
      * valid in the 'false' condition even if aio_ctx is set, and raw_set_aio()
      * won't override aio_ctx if aio_ctx is non-NULL */
     if (raw_set_aio(&s->aio_ctx, &raw_s->use_aio, state->flags)) {
+        error_setg(errp, "Could not set AIO state");
         return -1;
     }
 #endif
@@ -416,6 +426,7 @@ static int raw_reopen_prepare(BDRVReopenState *state,
         assert(!(raw_s->open_flags & O_CREAT));
         raw_s->fd = qemu_open(state->bs->filename, raw_s->open_flags);
         if (raw_s->fd == -1) {
+            error_setg_errno(errp, errno, "Could not reopen file");
             ret = -1;
         }
     }
@@ -1040,7 +1051,8 @@ static int64_t raw_get_allocated_file_size(BlockDriverState *bs)
     return (int64_t)st.st_blocks * 512;
 }
 
-static int raw_create(const char *filename, QEMUOptionParameter *options)
+static int raw_create(const char *filename, QEMUOptionParameter *options,
+                      Error **errp)
 {
     int fd;
     int result = 0;
@@ -1058,12 +1070,15 @@ static int raw_create(const char *filename, QEMUOptionParameter *options)
                    0644);
     if (fd < 0) {
         result = -errno;
+        error_setg_errno(errp, -result, "Could not create file");
     } else {
         if (ftruncate(fd, total_size * BDRV_SECTOR_SIZE) != 0) {
             result = -errno;
+            error_setg_errno(errp, -result, "Could not resize file");
         }
         if (qemu_close(fd) != 0) {
             result = -errno;
+            error_setg_errno(errp, -result, "Could not close the new file");
         }
     }
     return result;
@@ -1198,6 +1213,7 @@ static BlockDriver bdrv_file = {
     .format_name = "file",
     .protocol_name = "file",
     .instance_size = sizeof(BDRVRawState),
+    .bdrv_needs_filename = true,
     .bdrv_probe = NULL, /* no probe for protocols */
     .bdrv_file_open = raw_open,
     .bdrv_reopen_prepare = raw_reopen_prepare,
@@ -1331,9 +1347,11 @@ static int check_hdev_writable(BDRVRawState *s)
     return 0;
 }
 
-static int hdev_open(BlockDriverState *bs, QDict *options, int flags)
+static int hdev_open(BlockDriverState *bs, QDict *options, int flags,
+                     Error **errp)
 {
     BDRVRawState *s = bs->opaque;
+    Error *local_err = NULL;
     int ret;
     const char *filename = qdict_get_str(options, "filename");
 
@@ -1377,8 +1395,11 @@ static int hdev_open(BlockDriverState *bs, QDict *options, int flags)
     }
 #endif
 
-    ret = raw_open_common(bs, options, flags, 0);
+    ret = raw_open_common(bs, options, flags, 0, &local_err);
     if (ret < 0) {
+        if (error_is_set(&local_err)) {
+            error_propagate(errp, local_err);
+        }
         return ret;
     }
 
@@ -1386,6 +1407,7 @@ static int hdev_open(BlockDriverState *bs, QDict *options, int flags)
         ret = check_hdev_writable(s);
         if (ret < 0) {
             raw_close(bs);
+            error_setg_errno(errp, -ret, "The device is not writable");
             return ret;
         }
     }
@@ -1504,7 +1526,8 @@ static coroutine_fn BlockDriverAIOCB *hdev_aio_discard(BlockDriverState *bs,
                        cb, opaque, QEMU_AIO_DISCARD|QEMU_AIO_BLKDEV);
 }
 
-static int hdev_create(const char *filename, QEMUOptionParameter *options)
+static int hdev_create(const char *filename, QEMUOptionParameter *options,
+                       Error **errp)
 {
     int fd;
     int ret = 0;
@@ -1520,15 +1543,23 @@ static int hdev_create(const char *filename, QEMUOptionParameter *options)
     }
 
     fd = qemu_open(filename, O_WRONLY | O_BINARY);
-    if (fd < 0)
-        return -errno;
+    if (fd < 0) {
+        ret = -errno;
+        error_setg_errno(errp, -ret, "Could not open device");
+        return ret;
+    }
 
-    if (fstat(fd, &stat_buf) < 0)
+    if (fstat(fd, &stat_buf) < 0) {
         ret = -errno;
-    else if (!S_ISBLK(stat_buf.st_mode) && !S_ISCHR(stat_buf.st_mode))
+        error_setg_errno(errp, -ret, "Could not stat device");
+    } else if (!S_ISBLK(stat_buf.st_mode) && !S_ISCHR(stat_buf.st_mode)) {
+        error_setg(errp,
+                   "The given file is neither a block nor a character device");
         ret = -ENODEV;
-    else if (lseek(fd, 0, SEEK_END) < total_size * BDRV_SECTOR_SIZE)
+    } else if (lseek(fd, 0, SEEK_END) < total_size * BDRV_SECTOR_SIZE) {
+        error_setg(errp, "Device is too small");
         ret = -ENOSPC;
+    }
 
     qemu_close(fd);
     return ret;
@@ -1538,6 +1569,7 @@ static BlockDriver bdrv_host_device = {
     .format_name        = "host_device",
     .protocol_name        = "host_device",
     .instance_size      = sizeof(BDRVRawState),
+    .bdrv_needs_filename = true,
     .bdrv_probe_device  = hdev_probe_device,
     .bdrv_file_open     = hdev_open,
     .bdrv_close         = raw_close,
@@ -1565,17 +1597,23 @@ static BlockDriver bdrv_host_device = {
 };
 
 #ifdef __linux__
-static int floppy_open(BlockDriverState *bs, QDict *options, int flags)
+static int floppy_open(BlockDriverState *bs, QDict *options, int flags,
+                       Error **errp)
 {
     BDRVRawState *s = bs->opaque;
+    Error *local_err = NULL;
     int ret;
 
     s->type = FTYPE_FD;
 
     /* open will not fail even if no floppy is inserted, so add O_NONBLOCK */
-    ret = raw_open_common(bs, options, flags, O_NONBLOCK);
-    if (ret)
+    ret = raw_open_common(bs, options, flags, O_NONBLOCK, &local_err);
+    if (ret) {
+        if (error_is_set(&local_err)) {
+            error_propagate(errp, local_err);
+        }
         return ret;
+    }
 
     /* close fd so that we can reopen it as needed */
     qemu_close(s->fd);
@@ -1662,6 +1700,7 @@ static BlockDriver bdrv_host_floppy = {
     .format_name        = "host_floppy",
     .protocol_name      = "host_floppy",
     .instance_size      = sizeof(BDRVRawState),
+    .bdrv_needs_filename = true,
     .bdrv_probe_device	= floppy_probe_device,
     .bdrv_file_open     = floppy_open,
     .bdrv_close         = raw_close,
@@ -1686,14 +1725,21 @@ static BlockDriver bdrv_host_floppy = {
     .bdrv_eject         = floppy_eject,
 };
 
-static int cdrom_open(BlockDriverState *bs, QDict *options, int flags)
+static int cdrom_open(BlockDriverState *bs, QDict *options, int flags,
+                      Error **errp)
 {
     BDRVRawState *s = bs->opaque;
+    Error *local_err = NULL;
+    int ret;
 
     s->type = FTYPE_CD;
 
     /* open will not fail even if no CD is inserted, so add O_NONBLOCK */
-    return raw_open_common(bs, options, flags, O_NONBLOCK);
+    ret = raw_open_common(bs, options, flags, O_NONBLOCK, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(errp, local_err);
+    }
+    return ret;
 }
 
 static int cdrom_probe_device(const char *filename)
@@ -1763,6 +1809,7 @@ static BlockDriver bdrv_host_cdrom = {
     .format_name        = "host_cdrom",
     .protocol_name      = "host_cdrom",
     .instance_size      = sizeof(BDRVRawState),
+    .bdrv_needs_filename = true,
     .bdrv_probe_device	= cdrom_probe_device,
     .bdrv_file_open     = cdrom_open,
     .bdrv_close         = raw_close,
@@ -1796,13 +1843,18 @@ static BlockDriver bdrv_host_cdrom = {
 static int cdrom_open(BlockDriverState *bs, QDict *options, int flags)
 {
     BDRVRawState *s = bs->opaque;
+    Error *local_err = NULL;
     int ret;
 
     s->type = FTYPE_CD;
 
-    ret = raw_open_common(bs, options, flags, 0);
-    if (ret)
+    ret = raw_open_common(bs, options, flags, 0, &local_err);
+    if (ret) {
+        if (error_is_set(&local_err)) {
+            error_propagate(errp, local_err);
+        }
         return ret;
+    }
 
     /* make sure the door isn't locked at this time */
     ioctl(s->fd, CDIOCALLOW);
@@ -1884,6 +1936,7 @@ static BlockDriver bdrv_host_cdrom = {
     .format_name        = "host_cdrom",
     .protocol_name      = "host_cdrom",
     .instance_size      = sizeof(BDRVRawState),
+    .bdrv_needs_filename = true,
     .bdrv_probe_device	= cdrom_probe_device,
     .bdrv_file_open     = cdrom_open,
     .bdrv_close         = raw_close,
diff --git a/block/raw-win32.c b/block/raw-win32.c
index d2d2d9f4d4..676b5701db 100644
--- a/block/raw-win32.c
+++ b/block/raw-win32.c
@@ -85,6 +85,7 @@ static size_t handle_aiocb_rw(RawWin32AIOData *aiocb)
             ret_count = 0;
         }
         if (ret_count != len) {
+            offset += ret_count;
             break;
         }
         offset += len;
@@ -234,7 +235,8 @@ static QemuOptsList raw_runtime_opts = {
     },
 };
 
-static int raw_open(BlockDriverState *bs, QDict *options, int flags)
+static int raw_open(BlockDriverState *bs, QDict *options, int flags,
+                    Error **errp)
 {
     BDRVRawState *s = bs->opaque;
     int access_flags;
@@ -249,8 +251,7 @@ static int raw_open(BlockDriverState *bs, QDict *options, int flags)
     opts = qemu_opts_create_nofail(&raw_runtime_opts);
     qemu_opts_absorb_qdict(opts, options, &local_err);
     if (error_is_set(&local_err)) {
-        qerror_report_err(local_err);
-        error_free(local_err);
+        error_propagate(errp, local_err);
         ret = -EINVAL;
         goto fail;
     }
@@ -262,6 +263,7 @@ static int raw_open(BlockDriverState *bs, QDict *options, int flags)
     if ((flags & BDRV_O_NATIVE_AIO) && aio == NULL) {
         aio = win32_aio_init();
         if (aio == NULL) {
+            error_setg(errp, "Could not initialize AIO");
             ret = -EINVAL;
             goto fail;
         }
@@ -278,6 +280,7 @@ static int raw_open(BlockDriverState *bs, QDict *options, int flags)
         } else {
             ret = -EINVAL;
         }
+        error_setg_errno(errp, -ret, "Could not open file");
         goto fail;
     }
 
@@ -285,6 +288,7 @@ static int raw_open(BlockDriverState *bs, QDict *options, int flags)
         ret = win32_aio_attach(aio, s->hfile);
         if (ret < 0) {
             CloseHandle(s->hfile);
+            error_setg_errno(errp, -ret, "Could not enable AIO");
             goto fail;
         }
         s->aio = aio;
@@ -420,7 +424,8 @@ static int64_t raw_get_allocated_file_size(BlockDriverState *bs)
     return st.st_size;
 }
 
-static int raw_create(const char *filename, QEMUOptionParameter *options)
+static int raw_create(const char *filename, QEMUOptionParameter *options,
+                      Error **errp)
 {
     int fd;
     int64_t total_size = 0;
@@ -435,8 +440,10 @@ static int raw_create(const char *filename, QEMUOptionParameter *options)
 
     fd = qemu_open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY,
                    0644);
-    if (fd < 0)
+    if (fd < 0) {
+        error_setg_errno(errp, errno, "Could not create file");
         return -EIO;
+    }
     set_sparse(fd);
     ftruncate(fd, total_size * 512);
     qemu_close(fd);
@@ -456,6 +463,7 @@ static BlockDriver bdrv_file = {
     .format_name	= "file",
     .protocol_name	= "file",
     .instance_size	= sizeof(BDRVRawState),
+    .bdrv_needs_filename = true,
     .bdrv_file_open	= raw_open,
     .bdrv_close		= raw_close,
     .bdrv_create	= raw_create,
@@ -531,7 +539,8 @@ static int hdev_probe_device(const char *filename)
     return 0;
 }
 
-static int hdev_open(BlockDriverState *bs, QDict *options, int flags)
+static int hdev_open(BlockDriverState *bs, QDict *options, int flags,
+                     Error **errp)
 {
     BDRVRawState *s = bs->opaque;
     int access_flags, create_flags;
@@ -545,8 +554,7 @@ static int hdev_open(BlockDriverState *bs, QDict *options, int flags)
     QemuOpts *opts = qemu_opts_create_nofail(&raw_runtime_opts);
     qemu_opts_absorb_qdict(opts, options, &local_err);
     if (error_is_set(&local_err)) {
-        qerror_report_err(local_err);
-        error_free(local_err);
+        error_propagate(errp, local_err);
         ret = -EINVAL;
         goto done;
     }
@@ -555,6 +563,7 @@ static int hdev_open(BlockDriverState *bs, QDict *options, int flags)
 
     if (strstart(filename, "/dev/cdrom", NULL)) {
         if (find_cdrom(device_name, sizeof(device_name)) < 0) {
+            error_setg(errp, "Could not open CD-ROM drive");
             ret = -ENOENT;
             goto done;
         }
@@ -583,8 +592,9 @@ static int hdev_open(BlockDriverState *bs, QDict *options, int flags)
         if (err == ERROR_ACCESS_DENIED) {
             ret = -EACCES;
         } else {
-            ret = -1;
+            ret = -EINVAL;
         }
+        error_setg_errno(errp, -ret, "Could not open device");
         goto done;
     }
 
@@ -597,6 +607,7 @@ static BlockDriver bdrv_host_device = {
     .format_name	= "host_device",
     .protocol_name	= "host_device",
     .instance_size	= sizeof(BDRVRawState),
+    .bdrv_needs_filename = true,
     .bdrv_probe_device	= hdev_probe_device,
     .bdrv_file_open	= hdev_open,
     .bdrv_close		= raw_close,
diff --git a/block/raw_bsd.c b/block/raw_bsd.c
index a9060caec4..0078c1baeb 100644
--- a/block/raw_bsd.c
+++ b/block/raw_bsd.c
@@ -62,7 +62,9 @@ static int64_t coroutine_fn raw_co_get_block_status(BlockDriverState *bs,
                                             int64_t sector_num,
                                             int nb_sectors, int *pnum)
 {
-    return bdrv_get_block_status(bs->file, sector_num, nb_sectors, pnum);
+    *pnum = nb_sectors;
+    return BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID | BDRV_BLOCK_DATA |
+           (sector_num << BDRV_SECTOR_BITS);
 }
 
 static int coroutine_fn raw_co_write_zeroes(BlockDriverState *bs,
@@ -130,12 +132,21 @@ static int raw_has_zero_init(BlockDriverState *bs)
     return bdrv_has_zero_init(bs->file);
 }
 
-static int raw_create(const char *filename, QEMUOptionParameter *options)
+static int raw_create(const char *filename, QEMUOptionParameter *options,
+                      Error **errp)
 {
-    return bdrv_create_file(filename, options);
+    Error *local_err = NULL;
+    int ret;
+
+    ret = bdrv_create_file(filename, options, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(errp, local_err);
+    }
+    return ret;
 }
 
-static int raw_open(BlockDriverState *bs, QDict *options, int flags)
+static int raw_open(BlockDriverState *bs, QDict *options, int flags,
+                    Error **errp)
 {
     bs->sg = bs->file->sg;
     return 0;
diff --git a/block/rbd.c b/block/rbd.c
index e798e19f81..4a1ea5b5ce 100644
--- a/block/rbd.c
+++ b/block/rbd.c
@@ -287,7 +287,8 @@ static int qemu_rbd_set_conf(rados_t cluster, const char *conf)
     return ret;
 }
 
-static int qemu_rbd_create(const char *filename, QEMUOptionParameter *options)
+static int qemu_rbd_create(const char *filename, QEMUOptionParameter *options,
+                           Error **errp)
 {
     int64_t bytes = 0;
     int64_t objsize;
@@ -446,7 +447,8 @@ static QemuOptsList runtime_opts = {
     },
 };
 
-static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags)
+static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
+                         Error **errp)
 {
     BDRVRBDState *s = bs->opaque;
     char pool[RBD_MAX_POOL_NAME_SIZE];
@@ -891,12 +893,31 @@ static int qemu_rbd_snap_create(BlockDriverState *bs,
 }
 
 static int qemu_rbd_snap_remove(BlockDriverState *bs,
-                                const char *snapshot_name)
+                                const char *snapshot_id,
+                                const char *snapshot_name,
+                                Error **errp)
 {
     BDRVRBDState *s = bs->opaque;
     int r;
 
+    if (!snapshot_name) {
+        error_setg(errp, "rbd need a valid snapshot name");
+        return -EINVAL;
+    }
+
+    /* If snapshot_id is specified, it must be equal to name, see
+       qemu_rbd_snap_list() */
+    if (snapshot_id && strcmp(snapshot_id, snapshot_name)) {
+        error_setg(errp,
+                   "rbd do not support snapshot id, it should be NULL or "
+                   "equal to snapshot name");
+        return -EINVAL;
+    }
+
     r = rbd_snap_remove(s->image, snapshot_name);
+    if (r < 0) {
+        error_setg_errno(errp, -r, "Failed to remove the snapshot");
+    }
     return r;
 }
 
@@ -922,7 +943,7 @@ static int qemu_rbd_snap_list(BlockDriverState *bs,
     do {
         snaps = g_malloc(sizeof(*snaps) * max_snaps);
         snap_count = rbd_snap_list(s->image, snaps, &max_snaps);
-        if (snap_count < 0) {
+        if (snap_count <= 0) {
             g_free(snaps);
         }
     } while (snap_count == -ERANGE);
@@ -946,6 +967,7 @@ static int qemu_rbd_snap_list(BlockDriverState *bs,
         sn_info->vm_clock_nsec = 0;
     }
     rbd_snap_list_end(snaps);
+    g_free(snaps);
 
  done:
     *psn_tab = sn_tab;
@@ -981,6 +1003,7 @@ static QEMUOptionParameter qemu_rbd_create_options[] = {
 static BlockDriver bdrv_rbd = {
     .format_name        = "rbd",
     .instance_size      = sizeof(BDRVRBDState),
+    .bdrv_needs_filename = true,
     .bdrv_file_open     = qemu_rbd_open,
     .bdrv_close         = qemu_rbd_close,
     .bdrv_create        = qemu_rbd_create,
diff --git a/block/sheepdog.c b/block/sheepdog.c
index f9988d35ba..5f81c93ee3 100644
--- a/block/sheepdog.c
+++ b/block/sheepdog.c
@@ -1242,7 +1242,8 @@ static QemuOptsList runtime_opts = {
     },
 };
 
-static int sd_open(BlockDriverState *bs, QDict *options, int flags)
+static int sd_open(BlockDriverState *bs, QDict *options, int flags,
+                   Error **errp)
 {
     int ret, fd;
     uint32_t vid = 0;
@@ -1400,10 +1401,13 @@ static int sd_prealloc(const char *filename)
     uint32_t idx, max_idx;
     int64_t vdi_size;
     void *buf = g_malloc0(SD_DATA_OBJ_SIZE);
+    Error *local_err = NULL;
     int ret;
 
-    ret = bdrv_file_open(&bs, filename, NULL, BDRV_O_RDWR);
+    ret = bdrv_file_open(&bs, filename, NULL, BDRV_O_RDWR, &local_err);
     if (ret < 0) {
+        qerror_report_err(local_err);
+        error_free(local_err);
         goto out;
     }
 
@@ -1437,7 +1441,8 @@ out:
     return ret;
 }
 
-static int sd_create(const char *filename, QEMUOptionParameter *options)
+static int sd_create(const char *filename, QEMUOptionParameter *options,
+                     Error **errp)
 {
     int ret = 0;
     uint32_t vid = 0, base_vid = 0;
@@ -1447,6 +1452,7 @@ static int sd_create(const char *filename, QEMUOptionParameter *options)
     char vdi[SD_MAX_VDI_LEN], tag[SD_MAX_VDI_TAG_LEN];
     uint32_t snapid;
     bool prealloc = false;
+    Error *local_err = NULL;
 
     s = g_malloc0(sizeof(BDRVSheepdogState));
 
@@ -1500,8 +1506,10 @@ static int sd_create(const char *filename, QEMUOptionParameter *options)
             goto out;
         }
 
-        ret = bdrv_file_open(&bs, backing_file, NULL, 0);
+        ret = bdrv_file_open(&bs, backing_file, NULL, 0, &local_err);
         if (ret < 0) {
+            qerror_report_err(local_err);
+            error_free(local_err);
             goto out;
         }
 
@@ -2072,7 +2080,10 @@ out:
     return ret;
 }
 
-static int sd_snapshot_delete(BlockDriverState *bs, const char *snapshot_id)
+static int sd_snapshot_delete(BlockDriverState *bs,
+                              const char *snapshot_id,
+                              const char *name,
+                              Error **errp)
 {
     /* FIXME: Delete specified snapshot id.  */
     return 0;
@@ -2327,6 +2338,7 @@ static BlockDriver bdrv_sheepdog = {
     .format_name    = "sheepdog",
     .protocol_name  = "sheepdog",
     .instance_size  = sizeof(BDRVSheepdogState),
+    .bdrv_needs_filename = true,
     .bdrv_file_open = sd_open,
     .bdrv_close     = sd_close,
     .bdrv_create    = sd_create,
@@ -2355,6 +2367,7 @@ static BlockDriver bdrv_sheepdog_tcp = {
     .format_name    = "sheepdog",
     .protocol_name  = "sheepdog+tcp",
     .instance_size  = sizeof(BDRVSheepdogState),
+    .bdrv_needs_filename = true,
     .bdrv_file_open = sd_open,
     .bdrv_close     = sd_close,
     .bdrv_create    = sd_create,
@@ -2383,6 +2396,7 @@ static BlockDriver bdrv_sheepdog_unix = {
     .format_name    = "sheepdog",
     .protocol_name  = "sheepdog+unix",
     .instance_size  = sizeof(BDRVSheepdogState),
+    .bdrv_needs_filename = true,
     .bdrv_file_open = sd_open,
     .bdrv_close     = sd_close,
     .bdrv_create    = sd_create,
diff --git a/block/snapshot.c b/block/snapshot.c
index 8f61cc0745..a05c0c0be0 100644
--- a/block/snapshot.c
+++ b/block/snapshot.c
@@ -48,6 +48,79 @@ int bdrv_snapshot_find(BlockDriverState *bs, QEMUSnapshotInfo *sn_info,
     return ret;
 }
 
+/**
+ * Look up an internal snapshot by @id and @name.
+ * @bs: block device to search
+ * @id: unique snapshot ID, or NULL
+ * @name: snapshot name, or NULL
+ * @sn_info: location to store information on the snapshot found
+ * @errp: location to store error, will be set only for exception
+ *
+ * This function will traverse snapshot list in @bs to search the matching
+ * one, @id and @name are the matching condition:
+ * If both @id and @name are specified, find the first one with id @id and
+ * name @name.
+ * If only @id is specified, find the first one with id @id.
+ * If only @name is specified, find the first one with name @name.
+ * if none is specified, abort().
+ *
+ * Returns: true when a snapshot is found and @sn_info will be filled, false
+ * when error or not found. If all operation succeed but no matching one is
+ * found, @errp will NOT be set.
+ */
+bool bdrv_snapshot_find_by_id_and_name(BlockDriverState *bs,
+                                       const char *id,
+                                       const char *name,
+                                       QEMUSnapshotInfo *sn_info,
+                                       Error **errp)
+{
+    QEMUSnapshotInfo *sn_tab, *sn;
+    int nb_sns, i;
+    bool ret = false;
+
+    assert(id || name);
+
+    nb_sns = bdrv_snapshot_list(bs, &sn_tab);
+    if (nb_sns < 0) {
+        error_setg_errno(errp, -nb_sns, "Failed to get a snapshot list");
+        return false;
+    } else if (nb_sns == 0) {
+        return false;
+    }
+
+    if (id && name) {
+        for (i = 0; i < nb_sns; i++) {
+            sn = &sn_tab[i];
+            if (!strcmp(sn->id_str, id) && !strcmp(sn->name, name)) {
+                *sn_info = *sn;
+                ret = true;
+                break;
+            }
+        }
+    } else if (id) {
+        for (i = 0; i < nb_sns; i++) {
+            sn = &sn_tab[i];
+            if (!strcmp(sn->id_str, id)) {
+                *sn_info = *sn;
+                ret = true;
+                break;
+            }
+        }
+    } else if (name) {
+        for (i = 0; i < nb_sns; i++) {
+            sn = &sn_tab[i];
+            if (!strcmp(sn->name, name)) {
+                *sn_info = *sn;
+                ret = true;
+                break;
+            }
+        }
+    }
+
+    g_free(sn_tab);
+    return ret;
+}
+
 int bdrv_can_snapshot(BlockDriverState *bs)
 {
     BlockDriver *drv = bs->drv;
@@ -97,7 +170,7 @@ int bdrv_snapshot_goto(BlockDriverState *bs,
     if (bs->file) {
         drv->bdrv_close(bs);
         ret = bdrv_snapshot_goto(bs->file, snapshot_id);
-        open_ret = drv->bdrv_open(bs, NULL, bs->open_flags);
+        open_ret = drv->bdrv_open(bs, NULL, bs->open_flags, NULL);
         if (open_ret < 0) {
             bdrv_unref(bs->file);
             bs->drv = NULL;
@@ -109,21 +182,73 @@ int bdrv_snapshot_goto(BlockDriverState *bs,
     return -ENOTSUP;
 }
 
-int bdrv_snapshot_delete(BlockDriverState *bs, const char *snapshot_id)
+/**
+ * Delete an internal snapshot by @snapshot_id and @name.
+ * @bs: block device used in the operation
+ * @snapshot_id: unique snapshot ID, or NULL
+ * @name: snapshot name, or NULL
+ * @errp: location to store error
+ *
+ * If both @snapshot_id and @name are specified, delete the first one with
+ * id @snapshot_id and name @name.
+ * If only @snapshot_id is specified, delete the first one with id
+ * @snapshot_id.
+ * If only @name is specified, delete the first one with name @name.
+ * if none is specified, return -ENINVAL.
+ *
+ * Returns: 0 on success, -errno on failure. If @bs is not inserted, return
+ * -ENOMEDIUM. If @snapshot_id and @name are both NULL, return -EINVAL. If @bs
+ * does not support internal snapshot deletion, return -ENOTSUP. If @bs does
+ * not support parameter @snapshot_id or @name, or one of them is not correctly
+ * specified, return -EINVAL. If @bs can't find one matching @id and @name,
+ * return -ENOENT. If @errp != NULL, it will always be filled with error
+ * message on failure.
+ */
+int bdrv_snapshot_delete(BlockDriverState *bs,
+                         const char *snapshot_id,
+                         const char *name,
+                         Error **errp)
 {
     BlockDriver *drv = bs->drv;
     if (!drv) {
+        error_set(errp, QERR_DEVICE_HAS_NO_MEDIUM, bdrv_get_device_name(bs));
         return -ENOMEDIUM;
     }
+    if (!snapshot_id && !name) {
+        error_setg(errp, "snapshot_id and name are both NULL");
+        return -EINVAL;
+    }
     if (drv->bdrv_snapshot_delete) {
-        return drv->bdrv_snapshot_delete(bs, snapshot_id);
+        return drv->bdrv_snapshot_delete(bs, snapshot_id, name, errp);
     }
     if (bs->file) {
-        return bdrv_snapshot_delete(bs->file, snapshot_id);
+        return bdrv_snapshot_delete(bs->file, snapshot_id, name, errp);
     }
+    error_set(errp, QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
+              drv->format_name, bdrv_get_device_name(bs),
+              "internal snapshot deletion");
     return -ENOTSUP;
 }
 
+void bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs,
+                                        const char *id_or_name,
+                                        Error **errp)
+{
+    int ret;
+    Error *local_err = NULL;
+
+    ret = bdrv_snapshot_delete(bs, id_or_name, NULL, &local_err);
+    if (ret == -ENOENT || ret == -EINVAL) {
+        error_free(local_err);
+        local_err = NULL;
+        ret = bdrv_snapshot_delete(bs, NULL, id_or_name, &local_err);
+    }
+
+    if (ret < 0) {
+        error_propagate(errp, local_err);
+    }
+}
+
 int bdrv_snapshot_list(BlockDriverState *bs,
                        QEMUSnapshotInfo **psn_info)
 {
diff --git a/block/ssh.c b/block/ssh.c
index 27691b4ad5..aa63c9d20e 100644
--- a/block/ssh.c
+++ b/block/ssh.c
@@ -608,7 +608,8 @@ static int connect_to_ssh(BDRVSSHState *s, QDict *options,
     return ret;
 }
 
-static int ssh_file_open(BlockDriverState *bs, QDict *options, int bdrv_flags)
+static int ssh_file_open(BlockDriverState *bs, QDict *options, int bdrv_flags,
+                         Error **errp)
 {
     BDRVSSHState *s = bs->opaque;
     int ret;
@@ -650,7 +651,8 @@ static QEMUOptionParameter ssh_create_options[] = {
     { NULL }
 };
 
-static int ssh_create(const char *filename, QEMUOptionParameter *options)
+static int ssh_create(const char *filename, QEMUOptionParameter *options,
+                      Error **errp)
 {
     int r, ret;
     Error *local_err = NULL;
diff --git a/block/stream.c b/block/stream.c
index 078ce4aa6a..694fd42e41 100644
--- a/block/stream.c
+++ b/block/stream.c
@@ -119,11 +119,12 @@ wait:
             break;
         }
 
+        copy = false;
+
         ret = bdrv_is_allocated(bs, sector_num,
                                 STREAM_BUFFER_SIZE / BDRV_SECTOR_SIZE, &n);
         if (ret == 1) {
             /* Allocated in the top, no need to copy.  */
-            copy = false;
         } else if (ret >= 0) {
             /* Copy if allocated in the intermediate images.  Limit to the
              * known-unallocated area [sector_num, sector_num+n).  */
@@ -138,7 +139,7 @@ wait:
             copy = (ret == 1);
         }
         trace_stream_one_iteration(s, sector_num, n, ret);
-        if (ret >= 0 && copy) {
+        if (copy) {
             if (s->common.speed) {
                 delay_ns = ratelimit_calculate_delay(&s->limit, n);
                 if (delay_ns > 0) {
@@ -202,9 +203,9 @@ static void stream_set_speed(BlockJob *job, int64_t speed, Error **errp)
     ratelimit_set_speed(&s->limit, speed / BDRV_SECTOR_SIZE, SLICE_TIME);
 }
 
-static const BlockJobType stream_job_type = {
+static const BlockJobDriver stream_job_driver = {
     .instance_size = sizeof(StreamBlockJob),
-    .job_type      = "stream",
+    .job_type      = BLOCK_JOB_TYPE_STREAM,
     .set_speed     = stream_set_speed,
 };
 
@@ -223,7 +224,7 @@ void stream_start(BlockDriverState *bs, BlockDriverState *base,
         return;
     }
 
-    s = block_job_create(&stream_job_type, bs, speed, cb, opaque, errp);
+    s = block_job_create(&stream_job_driver, bs, speed, cb, opaque, errp);
     if (!s) {
         return;
     }
diff --git a/block/vdi.c b/block/vdi.c
index 1bf7dc575a..b6ec0020dc 100644
--- a/block/vdi.c
+++ b/block/vdi.c
@@ -165,7 +165,7 @@ typedef struct {
     uuid_t uuid_link;
     uuid_t uuid_parent;
     uint64_t unused2[7];
-} VdiHeader;
+} QEMU_PACKED VdiHeader;
 
 typedef struct {
     /* The block map entries are little endian (even in memory). */
@@ -364,7 +364,8 @@ static int vdi_probe(const uint8_t *buf, int buf_size, const char *filename)
     return result;
 }
 
-static int vdi_open(BlockDriverState *bs, QDict *options, int flags)
+static int vdi_open(BlockDriverState *bs, QDict *options, int flags,
+                    Error **errp)
 {
     BDRVVdiState *s = bs->opaque;
     VdiHeader header;
@@ -644,7 +645,8 @@ static int vdi_co_write(BlockDriverState *bs,
     return ret;
 }
 
-static int vdi_create(const char *filename, QEMUOptionParameter *options)
+static int vdi_create(const char *filename, QEMUOptionParameter *options,
+                      Error **errp)
 {
     int fd;
     int result = 0;
diff --git a/block/vhdx.c b/block/vhdx.c
index e9704b1fdc..6cb04122bb 100644
--- a/block/vhdx.c
+++ b/block/vhdx.c
@@ -20,6 +20,7 @@
 #include "qemu/module.h"
 #include "qemu/crc32c.h"
 #include "block/vhdx.h"
+#include "migration/migration.h"
 
 
 /* Several metadata and region table data entries are identified by
@@ -159,6 +160,7 @@ typedef struct BDRVVHDXState {
     VHDXParentLocatorHeader parent_header;
     VHDXParentLocatorEntry *parent_entries;
 
+    Error *migration_blocker;
 } BDRVVHDXState;
 
 uint32_t vhdx_checksum_calc(uint32_t crc, uint8_t *buf, size_t size,
@@ -715,7 +717,8 @@ exit:
 }
 
 
-static int vhdx_open(BlockDriverState *bs, QDict *options, int flags)
+static int vhdx_open(BlockDriverState *bs, QDict *options, int flags,
+                     Error **errp)
 {
     BDRVVHDXState *s = bs->opaque;
     int ret = 0;
@@ -805,6 +808,12 @@ static int vhdx_open(BlockDriverState *bs, QDict *options, int flags)
 
     /* TODO: differencing files, write */
 
+    /* Disable migration when VHDX images are used */
+    error_set(&s->migration_blocker,
+            QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
+            "vhdx", bs->device_name, "live migration");
+    migrate_add_blocker(s->migration_blocker);
+
     return 0;
 fail:
     qemu_vfree(s->headers[0]);
@@ -951,6 +960,8 @@ static void vhdx_close(BlockDriverState *bs)
     qemu_vfree(s->headers[1]);
     qemu_vfree(s->bat);
     qemu_vfree(s->parent_entries);
+    migrate_del_blocker(s->migration_blocker);
+    error_free(s->migration_blocker);
 }
 
 static BlockDriver bdrv_vhdx = {
diff --git a/block/vmdk.c b/block/vmdk.c
index fb5b5297ce..32ec8b7766 100644
--- a/block/vmdk.c
+++ b/block/vmdk.c
@@ -105,13 +105,14 @@ typedef struct VmdkExtent {
     uint32_t l2_cache_offsets[L2_CACHE_SIZE];
     uint32_t l2_cache_counts[L2_CACHE_SIZE];
 
-    unsigned int cluster_sectors;
+    int64_t cluster_sectors;
 } VmdkExtent;
 
 typedef struct BDRVVmdkState {
     CoMutex lock;
     uint64_t desc_offset;
     bool cid_updated;
+    bool cid_checked;
     uint32_t parent_cid;
     int num_extents;
     /* Extent array with num_extents entries, ascend ordered by address */
@@ -197,8 +198,6 @@ static int vmdk_probe(const uint8_t *buf, int buf_size, const char *filename)
     }
 }
 
-#define CHECK_CID 1
-
 #define SECTOR_SIZE 512
 #define DESC_SIZE (20 * SECTOR_SIZE)    /* 20 sectors of 512 bytes each */
 #define BUF_SIZE 4096
@@ -301,19 +300,18 @@ static int vmdk_write_cid(BlockDriverState *bs, uint32_t cid)
 
 static int vmdk_is_cid_valid(BlockDriverState *bs)
 {
-#ifdef CHECK_CID
     BDRVVmdkState *s = bs->opaque;
     BlockDriverState *p_bs = bs->backing_hd;
     uint32_t cur_pcid;
 
-    if (p_bs) {
+    if (!s->cid_checked && p_bs) {
         cur_pcid = vmdk_read_cid(p_bs, 0);
         if (s->parent_cid != cur_pcid) {
             /* CID not valid */
             return 0;
         }
     }
-#endif
+    s->cid_checked = true;
     /* CID valid */
     return 1;
 }
@@ -331,8 +329,7 @@ static int vmdk_reopen_prepare(BDRVReopenState *state,
     assert(state->bs != NULL);
 
     if (queue == NULL) {
-        error_set(errp, ERROR_CLASS_GENERIC_ERROR,
-                 "No reopen queue for VMDK extents");
+        error_setg(errp, "No reopen queue for VMDK extents");
         goto exit;
     }
 
@@ -391,22 +388,23 @@ static int vmdk_add_extent(BlockDriverState *bs,
                            int64_t l1_offset, int64_t l1_backup_offset,
                            uint32_t l1_size,
                            int l2_size, uint64_t cluster_sectors,
-                           VmdkExtent **new_extent)
+                           VmdkExtent **new_extent,
+                           Error **errp)
 {
     VmdkExtent *extent;
     BDRVVmdkState *s = bs->opaque;
 
     if (cluster_sectors > 0x200000) {
         /* 0x200000 * 512Bytes = 1GB for one cluster is unrealistic */
-        error_report("invalid granularity, image may be corrupt");
-        return -EINVAL;
+        error_setg(errp, "Invalid granularity, image may be corrupt");
+        return -EFBIG;
     }
     if (l1_size > 512 * 1024 * 1024) {
         /* Although with big capacity and small l1_entry_sectors, we can get a
          * big l1_size, we don't want unbounded value to allocate the table.
          * Limit it to 512M, which is 16PB for default cluster and L2 table
          * size */
-        error_report("L1 size too big");
+        error_setg(errp, "L1 size too big");
         return -EFBIG;
     }
 
@@ -424,7 +422,7 @@ static int vmdk_add_extent(BlockDriverState *bs,
     extent->l1_size = l1_size;
     extent->l1_entry_sectors = l2_size * cluster_sectors;
     extent->l2_size = l2_size;
-    extent->cluster_sectors = cluster_sectors;
+    extent->cluster_sectors = flat ? sectors : cluster_sectors;
 
     if (s->num_extents > 1) {
         extent->end_sector = (*(extent - 1)).end_sector + extent->sectors;
@@ -438,7 +436,8 @@ static int vmdk_add_extent(BlockDriverState *bs,
     return 0;
 }
 
-static int vmdk_init_tables(BlockDriverState *bs, VmdkExtent *extent)
+static int vmdk_init_tables(BlockDriverState *bs, VmdkExtent *extent,
+                            Error **errp)
 {
     int ret;
     int l1_size, i;
@@ -447,10 +446,13 @@ static int vmdk_init_tables(BlockDriverState *bs, VmdkExtent *extent)
     l1_size = extent->l1_size * sizeof(uint32_t);
     extent->l1_table = g_malloc(l1_size);
     ret = bdrv_pread(extent->file,
-                    extent->l1_table_offset,
-                    extent->l1_table,
-                    l1_size);
+                     extent->l1_table_offset,
+                     extent->l1_table,
+                     l1_size);
     if (ret < 0) {
+        error_setg_errno(errp, -ret,
+                         "Could not read l1 table from extent '%s'",
+                         extent->file->filename);
         goto fail_l1;
     }
     for (i = 0; i < extent->l1_size; i++) {
@@ -460,10 +462,13 @@ static int vmdk_init_tables(BlockDriverState *bs, VmdkExtent *extent)
     if (extent->l1_backup_table_offset) {
         extent->l1_backup_table = g_malloc(l1_size);
         ret = bdrv_pread(extent->file,
-                        extent->l1_backup_table_offset,
-                        extent->l1_backup_table,
-                        l1_size);
+                         extent->l1_backup_table_offset,
+                         extent->l1_backup_table,
+                         l1_size);
         if (ret < 0) {
+            error_setg_errno(errp, -ret,
+                             "Could not read l1 backup table from extent '%s'",
+                             extent->file->filename);
             goto fail_l1b;
         }
         for (i = 0; i < extent->l1_size; i++) {
@@ -483,7 +488,7 @@ static int vmdk_init_tables(BlockDriverState *bs, VmdkExtent *extent)
 
 static int vmdk_open_vmfs_sparse(BlockDriverState *bs,
                                  BlockDriverState *file,
-                                 int flags)
+                                 int flags, Error **errp)
 {
     int ret;
     uint32_t magic;
@@ -492,6 +497,9 @@ static int vmdk_open_vmfs_sparse(BlockDriverState *bs,
 
     ret = bdrv_pread(file, sizeof(magic), &header, sizeof(header));
     if (ret < 0) {
+        error_setg_errno(errp, -ret,
+                         "Could not read header from file '%s'",
+                         file->filename);
         return ret;
     }
     ret = vmdk_add_extent(bs, file, false,
@@ -501,11 +509,12 @@ static int vmdk_open_vmfs_sparse(BlockDriverState *bs,
                           le32_to_cpu(header.l1dir_size),
                           4096,
                           le32_to_cpu(header.granularity),
-                          &extent);
+                          &extent,
+                          errp);
     if (ret < 0) {
         return ret;
     }
-    ret = vmdk_init_tables(bs, extent);
+    ret = vmdk_init_tables(bs, extent, errp);
     if (ret) {
         /* free extent allocated by vmdk_add_extent */
         vmdk_free_last_extent(bs);
@@ -514,11 +523,11 @@ static int vmdk_open_vmfs_sparse(BlockDriverState *bs,
 }
 
 static int vmdk_open_desc_file(BlockDriverState *bs, int flags,
-                               uint64_t desc_offset);
+                               uint64_t desc_offset, Error **errp);
 
 static int vmdk_open_vmdk4(BlockDriverState *bs,
                            BlockDriverState *file,
-                           int flags)
+                           int flags, Error **errp)
 {
     int ret;
     uint32_t magic;
@@ -529,12 +538,14 @@ static int vmdk_open_vmdk4(BlockDriverState *bs,
 
     ret = bdrv_pread(file, sizeof(magic), &header, sizeof(header));
     if (ret < 0) {
-        return ret;
+        error_setg_errno(errp, -ret,
+                         "Could not read header from file '%s'",
+                         file->filename);
     }
     if (header.capacity == 0) {
         uint64_t desc_offset = le64_to_cpu(header.desc_offset);
         if (desc_offset) {
-            return vmdk_open_desc_file(bs, flags, desc_offset << 9);
+            return vmdk_open_desc_file(bs, flags, desc_offset << 9, errp);
         }
     }
 
@@ -616,7 +627,8 @@ static int vmdk_open_vmdk4(BlockDriverState *bs,
                           l1_size,
                           le32_to_cpu(header.num_gtes_per_gt),
                           le64_to_cpu(header.granularity),
-                          &extent);
+                          &extent,
+                          errp);
     if (ret < 0) {
         return ret;
     }
@@ -625,7 +637,7 @@ static int vmdk_open_vmdk4(BlockDriverState *bs,
     extent->has_marker = le32_to_cpu(header.flags) & VMDK4_FLAG_MARKER;
     extent->version = le32_to_cpu(header.version);
     extent->has_zero_grain = le32_to_cpu(header.flags) & VMDK4_FLAG_ZERO_GRAIN;
-    ret = vmdk_init_tables(bs, extent);
+    ret = vmdk_init_tables(bs, extent, errp);
     if (ret) {
         /* free extent allocated by vmdk_add_extent */
         vmdk_free_last_extent(bs);
@@ -663,7 +675,7 @@ static int vmdk_parse_description(const char *desc, const char *opt_name,
 /* Open an extent file and append to bs array */
 static int vmdk_open_sparse(BlockDriverState *bs,
                             BlockDriverState *file,
-                            int flags)
+                            int flags, Error **errp)
 {
     uint32_t magic;
 
@@ -674,10 +686,10 @@ static int vmdk_open_sparse(BlockDriverState *bs,
     magic = be32_to_cpu(magic);
     switch (magic) {
         case VMDK3_MAGIC:
-            return vmdk_open_vmfs_sparse(bs, file, flags);
+            return vmdk_open_vmfs_sparse(bs, file, flags, errp);
             break;
         case VMDK4_MAGIC:
-            return vmdk_open_vmdk4(bs, file, flags);
+            return vmdk_open_vmdk4(bs, file, flags, errp);
             break;
         default:
             return -EMEDIUMTYPE;
@@ -686,7 +698,7 @@ static int vmdk_open_sparse(BlockDriverState *bs,
 }
 
 static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
-        const char *desc_file_path)
+                              const char *desc_file_path, Error **errp)
 {
     int ret;
     char access[11];
@@ -711,9 +723,13 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
             goto next_line;
         } else if (!strcmp(type, "FLAT")) {
             if (ret != 5 || flat_offset < 0) {
+                error_setg(errp, "Invalid extent lines: \n%s", p);
                 return -EINVAL;
             }
+        } else if (!strcmp(type, "VMFS")) {
+            flat_offset = 0;
         } else if (ret != 4) {
+            error_setg(errp, "Invalid extent lines: \n%s", p);
             return -EINVAL;
         }
 
@@ -726,7 +742,8 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
 
         path_combine(extent_path, sizeof(extent_path),
                 desc_file_path, fname);
-        ret = bdrv_file_open(&extent_file, extent_path, NULL, bs->open_flags);
+        ret = bdrv_file_open(&extent_file, extent_path, NULL, bs->open_flags,
+                             errp);
         if (ret) {
             return ret;
         }
@@ -737,35 +754,37 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
             VmdkExtent *extent;
 
             ret = vmdk_add_extent(bs, extent_file, true, sectors,
-                            0, 0, 0, 0, sectors, &extent);
+                            0, 0, 0, 0, 0, &extent, errp);
             if (ret < 0) {
                 return ret;
             }
             extent->flat_start_offset = flat_offset << 9;
         } else if (!strcmp(type, "SPARSE") || !strcmp(type, "VMFSSPARSE")) {
             /* SPARSE extent and VMFSSPARSE extent are both "COWD" sparse file*/
-            ret = vmdk_open_sparse(bs, extent_file, bs->open_flags);
+            ret = vmdk_open_sparse(bs, extent_file, bs->open_flags, errp);
             if (ret) {
                 bdrv_unref(extent_file);
                 return ret;
             }
         } else {
-            fprintf(stderr,
-                "VMDK: Not supported extent type \"%s\""".\n", type);
+            error_setg(errp, "Unsupported extent type '%s'", type);
             return -ENOTSUP;
         }
 next_line:
         /* move to next line */
-        while (*p && *p != '\n') {
+        while (*p) {
+            if (*p == '\n') {
+                p++;
+                break;
+            }
             p++;
         }
-        p++;
     }
     return 0;
 }
 
 static int vmdk_open_desc_file(BlockDriverState *bs, int flags,
-                               uint64_t desc_offset)
+                               uint64_t desc_offset, Error **errp)
 {
     int ret;
     char *buf = NULL;
@@ -794,27 +813,27 @@ static int vmdk_open_desc_file(BlockDriverState *bs, int flags,
         strcmp(ct, "vmfsSparse") &&
         strcmp(ct, "twoGbMaxExtentSparse") &&
         strcmp(ct, "twoGbMaxExtentFlat")) {
-        fprintf(stderr,
-                "VMDK: Not supported image type \"%s\""".\n", ct);
+        error_setg(errp, "Unsupported image type '%s'", ct);
         ret = -ENOTSUP;
         goto exit;
     }
     s->desc_offset = 0;
-    ret = vmdk_parse_extents(buf, bs, bs->file->filename);
+    ret = vmdk_parse_extents(buf, bs, bs->file->filename, errp);
 exit:
     g_free(buf);
     return ret;
 }
 
-static int vmdk_open(BlockDriverState *bs, QDict *options, int flags)
+static int vmdk_open(BlockDriverState *bs, QDict *options, int flags,
+                     Error **errp)
 {
     int ret;
     BDRVVmdkState *s = bs->opaque;
 
-    if (vmdk_open_sparse(bs, bs->file, flags) == 0) {
+    if (vmdk_open_sparse(bs, bs->file, flags, errp) == 0) {
         s->desc_offset = 0x200;
     } else {
-        ret = vmdk_open_desc_file(bs, flags, 0);
+        ret = vmdk_open_desc_file(bs, flags, 0, errp);
         if (ret) {
             goto fail;
         }
@@ -1281,8 +1300,7 @@ static int vmdk_write(BlockDriverState *bs, int64_t sector_num,
     VmdkMetaData m_data;
 
     if (sector_num > bs->total_sectors) {
-        fprintf(stderr,
-                "(VMDK) Wrong offset: sector_num=0x%" PRIx64
+        error_report("Wrong offset: sector_num=0x%" PRIx64
                 " total_sectors=0x%" PRIx64 "\n",
                 sector_num, bs->total_sectors);
         return -EIO;
@@ -1302,9 +1320,8 @@ static int vmdk_write(BlockDriverState *bs, int64_t sector_num,
         if (extent->compressed) {
             if (ret == VMDK_OK) {
                 /* Refuse write to allocated cluster for streamOptimized */
-                fprintf(stderr,
-                        "VMDK: can't write to allocated cluster"
-                        " for streamOptimized\n");
+                error_report("Could not write to allocated cluster"
+                              " for streamOptimized");
                 return -EIO;
             } else {
                 /* allocate */
@@ -1512,12 +1529,12 @@ static int vmdk_create_extent(const char *filename, int64_t filesize,
 }
 
 static int filename_decompose(const char *filename, char *path, char *prefix,
-        char *postfix, size_t buf_len)
+                              char *postfix, size_t buf_len, Error **errp)
 {
     const char *p, *q;
 
     if (filename == NULL || !strlen(filename)) {
-        fprintf(stderr, "Vmdk: no filename provided.\n");
+        error_setg(errp, "No filename provided");
         return VMDK_ERROR;
     }
     p = strrchr(filename, '/');
@@ -1551,7 +1568,8 @@ static int filename_decompose(const char *filename, char *path, char *prefix,
     return VMDK_OK;
 }
 
-static int vmdk_create(const char *filename, QEMUOptionParameter *options)
+static int vmdk_create(const char *filename, QEMUOptionParameter *options,
+                       Error **errp)
 {
     int fd, idx = 0;
     char desc[BUF_SIZE];
@@ -1590,7 +1608,7 @@ static int vmdk_create(const char *filename, QEMUOptionParameter *options)
         "ddb.geometry.sectors = \"63\"\n"
         "ddb.adapterType = \"%s\"\n";
 
-    if (filename_decompose(filename, path, prefix, postfix, PATH_MAX)) {
+    if (filename_decompose(filename, path, prefix, postfix, PATH_MAX, errp)) {
         return -EINVAL;
     }
     /* Read out options */
@@ -1616,7 +1634,7 @@ static int vmdk_create(const char *filename, QEMUOptionParameter *options)
                strcmp(adapter_type, "buslogic") &&
                strcmp(adapter_type, "lsilogic") &&
                strcmp(adapter_type, "legacyESX")) {
-        fprintf(stderr, "VMDK: Unknown adapter type: '%s'.\n", adapter_type);
+        error_setg(errp, "Unknown adapter type: '%s'", adapter_type);
         return -EINVAL;
     }
     if (strcmp(adapter_type, "ide") != 0) {
@@ -1632,7 +1650,7 @@ static int vmdk_create(const char *filename, QEMUOptionParameter *options)
                strcmp(fmt, "twoGbMaxExtentSparse") &&
                strcmp(fmt, "twoGbMaxExtentFlat") &&
                strcmp(fmt, "streamOptimized")) {
-        fprintf(stderr, "VMDK: Unknown subformat: %s\n", fmt);
+        error_setg(errp, "Unknown subformat: '%s'", fmt);
         return -EINVAL;
     }
     split = !(strcmp(fmt, "twoGbMaxExtentFlat") &&
@@ -1646,12 +1664,16 @@ static int vmdk_create(const char *filename, QEMUOptionParameter *options)
         desc_extent_line = "RW %lld SPARSE \"%s\"\n";
     }
     if (flat && backing_file) {
-        /* not supporting backing file for flat image */
+        error_setg(errp, "Flat image can't have backing file");
+        return -ENOTSUP;
+    }
+    if (flat && zeroed_grain) {
+        error_setg(errp, "Flat image can't enable zeroed grain");
         return -ENOTSUP;
     }
     if (backing_file) {
         BlockDriverState *bs = bdrv_new("");
-        ret = bdrv_open(bs, backing_file, NULL, 0, NULL);
+        ret = bdrv_open(bs, backing_file, NULL, 0, NULL, errp);
         if (ret != 0) {
             bdrv_unref(bs);
             return ret;
diff --git a/block/vpc.c b/block/vpc.c
index fe4f311d50..b5dca3961e 100644
--- a/block/vpc.c
+++ b/block/vpc.c
@@ -46,7 +46,7 @@ enum vhd_type {
 #define VHD_TIMESTAMP_BASE 946684800
 
 // always big-endian
-struct vhd_footer {
+typedef struct vhd_footer {
     char        creator[8]; // "conectix"
     uint32_t    features;
     uint32_t    version;
@@ -79,9 +79,9 @@ struct vhd_footer {
     uint8_t     uuid[16];
 
     uint8_t     in_saved_state;
-};
+} QEMU_PACKED VHDFooter;
 
-struct vhd_dyndisk_header {
+typedef struct vhd_dyndisk_header {
     char        magic[8]; // "cxsparse"
 
     // Offset of next header structure, 0xFFFFFFFF if none
@@ -111,7 +111,7 @@ struct vhd_dyndisk_header {
         uint32_t    reserved;
         uint64_t    data_offset;
     } parent_locator[8];
-};
+} QEMU_PACKED VHDDynDiskHeader;
 
 typedef struct BDRVVPCState {
     CoMutex lock;
@@ -155,12 +155,13 @@ static int vpc_probe(const uint8_t *buf, int buf_size, const char *filename)
     return 0;
 }
 
-static int vpc_open(BlockDriverState *bs, QDict *options, int flags)
+static int vpc_open(BlockDriverState *bs, QDict *options, int flags,
+                    Error **errp)
 {
     BDRVVPCState *s = bs->opaque;
     int i;
-    struct vhd_footer* footer;
-    struct vhd_dyndisk_header* dyndisk_header;
+    VHDFooter *footer;
+    VHDDynDiskHeader *dyndisk_header;
     uint8_t buf[HEADER_SIZE];
     uint32_t checksum;
     int disk_type = VHD_DYNAMIC;
@@ -171,7 +172,7 @@ static int vpc_open(BlockDriverState *bs, QDict *options, int flags)
         goto fail;
     }
 
-    footer = (struct vhd_footer*) s->footer_buf;
+    footer = (VHDFooter *) s->footer_buf;
     if (strncmp(footer->creator, "conectix", 8)) {
         int64_t offset = bdrv_getlength(bs->file);
         if (offset < 0) {
@@ -223,7 +224,7 @@ static int vpc_open(BlockDriverState *bs, QDict *options, int flags)
             goto fail;
         }
 
-        dyndisk_header = (struct vhd_dyndisk_header *) buf;
+        dyndisk_header = (VHDDynDiskHeader *) buf;
 
         if (strncmp(dyndisk_header->magic, "cxsparse", 8)) {
             ret = -EINVAL;
@@ -445,7 +446,7 @@ static int vpc_read(BlockDriverState *bs, int64_t sector_num,
     int ret;
     int64_t offset;
     int64_t sectors, sectors_per_block;
-    struct vhd_footer *footer = (struct vhd_footer *) s->footer_buf;
+    VHDFooter *footer = (VHDFooter *) s->footer_buf;
 
     if (cpu_to_be32(footer->type) == VHD_FIXED) {
         return bdrv_read(bs->file, sector_num, buf, nb_sectors);
@@ -494,7 +495,7 @@ static int vpc_write(BlockDriverState *bs, int64_t sector_num,
     int64_t offset;
     int64_t sectors, sectors_per_block;
     int ret;
-    struct vhd_footer *footer =  (struct vhd_footer *) s->footer_buf;
+    VHDFooter *footer =  (VHDFooter *) s->footer_buf;
 
     if (cpu_to_be32(footer->type) == VHD_FIXED) {
         return bdrv_write(bs->file, sector_num, buf, nb_sectors);
@@ -596,8 +597,8 @@ static int calculate_geometry(int64_t total_sectors, uint16_t* cyls,
 
 static int create_dynamic_disk(int fd, uint8_t *buf, int64_t total_sectors)
 {
-    struct vhd_dyndisk_header* dyndisk_header =
-        (struct vhd_dyndisk_header*) buf;
+    VHDDynDiskHeader *dyndisk_header =
+        (VHDDynDiskHeader *) buf;
     size_t block_size, num_bat_entries;
     int i;
     int ret = -EIO;
@@ -683,10 +684,11 @@ static int create_fixed_disk(int fd, uint8_t *buf, int64_t total_size)
     return ret;
 }
 
-static int vpc_create(const char *filename, QEMUOptionParameter *options)
+static int vpc_create(const char *filename, QEMUOptionParameter *options,
+                      Error **errp)
 {
     uint8_t buf[1024];
-    struct vhd_footer *footer = (struct vhd_footer *) buf;
+    VHDFooter *footer = (VHDFooter *) buf;
     QEMUOptionParameter *disk_type_param;
     int fd, i;
     uint16_t cyls = 0;
@@ -789,7 +791,7 @@ static int vpc_create(const char *filename, QEMUOptionParameter *options)
 static int vpc_has_zero_init(BlockDriverState *bs)
 {
     BDRVVPCState *s = bs->opaque;
-    struct vhd_footer *footer =  (struct vhd_footer *) s->footer_buf;
+    VHDFooter *footer =  (VHDFooter *) s->footer_buf;
 
     if (cpu_to_be32(footer->type) == VHD_FIXED) {
         return bdrv_has_zero_init(bs->file);
diff --git a/block/vvfat.c b/block/vvfat.c
index 0129195e29..3ddaa0bcce 100644
--- a/block/vvfat.c
+++ b/block/vvfat.c
@@ -1065,7 +1065,8 @@ static void vvfat_parse_filename(const char *filename, QDict *options,
     qdict_put(options, "rw", qbool_from_int(rw));
 }
 
-static int vvfat_open(BlockDriverState *bs, QDict *options, int flags)
+static int vvfat_open(BlockDriverState *bs, QDict *options, int flags,
+                      Error **errp)
 {
     BDRVVVFATState *s = bs->opaque;
     int cyls, heads, secs;
@@ -2909,6 +2910,7 @@ static int enable_write_target(BDRVVVFATState *s)
 {
     BlockDriver *bdrv_qcow;
     QEMUOptionParameter *options;
+    Error *local_err = NULL;
     int ret;
     int size = sector2cluster(s, s->sector_count);
     s->used_clusters = calloc(size, 1);
@@ -2926,16 +2928,21 @@ static int enable_write_target(BDRVVVFATState *s)
     set_option_parameter_int(options, BLOCK_OPT_SIZE, s->sector_count * 512);
     set_option_parameter(options, BLOCK_OPT_BACKING_FILE, "fat:");
 
-    ret = bdrv_create(bdrv_qcow, s->qcow_filename, options);
+    ret = bdrv_create(bdrv_qcow, s->qcow_filename, options, &local_err);
     if (ret < 0) {
+        qerror_report_err(local_err);
+        error_free(local_err);
         goto err;
     }
 
     s->qcow = bdrv_new("");
 
     ret = bdrv_open(s->qcow, s->qcow_filename, NULL,
-            BDRV_O_RDWR | BDRV_O_CACHE_WB | BDRV_O_NO_FLUSH, bdrv_qcow);
+            BDRV_O_RDWR | BDRV_O_CACHE_WB | BDRV_O_NO_FLUSH, bdrv_qcow,
+            &local_err);
     if (ret < 0) {
+        qerror_report_err(local_err);
+        error_free(local_err);
         bdrv_unref(s->qcow);
         goto err;
     }
diff --git a/blockdev.c b/blockdev.c
index 07dac05a2c..b260477f1b 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -38,6 +38,8 @@
 #include "qemu/option.h"
 #include "qemu/config-file.h"
 #include "qapi/qmp/types.h"
+#include "qapi-visit.h"
+#include "qapi/qmp-output-visitor.h"
 #include "sysemu/sysemu.h"
 #include "block/block_int.h"
 #include "qmp-commands.h"
@@ -89,6 +91,10 @@ void blockdev_mark_auto_del(BlockDriverState *bs)
 {
     DriveInfo *dinfo = drive_get_by_blockdev(bs);
 
+    if (dinfo && !dinfo->enable_auto_del) {
+        return;
+    }
+
     if (bs->job) {
         block_job_cancel(bs->job);
     }
@@ -211,7 +217,10 @@ static void bdrv_format_print(void *opaque, const char *name)
 
 static void drive_uninit(DriveInfo *dinfo)
 {
-    qemu_opts_del(dinfo->opts);
+    if (dinfo->opts) {
+        qemu_opts_del(dinfo->opts);
+    }
+
     bdrv_unref(dinfo->bdrv);
     g_free(dinfo->id);
     QTAILQ_REMOVE(&drives, dinfo, next);
@@ -263,7 +272,7 @@ static void bdrv_put_ref_bh_schedule(BlockDriverState *bs)
     qemu_bh_schedule(s->bh);
 }
 
-static int parse_block_error_action(const char *buf, bool is_read)
+static int parse_block_error_action(const char *buf, bool is_read, Error **errp)
 {
     if (!strcmp(buf, "ignore")) {
         return BLOCKDEV_ON_ERROR_IGNORE;
@@ -274,8 +283,8 @@ static int parse_block_error_action(const char *buf, bool is_read)
     } else if (!strcmp(buf, "report")) {
         return BLOCKDEV_ON_ERROR_REPORT;
     } else {
-        error_report("'%s' invalid %s error action",
-                     buf, is_read ? "read" : "write");
+        error_setg(errp, "'%s' invalid %s error action",
+                   buf, is_read ? "read" : "write");
         return -1;
     }
 }
@@ -296,23 +305,19 @@ static bool check_throttle_config(ThrottleConfig *cfg, Error **errp)
     return true;
 }
 
-static DriveInfo *blockdev_init(QemuOpts *all_opts,
-                                BlockInterfaceType block_default_type)
+typedef enum { MEDIA_DISK, MEDIA_CDROM } DriveMediaType;
+
+/* Takes the ownership of bs_opts */
+static DriveInfo *blockdev_init(QDict *bs_opts,
+                                BlockInterfaceType type,
+                                Error **errp)
 {
     const char *buf;
     const char *file = NULL;
     const char *serial;
-    const char *mediastr = "";
-    BlockInterfaceType type;
-    enum { MEDIA_DISK, MEDIA_CDROM } media;
-    int bus_id, unit_id;
-    int cyls, heads, secs, translation;
-    int max_devs;
-    int index;
     int ro = 0;
     int bdrv_flags = 0;
     int on_read_error, on_write_error;
-    const char *devaddr;
     DriveInfo *dinfo;
     ThrottleConfig cfg;
     int snapshot = 0;
@@ -320,30 +325,22 @@ static DriveInfo *blockdev_init(QemuOpts *all_opts,
     int ret;
     Error *error = NULL;
     QemuOpts *opts;
-    QDict *bs_opts;
     const char *id;
     bool has_driver_specific_opts;
     BlockDriver *drv = NULL;
 
-    translation = BIOS_ATA_TRANSLATION_AUTO;
-    media = MEDIA_DISK;
-
-    /* Check common options by copying from all_opts to opts, all other options
-     * are stored in bs_opts. */
-    id = qemu_opts_id(all_opts);
+    /* Check common options by copying from bs_opts to opts, all other options
+     * stay in bs_opts for processing by bdrv_open(). */
+    id = qdict_get_try_str(bs_opts, "id");
     opts = qemu_opts_create(&qemu_common_drive_opts, id, 1, &error);
     if (error_is_set(&error)) {
-        qerror_report_err(error);
-        error_free(error);
+        error_propagate(errp, error);
         return NULL;
     }
 
-    bs_opts = qdict_new();
-    qemu_opts_to_qdict(all_opts, bs_opts);
     qemu_opts_absorb_qdict(opts, bs_opts, &error);
     if (error_is_set(&error)) {
-        qerror_report_err(error);
-        error_free(error);
+        error_propagate(errp, error);
         return NULL;
     }
 
@@ -354,14 +351,6 @@ static DriveInfo *blockdev_init(QemuOpts *all_opts,
     has_driver_specific_opts = !!qdict_size(bs_opts);
 
     /* extract parameters */
-    bus_id  = qemu_opt_get_number(opts, "bus", 0);
-    unit_id = qemu_opt_get_number(opts, "unit", -1);
-    index   = qemu_opt_get_number(opts, "index", -1);
-
-    cyls  = qemu_opt_get_number(opts, "cyls", 0);
-    heads = qemu_opt_get_number(opts, "heads", 0);
-    secs  = qemu_opt_get_number(opts, "secs", 0);
-
     snapshot = qemu_opt_get_bool(opts, "snapshot", 0);
     ro = qemu_opt_get_bool(opts, "read-only", 0);
     copy_on_read = qemu_opt_get_bool(opts, "copy-on-read", false);
@@ -369,70 +358,9 @@ static DriveInfo *blockdev_init(QemuOpts *all_opts,
     file = qemu_opt_get(opts, "file");
     serial = qemu_opt_get(opts, "serial");
 
-    if ((buf = qemu_opt_get(opts, "if")) != NULL) {
-        for (type = 0; type < IF_COUNT && strcmp(buf, if_name[type]); type++)
-            ;
-        if (type == IF_COUNT) {
-            error_report("unsupported bus type '%s'", buf);
-            return NULL;
-	}
-    } else {
-        type = block_default_type;
-    }
-
-    max_devs = if_max_devs[type];
-
-    if (cyls || heads || secs) {
-        if (cyls < 1) {
-            error_report("invalid physical cyls number");
-	    return NULL;
-	}
-        if (heads < 1) {
-            error_report("invalid physical heads number");
-	    return NULL;
-	}
-        if (secs < 1) {
-            error_report("invalid physical secs number");
-	    return NULL;
-	}
-    }
-
-    if ((buf = qemu_opt_get(opts, "trans")) != NULL) {
-        if (!cyls) {
-            error_report("'%s' trans must be used with cyls, heads and secs",
-                         buf);
-            return NULL;
-        }
-        if (!strcmp(buf, "none"))
-            translation = BIOS_ATA_TRANSLATION_NONE;
-        else if (!strcmp(buf, "lba"))
-            translation = BIOS_ATA_TRANSLATION_LBA;
-        else if (!strcmp(buf, "auto"))
-            translation = BIOS_ATA_TRANSLATION_AUTO;
-	else {
-            error_report("'%s' invalid translation type", buf);
-	    return NULL;
-	}
-    }
-
-    if ((buf = qemu_opt_get(opts, "media")) != NULL) {
-        if (!strcmp(buf, "disk")) {
-	    media = MEDIA_DISK;
-	} else if (!strcmp(buf, "cdrom")) {
-            if (cyls || secs || heads) {
-                error_report("CHS can't be set with media=%s", buf);
-	        return NULL;
-            }
-	    media = MEDIA_CDROM;
-	} else {
-	    error_report("'%s' invalid media", buf);
-	    return NULL;
-	}
-    }
-
     if ((buf = qemu_opt_get(opts, "discard")) != NULL) {
         if (bdrv_parse_discard_flags(buf, &bdrv_flags) != 0) {
-            error_report("invalid discard option");
+            error_setg(errp, "invalid discard option");
             return NULL;
         }
     }
@@ -443,7 +371,7 @@ static DriveInfo *blockdev_init(QemuOpts *all_opts,
     if (qemu_opt_get_bool(opts, "cache.direct", false)) {
         bdrv_flags |= BDRV_O_NOCACHE;
     }
-    if (qemu_opt_get_bool(opts, "cache.no-flush", true)) {
+    if (qemu_opt_get_bool(opts, "cache.no-flush", false)) {
         bdrv_flags |= BDRV_O_NO_FLUSH;
     }
 
@@ -454,7 +382,7 @@ static DriveInfo *blockdev_init(QemuOpts *all_opts,
         } else if (!strcmp(buf, "threads")) {
             /* this is the default */
         } else {
-           error_report("invalid aio option");
+           error_setg(errp, "invalid aio option");
            return NULL;
         }
     }
@@ -468,13 +396,9 @@ static DriveInfo *blockdev_init(QemuOpts *all_opts,
             return NULL;
         }
 
-        drv = bdrv_find_whitelisted_format(buf, ro);
+        drv = bdrv_find_format(buf);
         if (!drv) {
-            if (!ro && bdrv_find_whitelisted_format(buf, !ro)) {
-                error_report("'%s' can be only used as read-only device.", buf);
-            } else {
-                error_report("'%s' invalid format", buf);
-            }
+            error_setg(errp, "'%s' invalid format", buf);
             return NULL;
         }
     }
@@ -510,26 +434,20 @@ static DriveInfo *blockdev_init(QemuOpts *all_opts,
     cfg.op_size = qemu_opt_get_number(opts, "throttling.iops-size", 0);
 
     if (!check_throttle_config(&cfg, &error)) {
-        error_report("%s", error_get_pretty(error));
-        error_free(error);
+        error_propagate(errp, error);
         return NULL;
     }
 
-    if (qemu_opt_get(opts, "boot") != NULL) {
-        fprintf(stderr, "qemu-kvm: boot=on|off is deprecated and will be "
-                "ignored. Future versions will reject this parameter. Please "
-                "update your scripts.\n");
-    }
-
     on_write_error = BLOCKDEV_ON_ERROR_ENOSPC;
     if ((buf = qemu_opt_get(opts, "werror")) != NULL) {
         if (type != IF_IDE && type != IF_SCSI && type != IF_VIRTIO && type != IF_NONE) {
-            error_report("werror is not supported by this bus type");
+            error_setg(errp, "werror is not supported by this bus type");
             return NULL;
         }
 
-        on_write_error = parse_block_error_action(buf, 0);
-        if (on_write_error < 0) {
+        on_write_error = parse_block_error_action(buf, 0, &error);
+        if (error_is_set(&error)) {
+            error_propagate(errp, error);
             return NULL;
         }
     }
@@ -541,92 +459,20 @@ static DriveInfo *blockdev_init(QemuOpts *all_opts,
             return NULL;
         }
 
-        on_read_error = parse_block_error_action(buf, 1);
-        if (on_read_error < 0) {
-            return NULL;
-        }
-    }
-
-    if ((devaddr = qemu_opt_get(opts, "addr")) != NULL) {
-        if (type != IF_VIRTIO) {
-            error_report("addr is not supported by this bus type");
-            return NULL;
-        }
-    }
-
-    /* compute bus and unit according index */
-
-    if (index != -1) {
-        if (bus_id != 0 || unit_id != -1) {
-            error_report("index cannot be used with bus and unit");
+        on_read_error = parse_block_error_action(buf, 1, &error);
+        if (error_is_set(&error)) {
+            error_propagate(errp, error);
             return NULL;
         }
-        bus_id = drive_index_to_bus_id(type, index);
-        unit_id = drive_index_to_unit_id(type, index);
-    }
-
-    /* if user doesn't specify a unit_id,
-     * try to find the first free
-     */
-
-    if (unit_id == -1) {
-       unit_id = 0;
-       while (drive_get(type, bus_id, unit_id) != NULL) {
-           unit_id++;
-           if (max_devs && unit_id >= max_devs) {
-               unit_id -= max_devs;
-               bus_id++;
-           }
-       }
-    }
-
-    /* check unit id */
-
-    if (max_devs && unit_id >= max_devs) {
-        error_report("unit %d too big (max is %d)",
-                     unit_id, max_devs - 1);
-        return NULL;
-    }
-
-    /*
-     * catch multiple definitions
-     */
-
-    if (drive_get(type, bus_id, unit_id) != NULL) {
-        error_report("drive with bus=%d, unit=%d (index=%d) exists",
-                     bus_id, unit_id, index);
-        return NULL;
     }
 
     /* init */
-
     dinfo = g_malloc0(sizeof(*dinfo));
-    if ((buf = qemu_opts_id(opts)) != NULL) {
-        dinfo->id = g_strdup(buf);
-    } else {
-        /* no id supplied -> create one */
-        dinfo->id = g_malloc0(32);
-        if (type == IF_IDE || type == IF_SCSI)
-            mediastr = (media == MEDIA_CDROM) ? "-cd" : "-hd";
-        if (max_devs)
-            snprintf(dinfo->id, 32, "%s%i%s%i",
-                     if_name[type], bus_id, mediastr, unit_id);
-        else
-            snprintf(dinfo->id, 32, "%s%s%i",
-                     if_name[type], mediastr, unit_id);
-    }
+    dinfo->id = g_strdup(qemu_opts_id(opts));
     dinfo->bdrv = bdrv_new(dinfo->id);
     dinfo->bdrv->open_flags = snapshot ? BDRV_O_SNAPSHOT : 0;
     dinfo->bdrv->read_only = ro;
-    dinfo->devaddr = devaddr;
     dinfo->type = type;
-    dinfo->bus = bus_id;
-    dinfo->unit = unit_id;
-    dinfo->cyls = cyls;
-    dinfo->heads = heads;
-    dinfo->secs = secs;
-    dinfo->trans = translation;
-    dinfo->opts = all_opts;
     dinfo->refcount = 1;
     if (serial != NULL) {
         dinfo->serial = g_strdup(serial);
@@ -641,36 +487,6 @@ static DriveInfo *blockdev_init(QemuOpts *all_opts,
         bdrv_set_io_limits(dinfo->bdrv, &cfg);
     }
 
-    switch(type) {
-    case IF_IDE:
-    case IF_SCSI:
-    case IF_XEN:
-    case IF_NONE:
-        dinfo->media_cd = media == MEDIA_CDROM;
-        break;
-    case IF_SD:
-    case IF_FLOPPY:
-    case IF_PFLASH:
-    case IF_MTD:
-        break;
-    case IF_VIRTIO:
-    {
-        /* add virtio block device */
-        QemuOpts *devopts;
-        devopts = qemu_opts_create_nofail(qemu_find_opts("device"));
-        if (arch_type == QEMU_ARCH_S390X) {
-            qemu_opt_set(devopts, "driver", "virtio-blk-s390");
-        } else {
-            qemu_opt_set(devopts, "driver", "virtio-blk-pci");
-        }
-        qemu_opt_set(devopts, "drive", dinfo->id);
-        if (devaddr)
-            qemu_opt_set(devopts, "addr", devaddr);
-        break;
-    }
-    default:
-        abort();
-    }
     if (!file || !*file) {
         if (has_driver_specific_opts) {
             file = NULL;
@@ -692,35 +508,15 @@ static DriveInfo *blockdev_init(QemuOpts *all_opts,
         bdrv_flags |= BDRV_O_INCOMING;
     }
 
-    if (media == MEDIA_CDROM) {
-        /* CDROM is fine for any interface, don't check.  */
-        ro = 1;
-    } else if (ro == 1) {
-        if (type != IF_SCSI && type != IF_VIRTIO && type != IF_FLOPPY &&
-            type != IF_NONE && type != IF_PFLASH) {
-            error_report("read-only not supported by this bus type");
-            goto err;
-        }
-    }
-
     bdrv_flags |= ro ? 0 : BDRV_O_RDWR;
 
-    if (ro && copy_on_read) {
-        error_report("warning: disabling copy_on_read on read-only drive");
-    }
-
     QINCREF(bs_opts);
-    ret = bdrv_open(dinfo->bdrv, file, bs_opts, bdrv_flags, drv);
+    ret = bdrv_open(dinfo->bdrv, file, bs_opts, bdrv_flags, drv, &error);
 
     if (ret < 0) {
-        if (ret == -EMEDIUMTYPE) {
-            error_report("could not open disk image %s: not in %s format",
-                         file ?: dinfo->id, drv ? drv->format_name :
-                         qdict_get_str(bs_opts, "driver"));
-        } else {
-            error_report("could not open disk image %s: %s",
-                         file ?: dinfo->id, strerror(-ret));
-        }
+        error_setg(errp, "could not open disk image %s: %s",
+                   file ?: dinfo->id, error_get_pretty(error));
+        error_free(error);
         goto err;
     }
 
@@ -753,9 +549,85 @@ static void qemu_opt_rename(QemuOpts *opts, const char *from, const char *to)
     }
 }
 
+QemuOptsList qemu_legacy_drive_opts = {
+    .name = "drive",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_legacy_drive_opts.head),
+    .desc = {
+        {
+            .name = "bus",
+            .type = QEMU_OPT_NUMBER,
+            .help = "bus number",
+        },{
+            .name = "unit",
+            .type = QEMU_OPT_NUMBER,
+            .help = "unit number (i.e. lun for scsi)",
+        },{
+            .name = "index",
+            .type = QEMU_OPT_NUMBER,
+            .help = "index number",
+        },{
+            .name = "media",
+            .type = QEMU_OPT_STRING,
+            .help = "media type (disk, cdrom)",
+        },{
+            .name = "if",
+            .type = QEMU_OPT_STRING,
+            .help = "interface (ide, scsi, sd, mtd, floppy, pflash, virtio)",
+        },{
+            .name = "cyls",
+            .type = QEMU_OPT_NUMBER,
+            .help = "number of cylinders (ide disk geometry)",
+        },{
+            .name = "heads",
+            .type = QEMU_OPT_NUMBER,
+            .help = "number of heads (ide disk geometry)",
+        },{
+            .name = "secs",
+            .type = QEMU_OPT_NUMBER,
+            .help = "number of sectors (ide disk geometry)",
+        },{
+            .name = "trans",
+            .type = QEMU_OPT_STRING,
+            .help = "chs translation (auto, lba, none)",
+        },{
+            .name = "boot",
+            .type = QEMU_OPT_BOOL,
+            .help = "(deprecated, ignored)",
+        },{
+            .name = "addr",
+            .type = QEMU_OPT_STRING,
+            .help = "pci address (virtio only)",
+        },
+
+        /* Options that are passed on, but have special semantics with -drive */
+        {
+            .name = "read-only",
+            .type = QEMU_OPT_BOOL,
+            .help = "open drive file as read-only",
+        },{
+            .name = "copy-on-read",
+            .type = QEMU_OPT_BOOL,
+            .help = "copy read data from backing file into image file",
+        },
+
+        { /* end of list */ }
+    },
+};
+
 DriveInfo *drive_init(QemuOpts *all_opts, BlockInterfaceType block_default_type)
 {
     const char *value;
+    DriveInfo *dinfo = NULL;
+    QDict *bs_opts;
+    QemuOpts *legacy_opts;
+    DriveMediaType media = MEDIA_DISK;
+    BlockInterfaceType type;
+    int cyls, heads, secs, translation;
+    int max_devs, bus_id, unit_id, index;
+    const char *devaddr;
+    bool read_only = false;
+    bool copy_on_read;
+    Error *local_err = NULL;
 
     /* Change legacy command line options into QMP ones */
     qemu_opt_rename(all_opts, "iops", "throttling.iops-total");
@@ -804,7 +676,232 @@ DriveInfo *drive_init(QemuOpts *all_opts, BlockInterfaceType block_default_type)
         qemu_opt_unset(all_opts, "cache");
     }
 
-    return blockdev_init(all_opts, block_default_type);
+    /* Get a QDict for processing the options */
+    bs_opts = qdict_new();
+    qemu_opts_to_qdict(all_opts, bs_opts);
+
+    legacy_opts = qemu_opts_create_nofail(&qemu_legacy_drive_opts);
+    qemu_opts_absorb_qdict(legacy_opts, bs_opts, &local_err);
+    if (error_is_set(&local_err)) {
+        qerror_report_err(local_err);
+        error_free(local_err);
+        goto fail;
+    }
+
+    /* Deprecated option boot=[on|off] */
+    if (qemu_opt_get(legacy_opts, "boot") != NULL) {
+        fprintf(stderr, "qemu-kvm: boot=on|off is deprecated and will be "
+                "ignored. Future versions will reject this parameter. Please "
+                "update your scripts.\n");
+    }
+
+    /* Media type */
+    value = qemu_opt_get(legacy_opts, "media");
+    if (value) {
+        if (!strcmp(value, "disk")) {
+            media = MEDIA_DISK;
+        } else if (!strcmp(value, "cdrom")) {
+            media = MEDIA_CDROM;
+            read_only = true;
+        } else {
+            error_report("'%s' invalid media", value);
+            goto fail;
+        }
+    }
+
+    /* copy-on-read is disabled with a warning for read-only devices */
+    read_only |= qemu_opt_get_bool(legacy_opts, "read-only", false);
+    copy_on_read = qemu_opt_get_bool(legacy_opts, "copy-on-read", false);
+
+    if (read_only && copy_on_read) {
+        error_report("warning: disabling copy-on-read on read-only drive");
+        copy_on_read = false;
+    }
+
+    qdict_put(bs_opts, "read-only",
+              qstring_from_str(read_only ? "on" : "off"));
+    qdict_put(bs_opts, "copy-on-read",
+              qstring_from_str(copy_on_read ? "on" :"off"));
+
+    /* Controller type */
+    value = qemu_opt_get(legacy_opts, "if");
+    if (value) {
+        for (type = 0;
+             type < IF_COUNT && strcmp(value, if_name[type]);
+             type++) {
+        }
+        if (type == IF_COUNT) {
+            error_report("unsupported bus type '%s'", value);
+            goto fail;
+        }
+    } else {
+        type = block_default_type;
+    }
+
+    /* Geometry */
+    cyls  = qemu_opt_get_number(legacy_opts, "cyls", 0);
+    heads = qemu_opt_get_number(legacy_opts, "heads", 0);
+    secs  = qemu_opt_get_number(legacy_opts, "secs", 0);
+
+    if (cyls || heads || secs) {
+        if (cyls < 1) {
+            error_report("invalid physical cyls number");
+            goto fail;
+        }
+        if (heads < 1) {
+            error_report("invalid physical heads number");
+            goto fail;
+        }
+        if (secs < 1) {
+            error_report("invalid physical secs number");
+            goto fail;
+        }
+    }
+
+    translation = BIOS_ATA_TRANSLATION_AUTO;
+    value = qemu_opt_get(legacy_opts, "trans");
+    if (value != NULL) {
+        if (!cyls) {
+            error_report("'%s' trans must be used with cyls, heads and secs",
+                         value);
+            goto fail;
+        }
+        if (!strcmp(value, "none")) {
+            translation = BIOS_ATA_TRANSLATION_NONE;
+        } else if (!strcmp(value, "lba")) {
+            translation = BIOS_ATA_TRANSLATION_LBA;
+        } else if (!strcmp(value, "auto")) {
+            translation = BIOS_ATA_TRANSLATION_AUTO;
+        } else {
+            error_report("'%s' invalid translation type", value);
+            goto fail;
+        }
+    }
+
+    if (media == MEDIA_CDROM) {
+        if (cyls || secs || heads) {
+            error_report("CHS can't be set with media=cdrom");
+            goto fail;
+        }
+    }
+
+    /* Device address specified by bus/unit or index.
+     * If none was specified, try to find the first free one. */
+    bus_id  = qemu_opt_get_number(legacy_opts, "bus", 0);
+    unit_id = qemu_opt_get_number(legacy_opts, "unit", -1);
+    index   = qemu_opt_get_number(legacy_opts, "index", -1);
+
+    max_devs = if_max_devs[type];
+
+    if (index != -1) {
+        if (bus_id != 0 || unit_id != -1) {
+            error_report("index cannot be used with bus and unit");
+            goto fail;
+        }
+        bus_id = drive_index_to_bus_id(type, index);
+        unit_id = drive_index_to_unit_id(type, index);
+    }
+
+    if (unit_id == -1) {
+       unit_id = 0;
+       while (drive_get(type, bus_id, unit_id) != NULL) {
+           unit_id++;
+           if (max_devs && unit_id >= max_devs) {
+               unit_id -= max_devs;
+               bus_id++;
+           }
+       }
+    }
+
+    if (max_devs && unit_id >= max_devs) {
+        error_report("unit %d too big (max is %d)", unit_id, max_devs - 1);
+        goto fail;
+    }
+
+    if (drive_get(type, bus_id, unit_id) != NULL) {
+        error_report("drive with bus=%d, unit=%d (index=%d) exists",
+                     bus_id, unit_id, index);
+        goto fail;
+    }
+
+    /* no id supplied -> create one */
+    if (qemu_opts_id(all_opts) == NULL) {
+        char *new_id;
+        const char *mediastr = "";
+        if (type == IF_IDE || type == IF_SCSI) {
+            mediastr = (media == MEDIA_CDROM) ? "-cd" : "-hd";
+        }
+        if (max_devs) {
+            new_id = g_strdup_printf("%s%i%s%i", if_name[type], bus_id,
+                                     mediastr, unit_id);
+        } else {
+            new_id = g_strdup_printf("%s%s%i", if_name[type],
+                                     mediastr, unit_id);
+        }
+        qdict_put(bs_opts, "id", qstring_from_str(new_id));
+        g_free(new_id);
+    }
+
+    /* Add virtio block device */
+    devaddr = qemu_opt_get(legacy_opts, "addr");
+    if (devaddr && type != IF_VIRTIO) {
+        error_report("addr is not supported by this bus type");
+        goto fail;
+    }
+
+    if (type == IF_VIRTIO) {
+        QemuOpts *devopts;
+        devopts = qemu_opts_create_nofail(qemu_find_opts("device"));
+        if (arch_type == QEMU_ARCH_S390X) {
+            qemu_opt_set(devopts, "driver", "virtio-blk-s390");
+        } else {
+            qemu_opt_set(devopts, "driver", "virtio-blk-pci");
+        }
+        qemu_opt_set(devopts, "drive", qdict_get_str(bs_opts, "id"));
+        if (devaddr) {
+            qemu_opt_set(devopts, "addr", devaddr);
+        }
+    }
+
+    /* Actual block device init: Functionality shared with blockdev-add */
+    dinfo = blockdev_init(bs_opts, type, &local_err);
+    if (dinfo == NULL) {
+        if (error_is_set(&local_err)) {
+            qerror_report_err(local_err);
+            error_free(local_err);
+        }
+        goto fail;
+    } else {
+        assert(!error_is_set(&local_err));
+    }
+
+    /* Set legacy DriveInfo fields */
+    dinfo->enable_auto_del = true;
+    dinfo->opts = all_opts;
+
+    dinfo->cyls = cyls;
+    dinfo->heads = heads;
+    dinfo->secs = secs;
+    dinfo->trans = translation;
+
+    dinfo->bus = bus_id;
+    dinfo->unit = unit_id;
+    dinfo->devaddr = devaddr;
+
+    switch(type) {
+    case IF_IDE:
+    case IF_SCSI:
+    case IF_XEN:
+    case IF_NONE:
+        dinfo->media_cd = media == MEDIA_CDROM;
+        break;
+    default:
+        break;
+    }
+
+fail:
+    qemu_opts_del(legacy_opts);
+    return dinfo;
 }
 
 void do_commit(Monitor *mon, const QDict *qdict)
@@ -858,6 +955,80 @@ void qmp_blockdev_snapshot_sync(const char *device, const char *snapshot_file,
                        &snapshot, errp);
 }
 
+void qmp_blockdev_snapshot_internal_sync(const char *device,
+                                         const char *name,
+                                         Error **errp)
+{
+    BlockdevSnapshotInternal snapshot = {
+        .device = (char *) device,
+        .name = (char *) name
+    };
+
+    blockdev_do_action(TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_INTERNAL_SYNC,
+                       &snapshot, errp);
+}
+
+SnapshotInfo *qmp_blockdev_snapshot_delete_internal_sync(const char *device,
+                                                         bool has_id,
+                                                         const char *id,
+                                                         bool has_name,
+                                                         const char *name,
+                                                         Error **errp)
+{
+    BlockDriverState *bs = bdrv_find(device);
+    QEMUSnapshotInfo sn;
+    Error *local_err = NULL;
+    SnapshotInfo *info = NULL;
+    int ret;
+
+    if (!bs) {
+        error_set(errp, QERR_DEVICE_NOT_FOUND, device);
+        return NULL;
+    }
+
+    if (!has_id) {
+        id = NULL;
+    }
+
+    if (!has_name) {
+        name = NULL;
+    }
+
+    if (!id && !name) {
+        error_setg(errp, "Name or id must be provided");
+        return NULL;
+    }
+
+    ret = bdrv_snapshot_find_by_id_and_name(bs, id, name, &sn, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(errp, local_err);
+        return NULL;
+    }
+    if (!ret) {
+        error_setg(errp,
+                   "Snapshot with id '%s' and name '%s' does not exist on "
+                   "device '%s'",
+                   STR_OR_NULL(id), STR_OR_NULL(name), device);
+        return NULL;
+    }
+
+    bdrv_snapshot_delete(bs, id, name, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(errp, local_err);
+        return NULL;
+    }
+
+    info = g_malloc0(sizeof(SnapshotInfo));
+    info->id = g_strdup(sn.id_str);
+    info->name = g_strdup(sn.name);
+    info->date_nsec = sn.date_nsec;
+    info->date_sec = sn.date_sec;
+    info->vm_state_size = sn.vm_state_size;
+    info->vm_clock_nsec = sn.vm_clock_nsec % 1000000000;
+    info->vm_clock_sec = sn.vm_clock_nsec / 1000000000;
+
+    return info;
+}
 
 /* New and old BlockDriverState structs for group snapshots */
 
@@ -889,6 +1060,117 @@ struct BlkTransactionState {
     QSIMPLEQ_ENTRY(BlkTransactionState) entry;
 };
 
+/* internal snapshot private data */
+typedef struct InternalSnapshotState {
+    BlkTransactionState common;
+    BlockDriverState *bs;
+    QEMUSnapshotInfo sn;
+} InternalSnapshotState;
+
+static void internal_snapshot_prepare(BlkTransactionState *common,
+                                      Error **errp)
+{
+    const char *device;
+    const char *name;
+    BlockDriverState *bs;
+    QEMUSnapshotInfo old_sn, *sn;
+    bool ret;
+    qemu_timeval tv;
+    BlockdevSnapshotInternal *internal;
+    InternalSnapshotState *state;
+    int ret1;
+
+    g_assert(common->action->kind ==
+             TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_INTERNAL_SYNC);
+    internal = common->action->blockdev_snapshot_internal_sync;
+    state = DO_UPCAST(InternalSnapshotState, common, common);
+
+    /* 1. parse input */
+    device = internal->device;
+    name = internal->name;
+
+    /* 2. check for validation */
+    bs = bdrv_find(device);
+    if (!bs) {
+        error_set(errp, QERR_DEVICE_NOT_FOUND, device);
+        return;
+    }
+
+    if (!bdrv_is_inserted(bs)) {
+        error_set(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
+        return;
+    }
+
+    if (bdrv_is_read_only(bs)) {
+        error_set(errp, QERR_DEVICE_IS_READ_ONLY, device);
+        return;
+    }
+
+    if (!bdrv_can_snapshot(bs)) {
+        error_set(errp, QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
+                  bs->drv->format_name, device, "internal snapshot");
+        return;
+    }
+
+    if (!strlen(name)) {
+        error_setg(errp, "Name is empty");
+        return;
+    }
+
+    /* check whether a snapshot with name exist */
+    ret = bdrv_snapshot_find_by_id_and_name(bs, NULL, name, &old_sn, errp);
+    if (error_is_set(errp)) {
+        return;
+    } else if (ret) {
+        error_setg(errp,
+                   "Snapshot with name '%s' already exists on device '%s'",
+                   name, device);
+        return;
+    }
+
+    /* 3. take the snapshot */
+    sn = &state->sn;
+    pstrcpy(sn->name, sizeof(sn->name), name);
+    qemu_gettimeofday(&tv);
+    sn->date_sec = tv.tv_sec;
+    sn->date_nsec = tv.tv_usec * 1000;
+    sn->vm_clock_nsec = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+
+    ret1 = bdrv_snapshot_create(bs, sn);
+    if (ret1 < 0) {
+        error_setg_errno(errp, -ret1,
+                         "Failed to create snapshot '%s' on device '%s'",
+                         name, device);
+        return;
+    }
+
+    /* 4. succeed, mark a snapshot is created */
+    state->bs = bs;
+}
+
+static void internal_snapshot_abort(BlkTransactionState *common)
+{
+    InternalSnapshotState *state =
+                             DO_UPCAST(InternalSnapshotState, common, common);
+    BlockDriverState *bs = state->bs;
+    QEMUSnapshotInfo *sn = &state->sn;
+    Error *local_error = NULL;
+
+    if (!bs) {
+        return;
+    }
+
+    if (bdrv_snapshot_delete(bs, sn->id_str, sn->name, &local_error) < 0) {
+        error_report("Failed to delete snapshot with id '%s' and name '%s' on "
+                     "device '%s' in abort: %s",
+                     sn->id_str,
+                     sn->name,
+                     bdrv_get_device_name(bs),
+                     error_get_pretty(local_error));
+        error_free(local_error);
+    }
+}
+
 /* external snapshot private data */
 typedef struct ExternalSnapshotState {
     BlkTransactionState common;
@@ -952,6 +1234,11 @@ static void external_snapshot_prepare(BlkTransactionState *common,
         }
     }
 
+    if (bdrv_check_ext_snapshot(state->old_bs) != EXT_SNAPSHOT_ALLOWED) {
+        error_set(errp, QERR_FEATURE_DISABLED, "snapshot");
+        return;
+    }
+
     flags = state->old_bs->open_flags;
 
     /* create new image w/backing file */
@@ -971,9 +1258,9 @@ static void external_snapshot_prepare(BlkTransactionState *common,
     /* TODO Inherit bs->options or only take explicit options with an
      * extended QMP command? */
     ret = bdrv_open(state->new_bs, new_image_file, NULL,
-                    flags | BDRV_O_NO_BACKING, drv);
+                    flags | BDRV_O_NO_BACKING, drv, &local_err);
     if (ret != 0) {
-        error_setg_file_open(errp, -ret, new_image_file);
+        error_propagate(errp, local_err);
     }
 }
 
@@ -1072,6 +1359,11 @@ static const BdrvActionOps actions[] = {
         .prepare = abort_prepare,
         .commit = abort_commit,
     },
+    [TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_INTERNAL_SYNC] = {
+        .instance_size = sizeof(InternalSnapshotState),
+        .prepare  = internal_snapshot_prepare,
+        .abort = internal_snapshot_abort,
+    },
 };
 
 /*
@@ -1102,6 +1394,8 @@ void qmp_transaction(TransactionActionList *dev_list, Error **errp)
         assert(dev_info->kind < ARRAY_SIZE(actions));
 
         ops = &actions[dev_info->kind];
+        assert(ops->instance_size > 0);
+
         state = g_malloc0(ops->instance_size);
         state->ops = ops;
         state->action = dev_info;
@@ -1203,11 +1497,12 @@ static void qmp_bdrv_open_encrypted(BlockDriverState *bs, const char *filename,
                                     int bdrv_flags, BlockDriver *drv,
                                     const char *password, Error **errp)
 {
+    Error *local_err = NULL;
     int ret;
 
-    ret = bdrv_open(bs, filename, NULL, bdrv_flags, drv);
+    ret = bdrv_open(bs, filename, NULL, bdrv_flags, drv, &local_err);
     if (ret < 0) {
-        error_setg_file_open(errp, -ret, filename);
+        error_propagate(errp, local_err);
         return;
     }
 
@@ -1627,10 +1922,10 @@ void qmp_drive_backup(const char *device, const char *target,
     }
 
     target_bs = bdrv_new("");
-    ret = bdrv_open(target_bs, target, NULL, flags, drv);
+    ret = bdrv_open(target_bs, target, NULL, flags, drv, &local_err);
     if (ret < 0) {
         bdrv_unref(target_bs);
-        error_setg_file_open(errp, -ret, target);
+        error_propagate(errp, local_err);
         return;
     }
 
@@ -1739,7 +2034,6 @@ void qmp_drive_mirror(const char *device, const char *target,
     } else {
         switch (mode) {
         case NEW_IMAGE_MODE_EXISTING:
-            ret = 0;
             break;
         case NEW_IMAGE_MODE_ABSOLUTE_PATHS:
             /* create new image with backing file */
@@ -1762,10 +2056,11 @@ void qmp_drive_mirror(const char *device, const char *target,
      * file.
      */
     target_bs = bdrv_new("");
-    ret = bdrv_open(target_bs, target, NULL, flags | BDRV_O_NO_BACKING, drv);
+    ret = bdrv_open(target_bs, target, NULL, flags | BDRV_O_NO_BACKING, drv,
+                    &local_err);
     if (ret < 0) {
         bdrv_unref(target_bs);
-        error_setg_file_open(errp, -ret, target);
+        error_propagate(errp, local_err);
         return;
     }
 
@@ -1863,6 +2158,54 @@ void qmp_block_job_complete(const char *device, Error **errp)
     block_job_complete(job, errp);
 }
 
+void qmp_blockdev_add(BlockdevOptions *options, Error **errp)
+{
+    QmpOutputVisitor *ov = qmp_output_visitor_new();
+    QObject *obj;
+    QDict *qdict;
+    Error *local_err = NULL;
+
+    /* Require an ID in the top level */
+    if (!options->has_id) {
+        error_setg(errp, "Block device needs an ID");
+        goto fail;
+    }
+
+    /* TODO Sort it out in raw-posix and drive_init: Reject aio=native with
+     * cache.direct=false instead of silently switching to aio=threads, except
+     * if called from drive_init.
+     *
+     * For now, simply forbidding the combination for all drivers will do. */
+    if (options->has_aio && options->aio == BLOCKDEV_AIO_OPTIONS_NATIVE) {
+        bool direct = options->cache->has_direct && options->cache->direct;
+        if (!options->has_cache && !direct) {
+            error_setg(errp, "aio=native requires cache.direct=true");
+            goto fail;
+        }
+    }
+
+    visit_type_BlockdevOptions(qmp_output_get_visitor(ov),
+                               &options, NULL, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(errp, local_err);
+        goto fail;
+    }
+
+    obj = qmp_output_get_qobject(ov);
+    qdict = qobject_to_qdict(obj);
+
+    qdict_flatten(qdict);
+
+    blockdev_init(qdict, IF_NONE, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(errp, local_err);
+        goto fail;
+    }
+
+fail:
+    qmp_output_visitor_cleanup(ov);
+}
+
 static void do_qmp_query_block_jobs_one(void *opaque, BlockDriverState *bs)
 {
     BlockJobInfoList **prev = opaque;
@@ -1890,42 +2233,6 @@ QemuOptsList qemu_common_drive_opts = {
     .head = QTAILQ_HEAD_INITIALIZER(qemu_common_drive_opts.head),
     .desc = {
         {
-            .name = "bus",
-            .type = QEMU_OPT_NUMBER,
-            .help = "bus number",
-        },{
-            .name = "unit",
-            .type = QEMU_OPT_NUMBER,
-            .help = "unit number (i.e. lun for scsi)",
-        },{
-            .name = "if",
-            .type = QEMU_OPT_STRING,
-            .help = "interface (ide, scsi, sd, mtd, floppy, pflash, virtio)",
-        },{
-            .name = "index",
-            .type = QEMU_OPT_NUMBER,
-            .help = "index number",
-        },{
-            .name = "cyls",
-            .type = QEMU_OPT_NUMBER,
-            .help = "number of cylinders (ide disk geometry)",
-        },{
-            .name = "heads",
-            .type = QEMU_OPT_NUMBER,
-            .help = "number of heads (ide disk geometry)",
-        },{
-            .name = "secs",
-            .type = QEMU_OPT_NUMBER,
-            .help = "number of sectors (ide disk geometry)",
-        },{
-            .name = "trans",
-            .type = QEMU_OPT_STRING,
-            .help = "chs translation (auto, lba. none)",
-        },{
-            .name = "media",
-            .type = QEMU_OPT_STRING,
-            .help = "media type (disk, cdrom)",
-        },{
             .name = "snapshot",
             .type = QEMU_OPT_BOOL,
             .help = "enable/disable snapshot mode",
@@ -1970,10 +2277,6 @@ QemuOptsList qemu_common_drive_opts = {
             .type = QEMU_OPT_STRING,
             .help = "write error action",
         },{
-            .name = "addr",
-            .type = QEMU_OPT_STRING,
-            .help = "pci address (virtio only)",
-        },{
             .name = "read-only",
             .type = QEMU_OPT_BOOL,
             .help = "open drive file as read-only",
@@ -2033,10 +2336,6 @@ QemuOptsList qemu_common_drive_opts = {
             .name = "copy-on-read",
             .type = QEMU_OPT_BOOL,
             .help = "copy read data from backing file into image file",
-        },{
-            .name = "boot",
-            .type = QEMU_OPT_BOOL,
-            .help = "(deprecated, ignored)",
         },
         { /* end of list */ }
     },
diff --git a/blockjob.c b/blockjob.c
index e7d49b7169..9e5fd5c162 100644
--- a/blockjob.c
+++ b/blockjob.c
@@ -35,7 +35,7 @@
 #include "qmp-commands.h"
 #include "qemu/timer.h"
 
-void *block_job_create(const BlockJobType *job_type, BlockDriverState *bs,
+void *block_job_create(const BlockJobDriver *driver, BlockDriverState *bs,
                        int64_t speed, BlockDriverCompletionFunc *cb,
                        void *opaque, Error **errp)
 {
@@ -48,8 +48,8 @@ void *block_job_create(const BlockJobType *job_type, BlockDriverState *bs,
     bdrv_ref(bs);
     bdrv_set_in_use(bs, 1);
 
-    job = g_malloc0(job_type->instance_size);
-    job->job_type      = job_type;
+    job = g_malloc0(driver->instance_size);
+    job->driver        = driver;
     job->bs            = bs;
     job->cb            = cb;
     job->opaque        = opaque;
@@ -87,11 +87,11 @@ void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp)
 {
     Error *local_err = NULL;
 
-    if (!job->job_type->set_speed) {
+    if (!job->driver->set_speed) {
         error_set(errp, QERR_NOT_SUPPORTED);
         return;
     }
-    job->job_type->set_speed(job, speed, &local_err);
+    job->driver->set_speed(job, speed, &local_err);
     if (error_is_set(&local_err)) {
         error_propagate(errp, local_err);
         return;
@@ -102,12 +102,12 @@ void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp)
 
 void block_job_complete(BlockJob *job, Error **errp)
 {
-    if (job->paused || job->cancelled || !job->job_type->complete) {
+    if (job->paused || job->cancelled || !job->driver->complete) {
         error_set(errp, QERR_BLOCK_JOB_NOT_READY, job->bs->device_name);
         return;
     }
 
-    job->job_type->complete(job, errp);
+    job->driver->complete(job, errp);
 }
 
 void block_job_pause(BlockJob *job)
@@ -143,8 +143,8 @@ bool block_job_is_cancelled(BlockJob *job)
 void block_job_iostatus_reset(BlockJob *job)
 {
     job->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
-    if (job->job_type->iostatus_reset) {
-        job->job_type->iostatus_reset(job);
+    if (job->driver->iostatus_reset) {
+        job->driver->iostatus_reset(job);
     }
 }
 
@@ -209,7 +209,7 @@ void block_job_sleep_ns(BlockJob *job, QEMUClockType type, int64_t ns)
 BlockJobInfo *block_job_query(BlockJob *job)
 {
     BlockJobInfo *info = g_new0(BlockJobInfo, 1);
-    info->type      = g_strdup(job->job_type->job_type);
+    info->type      = g_strdup(BlockJobType_lookup[job->driver->job_type]);
     info->device    = g_strdup(bdrv_get_device_name(job->bs));
     info->len       = job->len;
     info->busy      = job->busy;
@@ -236,7 +236,7 @@ QObject *qobject_from_block_job(BlockJob *job)
                               "'len': %" PRId64 ","
                               "'offset': %" PRId64 ","
                               "'speed': %" PRId64 " }",
-                              job->job_type->job_type,
+                              BlockJobType_lookup[job->driver->job_type],
                               bdrv_get_device_name(job->bs),
                               job->len,
                               job->offset,
diff --git a/bsd-user/qemu.h b/bsd-user/qemu.h
index 325f564f80..ddc74ed0d7 100644
--- a/bsd-user/qemu.h
+++ b/bsd-user/qemu.h
@@ -323,9 +323,9 @@ abi_long copy_from_user(void *hptr, abi_ulong gaddr, size_t len);
 abi_long copy_to_user(abi_ulong gaddr, void *hptr, size_t len);
 
 /* Functions for accessing guest memory.  The tget and tput functions
-   read/write single values, byteswapping as necessary.  The lock_user
+   read/write single values, byteswapping as necessary.  The lock_user function
    gets a pointer to a contiguous area of guest memory, but does not perform
-   and byteswapping.  lock_user may return either a pointer to the guest
+   any byteswapping.  lock_user may return either a pointer to the guest
    memory, or a temporary buffer.  */
 
 /* Lock an area of guest memory into the host.  If copy is true then the
@@ -381,7 +381,7 @@ static inline void *lock_user_string(abi_ulong guest_addr)
     return lock_user(VERIFY_READ, guest_addr, (long)(len + 1), 1);
 }
 
-/* Helper macros for locking/ulocking a target struct.  */
+/* Helper macros for locking/unlocking a target struct.  */
 #define lock_user_struct(type, host_ptr, guest_addr, copy)      \
     (host_ptr = lock_user(type, guest_addr, sizeof(*host_ptr), copy))
 #define unlock_user_struct(host_ptr, guest_addr, copy)          \
diff --git a/configure b/configure
index 15405e1113..03d82f59b8 100755
--- a/configure
+++ b/configure
@@ -216,7 +216,6 @@ linux_user="no"
 bsd_user="no"
 guest_base="yes"
 uname_release=""
-mixemu="no"
 aix="no"
 blobs="yes"
 pkgversion=""
@@ -239,6 +238,7 @@ win_sdk="no"
 want_tools="yes"
 libiscsi=""
 coroutine=""
+coroutine_pool=""
 seccomp=""
 glusterfs=""
 glusterfs_discard="no"
@@ -432,9 +432,6 @@ case "$cpu" in
   aarch64)
     cpu="aarch64"
   ;;
-  hppa|parisc|parisc64)
-    cpu="hppa"
-  ;;
   mips*)
     cpu="mips"
   ;;
@@ -564,7 +561,6 @@ Haiku)
   audio_possible_drivers="oss alsa sdl esd pa"
   linux="yes"
   linux_user="yes"
-  usb="linux"
   kvm="yes"
   vhost_net="yes"
   vhost_scsi="yes"
@@ -577,9 +573,6 @@ esac
 
 if [ "$bsd" = "yes" ] ; then
   if [ "$darwin" != "yes" ] ; then
-    if [ "$targetos" != "FreeBSD" ]; then
-      usb="bsd"
-    fi
     bsd_user="yes"
   fi
 fi
@@ -875,8 +868,6 @@ for opt do
   ;;
   --enable-fdt) fdt="yes"
   ;;
-  --enable-mixemu) mixemu="yes"
-  ;;
   --disable-linux-aio) linux_aio="no"
   ;;
   --enable-linux-aio) linux_aio="yes"
@@ -891,6 +882,10 @@ for opt do
   ;;
   --with-coroutine=*) coroutine="$optarg"
   ;;
+  --disable-coroutine-pool) coroutine_pool="no"
+  ;;
+  --enable-coroutine-pool) coroutine_pool="yes"
+  ;;
   --disable-docs) docs="no"
   ;;
   --enable-docs) docs="yes"
@@ -983,6 +978,14 @@ for opt do
 done
 
 case "$cpu" in
+    ppc)
+           CPU_CFLAGS="-m32"
+           LDFLAGS="-m32 $LDFLAGS"
+           ;;
+    ppc64)
+           CPU_CFLAGS="-m64"
+           LDFLAGS="-m64 $LDFLAGS"
+           ;;
     sparc)
            LDFLAGS="-m32 $LDFLAGS"
            CPU_CFLAGS="-m32 -mcpu=ultrasparc"
@@ -1106,7 +1109,6 @@ echo "                           (affects only QEMU, not qemu-img)"
 echo "  --block-drv-ro-whitelist=L"
 echo "                           set block driver read-only whitelist"
 echo "                           (affects only QEMU, not qemu-img)"
-echo "  --enable-mixemu          enable mixer emulation"
 echo "  --disable-xen            disable xen backend driver support"
 echo "  --enable-xen             enable xen backend driver support"
 echo "  --disable-xen-pci-passthrough"
@@ -1193,6 +1195,8 @@ echo "  --disable-seccomp        disable seccomp support"
 echo "  --enable-seccomp         enables seccomp support"
 echo "  --with-coroutine=BACKEND coroutine backend. Supported options:"
 echo "                           gthread, ucontext, sigaltstack, windows"
+echo "  --disable-coroutine-pool disable coroutine freelist (worse performance)"
+echo "  --enable-coroutine-pool  enable coroutine freelist (better performance)"
 echo "  --enable-glusterfs       enable GlusterFS backend"
 echo "  --disable-glusterfs      disable GlusterFS backend"
 echo "  --enable-gcov            enable test coverage analysis with gcov"
@@ -3123,7 +3127,6 @@ fi
 if test "$libusb" != "no" ; then
     if $pkg_config --atleast-version=1.0.13 libusb-1.0; then
         libusb="yes"
-	usb="libusb"
         libusb_cflags=$($pkg_config --cflags libusb-1.0)
         libusb_libs=$($pkg_config --libs libusb-1.0)
         QEMU_CFLAGS="$QEMU_CFLAGS $libusb_cflags"
@@ -3366,6 +3369,17 @@ else
   esac
 fi
 
+if test "$coroutine_pool" = ""; then
+  if test "$coroutine" = "gthread"; then
+    coroutine_pool=no
+  else
+    coroutine_pool=yes
+  fi
+fi
+if test "$coroutine" = "gthread" -a "$coroutine_pool" = "yes"; then
+  error_exit "'gthread' coroutine backend does not support pool (use --disable-coroutine-pool)"
+fi
+
 ##########################################
 # check if we have open_by_handle_at
 
@@ -3506,7 +3520,7 @@ if test "$gcov" = "yes" ; then
   CFLAGS="-fprofile-arcs -ftest-coverage -g $CFLAGS"
   LDFLAGS="-fprofile-arcs -ftest-coverage $LDFLAGS"
 elif test "$debug" = "no" ; then
-  CFLAGS="-O2 -D_FORTIFY_SOURCE=2 $CFLAGS"
+  CFLAGS="-O2 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 $CFLAGS"
 fi
 
 
@@ -3685,7 +3699,6 @@ echo "mingw32 support   $mingw32"
 echo "Audio drivers     $audio_drv_list"
 echo "Block whitelist (rw) $block_drv_rw_whitelist"
 echo "Block whitelist (ro) $block_drv_ro_whitelist"
-echo "Mixer emulation   $mixemu"
 echo "VirtFS support    $virtfs"
 echo "VNC support       $vnc"
 if test "$vnc" = "yes" ; then
@@ -3737,6 +3750,7 @@ echo "build guest agent $guest_agent"
 echo "QGA VSS support   $guest_agent_with_vss"
 echo "seccomp support   $seccomp"
 echo "coroutine backend $coroutine"
+echo "coroutine pool    $coroutine_pool"
 echo "GlusterFS support $glusterfs"
 echo "virtio-blk-data-plane $virtio_blk_data_plane"
 echo "gcov              $gcov_tool"
@@ -3781,14 +3795,6 @@ echo "libs_softmmu=$libs_softmmu" >> $config_host_mak
 
 echo "ARCH=$ARCH" >> $config_host_mak
 
-case "$cpu" in
-  arm|i386|x86_64|x32|ppc|aarch64)
-    # The TCG interpreter currently does not support ld/st optimization.
-    if test "$tcg_interpreter" = "no" ; then
-        echo "CONFIG_QEMU_LDST_OPTIMIZATION=y" >> $config_host_mak
-    fi
-  ;;
-esac
 if test "$debug_tcg" = "yes" ; then
   echo "CONFIG_DEBUG_TCG=y" >> $config_host_mak
 fi
@@ -3871,9 +3877,6 @@ if test "$audio_win_int" = "yes" ; then
 fi
 echo "CONFIG_BDRV_RW_WHITELIST=$block_drv_rw_whitelist" >> $config_host_mak
 echo "CONFIG_BDRV_RO_WHITELIST=$block_drv_ro_whitelist" >> $config_host_mak
-if test "$mixemu" = "yes" ; then
-  echo "CONFIG_MIXEMU=y" >> $config_host_mak
-fi
 if test "$vnc" = "yes" ; then
   echo "CONFIG_VNC=y" >> $config_host_mak
 fi
@@ -4096,6 +4099,11 @@ if test "$rbd" = "yes" ; then
 fi
 
 echo "CONFIG_COROUTINE_BACKEND=$coroutine" >> $config_host_mak
+if test "$coroutine_pool" = "yes" ; then
+  echo "CONFIG_COROUTINE_POOL=1" >> $config_host_mak
+else
+  echo "CONFIG_COROUTINE_POOL=0" >> $config_host_mak
+fi
 
 if test "$open_by_handle_at" = "yes" ; then
   echo "CONFIG_OPEN_BY_HANDLE=y" >> $config_host_mak
@@ -4146,24 +4154,11 @@ if test "$virtio_blk_data_plane" = "yes" ; then
 fi
 
 # USB host support
-case "$usb" in
-linux)
-  echo "HOST_USB=linux legacy" >> $config_host_mak
-;;
-bsd)
-  echo "HOST_USB=bsd" >> $config_host_mak
-;;
-libusb)
-  if test "$linux" = "yes"; then
-    echo "HOST_USB=libusb linux legacy" >> $config_host_mak
-  else
-    echo "HOST_USB=libusb legacy" >> $config_host_mak
-  fi
-;;
-*)
+if test "$libusb" = "yes"; then
+  echo "HOST_USB=libusb legacy" >> $config_host_mak
+else
   echo "HOST_USB=stub" >> $config_host_mak
-;;
-esac
+fi
 
 # TPM passthrough support?
 if test "$tpm" = "yes"; then
@@ -4658,7 +4653,7 @@ if [ "$dtc_internal" = "yes" ]; then
 fi
 
 # build tree in object directory in case the source is not in the current directory
-DIRS="tests tests/tcg tests/tcg/cris tests/tcg/lm32 tests/libqos tests/qapi-schema tests/tcg/xtensa"
+DIRS="tests tests/tcg tests/tcg/cris tests/tcg/lm32 tests/libqos tests/qapi-schema tests/tcg/xtensa tests/qemu-iotests"
 DIRS="$DIRS pc-bios/optionrom pc-bios/spapr-rtas pc-bios/s390-ccw"
 DIRS="$DIRS roms/seabios roms/vgabios"
 DIRS="$DIRS qapi-generated"
diff --git a/cpu-exec.c b/cpu-exec.c
index 5a4399509e..30cfa2a63a 100644
--- a/cpu-exec.c
+++ b/cpu-exec.c
@@ -681,6 +681,10 @@ int cpu_exec(CPUArchState *env)
              * local variables as longjmp is marked 'noreturn'. */
             cpu = current_cpu;
             env = cpu->env_ptr;
+#if !(defined(CONFIG_USER_ONLY) && \
+      (defined(TARGET_M68K) || defined(TARGET_PPC) || defined(TARGET_S390X)))
+            cc = CPU_GET_CLASS(cpu);
+#endif
         }
     } /* for(;;) */
 
diff --git a/cpus.c b/cpus.c
index e566297bd3..398229ecbd 100644
--- a/cpus.c
+++ b/cpus.c
@@ -37,6 +37,7 @@
 #include "sysemu/qtest.h"
 #include "qemu/main-loop.h"
 #include "qemu/bitmap.h"
+#include "qemu/seqlock.h"
 
 #ifndef _WIN32
 #include "qemu/compatfd.h"
@@ -97,21 +98,32 @@ static bool all_cpu_threads_idle(void)
 /***********************************************************/
 /* guest cycle counter */
 
+/* Protected by TimersState seqlock */
+
+/* Compensate for varying guest execution speed.  */
+static int64_t qemu_icount_bias;
+static int64_t vm_clock_warp_start;
 /* Conversion factor from emulated instructions to virtual clock ticks.  */
 static int icount_time_shift;
 /* Arbitrarily pick 1MIPS as the minimum allowable speed.  */
 #define MAX_ICOUNT_SHIFT 10
-/* Compensate for varying guest execution speed.  */
-static int64_t qemu_icount_bias;
+
+/* Only written by TCG thread */
+static int64_t qemu_icount;
+
 static QEMUTimer *icount_rt_timer;
 static QEMUTimer *icount_vm_timer;
 static QEMUTimer *icount_warp_timer;
-static int64_t vm_clock_warp_start;
-static int64_t qemu_icount;
 
 typedef struct TimersState {
+    /* Protected by BQL.  */
     int64_t cpu_ticks_prev;
     int64_t cpu_ticks_offset;
+
+    /* cpu_clock_offset can be read out of BQL, so protect it with
+     * this lock.
+     */
+    QemuSeqLock vm_clock_seqlock;
     int64_t cpu_clock_offset;
     int32_t cpu_ticks_enabled;
     int64_t dummy;
@@ -120,7 +132,7 @@ typedef struct TimersState {
 static TimersState timers_state;
 
 /* Return the virtual CPU time, based on the instruction counter.  */
-int64_t cpu_get_icount(void)
+static int64_t cpu_get_icount_locked(void)
 {
     int64_t icount;
     CPUState *cpu = current_cpu;
@@ -136,7 +148,21 @@ int64_t cpu_get_icount(void)
     return qemu_icount_bias + (icount << icount_time_shift);
 }
 
+int64_t cpu_get_icount(void)
+{
+    int64_t icount;
+    unsigned start;
+
+    do {
+        start = seqlock_read_begin(&timers_state.vm_clock_seqlock);
+        icount = cpu_get_icount_locked();
+    } while (seqlock_read_retry(&timers_state.vm_clock_seqlock, start));
+
+    return icount;
+}
+
 /* return the host CPU cycle counter and handle stop/restart */
+/* Caller must hold the BQL */
 int64_t cpu_get_ticks(void)
 {
     if (use_icount) {
@@ -157,37 +183,63 @@ int64_t cpu_get_ticks(void)
     }
 }
 
-/* return the host CPU monotonic timer and handle stop/restart */
-int64_t cpu_get_clock(void)
+static int64_t cpu_get_clock_locked(void)
 {
     int64_t ti;
+
     if (!timers_state.cpu_ticks_enabled) {
-        return timers_state.cpu_clock_offset;
+        ti = timers_state.cpu_clock_offset;
     } else {
         ti = get_clock();
-        return ti + timers_state.cpu_clock_offset;
+        ti += timers_state.cpu_clock_offset;
     }
+
+    return ti;
 }
 
-/* enable cpu_get_ticks() */
+/* return the host CPU monotonic timer and handle stop/restart */
+int64_t cpu_get_clock(void)
+{
+    int64_t ti;
+    unsigned start;
+
+    do {
+        start = seqlock_read_begin(&timers_state.vm_clock_seqlock);
+        ti = cpu_get_clock_locked();
+    } while (seqlock_read_retry(&timers_state.vm_clock_seqlock, start));
+
+    return ti;
+}
+
+/* enable cpu_get_ticks()
+ * Caller must hold BQL which server as mutex for vm_clock_seqlock.
+ */
 void cpu_enable_ticks(void)
 {
+    /* Here, the really thing protected by seqlock is cpu_clock_offset. */
+    seqlock_write_lock(&timers_state.vm_clock_seqlock);
     if (!timers_state.cpu_ticks_enabled) {
         timers_state.cpu_ticks_offset -= cpu_get_real_ticks();
         timers_state.cpu_clock_offset -= get_clock();
         timers_state.cpu_ticks_enabled = 1;
     }
+    seqlock_write_unlock(&timers_state.vm_clock_seqlock);
 }
 
 /* disable cpu_get_ticks() : the clock is stopped. You must not call
-   cpu_get_ticks() after that.  */
+ * cpu_get_ticks() after that.
+ * Caller must hold BQL which server as mutex for vm_clock_seqlock.
+ */
 void cpu_disable_ticks(void)
 {
+    /* Here, the really thing protected by seqlock is cpu_clock_offset. */
+    seqlock_write_lock(&timers_state.vm_clock_seqlock);
     if (timers_state.cpu_ticks_enabled) {
         timers_state.cpu_ticks_offset = cpu_get_ticks();
-        timers_state.cpu_clock_offset = cpu_get_clock();
+        timers_state.cpu_clock_offset = cpu_get_clock_locked();
         timers_state.cpu_ticks_enabled = 0;
     }
+    seqlock_write_unlock(&timers_state.vm_clock_seqlock);
 }
 
 /* Correlation between real and virtual time is always going to be
@@ -201,13 +253,19 @@ static void icount_adjust(void)
     int64_t cur_time;
     int64_t cur_icount;
     int64_t delta;
+
+    /* Protected by TimersState mutex.  */
     static int64_t last_delta;
+
     /* If the VM is not running, then do nothing.  */
     if (!runstate_is_running()) {
         return;
     }
-    cur_time = cpu_get_clock();
-    cur_icount = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+
+    seqlock_write_lock(&timers_state.vm_clock_seqlock);
+    cur_time = cpu_get_clock_locked();
+    cur_icount = cpu_get_icount_locked();
+
     delta = cur_icount - cur_time;
     /* FIXME: This is a very crude algorithm, somewhat prone to oscillation.  */
     if (delta > 0
@@ -224,6 +282,7 @@ static void icount_adjust(void)
     }
     last_delta = delta;
     qemu_icount_bias = cur_icount - (qemu_icount << icount_time_shift);
+    seqlock_write_unlock(&timers_state.vm_clock_seqlock);
 }
 
 static void icount_adjust_rt(void *opaque)
@@ -248,30 +307,37 @@ static int64_t qemu_icount_round(int64_t count)
 
 static void icount_warp_rt(void *opaque)
 {
-    if (vm_clock_warp_start == -1) {
+    /* The icount_warp_timer is rescheduled soon after vm_clock_warp_start
+     * changes from -1 to another value, so the race here is okay.
+     */
+    if (atomic_read(&vm_clock_warp_start) == -1) {
         return;
     }
 
+    seqlock_write_lock(&timers_state.vm_clock_seqlock);
     if (runstate_is_running()) {
         int64_t clock = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
-        int64_t warp_delta = clock - vm_clock_warp_start;
-        if (use_icount == 1) {
-            qemu_icount_bias += warp_delta;
-        } else {
+        int64_t warp_delta;
+
+        warp_delta = clock - vm_clock_warp_start;
+        if (use_icount == 2) {
             /*
              * In adaptive mode, do not let QEMU_CLOCK_VIRTUAL run too
              * far ahead of real time.
              */
-            int64_t cur_time = cpu_get_clock();
-            int64_t cur_icount = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+            int64_t cur_time = cpu_get_clock_locked();
+            int64_t cur_icount = cpu_get_icount_locked();
             int64_t delta = cur_time - cur_icount;
-            qemu_icount_bias += MIN(warp_delta, delta);
-        }
-        if (qemu_clock_expired(QEMU_CLOCK_VIRTUAL)) {
-            qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
+            warp_delta = MIN(warp_delta, delta);
         }
+        qemu_icount_bias += warp_delta;
     }
     vm_clock_warp_start = -1;
+    seqlock_write_unlock(&timers_state.vm_clock_seqlock);
+
+    if (qemu_clock_expired(QEMU_CLOCK_VIRTUAL)) {
+        qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
+    }
 }
 
 void qtest_clock_warp(int64_t dest)
@@ -281,7 +347,10 @@ void qtest_clock_warp(int64_t dest)
     while (clock < dest) {
         int64_t deadline = qemu_clock_deadline_ns_all(QEMU_CLOCK_VIRTUAL);
         int64_t warp = MIN(dest - clock, deadline);
+        seqlock_write_lock(&timers_state.vm_clock_seqlock);
         qemu_icount_bias += warp;
+        seqlock_write_unlock(&timers_state.vm_clock_seqlock);
+
         qemu_clock_run_timers(QEMU_CLOCK_VIRTUAL);
         clock = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
     }
@@ -290,6 +359,7 @@ void qtest_clock_warp(int64_t dest)
 
 void qemu_clock_warp(QEMUClockType type)
 {
+    int64_t clock;
     int64_t deadline;
 
     /*
@@ -309,8 +379,8 @@ void qemu_clock_warp(QEMUClockType type)
      * the earliest QEMU_CLOCK_VIRTUAL timer.
      */
     icount_warp_rt(NULL);
-    if (!all_cpu_threads_idle() || !qemu_clock_has_timers(QEMU_CLOCK_VIRTUAL)) {
-        timer_del(icount_warp_timer);
+    timer_del(icount_warp_timer);
+    if (!all_cpu_threads_idle()) {
         return;
     }
 
@@ -319,17 +389,11 @@ void qemu_clock_warp(QEMUClockType type)
 	return;
     }
 
-    vm_clock_warp_start = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
     /* We want to use the earliest deadline from ALL vm_clocks */
+    clock = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
     deadline = qemu_clock_deadline_ns_all(QEMU_CLOCK_VIRTUAL);
-
-    /* Maintain prior (possibly buggy) behaviour where if no deadline
-     * was set (as there is no QEMU_CLOCK_VIRTUAL timer) or it is more than
-     * INT32_MAX nanoseconds ahead, we still use INT32_MAX
-     * nanoseconds.
-     */
-    if ((deadline < 0) || (deadline > INT32_MAX)) {
-        deadline = INT32_MAX;
+    if (deadline < 0) {
+        return;
     }
 
     if (deadline > 0) {
@@ -350,7 +414,12 @@ void qemu_clock_warp(QEMUClockType type)
          * you will not be sending network packets continuously instead of
          * every 100ms.
          */
-        timer_mod(icount_warp_timer, vm_clock_warp_start + deadline);
+        seqlock_write_lock(&timers_state.vm_clock_seqlock);
+        if (vm_clock_warp_start == -1 || vm_clock_warp_start > clock) {
+            vm_clock_warp_start = clock;
+        }
+        seqlock_write_unlock(&timers_state.vm_clock_seqlock);
+        timer_mod_anticipate(icount_warp_timer, clock + deadline);
     } else if (deadline == 0) {
         qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
     }
@@ -371,6 +440,7 @@ static const VMStateDescription vmstate_timers = {
 
 void configure_icount(const char *option)
 {
+    seqlock_init(&timers_state.vm_clock_seqlock, NULL);
     vmstate_register(NULL, 0, &vmstate_timers, &timers_state);
     if (!option) {
         return;
diff --git a/cputlb.c b/cputlb.c
index 19ecf60983..fff0afbd4a 100644
--- a/cputlb.c
+++ b/cputlb.c
@@ -169,21 +169,6 @@ static inline ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
     return ram_addr;
 }
 
-static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
-{
-    ram_addr_t ram_addr;
-    void *p;
-
-    if (tlb_is_dirty_ram(tlb_entry)) {
-        p = (void *)(uintptr_t)((tlb_entry->addr_write & TARGET_PAGE_MASK)
-            + tlb_entry->addend);
-        ram_addr = qemu_ram_addr_from_host_nofail(p);
-        if (!cpu_physical_memory_is_dirty(ram_addr)) {
-            tlb_entry->addr_write |= TLB_NOTDIRTY;
-        }
-    }
-}
-
 void cpu_tlb_reset_dirty_all(ram_addr_t start1, ram_addr_t length)
 {
     CPUState *cpu;
diff --git a/default-configs/arm-linux-user.mak b/default-configs/arm-linux-user.mak
index 46d4aa2d71..413361a022 100644
--- a/default-configs/arm-linux-user.mak
+++ b/default-configs/arm-linux-user.mak
@@ -1,3 +1 @@
 # Default configuration for arm-linux-user
-
-CONFIG_GDBSTUB_XML=y
diff --git a/default-configs/arm-softmmu.mak b/default-configs/arm-softmmu.mak
index ac0815d663..7e6913782e 100644
--- a/default-configs/arm-softmmu.mak
+++ b/default-configs/arm-softmmu.mak
@@ -2,7 +2,6 @@
 
 include pci.mak
 include usb.mak
-CONFIG_GDBSTUB_XML=y
 CONFIG_VGA=y
 CONFIG_ISA_MMIO=y
 CONFIG_NAND=y
@@ -80,3 +79,4 @@ CONFIG_VERSATILE_PCI=y
 CONFIG_VERSATILE_I2C=y
 
 CONFIG_SDHCI=y
+CONFIG_INTEGRATOR_DEBUG=y
diff --git a/default-configs/armeb-linux-user.mak b/default-configs/armeb-linux-user.mak
index 41d0cc4926..bf2ffe7038 100644
--- a/default-configs/armeb-linux-user.mak
+++ b/default-configs/armeb-linux-user.mak
@@ -1,3 +1 @@
 # Default configuration for armeb-linux-user
-
-CONFIG_GDBSTUB_XML=y
diff --git a/default-configs/m68k-linux-user.mak b/default-configs/m68k-linux-user.mak
index f3487aa3d9..06cd5ed7ed 100644
--- a/default-configs/m68k-linux-user.mak
+++ b/default-configs/m68k-linux-user.mak
@@ -1,3 +1 @@
 # Default configuration for m68k-linux-user
-
-CONFIG_GDBSTUB_XML=y
diff --git a/default-configs/m68k-softmmu.mak b/default-configs/m68k-softmmu.mak
index 51fe5bb321..d9552df076 100644
--- a/default-configs/m68k-softmmu.mak
+++ b/default-configs/m68k-softmmu.mak
@@ -3,5 +3,4 @@
 include pci.mak
 include usb.mak
 CONFIG_COLDFIRE=y
-CONFIG_GDBSTUB_XML=y
 CONFIG_PTIMER=y
diff --git a/default-configs/ppc-linux-user.mak b/default-configs/ppc-linux-user.mak
index 681a94598e..6273df2930 100644
--- a/default-configs/ppc-linux-user.mak
+++ b/default-configs/ppc-linux-user.mak
@@ -1,3 +1 @@
 # Default configuration for ppc-linux-user
-
-CONFIG_GDBSTUB_XML=y
diff --git a/default-configs/ppc-softmmu.mak b/default-configs/ppc-softmmu.mak
index eac0b28fb9..f5cd0bdcc0 100644
--- a/default-configs/ppc-softmmu.mak
+++ b/default-configs/ppc-softmmu.mak
@@ -3,7 +3,6 @@
 include pci.mak
 include sound.mak
 include usb.mak
-CONFIG_GDBSTUB_XML=y
 CONFIG_ISA_MMIO=y
 CONFIG_ESCC=y
 CONFIG_M48T59=y
diff --git a/default-configs/ppc64-linux-user.mak b/default-configs/ppc64-linux-user.mak
index 089c08f3a0..422d3fbaeb 100644
--- a/default-configs/ppc64-linux-user.mak
+++ b/default-configs/ppc64-linux-user.mak
@@ -1,3 +1 @@
 # Default configuration for ppc64-linux-user
-
-CONFIG_GDBSTUB_XML=y
diff --git a/default-configs/ppc64-softmmu.mak b/default-configs/ppc64-softmmu.mak
index 7831c2bf57..975112acf7 100644
--- a/default-configs/ppc64-softmmu.mak
+++ b/default-configs/ppc64-softmmu.mak
@@ -3,7 +3,6 @@
 include pci.mak
 include sound.mak
 include usb.mak
-CONFIG_GDBSTUB_XML=y
 CONFIG_ISA_MMIO=y
 CONFIG_ESCC=y
 CONFIG_M48T59=y
diff --git a/default-configs/ppc64abi32-linux-user.mak b/default-configs/ppc64abi32-linux-user.mak
index f038ffd97c..1c657ec9bb 100644
--- a/default-configs/ppc64abi32-linux-user.mak
+++ b/default-configs/ppc64abi32-linux-user.mak
@@ -1,3 +1 @@
 # Default configuration for ppc64abi32-linux-user
-
-CONFIG_GDBSTUB_XML=y
diff --git a/default-configs/ppcemb-softmmu.mak b/default-configs/ppcemb-softmmu.mak
index 86080a7574..4411203a9a 100644
--- a/default-configs/ppcemb-softmmu.mak
+++ b/default-configs/ppcemb-softmmu.mak
@@ -3,7 +3,6 @@
 include pci.mak
 include sound.mak
 include usb.mak
-CONFIG_GDBSTUB_XML=y
 CONFIG_ISA_MMIO=y
 CONFIG_ESCC=y
 CONFIG_M48T59=y
diff --git a/docs/qapi-code-gen.txt b/docs/qapi-code-gen.txt
index 0ce045c0b3..91f44d01b9 100644
--- a/docs/qapi-code-gen.txt
+++ b/docs/qapi-code-gen.txt
@@ -53,6 +53,23 @@ The use of '*' as a prefix to the name means the member is optional.  Optional
 members should always be added to the end of the dictionary to preserve
 backwards compatibility.
 
+
+A complex type definition can specify another complex type as its base.
+In this case, the fields of the base type are included as top-level fields
+of the new complex type's dictionary in the QMP wire format. An example
+definition is:
+
+ { 'type': 'BlockdevOptionsGenericFormat', 'data': { 'file': 'str' } }
+ { 'type': 'BlockdevOptionsGenericCOWFormat',
+   'base': 'BlockdevOptionsGenericFormat',
+   'data': { '*backing': 'str' } }
+
+An example BlockdevOptionsGenericCOWFormat object on the wire could use
+both fields like this:
+
+ { "file": "/some/place/my-image",
+   "backing": "/some/place/my-backing-file" }
+
 === Enumeration types ===
 
 An enumeration type is a dictionary containing a single key whose value is a
diff --git a/docs/qmp/README b/docs/qmp/README
new file mode 100644
index 0000000000..f6a3a031e9
--- /dev/null
+++ b/docs/qmp/README
@@ -0,0 +1,87 @@
+                          QEMU Machine Protocol
+                          =====================
+
+Introduction
+------------
+
+The QEMU Machine Protocol (QMP) allows applications to operate a
+QEMU instance.
+
+QMP is JSON[1] based and features the following:
+
+- Lightweight, text-based, easy to parse data format
+- Asynchronous messages support (ie. events)
+- Capabilities Negotiation
+
+For detailed information on QMP's usage, please, refer to the following files:
+
+o qmp-spec.txt      QEMU Machine Protocol current specification
+o qmp-commands.txt  QMP supported commands (auto-generated at build-time)
+o qmp-events.txt    List of available asynchronous events
+
+[1] http://www.json.org
+
+Usage
+-----
+
+You can use the -qmp option to enable QMP. For example, the following
+makes QMP available on localhost port 4444:
+
+$ qemu [...] -qmp tcp:localhost:4444,server,nowait
+
+However, for more flexibility and to make use of more options, the -mon
+command-line option should be used. For instance, the following example
+creates one HMP instance (human monitor) on stdio and one QMP instance
+on localhost port 4444:
+
+$ qemu [...] -chardev stdio,id=mon0 -mon chardev=mon0,mode=readline \
+             -chardev socket,id=mon1,host=localhost,port=4444,server,nowait \
+             -mon chardev=mon1,mode=control,pretty=on
+
+Please, refer to QEMU's manpage for more information.
+
+Simple Testing
+--------------
+
+To manually test QMP one can connect with telnet and issue commands by hand:
+
+$ telnet localhost 4444
+Trying 127.0.0.1...
+Connected to localhost.
+Escape character is '^]'.
+{
+    "QMP": {
+        "version": {
+            "qemu": {
+                "micro": 50, 
+                "minor": 6, 
+                "major": 1
+            }, 
+            "package": ""
+        }, 
+        "capabilities": [
+        ]
+    }
+}
+
+{ "execute": "qmp_capabilities" }
+{
+    "return": {
+    }
+}
+
+{ "execute": "query-status" }
+{
+    "return": {
+        "status": "prelaunch", 
+        "singlestep": false, 
+        "running": false
+    }
+}
+
+Please, refer to the qapi-schema.json file for a complete command reference.
+
+QMP wiki page
+-------------
+
+http://wiki.qemu-project.org/QMP
diff --git a/QMP/qmp-events.txt b/docs/qmp/qmp-events.txt
index 4b24ec900d..6b87e9786a 100644
--- a/QMP/qmp-events.txt
+++ b/docs/qmp/qmp-events.txt
@@ -1,4 +1,4 @@
-                   QEMU Monitor Protocol Events
+                   QEMU Machine Protocol Events
                    ============================
 
 BALLOON_CHANGE
@@ -159,7 +159,7 @@ Note: The "ready to complete" status is always reset by a BLOCK_JOB_ERROR
 event.
 
 DEVICE_DELETED
------------------
+--------------
 
 Emitted whenever the device removal completion is acknowledged
 by the guest.
@@ -194,8 +194,22 @@ Data:
   },
   "timestamp": { "seconds": 1265044230, "microseconds": 450486 } }
 
+GUEST_PANICKED
+--------------
+
+Emitted when guest OS panic is detected.
+
+Data:
+
+- "action": Action that has been taken (json-string, currently always "pause").
+
+Example:
+
+{ "event": "GUEST_PANICKED",
+     "data": { "action": "pause" } }
+
 NIC_RX_FILTER_CHANGED
------------------
+---------------------
 
 The event is emitted once until the query command is executed,
 the first event will always be emitted.
@@ -486,17 +500,3 @@ Example:
 
 Note: If action is "reset", "shutdown", or "pause" the WATCHDOG event is
 followed respectively by the RESET, SHUTDOWN, or STOP events.
-
-GUEST_PANICKED
---------------
-
-Emitted when guest OS panic is detected.
-
-Data:
-
-- "action": Action that has been taken (json-string, currently always "pause").
-
-Example:
-
-{ "event": "GUEST_PANICKED",
-     "data": { "action": "pause" } }
diff --git a/QMP/qmp-spec.txt b/docs/qmp/qmp-spec.txt
index a27789692b..22568c644e 100644
--- a/QMP/qmp-spec.txt
+++ b/docs/qmp/qmp-spec.txt
@@ -1,21 +1,17 @@
-           QEMU Monitor Protocol Specification - Version 0.1
+                      QEMU Machine Protocol Specification
 
 1. Introduction
 ===============
 
-This document specifies the QEMU Monitor Protocol (QMP), a JSON-based protocol
-which is available for applications to control QEMU at the machine-level.
-
-To enable QMP support, QEMU has to be run in "control mode". This is done by
-starting QEMU with the appropriate command-line options. Please, refer to the
-QEMU manual page for more information.
+This document specifies the QEMU Machine Protocol (QMP), a JSON-based protocol
+which is available for applications to operate QEMU at the machine-level.
 
 2. Protocol Specification
 =========================
 
 This section details the protocol format. For the purpose of this document
-"Client" is any application which is communicating with QEMU in control mode,
-and "Server" is QEMU itself.
+"Client" is any application which is using QMP to communicate with QEMU and
+"Server" is QEMU itself.
 
 JSON data structures, when mentioned in this document, are always in the
 following format:
@@ -47,14 +43,14 @@ that the connection has been successfully established and that the Server is
 ready for capabilities negotiation (for more information refer to section
 '4. Capabilities Negotiation').
 
-The format is:
+The greeting message format is:
 
 { "QMP": { "version": json-object, "capabilities": json-array } }
 
  Where,
 
 - The "version" member contains the Server's version information (the format
-  is the same of the 'query-version' command)
+  is the same of the query-version command)
 - The "capabilities" member specify the availability of features beyond the
   baseline specification
 
@@ -83,10 +79,7 @@ of a command execution: success or error.
 2.4.1 success
 -------------
 
-The success response is issued when the command execution has finished
-without errors.
-
-The format is:
+The format of a success response is:
 
 { "return": json-object, "id": json-value }
 
@@ -96,15 +89,12 @@ The format is:
   in a per-command basis or an empty json-object if the command does not
   return data
 - The "id" member contains the transaction identification associated
-  with the command execution (if issued by the Client)
+  with the command execution if issued by the Client
 
 2.4.2 error
 -----------
 
-The error response is issued when the command execution could not be
-completed because of an error condition.
-
-The format is:
+The format of an error response is:
 
 { "error": { "class": json-string, "desc": json-string }, "id": json-value }
 
@@ -114,7 +104,7 @@ The format is:
 - The "desc" member is a human-readable error message. Clients should
   not attempt to parse this message.
 - The "id" member contains the transaction identification associated with
-  the command execution (if issued by the Client)
+  the command execution if issued by the Client
 
 NOTE: Some errors can occur before the Server is able to read the "id" member,
 in these cases the "id" member will not be part of the error response, even
@@ -124,9 +114,9 @@ if provided by the client.
 -----------------------
 
 As a result of state changes, the Server may send messages unilaterally
-to the Client at any time. They are called 'asynchronous events'.
+to the Client at any time. They are called "asynchronous events".
 
-The format is:
+The format of asynchronous events is:
 
 { "event": json-string, "data": json-object,
   "timestamp": { "seconds": json-number, "microseconds": json-number } }
@@ -147,36 +137,37 @@ qmp-events.txt file.
 ===============
 
 This section provides some examples of real QMP usage, in all of them
-'C' stands for 'Client' and 'S' stands for 'Server'.
+"C" stands for "Client" and "S" stands for "Server".
 
 3.1 Server greeting
 -------------------
 
-S: {"QMP": {"version": {"qemu": "0.12.50", "package": ""}, "capabilities": []}}
+S: { "QMP": { "version": { "qemu": { "micro": 50, "minor": 6, "major": 1 },
+     "package": ""}, "capabilities": []}}
 
 3.2 Simple 'stop' execution
 ---------------------------
 
 C: { "execute": "stop" }
-S: {"return": {}}
+S: { "return": {} }
 
 3.3 KVM information
 -------------------
 
 C: { "execute": "query-kvm", "id": "example" }
-S: {"return": {"enabled": true, "present": true}, "id": "example"}
+S: { "return": { "enabled": true, "present": true }, "id": "example"}
 
 3.4 Parsing error
 ------------------
 
 C: { "execute": }
-S: {"error": {"class": "GenericError", "desc": "Invalid JSON syntax" } }
+S: { "error": { "class": "GenericError", "desc": "Invalid JSON syntax" } }
 
 3.5 Powerdown event
 -------------------
 
-S: {"timestamp": {"seconds": 1258551470, "microseconds": 802384}, "event":
-"POWERDOWN"}
+S: { "timestamp": { "seconds": 1258551470, "microseconds": 802384 },
+    "event": "POWERDOWN" }
 
 4. Capabilities Negotiation
 ----------------------------
@@ -184,17 +175,17 @@ S: {"timestamp": {"seconds": 1258551470, "microseconds": 802384}, "event":
 When a Client successfully establishes a connection, the Server is in
 Capabilities Negotiation mode.
 
-In this mode only the 'qmp_capabilities' command is allowed to run, all
-other commands will return the CommandNotFound error. Asynchronous messages
-are not delivered either.
+In this mode only the qmp_capabilities command is allowed to run, all
+other commands will return the CommandNotFound error. Asynchronous
+messages are not delivered either.
 
-Clients should use the 'qmp_capabilities' command to enable capabilities
+Clients should use the qmp_capabilities command to enable capabilities
 advertised in the Server's greeting (section '2.2 Server Greeting') they
 support.
 
-When the 'qmp_capabilities' command is issued, and if it does not return an
+When the qmp_capabilities command is issued, and if it does not return an
 error, the Server enters in Command mode where capabilities changes take
-effect, all commands (except 'qmp_capabilities') are allowed and asynchronous
+effect, all commands (except qmp_capabilities) are allowed and asynchronous
 messages are delivered.
 
 5 Compatibility Considerations
@@ -245,7 +236,7 @@ arguments, errors, asynchronous events, and so forth.
 
 Any new names downstream wishes to add must begin with '__'.  To
 ensure compatibility with other downstreams, it is strongly
-recommended that you prefix your downstram names with '__RFQDN_' where
+recommended that you prefix your downstream names with '__RFQDN_' where
 RFQDN is a valid, reverse fully qualified domain name which you
 control.  For example, a qemu-kvm specific monitor command would be:
 
diff --git a/docs/rdma.txt b/docs/rdma.txt
index 8d1e003f92..2aca63bd72 100644
--- a/docs/rdma.txt
+++ b/docs/rdma.txt
@@ -1,7 +1,7 @@
 (RDMA: Remote Direct Memory Access)
 RDMA Live Migration Specification, Version # 1
 ==============================================
-Wiki: http://wiki.qemu.org/Features/RDMALiveMigration
+Wiki: http://wiki.qemu-project.org/Features/RDMALiveMigration
 Github: git@github.com:hinesmr/qemu.git, 'rdma' branch
 
 Copyright (C) 2013 Michael R. Hines <mrhines@us.ibm.com>
diff --git a/docs/specs/acpi_cpu_hotplug.txt b/docs/specs/acpi_cpu_hotplug.txt
index 5dec0c5010..f6f577457d 100644
--- a/docs/specs/acpi_cpu_hotplug.txt
+++ b/docs/specs/acpi_cpu_hotplug.txt
@@ -10,7 +10,7 @@ ACPI GPE block (IO ports 0xafe0-0xafe3, byte access):
 Generic ACPI GPE block. Bit 2 (GPE.2) used to notify CPU
 hot-add/remove event to ACPI BIOS, via SCI interrupt.
 
-CPU present bitmap (IO port 0xaf00-0xae1f, 1-byte access):
+CPU present bitmap (IO port 0xaf00-0xaf1f, 1-byte access):
 ---------------------------------------------------------------
 One bit per CPU. Bit position reflects corresponding CPU APIC ID.
 Read-only.
diff --git a/docs/specs/qcow2.txt b/docs/specs/qcow2.txt
index 33eca360cc..f19536a46f 100644
--- a/docs/specs/qcow2.txt
+++ b/docs/specs/qcow2.txt
@@ -355,3 +355,6 @@ Snapshot table entry:
         variable:   Unique ID string for the snapshot (not null terminated)
 
         variable:   Name of the snapshot (not null terminated)
+
+        variable:   Padding to round up the snapshot table entry size to the
+                    next multiple of 8.
diff --git a/exec.c b/exec.c
index 030118e68e..2e31ffcb2c 100644
--- a/exec.c
+++ b/exec.c
@@ -129,7 +129,6 @@ static PhysPageMap next_map;
 
 static void io_mem_init(void);
 static void memory_map_init(void);
-static void *qemu_safe_ram_ptr(ram_addr_t addr);
 
 static MemoryRegion io_mem_watch;
 #endif
@@ -625,55 +624,40 @@ void cpu_abort(CPUArchState *env, const char *fmt, ...)
     abort();
 }
 
-CPUArchState *cpu_copy(CPUArchState *env)
+#if !defined(CONFIG_USER_ONLY)
+static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
 {
-    CPUArchState *new_env = cpu_init(env->cpu_model_str);
-#if defined(TARGET_HAS_ICE)
-    CPUBreakpoint *bp;
-    CPUWatchpoint *wp;
-#endif
-
-    /* Reset non arch specific state */
-    cpu_reset(ENV_GET_CPU(new_env));
-
-    /* Copy arch specific state into the new CPU */
-    memcpy(new_env, env, sizeof(CPUArchState));
+    RAMBlock *block;
 
-    /* Clone all break/watchpoints.
-       Note: Once we support ptrace with hw-debug register access, make sure
-       BP_CPU break/watchpoints are handled correctly on clone. */
-    QTAILQ_INIT(&env->breakpoints);
-    QTAILQ_INIT(&env->watchpoints);
-#if defined(TARGET_HAS_ICE)
-    QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
-        cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
+    /* The list is protected by the iothread lock here.  */
+    block = ram_list.mru_block;
+    if (block && addr - block->offset < block->length) {
+        goto found;
     }
-    QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
-        cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
-                              wp->flags, NULL);
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
+        if (addr - block->offset < block->length) {
+            goto found;
+        }
     }
-#endif
 
-    return new_env;
+    fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
+    abort();
+
+found:
+    ram_list.mru_block = block;
+    return block;
 }
 
-#if !defined(CONFIG_USER_ONLY)
 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
                                       uintptr_t length)
 {
-    uintptr_t start1;
+    RAMBlock *block;
+    ram_addr_t start1;
 
-    /* we modify the TLB cache so that the dirty bit will be set again
-       when accessing the range */
-    start1 = (uintptr_t)qemu_safe_ram_ptr(start);
-    /* Check that we don't span multiple blocks - this breaks the
-       address comparisons below.  */
-    if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
-            != (end - 1) - start) {
-        abort();
-    }
+    block = qemu_get_ram_block(start);
+    assert(block == qemu_get_ram_block(end - 1));
+    start1 = (uintptr_t)block->host + (start - block->offset);
     cpu_tlb_reset_dirty_all(start1, length);
-
 }
 
 /* Note: start and end must be within the same ram block.  */
@@ -749,6 +733,18 @@ static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
                              uint16_t section);
 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
 
+static void *(*phys_mem_alloc)(size_t size) = qemu_anon_ram_alloc;
+
+/*
+ * Set a custom physical guest memory alloator.
+ * Accelerators with unusual needs may need this.  Hopefully, we can
+ * get rid of it eventually.
+ */
+void phys_mem_set_alloc(void *(*alloc)(size_t))
+{
+    phys_mem_alloc = alloc;
+}
+
 static uint16_t phys_section_add(MemoryRegionSection *section)
 {
     /* The physical section number is ORed with a page-aligned
@@ -880,7 +876,7 @@ void qemu_mutex_unlock_ramlist(void)
     qemu_mutex_unlock(&ram_list.mutex);
 }
 
-#if defined(__linux__) && !defined(TARGET_S390X)
+#ifdef __linux__
 
 #include <sys/vfs.h>
 
@@ -983,6 +979,14 @@ static void *file_ram_alloc(RAMBlock *block,
     block->fd = fd;
     return area;
 }
+#else
+static void *file_ram_alloc(RAMBlock *block,
+                            ram_addr_t memory,
+                            const char *path)
+{
+    fprintf(stderr, "-mem-path not supported on this host\n");
+    exit(1);
+}
 #endif
 
 static ram_addr_t find_ram_offset(ram_addr_t size)
@@ -1099,6 +1103,7 @@ ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
 
     size = TARGET_PAGE_ALIGN(size);
     new_block = g_malloc0(sizeof(*new_block));
+    new_block->fd = -1;
 
     /* This assumes the iothread lock is taken here too.  */
     qemu_mutex_lock_ramlist();
@@ -1107,26 +1112,32 @@ ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
     if (host) {
         new_block->host = host;
         new_block->flags |= RAM_PREALLOC_MASK;
+    } else if (xen_enabled()) {
+        if (mem_path) {
+            fprintf(stderr, "-mem-path not supported with Xen\n");
+            exit(1);
+        }
+        xen_ram_alloc(new_block->offset, size, mr);
     } else {
         if (mem_path) {
-#if defined (__linux__) && !defined(TARGET_S390X)
+            if (phys_mem_alloc != qemu_anon_ram_alloc) {
+                /*
+                 * file_ram_alloc() needs to allocate just like
+                 * phys_mem_alloc, but we haven't bothered to provide
+                 * a hook there.
+                 */
+                fprintf(stderr,
+                        "-mem-path not supported with this accelerator\n");
+                exit(1);
+            }
             new_block->host = file_ram_alloc(new_block, size, mem_path);
+        }
+        if (!new_block->host) {
+            new_block->host = phys_mem_alloc(size);
             if (!new_block->host) {
-                new_block->host = qemu_anon_ram_alloc(size);
-                memory_try_enable_merging(new_block->host, size);
-            }
-#else
-            fprintf(stderr, "-mem-path option unsupported\n");
-            exit(1);
-#endif
-        } else {
-            if (xen_enabled()) {
-                xen_ram_alloc(new_block->offset, size, mr);
-            } else if (kvm_enabled()) {
-                /* some s390/kvm configurations have special constraints */
-                new_block->host = kvm_ram_alloc(size);
-            } else {
-                new_block->host = qemu_anon_ram_alloc(size);
+                fprintf(stderr, "Cannot set up guest memory '%s': %s\n",
+                        new_block->mr->name, strerror(errno));
+                exit(1);
             }
             memory_try_enable_merging(new_block->host, size);
         }
@@ -1157,6 +1168,7 @@ ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
 
     qemu_ram_setup_dump(new_block->host, size);
     qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
+    qemu_madvise(new_block->host, size, QEMU_MADV_DONTFORK);
 
     if (kvm_enabled())
         kvm_setup_guest_memory(new_block->host, size);
@@ -1200,23 +1212,15 @@ void qemu_ram_free(ram_addr_t addr)
             ram_list.version++;
             if (block->flags & RAM_PREALLOC_MASK) {
                 ;
-            } else if (mem_path) {
-#if defined (__linux__) && !defined(TARGET_S390X)
-                if (block->fd) {
-                    munmap(block->host, block->length);
-                    close(block->fd);
-                } else {
-                    qemu_anon_ram_free(block->host, block->length);
-                }
-#else
-                abort();
+            } else if (xen_enabled()) {
+                xen_invalidate_map_cache_entry(block->host);
+#ifndef _WIN32
+            } else if (block->fd >= 0) {
+                munmap(block->host, block->length);
+                close(block->fd);
 #endif
             } else {
-                if (xen_enabled()) {
-                    xen_invalidate_map_cache_entry(block->host);
-                } else {
-                    qemu_anon_ram_free(block->host, block->length);
-                }
+                qemu_anon_ram_free(block->host, block->length);
             }
             g_free(block);
             break;
@@ -1240,38 +1244,31 @@ void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
             vaddr = block->host + offset;
             if (block->flags & RAM_PREALLOC_MASK) {
                 ;
+            } else if (xen_enabled()) {
+                abort();
             } else {
                 flags = MAP_FIXED;
                 munmap(vaddr, length);
-                if (mem_path) {
-#if defined(__linux__) && !defined(TARGET_S390X)
-                    if (block->fd) {
+                if (block->fd >= 0) {
 #ifdef MAP_POPULATE
-                        flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
-                            MAP_PRIVATE;
+                    flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
+                        MAP_PRIVATE;
 #else
-                        flags |= MAP_PRIVATE;
-#endif
-                        area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
-                                    flags, block->fd, offset);
-                    } else {
-                        flags |= MAP_PRIVATE | MAP_ANONYMOUS;
-                        area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
-                                    flags, -1, 0);
-                    }
-#else
-                    abort();
+                    flags |= MAP_PRIVATE;
 #endif
+                    area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
+                                flags, block->fd, offset);
                 } else {
-#if defined(TARGET_S390X) && defined(CONFIG_KVM)
-                    flags |= MAP_SHARED | MAP_ANONYMOUS;
-                    area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
-                                flags, -1, 0);
-#else
+                    /*
+                     * Remap needs to match alloc.  Accelerators that
+                     * set phys_mem_alloc never remap.  If they did,
+                     * we'd need a remap hook here.
+                     */
+                    assert(phys_mem_alloc == qemu_anon_ram_alloc);
+
                     flags |= MAP_PRIVATE | MAP_ANONYMOUS;
                     area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
                                 flags, -1, 0);
-#endif
                 }
                 if (area != vaddr) {
                     fprintf(stderr, "Could not remap addr: "
@@ -1288,29 +1285,6 @@ void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
 }
 #endif /* !_WIN32 */
 
-static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
-{
-    RAMBlock *block;
-
-    /* The list is protected by the iothread lock here.  */
-    block = ram_list.mru_block;
-    if (block && addr - block->offset < block->length) {
-        goto found;
-    }
-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
-        if (addr - block->offset < block->length) {
-            goto found;
-        }
-    }
-
-    fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
-    abort();
-
-found:
-    ram_list.mru_block = block;
-    return block;
-}
-
 /* Return a host pointer to ram allocated with qemu_ram_alloc.
    With the exception of the softmmu code in this file, this should
    only be used for local memory (e.g. video ram) that the device owns,
@@ -1338,40 +1312,6 @@ void *qemu_get_ram_ptr(ram_addr_t addr)
     return block->host + (addr - block->offset);
 }
 
-/* Return a host pointer to ram allocated with qemu_ram_alloc.  Same as
- * qemu_get_ram_ptr but do not touch ram_list.mru_block.
- *
- * ??? Is this still necessary?
- */
-static void *qemu_safe_ram_ptr(ram_addr_t addr)
-{
-    RAMBlock *block;
-
-    /* The list is protected by the iothread lock here.  */
-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
-        if (addr - block->offset < block->length) {
-            if (xen_enabled()) {
-                /* We need to check if the requested address is in the RAM
-                 * because we don't want to map the entire memory in QEMU.
-                 * In that case just map until the end of the page.
-                 */
-                if (block->offset == 0) {
-                    return xen_map_cache(addr, 0, 0);
-                } else if (block->host == NULL) {
-                    block->host =
-                        xen_map_cache(block->offset, block->length, 1);
-                }
-            }
-            return block->host + (addr - block->offset);
-        }
-    }
-
-    fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
-    abort();
-
-    return NULL;
-}
-
 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
  * but takes a size argument */
 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
@@ -1560,7 +1500,7 @@ static uint64_t subpage_read(void *opaque, hwaddr addr,
     uint8_t buf[4];
 
 #if defined(DEBUG_SUBPAGE)
-    printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
+    printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
            subpage, len, addr);
 #endif
     address_space_read(subpage->as, addr + subpage->base, buf, len);
@@ -1583,7 +1523,7 @@ static void subpage_write(void *opaque, hwaddr addr,
     uint8_t buf[4];
 
 #if defined(DEBUG_SUBPAGE)
-    printf("%s: subpage %p len %d addr " TARGET_FMT_plx
+    printf("%s: subpage %p len %u addr " TARGET_FMT_plx
            " value %"PRIx64"\n",
            __func__, subpage, len, addr, value);
 #endif
@@ -1604,16 +1544,16 @@ static void subpage_write(void *opaque, hwaddr addr,
 }
 
 static bool subpage_accepts(void *opaque, hwaddr addr,
-                            unsigned size, bool is_write)
+                            unsigned len, bool is_write)
 {
     subpage_t *subpage = opaque;
 #if defined(DEBUG_SUBPAGE)
-    printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
+    printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
            __func__, subpage, is_write ? 'w' : 'r', len, addr);
 #endif
 
     return address_space_access_valid(subpage->as, addr + subpage->base,
-                                      size, is_write);
+                                      len, is_write);
 }
 
 static const MemoryRegionOps subpage_ops = {
@@ -1633,8 +1573,8 @@ static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
     idx = SUBPAGE_IDX(start);
     eidx = SUBPAGE_IDX(end);
 #if defined(DEBUG_SUBPAGE)
-    printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
-           mmio, start, end, idx, eidx, memory);
+    printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
+           __func__, mmio, start, end, idx, eidx, section);
 #endif
     for (; idx <= eidx; idx++) {
         mmio->sub_section[idx] = section;
@@ -1655,8 +1595,8 @@ static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
                           "subpage", TARGET_PAGE_SIZE);
     mmio->iomem.subpage = true;
 #if defined(DEBUG_SUBPAGE)
-    printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
-           mmio, base, TARGET_PAGE_SIZE, subpage_memory);
+    printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
+           mmio, base, TARGET_PAGE_SIZE);
 #endif
     subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
 
diff --git a/gdbstub.c b/gdbstub.c
index 2b7f22b2d2..0e5a3f5bf9 100644
--- a/gdbstub.c
+++ b/gdbstub.c
@@ -1553,7 +1553,7 @@ static void gdb_accept(void)
 static int gdbserver_open(int port)
 {
     struct sockaddr_in sockaddr;
-    int fd, val, ret;
+    int fd, ret;
 
     fd = socket(PF_INET, SOCK_STREAM, 0);
     if (fd < 0) {
@@ -1564,9 +1564,7 @@ static int gdbserver_open(int port)
     fcntl(fd, F_SETFD, FD_CLOEXEC);
 #endif
 
-    /* allow fast reuse */
-    val = 1;
-    qemu_setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val));
+    socket_set_fast_reuse(fd);
 
     sockaddr.sin_family = AF_INET;
     sockaddr.sin_port = htons(port);
diff --git a/hmp-commands.hx b/hmp-commands.hx
index 65b7f6076c..caae5ad9e9 100644
--- a/hmp-commands.hx
+++ b/hmp-commands.hx
@@ -1023,8 +1023,7 @@ ETEXI
                       "of device. If a new image file is specified, the\n\t\t\t"
                       "new image file will become the new root image.\n\t\t\t"
                       "If format is specified, the snapshot file will\n\t\t\t"
-                      "be created in that format. Otherwise the\n\t\t\t"
-                      "snapshot will be internal! (currently unsupported).\n\t\t\t"
+                      "be created in that format.\n\t\t\t"
                       "The default format is qcow2.  The -n flag requests QEMU\n\t\t\t"
                       "to reuse the image found in new-image-file, instead of\n\t\t\t"
                       "recreating it from scratch.",
@@ -1038,6 +1037,40 @@ Snapshot device, using snapshot file as target if provided
 ETEXI
 
     {
+        .name       = "snapshot_blkdev_internal",
+        .args_type  = "device:B,name:s",
+        .params     = "device name",
+        .help       = "take an internal snapshot of device.\n\t\t\t"
+                      "The format of the image used by device must\n\t\t\t"
+                      "support it, such as qcow2.\n\t\t\t",
+        .mhandler.cmd = hmp_snapshot_blkdev_internal,
+    },
+
+STEXI
+@item snapshot_blkdev_internal
+@findex snapshot_blkdev_internal
+Take an internal snapshot on device if it support
+ETEXI
+
+    {
+        .name       = "snapshot_delete_blkdev_internal",
+        .args_type  = "device:B,name:s,id:s?",
+        .params     = "device name [id]",
+        .help       = "delete an internal snapshot of device.\n\t\t\t"
+                      "If id is specified, qemu will try delete\n\t\t\t"
+                      "the snapshot matching both id and name.\n\t\t\t"
+                      "The format of the image used by device must\n\t\t\t"
+                      "support it, such as qcow2.\n\t\t\t",
+        .mhandler.cmd = hmp_snapshot_delete_blkdev_internal,
+    },
+
+STEXI
+@item snapshot_delete_blkdev_internal
+@findex snapshot_delete_blkdev_internal
+Delete an internal snapshot on device if it support
+ETEXI
+
+    {
         .name       = "drive_mirror",
         .args_type  = "reuse:-n,full:-f,device:B,target:s,format:s?",
         .params     = "[-n] [-f] device target [format]",
diff --git a/hmp.c b/hmp.c
index b4a6422e7a..32ee285a1e 100644
--- a/hmp.c
+++ b/hmp.c
@@ -366,8 +366,6 @@ void hmp_info_block(Monitor *mon, const QDict *qdict)
                             info->value->inserted->iops_rd_max,
                             info->value->inserted->iops_wr_max,
                             info->value->inserted->iops_size);
-        } else {
-            monitor_printf(mon, " [not inserted]");
         }
 
         if (verbose) {
@@ -978,6 +976,28 @@ void hmp_snapshot_blkdev(Monitor *mon, const QDict *qdict)
     hmp_handle_error(mon, &errp);
 }
 
+void hmp_snapshot_blkdev_internal(Monitor *mon, const QDict *qdict)
+{
+    const char *device = qdict_get_str(qdict, "device");
+    const char *name = qdict_get_str(qdict, "name");
+    Error *errp = NULL;
+
+    qmp_blockdev_snapshot_internal_sync(device, name, &errp);
+    hmp_handle_error(mon, &errp);
+}
+
+void hmp_snapshot_delete_blkdev_internal(Monitor *mon, const QDict *qdict)
+{
+    const char *device = qdict_get_str(qdict, "device");
+    const char *name = qdict_get_str(qdict, "name");
+    const char *id = qdict_get_try_str(qdict, "id");
+    Error *errp = NULL;
+
+    qmp_blockdev_snapshot_delete_internal_sync(device, !!id, id,
+                                               true, name, &errp);
+    hmp_handle_error(mon, &errp);
+}
+
 void hmp_migrate_cancel(Monitor *mon, const QDict *qdict)
 {
     qmp_migrate_cancel(NULL);
@@ -1141,7 +1161,7 @@ void hmp_block_stream(Monitor *mon, const QDict *qdict)
 
     qmp_block_stream(device, base != NULL, base,
                      qdict_haskey(qdict, "speed"), speed,
-                     BLOCKDEV_ON_ERROR_REPORT, true, &error);
+                     true, BLOCKDEV_ON_ERROR_REPORT, &error);
 
     hmp_handle_error(mon, &error);
 }
diff --git a/hmp.h b/hmp.h
index 6c3bdcd4c2..54cf71fb94 100644
--- a/hmp.h
+++ b/hmp.h
@@ -54,6 +54,8 @@ void hmp_block_passwd(Monitor *mon, const QDict *qdict);
 void hmp_balloon(Monitor *mon, const QDict *qdict);
 void hmp_block_resize(Monitor *mon, const QDict *qdict);
 void hmp_snapshot_blkdev(Monitor *mon, const QDict *qdict);
+void hmp_snapshot_blkdev_internal(Monitor *mon, const QDict *qdict);
+void hmp_snapshot_delete_blkdev_internal(Monitor *mon, const QDict *qdict);
 void hmp_drive_mirror(Monitor *mon, const QDict *qdict);
 void hmp_drive_backup(Monitor *mon, const QDict *qdict);
 void hmp_migrate_cancel(Monitor *mon, const QDict *qdict);
diff --git a/hw/9pfs/virtio-9p-xattr.c b/hw/9pfs/virtio-9p-xattr.c
index 90ae565c19..3fae557a84 100644
--- a/hw/9pfs/virtio-9p-xattr.c
+++ b/hw/9pfs/virtio-9p-xattr.c
@@ -36,7 +36,7 @@ ssize_t v9fs_get_xattr(FsContext *ctx, const char *path,
     if (xops) {
         return xops->getxattr(ctx, path, name, value, size);
     }
-    errno = -EOPNOTSUPP;
+    errno = EOPNOTSUPP;
     return -1;
 }
 
@@ -123,7 +123,7 @@ int v9fs_set_xattr(FsContext *ctx, const char *path, const char *name,
     if (xops) {
         return xops->setxattr(ctx, path, name, value, size, flags);
     }
-    errno = -EOPNOTSUPP;
+    errno = EOPNOTSUPP;
     return -1;
 
 }
@@ -135,7 +135,7 @@ int v9fs_remove_xattr(FsContext *ctx,
     if (xops) {
         return xops->removexattr(ctx, path, name);
     }
-    errno = -EOPNOTSUPP;
+    errno = EOPNOTSUPP;
     return -1;
 
 }
diff --git a/hw/acpi/core.c b/hw/acpi/core.c
index 4d25d8eff0..d8dff5b9d0 100644
--- a/hw/acpi/core.c
+++ b/hw/acpi/core.c
@@ -364,12 +364,13 @@ static void acpi_notify_wakeup(Notifier *notifier, void *data)
             (ACPI_BITMASK_WAKE_STATUS | ACPI_BITMASK_TIMER_STATUS);
         break;
     case QEMU_WAKEUP_REASON_OTHER:
-    default:
         /* ACPI_BITMASK_WAKE_STATUS should be set on resume.
            Pretend that resume was caused by power button */
         ar->pm1.evt.sts |=
             (ACPI_BITMASK_WAKE_STATUS | ACPI_BITMASK_POWER_BUTTON_STATUS);
         break;
+    default:
+        break;
     }
 }
 
diff --git a/hw/alpha/typhoon.c b/hw/alpha/typhoon.c
index aac9a32e0c..59e1bb8388 100644
--- a/hw/alpha/typhoon.c
+++ b/hw/alpha/typhoon.c
@@ -700,7 +700,7 @@ static IOMMUTLBEntry typhoon_translate_iommu(MemoryRegion *iommu, hwaddr addr)
             }
         }
 
-        if (addr >= 0x80000000000 && addr <= 0xfffffffffff) {
+        if (addr >= 0x80000000000ull && addr <= 0xfffffffffffull) {
             /* Check the fourth window for DAC enable and window enable.  */
             if ((pchip->win[3].wba & 0x80000000001ull) == 0x80000000001ull) {
                 uint64_t pte_addr;
diff --git a/hw/arm/boot.c b/hw/arm/boot.c
index 1e313afe8d..583ec7992e 100644
--- a/hw/arm/boot.c
+++ b/hw/arm/boot.c
@@ -354,8 +354,10 @@ void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
 
     /* Load the kernel.  */
     if (!info->kernel_filename) {
-        fprintf(stderr, "Kernel image must be specified\n");
-        exit(1);
+        /* If no kernel specified, do nothing; we will start from address 0
+         * (typically a boot ROM image) in the same way as hardware.
+         */
+        return;
     }
 
     info->dtb_filename = qemu_opt_get(qemu_get_machine_opts(), "dtb");
diff --git a/hw/arm/integratorcp.c b/hw/arm/integratorcp.c
index 2ef93ed8d6..c44b2a499c 100644
--- a/hw/arm/integratorcp.c
+++ b/hw/arm/integratorcp.c
@@ -11,6 +11,7 @@
 #include "hw/devices.h"
 #include "hw/boards.h"
 #include "hw/arm/arm.h"
+#include "hw/misc/arm_integrator_debug.h"
 #include "net/net.h"
 #include "exec/address-spaces.h"
 #include "sysemu/sysemu.h"
@@ -508,6 +509,7 @@ static void integratorcp_init(QEMUMachineInitArgs *args)
     icp_control_init(0xcb000000);
     sysbus_create_simple("pl050_keyboard", 0x18000000, pic[3]);
     sysbus_create_simple("pl050_mouse", 0x19000000, pic[4]);
+    sysbus_create_simple(TYPE_INTEGRATOR_DEBUG, 0x1a000000, 0);
     sysbus_create_varargs("pl181", 0x1c000000, pic[23], pic[24], NULL);
     if (nd_table[0].used)
         smc91c111_init(&nd_table[0], 0xc8000000, pic[27]);
diff --git a/hw/arm/omap_sx1.c b/hw/arm/omap_sx1.c
index b0f8664607..03b381688b 100644
--- a/hw/arm/omap_sx1.c
+++ b/hw/arm/omap_sx1.c
@@ -194,12 +194,10 @@ static void sx1_init(QEMUMachineInitArgs *args, const int version)
     }
 
     /* Load the kernel.  */
-    if (args->kernel_filename) {
-        sx1_binfo.kernel_filename = args->kernel_filename;
-        sx1_binfo.kernel_cmdline = args->kernel_cmdline;
-        sx1_binfo.initrd_filename = args->initrd_filename;
-        arm_load_kernel(mpu->cpu, &sx1_binfo);
-    }
+    sx1_binfo.kernel_filename = args->kernel_filename;
+    sx1_binfo.kernel_cmdline = args->kernel_cmdline;
+    sx1_binfo.initrd_filename = args->initrd_filename;
+    arm_load_kernel(mpu->cpu, &sx1_binfo);
 
     /* TODO: fix next line */
     //~ qemu_console_resize(ds, 640, 480);
diff --git a/hw/arm/palm.c b/hw/arm/palm.c
index 3e390448e2..0b72bbe5fb 100644
--- a/hw/arm/palm.c
+++ b/hw/arm/palm.c
@@ -261,12 +261,10 @@ static void palmte_init(QEMUMachineInitArgs *args)
     }
 
     /* Load the kernel.  */
-    if (kernel_filename) {
-        palmte_binfo.kernel_filename = kernel_filename;
-        palmte_binfo.kernel_cmdline = kernel_cmdline;
-        palmte_binfo.initrd_filename = initrd_filename;
-        arm_load_kernel(mpu->cpu, &palmte_binfo);
-    }
+    palmte_binfo.kernel_filename = kernel_filename;
+    palmte_binfo.kernel_cmdline = kernel_cmdline;
+    palmte_binfo.initrd_filename = initrd_filename;
+    arm_load_kernel(mpu->cpu, &palmte_binfo);
 }
 
 static QEMUMachine palmte_machine = {
diff --git a/hw/arm/z2.c b/hw/arm/z2.c
index 2e0d5d4bcc..a00fcc042d 100644
--- a/hw/arm/z2.c
+++ b/hw/arm/z2.c
@@ -360,13 +360,11 @@ static void z2_init(QEMUMachineInitArgs *args)
     qdev_connect_gpio_out(mpu->gpio, Z2_GPIO_LCD_CS,
         qemu_allocate_irqs(z2_lcd_cs, z2_lcd, 1)[0]);
 
-    if (kernel_filename) {
-        z2_binfo.kernel_filename = kernel_filename;
-        z2_binfo.kernel_cmdline = kernel_cmdline;
-        z2_binfo.initrd_filename = initrd_filename;
-        z2_binfo.board_id = 0x6dd;
-        arm_load_kernel(mpu->cpu, &z2_binfo);
-    }
+    z2_binfo.kernel_filename = kernel_filename;
+    z2_binfo.kernel_cmdline = kernel_cmdline;
+    z2_binfo.initrd_filename = initrd_filename;
+    z2_binfo.board_id = 0x6dd;
+    arm_load_kernel(mpu->cpu, &z2_binfo);
 }
 
 static QEMUMachine z2_machine = {
diff --git a/hw/audio/hda-codec-common.h b/hw/audio/hda-codec-common.h
new file mode 100644
index 0000000000..b4fdb51e8b
--- /dev/null
+++ b/hw/audio/hda-codec-common.h
@@ -0,0 +1,456 @@
+/*
+ * Common code to disable/enable mixer emulation at run time
+ *
+ * Copyright (C) 2013 Red Hat, Inc.
+ *
+ * Written by Bandan Das <bsd@redhat.com>
+ * with important bits picked up from hda-codec.c
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 or
+ * (at your option) version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * HDA codec descriptions
+ */
+
+#ifdef HDA_MIXER
+#define QEMU_HDA_ID_OUTPUT  ((QEMU_HDA_ID_VENDOR << 16) | 0x12)
+#define QEMU_HDA_ID_DUPLEX  ((QEMU_HDA_ID_VENDOR << 16) | 0x22)
+#define QEMU_HDA_ID_MICRO   ((QEMU_HDA_ID_VENDOR << 16) | 0x32)
+#define QEMU_HDA_AMP_CAPS                                               \
+    (AC_AMPCAP_MUTE |                                                   \
+     (QEMU_HDA_AMP_STEPS << AC_AMPCAP_OFFSET_SHIFT)    |                \
+     (QEMU_HDA_AMP_STEPS << AC_AMPCAP_NUM_STEPS_SHIFT) |                \
+     (3                  << AC_AMPCAP_STEP_SIZE_SHIFT))
+#else
+#define QEMU_HDA_ID_OUTPUT  ((QEMU_HDA_ID_VENDOR << 16) | 0x11)
+#define QEMU_HDA_ID_DUPLEX  ((QEMU_HDA_ID_VENDOR << 16) | 0x21)
+#define QEMU_HDA_ID_MICRO   ((QEMU_HDA_ID_VENDOR << 16) | 0x31)
+#define QEMU_HDA_AMP_CAPS   QEMU_HDA_AMP_NONE
+#endif
+
+
+/* common: audio output widget */
+static const desc_param glue(common_params_audio_dac_, PARAM)[] = {
+    {
+        .id  = AC_PAR_AUDIO_WIDGET_CAP,
+        .val = ((AC_WID_AUD_OUT << AC_WCAP_TYPE_SHIFT) |
+                AC_WCAP_FORMAT_OVRD |
+                AC_WCAP_AMP_OVRD |
+                AC_WCAP_OUT_AMP |
+                AC_WCAP_STEREO),
+    },{
+        .id  = AC_PAR_PCM,
+        .val = QEMU_HDA_PCM_FORMATS,
+    },{
+        .id  = AC_PAR_STREAM,
+        .val = AC_SUPFMT_PCM,
+    },{
+        .id  = AC_PAR_AMP_IN_CAP,
+        .val = QEMU_HDA_AMP_NONE,
+    },{
+        .id  = AC_PAR_AMP_OUT_CAP,
+        .val = QEMU_HDA_AMP_CAPS,
+    },
+};
+
+/* common: audio input widget */
+static const desc_param glue(common_params_audio_adc_, PARAM)[] = {
+    {
+        .id  = AC_PAR_AUDIO_WIDGET_CAP,
+        .val = ((AC_WID_AUD_IN << AC_WCAP_TYPE_SHIFT) |
+                AC_WCAP_CONN_LIST |
+                AC_WCAP_FORMAT_OVRD |
+                AC_WCAP_AMP_OVRD |
+                AC_WCAP_IN_AMP |
+                AC_WCAP_STEREO),
+    },{
+        .id  = AC_PAR_CONNLIST_LEN,
+        .val = 1,
+    },{
+        .id  = AC_PAR_PCM,
+        .val = QEMU_HDA_PCM_FORMATS,
+    },{
+        .id  = AC_PAR_STREAM,
+        .val = AC_SUPFMT_PCM,
+    },{
+        .id  = AC_PAR_AMP_IN_CAP,
+        .val = QEMU_HDA_AMP_CAPS,
+    },{
+        .id  = AC_PAR_AMP_OUT_CAP,
+        .val = QEMU_HDA_AMP_NONE,
+    },
+};
+
+/* common: pin widget (line-out) */
+static const desc_param glue(common_params_audio_lineout_, PARAM)[] = {
+    {
+        .id  = AC_PAR_AUDIO_WIDGET_CAP,
+        .val = ((AC_WID_PIN << AC_WCAP_TYPE_SHIFT) |
+                AC_WCAP_CONN_LIST |
+                AC_WCAP_STEREO),
+    },{
+        .id  = AC_PAR_PIN_CAP,
+        .val = AC_PINCAP_OUT,
+    },{
+        .id  = AC_PAR_CONNLIST_LEN,
+        .val = 1,
+    },{
+        .id  = AC_PAR_AMP_IN_CAP,
+        .val = QEMU_HDA_AMP_NONE,
+    },{
+        .id  = AC_PAR_AMP_OUT_CAP,
+        .val = QEMU_HDA_AMP_NONE,
+    },
+};
+
+/* common: pin widget (line-in) */
+static const desc_param glue(common_params_audio_linein_, PARAM)[] = {
+    {
+        .id  = AC_PAR_AUDIO_WIDGET_CAP,
+        .val = ((AC_WID_PIN << AC_WCAP_TYPE_SHIFT) |
+                AC_WCAP_STEREO),
+    },{
+        .id  = AC_PAR_PIN_CAP,
+        .val = AC_PINCAP_IN,
+    },{
+        .id  = AC_PAR_AMP_IN_CAP,
+        .val = QEMU_HDA_AMP_NONE,
+    },{
+        .id  = AC_PAR_AMP_OUT_CAP,
+        .val = QEMU_HDA_AMP_NONE,
+    },
+};
+
+/* output: root node */
+static const desc_param glue(output_params_root_, PARAM)[] = {
+    {
+        .id  = AC_PAR_VENDOR_ID,
+        .val = QEMU_HDA_ID_OUTPUT,
+    },{
+        .id  = AC_PAR_SUBSYSTEM_ID,
+        .val = QEMU_HDA_ID_OUTPUT,
+    },{
+        .id  = AC_PAR_REV_ID,
+        .val = 0x00100101,
+    },{
+        .id  = AC_PAR_NODE_COUNT,
+        .val = 0x00010001,
+    },
+};
+
+/* output: audio function */
+static const desc_param glue(output_params_audio_func_, PARAM)[] = {
+    {
+        .id  = AC_PAR_FUNCTION_TYPE,
+        .val = AC_GRP_AUDIO_FUNCTION,
+    },{
+        .id  = AC_PAR_SUBSYSTEM_ID,
+        .val = QEMU_HDA_ID_OUTPUT,
+    },{
+        .id  = AC_PAR_NODE_COUNT,
+        .val = 0x00020002,
+    },{
+        .id  = AC_PAR_PCM,
+        .val = QEMU_HDA_PCM_FORMATS,
+    },{
+        .id  = AC_PAR_STREAM,
+        .val = AC_SUPFMT_PCM,
+    },{
+        .id  = AC_PAR_AMP_IN_CAP,
+        .val = QEMU_HDA_AMP_NONE,
+    },{
+        .id  = AC_PAR_AMP_OUT_CAP,
+        .val = QEMU_HDA_AMP_NONE,
+    },{
+        .id  = AC_PAR_GPIO_CAP,
+        .val = 0,
+    },{
+        .id  = AC_PAR_AUDIO_FG_CAP,
+        .val = 0x00000808,
+    },{
+        .id  = AC_PAR_POWER_STATE,
+        .val = 0,
+    },
+};
+
+/* output: nodes */
+static const desc_node glue(output_nodes_, PARAM)[] = {
+    {
+        .nid     = AC_NODE_ROOT,
+        .name    = "root",
+        .params  = glue(output_params_root_, PARAM),
+        .nparams = ARRAY_SIZE(glue(output_params_root_, PARAM)),
+    },{
+        .nid     = 1,
+        .name    = "func",
+        .params  = glue(output_params_audio_func_, PARAM),
+        .nparams = ARRAY_SIZE(glue(output_params_audio_func_, PARAM)),
+    },{
+        .nid     = 2,
+        .name    = "dac",
+        .params  = glue(common_params_audio_dac_, PARAM),
+        .nparams = ARRAY_SIZE(glue(common_params_audio_dac_, PARAM)),
+        .stindex = 0,
+    },{
+        .nid     = 3,
+        .name    = "out",
+        .params  = glue(common_params_audio_lineout_, PARAM),
+        .nparams = ARRAY_SIZE(glue(common_params_audio_lineout_, PARAM)),
+        .config  = ((AC_JACK_PORT_COMPLEX << AC_DEFCFG_PORT_CONN_SHIFT) |
+                    (AC_JACK_LINE_OUT     << AC_DEFCFG_DEVICE_SHIFT)    |
+                    (AC_JACK_CONN_UNKNOWN << AC_DEFCFG_CONN_TYPE_SHIFT) |
+                    (AC_JACK_COLOR_GREEN  << AC_DEFCFG_COLOR_SHIFT)     |
+                    0x10),
+        .pinctl  = AC_PINCTL_OUT_EN,
+        .conn    = (uint32_t[]) { 2 },
+    }
+};
+
+/* output: codec */
+static const desc_codec glue(output_, PARAM) = {
+    .name   = "output",
+    .iid    = QEMU_HDA_ID_OUTPUT,
+    .nodes  = glue(output_nodes_, PARAM),
+    .nnodes = ARRAY_SIZE(glue(output_nodes_, PARAM)),
+};
+
+/* duplex: root node */
+static const desc_param glue(duplex_params_root_, PARAM)[] = {
+    {
+        .id  = AC_PAR_VENDOR_ID,
+        .val = QEMU_HDA_ID_DUPLEX,
+    },{
+        .id  = AC_PAR_SUBSYSTEM_ID,
+        .val = QEMU_HDA_ID_DUPLEX,
+    },{
+        .id  = AC_PAR_REV_ID,
+        .val = 0x00100101,
+    },{
+        .id  = AC_PAR_NODE_COUNT,
+        .val = 0x00010001,
+    },
+};
+
+/* duplex: audio function */
+static const desc_param glue(duplex_params_audio_func_, PARAM)[] = {
+    {
+        .id  = AC_PAR_FUNCTION_TYPE,
+        .val = AC_GRP_AUDIO_FUNCTION,
+    },{
+        .id  = AC_PAR_SUBSYSTEM_ID,
+        .val = QEMU_HDA_ID_DUPLEX,
+    },{
+        .id  = AC_PAR_NODE_COUNT,
+        .val = 0x00020004,
+    },{
+        .id  = AC_PAR_PCM,
+        .val = QEMU_HDA_PCM_FORMATS,
+    },{
+        .id  = AC_PAR_STREAM,
+        .val = AC_SUPFMT_PCM,
+    },{
+        .id  = AC_PAR_AMP_IN_CAP,
+        .val = QEMU_HDA_AMP_NONE,
+    },{
+        .id  = AC_PAR_AMP_OUT_CAP,
+        .val = QEMU_HDA_AMP_NONE,
+    },{
+        .id  = AC_PAR_GPIO_CAP,
+        .val = 0,
+    },{
+        .id  = AC_PAR_AUDIO_FG_CAP,
+        .val = 0x00000808,
+    },{
+        .id  = AC_PAR_POWER_STATE,
+        .val = 0,
+    },
+};
+
+/* duplex: nodes */
+static const desc_node glue(duplex_nodes_, PARAM)[] = {
+    {
+        .nid     = AC_NODE_ROOT,
+        .name    = "root",
+        .params  = glue(duplex_params_root_, PARAM),
+        .nparams = ARRAY_SIZE(glue(duplex_params_root_, PARAM)),
+    },{
+        .nid     = 1,
+        .name    = "func",
+        .params  = glue(duplex_params_audio_func_, PARAM),
+        .nparams = ARRAY_SIZE(glue(duplex_params_audio_func_, PARAM)),
+    },{
+        .nid     = 2,
+        .name    = "dac",
+        .params  = glue(common_params_audio_dac_, PARAM),
+        .nparams = ARRAY_SIZE(glue(common_params_audio_dac_, PARAM)),
+        .stindex = 0,
+    },{
+        .nid     = 3,
+        .name    = "out",
+        .params  = glue(common_params_audio_lineout_, PARAM),
+        .nparams = ARRAY_SIZE(glue(common_params_audio_lineout_, PARAM)),
+        .config  = ((AC_JACK_PORT_COMPLEX << AC_DEFCFG_PORT_CONN_SHIFT) |
+                    (AC_JACK_LINE_OUT     << AC_DEFCFG_DEVICE_SHIFT)    |
+                    (AC_JACK_CONN_UNKNOWN << AC_DEFCFG_CONN_TYPE_SHIFT) |
+                    (AC_JACK_COLOR_GREEN  << AC_DEFCFG_COLOR_SHIFT)     |
+                    0x10),
+        .pinctl  = AC_PINCTL_OUT_EN,
+        .conn    = (uint32_t[]) { 2 },
+    },{
+        .nid     = 4,
+        .name    = "adc",
+        .params  = glue(common_params_audio_adc_, PARAM),
+        .nparams = ARRAY_SIZE(glue(common_params_audio_adc_, PARAM)),
+        .stindex = 1,
+        .conn    = (uint32_t[]) { 5 },
+    },{
+        .nid     = 5,
+        .name    = "in",
+        .params  = glue(common_params_audio_linein_, PARAM),
+        .nparams = ARRAY_SIZE(glue(common_params_audio_linein_, PARAM)),
+        .config  = ((AC_JACK_PORT_COMPLEX << AC_DEFCFG_PORT_CONN_SHIFT) |
+                    (AC_JACK_LINE_IN      << AC_DEFCFG_DEVICE_SHIFT)    |
+                    (AC_JACK_CONN_UNKNOWN << AC_DEFCFG_CONN_TYPE_SHIFT) |
+                    (AC_JACK_COLOR_RED    << AC_DEFCFG_COLOR_SHIFT)     |
+                    0x20),
+        .pinctl  = AC_PINCTL_IN_EN,
+    }
+};
+
+/* duplex: codec */
+static const desc_codec glue(duplex_, PARAM) = {
+    .name   = "duplex",
+    .iid    = QEMU_HDA_ID_DUPLEX,
+    .nodes  = glue(duplex_nodes_, PARAM),
+    .nnodes = ARRAY_SIZE(glue(duplex_nodes_, PARAM)),
+};
+
+/* micro: root node */
+static const desc_param glue(micro_params_root_, PARAM)[] = {
+    {
+        .id  = AC_PAR_VENDOR_ID,
+        .val = QEMU_HDA_ID_MICRO,
+    },{
+        .id  = AC_PAR_SUBSYSTEM_ID,
+        .val = QEMU_HDA_ID_MICRO,
+    },{
+        .id  = AC_PAR_REV_ID,
+        .val = 0x00100101,
+    },{
+        .id  = AC_PAR_NODE_COUNT,
+        .val = 0x00010001,
+    },
+};
+
+/* micro: audio function */
+static const desc_param glue(micro_params_audio_func_, PARAM)[] = {
+    {
+        .id  = AC_PAR_FUNCTION_TYPE,
+        .val = AC_GRP_AUDIO_FUNCTION,
+    },{
+        .id  = AC_PAR_SUBSYSTEM_ID,
+        .val = QEMU_HDA_ID_MICRO,
+    },{
+        .id  = AC_PAR_NODE_COUNT,
+        .val = 0x00020004,
+    },{
+        .id  = AC_PAR_PCM,
+        .val = QEMU_HDA_PCM_FORMATS,
+    },{
+        .id  = AC_PAR_STREAM,
+        .val = AC_SUPFMT_PCM,
+    },{
+        .id  = AC_PAR_AMP_IN_CAP,
+        .val = QEMU_HDA_AMP_NONE,
+    },{
+        .id  = AC_PAR_AMP_OUT_CAP,
+        .val = QEMU_HDA_AMP_NONE,
+    },{
+        .id  = AC_PAR_GPIO_CAP,
+        .val = 0,
+    },{
+        .id  = AC_PAR_AUDIO_FG_CAP,
+        .val = 0x00000808,
+    },{
+        .id  = AC_PAR_POWER_STATE,
+        .val = 0,
+    },
+};
+
+/* micro: nodes */
+static const desc_node glue(micro_nodes_, PARAM)[] = {
+    {
+        .nid     = AC_NODE_ROOT,
+        .name    = "root",
+        .params  = glue(micro_params_root_, PARAM),
+        .nparams = ARRAY_SIZE(glue(micro_params_root_, PARAM)),
+    },{
+        .nid     = 1,
+        .name    = "func",
+        .params  = glue(micro_params_audio_func_, PARAM),
+        .nparams = ARRAY_SIZE(glue(micro_params_audio_func_, PARAM)),
+    },{
+        .nid     = 2,
+        .name    = "dac",
+        .params  = glue(common_params_audio_dac_, PARAM),
+        .nparams = ARRAY_SIZE(glue(common_params_audio_dac_, PARAM)),
+        .stindex = 0,
+    },{
+        .nid     = 3,
+        .name    = "out",
+        .params  = glue(common_params_audio_lineout_, PARAM),
+        .nparams = ARRAY_SIZE(glue(common_params_audio_lineout_, PARAM)),
+        .config  = ((AC_JACK_PORT_COMPLEX << AC_DEFCFG_PORT_CONN_SHIFT) |
+                    (AC_JACK_SPEAKER      << AC_DEFCFG_DEVICE_SHIFT)    |
+                    (AC_JACK_CONN_UNKNOWN << AC_DEFCFG_CONN_TYPE_SHIFT) |
+                    (AC_JACK_COLOR_GREEN  << AC_DEFCFG_COLOR_SHIFT)     |
+                    0x10),
+        .pinctl  = AC_PINCTL_OUT_EN,
+        .conn    = (uint32_t[]) { 2 },
+    },{
+        .nid     = 4,
+        .name    = "adc",
+        .params  = glue(common_params_audio_adc_, PARAM),
+        .nparams = ARRAY_SIZE(glue(common_params_audio_adc_, PARAM)),
+        .stindex = 1,
+        .conn    = (uint32_t[]) { 5 },
+    },{
+        .nid     = 5,
+        .name    = "in",
+        .params  = glue(common_params_audio_linein_, PARAM),
+        .nparams = ARRAY_SIZE(glue(common_params_audio_linein_, PARAM)),
+        .config  = ((AC_JACK_PORT_COMPLEX << AC_DEFCFG_PORT_CONN_SHIFT) |
+                    (AC_JACK_MIC_IN       << AC_DEFCFG_DEVICE_SHIFT)    |
+                    (AC_JACK_CONN_UNKNOWN << AC_DEFCFG_CONN_TYPE_SHIFT) |
+                    (AC_JACK_COLOR_RED    << AC_DEFCFG_COLOR_SHIFT)     |
+                    0x20),
+        .pinctl  = AC_PINCTL_IN_EN,
+    }
+};
+
+/* micro: codec */
+static const desc_codec glue(micro_, PARAM) = {
+    .name   = "micro",
+    .iid    = QEMU_HDA_ID_MICRO,
+    .nodes  = glue(micro_nodes_, PARAM),
+    .nnodes = ARRAY_SIZE(glue(micro_nodes_, PARAM)),
+};
+
+#undef PARAM
+#undef HDA_MIXER
+#undef QEMU_HDA_ID_OUTPUT
+#undef QEMU_HDA_ID_DUPLEX
+#undef QEMU_HDA_ID_MICRO
+#undef QEMU_HDA_AMP_CAPS
diff --git a/hw/audio/hda-codec.c b/hw/audio/hda-codec.c
index 9550c97e65..07a43bfe89 100644
--- a/hw/audio/hda-codec.c
+++ b/hw/audio/hda-codec.c
@@ -118,428 +118,12 @@ static void hda_codec_parse_fmt(uint32_t format, struct audsettings *as)
 #define QEMU_HDA_AMP_NONE    (0)
 #define QEMU_HDA_AMP_STEPS   0x4a
 
-#ifdef CONFIG_MIXEMU
-# define QEMU_HDA_ID_OUTPUT  ((QEMU_HDA_ID_VENDOR << 16) | 0x12)
-# define QEMU_HDA_ID_DUPLEX  ((QEMU_HDA_ID_VENDOR << 16) | 0x22)
-# define QEMU_HDA_ID_MICRO   ((QEMU_HDA_ID_VENDOR << 16) | 0x32)
-# define QEMU_HDA_AMP_CAPS                                              \
-    (AC_AMPCAP_MUTE |                                                   \
-     (QEMU_HDA_AMP_STEPS << AC_AMPCAP_OFFSET_SHIFT)    |                \
-     (QEMU_HDA_AMP_STEPS << AC_AMPCAP_NUM_STEPS_SHIFT) |                \
-     (3                  << AC_AMPCAP_STEP_SIZE_SHIFT))
-#else
-# define QEMU_HDA_ID_OUTPUT  ((QEMU_HDA_ID_VENDOR << 16) | 0x11)
-# define QEMU_HDA_ID_DUPLEX  ((QEMU_HDA_ID_VENDOR << 16) | 0x21)
-# define QEMU_HDA_ID_MICRO   ((QEMU_HDA_ID_VENDOR << 16) | 0x31)
-# define QEMU_HDA_AMP_CAPS   QEMU_HDA_AMP_NONE
-#endif
-
-/* common: audio output widget */
-static const desc_param common_params_audio_dac[] = {
-    {
-        .id  = AC_PAR_AUDIO_WIDGET_CAP,
-        .val = ((AC_WID_AUD_OUT << AC_WCAP_TYPE_SHIFT) |
-                AC_WCAP_FORMAT_OVRD |
-                AC_WCAP_AMP_OVRD |
-                AC_WCAP_OUT_AMP |
-                AC_WCAP_STEREO),
-    },{
-        .id  = AC_PAR_PCM,
-        .val = QEMU_HDA_PCM_FORMATS,
-    },{
-        .id  = AC_PAR_STREAM,
-        .val = AC_SUPFMT_PCM,
-    },{
-        .id  = AC_PAR_AMP_IN_CAP,
-        .val = QEMU_HDA_AMP_NONE,
-    },{
-        .id  = AC_PAR_AMP_OUT_CAP,
-        .val = QEMU_HDA_AMP_CAPS,
-    },
-};
-
-/* common: audio input widget */
-static const desc_param common_params_audio_adc[] = {
-    {
-        .id  = AC_PAR_AUDIO_WIDGET_CAP,
-        .val = ((AC_WID_AUD_IN << AC_WCAP_TYPE_SHIFT) |
-                AC_WCAP_CONN_LIST |
-                AC_WCAP_FORMAT_OVRD |
-                AC_WCAP_AMP_OVRD |
-                AC_WCAP_IN_AMP |
-                AC_WCAP_STEREO),
-    },{
-        .id  = AC_PAR_CONNLIST_LEN,
-        .val = 1,
-    },{
-        .id  = AC_PAR_PCM,
-        .val = QEMU_HDA_PCM_FORMATS,
-    },{
-        .id  = AC_PAR_STREAM,
-        .val = AC_SUPFMT_PCM,
-    },{
-        .id  = AC_PAR_AMP_IN_CAP,
-        .val = QEMU_HDA_AMP_CAPS,
-    },{
-        .id  = AC_PAR_AMP_OUT_CAP,
-        .val = QEMU_HDA_AMP_NONE,
-    },
-};
-
-/* common: pin widget (line-out) */
-static const desc_param common_params_audio_lineout[] = {
-    {
-        .id  = AC_PAR_AUDIO_WIDGET_CAP,
-        .val = ((AC_WID_PIN << AC_WCAP_TYPE_SHIFT) |
-                AC_WCAP_CONN_LIST |
-                AC_WCAP_STEREO),
-    },{
-        .id  = AC_PAR_PIN_CAP,
-        .val = AC_PINCAP_OUT,
-    },{
-        .id  = AC_PAR_CONNLIST_LEN,
-        .val = 1,
-    },{
-        .id  = AC_PAR_AMP_IN_CAP,
-        .val = QEMU_HDA_AMP_NONE,
-    },{
-        .id  = AC_PAR_AMP_OUT_CAP,
-        .val = QEMU_HDA_AMP_NONE,
-    },
-};
-
-/* common: pin widget (line-in) */
-static const desc_param common_params_audio_linein[] = {
-    {
-        .id  = AC_PAR_AUDIO_WIDGET_CAP,
-        .val = ((AC_WID_PIN << AC_WCAP_TYPE_SHIFT) |
-                AC_WCAP_STEREO),
-    },{
-        .id  = AC_PAR_PIN_CAP,
-        .val = AC_PINCAP_IN,
-    },{
-        .id  = AC_PAR_AMP_IN_CAP,
-        .val = QEMU_HDA_AMP_NONE,
-    },{
-        .id  = AC_PAR_AMP_OUT_CAP,
-        .val = QEMU_HDA_AMP_NONE,
-    },
-};
-
-/* output: root node */
-static const desc_param output_params_root[] = {
-    {
-        .id  = AC_PAR_VENDOR_ID,
-        .val = QEMU_HDA_ID_OUTPUT,
-    },{
-        .id  = AC_PAR_SUBSYSTEM_ID,
-        .val = QEMU_HDA_ID_OUTPUT,
-    },{
-        .id  = AC_PAR_REV_ID,
-        .val = 0x00100101,
-    },{
-        .id  = AC_PAR_NODE_COUNT,
-        .val = 0x00010001,
-    },
-};
+#define   PARAM mixemu
+#define   HDA_MIXER
+#include "hda-codec-common.h"
 
-/* output: audio function */
-static const desc_param output_params_audio_func[] = {
-    {
-        .id  = AC_PAR_FUNCTION_TYPE,
-        .val = AC_GRP_AUDIO_FUNCTION,
-    },{
-        .id  = AC_PAR_SUBSYSTEM_ID,
-        .val = QEMU_HDA_ID_OUTPUT,
-    },{
-        .id  = AC_PAR_NODE_COUNT,
-        .val = 0x00020002,
-    },{
-        .id  = AC_PAR_PCM,
-        .val = QEMU_HDA_PCM_FORMATS,
-    },{
-        .id  = AC_PAR_STREAM,
-        .val = AC_SUPFMT_PCM,
-    },{
-        .id  = AC_PAR_AMP_IN_CAP,
-        .val = QEMU_HDA_AMP_NONE,
-    },{
-        .id  = AC_PAR_AMP_OUT_CAP,
-        .val = QEMU_HDA_AMP_NONE,
-    },{
-        .id  = AC_PAR_GPIO_CAP,
-        .val = 0,
-    },{
-        .id  = AC_PAR_AUDIO_FG_CAP,
-        .val = 0x00000808,
-    },{
-        .id  = AC_PAR_POWER_STATE,
-        .val = 0,
-    },
-};
-
-/* output: nodes */
-static const desc_node output_nodes[] = {
-    {
-        .nid     = AC_NODE_ROOT,
-        .name    = "root",
-        .params  = output_params_root,
-        .nparams = ARRAY_SIZE(output_params_root),
-    },{
-        .nid     = 1,
-        .name    = "func",
-        .params  = output_params_audio_func,
-        .nparams = ARRAY_SIZE(output_params_audio_func),
-    },{
-        .nid     = 2,
-        .name    = "dac",
-        .params  = common_params_audio_dac,
-        .nparams = ARRAY_SIZE(common_params_audio_dac),
-        .stindex = 0,
-    },{
-        .nid     = 3,
-        .name    = "out",
-        .params  = common_params_audio_lineout,
-        .nparams = ARRAY_SIZE(common_params_audio_lineout),
-        .config  = ((AC_JACK_PORT_COMPLEX << AC_DEFCFG_PORT_CONN_SHIFT) |
-                    (AC_JACK_LINE_OUT     << AC_DEFCFG_DEVICE_SHIFT)    |
-                    (AC_JACK_CONN_UNKNOWN << AC_DEFCFG_CONN_TYPE_SHIFT) |
-                    (AC_JACK_COLOR_GREEN  << AC_DEFCFG_COLOR_SHIFT)     |
-                    0x10),
-        .pinctl  = AC_PINCTL_OUT_EN,
-        .conn    = (uint32_t[]) { 2 },
-    }
-};
-
-/* output: codec */
-static const desc_codec output = {
-    .name   = "output",
-    .iid    = QEMU_HDA_ID_OUTPUT,
-    .nodes  = output_nodes,
-    .nnodes = ARRAY_SIZE(output_nodes),
-};
-
-/* duplex: root node */
-static const desc_param duplex_params_root[] = {
-    {
-        .id  = AC_PAR_VENDOR_ID,
-        .val = QEMU_HDA_ID_DUPLEX,
-    },{
-        .id  = AC_PAR_SUBSYSTEM_ID,
-        .val = QEMU_HDA_ID_DUPLEX,
-    },{
-        .id  = AC_PAR_REV_ID,
-        .val = 0x00100101,
-    },{
-        .id  = AC_PAR_NODE_COUNT,
-        .val = 0x00010001,
-    },
-};
-
-/* duplex: audio function */
-static const desc_param duplex_params_audio_func[] = {
-    {
-        .id  = AC_PAR_FUNCTION_TYPE,
-        .val = AC_GRP_AUDIO_FUNCTION,
-    },{
-        .id  = AC_PAR_SUBSYSTEM_ID,
-        .val = QEMU_HDA_ID_DUPLEX,
-    },{
-        .id  = AC_PAR_NODE_COUNT,
-        .val = 0x00020004,
-    },{
-        .id  = AC_PAR_PCM,
-        .val = QEMU_HDA_PCM_FORMATS,
-    },{
-        .id  = AC_PAR_STREAM,
-        .val = AC_SUPFMT_PCM,
-    },{
-        .id  = AC_PAR_AMP_IN_CAP,
-        .val = QEMU_HDA_AMP_NONE,
-    },{
-        .id  = AC_PAR_AMP_OUT_CAP,
-        .val = QEMU_HDA_AMP_NONE,
-    },{
-        .id  = AC_PAR_GPIO_CAP,
-        .val = 0,
-    },{
-        .id  = AC_PAR_AUDIO_FG_CAP,
-        .val = 0x00000808,
-    },{
-        .id  = AC_PAR_POWER_STATE,
-        .val = 0,
-    },
-};
-
-/* duplex: nodes */
-static const desc_node duplex_nodes[] = {
-    {
-        .nid     = AC_NODE_ROOT,
-        .name    = "root",
-        .params  = duplex_params_root,
-        .nparams = ARRAY_SIZE(duplex_params_root),
-    },{
-        .nid     = 1,
-        .name    = "func",
-        .params  = duplex_params_audio_func,
-        .nparams = ARRAY_SIZE(duplex_params_audio_func),
-    },{
-        .nid     = 2,
-        .name    = "dac",
-        .params  = common_params_audio_dac,
-        .nparams = ARRAY_SIZE(common_params_audio_dac),
-        .stindex = 0,
-    },{
-        .nid     = 3,
-        .name    = "out",
-        .params  = common_params_audio_lineout,
-        .nparams = ARRAY_SIZE(common_params_audio_lineout),
-        .config  = ((AC_JACK_PORT_COMPLEX << AC_DEFCFG_PORT_CONN_SHIFT) |
-                    (AC_JACK_LINE_OUT     << AC_DEFCFG_DEVICE_SHIFT)    |
-                    (AC_JACK_CONN_UNKNOWN << AC_DEFCFG_CONN_TYPE_SHIFT) |
-                    (AC_JACK_COLOR_GREEN  << AC_DEFCFG_COLOR_SHIFT)     |
-                    0x10),
-        .pinctl  = AC_PINCTL_OUT_EN,
-        .conn    = (uint32_t[]) { 2 },
-    },{
-        .nid     = 4,
-        .name    = "adc",
-        .params  = common_params_audio_adc,
-        .nparams = ARRAY_SIZE(common_params_audio_adc),
-        .stindex = 1,
-        .conn    = (uint32_t[]) { 5 },
-    },{
-        .nid     = 5,
-        .name    = "in",
-        .params  = common_params_audio_linein,
-        .nparams = ARRAY_SIZE(common_params_audio_linein),
-        .config  = ((AC_JACK_PORT_COMPLEX << AC_DEFCFG_PORT_CONN_SHIFT) |
-                    (AC_JACK_LINE_IN      << AC_DEFCFG_DEVICE_SHIFT)    |
-                    (AC_JACK_CONN_UNKNOWN << AC_DEFCFG_CONN_TYPE_SHIFT) |
-                    (AC_JACK_COLOR_RED    << AC_DEFCFG_COLOR_SHIFT)     |
-                    0x20),
-        .pinctl  = AC_PINCTL_IN_EN,
-    }
-};
-
-/* duplex: codec */
-static const desc_codec duplex = {
-    .name   = "duplex",
-    .iid    = QEMU_HDA_ID_DUPLEX,
-    .nodes  = duplex_nodes,
-    .nnodes = ARRAY_SIZE(duplex_nodes),
-};
-
-/* micro: root node */
-static const desc_param micro_params_root[] = {
-    {
-        .id  = AC_PAR_VENDOR_ID,
-        .val = QEMU_HDA_ID_MICRO,
-    },{
-        .id  = AC_PAR_SUBSYSTEM_ID,
-        .val = QEMU_HDA_ID_MICRO,
-    },{
-        .id  = AC_PAR_REV_ID,
-        .val = 0x00100101,
-    },{
-        .id  = AC_PAR_NODE_COUNT,
-        .val = 0x00010001,
-    },
-};
-
-/* micro: audio function */
-static const desc_param micro_params_audio_func[] = {
-    {
-        .id  = AC_PAR_FUNCTION_TYPE,
-        .val = AC_GRP_AUDIO_FUNCTION,
-    },{
-        .id  = AC_PAR_SUBSYSTEM_ID,
-        .val = QEMU_HDA_ID_MICRO,
-    },{
-        .id  = AC_PAR_NODE_COUNT,
-        .val = 0x00020004,
-    },{
-        .id  = AC_PAR_PCM,
-        .val = QEMU_HDA_PCM_FORMATS,
-    },{
-        .id  = AC_PAR_STREAM,
-        .val = AC_SUPFMT_PCM,
-    },{
-        .id  = AC_PAR_AMP_IN_CAP,
-        .val = QEMU_HDA_AMP_NONE,
-    },{
-        .id  = AC_PAR_AMP_OUT_CAP,
-        .val = QEMU_HDA_AMP_NONE,
-    },{
-        .id  = AC_PAR_GPIO_CAP,
-        .val = 0,
-    },{
-        .id  = AC_PAR_AUDIO_FG_CAP,
-        .val = 0x00000808,
-    },{
-        .id  = AC_PAR_POWER_STATE,
-        .val = 0,
-    },
-};
-
-/* micro: nodes */
-static const desc_node micro_nodes[] = {
-    {
-        .nid     = AC_NODE_ROOT,
-        .name    = "root",
-        .params  = micro_params_root,
-        .nparams = ARRAY_SIZE(micro_params_root),
-    },{
-        .nid     = 1,
-        .name    = "func",
-        .params  = micro_params_audio_func,
-        .nparams = ARRAY_SIZE(micro_params_audio_func),
-    },{
-        .nid     = 2,
-        .name    = "dac",
-        .params  = common_params_audio_dac,
-        .nparams = ARRAY_SIZE(common_params_audio_dac),
-        .stindex = 0,
-    },{
-        .nid     = 3,
-        .name    = "out",
-        .params  = common_params_audio_lineout,
-        .nparams = ARRAY_SIZE(common_params_audio_lineout),
-        .config  = ((AC_JACK_PORT_COMPLEX << AC_DEFCFG_PORT_CONN_SHIFT) |
-                    (AC_JACK_SPEAKER      << AC_DEFCFG_DEVICE_SHIFT)    |
-                    (AC_JACK_CONN_UNKNOWN << AC_DEFCFG_CONN_TYPE_SHIFT) |
-                    (AC_JACK_COLOR_GREEN  << AC_DEFCFG_COLOR_SHIFT)     |
-                    0x10),
-        .pinctl  = AC_PINCTL_OUT_EN,
-        .conn    = (uint32_t[]) { 2 },
-    },{
-        .nid     = 4,
-        .name    = "adc",
-        .params  = common_params_audio_adc,
-        .nparams = ARRAY_SIZE(common_params_audio_adc),
-        .stindex = 1,
-        .conn    = (uint32_t[]) { 5 },
-    },{
-        .nid     = 5,
-        .name    = "in",
-        .params  = common_params_audio_linein,
-        .nparams = ARRAY_SIZE(common_params_audio_linein),
-        .config  = ((AC_JACK_PORT_COMPLEX << AC_DEFCFG_PORT_CONN_SHIFT) |
-                    (AC_JACK_MIC_IN       << AC_DEFCFG_DEVICE_SHIFT)    |
-                    (AC_JACK_CONN_UNKNOWN << AC_DEFCFG_CONN_TYPE_SHIFT) |
-                    (AC_JACK_COLOR_RED    << AC_DEFCFG_COLOR_SHIFT)     |
-                    0x20),
-        .pinctl  = AC_PINCTL_IN_EN,
-    }
-};
-
-/* micro: codec */
-static const desc_codec micro = {
-    .name   = "micro",
-    .iid    = QEMU_HDA_ID_MICRO,
-    .nodes  = micro_nodes,
-    .nnodes = ARRAY_SIZE(micro_nodes),
-};
+#define   PARAM nomixemu
+#include  "hda-codec-common.h"
 
 /* -------------------------------------------------------------------------- */
 
@@ -585,6 +169,7 @@ struct HDAAudioState {
 
     /* properties */
     uint32_t debug;
+    bool     mixer;
 };
 
 static void hda_audio_input_cb(void *opaque, int avail)
@@ -1006,23 +591,42 @@ static const VMStateDescription vmstate_hda_audio = {
 };
 
 static Property hda_audio_properties[] = {
-    DEFINE_PROP_UINT32("debug", HDAAudioState, debug, 0),
+    DEFINE_PROP_UINT32("debug", HDAAudioState, debug,   0),
+    DEFINE_PROP_BOOL("mixer", HDAAudioState, mixer,  true),
     DEFINE_PROP_END_OF_LIST(),
 };
 
 static int hda_audio_init_output(HDACodecDevice *hda)
 {
-    return hda_audio_init(hda, &output);
+    HDAAudioState *a = DO_UPCAST(HDAAudioState, hda, hda);
+
+    if (!a->mixer) {
+        return hda_audio_init(hda, &output_nomixemu);
+    } else {
+        return hda_audio_init(hda, &output_mixemu);
+    }
 }
 
 static int hda_audio_init_duplex(HDACodecDevice *hda)
 {
-    return hda_audio_init(hda, &duplex);
+    HDAAudioState *a = DO_UPCAST(HDAAudioState, hda, hda);
+
+    if (!a->mixer) {
+        return hda_audio_init(hda, &duplex_nomixemu);
+    } else {
+        return hda_audio_init(hda, &duplex_mixemu);
+    }
 }
 
 static int hda_audio_init_micro(HDACodecDevice *hda)
 {
-    return hda_audio_init(hda, &micro);
+    HDAAudioState *a = DO_UPCAST(HDAAudioState, hda, hda);
+
+    if (!a->mixer) {
+        return hda_audio_init(hda, &micro_nomixemu);
+    } else {
+        return hda_audio_init(hda, &micro_mixemu);
+    }
 }
 
 static void hda_audio_output_class_init(ObjectClass *klass, void *data)
diff --git a/hw/block/m25p80.c b/hw/block/m25p80.c
index 8c3b7f0d3b..02a15441fa 100644
--- a/hw/block/m25p80.c
+++ b/hw/block/m25p80.c
@@ -624,6 +624,11 @@ static int m25p80_init(SSISlave *ss)
     if (dinfo && dinfo->bdrv) {
         DB_PRINT_L(0, "Binding to IF_MTD drive\n");
         s->bdrv = dinfo->bdrv;
+        if (bdrv_is_read_only(s->bdrv)) {
+            fprintf(stderr, "Can't use a read-only drive");
+            return 1;
+        }
+
         /* FIXME: Move to late init */
         if (bdrv_read(s->bdrv, 0, s->storage, DIV_ROUND_UP(s->size,
                                                     BDRV_SECTOR_SIZE))) {
diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c
index e2f55cc946..13f6d8276e 100644
--- a/hw/block/virtio-blk.c
+++ b/hw/block/virtio-blk.c
@@ -460,9 +460,9 @@ static void virtio_blk_dma_restart_cb(void *opaque, int running,
 
 static void virtio_blk_reset(VirtIODevice *vdev)
 {
-#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
     VirtIOBlock *s = VIRTIO_BLK(vdev);
 
+#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
     if (s->dataplane) {
         virtio_blk_data_plane_stop(s->dataplane);
     }
@@ -473,6 +473,7 @@ static void virtio_blk_reset(VirtIODevice *vdev)
      * are per-device request lists.
      */
     bdrv_drain_all();
+    bdrv_set_enable_write_cache(s->bs, s->original_wce);
 }
 
 /* coalesce internal state, copy to pci i/o region 0
@@ -564,7 +565,25 @@ static void virtio_blk_set_status(VirtIODevice *vdev, uint8_t status)
     }
 
     features = vdev->guest_features;
-    bdrv_set_enable_write_cache(s->bs, !!(features & (1 << VIRTIO_BLK_F_WCE)));
+
+    /* A guest that supports VIRTIO_BLK_F_CONFIG_WCE must be able to send
+     * cache flushes.  Thus, the "auto writethrough" behavior is never
+     * necessary for guests that support the VIRTIO_BLK_F_CONFIG_WCE feature.
+     * Leaving it enabled would break the following sequence:
+     *
+     *     Guest started with "-drive cache=writethrough"
+     *     Guest sets status to 0
+     *     Guest sets DRIVER bit in status field
+     *     Guest reads host features (WCE=0, CONFIG_WCE=1)
+     *     Guest writes guest features (WCE=0, CONFIG_WCE=1)
+     *     Guest writes 1 to the WCE configuration field (writeback mode)
+     *     Guest sets DRIVER_OK bit in status field
+     *
+     * s->bs would erroneously be placed in writethrough mode.
+     */
+    if (!(features & (1 << VIRTIO_BLK_F_CONFIG_WCE))) {
+        bdrv_set_enable_write_cache(s->bs, !!(features & (1 << VIRTIO_BLK_F_WCE)));
+    }
 }
 
 static void virtio_blk_save(QEMUFile *f, void *opaque)
@@ -674,6 +693,7 @@ static int virtio_blk_device_init(VirtIODevice *vdev)
     }
 
     blkconf_serial(&blk->conf, &blk->serial);
+    s->original_wce = bdrv_enable_write_cache(blk->conf.bs);
     if (blkconf_geometry(&blk->conf, NULL, 65535, 255, 255) < 0) {
         return -1;
     }
@@ -683,7 +703,6 @@ static int virtio_blk_device_init(VirtIODevice *vdev)
 
     s->bs = blk->conf.bs;
     s->conf = &blk->conf;
-    memcpy(&(s->blk), blk, sizeof(struct VirtIOBlkConf));
     s->rq = NULL;
     s->sector_mask = (s->conf->logical_block_size / BDRV_SECTOR_SIZE) - 1;
 
diff --git a/hw/block/xen_disk.c b/hw/block/xen_disk.c
index 668cc069ff..098f6c62c7 100644
--- a/hw/block/xen_disk.c
+++ b/hw/block/xen_disk.c
@@ -405,6 +405,7 @@ static int ioreq_map(struct ioreq *ioreq)
                 xen_be_printf(&ioreq->blkdev->xendev, 0,
                               "can't map grant ref %d (%s, %d maps)\n",
                               refs[i], strerror(errno), ioreq->blkdev->cnt_map);
+                ioreq->mapped = 1;
                 ioreq_unmap(ioreq);
                 return -1;
             }
@@ -809,10 +810,15 @@ static int blk_connect(struct XenDevice *xendev)
         xen_be_printf(&blkdev->xendev, 2, "create new bdrv (xenbus setup)\n");
         blkdev->bs = bdrv_new(blkdev->dev);
         if (blkdev->bs) {
+            Error *local_err = NULL;
             BlockDriver *drv = bdrv_find_whitelisted_format(blkdev->fileproto,
                                                            readonly);
             if (bdrv_open(blkdev->bs,
-                          blkdev->filename, NULL, qflags, drv) != 0) {
+                          blkdev->filename, NULL, qflags, drv, &local_err) != 0)
+            {
+                xen_be_printf(&blkdev->xendev, 0, "error: %s\n",
+                              error_get_pretty(local_err));
+                error_free(local_err);
                 bdrv_unref(blkdev->bs);
                 blkdev->bs = NULL;
             }
@@ -824,6 +830,11 @@ static int blk_connect(struct XenDevice *xendev)
         /* setup via qemu cmdline -> already setup for us */
         xen_be_printf(&blkdev->xendev, 2, "get configured bdrv (cmdline setup)\n");
         blkdev->bs = blkdev->dinfo->bdrv;
+        if (bdrv_is_read_only(blkdev->bs) && !readonly) {
+            xen_be_printf(&blkdev->xendev, 0, "Unexpected read-only drive");
+            blkdev->bs = NULL;
+            return -1;
+        }
         /* blkdev->bs is not create by us, we get a reference
          * so we can bdrv_unref() unconditionally */
         bdrv_ref(blkdev->bs);
diff --git a/hw/char/Makefile.objs b/hw/char/Makefile.objs
index f8f3dbca3e..cbd6a006f4 100644
--- a/hw/char/Makefile.objs
+++ b/hw/char/Makefile.objs
@@ -22,6 +22,6 @@ common-obj-$(CONFIG_IMX) += imx_serial.o
 common-obj-$(CONFIG_LM32) += lm32_juart.o
 common-obj-$(CONFIG_LM32) += lm32_uart.o
 common-obj-$(CONFIG_MILKYMIST) += milkymist-uart.o
-common-obj-$(CONFIG_SCLPCONSOLE) += sclpconsole.o
+common-obj-$(CONFIG_SCLPCONSOLE) += sclpconsole.o sclpconsole-lm.o
 
 obj-$(CONFIG_VIRTIO) += virtio-serial-bus.o
diff --git a/hw/char/sclpconsole-lm.c b/hw/char/sclpconsole-lm.c
new file mode 100644
index 0000000000..93390675d6
--- /dev/null
+++ b/hw/char/sclpconsole-lm.c
@@ -0,0 +1,398 @@
+/*
+ * SCLP event types
+ *    Operations Command - Line Mode input
+ *    Message            - Line Mode output
+ *
+ * Copyright IBM, Corp. 2013
+ *
+ * Authors:
+ *  Heinz Graalfs <graalfs@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version.  See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "hw/qdev.h"
+#include "qemu/thread.h"
+#include "qemu/error-report.h"
+#include "sysemu/char.h"
+
+#include "hw/s390x/sclp.h"
+#include "hw/s390x/event-facility.h"
+#include "hw/s390x/ebcdic.h"
+
+#define SIZE_BUFFER 4096
+#define NEWLINE     "\n"
+
+typedef struct OprtnsCommand {
+    EventBufferHeader header;
+    MDMSU message_unit;
+    char data[0];
+} QEMU_PACKED OprtnsCommand;
+
+/* max size for line-mode data in 4K SCCB page */
+#define SIZE_CONSOLE_BUFFER (SCCB_DATA_LEN - sizeof(OprtnsCommand))
+
+typedef struct SCLPConsoleLM {
+    SCLPEvent event;
+    CharDriverState *chr;
+    bool echo;                  /* immediate echo of input if true        */
+    uint32_t write_errors;      /* errors writing to char layer           */
+    uint32_t length;            /* length of byte stream in buffer        */
+    uint8_t buf[SIZE_CONSOLE_BUFFER];
+    qemu_irq irq_console_read;
+} SCLPConsoleLM;
+
+/*
+*  Character layer call-back functions
+ *
+ * Allow 1 character at a time
+ *
+ * Accumulate bytes from character layer in console buffer,
+ * event_pending is set when a newline character is encountered
+ *
+ * The maximum command line length is limited by the maximum
+ * space available in an SCCB
+ */
+
+static int chr_can_read(void *opaque)
+{
+    SCLPConsoleLM *scon = opaque;
+
+    if (scon->event.event_pending) {
+        return 0;
+    } else if (SIZE_CONSOLE_BUFFER - scon->length) {
+        return 1;
+    }
+    return 0;
+}
+
+static void receive_from_chr_layer(SCLPConsoleLM *scon, const uint8_t *buf,
+                                   int size)
+{
+    assert(size == 1);
+
+    if (*buf == '\r' || *buf == '\n') {
+        scon->event.event_pending = true;
+        return;
+    }
+    scon->buf[scon->length] = *buf;
+    scon->length += 1;
+    if (scon->echo) {
+        qemu_chr_fe_write(scon->chr, buf, size);
+    }
+}
+
+/*
+ * Send data from a char device over to the guest
+ */
+static void chr_read(void *opaque, const uint8_t *buf, int size)
+{
+    SCLPConsoleLM *scon = opaque;
+
+    receive_from_chr_layer(scon, buf, size);
+    if (scon->event.event_pending) {
+        /* trigger SCLP read operation */
+        qemu_irq_raise(scon->irq_console_read);
+    }
+}
+
+/* functions to be called by event facility */
+
+static bool can_handle_event(uint8_t type)
+{
+    return type == SCLP_EVENT_MESSAGE || type == SCLP_EVENT_PMSGCMD;
+}
+
+static unsigned int send_mask(void)
+{
+    return SCLP_EVENT_MASK_OP_CMD | SCLP_EVENT_MASK_PMSGCMD;
+}
+
+static unsigned int receive_mask(void)
+{
+    return SCLP_EVENT_MASK_MSG | SCLP_EVENT_MASK_PMSGCMD;
+}
+
+/*
+ * Triggered by SCLP's read_event_data
+ * - convert ASCII byte stream to EBCDIC and
+ * - copy converted data into provided (SCLP) buffer
+ */
+static int get_console_data(SCLPEvent *event, uint8_t *buf, size_t *size,
+                            int avail)
+{
+    int len;
+
+    SCLPConsoleLM *cons = DO_UPCAST(SCLPConsoleLM, event, event);
+
+    len = cons->length;
+    /* data need to fit into provided SCLP buffer */
+    if (len > avail) {
+        return 1;
+    }
+
+    ebcdic_put(buf, (char *)&cons->buf, len);
+    *size = len;
+    cons->length = 0;
+    /* data provided and no more data pending */
+    event->event_pending = false;
+    return 0;
+}
+
+static int read_event_data(SCLPEvent *event, EventBufferHeader *evt_buf_hdr,
+                           int *slen)
+{
+    int avail, rc;
+    size_t src_len;
+    uint8_t *to;
+    OprtnsCommand *oc = (OprtnsCommand *) evt_buf_hdr;
+
+    if (!event->event_pending) {
+        /* no data pending */
+        return 0;
+    }
+
+    to = (uint8_t *)&oc->data;
+    avail = *slen - sizeof(OprtnsCommand);
+    rc = get_console_data(event, to, &src_len, avail);
+    if (rc) {
+        /* data didn't fit, try next SCCB */
+        return 1;
+    }
+
+    oc->message_unit.mdmsu.gds_id = GDS_ID_MDSMU;
+    oc->message_unit.mdmsu.length = cpu_to_be16(sizeof(struct MDMSU));
+
+    oc->message_unit.cpmsu.gds_id = GDS_ID_CPMSU;
+    oc->message_unit.cpmsu.length =
+        cpu_to_be16(sizeof(struct MDMSU) - sizeof(GdsVector));
+
+    oc->message_unit.text_command.gds_id = GDS_ID_TEXTCMD;
+    oc->message_unit.text_command.length =
+        cpu_to_be16(sizeof(struct MDMSU) - (2 * sizeof(GdsVector)));
+
+    oc->message_unit.self_def_text_message.key = GDS_KEY_SELFDEFTEXTMSG;
+    oc->message_unit.self_def_text_message.length =
+        cpu_to_be16(sizeof(struct MDMSU) - (3 * sizeof(GdsVector)));
+
+    oc->message_unit.text_message.key = GDS_KEY_TEXTMSG;
+    oc->message_unit.text_message.length =
+        cpu_to_be16(sizeof(GdsSubvector) + src_len);
+
+    oc->header.length = cpu_to_be16(sizeof(OprtnsCommand) + src_len);
+    oc->header.type = SCLP_EVENT_OPRTNS_COMMAND;
+    *slen = avail - src_len;
+
+    return 1;
+}
+
+/*
+ * Triggered by SCLP's write_event_data
+ *  - write console data to character layer
+ *  returns < 0 if an error occurred
+ */
+static int write_console_data(SCLPEvent *event, const uint8_t *buf, int len)
+{
+    int ret = 0;
+    const uint8_t *buf_offset;
+
+    SCLPConsoleLM *scon = DO_UPCAST(SCLPConsoleLM, event, event);
+
+    if (!scon->chr) {
+        /* If there's no backend, we can just say we consumed all data. */
+        return len;
+    }
+
+    buf_offset = buf;
+    while (len > 0) {
+        ret = qemu_chr_fe_write(scon->chr, buf, len);
+        if (ret == 0) {
+            /* a pty doesn't seem to be connected - no error */
+            len = 0;
+        } else if (ret == -EAGAIN || (ret > 0 && ret < len)) {
+            len -= ret;
+            buf_offset += ret;
+        } else {
+            len = 0;
+        }
+    }
+
+    return ret;
+}
+
+static int process_mdb(SCLPEvent *event, MDBO *mdbo)
+{
+    int rc;
+    int len;
+    uint8_t buffer[SIZE_BUFFER];
+
+    len = be16_to_cpu(mdbo->length);
+    len -= sizeof(mdbo->length) + sizeof(mdbo->type)
+            + sizeof(mdbo->mto.line_type_flags)
+            + sizeof(mdbo->mto.alarm_control)
+            + sizeof(mdbo->mto._reserved);
+
+    assert(len <= SIZE_BUFFER);
+
+    /* convert EBCDIC SCLP contents to ASCII console message */
+    ascii_put(buffer, mdbo->mto.message, len);
+    rc = write_console_data(event, (uint8_t *)NEWLINE, 1);
+    if (rc < 0) {
+        return rc;
+    }
+    return write_console_data(event, buffer, len);
+}
+
+static int write_event_data(SCLPEvent *event, EventBufferHeader *ebh)
+{
+    int len;
+    int written;
+    int errors = 0;
+    MDBO *mdbo;
+    SclpMsg *data = (SclpMsg *) ebh;
+    SCLPConsoleLM *scon = DO_UPCAST(SCLPConsoleLM, event, event);
+
+    len = be16_to_cpu(data->mdb.header.length);
+    if (len < sizeof(data->mdb.header)) {
+        return SCLP_RC_INCONSISTENT_LENGTHS;
+    }
+    len -= sizeof(data->mdb.header);
+
+    /* first check message buffers */
+    mdbo = data->mdb.mdbo;
+    while (len > 0) {
+        if (be16_to_cpu(mdbo->length) > len
+                || be16_to_cpu(mdbo->length) == 0) {
+            return SCLP_RC_INCONSISTENT_LENGTHS;
+        }
+        len -= be16_to_cpu(mdbo->length);
+        mdbo = (void *) mdbo + be16_to_cpu(mdbo->length);
+    }
+
+    /* then execute */
+    len = be16_to_cpu(data->mdb.header.length) - sizeof(data->mdb.header);
+    mdbo = data->mdb.mdbo;
+    while (len > 0) {
+        switch (be16_to_cpu(mdbo->type)) {
+        case MESSAGE_TEXT:
+            /* message text object */
+            written = process_mdb(event, mdbo);
+            if (written < 0) {
+                /* character layer error */
+                errors++;
+            }
+            break;
+        default: /* ignore */
+            break;
+        }
+        len -= be16_to_cpu(mdbo->length);
+        mdbo = (void *) mdbo + be16_to_cpu(mdbo->length);
+    }
+    if (errors) {
+        scon->write_errors += errors;
+    }
+    data->header.flags = SCLP_EVENT_BUFFER_ACCEPTED;
+
+    return SCLP_RC_NORMAL_COMPLETION;
+}
+
+static void trigger_console_data(void *opaque, int n, int level)
+{
+    sclp_service_interrupt(0);
+}
+
+/* functions for live migration */
+
+static const VMStateDescription vmstate_sclplmconsole = {
+    .name = "sclplmconsole",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .minimum_version_id_old = 0,
+    .fields      = (VMStateField[]) {
+        VMSTATE_BOOL(event.event_pending, SCLPConsoleLM),
+        VMSTATE_UINT32(write_errors, SCLPConsoleLM),
+        VMSTATE_UINT32(length, SCLPConsoleLM),
+        VMSTATE_UINT8_ARRAY(buf, SCLPConsoleLM, SIZE_CONSOLE_BUFFER),
+        VMSTATE_END_OF_LIST()
+     }
+};
+
+/* qemu object creation and initialization functions */
+
+/* tell character layer our call-back functions */
+
+static int console_init(SCLPEvent *event)
+{
+    static bool console_available;
+
+    SCLPConsoleLM *scon = DO_UPCAST(SCLPConsoleLM, event, event);
+
+    if (console_available) {
+        error_report("Multiple line-mode operator consoles are not supported");
+        return -1;
+    }
+    console_available = true;
+
+    if (scon->chr) {
+        qemu_chr_add_handlers(scon->chr, chr_can_read, chr_read, NULL, scon);
+    }
+    scon->irq_console_read = *qemu_allocate_irqs(trigger_console_data, NULL, 1);
+
+    return 0;
+}
+
+static int console_exit(SCLPEvent *event)
+{
+    return 0;
+}
+
+static void console_reset(DeviceState *dev)
+{
+   SCLPEvent *event = SCLP_EVENT(dev);
+   SCLPConsoleLM *scon = DO_UPCAST(SCLPConsoleLM, event, event);
+
+   event->event_pending = false;
+   scon->length = 0;
+   scon->write_errors = 0;
+}
+
+static Property console_properties[] = {
+    DEFINE_PROP_CHR("chardev", SCLPConsoleLM, chr),
+    DEFINE_PROP_UINT32("write_errors", SCLPConsoleLM, write_errors, 0),
+    DEFINE_PROP_BOOL("echo", SCLPConsoleLM, echo, true),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void console_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SCLPEventClass *ec = SCLP_EVENT_CLASS(klass);
+
+    dc->props = console_properties;
+    dc->reset = console_reset;
+    dc->vmsd = &vmstate_sclplmconsole;
+    ec->init = console_init;
+    ec->exit = console_exit;
+    ec->get_send_mask = send_mask;
+    ec->get_receive_mask = receive_mask;
+    ec->can_handle_event = can_handle_event;
+    ec->read_event_data = read_event_data;
+    ec->write_event_data = write_event_data;
+}
+
+static const TypeInfo sclp_console_info = {
+    .name          = "sclplmconsole",
+    .parent        = TYPE_SCLP_EVENT,
+    .instance_size = sizeof(SCLPConsoleLM),
+    .class_init    = console_class_init,
+    .class_size    = sizeof(SCLPEventClass),
+};
+
+static void register_types(void)
+{
+    type_register_static(&sclp_console_info);
+}
+
+type_init(register_types)
diff --git a/hw/char/sclpconsole.c b/hw/char/sclpconsole.c
index eb3988c2e4..16d77c5e27 100644
--- a/hw/char/sclpconsole.c
+++ b/hw/char/sclpconsole.c
@@ -31,12 +31,11 @@ typedef struct ASCIIConsoleData {
 typedef struct SCLPConsole {
     SCLPEvent event;
     CharDriverState *chr;
-    /* io vector                                                       */
-    uint8_t *iov;           /* iov buffer pointer                      */
-    uint8_t *iov_sclp;      /* pointer to SCLP read offset             */
-    uint8_t *iov_bs;        /* pointer byte stream read offset         */
-    uint32_t iov_data_len;  /* length of byte stream in buffer         */
-    uint32_t iov_sclp_rest; /* length of byte stream not read via SCLP */
+    uint8_t iov[SIZE_BUFFER_VT220];
+    uint32_t iov_sclp;      /* offset in buf for SCLP read operation       */
+    uint32_t iov_bs;        /* offset in buf for char layer read operation */
+    uint32_t iov_data_len;  /* length of byte stream in buffer             */
+    uint32_t iov_sclp_rest; /* length of byte stream not read via SCLP     */
     qemu_irq irq_read_vt220;
 } SCLPConsole;
 
@@ -47,7 +46,7 @@ static int chr_can_read(void *opaque)
 {
     SCLPConsole *scon = opaque;
 
-    return scon->iov ? SIZE_BUFFER_VT220 - scon->iov_data_len : 0;
+    return SIZE_BUFFER_VT220 - scon->iov_data_len;
 }
 
 /* Receive n bytes from character layer, save in iov buffer,
@@ -55,13 +54,11 @@ static int chr_can_read(void *opaque)
 static void receive_from_chr_layer(SCLPConsole *scon, const uint8_t *buf,
                                    int size)
 {
-    assert(scon->iov);
-
     /* read data must fit into current buffer */
     assert(size <= SIZE_BUFFER_VT220 - scon->iov_data_len);
 
     /* put byte-stream from character layer into buffer */
-    memcpy(scon->iov_bs, buf, size);
+    memcpy(&scon->iov[scon->iov_bs], buf, size);
     scon->iov_data_len += size;
     scon->iov_sclp_rest += size;
     scon->iov_bs += size;
@@ -80,34 +77,11 @@ static void chr_read(void *opaque, const uint8_t *buf, int size)
     qemu_irq_raise(scon->irq_read_vt220);
 }
 
-static void chr_event(void *opaque, int event)
-{
-    SCLPConsole *scon = opaque;
-
-    switch (event) {
-    case CHR_EVENT_OPENED:
-        if (!scon->iov) {
-            scon->iov = g_malloc0(SIZE_BUFFER_VT220);
-            scon->iov_sclp = scon->iov;
-            scon->iov_bs = scon->iov;
-            scon->iov_data_len = 0;
-            scon->iov_sclp_rest = 0;
-        }
-        break;
-    case CHR_EVENT_CLOSED:
-        if (scon->iov) {
-            g_free(scon->iov);
-            scon->iov = NULL;
-        }
-        break;
-    }
-}
-
 /* functions to be called by event facility */
 
-static int event_type(void)
+static bool can_handle_event(uint8_t type)
 {
-    return SCLP_EVENT_ASCII_CONSOLE_DATA;
+    return type == SCLP_EVENT_ASCII_CONSOLE_DATA;
 }
 
 static unsigned int send_mask(void)
@@ -134,17 +108,17 @@ static void get_console_data(SCLPEvent *event, uint8_t *buf, size_t *size,
     /* if all data fit into provided SCLP buffer */
     if (avail >= cons->iov_sclp_rest) {
         /* copy character byte-stream to SCLP buffer */
-        memcpy(buf, cons->iov_sclp, cons->iov_sclp_rest);
+        memcpy(buf, &cons->iov[cons->iov_sclp], cons->iov_sclp_rest);
         *size = cons->iov_sclp_rest + 1;
-        cons->iov_sclp = cons->iov;
-        cons->iov_bs = cons->iov;
+        cons->iov_sclp = 0;
+        cons->iov_bs = 0;
         cons->iov_data_len = 0;
         cons->iov_sclp_rest = 0;
         event->event_pending = false;
         /* data provided and no more data pending */
     } else {
         /* if provided buffer is too small, just copy part */
-        memcpy(buf, cons->iov_sclp, avail);
+        memcpy(buf, &cons->iov[cons->iov_sclp], avail);
         *size = avail + 1;
         cons->iov_sclp_rest -= avail;
         cons->iov_sclp += avail;
@@ -223,9 +197,26 @@ static void trigger_ascii_console_data(void *opaque, int n, int level)
     sclp_service_interrupt(0);
 }
 
+static const VMStateDescription vmstate_sclpconsole = {
+    .name = "sclpconsole",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .minimum_version_id_old = 0,
+    .fields      = (VMStateField[]) {
+        VMSTATE_BOOL(event.event_pending, SCLPConsole),
+        VMSTATE_UINT8_ARRAY(iov, SCLPConsole, SIZE_BUFFER_VT220),
+        VMSTATE_UINT32(iov_sclp, SCLPConsole),
+        VMSTATE_UINT32(iov_bs, SCLPConsole),
+        VMSTATE_UINT32(iov_data_len, SCLPConsole),
+        VMSTATE_UINT32(iov_sclp_rest, SCLPConsole),
+        VMSTATE_END_OF_LIST()
+     }
+};
+
 /* qemu object creation and initialization functions */
 
 /* tell character layer our call-back functions */
+
 static int console_init(SCLPEvent *event)
 {
     static bool console_available;
@@ -237,10 +228,9 @@ static int console_init(SCLPEvent *event)
         return -1;
     }
     console_available = true;
-    event->event_type = SCLP_EVENT_ASCII_CONSOLE_DATA;
     if (scon->chr) {
         qemu_chr_add_handlers(scon->chr, chr_can_read,
-                              chr_read, chr_event, scon);
+                              chr_read, NULL, scon);
     }
     scon->irq_read_vt220 = *qemu_allocate_irqs(trigger_ascii_console_data,
                                                NULL, 1);
@@ -248,6 +238,18 @@ static int console_init(SCLPEvent *event)
     return 0;
 }
 
+static void console_reset(DeviceState *dev)
+{
+   SCLPEvent *event = SCLP_EVENT(dev);
+   SCLPConsole *scon = DO_UPCAST(SCLPConsole, event, event);
+
+   event->event_pending = false;
+   scon->iov_sclp = 0;
+   scon->iov_bs = 0;
+   scon->iov_data_len = 0;
+   scon->iov_sclp_rest = 0;
+}
+
 static int console_exit(SCLPEvent *event)
 {
     return 0;
@@ -264,11 +266,13 @@ static void console_class_init(ObjectClass *klass, void *data)
     SCLPEventClass *ec = SCLP_EVENT_CLASS(klass);
 
     dc->props = console_properties;
+    dc->reset = console_reset;
+    dc->vmsd = &vmstate_sclpconsole;
     ec->init = console_init;
     ec->exit = console_exit;
     ec->get_send_mask = send_mask;
     ec->get_receive_mask = receive_mask;
-    ec->event_type = event_type;
+    ec->can_handle_event = can_handle_event;
     ec->read_event_data = read_event_data;
     ec->write_event_data = write_event_data;
 }
diff --git a/hw/char/sh_serial.c b/hw/char/sh_serial.c
index 6223a557b2..9328dd1b57 100644
--- a/hw/char/sh_serial.c
+++ b/hw/char/sh_serial.c
@@ -248,11 +248,9 @@ static uint64_t sh_serial_read(void *opaque, hwaddr offs,
                     s->flags &= ~SH_SERIAL_FLAG_RDF;
             }
             break;
-#if 0
         case 0x18:
             ret = s->fcr;
             break;
-#endif
         case 0x1c:
             ret = s->rx_cnt;
             break;
diff --git a/hw/display/Makefile.objs b/hw/display/Makefile.objs
index 6e9fb3b695..540df82600 100644
--- a/hw/display/Makefile.objs
+++ b/hw/display/Makefile.objs
@@ -31,5 +31,4 @@ obj-$(CONFIG_TCX) += tcx.o
 
 obj-$(CONFIG_VGA) += vga.o
 
-common-obj-$(CONFIG_QXL) += qxl-logger.o qxl-render.o
-obj-$(CONFIG_QXL) += qxl.o
+common-obj-$(CONFIG_QXL) += qxl.o qxl-logger.o qxl-render.o
diff --git a/hw/display/cirrus_vga.c b/hw/display/cirrus_vga.c
index dbd1f4a47b..e4c345fa82 100644
--- a/hw/display/cirrus_vga.c
+++ b/hw/display/cirrus_vga.c
@@ -2447,7 +2447,6 @@ static uint64_t cirrus_vga_ioport_read(void *opaque, hwaddr addr,
     VGACommonState *s = &c->vga;
     int val, index;
 
-    qemu_flush_coalesced_mmio_buffer();
     addr += 0x3b0;
 
     if (vga_ioport_invalid(s, addr)) {
@@ -2544,7 +2543,6 @@ static void cirrus_vga_ioport_write(void *opaque, hwaddr addr, uint64_t val,
     VGACommonState *s = &c->vga;
     int index;
 
-    qemu_flush_coalesced_mmio_buffer();
     addr += 0x3b0;
 
     /* check port range access depending on color/monochrome mode */
@@ -2843,6 +2841,7 @@ static void cirrus_init_common(CirrusVGAState *s, Object *owner,
     /* Register ioport 0x3b0 - 0x3df */
     memory_region_init_io(&s->cirrus_vga_io, owner, &cirrus_vga_io_ops, s,
                           "cirrus-io", 0x30);
+    memory_region_set_flush_coalesced(&s->cirrus_vga_io);
     memory_region_add_subregion(system_io, 0x3b0, &s->cirrus_vga_io);
 
     memory_region_init(&s->low_mem_container, owner,
diff --git a/hw/display/qxl.c b/hw/display/qxl.c
index 0e2231ce89..5977d526d4 100644
--- a/hw/display/qxl.c
+++ b/hw/display/qxl.c
@@ -162,7 +162,7 @@ void qxl_spice_update_area(PCIQXLDevice *qxl, uint32_t surface_id,
     trace_qxl_spice_update_area_rest(qxl->id, num_dirty_rects,
                                      clear_dirty_region);
     if (async == QXL_SYNC) {
-        qxl->ssd.worker->update_area(qxl->ssd.worker, surface_id, area,
+        spice_qxl_update_area(&qxl->ssd.qxl, surface_id, area,
                         dirty_rects, num_dirty_rects, clear_dirty_region);
     } else {
         assert(cookie != NULL);
@@ -193,7 +193,7 @@ static void qxl_spice_destroy_surface_wait(PCIQXLDevice *qxl, uint32_t id,
         cookie->u.surface_id = id;
         spice_qxl_destroy_surface_async(&qxl->ssd.qxl, id, (uintptr_t)cookie);
     } else {
-        qxl->ssd.worker->destroy_surface_wait(qxl->ssd.worker, id);
+        spice_qxl_destroy_surface_wait(&qxl->ssd.qxl, id);
         qxl_spice_destroy_surface_wait_complete(qxl, id);
     }
 }
@@ -211,19 +211,19 @@ void qxl_spice_loadvm_commands(PCIQXLDevice *qxl, struct QXLCommandExt *ext,
                                uint32_t count)
 {
     trace_qxl_spice_loadvm_commands(qxl->id, ext, count);
-    qxl->ssd.worker->loadvm_commands(qxl->ssd.worker, ext, count);
+    spice_qxl_loadvm_commands(&qxl->ssd.qxl, ext, count);
 }
 
 void qxl_spice_oom(PCIQXLDevice *qxl)
 {
     trace_qxl_spice_oom(qxl->id);
-    qxl->ssd.worker->oom(qxl->ssd.worker);
+    spice_qxl_oom(&qxl->ssd.qxl);
 }
 
 void qxl_spice_reset_memslots(PCIQXLDevice *qxl)
 {
     trace_qxl_spice_reset_memslots(qxl->id);
-    qxl->ssd.worker->reset_memslots(qxl->ssd.worker);
+    spice_qxl_reset_memslots(&qxl->ssd.qxl);
 }
 
 static void qxl_spice_destroy_surfaces_complete(PCIQXLDevice *qxl)
@@ -244,7 +244,7 @@ static void qxl_spice_destroy_surfaces(PCIQXLDevice *qxl, qxl_async_io async)
                 (uintptr_t)qxl_cookie_new(QXL_COOKIE_TYPE_IO,
                                           QXL_IO_DESTROY_ALL_SURFACES_ASYNC));
     } else {
-        qxl->ssd.worker->destroy_surfaces(qxl->ssd.worker);
+        spice_qxl_destroy_surfaces(&qxl->ssd.qxl);
         qxl_spice_destroy_surfaces_complete(qxl);
     }
 }
@@ -278,13 +278,13 @@ static void qxl_spice_monitors_config_async(PCIQXLDevice *qxl, int replay)
 void qxl_spice_reset_image_cache(PCIQXLDevice *qxl)
 {
     trace_qxl_spice_reset_image_cache(qxl->id);
-    qxl->ssd.worker->reset_image_cache(qxl->ssd.worker);
+    spice_qxl_reset_image_cache(&qxl->ssd.qxl);
 }
 
 void qxl_spice_reset_cursor(PCIQXLDevice *qxl)
 {
     trace_qxl_spice_reset_cursor(qxl->id);
-    qxl->ssd.worker->reset_cursor(qxl->ssd.worker);
+    spice_qxl_reset_cursor(&qxl->ssd.qxl);
     qemu_mutex_lock(&qxl->track_lock);
     qxl->guest_cursor = 0;
     qemu_mutex_unlock(&qxl->track_lock);
@@ -313,9 +313,7 @@ static ram_addr_t qxl_rom_size(void)
                                  sizeof(qxl_modes);
     uint32_t rom_size = 8192; /* two pages */
 
-    required_rom_size = MAX(required_rom_size, TARGET_PAGE_SIZE);
-    required_rom_size = msb_mask(required_rom_size * 2 - 1);
-    assert(required_rom_size <= rom_size);
+    QEMU_BUILD_BUG_ON(required_rom_size > rom_size);
     return rom_size;
 }
 
@@ -364,7 +362,7 @@ static void init_qxl_rom(PCIQXLDevice *d)
     num_pages          = d->vga.vram_size;
     num_pages         -= ram_header_size;
     num_pages         -= surface0_area_size;
-    num_pages          = num_pages / TARGET_PAGE_SIZE;
+    num_pages          = num_pages / QXL_PAGE_SIZE;
 
     rom->draw_area_offset   = cpu_to_le32(0);
     rom->surface0_area_size = cpu_to_le32(surface0_area_size);
@@ -416,9 +414,8 @@ static void qxl_ram_set_dirty(PCIQXLDevice *qxl, void *ptr)
     intptr_t offset;
 
     offset = ptr - base;
-    offset &= ~(TARGET_PAGE_SIZE-1);
     assert(offset < qxl->vga.vram_size);
-    qxl_set_dirty(&qxl->vga.vram, offset, offset + TARGET_PAGE_SIZE);
+    qxl_set_dirty(&qxl->vga.vram, offset, offset + 3);
 }
 
 /* can be called from spice server thread context */
@@ -528,7 +525,8 @@ static void interface_get_init_info(QXLInstance *sin, QXLDevInitInfo *info)
     info->num_memslots = NUM_MEMSLOTS;
     info->num_memslots_groups = NUM_MEMSLOTS_GROUPS;
     info->internal_groupslot_id = 0;
-    info->qxl_ram_size = le32_to_cpu(qxl->shadow_rom.num_pages) << TARGET_PAGE_BITS;
+    info->qxl_ram_size =
+        le32_to_cpu(qxl->shadow_rom.num_pages) << QXL_PAGE_BITS;
     info->n_surfaces = qxl->ssd.num_surfaces;
 }
 
@@ -2039,8 +2037,7 @@ static int qxl_init_common(PCIQXLDevice *qxl)
            qxl->vram32_size < qxl->vram_size ? "[region 4]" : "[unmapped]");
 
     qxl->ssd.qxl.base.sif = &qxl_interface.base;
-    qxl->ssd.qxl.id = qxl->id;
-    if (qemu_spice_add_interface(&qxl->ssd.qxl.base) != 0) {
+    if (qemu_spice_add_display_interface(&qxl->ssd.qxl, qxl->vga.con) != 0) {
         error_report("qxl interface %d.%d not supported by spice-server",
                      SPICE_INTERFACE_QXL_MAJOR, SPICE_INTERFACE_QXL_MINOR);
         return -1;
@@ -2076,6 +2073,7 @@ static int qxl_init_primary(PCIDevice *dev)
              pci_address_space(dev), pci_address_space_io(dev), false);
     portio_list_init(qxl_vga_port_list, OBJECT(dev), qxl_vga_portio_list,
                      vga, "vga");
+    portio_list_set_flush_coalesced(qxl_vga_port_list);
     portio_list_add(qxl_vga_port_list, pci_address_space_io(dev), 0x3b0);
 
     vga->con = graphic_console_init(DEVICE(dev), &qxl_ops, qxl);
diff --git a/hw/display/qxl.h b/hw/display/qxl.h
index 8e9b0c299e..84f0182383 100644
--- a/hw/display/qxl.h
+++ b/hw/display/qxl.h
@@ -27,6 +27,9 @@ enum qxl_mode {
 
 #define QXL_NUM_DIRTY_RECTS 64
 
+#define QXL_PAGE_BITS 12
+#define QXL_PAGE_SIZE (1 << QXL_PAGE_BITS);
+
 typedef struct PCIQXLDevice {
     PCIDevice          pci;
     SimpleSpiceDisplay ssd;
diff --git a/hw/display/vga.c b/hw/display/vga.c
index 7b91d9c54e..b5e22849ab 100644
--- a/hw/display/vga.c
+++ b/hw/display/vga.c
@@ -359,8 +359,6 @@ uint32_t vga_ioport_read(void *opaque, uint32_t addr)
     VGACommonState *s = opaque;
     int val, index;
 
-    qemu_flush_coalesced_mmio_buffer();
-
     if (vga_ioport_invalid(s, addr)) {
         val = 0xff;
     } else {
@@ -453,8 +451,6 @@ void vga_ioport_write(void *opaque, uint32_t addr, uint32_t val)
     VGACommonState *s = opaque;
     int index;
 
-    qemu_flush_coalesced_mmio_buffer();
-
     /* check port range access depending on color/monochrome mode */
     if (vga_ioport_invalid(s, addr)) {
         return;
@@ -2373,6 +2369,7 @@ void vga_init(VGACommonState *s, Object *obj, MemoryRegion *address_space,
     memory_region_set_coalescing(vga_io_memory);
     if (init_vga_ports) {
         portio_list_init(vga_port_list, obj, vga_ports, s, "vga");
+        portio_list_set_flush_coalesced(vga_port_list);
         portio_list_add(vga_port_list, address_space_io, 0x3b0);
     }
     if (vbe_ports) {
diff --git a/hw/i386/kvm/clock.c b/hw/i386/kvm/clock.c
index 92aabb83b5..383938d1bc 100644
--- a/hw/i386/kvm/clock.c
+++ b/hw/i386/kvm/clock.c
@@ -39,7 +39,7 @@ static void kvmclock_vm_state_change(void *opaque, int running,
                                      RunState state)
 {
     KVMClockState *s = opaque;
-    CPUState *cpu = first_cpu;
+    CPUState *cpu;
     int cap_clock_ctrl = kvm_check_extension(kvm_state, KVM_CAP_KVMCLOCK_CTRL);
     int ret;
 
diff --git a/hw/i386/kvmvapic.c b/hw/i386/kvmvapic.c
index d3a6fbe1f9..2d876009fc 100644
--- a/hw/i386/kvmvapic.c
+++ b/hw/i386/kvmvapic.c
@@ -510,9 +510,8 @@ static void vapic_reset(DeviceState *dev)
 {
     VAPICROMState *s = VAPIC(dev);
 
-    if (s->state == VAPIC_ACTIVE) {
-        s->state = VAPIC_STANDBY;
-    }
+    s->state = VAPIC_INACTIVE;
+    s->rom_state_paddr = 0;
     vapic_enable_tpr_reporting(false);
 }
 
@@ -578,7 +577,7 @@ static int patch_hypercalls(VAPICROMState *s)
  * enable write access to the option ROM so that variables can be updated by
  * the guest.
  */
-static void vapic_map_rom_writable(VAPICROMState *s)
+static int vapic_map_rom_writable(VAPICROMState *s)
 {
     hwaddr rom_paddr = s->rom_state_paddr & ROM_BLOCK_MASK;
     MemoryRegionSection section;
@@ -597,8 +596,14 @@ static void vapic_map_rom_writable(VAPICROMState *s)
     section = memory_region_find(as, 0, 1);
 
     /* read ROM size from RAM region */
+    if (rom_paddr + 2 >= memory_region_size(section.mr)) {
+        return -1;
+    }
     ram = memory_region_get_ram_ptr(section.mr);
     rom_size = ram[rom_paddr + 2] * ROM_BLOCK_SIZE;
+    if (rom_size == 0) {
+        return -1;
+    }
     s->rom_size = rom_size;
 
     /* We need to round to avoid creating subpages
@@ -612,11 +617,15 @@ static void vapic_map_rom_writable(VAPICROMState *s)
     memory_region_add_subregion_overlap(as, rom_paddr, &s->rom, 1000);
     s->rom_mapped_writable = true;
     memory_region_unref(section.mr);
+
+    return 0;
 }
 
 static int vapic_prepare(VAPICROMState *s)
 {
-    vapic_map_rom_writable(s);
+    if (vapic_map_rom_writable(s) < 0) {
+        return -1;
+    }
 
     if (patch_hypercalls(s) < 0) {
         return -1;
@@ -659,6 +668,7 @@ static void vapic_write(void *opaque, hwaddr addr, uint64_t data,
         }
         if (vapic_prepare(s) < 0) {
             s->state = VAPIC_INACTIVE;
+            s->rom_state_paddr = 0;
             break;
         }
         break;
diff --git a/hw/i386/pc_piix.c b/hw/i386/pc_piix.c
index 63ae2ae859..24a98cbee7 100644
--- a/hw/i386/pc_piix.c
+++ b/hw/i386/pc_piix.c
@@ -746,6 +746,7 @@ static QEMUMachine xenfv_machine = {
     .init = pc_xen_hvm_init,
     .max_cpus = HVM_MAX_VCPUS,
     .default_machine_opts = "accel=xen",
+    .hot_add_cpu = pc_hot_add_cpu,
 };
 #endif
 
diff --git a/hw/i386/pc_sysfw.c b/hw/i386/pc_sysfw.c
index 8246a1bdd4..e917c83540 100644
--- a/hw/i386/pc_sysfw.c
+++ b/hw/i386/pc_sysfw.c
@@ -53,10 +53,7 @@ static void pc_isa_bios_init(MemoryRegion *rom_memory,
     flash_size = memory_region_size(flash_mem);
 
     /* map the last 128KB of the BIOS in ISA space */
-    isa_bios_size = flash_size;
-    if (isa_bios_size > (128 * 1024)) {
-        isa_bios_size = 128 * 1024;
-    }
+    isa_bios_size = MIN(flash_size, 128 * 1024);
     isa_bios = g_malloc(sizeof(*isa_bios));
     memory_region_init_ram(isa_bios, NULL, "isa-bios", isa_bios_size);
     vmstate_register_ram_global(isa_bios);
diff --git a/hw/ide/ahci.c b/hw/ide/ahci.c
index a71a4ca47c..a8be62cf99 100644
--- a/hw/ide/ahci.c
+++ b/hw/ide/ahci.c
@@ -1198,7 +1198,15 @@ void ahci_reset(AHCIState *s)
     int i;
 
     s->control_regs.irqstatus = 0;
-    s->control_regs.ghc = 0;
+    /* AHCI Enable (AE)
+     * The implementation of this bit is dependent upon the value of the
+     * CAP.SAM bit. If CAP.SAM is '0', then GHC.AE shall be read-write and
+     * shall have a reset value of '0'. If CAP.SAM is '1', then AE shall be
+     * read-only and shall have a reset value of '1'.
+     *
+     * We set HOST_CAP_AHCI so we must enable AHCI at reset.
+     */
+    s->control_regs.ghc = HOST_CTL_AHCI_EN;
 
     for (i = 0; i < s->ports; i++) {
         pr = &s->dev[i].port_regs;
diff --git a/hw/microblaze/boot.c b/hw/microblaze/boot.c
index 5b057f7880..2a7ea5c0f9 100644
--- a/hw/microblaze/boot.c
+++ b/hw/microblaze/boot.c
@@ -26,6 +26,7 @@
 
 #include "qemu/option.h"
 #include "qemu/config-file.h"
+#include "qemu/error-report.h"
 #include "qemu-common.h"
 #include "sysemu/device_tree.h"
 #include "sysemu/sysemu.h"
@@ -39,6 +40,8 @@ static struct
     void (*machine_cpu_reset)(MicroBlazeCPU *);
     uint32_t bootstrap_pc;
     uint32_t cmdline;
+    uint32_t initrd_start;
+    uint32_t initrd_end;
     uint32_t fdt;
 } boot_info;
 
@@ -49,6 +52,7 @@ static void main_cpu_reset(void *opaque)
 
     cpu_reset(CPU(cpu));
     env->regs[5] = boot_info.cmdline;
+    env->regs[6] = boot_info.initrd_start;
     env->regs[7] = boot_info.fdt;
     env->sregs[SR_PC] = boot_info.bootstrap_pc;
     if (boot_info.machine_cpu_reset) {
@@ -57,9 +61,11 @@ static void main_cpu_reset(void *opaque)
 }
 
 static int microblaze_load_dtb(hwaddr addr,
-                                      uint32_t ramsize,
-                                      const char *kernel_cmdline,
-                                      const char *dtb_filename)
+                               uint32_t ramsize,
+                               uint32_t initrd_start,
+                               uint32_t initrd_end,
+                               const char *kernel_cmdline,
+                               const char *dtb_filename)
 {
     int fdt_size;
     void *fdt = NULL;
@@ -80,6 +86,14 @@ static int microblaze_load_dtb(hwaddr addr,
         }
     }
 
+    if (initrd_start) {
+        qemu_devtree_setprop_cell(fdt, "/chosen", "linux,initrd-start",
+                                  initrd_start);
+
+        qemu_devtree_setprop_cell(fdt, "/chosen", "linux,initrd-end",
+                                  initrd_end);
+    }
+
     cpu_physical_memory_write(addr, fdt, fdt_size);
     return fdt_size;
 }
@@ -90,7 +104,9 @@ static uint64_t translate_kernel_address(void *opaque, uint64_t addr)
 }
 
 void microblaze_load_kernel(MicroBlazeCPU *cpu, hwaddr ddr_base,
-                            uint32_t ramsize, const char *dtb_filename,
+                            uint32_t ramsize,
+                            const char *initrd_filename,
+                            const char *dtb_filename,
                             void (*machine_cpu_reset)(MicroBlazeCPU *))
 {
     QemuOpts *machine_opts;
@@ -151,14 +167,36 @@ void microblaze_load_kernel(MicroBlazeCPU *cpu, hwaddr ddr_base,
             high = (ddr_base + kernel_size + 3) & ~3;
         }
 
+        if (initrd_filename) {
+            int initrd_size;
+            uint32_t initrd_offset;
+
+            high = ROUND_UP(high + kernel_size, 4);
+            boot_info.initrd_start = high;
+            initrd_offset = boot_info.initrd_start - ddr_base;
+            initrd_size = load_image_targphys(initrd_filename,
+                                              boot_info.initrd_start,
+                                              ram_size - initrd_offset);
+            if (initrd_size < 0) {
+                error_report("qemu: could not load initrd '%s'\n",
+                             initrd_filename);
+                exit(EXIT_FAILURE);
+            }
+            boot_info.initrd_end = boot_info.initrd_start + initrd_size;
+            high = ROUND_UP(high + initrd_size, 4);
+        }
+
         boot_info.cmdline = high + 4096;
         if (kernel_cmdline && strlen(kernel_cmdline)) {
             pstrcpy_targphys("cmdline", boot_info.cmdline, 256, kernel_cmdline);
         }
         /* Provide a device-tree.  */
         boot_info.fdt = boot_info.cmdline + 4096;
-        microblaze_load_dtb(boot_info.fdt, ram_size, kernel_cmdline,
-                                                     dtb_filename);
+        microblaze_load_dtb(boot_info.fdt, ram_size,
+                            boot_info.initrd_start,
+                            boot_info.initrd_end,
+                            kernel_cmdline,
+                            dtb_filename);
     }
 
 }
diff --git a/hw/microblaze/boot.h b/hw/microblaze/boot.h
index b14ef2b992..0eb7f8e4f6 100644
--- a/hw/microblaze/boot.h
+++ b/hw/microblaze/boot.h
@@ -4,7 +4,9 @@
 #include "hw/hw.h"
 
 void microblaze_load_kernel(MicroBlazeCPU *cpu, hwaddr ddr_base,
-                            uint32_t ramsize, const char *dtb_filename,
+                            uint32_t ramsize,
+                            const char *initrd_filename,
+                            const char *dtb_filename,
                             void (*machine_cpu_reset)(MicroBlazeCPU *));
 
 #endif /* __MICROBLAZE_BOOT __ */
diff --git a/hw/microblaze/petalogix_ml605_mmu.c b/hw/microblaze/petalogix_ml605_mmu.c
index e003c7c7b4..10970e0f3f 100644
--- a/hw/microblaze/petalogix_ml605_mmu.c
+++ b/hw/microblaze/petalogix_ml605_mmu.c
@@ -176,8 +176,10 @@ petalogix_ml605_init(QEMUMachineInitArgs *args)
         }
     }
 
-    microblaze_load_kernel(cpu, ddr_base, ram_size, BINARY_DEVICE_TREE_FILE,
-                                                            machine_cpu_reset);
+    microblaze_load_kernel(cpu, ddr_base, ram_size,
+                           args->initrd_filename,
+                           BINARY_DEVICE_TREE_FILE,
+                           machine_cpu_reset);
 
 }
 
diff --git a/hw/microblaze/petalogix_s3adsp1800_mmu.c b/hw/microblaze/petalogix_s3adsp1800_mmu.c
index 00af2b5abc..ec6489c2d3 100644
--- a/hw/microblaze/petalogix_s3adsp1800_mmu.c
+++ b/hw/microblaze/petalogix_s3adsp1800_mmu.c
@@ -108,7 +108,9 @@ petalogix_s3adsp1800_init(QEMUMachineInitArgs *args)
     xilinx_ethlite_create(&nd_table[0], ETHLITE_BASEADDR, irq[1], 0, 0);
 
     microblaze_load_kernel(cpu, ddr_base, ram_size,
-                    BINARY_DEVICE_TREE_FILE, machine_cpu_reset);
+                           args->initrd_filename,
+                           BINARY_DEVICE_TREE_FILE,
+                           machine_cpu_reset);
 }
 
 static QEMUMachine petalogix_s3adsp1800_machine = {
diff --git a/hw/misc/Makefile.objs b/hw/misc/Makefile.objs
index 2578e2939d..cca5c058cc 100644
--- a/hw/misc/Makefile.objs
+++ b/hw/misc/Makefile.objs
@@ -10,6 +10,7 @@ obj-$(CONFIG_VMPORT) += vmport.o
 
 # ARM devices
 common-obj-$(CONFIG_PL310) += arm_l2x0.o
+common-obj-$(CONFIG_INTEGRATOR_DEBUG) += arm_integrator_debug.o
 
 # PKUnity SoC devices
 common-obj-$(CONFIG_PUV3) += puv3_pm.o
diff --git a/hw/misc/arm_integrator_debug.c b/hw/misc/arm_integrator_debug.c
new file mode 100644
index 0000000000..99b720fbb9
--- /dev/null
+++ b/hw/misc/arm_integrator_debug.c
@@ -0,0 +1,99 @@
+/*
+ * LED, Switch and Debug control registers for ARM Integrator Boards
+ *
+ * This is currently a stub for this functionality but at least
+ * ensures something other than unassigned_mem_read() handles access
+ * to this area.
+ *
+ * The real h/w is described at:
+ *  http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.dui0159b/Babbfijf.html
+ *
+ * Copyright (c) 2013 Alex Bennée <alex@bennee.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "hw/hw.h"
+#include "hw/sysbus.h"
+#include "exec/address-spaces.h"
+#include "hw/misc/arm_integrator_debug.h"
+
+#define INTEGRATOR_DEBUG(obj) \
+    OBJECT_CHECK(IntegratorDebugState, (obj), TYPE_INTEGRATOR_DEBUG)
+
+typedef struct {
+    SysBusDevice parent_obj;
+
+    MemoryRegion iomem;
+} IntegratorDebugState;
+
+static uint64_t intdbg_control_read(void *opaque, hwaddr offset,
+                                    unsigned size)
+{
+    switch (offset >> 2) {
+    case 0: /* ALPHA */
+    case 1: /* LEDS */
+    case 2: /* SWITCHES */
+        qemu_log_mask(LOG_UNIMP,
+                      "%s: returning zero from %" HWADDR_PRIx ":%u\n",
+                      __func__, offset, size);
+        return 0;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Bad offset %" HWADDR_PRIx,
+                      __func__, offset);
+        return 0;
+    }
+}
+
+static void intdbg_control_write(void *opaque, hwaddr offset,
+                                 uint64_t value, unsigned size)
+{
+    switch (offset >> 2) {
+    case 1: /* ALPHA */
+    case 2: /* LEDS */
+    case 3: /* SWITCHES */
+        /* Nothing interesting implemented yet.  */
+        qemu_log_mask(LOG_UNIMP,
+                      "%s: ignoring write of %" PRIu64
+                      " to %" HWADDR_PRIx ":%u\n",
+                      __func__, value, offset, size);
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: write of %" PRIu64
+                      " to bad offset %" HWADDR_PRIx "\n",
+                      __func__, value, offset);
+    }
+}
+
+static const MemoryRegionOps intdbg_control_ops = {
+    .read = intdbg_control_read,
+    .write = intdbg_control_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static void intdbg_control_init(Object *obj)
+{
+    SysBusDevice *sd = SYS_BUS_DEVICE(obj);
+    IntegratorDebugState *s = INTEGRATOR_DEBUG(obj);
+
+    memory_region_init_io(&s->iomem, NULL, &intdbg_control_ops,
+                          NULL, "dbg-leds", 0x1000000);
+    sysbus_init_mmio(sd, &s->iomem);
+}
+
+static const TypeInfo intdbg_info = {
+    .name          = TYPE_INTEGRATOR_DEBUG,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(IntegratorDebugState),
+    .instance_init = intdbg_control_init,
+};
+
+static void intdbg_register_types(void)
+{
+    type_register_static(&intdbg_info);
+}
+
+type_init(intdbg_register_types)
diff --git a/hw/misc/vfio.c b/hw/misc/vfio.c
index 9d02e49a3e..fe95e03d1d 100644
--- a/hw/misc/vfio.c
+++ b/hw/misc/vfio.c
@@ -119,6 +119,7 @@ typedef struct VFIOINTx {
 typedef struct VFIOMSIVector {
     EventNotifier interrupt; /* eventfd triggered on interrupt */
     struct VFIODevice *vdev; /* back pointer to device */
+    MSIMessage msg; /* cache the MSI message so we know when it changes */
     int virq; /* KVM irqchip route for QEMU bypass */
     bool use;
 } VFIOMSIVector;
@@ -165,6 +166,7 @@ typedef struct VFIODevice {
     off_t config_offset; /* Offset of config space region within device fd */
     unsigned int rom_size;
     off_t rom_offset; /* Offset of ROM region within device fd */
+    void *rom;
     int msi_cap_size;
     VFIOMSIVector *msi_vectors;
     VFIOMSIXInfo *msix;
@@ -184,6 +186,9 @@ typedef struct VFIODevice {
     bool reset_works;
     bool has_vga;
     bool pci_aer;
+    bool has_flr;
+    bool has_pm_reset;
+    bool needs_reset;
 } VFIODevice;
 
 typedef struct VFIOGroup {
@@ -796,7 +801,6 @@ retry:
     vdev->msi_vectors = g_malloc0(vdev->nr_vectors * sizeof(VFIOMSIVector));
 
     for (i = 0; i < vdev->nr_vectors; i++) {
-        MSIMessage msg;
         VFIOMSIVector *vector = &vdev->msi_vectors[i];
 
         vector->vdev = vdev;
@@ -806,13 +810,13 @@ retry:
             error_report("vfio: Error: event_notifier_init failed");
         }
 
-        msg = msi_get_message(&vdev->pdev, i);
+        vector->msg = msi_get_message(&vdev->pdev, i);
 
         /*
          * Attempt to enable route through KVM irqchip,
          * default to userspace handling if unavailable.
          */
-        vector->virq = kvm_irqchip_add_msi_route(kvm_state, msg);
+        vector->virq = kvm_irqchip_add_msi_route(kvm_state, vector->msg);
         if (vector->virq < 0 ||
             kvm_irqchip_add_irqfd_notifier(kvm_state, &vector->interrupt,
                                            NULL, vector->virq) < 0) {
@@ -918,6 +922,33 @@ static void vfio_disable_msi(VFIODevice *vdev)
             vdev->host.bus, vdev->host.slot, vdev->host.function);
 }
 
+static void vfio_update_msi(VFIODevice *vdev)
+{
+    int i;
+
+    for (i = 0; i < vdev->nr_vectors; i++) {
+        VFIOMSIVector *vector = &vdev->msi_vectors[i];
+        MSIMessage msg;
+
+        if (!vector->use || vector->virq < 0) {
+            continue;
+        }
+
+        msg = msi_get_message(&vdev->pdev, i);
+
+        if (msg.address != vector->msg.address ||
+            msg.data != vector->msg.data) {
+
+            DPRINTF("%s(%04x:%02x:%02x.%x) MSI vector %d changed\n",
+                    __func__, vdev->host.domain, vdev->host.bus,
+                    vdev->host.slot, vdev->host.function, i);
+
+            kvm_irqchip_update_msi_route(kvm_state, vector->virq, msg);
+            vector->msg = msg;
+        }
+    }
+}
+
 /*
  * IO Port/MMIO - Beware of the endians, VFIO is always little endian
  */
@@ -1030,6 +1061,131 @@ static const MemoryRegionOps vfio_bar_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
+static void vfio_pci_load_rom(VFIODevice *vdev)
+{
+    struct vfio_region_info reg_info = {
+        .argsz = sizeof(reg_info),
+        .index = VFIO_PCI_ROM_REGION_INDEX
+    };
+    uint64_t size;
+    off_t off = 0;
+    size_t bytes;
+
+    if (ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info)) {
+        error_report("vfio: Error getting ROM info: %m");
+        return;
+    }
+
+    DPRINTF("Device %04x:%02x:%02x.%x ROM:\n", vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function);
+    DPRINTF("  size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
+            (unsigned long)reg_info.size, (unsigned long)reg_info.offset,
+            (unsigned long)reg_info.flags);
+
+    vdev->rom_size = size = reg_info.size;
+    vdev->rom_offset = reg_info.offset;
+
+    if (!vdev->rom_size) {
+        return;
+    }
+
+    vdev->rom = g_malloc(size);
+    memset(vdev->rom, 0xff, size);
+
+    while (size) {
+        bytes = pread(vdev->fd, vdev->rom + off, size, vdev->rom_offset + off);
+        if (bytes == 0) {
+            break;
+        } else if (bytes > 0) {
+            off += bytes;
+            size -= bytes;
+        } else {
+            if (errno == EINTR || errno == EAGAIN) {
+                continue;
+            }
+            error_report("vfio: Error reading device ROM: %m");
+            break;
+        }
+    }
+}
+
+static uint64_t vfio_rom_read(void *opaque, hwaddr addr, unsigned size)
+{
+    VFIODevice *vdev = opaque;
+    uint64_t val = ((uint64_t)1 << (size * 8)) - 1;
+
+    /* Load the ROM lazily when the guest tries to read it */
+    if (unlikely(!vdev->rom)) {
+        vfio_pci_load_rom(vdev);
+    }
+
+    memcpy(&val, vdev->rom + addr,
+           (addr < vdev->rom_size) ? MIN(size, vdev->rom_size - addr) : 0);
+
+    DPRINTF("%s(%04x:%02x:%02x.%x, 0x%"HWADDR_PRIx", 0x%x) = 0x%"PRIx64"\n",
+            __func__, vdev->host.domain, vdev->host.bus, vdev->host.slot,
+            vdev->host.function, addr, size, val);
+
+    return val;
+}
+
+static void vfio_rom_write(void *opaque, hwaddr addr,
+                           uint64_t data, unsigned size)
+{
+}
+
+static const MemoryRegionOps vfio_rom_ops = {
+    .read = vfio_rom_read,
+    .write = vfio_rom_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void vfio_pci_size_rom(VFIODevice *vdev)
+{
+    uint32_t orig, size = cpu_to_le32((uint32_t)PCI_ROM_ADDRESS_MASK);
+    off_t offset = vdev->config_offset + PCI_ROM_ADDRESS;
+    char name[32];
+
+    if (vdev->pdev.romfile || !vdev->pdev.rom_bar) {
+        return;
+    }
+
+    /*
+     * Use the same size ROM BAR as the physical device.  The contents
+     * will get filled in later when the guest tries to read it.
+     */
+    if (pread(vdev->fd, &orig, 4, offset) != 4 ||
+        pwrite(vdev->fd, &size, 4, offset) != 4 ||
+        pread(vdev->fd, &size, 4, offset) != 4 ||
+        pwrite(vdev->fd, &orig, 4, offset) != 4) {
+        error_report("%s(%04x:%02x:%02x.%x) failed: %m",
+                     __func__, vdev->host.domain, vdev->host.bus,
+                     vdev->host.slot, vdev->host.function);
+        return;
+    }
+
+    size = ~(le32_to_cpu(size) & PCI_ROM_ADDRESS_MASK) + 1;
+
+    if (!size) {
+        return;
+    }
+
+    DPRINTF("%04x:%02x:%02x.%x ROM size 0x%x\n", vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function, size);
+
+    snprintf(name, sizeof(name), "vfio[%04x:%02x:%02x.%x].rom",
+             vdev->host.domain, vdev->host.bus, vdev->host.slot,
+             vdev->host.function);
+
+    memory_region_init_io(&vdev->pdev.rom, OBJECT(vdev),
+                          &vfio_rom_ops, vdev, name, size);
+
+    pci_register_bar(&vdev->pdev, PCI_ROM_SLOT,
+                     PCI_BASE_ADDRESS_SPACE_MEMORY, &vdev->pdev.rom);
+
+    vdev->pdev.has_rom = true;
+}
+
 static void vfio_vga_write(void *opaque, hwaddr addr,
                            uint64_t data, unsigned size)
 {
@@ -1835,10 +1991,16 @@ static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr,
 
         is_enabled = msi_enabled(pdev);
 
-        if (!was_enabled && is_enabled) {
-            vfio_enable_msi(vdev);
-        } else if (was_enabled && !is_enabled) {
-            vfio_disable_msi(vdev);
+        if (!was_enabled) {
+            if (is_enabled) {
+                vfio_enable_msi(vdev);
+            }
+        } else {
+            if (!is_enabled) {
+                vfio_disable_msi(vdev);
+            } else {
+                vfio_update_msi(vdev);
+            }
         }
     } else if (pdev->cap_present & QEMU_PCI_CAP_MSIX &&
         ranges_overlap(addr, len, pdev->msix_cap, MSIX_CAP_LENGTH)) {
@@ -1929,7 +2091,8 @@ static void vfio_listener_region_add(MemoryListener *listener,
     if (vfio_listener_skipped_section(section)) {
         DPRINTF("SKIPPING region_add %"HWADDR_PRIx" - %"PRIx64"\n",
                 section->offset_within_address_space,
-                section->offset_within_address_space + section->size - 1);
+                section->offset_within_address_space +
+                int128_get64(int128_sub(section->size, int128_one())));
         return;
     }
 
@@ -1974,7 +2137,8 @@ static void vfio_listener_region_del(MemoryListener *listener,
     if (vfio_listener_skipped_section(section)) {
         DPRINTF("SKIPPING region_del %"HWADDR_PRIx" - %"PRIx64"\n",
                 section->offset_within_address_space,
-                section->offset_within_address_space + section->size - 1);
+                section->offset_within_address_space +
+                int128_get64(int128_sub(section->size, int128_one())));
         return;
     }
 
@@ -2481,6 +2645,42 @@ static int vfio_setup_pcie_cap(VFIODevice *vdev, int pos, uint8_t size)
     return pos;
 }
 
+static void vfio_check_pcie_flr(VFIODevice *vdev, uint8_t pos)
+{
+    uint32_t cap = pci_get_long(vdev->pdev.config + pos + PCI_EXP_DEVCAP);
+
+    if (cap & PCI_EXP_DEVCAP_FLR) {
+        DPRINTF("%04x:%02x:%02x.%x Supports FLR via PCIe cap\n",
+                vdev->host.domain, vdev->host.bus, vdev->host.slot,
+                vdev->host.function);
+        vdev->has_flr = true;
+    }
+}
+
+static void vfio_check_pm_reset(VFIODevice *vdev, uint8_t pos)
+{
+    uint16_t csr = pci_get_word(vdev->pdev.config + pos + PCI_PM_CTRL);
+
+    if (!(csr & PCI_PM_CTRL_NO_SOFT_RESET)) {
+        DPRINTF("%04x:%02x:%02x.%x Supports PM reset\n",
+                vdev->host.domain, vdev->host.bus, vdev->host.slot,
+                vdev->host.function);
+        vdev->has_pm_reset = true;
+    }
+}
+
+static void vfio_check_af_flr(VFIODevice *vdev, uint8_t pos)
+{
+    uint8_t cap = pci_get_byte(vdev->pdev.config + pos + PCI_AF_CAP);
+
+    if ((cap & PCI_AF_CAP_TP) && (cap & PCI_AF_CAP_FLR)) {
+        DPRINTF("%04x:%02x:%02x.%x Supports FLR via AF cap\n",
+                vdev->host.domain, vdev->host.bus, vdev->host.slot,
+                vdev->host.function);
+        vdev->has_flr = true;
+    }
+}
+
 static int vfio_add_std_cap(VFIODevice *vdev, uint8_t pos)
 {
     PCIDevice *pdev = &vdev->pdev;
@@ -2525,13 +2725,21 @@ static int vfio_add_std_cap(VFIODevice *vdev, uint8_t pos)
         ret = vfio_setup_msi(vdev, pos);
         break;
     case PCI_CAP_ID_EXP:
+        vfio_check_pcie_flr(vdev, pos);
         ret = vfio_setup_pcie_cap(vdev, pos, size);
         break;
     case PCI_CAP_ID_MSIX:
         ret = vfio_setup_msix(vdev, pos);
         break;
     case PCI_CAP_ID_PM:
+        vfio_check_pm_reset(vdev, pos);
         vdev->pm_cap = pos;
+        ret = pci_add_capability(pdev, cap_id, pos, size);
+        break;
+    case PCI_CAP_ID_AF:
+        vfio_check_af_flr(vdev, pos);
+        ret = pci_add_capability(pdev, cap_id, pos, size);
+        break;
     default:
         ret = pci_add_capability(pdev, cap_id, pos, size);
         break;
@@ -2560,49 +2768,277 @@ static int vfio_add_capabilities(VFIODevice *vdev)
     return vfio_add_std_cap(vdev, pdev->config[PCI_CAPABILITY_LIST]);
 }
 
-static int vfio_load_rom(VFIODevice *vdev)
+static void vfio_pci_pre_reset(VFIODevice *vdev)
 {
-    uint64_t size = vdev->rom_size;
-    char name[32];
-    off_t off = 0, voff = vdev->rom_offset;
-    ssize_t bytes;
-    void *ptr;
+    PCIDevice *pdev = &vdev->pdev;
+    uint16_t cmd;
 
-    /* If loading ROM from file, pci handles it */
-    if (vdev->pdev.romfile || !vdev->pdev.rom_bar || !size) {
-        return 0;
+    vfio_disable_interrupts(vdev);
+
+    /* Make sure the device is in D0 */
+    if (vdev->pm_cap) {
+        uint16_t pmcsr;
+        uint8_t state;
+
+        pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
+        state = pmcsr & PCI_PM_CTRL_STATE_MASK;
+        if (state) {
+            pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
+            vfio_pci_write_config(pdev, vdev->pm_cap + PCI_PM_CTRL, pmcsr, 2);
+            /* vfio handles the necessary delay here */
+            pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
+            state = pmcsr & PCI_PM_CTRL_STATE_MASK;
+            if (state) {
+                error_report("vfio: Unable to power on device, stuck in D%d\n",
+                             state);
+            }
+        }
     }
 
-    DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
-            vdev->host.bus, vdev->host.slot, vdev->host.function);
+    /*
+     * Stop any ongoing DMA by disconecting I/O, MMIO, and bus master.
+     * Also put INTx Disable in known state.
+     */
+    cmd = vfio_pci_read_config(pdev, PCI_COMMAND, 2);
+    cmd &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
+             PCI_COMMAND_INTX_DISABLE);
+    vfio_pci_write_config(pdev, PCI_COMMAND, cmd, 2);
+}
 
-    snprintf(name, sizeof(name), "vfio[%04x:%02x:%02x.%x].rom",
-             vdev->host.domain, vdev->host.bus, vdev->host.slot,
-             vdev->host.function);
-    memory_region_init_ram(&vdev->pdev.rom, OBJECT(vdev), name, size);
-    ptr = memory_region_get_ram_ptr(&vdev->pdev.rom);
-    memset(ptr, 0xff, size);
+static void vfio_pci_post_reset(VFIODevice *vdev)
+{
+    vfio_enable_intx(vdev);
+}
 
-    while (size) {
-        bytes = pread(vdev->fd, ptr + off, size, voff + off);
-        if (bytes == 0) {
-            break; /* expect that we could get back less than the ROM BAR */
-        } else if (bytes > 0) {
-            off += bytes;
-            size -= bytes;
-        } else {
-            if (errno == EINTR || errno == EAGAIN) {
-                continue;
+static bool vfio_pci_host_match(PCIHostDeviceAddress *host1,
+                                PCIHostDeviceAddress *host2)
+{
+    return (host1->domain == host2->domain && host1->bus == host2->bus &&
+            host1->slot == host2->slot && host1->function == host2->function);
+}
+
+static int vfio_pci_hot_reset(VFIODevice *vdev, bool single)
+{
+    VFIOGroup *group;
+    struct vfio_pci_hot_reset_info *info;
+    struct vfio_pci_dependent_device *devices;
+    struct vfio_pci_hot_reset *reset;
+    int32_t *fds;
+    int ret, i, count;
+    bool multi = false;
+
+    DPRINTF("%s(%04x:%02x:%02x.%x) %s\n", __func__, vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function,
+            single ? "one" : "multi");
+
+    vfio_pci_pre_reset(vdev);
+    vdev->needs_reset = false;
+
+    info = g_malloc0(sizeof(*info));
+    info->argsz = sizeof(*info);
+
+    ret = ioctl(vdev->fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
+    if (ret && errno != ENOSPC) {
+        ret = -errno;
+        if (!vdev->has_pm_reset) {
+            error_report("vfio: Cannot reset device %04x:%02x:%02x.%x, "
+                         "no available reset mechanism.", vdev->host.domain,
+                         vdev->host.bus, vdev->host.slot, vdev->host.function);
+        }
+        goto out_single;
+    }
+
+    count = info->count;
+    info = g_realloc(info, sizeof(*info) + (count * sizeof(*devices)));
+    info->argsz = sizeof(*info) + (count * sizeof(*devices));
+    devices = &info->devices[0];
+
+    ret = ioctl(vdev->fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
+    if (ret) {
+        ret = -errno;
+        error_report("vfio: hot reset info failed: %m");
+        goto out_single;
+    }
+
+    DPRINTF("%04x:%02x:%02x.%x: hot reset dependent devices:\n",
+            vdev->host.domain, vdev->host.bus, vdev->host.slot,
+            vdev->host.function);
+
+    /* Verify that we have all the groups required */
+    for (i = 0; i < info->count; i++) {
+        PCIHostDeviceAddress host;
+        VFIODevice *tmp;
+
+        host.domain = devices[i].segment;
+        host.bus = devices[i].bus;
+        host.slot = PCI_SLOT(devices[i].devfn);
+        host.function = PCI_FUNC(devices[i].devfn);
+
+        DPRINTF("\t%04x:%02x:%02x.%x group %d\n", host.domain,
+                host.bus, host.slot, host.function, devices[i].group_id);
+
+        if (vfio_pci_host_match(&host, &vdev->host)) {
+            continue;
+        }
+
+        QLIST_FOREACH(group, &group_list, next) {
+            if (group->groupid == devices[i].group_id) {
+                break;
+            }
+        }
+
+        if (!group) {
+            if (!vdev->has_pm_reset) {
+                error_report("vfio: Cannot reset device %04x:%02x:%02x.%x, "
+                             "depends on group %d which is not owned.",
+                             vdev->host.domain, vdev->host.bus, vdev->host.slot,
+                             vdev->host.function, devices[i].group_id);
+            }
+            ret = -EPERM;
+            goto out;
+        }
+
+        /* Prep dependent devices for reset and clear our marker. */
+        QLIST_FOREACH(tmp, &group->device_list, next) {
+            if (vfio_pci_host_match(&host, &tmp->host)) {
+                if (single) {
+                    DPRINTF("vfio: found another in-use device "
+                            "%04x:%02x:%02x.%x\n", host.domain, host.bus,
+                            host.slot, host.function);
+                    ret = -EINVAL;
+                    goto out_single;
+                }
+                vfio_pci_pre_reset(tmp);
+                tmp->needs_reset = false;
+                multi = true;
+                break;
             }
-            error_report("vfio: Error reading device ROM: %m");
-            memory_region_destroy(&vdev->pdev.rom);
-            return -errno;
         }
     }
 
-    pci_register_bar(&vdev->pdev, PCI_ROM_SLOT, 0, &vdev->pdev.rom);
-    vdev->pdev.has_rom = true;
-    return 0;
+    if (!single && !multi) {
+        DPRINTF("vfio: No other in-use devices for multi hot reset\n");
+        ret = -EINVAL;
+        goto out_single;
+    }
+
+    /* Determine how many group fds need to be passed */
+    count = 0;
+    QLIST_FOREACH(group, &group_list, next) {
+        for (i = 0; i < info->count; i++) {
+            if (group->groupid == devices[i].group_id) {
+                count++;
+                break;
+            }
+        }
+    }
+
+    reset = g_malloc0(sizeof(*reset) + (count * sizeof(*fds)));
+    reset->argsz = sizeof(*reset) + (count * sizeof(*fds));
+    fds = &reset->group_fds[0];
+
+    /* Fill in group fds */
+    QLIST_FOREACH(group, &group_list, next) {
+        for (i = 0; i < info->count; i++) {
+            if (group->groupid == devices[i].group_id) {
+                fds[reset->count++] = group->fd;
+                break;
+            }
+        }
+    }
+
+    /* Bus reset! */
+    ret = ioctl(vdev->fd, VFIO_DEVICE_PCI_HOT_RESET, reset);
+    g_free(reset);
+
+    DPRINTF("%04x:%02x:%02x.%x hot reset: %s\n", vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function,
+            ret ? "%m" : "Success");
+
+out:
+    /* Re-enable INTx on affected devices */
+    for (i = 0; i < info->count; i++) {
+        PCIHostDeviceAddress host;
+        VFIODevice *tmp;
+
+        host.domain = devices[i].segment;
+        host.bus = devices[i].bus;
+        host.slot = PCI_SLOT(devices[i].devfn);
+        host.function = PCI_FUNC(devices[i].devfn);
+
+        if (vfio_pci_host_match(&host, &vdev->host)) {
+            continue;
+        }
+
+        QLIST_FOREACH(group, &group_list, next) {
+            if (group->groupid == devices[i].group_id) {
+                break;
+            }
+        }
+
+        if (!group) {
+            break;
+        }
+
+        QLIST_FOREACH(tmp, &group->device_list, next) {
+            if (vfio_pci_host_match(&host, &tmp->host)) {
+                vfio_pci_post_reset(tmp);
+                break;
+            }
+        }
+    }
+out_single:
+    vfio_pci_post_reset(vdev);
+    g_free(info);
+
+    return ret;
+}
+
+/*
+ * We want to differentiate hot reset of mulitple in-use devices vs hot reset
+ * of a single in-use device.  VFIO_DEVICE_RESET will already handle the case
+ * of doing hot resets when there is only a single device per bus.  The in-use
+ * here refers to how many VFIODevices are affected.  A hot reset that affects
+ * multiple devices, but only a single in-use device, means that we can call
+ * it from our bus ->reset() callback since the extent is effectively a single
+ * device.  This allows us to make use of it in the hotplug path.  When there
+ * are multiple in-use devices, we can only trigger the hot reset during a
+ * system reset and thus from our reset handler.  We separate _one vs _multi
+ * here so that we don't overlap and do a double reset on the system reset
+ * path where both our reset handler and ->reset() callback are used.  Calling
+ * _one() will only do a hot reset for the one in-use devices case, calling
+ * _multi() will do nothing if a _one() would have been sufficient.
+ */
+static int vfio_pci_hot_reset_one(VFIODevice *vdev)
+{
+    return vfio_pci_hot_reset(vdev, true);
+}
+
+static int vfio_pci_hot_reset_multi(VFIODevice *vdev)
+{
+    return vfio_pci_hot_reset(vdev, false);
+}
+
+static void vfio_pci_reset_handler(void *opaque)
+{
+    VFIOGroup *group;
+    VFIODevice *vdev;
+
+    QLIST_FOREACH(group, &group_list, next) {
+        QLIST_FOREACH(vdev, &group->device_list, next) {
+            if (!vdev->reset_works || (!vdev->has_flr && vdev->has_pm_reset)) {
+                vdev->needs_reset = true;
+            }
+        }
+    }
+
+    QLIST_FOREACH(group, &group_list, next) {
+        QLIST_FOREACH(vdev, &group->device_list, next) {
+            if (vdev->needs_reset) {
+                vfio_pci_hot_reset_multi(vdev);
+            }
+        }
+    }
 }
 
 static int vfio_connect_container(VFIOGroup *group)
@@ -2747,6 +3183,10 @@ static VFIOGroup *vfio_get_group(int groupid)
         return NULL;
     }
 
+    if (QLIST_EMPTY(&group_list)) {
+        qemu_register_reset(vfio_pci_reset_handler, NULL);
+    }
+
     QLIST_INSERT_HEAD(&group_list, group, next);
 
     return group;
@@ -2763,6 +3203,10 @@ static void vfio_put_group(VFIOGroup *group)
     DPRINTF("vfio_put_group: close group->fd\n");
     close(group->fd);
     g_free(group);
+
+    if (QLIST_EMPTY(&group_list)) {
+        qemu_unregister_reset(vfio_pci_reset_handler, NULL);
+    }
 }
 
 static int vfio_get_device(VFIOGroup *group, const char *name, VFIODevice *vdev)
@@ -2801,9 +3245,6 @@ static int vfio_get_device(VFIOGroup *group, const char *name, VFIODevice *vdev)
     }
 
     vdev->reset_works = !!(dev_info.flags & VFIO_DEVICE_FLAGS_RESET);
-    if (!vdev->reset_works) {
-        error_report("Warning, device %s does not support reset", name);
-    }
 
     if (dev_info.num_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1) {
         error_report("vfio: unexpected number of io regions %u",
@@ -2838,22 +3279,6 @@ static int vfio_get_device(VFIOGroup *group, const char *name, VFIODevice *vdev)
         QLIST_INIT(&vdev->bars[i].quirks);
     }
 
-    reg_info.index = VFIO_PCI_ROM_REGION_INDEX;
-
-    ret = ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
-    if (ret) {
-        error_report("vfio: Error getting ROM info: %m");
-        goto error;
-    }
-
-    DPRINTF("Device %s ROM:\n", name);
-    DPRINTF("  size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
-            (unsigned long)reg_info.size, (unsigned long)reg_info.offset,
-            (unsigned long)reg_info.flags);
-
-    vdev->rom_size = reg_info.size;
-    vdev->rom_offset = reg_info.offset;
-
     reg_info.index = VFIO_PCI_CONFIG_REGION_INDEX;
 
     ret = ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
@@ -2918,13 +3343,15 @@ static int vfio_get_device(VFIOGroup *group, const char *name, VFIODevice *vdev)
     ret = ioctl(vdev->fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info);
     if (ret) {
         /* This can fail for an old kernel or legacy PCI dev */
-        DPRINTF("VFIO_DEVICE_GET_IRQ_INFO failure ret=%d\n", ret);
+        DPRINTF("VFIO_DEVICE_GET_IRQ_INFO failure: %m\n");
         ret = 0;
     } else if (irq_info.count == 1) {
         vdev->pci_aer = true;
     } else {
-        error_report("vfio: Warning: "
-                     "Could not enable error recovery for the device\n");
+        error_report("vfio: %04x:%02x:%02x.%x "
+                     "Could not enable error recovery for the device",
+                     vdev->host.domain, vdev->host.bus, vdev->host.slot,
+                     vdev->host.function);
     }
 
 error:
@@ -2965,11 +3392,10 @@ static void vfio_err_notifier_handler(void *opaque)
      * guest to contain the error.
      */
 
-    error_report("%s (%04x:%02x:%02x.%x)"
-        "Unrecoverable error detected...\n"
-        "Please collect any data possible and then kill the guest",
-        __func__, vdev->host.domain, vdev->host.bus,
-        vdev->host.slot, vdev->host.function);
+    error_report("%s(%04x:%02x:%02x.%x) Unrecoverable error detected.  "
+                 "Please collect any data possible and then kill the guest",
+                 __func__, vdev->host.domain, vdev->host.bus,
+                 vdev->host.slot, vdev->host.function);
 
     vm_stop(RUN_STATE_IO_ERROR);
 }
@@ -2992,8 +3418,7 @@ static void vfio_register_err_notifier(VFIODevice *vdev)
     }
 
     if (event_notifier_init(&vdev->err_notifier, 0)) {
-        error_report("vfio: Warning: "
-                     "Unable to init event notifier for error detection\n");
+        error_report("vfio: Unable to init event notifier for error detection");
         vdev->pci_aer = false;
         return;
     }
@@ -3014,7 +3439,7 @@ static void vfio_register_err_notifier(VFIODevice *vdev)
 
     ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
     if (ret) {
-        error_report("vfio: Failed to set up error notification\n");
+        error_report("vfio: Failed to set up error notification");
         qemu_set_fd_handler(*pfd, NULL, NULL, vdev);
         event_notifier_cleanup(&vdev->err_notifier);
         vdev->pci_aer = false;
@@ -3047,7 +3472,7 @@ static void vfio_unregister_err_notifier(VFIODevice *vdev)
 
     ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
     if (ret) {
-        error_report("vfio: Failed to de-assign error fd: %d\n", ret);
+        error_report("vfio: Failed to de-assign error fd: %m");
     }
     g_free(irq_set);
     qemu_set_fd_handler(event_notifier_get_fd(&vdev->err_notifier),
@@ -3151,7 +3576,7 @@ static int vfio_initfn(PCIDevice *pdev)
     memset(&vdev->pdev.config[PCI_BASE_ADDRESS_0], 0, 24);
     memset(&vdev->pdev.config[PCI_ROM_ADDRESS], 0, 4);
 
-    vfio_load_rom(vdev);
+    vfio_pci_size_rom(vdev);
 
     ret = vfio_early_setup_msix(vdev);
     if (ret) {
@@ -3216,6 +3641,7 @@ static void vfio_exitfn(PCIDevice *pdev)
     vfio_teardown_msi(vdev);
     vfio_unmap_bars(vdev);
     g_free(vdev->emulated_config_bits);
+    g_free(vdev->rom);
     vfio_put_device(vdev);
     vfio_put_group(group);
 }
@@ -3224,51 +3650,34 @@ static void vfio_pci_reset(DeviceState *dev)
 {
     PCIDevice *pdev = DO_UPCAST(PCIDevice, qdev, dev);
     VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
-    uint16_t cmd;
 
     DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
             vdev->host.bus, vdev->host.slot, vdev->host.function);
 
-    vfio_disable_interrupts(vdev);
-
-    /* Make sure the device is in D0 */
-    if (vdev->pm_cap) {
-        uint16_t pmcsr;
-        uint8_t state;
+    vfio_pci_pre_reset(vdev);
 
-        pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
-        state = pmcsr & PCI_PM_CTRL_STATE_MASK;
-        if (state) {
-            pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
-            vfio_pci_write_config(pdev, vdev->pm_cap + PCI_PM_CTRL, pmcsr, 2);
-            /* vfio handles the necessary delay here */
-            pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
-            state = pmcsr & PCI_PM_CTRL_STATE_MASK;
-            if (state) {
-                error_report("vfio: Unable to power on device, stuck in D%d\n",
-                             state);
-            }
-        }
+    if (vdev->reset_works && (vdev->has_flr || !vdev->has_pm_reset) &&
+        !ioctl(vdev->fd, VFIO_DEVICE_RESET)) {
+        DPRINTF("%04x:%02x:%02x.%x FLR/VFIO_DEVICE_RESET\n", vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function);
+        goto post_reset;
     }
 
-    /*
-     * Stop any ongoing DMA by disconecting I/O, MMIO, and bus master.
-     * Also put INTx Disable in known state.
-     */
-    cmd = vfio_pci_read_config(pdev, PCI_COMMAND, 2);
-    cmd &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
-             PCI_COMMAND_INTX_DISABLE);
-    vfio_pci_write_config(pdev, PCI_COMMAND, cmd, 2);
+    /* See if we can do our own bus reset */
+    if (!vfio_pci_hot_reset_one(vdev)) {
+        goto post_reset;
+    }
 
-    if (vdev->reset_works) {
-        if (ioctl(vdev->fd, VFIO_DEVICE_RESET)) {
-            error_report("vfio: Error unable to reset physical device "
-                         "(%04x:%02x:%02x.%x): %m", vdev->host.domain,
-                         vdev->host.bus, vdev->host.slot, vdev->host.function);
-        }
+    /* If nothing else works and the device supports PM reset, use it */
+    if (vdev->reset_works && vdev->has_pm_reset &&
+        !ioctl(vdev->fd, VFIO_DEVICE_RESET)) {
+        DPRINTF("%04x:%02x:%02x.%x PCI PM Reset\n", vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function);
+        goto post_reset;
     }
 
-    vfio_enable_intx(vdev);
+post_reset:
+    vfio_pci_post_reset(vdev);
 }
 
 static Property vfio_pci_dev_properties[] = {
diff --git a/hw/net/e1000.c b/hw/net/e1000.c
index a37a3dfbe4..ec8ecd7b94 100644
--- a/hw/net/e1000.c
+++ b/hw/net/e1000.c
@@ -32,6 +32,7 @@
 #include "hw/loader.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/dma.h"
+#include "qemu/iov.h"
 
 #include "e1000_regs.h"
 
@@ -64,6 +65,8 @@ static int debugflags = DBGBIT(TXERR) | DBGBIT(GENERAL);
 /* this is the size past which hardware will drop packets when setting LPE=1 */
 #define MAXIMUM_ETHERNET_LPE_SIZE 16384
 
+#define MAXIMUM_ETHERNET_HDR_LEN (14+4)
+
 /*
  * HW models:
  *  E1000_DEV_ID_82540EM works with Windows and Linux
@@ -398,6 +401,7 @@ static void e1000_reset(void *opaque)
         d->mac_reg[RA] |= macaddr[i] << (8 * i);
         d->mac_reg[RA + 1] |= (i < 2) ? macaddr[i + 4] << (8 * i) : 0;
     }
+    qemu_format_nic_info_str(qemu_get_queue(d->nic), macaddr);
 }
 
 static void
@@ -899,7 +903,7 @@ static uint64_t rx_desc_base(E1000State *s)
 }
 
 static ssize_t
-e1000_receive(NetClientState *nc, const uint8_t *buf, size_t size)
+e1000_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt)
 {
     E1000State *s = qemu_get_nic_opaque(nc);
     PCIDevice *d = PCI_DEVICE(s);
@@ -908,8 +912,12 @@ e1000_receive(NetClientState *nc, const uint8_t *buf, size_t size)
     unsigned int n, rdt;
     uint32_t rdh_start;
     uint16_t vlan_special = 0;
-    uint8_t vlan_status = 0, vlan_offset = 0;
+    uint8_t vlan_status = 0;
     uint8_t min_buf[MIN_BUF_SIZE];
+    struct iovec min_iov;
+    uint8_t *filter_buf = iov->iov_base;
+    size_t size = iov_size(iov, iovcnt);
+    size_t iov_ofs = 0;
     size_t desc_offset;
     size_t desc_size;
     size_t total_size;
@@ -924,10 +932,16 @@ e1000_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 
     /* Pad to minimum Ethernet frame length */
     if (size < sizeof(min_buf)) {
-        memcpy(min_buf, buf, size);
+        iov_to_buf(iov, iovcnt, 0, min_buf, size);
         memset(&min_buf[size], 0, sizeof(min_buf) - size);
-        buf = min_buf;
-        size = sizeof(min_buf);
+        min_iov.iov_base = filter_buf = min_buf;
+        min_iov.iov_len = size = sizeof(min_buf);
+        iovcnt = 1;
+        iov = &min_iov;
+    } else if (iov->iov_len < MAXIMUM_ETHERNET_HDR_LEN) {
+        /* This is very unlikely, but may happen. */
+        iov_to_buf(iov, iovcnt, 0, min_buf, MAXIMUM_ETHERNET_HDR_LEN);
+        filter_buf = min_buf;
     }
 
     /* Discard oversized packets if !LPE and !SBP. */
@@ -938,14 +952,24 @@ e1000_receive(NetClientState *nc, const uint8_t *buf, size_t size)
         return size;
     }
 
-    if (!receive_filter(s, buf, size))
+    if (!receive_filter(s, filter_buf, size)) {
         return size;
+    }
 
-    if (vlan_enabled(s) && is_vlan_packet(s, buf)) {
-        vlan_special = cpu_to_le16(be16_to_cpup((uint16_t *)(buf + 14)));
-        memmove((uint8_t *)buf + 4, buf, 12);
+    if (vlan_enabled(s) && is_vlan_packet(s, filter_buf)) {
+        vlan_special = cpu_to_le16(be16_to_cpup((uint16_t *)(filter_buf
+                                                                + 14)));
+        iov_ofs = 4;
+        if (filter_buf == iov->iov_base) {
+            memmove(filter_buf + 4, filter_buf, 12);
+        } else {
+            iov_from_buf(iov, iovcnt, 4, filter_buf, 12);
+            while (iov->iov_len <= iov_ofs) {
+                iov_ofs -= iov->iov_len;
+                iov++;
+            }
+        }
         vlan_status = E1000_RXD_STAT_VP;
-        vlan_offset = 4;
         size -= 4;
     }
 
@@ -967,12 +991,23 @@ e1000_receive(NetClientState *nc, const uint8_t *buf, size_t size)
         desc.status |= (vlan_status | E1000_RXD_STAT_DD);
         if (desc.buffer_addr) {
             if (desc_offset < size) {
+                size_t iov_copy;
+                hwaddr ba = le64_to_cpu(desc.buffer_addr);
                 size_t copy_size = size - desc_offset;
                 if (copy_size > s->rxbuf_size) {
                     copy_size = s->rxbuf_size;
                 }
-                pci_dma_write(d, le64_to_cpu(desc.buffer_addr),
-                              buf + desc_offset + vlan_offset, copy_size);
+                do {
+                    iov_copy = MIN(copy_size, iov->iov_len - iov_ofs);
+                    pci_dma_write(d, ba, iov->iov_base + iov_ofs, iov_copy);
+                    copy_size -= iov_copy;
+                    ba += iov_copy;
+                    iov_ofs += iov_copy;
+                    if (iov_ofs == iov->iov_len) {
+                        iov++;
+                        iov_ofs = 0;
+                    }
+                } while (copy_size);
             }
             desc_offset += desc_size;
             desc.length = cpu_to_le16(desc_size);
@@ -1022,6 +1057,17 @@ e1000_receive(NetClientState *nc, const uint8_t *buf, size_t size)
     return size;
 }
 
+static ssize_t
+e1000_receive(NetClientState *nc, const uint8_t *buf, size_t size)
+{
+    const struct iovec iov = {
+        .iov_base = (uint8_t *)buf,
+        .iov_len = size
+    };
+
+    return e1000_receive_iov(nc, &iov, 1);
+}
+
 static uint32_t
 mac_readreg(E1000State *s, int index)
 {
@@ -1060,7 +1106,15 @@ mac_read_clr8(E1000State *s, int index)
 static void
 mac_writereg(E1000State *s, int index, uint32_t val)
 {
+    uint32_t macaddr[2];
+
     s->mac_reg[index] = val;
+
+    if (index == RA + 1) {
+        macaddr[0] = cpu_to_le32(s->mac_reg[RA]);
+        macaddr[1] = cpu_to_le32(s->mac_reg[RA + 1]);
+        qemu_format_nic_info_str(qemu_get_queue(s->nic), (uint8_t *)macaddr);
+    }
 }
 
 static void
@@ -1448,6 +1502,7 @@ static NetClientInfo net_e1000_info = {
     .size = sizeof(NICState),
     .can_receive = e1000_can_receive,
     .receive = e1000_receive,
+    .receive_iov = e1000_receive_iov,
     .cleanup = e1000_cleanup,
     .link_status_changed = e1000_set_link_status,
 };
diff --git a/hw/net/pcnet-pci.c b/hw/net/pcnet-pci.c
index 311594d4fe..6a5d8064bb 100644
--- a/hw/net/pcnet-pci.c
+++ b/hw/net/pcnet-pci.c
@@ -134,7 +134,7 @@ static void pcnet_ioport_write(void *opaque, hwaddr addr,
 static const MemoryRegionOps pcnet_io_ops = {
     .read = pcnet_ioport_read,
     .write = pcnet_ioport_write,
-    .endianness = DEVICE_NATIVE_ENDIAN,
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static void pcnet_mmio_writeb(void *opaque, hwaddr addr, uint32_t val)
@@ -256,7 +256,7 @@ static const MemoryRegionOps pcnet_mmio_ops = {
         .read = { pcnet_mmio_readb, pcnet_mmio_readw, pcnet_mmio_readl },
         .write = { pcnet_mmio_writeb, pcnet_mmio_writew, pcnet_mmio_writel },
     },
-    .endianness = DEVICE_NATIVE_ENDIAN,
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static void pci_physical_memory_write(void *dma_opaque, hwaddr addr,
diff --git a/hw/net/rtl8139.c b/hw/net/rtl8139.c
index 7d72b211fc..5329f44a9d 100644
--- a/hw/net/rtl8139.c
+++ b/hw/net/rtl8139.c
@@ -1214,6 +1214,7 @@ static void rtl8139_reset(DeviceState *d)
 
     /* restore MAC address */
     memcpy(s->phys, s->conf.macaddr.a, 6);
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->phys);
 
     /* reset interrupt mask */
     s->IntrStatus = 0;
@@ -2740,8 +2741,12 @@ static void rtl8139_io_writeb(void *opaque, uint8_t addr, uint32_t val)
 
     switch (addr)
     {
-        case MAC0 ... MAC0+5:
+        case MAC0 ... MAC0+4:
+            s->phys[addr - MAC0] = val;
+            break;
+        case MAC0+5:
             s->phys[addr - MAC0] = val;
+            qemu_format_nic_info_str(qemu_get_queue(s->nic), s->phys);
             break;
         case MAC0+6 ... MAC0+7:
             /* reserved */
diff --git a/hw/pci-host/q35.c b/hw/pci-host/q35.c
index 50063f83f3..b8feed1254 100644
--- a/hw/pci-host/q35.c
+++ b/hw/pci-host/q35.c
@@ -412,7 +412,7 @@ static void mch_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_mch;
     k->vendor_id = PCI_VENDOR_ID_INTEL;
     k->device_id = PCI_DEVICE_ID_INTEL_Q35_MCH;
-    k->revision = MCH_HOST_BRIDGE_REVISION_DEFUALT;
+    k->revision = MCH_HOST_BRIDGE_REVISION_DEFAULT;
     k->class_id = PCI_CLASS_BRIDGE_HOST;
 }
 
diff --git a/hw/pci/Makefile.objs b/hw/pci/Makefile.objs
index 720f438ac9..80f8aa69ac 100644
--- a/hw/pci/Makefile.objs
+++ b/hw/pci/Makefile.objs
@@ -5,7 +5,7 @@ common-obj-$(CONFIG_PCI) += slotid_cap.o
 common-obj-$(CONFIG_PCI) += pci_host.o pcie_host.o
 common-obj-$(CONFIG_PCI) += pcie.o pcie_aer.o pcie_port.o
 
-common-obj-$(CONFIG_NO_PCI) += pci-stub.o
+common-obj-$(call lnot,$(CONFIG_PCI)) += pci-stub.o
 common-obj-$(CONFIG_ALL) += pci-stub.o
 
 common-obj-$(CONFIG_PCI_HOTPLUG_OLD) += pci-hotplug-old.o
diff --git a/hw/s390x/event-facility.c b/hw/s390x/event-facility.c
index a3aceef8f5..25951a020a 100644
--- a/hw/s390x/event-facility.c
+++ b/hw/s390x/event-facility.c
@@ -120,7 +120,7 @@ static uint16_t handle_write_event_buf(SCLPEventFacility *ef,
         ec = SCLP_EVENT_GET_CLASS(event);
 
         if (ec->write_event_data &&
-            ec->event_type() == event_buf->type) {
+            ec->can_handle_event(event_buf->type)) {
             rc = ec->write_event_data(event, event_buf);
             break;
         }
@@ -183,7 +183,7 @@ static uint16_t handle_sccb_read_events(SCLPEventFacility *ef, SCCB *sccb,
 {
     uint16_t rc;
     int slen;
-    unsigned elen = 0;
+    unsigned elen;
     BusChild *kid;
     SCLPEvent *event;
     SCLPEventClass *ec;
@@ -203,11 +203,11 @@ static uint16_t handle_sccb_read_events(SCLPEventFacility *ef, SCCB *sccb,
 
         if (mask & ec->get_send_mask()) {
             if (ec->read_event_data(event, event_buf, &slen)) {
+                elen = be16_to_cpu(event_buf->length);
+                event_buf = (EventBufferHeader *) ((char *)event_buf + elen);
                 rc = SCLP_RC_NORMAL_COMPLETION;
             }
         }
-        elen = be16_to_cpu(event_buf->length);
-        event_buf = (void *) event_buf + elen;
     }
 
     if (sccb->h.control_mask[2] & SCLP_VARIABLE_LENGTH_RESPONSE) {
@@ -338,10 +338,19 @@ static int init_event_facility(S390SCLPDevice *sdev)
     return 0;
 }
 
+static void reset_event_facility(DeviceState *dev)
+{
+    S390SCLPDevice *sdev = SCLP_S390_DEVICE(dev);
+
+    sdev->ef->receive_mask = 0;
+}
+
 static void init_event_facility_class(ObjectClass *klass, void *data)
 {
+    DeviceClass *dc = DEVICE_CLASS(klass);
     S390SCLPDeviceClass *k = SCLP_S390_DEVICE_CLASS(klass);
 
+    dc->reset = reset_event_facility;
     k->init = init_event_facility;
 }
 
diff --git a/hw/s390x/sclpquiesce.c b/hw/s390x/sclpquiesce.c
index 5fadc86d42..a3c4bd6272 100644
--- a/hw/s390x/sclpquiesce.c
+++ b/hw/s390x/sclpquiesce.c
@@ -22,9 +22,9 @@ typedef struct SignalQuiesce {
     uint8_t unit;
 } QEMU_PACKED SignalQuiesce;
 
-static int event_type(void)
+static bool can_handle_event(uint8_t type)
 {
-    return SCLP_EVENT_SIGNAL_QUIESCE;
+    return type == SCLP_EVENT_SIGNAL_QUIESCE;
 }
 
 static unsigned int send_mask(void)
@@ -65,6 +65,17 @@ static int read_event_data(SCLPEvent *event, EventBufferHeader *evt_buf_hdr,
     return 1;
 }
 
+static const VMStateDescription vmstate_sclpquiesce = {
+    .name = "sclpquiesce",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .minimum_version_id_old = 0,
+    .fields      = (VMStateField[]) {
+        VMSTATE_BOOL(event_pending, SCLPEvent),
+        VMSTATE_END_OF_LIST()
+     }
+};
+
 typedef struct QuiesceNotifier QuiesceNotifier;
 
 static struct QuiesceNotifier {
@@ -84,8 +95,6 @@ static void quiesce_powerdown_req(Notifier *n, void *opaque)
 
 static int quiesce_init(SCLPEvent *event)
 {
-    event->event_type = SCLP_EVENT_SIGNAL_QUIESCE;
-
     qn.notifier.notify = quiesce_powerdown_req;
     qn.event = event;
 
@@ -94,15 +103,25 @@ static int quiesce_init(SCLPEvent *event)
     return 0;
 }
 
+static void quiesce_reset(DeviceState *dev)
+{
+   SCLPEvent *event = SCLP_EVENT(dev);
+
+   event->event_pending = false;
+}
+
 static void quiesce_class_init(ObjectClass *klass, void *data)
 {
+    DeviceClass *dc = DEVICE_CLASS(klass);
     SCLPEventClass *k = SCLP_EVENT_CLASS(klass);
 
+    dc->reset = quiesce_reset;
+    dc->vmsd = &vmstate_sclpquiesce;
     k->init = quiesce_init;
 
     k->get_send_mask = send_mask;
     k->get_receive_mask = receive_mask;
-    k->event_type = event_type;
+    k->can_handle_event = can_handle_event;
     k->read_event_data = read_event_data;
     k->write_event_data = NULL;
 }
diff --git a/hw/scsi/lsi53c895a.c b/hw/scsi/lsi53c895a.c
index 0e51b94748..cb30414849 100644
--- a/hw/scsi/lsi53c895a.c
+++ b/hw/scsi/lsi53c895a.c
@@ -7,8 +7,11 @@
  * This code is licensed under the LGPL.
  */
 
-/* ??? Need to check if the {read,write}[wl] routines work properly on
-   big-endian targets.  */
+/* Note:
+ * LSI53C810 emulation is incorrect, in the sense that it supports
+ * features added in later evolutions. This should not be a problem,
+ * as well-behaved operating systems will not try to use them.
+ */
 
 #include <assert.h>
 
@@ -278,6 +281,7 @@ typedef struct {
     uint32_t script_ram[2048];
 } LSIState;
 
+#define TYPE_LSI53C810  "lsi53c810"
 #define TYPE_LSI53C895A "lsi53c895a"
 
 #define LSI53C895A(obj) \
@@ -998,12 +1002,6 @@ bad:
     s->msg_action = 0;
 }
 
-/* Sign extend a 24-bit value.  */
-static inline int32_t sxt24(int32_t n)
-{
-    return (n << 8) >> 8;
-}
-
 #define LSI_BUF_SIZE 4096
 static void lsi_memcpy(LSIState *s, uint32_t dest, uint32_t src, int count)
 {
@@ -1083,7 +1081,7 @@ again:
             /* Table indirect addressing.  */
 
             /* 32-bit Table indirect */
-            offset = sxt24(addr);
+            offset = sextract32(addr, 0, 24);
             pci_dma_read(pci_dev, s->dsa + offset, buf, 8);
             /* byte count is stored in bits 0:23 only */
             s->dbc = cpu_to_le32(buf[0]) & 0xffffff;
@@ -1183,13 +1181,13 @@ again:
             uint32_t id;
 
             if (insn & (1 << 25)) {
-                id = read_dword(s, s->dsa + sxt24(insn));
+                id = read_dword(s, s->dsa + sextract32(insn, 0, 24));
             } else {
                 id = insn;
             }
             id = (id >> 16) & 0xf;
             if (insn & (1 << 26)) {
-                addr = s->dsp + sxt24(addr);
+                addr = s->dsp + sextract32(addr, 0, 24);
             }
             s->dnad = addr;
             switch (opcode) {
@@ -1385,7 +1383,7 @@ again:
             if (cond == jmp) {
                 if (insn & (1 << 23)) {
                     /* Relative address.  */
-                    addr = s->dsp + sxt24(addr);
+                    addr = s->dsp + sextract32(addr, 0, 24);
                 }
                 switch ((insn >> 27) & 7) {
                 case 0: /* Jump */
@@ -1438,7 +1436,7 @@ again:
             int i;
 
             if (insn & (1 << 28)) {
-                addr = s->dsa + sxt24(addr);
+                addr = s->dsa + sextract32(addr, 0, 24);
             }
             n = (insn & 7);
             reg = (insn >> 16) & 0xff;
@@ -1521,7 +1519,7 @@ static uint8_t lsi_reg_readb(LSIState *s, int offset)
            used for diagnostics, so should be ok.  */
         return 0;
     case 0xc: /* DSTAT */
-        tmp = s->dstat | 0x80;
+        tmp = s->dstat | LSI_DSTAT_DFE;
         if ((s->istat0 & LSI_ISTAT0_INTF) == 0)
             s->dstat = 0;
         lsi_update_irq(s);
@@ -1705,8 +1703,9 @@ static void lsi_reg_writeb(LSIState *s, int offset, uint8_t val)
         s->sxfer = val;
         break;
     case 0x06: /* SDID */
-        if ((val & 0xf) != (s->ssid & 0xf))
+        if ((s->ssid & 0x80) && (val & 0xf) != (s->ssid & 0xf)) {
             BADF("Destination ID does not match SSID\n");
+        }
         s->sdid = val & 0xf;
         break;
     case 0x07: /* GPREG0 */
@@ -1748,6 +1747,9 @@ static void lsi_reg_writeb(LSIState *s, int offset, uint8_t val)
     case 0x17: /* MBOX1 */
         s->mbox1 = val;
         break;
+    case 0x18: /* CTEST0 */
+        /* nothing to do */
+        break;
     case 0x1a: /* CTEST2 */
 	s->ctest2 = val & LSI_CTEST2_PCICIE;
 	break;
@@ -1876,8 +1878,7 @@ static void lsi_reg_writeb(LSIState *s, int offset, uint8_t val)
             int shift;
             n = (offset - 0x58) >> 2;
             shift = (offset & 3) * 8;
-            s->scratch[n] &= ~(0xff << shift);
-            s->scratch[n] |= (val & 0xff) << shift;
+            s->scratch[n] = deposit32(s->scratch[n], shift, 8, val);
         } else {
             BADF("Unhandled writeb 0x%x = 0x%x\n", offset, val);
         }
@@ -2113,7 +2114,7 @@ static int lsi_scsi_init(PCIDevice *dev)
                           "lsi-io", 256);
 
     pci_register_bar(dev, 0, PCI_BASE_ADDRESS_SPACE_IO, &s->io_io);
-    pci_register_bar(dev, 1, 0, &s->mmio_io);
+    pci_register_bar(dev, 1, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->mmio_io);
     pci_register_bar(dev, 2, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->ram_io);
     QTAILQ_INIT(&s->queue);
 
@@ -2151,9 +2152,23 @@ static const TypeInfo lsi_info = {
     .class_init    = lsi_class_init,
 };
 
+static void lsi53c810_class_init(ObjectClass *klass, void *data)
+{
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->device_id = PCI_DEVICE_ID_LSI_53C810;
+}
+
+static TypeInfo lsi53c810_info = {
+    .name          = TYPE_LSI53C810,
+    .parent        = TYPE_LSI53C895A,
+    .class_init    = lsi53c810_class_init,
+};
+
 static void lsi53c895a_register_types(void)
 {
     type_register_static(&lsi_info);
+    type_register_static(&lsi53c810_info);
 }
 
 type_init(lsi53c895a_register_types)
diff --git a/hw/scsi/scsi-bus.c b/hw/scsi/scsi-bus.c
index 5cd6137046..24ec52f8f9 100644
--- a/hw/scsi/scsi-bus.c
+++ b/hw/scsi/scsi-bus.c
@@ -11,6 +11,8 @@ static char *scsibus_get_dev_path(DeviceState *dev);
 static char *scsibus_get_fw_dev_path(DeviceState *dev);
 static int scsi_req_parse(SCSICommand *cmd, SCSIDevice *dev, uint8_t *buf);
 static void scsi_req_dequeue(SCSIRequest *req);
+static uint8_t *scsi_target_alloc_buf(SCSIRequest *req, size_t len);
+static void scsi_target_free_buf(SCSIRequest *req);
 
 static Property scsi_props[] = {
     DEFINE_PROP_UINT32("channel", SCSIDevice, channel, 0),
@@ -224,7 +226,7 @@ SCSIDevice *scsi_bus_legacy_add_drive(SCSIBus *bus, BlockDriverState *bdrv,
     if (object_property_find(OBJECT(dev), "removable", NULL)) {
         qdev_prop_set_bit(dev, "removable", removable);
     }
-    if (serial) {
+    if (serial && object_property_find(OBJECT(dev), "serial", NULL)) {
         qdev_prop_set_string(dev, "serial", serial);
     }
     if (qdev_prop_set_drive(dev, "drive", bdrv) < 0) {
@@ -317,7 +319,8 @@ typedef struct SCSITargetReq SCSITargetReq;
 struct SCSITargetReq {
     SCSIRequest req;
     int len;
-    uint8_t buf[2056];
+    uint8_t *buf;
+    int buf_len;
 };
 
 static void store_lun(uint8_t *outbuf, int lun)
@@ -361,14 +364,12 @@ static bool scsi_target_emulate_report_luns(SCSITargetReq *r)
     if (!found_lun0) {
         n += 8;
     }
-    len = MIN(n + 8, r->req.cmd.xfer & ~7);
-    if (len > sizeof(r->buf)) {
-        /* TODO: > 256 LUNs? */
-        return false;
-    }
 
+    scsi_target_alloc_buf(&r->req, n + 8);
+
+    len = MIN(n + 8, r->req.cmd.xfer & ~7);
     memset(r->buf, 0, len);
-    stl_be_p(&r->buf, n);
+    stl_be_p(&r->buf[0], n);
     i = found_lun0 ? 8 : 16;
     QTAILQ_FOREACH(kid, &r->req.bus->qbus.children, sibling) {
         DeviceState *qdev = kid->child;
@@ -387,6 +388,9 @@ static bool scsi_target_emulate_report_luns(SCSITargetReq *r)
 static bool scsi_target_emulate_inquiry(SCSITargetReq *r)
 {
     assert(r->req.dev->lun != r->req.lun);
+
+    scsi_target_alloc_buf(&r->req, SCSI_INQUIRY_LEN);
+
     if (r->req.cmd.buf[1] & 0x2) {
         /* Command support data - optional, not implemented */
         return false;
@@ -411,7 +415,7 @@ static bool scsi_target_emulate_inquiry(SCSITargetReq *r)
             return false;
         }
         /* done with EVPD */
-        assert(r->len < sizeof(r->buf));
+        assert(r->len < r->buf_len);
         r->len = MIN(r->req.cmd.xfer, r->len);
         return true;
     }
@@ -422,7 +426,7 @@ static bool scsi_target_emulate_inquiry(SCSITargetReq *r)
     }
 
     /* PAGE CODE == 0 */
-    r->len = MIN(r->req.cmd.xfer, 36);
+    r->len = MIN(r->req.cmd.xfer, SCSI_INQUIRY_LEN);
     memset(r->buf, 0, r->len);
     if (r->req.lun != 0) {
         r->buf[0] = TYPE_NO_LUN;
@@ -455,8 +459,9 @@ static int32_t scsi_target_send_command(SCSIRequest *req, uint8_t *buf)
         }
         break;
     case REQUEST_SENSE:
+        scsi_target_alloc_buf(&r->req, SCSI_SENSE_LEN);
         r->len = scsi_device_get_sense(r->req.dev, r->buf,
-                                       MIN(req->cmd.xfer, sizeof r->buf),
+                                       MIN(req->cmd.xfer, r->buf_len),
                                        (req->cmd.buf[1] & 1) == 0);
         if (r->req.dev->sense_is_ua) {
             scsi_device_unit_attention_reported(req->dev);
@@ -501,11 +506,29 @@ static uint8_t *scsi_target_get_buf(SCSIRequest *req)
     return r->buf;
 }
 
+static uint8_t *scsi_target_alloc_buf(SCSIRequest *req, size_t len)
+{
+    SCSITargetReq *r = DO_UPCAST(SCSITargetReq, req, req);
+
+    r->buf = g_malloc(len);
+    r->buf_len = len;
+
+    return r->buf;
+}
+
+static void scsi_target_free_buf(SCSIRequest *req)
+{
+    SCSITargetReq *r = DO_UPCAST(SCSITargetReq, req, req);
+
+    g_free(r->buf);
+}
+
 static const struct SCSIReqOps reqops_target_command = {
     .size         = sizeof(SCSITargetReq),
     .send_command = scsi_target_send_command,
     .read_data    = scsi_target_read_data,
     .get_buf      = scsi_target_get_buf,
+    .free_req     = scsi_target_free_buf,
 };
 
 
@@ -1365,7 +1388,7 @@ int scsi_build_sense(uint8_t *in_buf, int in_len,
         buf[7] = 10;
         buf[12] = sense.asc;
         buf[13] = sense.ascq;
-        return MIN(len, 18);
+        return MIN(len, SCSI_SENSE_LEN);
     } else {
         /* Return descriptor format sense buffer */
         buf[0] = 0x72;
diff --git a/hw/scsi/spapr_vscsi.c b/hw/scsi/spapr_vscsi.c
index b2fcd4b3e3..2a26042701 100644
--- a/hw/scsi/spapr_vscsi.c
+++ b/hw/scsi/spapr_vscsi.c
@@ -117,6 +117,20 @@ static struct vscsi_req *vscsi_get_req(VSCSIState *s)
     return NULL;
 }
 
+static struct vscsi_req *vscsi_find_req(VSCSIState *s, uint64_t srp_tag)
+{
+    vscsi_req *req;
+    int i;
+
+    for (i = 0; i < VSCSI_REQ_LIMIT; i++) {
+        req = &s->reqs[i];
+        if (req->iu.srp.cmd.tag == srp_tag) {
+            return req;
+        }
+    }
+    return NULL;
+}
+
 static void vscsi_put_req(vscsi_req *req)
 {
     if (req->sreq != NULL) {
@@ -755,40 +769,91 @@ static int vscsi_queue_cmd(VSCSIState *s, vscsi_req *req)
 static int vscsi_process_tsk_mgmt(VSCSIState *s, vscsi_req *req)
 {
     union viosrp_iu *iu = &req->iu;
-    int fn;
+    vscsi_req *tmpreq;
+    int i, lun = 0, resp = SRP_TSK_MGMT_COMPLETE;
+    SCSIDevice *d;
+    uint64_t tag = iu->srp.rsp.tag;
+    uint8_t sol_not = iu->srp.cmd.sol_not;
 
     fprintf(stderr, "vscsi_process_tsk_mgmt %02x\n",
             iu->srp.tsk_mgmt.tsk_mgmt_func);
 
-    switch (iu->srp.tsk_mgmt.tsk_mgmt_func) {
-#if 0 /* We really don't deal with these for now */
-    case SRP_TSK_ABORT_TASK:
-        fn = ABORT_TASK;
-        break;
-    case SRP_TSK_ABORT_TASK_SET:
-        fn = ABORT_TASK_SET;
-        break;
-    case SRP_TSK_CLEAR_TASK_SET:
-        fn = CLEAR_TASK_SET;
-        break;
-    case SRP_TSK_LUN_RESET:
-        fn = LOGICAL_UNIT_RESET;
-        break;
-    case SRP_TSK_CLEAR_ACA:
-        fn = CLEAR_ACA;
-        break;
-#endif
-    default:
-        fn = 0;
+    d = vscsi_device_find(&s->bus, be64_to_cpu(req->iu.srp.tsk_mgmt.lun), &lun);
+    if (!d) {
+        resp = SRP_TSK_MGMT_FIELDS_INVALID;
+    } else {
+        switch (iu->srp.tsk_mgmt.tsk_mgmt_func) {
+        case SRP_TSK_ABORT_TASK:
+            if (d->lun != lun) {
+                resp = SRP_TSK_MGMT_FIELDS_INVALID;
+                break;
+            }
+
+            tmpreq = vscsi_find_req(s, req->iu.srp.tsk_mgmt.task_tag);
+            if (tmpreq && tmpreq->sreq) {
+                assert(tmpreq->sreq->hba_private);
+                scsi_req_cancel(tmpreq->sreq);
+            }
+            break;
+
+        case SRP_TSK_LUN_RESET:
+            if (d->lun != lun) {
+                resp = SRP_TSK_MGMT_FIELDS_INVALID;
+                break;
+            }
+
+            qdev_reset_all(&d->qdev);
+            break;
+
+        case SRP_TSK_ABORT_TASK_SET:
+        case SRP_TSK_CLEAR_TASK_SET:
+            if (d->lun != lun) {
+                resp = SRP_TSK_MGMT_FIELDS_INVALID;
+                break;
+            }
+
+            for (i = 0; i < VSCSI_REQ_LIMIT; i++) {
+                tmpreq = &s->reqs[i];
+                if (tmpreq->iu.srp.cmd.lun != req->iu.srp.tsk_mgmt.lun) {
+                    continue;
+                }
+                if (!tmpreq->active || !tmpreq->sreq) {
+                    continue;
+                }
+                assert(tmpreq->sreq->hba_private);
+                scsi_req_cancel(tmpreq->sreq);
+            }
+            break;
+
+        case SRP_TSK_CLEAR_ACA:
+            resp = SRP_TSK_MGMT_NOT_SUPPORTED;
+            break;
+
+        default:
+            resp = SRP_TSK_MGMT_FIELDS_INVALID;
+            break;
+        }
     }
-    if (fn) {
-        /* XXX Send/Handle target task management */
-        ;
+
+    /* Compose the response here as  */
+    memset(iu, 0, sizeof(struct srp_rsp) + 4);
+    iu->srp.rsp.opcode = SRP_RSP;
+    iu->srp.rsp.req_lim_delta = cpu_to_be32(1);
+    iu->srp.rsp.tag = tag;
+    iu->srp.rsp.flags |= SRP_RSP_FLAG_RSPVALID;
+    iu->srp.rsp.resp_data_len = cpu_to_be32(4);
+    if (resp) {
+        iu->srp.rsp.sol_not = (sol_not & 0x04) >> 2;
     } else {
-        vscsi_makeup_sense(s, req, ILLEGAL_REQUEST, 0x20, 0);
-        vscsi_send_rsp(s, req, CHECK_CONDITION, 0, 0);
+        iu->srp.rsp.sol_not = (sol_not & 0x02) >> 1;
     }
-    return !fn;
+
+    iu->srp.rsp.status = GOOD;
+    iu->srp.rsp.data[3] = resp;
+
+    vscsi_send_iu(s, req, sizeof(iu->srp.rsp) + 4, VIOSRP_SRP_FORMAT);
+
+    return 1;
 }
 
 static int vscsi_handle_srp_req(VSCSIState *s, vscsi_req *req)
@@ -858,29 +923,97 @@ static int vscsi_send_adapter_info(VSCSIState *s, vscsi_req *req)
     return vscsi_send_iu(s, req, sizeof(*sinfo), VIOSRP_MAD_FORMAT);
 }
 
+static int vscsi_send_capabilities(VSCSIState *s, vscsi_req *req)
+{
+    struct viosrp_capabilities *vcap;
+    struct capabilities cap = { };
+    uint16_t len, req_len;
+    uint64_t buffer;
+    int rc;
+
+    vcap = &req->iu.mad.capabilities;
+    req_len = len = be16_to_cpu(vcap->common.length);
+    buffer = be64_to_cpu(vcap->buffer);
+    if (len > sizeof(cap)) {
+        fprintf(stderr, "vscsi_send_capabilities: capabilities size mismatch !\n");
+
+        /*
+         * Just read and populate the structure that is known.
+         * Zero rest of the structure.
+         */
+        len = sizeof(cap);
+    }
+    rc = spapr_vio_dma_read(&s->vdev, buffer, &cap, len);
+    if (rc)  {
+        fprintf(stderr, "vscsi_send_capabilities: DMA read failure !\n");
+    }
+
+    /*
+     * Current implementation does not suppport any migration or
+     * reservation capabilities. Construct the response telling the
+     * guest not to use them.
+     */
+    cap.flags = 0;
+    cap.migration.ecl = 0;
+    cap.reserve.type = 0;
+    cap.migration.common.server_support = 0;
+    cap.reserve.common.server_support = 0;
+
+    rc = spapr_vio_dma_write(&s->vdev, buffer, &cap, len);
+    if (rc)  {
+        fprintf(stderr, "vscsi_send_capabilities: DMA write failure !\n");
+    }
+    if (req_len > len) {
+        /*
+         * Being paranoid and lets not worry about the error code
+         * here. Actual write of the cap is done above.
+         */
+        spapr_vio_dma_set(&s->vdev, (buffer + len), 0, (req_len - len));
+    }
+    vcap->common.status = rc ? cpu_to_be32(1) : 0;
+    return vscsi_send_iu(s, req, sizeof(*vcap), VIOSRP_MAD_FORMAT);
+}
+
 static int vscsi_handle_mad_req(VSCSIState *s, vscsi_req *req)
 {
     union mad_iu *mad = &req->iu.mad;
+    bool request_handled = false;
+    uint64_t retlen = 0;
 
     switch (be32_to_cpu(mad->empty_iu.common.type)) {
     case VIOSRP_EMPTY_IU_TYPE:
         fprintf(stderr, "Unsupported EMPTY MAD IU\n");
+        retlen = sizeof(mad->empty_iu);
         break;
     case VIOSRP_ERROR_LOG_TYPE:
         fprintf(stderr, "Unsupported ERROR LOG MAD IU\n");
-        mad->error_log.common.status = cpu_to_be16(1);
-        vscsi_send_iu(s, req, sizeof(mad->error_log), VIOSRP_MAD_FORMAT);
+        retlen = sizeof(mad->error_log);
         break;
     case VIOSRP_ADAPTER_INFO_TYPE:
         vscsi_send_adapter_info(s, req);
+        request_handled = true;
         break;
     case VIOSRP_HOST_CONFIG_TYPE:
-        mad->host_config.common.status = cpu_to_be16(1);
-        vscsi_send_iu(s, req, sizeof(mad->host_config), VIOSRP_MAD_FORMAT);
+        retlen = sizeof(mad->host_config);
+        break;
+    case VIOSRP_CAPABILITIES_TYPE:
+        vscsi_send_capabilities(s, req);
+        request_handled = true;
         break;
     default:
         fprintf(stderr, "VSCSI: Unknown MAD type %02x\n",
                 be32_to_cpu(mad->empty_iu.common.type));
+        /*
+         * PAPR+ says that "The length field is set to the length
+         * of the data structure(s) used in the command".
+         * As we did not recognize the request type, put zero there.
+         */
+        retlen = 0;
+    }
+
+    if (!request_handled) {
+        mad->empty_iu.common.status = cpu_to_be16(VIOSRP_MAD_NOT_SUPPORTED);
+        vscsi_send_iu(s, req, retlen, VIOSRP_MAD_FORMAT);
     }
 
     return 1;
diff --git a/hw/scsi/srp.h b/hw/scsi/srp.h
index 5e0cad5c19..d27f31d2d5 100644
--- a/hw/scsi/srp.h
+++ b/hw/scsi/srp.h
@@ -90,6 +90,13 @@ enum {
     SRP_REV16A_IB_IO_CLASS = 0x0100
 };
 
+enum {
+    SRP_TSK_MGMT_COMPLETE       = 0x00,
+    SRP_TSK_MGMT_FIELDS_INVALID = 0x02,
+    SRP_TSK_MGMT_NOT_SUPPORTED  = 0x04,
+    SRP_TSK_MGMT_FAILED         = 0x05
+};
+
 struct srp_direct_buf {
     uint64_t    va;
     uint32_t    key;
diff --git a/hw/scsi/virtio-scsi.c b/hw/scsi/virtio-scsi.c
index 3bd690d128..26d95a14ec 100644
--- a/hw/scsi/virtio-scsi.c
+++ b/hw/scsi/virtio-scsi.c
@@ -693,6 +693,7 @@ static const TypeInfo virtio_scsi_common_info = {
     .name = TYPE_VIRTIO_SCSI_COMMON,
     .parent = TYPE_VIRTIO_DEVICE,
     .instance_size = sizeof(VirtIOSCSICommon),
+    .abstract = true,
     .class_init = virtio_scsi_common_class_init,
 };
 
diff --git a/hw/sd/milkymist-memcard.c b/hw/sd/milkymist-memcard.c
index 42613b3aff..d1168c9e04 100644
--- a/hw/sd/milkymist-memcard.c
+++ b/hw/sd/milkymist-memcard.c
@@ -255,6 +255,10 @@ static int milkymist_memcard_init(SysBusDevice *dev)
 
     dinfo = drive_get_next(IF_SD);
     s->card = sd_init(dinfo ? dinfo->bdrv : NULL, false);
+    if (s->card == NULL) {
+        return -1;
+    }
+
     s->enabled = dinfo ? bdrv_is_inserted(dinfo->bdrv) : 0;
 
     memory_region_init_io(&s->regs_region, OBJECT(s), &memcard_mmio_ops, s,
diff --git a/hw/sd/omap_mmc.c b/hw/sd/omap_mmc.c
index bf5d1fbf6d..937a47869a 100644
--- a/hw/sd/omap_mmc.c
+++ b/hw/sd/omap_mmc.c
@@ -593,6 +593,9 @@ struct omap_mmc_s *omap_mmc_init(hwaddr base,
 
     /* Instantiate the storage */
     s->card = sd_init(bd, false);
+    if (s->card == NULL) {
+        exit(1);
+    }
 
     return s;
 }
@@ -618,6 +621,9 @@ struct omap_mmc_s *omap2_mmc_init(struct omap_target_agent_s *ta,
 
     /* Instantiate the storage */
     s->card = sd_init(bd, false);
+    if (s->card == NULL) {
+        exit(1);
+    }
 
     s->cdet = qemu_allocate_irqs(omap_mmc_cover_cb, s, 1)[0];
     sd_set_cb(s->card, NULL, s->cdet);
diff --git a/hw/sd/pl181.c b/hw/sd/pl181.c
index 03875bf6ca..c35896d28c 100644
--- a/hw/sd/pl181.c
+++ b/hw/sd/pl181.c
@@ -491,6 +491,10 @@ static int pl181_init(SysBusDevice *sbd)
     qdev_init_gpio_out(dev, s->cardstatus, 2);
     dinfo = drive_get_next(IF_SD);
     s->card = sd_init(dinfo ? dinfo->bdrv : NULL, false);
+    if (s->card == NULL) {
+        return -1;
+    }
+
     return 0;
 }
 
diff --git a/hw/sd/pxa2xx_mmci.c b/hw/sd/pxa2xx_mmci.c
index 90c955fe62..b9d8b1a3e1 100644
--- a/hw/sd/pxa2xx_mmci.c
+++ b/hw/sd/pxa2xx_mmci.c
@@ -539,6 +539,9 @@ PXA2xxMMCIState *pxa2xx_mmci_init(MemoryRegion *sysmem,
 
     /* Instantiate the actual storage */
     s->card = sd_init(bd, false);
+    if (s->card == NULL) {
+        exit(1);
+    }
 
     register_savevm(NULL, "pxa2xx_mmci", 0, 0,
                     pxa2xx_mmci_save, pxa2xx_mmci_load, s);
diff --git a/hw/sd/sd.c b/hw/sd/sd.c
index 346d86f69c..4502ad143d 100644
--- a/hw/sd/sd.c
+++ b/hw/sd/sd.c
@@ -494,6 +494,11 @@ SDState *sd_init(BlockDriverState *bs, bool is_spi)
 {
     SDState *sd;
 
+    if (bs && bdrv_is_read_only(bs)) {
+        fprintf(stderr, "sd_init: Cannot use read-only drive\n");
+        return NULL;
+    }
+
     sd = (SDState *) g_malloc0(sizeof(SDState));
     sd->buf = qemu_blockalign(bs, 512);
     sd->spi = is_spi;
diff --git a/hw/sd/sdhci.c b/hw/sd/sdhci.c
index 1483e196cd..0906a1d62b 100644
--- a/hw/sd/sdhci.c
+++ b/hw/sd/sdhci.c
@@ -1166,6 +1166,9 @@ static void sdhci_initfn(Object *obj)
 
     di = drive_get_next(IF_SD);
     s->card = sd_init(di ? di->bdrv : NULL, false);
+    if (s->card == NULL) {
+        exit(1);
+    }
     s->eject_cb = qemu_allocate_irqs(sdhci_insert_eject_cb, s, 1)[0];
     s->ro_cb = qemu_allocate_irqs(sdhci_card_readonly_cb, s, 1)[0];
     sd_set_cb(s->card, s->ro_cb, s->eject_cb);
diff --git a/hw/sd/ssi-sd.c b/hw/sd/ssi-sd.c
index d47e2377f9..1bb56c4d54 100644
--- a/hw/sd/ssi-sd.c
+++ b/hw/sd/ssi-sd.c
@@ -246,6 +246,9 @@ static int ssi_sd_init(SSISlave *dev)
     s->mode = SSI_SD_CMD;
     dinfo = drive_get_next(IF_SD);
     s->sd = sd_init(dinfo ? dinfo->bdrv : NULL, true);
+    if (s->sd == NULL) {
+        return -1;
+    }
     register_savevm(&dev->qdev, "ssi_sd", -1, 1, ssi_sd_save, ssi_sd_load, s);
     return 0;
 }
diff --git a/hw/usb/combined-packet.c b/hw/usb/combined-packet.c
index 13f6602ad2..ad77705f8c 100644
--- a/hw/usb/combined-packet.c
+++ b/hw/usb/combined-packet.c
@@ -39,6 +39,7 @@ static void usb_combined_packet_remove(USBCombinedPacket *combined,
     p->combined = NULL;
     QTAILQ_REMOVE(&combined->packets, p, combined_entry);
     if (QTAILQ_EMPTY(&combined->packets)) {
+        qemu_iovec_destroy(&combined->iov);
         g_free(combined);
     }
 }
diff --git a/hw/usb/core.c b/hw/usb/core.c
index 31960c28a8..cf59a1abcf 100644
--- a/hw/usb/core.c
+++ b/hw/usb/core.c
@@ -622,6 +622,7 @@ void usb_ep_reset(USBDevice *dev)
     dev->ep_ctl.nr = 0;
     dev->ep_ctl.type = USB_ENDPOINT_XFER_CONTROL;
     dev->ep_ctl.ifnum = 0;
+    dev->ep_ctl.max_packet_size = 64;
     dev->ep_ctl.dev = dev;
     dev->ep_ctl.pipeline = false;
     for (ep = 0; ep < USB_MAX_ENDPOINTS; ep++) {
@@ -633,6 +634,8 @@ void usb_ep_reset(USBDevice *dev)
         dev->ep_out[ep].type = USB_ENDPOINT_XFER_INVALID;
         dev->ep_in[ep].ifnum = USB_INTERFACE_INVALID;
         dev->ep_out[ep].ifnum = USB_INTERFACE_INVALID;
+        dev->ep_in[ep].max_packet_size = 0;
+        dev->ep_out[ep].max_packet_size = 0;
         dev->ep_in[ep].dev = dev;
         dev->ep_out[ep].dev = dev;
         dev->ep_in[ep].pipeline = false;
diff --git a/hw/usb/hcd-ohci.c b/hw/usb/hcd-ohci.c
index 2b36ee5094..e38cdebfec 100644
--- a/hw/usb/hcd-ohci.c
+++ b/hw/usb/hcd-ohci.c
@@ -1143,7 +1143,9 @@ static int ohci_service_td(OHCIState *ohci, struct ohci_ed *ed)
             switch (ret) {
             case USB_RET_IOERROR:
             case USB_RET_NODEV:
+                DPRINTF("usb-ohci: got DEV ERROR\n");
                 OHCI_SET_BM(td.flags, TD_CC, OHCI_CC_DEVICENOTRESPONDING);
+                break;
             case USB_RET_NAK:
                 DPRINTF("usb-ohci: got NAK\n");
                 return 1;
diff --git a/hw/usb/hcd-xhci.c b/hw/usb/hcd-xhci.c
index cffefd7227..4f0bbb72fc 100644
--- a/hw/usb/hcd-xhci.c
+++ b/hw/usb/hcd-xhci.c
@@ -508,6 +508,8 @@ static TRBCCode xhci_disable_ep(XHCIState *xhci, unsigned int slotid,
                                 unsigned int epid);
 static void xhci_event(XHCIState *xhci, XHCIEvent *event, int v);
 static void xhci_write_event(XHCIState *xhci, XHCIEvent *event, int v);
+static USBEndpoint *xhci_epid_to_usbep(XHCIState *xhci,
+                                       unsigned int slotid, unsigned int epid);
 
 static const char *TRBType_names[] = {
     [TRB_RESERVED]                     = "TRB_RESERVED",
@@ -1137,7 +1139,7 @@ static void xhci_reset_streams(XHCIEPContext *epctx)
 static void xhci_alloc_streams(XHCIEPContext *epctx, dma_addr_t base)
 {
     assert(epctx->pstreams == NULL);
-    epctx->nr_pstreams = 2 << epctx->max_pstreams;
+    epctx->nr_pstreams = 2 << (epctx->max_pstreams + 1);
     epctx->pstreams = xhci_alloc_stream_contexts(epctx->nr_pstreams, base);
 }
 
@@ -1244,6 +1246,9 @@ static XHCIEPContext *xhci_alloc_epctx(XHCIState *xhci,
     epctx->epid = epid;
 
     for (i = 0; i < ARRAY_SIZE(epctx->transfers); i++) {
+        epctx->transfers[i].xhci = xhci;
+        epctx->transfers[i].slotid = slotid;
+        epctx->transfers[i].epid = epid;
         usb_packet_init(&epctx->transfers[i].packet);
     }
     epctx->kick_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, xhci_ep_kick_timer, epctx);
@@ -1357,13 +1362,12 @@ static int xhci_ep_nuke_xfers(XHCIState *xhci, unsigned int slotid,
 
     xferi = epctx->next_xfer;
     for (i = 0; i < TD_QUEUE; i++) {
-        if (epctx->transfers[xferi].packet.ep) {
-            ep = epctx->transfers[xferi].packet.ep;
-        }
         killed += xhci_ep_nuke_one_xfer(&epctx->transfers[xferi]);
         epctx->transfers[xferi].packet.ep = NULL;
         xferi = (xferi + 1) % TD_QUEUE;
     }
+
+    ep = xhci_epid_to_usbep(xhci, slotid, epid);
     if (ep) {
         usb_device_ep_stopped(ep->dev, ep);
     }
@@ -1375,6 +1379,7 @@ static TRBCCode xhci_disable_ep(XHCIState *xhci, unsigned int slotid,
 {
     XHCISlot *slot;
     XHCIEPContext *epctx;
+    int i;
 
     trace_usb_xhci_ep_disable(slotid, epid);
     assert(slotid >= 1 && slotid <= xhci->numslots);
@@ -1395,6 +1400,10 @@ static TRBCCode xhci_disable_ep(XHCIState *xhci, unsigned int slotid,
         xhci_free_streams(epctx);
     }
 
+    for (i = 0; i < ARRAY_SIZE(epctx->transfers); i++) {
+        usb_packet_cleanup(&epctx->transfers[i].packet);
+    }
+
     xhci_set_ep_state(xhci, epctx, NULL, EP_DISABLED);
 
     timer_free(epctx->kick_timer);
@@ -1695,7 +1704,6 @@ static int xhci_submit(XHCIState *xhci, XHCITransfer *xfer,
 static int xhci_setup_packet(XHCITransfer *xfer)
 {
     XHCIState *xhci = xfer->xhci;
-    USBDevice *dev;
     USBEndpoint *ep;
     int dir;
 
@@ -1703,15 +1711,13 @@ static int xhci_setup_packet(XHCITransfer *xfer)
 
     if (xfer->packet.ep) {
         ep = xfer->packet.ep;
-        dev = ep->dev;
     } else {
-        if (!xhci->slots[xfer->slotid-1].uport) {
+        ep = xhci_epid_to_usbep(xhci, xfer->slotid, xfer->epid);
+        if (!ep) {
             fprintf(stderr, "xhci: slot %d has no device\n",
                     xfer->slotid);
             return -1;
         }
-        dev = xhci->slots[xfer->slotid-1].uport->dev;
-        ep = usb_ep_get(dev, dir, xfer->epid >> 1);
     }
 
     xhci_xfer_create_sgl(xfer, dir == USB_TOKEN_IN); /* Also sets int_req */
@@ -1719,7 +1725,7 @@ static int xhci_setup_packet(XHCITransfer *xfer)
                      xfer->trbs[0].addr, false, xfer->int_req);
     usb_packet_map(&xfer->packet, &xfer->sgl);
     DPRINTF("xhci: setup packet pid 0x%x addr %d ep %d\n",
-            xfer->packet.pid, dev->addr, ep->nr);
+            xfer->packet.pid, ep->dev->addr, ep->nr);
     return 0;
 }
 
@@ -2059,9 +2065,6 @@ static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid,
         for (i = 0; i < length; i++) {
             assert(xhci_ring_fetch(xhci, ring, &xfer->trbs[i], NULL));
         }
-        xfer->xhci = xhci;
-        xfer->epid = epid;
-        xfer->slotid = slotid;
         xfer->streamid = streamid;
 
         if (epid == 1) {
@@ -2074,7 +2077,6 @@ static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid,
         } else {
             if (xhci_fire_transfer(xhci, xfer, epctx) >= 0) {
                 epctx->next_xfer = (epctx->next_xfer + 1) % TD_QUEUE;
-                ep = xfer->packet.ep;
             } else {
                 if (!xfer->timed_xfer) {
                     fprintf(stderr, "xhci: error firing data transfer\n");
@@ -2091,6 +2093,8 @@ static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid,
             break;
         }
     }
+
+    ep = xhci_epid_to_usbep(xhci, slotid, epid);
     if (ep) {
         usb_device_flush_ep_queue(ep->dev, ep);
     }
@@ -3320,6 +3324,19 @@ static int xhci_find_epid(USBEndpoint *ep)
     }
 }
 
+static USBEndpoint *xhci_epid_to_usbep(XHCIState *xhci,
+                                       unsigned int slotid, unsigned int epid)
+{
+    assert(slotid >= 1 && slotid <= xhci->numslots);
+
+    if (!xhci->slots[slotid - 1].uport) {
+        return NULL;
+    }
+
+    return usb_ep_get(xhci->slots[slotid - 1].uport->dev,
+                      (epid & 1) ? USB_TOKEN_IN : USB_TOKEN_OUT, epid >> 1);
+}
+
 static void xhci_wakeup_endpoint(USBBus *bus, USBEndpoint *ep,
                                  unsigned int stream)
 {
diff --git a/hw/usb/host-bsd.c b/hw/usb/host-bsd.c
deleted file mode 100644
index 39f22810b3..0000000000
--- a/hw/usb/host-bsd.c
+++ /dev/null
@@ -1,639 +0,0 @@
-/*
- * BSD host USB redirector
- *
- * Copyright (c) 2006 Lonnie Mendez
- * Portions of code and concepts borrowed from
- * usb-linux.c and libusb's bsd.c and are copyright their respective owners.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "qemu-common.h"
-#include "monitor/monitor.h"
-#include "hw/usb.h"
-
-/* usb.h declares these */
-#undef USB_SPEED_HIGH
-#undef USB_SPEED_FULL
-#undef USB_SPEED_LOW
-
-#include <sys/ioctl.h>
-#ifndef __DragonFly__
-#include <dev/usb/usb.h>
-#else
-#include <bus/usb/usb.h>
-#endif
-
-/* This value has maximum potential at 16.
- * You should also set hw.usb.debug to gain
- * more detailed view.
- */
-//#define DEBUG
-#define UGEN_DEBUG_LEVEL 0
-
-
-typedef int USBScanFunc(void *opaque, int bus_num, int addr, int class_id,
-                        int vendor_id, int product_id,
-                        const char *product_name, int speed);
-static int usb_host_find_device(int *pbus_num, int *paddr,
-                                const char *devname);
-
-typedef struct USBHostDevice {
-    USBDevice dev;
-    int ep_fd[USB_MAX_ENDPOINTS];
-    int devfd;
-    char devpath[32];
-} USBHostDevice;
-
-
-static int ensure_ep_open(USBHostDevice *dev, int ep, int mode)
-{
-    char buf[32];
-    int fd;
-
-    /* Get the address for this endpoint */
-    ep = UE_GET_ADDR(ep);
-
-    if (dev->ep_fd[ep] < 0) {
-#if defined(__FreeBSD__) || defined(__DragonFly__)
-        snprintf(buf, sizeof(buf) - 1, "%s.%d", dev->devpath, ep);
-#else
-        snprintf(buf, sizeof(buf) - 1, "%s.%02d", dev->devpath, ep);
-#endif
-        /* Try to open it O_RDWR first for those devices which have in and out
-         * endpoints with the same address (eg 0x02 and 0x82)
-         */
-        fd = open(buf, O_RDWR);
-        if (fd < 0 && errno == ENXIO)
-            fd = open(buf, mode);
-        if (fd < 0) {
-#ifdef DEBUG
-            printf("ensure_ep_open: failed to open device endpoint %s: %s\n",
-                   buf, strerror(errno));
-#endif
-        }
-        dev->ep_fd[ep] = fd;
-    }
-
-    return dev->ep_fd[ep];
-}
-
-static void ensure_eps_closed(USBHostDevice *dev)
-{
-    int epnum = 1;
-
-    if (!dev)
-        return;
-
-    while (epnum < USB_MAX_ENDPOINTS) {
-        if (dev->ep_fd[epnum] >= 0) {
-            close(dev->ep_fd[epnum]);
-            dev->ep_fd[epnum] = -1;
-        }
-        epnum++;
-    }
-}
-
-static void usb_host_handle_reset(USBDevice *dev)
-{
-#if 0
-    USBHostDevice *s = (USBHostDevice *)dev;
-#endif
-}
-
-/* XXX:
- * -check device states against transfer requests
- *  and return appropriate response
- */
-static void usb_host_handle_control(USBDevice *dev,
-                                   USBPacket *p,
-                                   int request,
-                                   int value,
-                                   int index,
-                                   int length,
-                                   uint8_t *data)
-{
-    USBHostDevice *s = (USBHostDevice *)dev;
-    struct usb_ctl_request req;
-    struct usb_alt_interface aiface;
-    int ret, timeout = 50;
-
-    if ((request >> 8) == UT_WRITE_DEVICE &&
-        (request & 0xff) == UR_SET_ADDRESS) {
-
-        /* specific SET_ADDRESS support */
-        dev->addr = value;
-    } else if ((request >> 8) == UT_WRITE_DEVICE &&
-               (request & 0xff) == UR_SET_CONFIG) {
-
-        ensure_eps_closed(s); /* can't do this without all eps closed */
-
-        ret = ioctl(s->devfd, USB_SET_CONFIG, &value);
-        if (ret < 0) {
-#ifdef DEBUG
-            printf("handle_control: failed to set configuration - %s\n",
-                   strerror(errno));
-#endif
-            p->status = USB_RET_STALL;
-        }
-    } else if ((request >> 8) == UT_WRITE_INTERFACE &&
-               (request & 0xff) == UR_SET_INTERFACE) {
-
-        aiface.uai_interface_index = index;
-        aiface.uai_alt_no = value;
-
-        ensure_eps_closed(s); /* can't do this without all eps closed */
-        ret = ioctl(s->devfd, USB_SET_ALTINTERFACE, &aiface);
-        if (ret < 0) {
-#ifdef DEBUG
-            printf("handle_control: failed to set alternate interface - %s\n",
-                   strerror(errno));
-#endif
-            p->status = USB_RET_STALL;
-        }
-    } else {
-        req.ucr_request.bmRequestType = request >> 8;
-        req.ucr_request.bRequest = request & 0xff;
-        USETW(req.ucr_request.wValue, value);
-        USETW(req.ucr_request.wIndex, index);
-        USETW(req.ucr_request.wLength, length);
-        req.ucr_data = data;
-        req.ucr_flags = USBD_SHORT_XFER_OK;
-
-        ret = ioctl(s->devfd, USB_SET_TIMEOUT, &timeout);
-#if defined(__NetBSD__) || defined(__OpenBSD__)
-        if (ret < 0 && errno != EINVAL) {
-#else
-        if (ret < 0) {
-#endif
-#ifdef DEBUG
-            printf("handle_control: setting timeout failed - %s\n",
-                   strerror(errno));
-#endif
-        }
-
-        ret = ioctl(s->devfd, USB_DO_REQUEST, &req);
-        /* ugen returns EIO for usbd_do_request_ no matter what
-         * happens with the transfer */
-        if (ret < 0) {
-#ifdef DEBUG
-            printf("handle_control: error after request - %s\n",
-                   strerror(errno));
-#endif
-            p->status = USB_RET_NAK; /* STALL */
-        } else {
-            p->actual_length = req.ucr_actlen;
-        }
-    }
-}
-
-static void usb_host_handle_data(USBDevice *dev, USBPacket *p)
-{
-    USBHostDevice *s = (USBHostDevice *)dev;
-    int ret, fd, mode;
-    int one = 1, shortpacket = 0, timeout = 50;
-    sigset_t new_mask, old_mask;
-    uint8_t devep = p->ep->nr;
-
-    /* protect data transfers from SIGALRM signal */
-    sigemptyset(&new_mask);
-    sigaddset(&new_mask, SIGALRM);
-    sigprocmask(SIG_BLOCK, &new_mask, &old_mask);
-
-    if (p->pid == USB_TOKEN_IN) {
-        devep |= 0x80;
-        mode = O_RDONLY;
-        shortpacket = 1;
-    } else {
-        mode = O_WRONLY;
-    }
-
-    fd = ensure_ep_open(s, devep, mode);
-    if (fd < 0) {
-        sigprocmask(SIG_SETMASK, &old_mask, NULL);
-        p->status = USB_RET_NODEV;
-        return;
-    }
-
-    if (ioctl(fd, USB_SET_TIMEOUT, &timeout) < 0) {
-#ifdef DEBUG
-        printf("handle_data: failed to set timeout - %s\n",
-               strerror(errno));
-#endif
-    }
-
-    if (shortpacket) {
-        if (ioctl(fd, USB_SET_SHORT_XFER, &one) < 0) {
-#ifdef DEBUG
-            printf("handle_data: failed to set short xfer mode - %s\n",
-                   strerror(errno));
-#endif
-            sigprocmask(SIG_SETMASK, &old_mask, NULL);
-        }
-    }
-
-    if (p->pid == USB_TOKEN_IN)
-        ret = readv(fd, p->iov.iov, p->iov.niov);
-    else
-        ret = writev(fd, p->iov.iov, p->iov.niov);
-
-    sigprocmask(SIG_SETMASK, &old_mask, NULL);
-
-    if (ret < 0) {
-#ifdef DEBUG
-        printf("handle_data: error after %s data - %s\n",
-               pid == USB_TOKEN_IN ? "reading" : "writing", strerror(errno));
-#endif
-        switch(errno) {
-        case ETIMEDOUT:
-        case EINTR:
-            p->status = USB_RET_NAK;
-            break;
-        default:
-            p->status = USB_RET_STALL;
-        }
-    } else {
-        p->actual_length = ret;
-    }
-}
-
-static void usb_host_handle_destroy(USBDevice *opaque)
-{
-    USBHostDevice *s = (USBHostDevice *)opaque;
-    int i;
-
-    for (i = 0; i < USB_MAX_ENDPOINTS; i++)
-        if (s->ep_fd[i] >= 0)
-            close(s->ep_fd[i]);
-
-    if (s->devfd < 0)
-        return;
-
-    close(s->devfd);
-
-    g_free(s);
-}
-
-static int usb_host_initfn(USBDevice *dev)
-{
-    dev->flags |= (1 << USB_DEV_FLAG_IS_HOST);
-    return 0;
-}
-
-USBDevice *usb_host_device_open(USBBus *guest_bus, const char *devname)
-{
-    struct usb_device_info bus_info, dev_info;
-    USBDevice *d = NULL, *ret = NULL;
-    USBHostDevice *dev;
-    char ctlpath[PATH_MAX + 1];
-    char buspath[PATH_MAX + 1];
-    int bfd, dfd, bus, address, i;
-    int ugendebug = UGEN_DEBUG_LEVEL;
-
-    if (usb_host_find_device(&bus, &address, devname) < 0) {
-        goto fail;
-    }
-
-    snprintf(buspath, PATH_MAX, "/dev/usb%d", bus);
-
-    bfd = open(buspath, O_RDWR);
-    if (bfd < 0) {
-#ifdef DEBUG
-        printf("usb_host_device_open: failed to open usb bus - %s\n",
-               strerror(errno));
-#endif
-        goto fail;
-    }
-
-    bus_info.udi_addr = address;
-    if (ioctl(bfd, USB_DEVICEINFO, &bus_info) < 0) {
-#ifdef DEBUG
-        printf("usb_host_device_open: failed to grab bus information - %s\n",
-               strerror(errno));
-#endif
-        goto fail_bfd;
-    }
-
-#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__DragonFly__)
-    snprintf(ctlpath, PATH_MAX, "/dev/%s", bus_info.udi_devnames[0]);
-#else
-    snprintf(ctlpath, PATH_MAX, "/dev/%s.00", bus_info.udi_devnames[0]);
-#endif
-
-    dfd  = open(ctlpath, O_RDWR);
-    if (dfd < 0) {
-        dfd = open(ctlpath, O_RDONLY);
-        if (dfd < 0) {
-#ifdef DEBUG
-            printf("usb_host_device_open: failed to open usb device %s - %s\n",
-                   ctlpath, strerror(errno));
-#endif
-        }
-        goto fail_dfd;
-    }
-
-    if (ioctl(dfd, USB_GET_DEVICEINFO, &dev_info) < 0) {
-#ifdef DEBUG
-        printf("usb_host_device_open: failed to grab device info - %s\n",
-               strerror(errno));
-#endif
-        goto fail_dfd;
-    }
-
-    d = usb_create(guest_bus, "usb-host");
-    dev = DO_UPCAST(USBHostDevice, dev, d);
-
-    if (dev_info.udi_speed == 1) {
-        dev->dev.speed = USB_SPEED_LOW - 1;
-        dev->dev.speedmask = USB_SPEED_MASK_LOW;
-    } else {
-        dev->dev.speed = USB_SPEED_FULL - 1;
-        dev->dev.speedmask = USB_SPEED_MASK_FULL;
-    }
-
-    if (strncmp(dev_info.udi_product, "product", 7) != 0) {
-        pstrcpy(dev->dev.product_desc, sizeof(dev->dev.product_desc),
-                dev_info.udi_product);
-    } else {
-        snprintf(dev->dev.product_desc, sizeof(dev->dev.product_desc),
-                 "host:%s", devname);
-    }
-
-    pstrcpy(dev->devpath, sizeof(dev->devpath), "/dev/");
-    pstrcat(dev->devpath, sizeof(dev->devpath), dev_info.udi_devnames[0]);
-
-    /* Mark the endpoints as not yet open */
-    for (i = 0; i < USB_MAX_ENDPOINTS; i++) {
-        dev->ep_fd[i] = -1;
-    }
-
-    ioctl(dfd, USB_SETDEBUG, &ugendebug);
-
-    ret = (USBDevice *)dev;
-
-fail_dfd:
-    close(dfd);
-fail_bfd:
-    close(bfd);
-fail:
-    return ret;
-}
-
-static void usb_host_class_initfn(ObjectClass *klass, void *data)
-{
-    USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
-
-    uc->product_desc   = "USB Host Device";
-    uc->init           = usb_host_initfn;
-    uc->handle_reset   = usb_host_handle_reset;
-    uc->handle_control = usb_host_handle_control;
-    uc->handle_data    = usb_host_handle_data;
-    uc->handle_destroy = usb_host_handle_destroy;
-}
-
-static const TypeInfo usb_host_dev_info = {
-    .name          = "usb-host",
-    .parent        = TYPE_USB_DEVICE,
-    .instance_size = sizeof(USBHostDevice),
-    .class_init    = usb_host_class_initfn,
-};
-
-static void usb_host_register_types(void)
-{
-    type_register_static(&usb_host_dev_info);
-}
-
-type_init(usb_host_register_types)
-
-static int usb_host_scan(void *opaque, USBScanFunc *func)
-{
-    struct usb_device_info bus_info;
-    struct usb_device_info dev_info;
-    uint16_t vendor_id, product_id, class_id, speed;
-    int bfd, dfd, bus, address;
-    char busbuf[20], devbuf[20], product_name[256];
-    int ret = 0;
-
-    for (bus = 0; bus < 10; bus++) {
-
-        snprintf(busbuf, sizeof(busbuf) - 1, "/dev/usb%d", bus);
-        bfd = open(busbuf, O_RDWR);
-        if (bfd < 0)
-	    continue;
-
-        for (address = 1; address < 127; address++) {
-
-            bus_info.udi_addr = address;
-            if (ioctl(bfd, USB_DEVICEINFO, &bus_info) < 0)
-                continue;
-
-            /* only list devices that can be used by generic layer */
-            if (strncmp(bus_info.udi_devnames[0], "ugen", 4) != 0)
-                continue;
-
-#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__DragonFly__)
-            snprintf(devbuf, sizeof(devbuf) - 1, "/dev/%s", bus_info.udi_devnames[0]);
-#else
-            snprintf(devbuf, sizeof(devbuf) - 1, "/dev/%s.00", bus_info.udi_devnames[0]);
-#endif
-
-            dfd = open(devbuf, O_RDONLY);
-            if (dfd < 0) {
-#ifdef DEBUG
-                printf("usb_host_scan: couldn't open device %s - %s\n", devbuf,
-                       strerror(errno));
-#endif
-                continue;
-            }
-
-            if (ioctl(dfd, USB_GET_DEVICEINFO, &dev_info) < 0)
-                printf("usb_host_scan: couldn't get device information for %s - %s\n",
-                       devbuf, strerror(errno));
-
-            /* XXX: might need to fixup endianness of word values before copying over */
-
-            vendor_id = dev_info.udi_vendorNo;
-            product_id = dev_info.udi_productNo;
-            class_id = dev_info.udi_class;
-            speed = dev_info.udi_speed;
-
-            if (strncmp(dev_info.udi_product, "product", 7) != 0)
-                pstrcpy(product_name, sizeof(product_name),
-                        dev_info.udi_product);
-            else
-                product_name[0] = '\0';
-
-            ret = func(opaque, bus, address, class_id, vendor_id,
-                       product_id, product_name, speed);
-
-            close(dfd);
-
-            if (ret)
-                goto the_end;
-        }
-
-        close(bfd);
-    }
-
-the_end:
-    return ret;
-}
-
-typedef struct FindDeviceState {
-    int vendor_id;
-    int product_id;
-    int bus_num;
-    int addr;
-} FindDeviceState;
-
-static int usb_host_find_device_scan(void *opaque, int bus_num, int addr,
-                                     int class_id,
-                                     int vendor_id, int product_id,
-                                     const char *product_name, int speed)
-{
-    FindDeviceState *s = opaque;
-    if (vendor_id == s->vendor_id &&
-        product_id == s->product_id) {
-        s->bus_num = bus_num;
-        s->addr = addr;
-        return 1;
-     } else {
-        return 0;
-     }
-}
-
-
-/* the syntax is :
-   'bus.addr' (decimal numbers) or
-   'vendor_id:product_id' (hexa numbers) */
-static int usb_host_find_device(int *pbus_num, int *paddr,
-                                const char *devname)
-{
-    const char *p;
-    int ret;
-    FindDeviceState fs;
-
-    p = strchr(devname, '.');
-    if (p) {
-        *pbus_num = strtoul(devname, NULL, 0);
-        *paddr = strtoul(p + 1, NULL, 0);
-        return 0;
-    }
-    p = strchr(devname, ':');
-    if (p) {
-        fs.vendor_id = strtoul(devname, NULL, 16);
-        fs.product_id = strtoul(p + 1, NULL, 16);
-        ret = usb_host_scan(&fs, usb_host_find_device_scan);
-        if (ret) {
-            *pbus_num = fs.bus_num;
-            *paddr = fs.addr;
-            return 0;
-        }
-     }
-     return -1;
-}
-
-/**********************/
-/* USB host device info */
-
-struct usb_class_info {
-    int class;
-    const char *class_name;
-};
-
-static const struct usb_class_info usb_class_info[] = {
-    { USB_CLASS_AUDIO, "Audio"},
-    { USB_CLASS_COMM, "Communication"},
-    { USB_CLASS_HID, "HID"},
-    { USB_CLASS_HUB, "Hub" },
-    { USB_CLASS_PHYSICAL, "Physical" },
-    { USB_CLASS_PRINTER, "Printer" },
-    { USB_CLASS_MASS_STORAGE, "Storage" },
-    { USB_CLASS_CDC_DATA, "Data" },
-    { USB_CLASS_APP_SPEC, "Application Specific" },
-    { USB_CLASS_VENDOR_SPEC, "Vendor Specific" },
-    { USB_CLASS_STILL_IMAGE, "Still Image" },
-    { USB_CLASS_CSCID, "Smart Card" },
-    { USB_CLASS_CONTENT_SEC, "Content Security" },
-    { -1, NULL }
-};
-
-static const char *usb_class_str(uint8_t class)
-{
-    const struct usb_class_info *p;
-    for (p = usb_class_info; p->class != -1; p++) {
-        if (p->class == class)
-            break;
-    }
-    return p->class_name;
-}
-
-static void usb_info_device(Monitor *mon, int bus_num, int addr, int class_id,
-                            int vendor_id, int product_id,
-                            const char *product_name,
-                            int speed)
-{
-    const char *class_str, *speed_str;
-
-    switch(speed) {
-    case USB_SPEED_LOW:
-        speed_str = "1.5";
-        break;
-    case USB_SPEED_FULL:
-        speed_str = "12";
-        break;
-    case USB_SPEED_HIGH:
-        speed_str = "480";
-        break;
-    default:
-        speed_str = "?";
-        break;
-    }
-
-    monitor_printf(mon, "  Device %d.%d, speed %s Mb/s\n",
-                   bus_num, addr, speed_str);
-    class_str = usb_class_str(class_id);
-    if (class_str)
-        monitor_printf(mon, "    %s:", class_str);
-    else
-        monitor_printf(mon, "    Class %02x:", class_id);
-    monitor_printf(mon, " USB device %04x:%04x", vendor_id, product_id);
-    if (product_name[0] != '\0')
-        monitor_printf(mon, ", %s", product_name);
-    monitor_printf(mon, "\n");
-}
-
-static int usb_host_info_device(void *opaque,
-                                int bus_num, int addr,
-                                int class_id,
-                                int vendor_id, int product_id,
-                                const char *product_name,
-                                int speed)
-{
-    Monitor *mon = opaque;
-
-    usb_info_device(mon, bus_num, addr, class_id, vendor_id, product_id,
-                    product_name, speed);
-    return 0;
-}
-
-void usb_host_info(Monitor *mon, const QDict *qdict)
-{
-    usb_host_scan(mon, usb_host_info_device);
-}
diff --git a/hw/usb/host-linux.c b/hw/usb/host-linux.c
deleted file mode 100644
index 65cd3b444c..0000000000
--- a/hw/usb/host-linux.c
+++ /dev/null
@@ -1,1911 +0,0 @@
-/*
- * Linux host USB redirector
- *
- * Copyright (c) 2005 Fabrice Bellard
- *
- * Copyright (c) 2008 Max Krasnyansky
- *      Support for host device auto connect & disconnect
- *      Major rewrite to support fully async operation
- *
- * Copyright 2008 TJ <linux@tjworld.net>
- *      Added flexible support for /dev/bus/usb /sys/bus/usb/devices in addition
- *      to the legacy /proc/bus/usb USB device discovery and handling
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "qemu-common.h"
-#include "qemu/timer.h"
-#include "monitor/monitor.h"
-#include "sysemu/sysemu.h"
-#include "trace.h"
-
-#include <dirent.h>
-#include <sys/ioctl.h>
-
-#include <linux/usbdevice_fs.h>
-#include <linux/version.h>
-#include "hw/usb.h"
-#include "hw/usb/desc.h"
-#include "hw/usb/host.h"
-
-#ifdef CONFIG_USB_LIBUSB
-# define DEVNAME "usb-host-linux"
-#else
-# define DEVNAME "usb-host"
-#endif
-
-/* We redefine it to avoid version problems */
-struct usb_ctrltransfer {
-    uint8_t  bRequestType;
-    uint8_t  bRequest;
-    uint16_t wValue;
-    uint16_t wIndex;
-    uint16_t wLength;
-    uint32_t timeout;
-    void *data;
-};
-
-typedef int USBScanFunc(void *opaque, int bus_num, int addr, const char *port,
-                        int class_id, int vendor_id, int product_id,
-                        const char *product_name, int speed);
-
-//#define DEBUG
-
-#ifdef DEBUG
-#define DPRINTF printf
-#else
-#define DPRINTF(...)
-#endif
-
-#define PRODUCT_NAME_SZ 32
-#define MAX_PORTLEN 16
-
-/* endpoint association data */
-#define ISO_FRAME_DESC_PER_URB 32
-
-/* devio.c limits single requests to 16k */
-#define MAX_USBFS_BUFFER_SIZE 16384
-
-typedef struct AsyncURB AsyncURB;
-
-struct endp_data {
-    uint8_t halted;
-    uint8_t iso_started;
-    AsyncURB *iso_urb;
-    int iso_urb_idx;
-    int iso_buffer_used;
-    int inflight;
-};
-
-enum USBHostDeviceOptions {
-    USB_HOST_OPT_PIPELINE,
-};
-
-typedef struct USBHostDevice {
-    USBDevice dev;
-    int       fd;
-    int       hub_fd;
-    int       hub_port;
-
-    uint8_t   descr[8192];
-    int       descr_len;
-    int       closing;
-    uint32_t  iso_urb_count;
-    uint32_t  options;
-    Notifier  exit;
-    QEMUBH    *bh;
-
-    struct endp_data ep_in[USB_MAX_ENDPOINTS];
-    struct endp_data ep_out[USB_MAX_ENDPOINTS];
-    QLIST_HEAD(, AsyncURB) aurbs;
-
-    /* Host side address */
-    int bus_num;
-    int addr;
-    char port[MAX_PORTLEN];
-    struct USBAutoFilter match;
-    int32_t bootindex;
-    int seen, errcount;
-
-    QTAILQ_ENTRY(USBHostDevice) next;
-} USBHostDevice;
-
-static QTAILQ_HEAD(, USBHostDevice) hostdevs = QTAILQ_HEAD_INITIALIZER(hostdevs);
-
-static int usb_host_close(USBHostDevice *dev);
-static void usb_host_auto_check(void *unused);
-static int usb_host_read_file(char *line, size_t line_size,
-                            const char *device_file, const char *device_name);
-static void usb_linux_update_endp_table(USBHostDevice *s);
-
-static int usb_host_usbfs_type(USBHostDevice *s, USBPacket *p)
-{
-    static const int usbfs[] = {
-        [USB_ENDPOINT_XFER_CONTROL] = USBDEVFS_URB_TYPE_CONTROL,
-        [USB_ENDPOINT_XFER_ISOC]    = USBDEVFS_URB_TYPE_ISO,
-        [USB_ENDPOINT_XFER_BULK]    = USBDEVFS_URB_TYPE_BULK,
-        [USB_ENDPOINT_XFER_INT]     = USBDEVFS_URB_TYPE_INTERRUPT,
-    };
-    uint8_t type = p->ep->type;
-    assert(type < ARRAY_SIZE(usbfs));
-    return usbfs[type];
-}
-
-static int usb_host_do_reset(USBHostDevice *dev)
-{
-    struct timeval s, e;
-    uint32_t usecs;
-    int ret;
-
-    gettimeofday(&s, NULL);
-    ret = ioctl(dev->fd, USBDEVFS_RESET);
-    gettimeofday(&e, NULL);
-    usecs = (e.tv_sec  - s.tv_sec) * 1000000;
-    usecs += e.tv_usec - s.tv_usec;
-    if (usecs > 1000000) {
-        /* more than a second, something is fishy, broken usb device? */
-        fprintf(stderr, "husb: device %d:%d reset took %d.%06d seconds\n",
-                dev->bus_num, dev->addr, usecs / 1000000, usecs % 1000000);
-    }
-    return ret;
-}
-
-static struct endp_data *get_endp(USBHostDevice *s, int pid, int ep)
-{
-    struct endp_data *eps = pid == USB_TOKEN_IN ? s->ep_in : s->ep_out;
-    assert(pid == USB_TOKEN_IN || pid == USB_TOKEN_OUT);
-    assert(ep > 0 && ep <= USB_MAX_ENDPOINTS);
-    return eps + ep - 1;
-}
-
-static int is_isoc(USBHostDevice *s, int pid, int ep)
-{
-    return usb_ep_get_type(&s->dev, pid, ep) == USB_ENDPOINT_XFER_ISOC;
-}
-
-static int is_valid(USBHostDevice *s, int pid, int ep)
-{
-    return usb_ep_get_type(&s->dev, pid, ep) != USB_ENDPOINT_XFER_INVALID;
-}
-
-static int is_halted(USBHostDevice *s, int pid, int ep)
-{
-    return get_endp(s, pid, ep)->halted;
-}
-
-static void clear_halt(USBHostDevice *s, int pid, int ep)
-{
-    trace_usb_host_ep_clear_halt(s->bus_num, s->addr, ep);
-    get_endp(s, pid, ep)->halted = 0;
-}
-
-static void set_halt(USBHostDevice *s, int pid, int ep)
-{
-    if (ep != 0) {
-        trace_usb_host_ep_set_halt(s->bus_num, s->addr, ep);
-        get_endp(s, pid, ep)->halted = 1;
-    }
-}
-
-static int is_iso_started(USBHostDevice *s, int pid, int ep)
-{
-    return get_endp(s, pid, ep)->iso_started;
-}
-
-static void clear_iso_started(USBHostDevice *s, int pid, int ep)
-{
-    trace_usb_host_iso_stop(s->bus_num, s->addr, ep);
-    get_endp(s, pid, ep)->iso_started = 0;
-}
-
-static void set_iso_started(USBHostDevice *s, int pid, int ep)
-{
-    struct endp_data *e = get_endp(s, pid, ep);
-
-    trace_usb_host_iso_start(s->bus_num, s->addr, ep);
-    if (!e->iso_started) {
-        e->iso_started = 1;
-        e->inflight = 0;
-    }
-}
-
-static int change_iso_inflight(USBHostDevice *s, int pid, int ep, int value)
-{
-    struct endp_data *e = get_endp(s, pid, ep);
-
-    e->inflight += value;
-    return e->inflight;
-}
-
-static void set_iso_urb(USBHostDevice *s, int pid, int ep, AsyncURB *iso_urb)
-{
-    get_endp(s, pid, ep)->iso_urb = iso_urb;
-}
-
-static AsyncURB *get_iso_urb(USBHostDevice *s, int pid, int ep)
-{
-    return get_endp(s, pid, ep)->iso_urb;
-}
-
-static void set_iso_urb_idx(USBHostDevice *s, int pid, int ep, int i)
-{
-    get_endp(s, pid, ep)->iso_urb_idx = i;
-}
-
-static int get_iso_urb_idx(USBHostDevice *s, int pid, int ep)
-{
-    return get_endp(s, pid, ep)->iso_urb_idx;
-}
-
-static void set_iso_buffer_used(USBHostDevice *s, int pid, int ep, int i)
-{
-    get_endp(s, pid, ep)->iso_buffer_used = i;
-}
-
-static int get_iso_buffer_used(USBHostDevice *s, int pid, int ep)
-{
-    return get_endp(s, pid, ep)->iso_buffer_used;
-}
-
-/*
- * Async URB state.
- * We always allocate iso packet descriptors even for bulk transfers
- * to simplify allocation and casts.
- */
-struct AsyncURB
-{
-    struct usbdevfs_urb urb;
-    struct usbdevfs_iso_packet_desc isocpd[ISO_FRAME_DESC_PER_URB];
-    USBHostDevice *hdev;
-    QLIST_ENTRY(AsyncURB) next;
-
-    /* For regular async urbs */
-    USBPacket     *packet;
-    int more; /* large transfer, more urbs follow */
-
-    /* For buffered iso handling */
-    int iso_frame_idx; /* -1 means in flight */
-};
-
-static AsyncURB *async_alloc(USBHostDevice *s)
-{
-    AsyncURB *aurb = g_malloc0(sizeof(AsyncURB));
-    aurb->hdev = s;
-    QLIST_INSERT_HEAD(&s->aurbs, aurb, next);
-    return aurb;
-}
-
-static void async_free(AsyncURB *aurb)
-{
-    QLIST_REMOVE(aurb, next);
-    g_free(aurb);
-}
-
-static void do_disconnect(USBHostDevice *s)
-{
-    usb_host_close(s);
-    usb_host_auto_check(NULL);
-}
-
-static void async_complete(void *opaque)
-{
-    USBHostDevice *s = opaque;
-    AsyncURB *aurb;
-    int urbs = 0;
-
-    while (1) {
-        USBPacket *p;
-
-        int r = ioctl(s->fd, USBDEVFS_REAPURBNDELAY, &aurb);
-        if (r < 0) {
-            if (errno == EAGAIN) {
-                if (urbs > 2) {
-                    /* indicates possible latency issues */
-                    trace_usb_host_iso_many_urbs(s->bus_num, s->addr, urbs);
-                }
-                return;
-            }
-            if (errno == ENODEV) {
-                if (!s->closing) {
-                    trace_usb_host_disconnect(s->bus_num, s->addr);
-                    do_disconnect(s);
-                }
-                return;
-            }
-
-            perror("USBDEVFS_REAPURBNDELAY");
-            return;
-        }
-
-        DPRINTF("husb: async completed. aurb %p status %d alen %d\n",
-                aurb, aurb->urb.status, aurb->urb.actual_length);
-
-        /* If this is a buffered iso urb mark it as complete and don't do
-           anything else (it is handled further in usb_host_handle_iso_data) */
-        if (aurb->iso_frame_idx == -1) {
-            int inflight;
-            int pid = (aurb->urb.endpoint & USB_DIR_IN) ?
-                USB_TOKEN_IN : USB_TOKEN_OUT;
-            int ep = aurb->urb.endpoint & 0xf;
-            if (aurb->urb.status == -EPIPE) {
-                set_halt(s, pid, ep);
-            }
-            aurb->iso_frame_idx = 0;
-            urbs++;
-            inflight = change_iso_inflight(s, pid, ep, -1);
-            if (inflight == 0 && is_iso_started(s, pid, ep)) {
-                /* can be latency issues, or simply end of stream */
-                trace_usb_host_iso_out_of_bufs(s->bus_num, s->addr, ep);
-            }
-            continue;
-        }
-
-        p = aurb->packet;
-        trace_usb_host_urb_complete(s->bus_num, s->addr, aurb, aurb->urb.status,
-                                    aurb->urb.actual_length, aurb->more);
-
-        if (p) {
-            switch (aurb->urb.status) {
-            case 0:
-                p->actual_length += aurb->urb.actual_length;
-                if (!aurb->more) {
-                    /* Clear previous ASYNC status */
-                    p->status = USB_RET_SUCCESS;
-                }
-                break;
-
-            case -EPIPE:
-                set_halt(s, p->pid, p->ep->nr);
-                p->status = USB_RET_STALL;
-                break;
-
-            case -EOVERFLOW:
-                p->status = USB_RET_BABBLE;
-                break;
-
-            default:
-                p->status = USB_RET_IOERROR;
-                break;
-            }
-
-            if (aurb->urb.type == USBDEVFS_URB_TYPE_CONTROL) {
-                trace_usb_host_req_complete(s->bus_num, s->addr, p,
-                                            p->status, aurb->urb.actual_length);
-                usb_generic_async_ctrl_complete(&s->dev, p);
-            } else if (!aurb->more) {
-                trace_usb_host_req_complete(s->bus_num, s->addr, p,
-                                            p->status, aurb->urb.actual_length);
-                usb_packet_complete(&s->dev, p);
-            }
-        }
-
-        async_free(aurb);
-    }
-}
-
-static void usb_host_async_cancel(USBDevice *dev, USBPacket *p)
-{
-    USBHostDevice *s = DO_UPCAST(USBHostDevice, dev, dev);
-    AsyncURB *aurb;
-
-    trace_usb_host_req_canceled(s->bus_num, s->addr, p);
-
-    QLIST_FOREACH(aurb, &s->aurbs, next) {
-        if (p != aurb->packet) {
-            continue;
-        }
-
-        trace_usb_host_urb_canceled(s->bus_num, s->addr, aurb);
-
-        /* Mark it as dead (see async_complete above) */
-        aurb->packet = NULL;
-
-        int r = ioctl(s->fd, USBDEVFS_DISCARDURB, aurb);
-        if (r < 0) {
-            DPRINTF("husb: async. discard urb failed errno %d\n", errno);
-        }
-    }
-}
-
-static int usb_host_open_device(int bus, int addr)
-{
-    const char *usbfs = NULL;
-    char filename[32];
-    struct stat st;
-    int fd, rc;
-
-    rc = stat("/dev/bus/usb", &st);
-    if (rc == 0 && S_ISDIR(st.st_mode)) {
-        /* udev-created device nodes available */
-        usbfs = "/dev/bus/usb";
-    } else {
-        /* fallback: usbfs mounted below /proc */
-        usbfs = "/proc/bus/usb";
-    }
-
-    snprintf(filename, sizeof(filename), "%s/%03d/%03d",
-             usbfs, bus, addr);
-    fd = open(filename, O_RDWR | O_NONBLOCK);
-    if (fd < 0) {
-        fprintf(stderr, "husb: open %s: %s\n", filename, strerror(errno));
-    }
-    return fd;
-}
-
-static int usb_host_claim_port(USBHostDevice *s)
-{
-#ifdef USBDEVFS_CLAIM_PORT
-    char *h, hub_name[64], line[1024];
-    int hub_addr, ret;
-
-    snprintf(hub_name, sizeof(hub_name), "%d-%s",
-             s->match.bus_num, s->match.port);
-
-    /* try strip off last ".$portnr" to get hub */
-    h = strrchr(hub_name, '.');
-    if (h != NULL) {
-        s->hub_port = atoi(h+1);
-        *h = '\0';
-    } else {
-        /* no dot in there -> it is the root hub */
-        snprintf(hub_name, sizeof(hub_name), "usb%d",
-                 s->match.bus_num);
-        s->hub_port = atoi(s->match.port);
-    }
-
-    if (!usb_host_read_file(line, sizeof(line), "devnum",
-                            hub_name)) {
-        return -1;
-    }
-    if (sscanf(line, "%d", &hub_addr) != 1) {
-        return -1;
-    }
-
-    s->hub_fd = usb_host_open_device(s->match.bus_num, hub_addr);
-    if (s->hub_fd < 0) {
-        return -1;
-    }
-
-    ret = ioctl(s->hub_fd, USBDEVFS_CLAIM_PORT, &s->hub_port);
-    if (ret < 0) {
-        close(s->hub_fd);
-        s->hub_fd = -1;
-        return -1;
-    }
-
-    trace_usb_host_claim_port(s->match.bus_num, hub_addr, s->hub_port);
-    return 0;
-#else
-    return -1;
-#endif
-}
-
-static void usb_host_release_port(USBHostDevice *s)
-{
-    if (s->hub_fd == -1) {
-        return;
-    }
-#ifdef USBDEVFS_RELEASE_PORT
-    ioctl(s->hub_fd, USBDEVFS_RELEASE_PORT, &s->hub_port);
-#endif
-    close(s->hub_fd);
-    s->hub_fd = -1;
-}
-
-static int usb_host_disconnect_ifaces(USBHostDevice *dev, int nb_interfaces)
-{
-    /* earlier Linux 2.4 do not support that */
-#ifdef USBDEVFS_DISCONNECT
-    struct usbdevfs_ioctl ctrl;
-    int ret, interface;
-
-    for (interface = 0; interface < nb_interfaces; interface++) {
-        ctrl.ioctl_code = USBDEVFS_DISCONNECT;
-        ctrl.ifno = interface;
-        ctrl.data = 0;
-        ret = ioctl(dev->fd, USBDEVFS_IOCTL, &ctrl);
-        if (ret < 0 && errno != ENODATA) {
-            perror("USBDEVFS_DISCONNECT");
-            return -1;
-        }
-    }
-#endif
-    return 0;
-}
-
-static int usb_linux_get_num_interfaces(USBHostDevice *s)
-{
-    char device_name[64], line[1024];
-    int num_interfaces = 0;
-
-    sprintf(device_name, "%d-%s", s->bus_num, s->port);
-    if (!usb_host_read_file(line, sizeof(line), "bNumInterfaces",
-                            device_name)) {
-        return -1;
-    }
-    if (sscanf(line, "%d", &num_interfaces) != 1) {
-        return -1;
-    }
-    return num_interfaces;
-}
-
-static int usb_host_claim_interfaces(USBHostDevice *dev, int configuration)
-{
-    const char *op = NULL;
-    int dev_descr_len, config_descr_len;
-    int interface, nb_interfaces;
-    int ret, i;
-
-    for (i = 0; i < USB_MAX_INTERFACES; i++) {
-        dev->dev.altsetting[i] = 0;
-    }
-
-    if (configuration == 0) { /* address state - ignore */
-        dev->dev.ninterfaces   = 0;
-        dev->dev.configuration = 0;
-        return 1;
-    }
-
-    DPRINTF("husb: claiming interfaces. config %d\n", configuration);
-
-    i = 0;
-    dev_descr_len = dev->descr[0];
-    if (dev_descr_len > dev->descr_len) {
-        fprintf(stderr, "husb: update iface failed. descr too short\n");
-        return 0;
-    }
-
-    i += dev_descr_len;
-    while (i < dev->descr_len) {
-        DPRINTF("husb: i is %d, descr_len is %d, dl %d, dt %d\n",
-                i, dev->descr_len,
-               dev->descr[i], dev->descr[i+1]);
-
-        if (dev->descr[i+1] != USB_DT_CONFIG) {
-            i += dev->descr[i];
-            continue;
-        }
-        config_descr_len = dev->descr[i];
-
-        DPRINTF("husb: config #%d need %d\n", dev->descr[i + 5], configuration);
-
-        if (configuration == dev->descr[i + 5]) {
-            configuration = dev->descr[i + 5];
-            break;
-        }
-
-        i += config_descr_len;
-    }
-
-    if (i >= dev->descr_len) {
-        fprintf(stderr,
-                "husb: update iface failed. no matching configuration\n");
-        return 0;
-    }
-    nb_interfaces = dev->descr[i + 4];
-
-    if (usb_host_disconnect_ifaces(dev, nb_interfaces) < 0) {
-        goto fail;
-    }
-
-    /* XXX: only grab if all interfaces are free */
-    for (interface = 0; interface < nb_interfaces; interface++) {
-        op = "USBDEVFS_CLAIMINTERFACE";
-        ret = ioctl(dev->fd, USBDEVFS_CLAIMINTERFACE, &interface);
-        if (ret < 0) {
-            goto fail;
-        }
-    }
-
-    trace_usb_host_claim_interfaces(dev->bus_num, dev->addr,
-                                    nb_interfaces, configuration);
-
-    dev->dev.ninterfaces   = nb_interfaces;
-    dev->dev.configuration = configuration;
-    return 1;
-
-fail:
-    if (errno == ENODEV) {
-        do_disconnect(dev);
-    }
-    perror(op);
-    return 0;
-}
-
-static int usb_host_release_interfaces(USBHostDevice *s)
-{
-    int ret, i;
-
-    trace_usb_host_release_interfaces(s->bus_num, s->addr);
-
-    for (i = 0; i < s->dev.ninterfaces; i++) {
-        ret = ioctl(s->fd, USBDEVFS_RELEASEINTERFACE, &i);
-        if (ret < 0) {
-            perror("USBDEVFS_RELEASEINTERFACE");
-            return 0;
-        }
-    }
-    return 1;
-}
-
-static void usb_host_handle_reset(USBDevice *dev)
-{
-    USBHostDevice *s = DO_UPCAST(USBHostDevice, dev, dev);
-
-    trace_usb_host_reset(s->bus_num, s->addr);
-
-    usb_host_do_reset(s);
-
-    usb_host_claim_interfaces(s, 0);
-    usb_linux_update_endp_table(s);
-}
-
-static void usb_host_handle_destroy(USBDevice *dev)
-{
-    USBHostDevice *s = (USBHostDevice *)dev;
-
-    usb_host_release_port(s);
-    usb_host_close(s);
-    QTAILQ_REMOVE(&hostdevs, s, next);
-    qemu_remove_exit_notifier(&s->exit);
-}
-
-/* iso data is special, we need to keep enough urbs in flight to make sure
-   that the controller never runs out of them, otherwise the device will
-   likely suffer a buffer underrun / overrun. */
-static AsyncURB *usb_host_alloc_iso(USBHostDevice *s, int pid, uint8_t ep)
-{
-    AsyncURB *aurb;
-    int i, j, len = usb_ep_get_max_packet_size(&s->dev, pid, ep);
-
-    aurb = g_malloc0(s->iso_urb_count * sizeof(*aurb));
-    for (i = 0; i < s->iso_urb_count; i++) {
-        aurb[i].urb.endpoint      = ep;
-        aurb[i].urb.buffer_length = ISO_FRAME_DESC_PER_URB * len;
-        aurb[i].urb.buffer        = g_malloc(aurb[i].urb.buffer_length);
-        aurb[i].urb.type          = USBDEVFS_URB_TYPE_ISO;
-        aurb[i].urb.flags         = USBDEVFS_URB_ISO_ASAP;
-        aurb[i].urb.number_of_packets = ISO_FRAME_DESC_PER_URB;
-        for (j = 0 ; j < ISO_FRAME_DESC_PER_URB; j++)
-            aurb[i].urb.iso_frame_desc[j].length = len;
-        if (pid == USB_TOKEN_IN) {
-            aurb[i].urb.endpoint |= 0x80;
-            /* Mark as fully consumed (idle) */
-            aurb[i].iso_frame_idx = ISO_FRAME_DESC_PER_URB;
-        }
-    }
-    set_iso_urb(s, pid, ep, aurb);
-
-    return aurb;
-}
-
-static void usb_host_stop_n_free_iso(USBHostDevice *s, int pid, uint8_t ep)
-{
-    AsyncURB *aurb;
-    int i, ret, killed = 0, free = 1;
-
-    aurb = get_iso_urb(s, pid, ep);
-    if (!aurb) {
-        return;
-    }
-
-    for (i = 0; i < s->iso_urb_count; i++) {
-        /* in flight? */
-        if (aurb[i].iso_frame_idx == -1) {
-            ret = ioctl(s->fd, USBDEVFS_DISCARDURB, &aurb[i]);
-            if (ret < 0) {
-                perror("USBDEVFS_DISCARDURB");
-                free = 0;
-                continue;
-            }
-            killed++;
-        }
-    }
-
-    /* Make sure any urbs we've killed are reaped before we free them */
-    if (killed) {
-        async_complete(s);
-    }
-
-    for (i = 0; i < s->iso_urb_count; i++) {
-        g_free(aurb[i].urb.buffer);
-    }
-
-    if (free)
-        g_free(aurb);
-    else
-        printf("husb: leaking iso urbs because of discard failure\n");
-    set_iso_urb(s, pid, ep, NULL);
-    set_iso_urb_idx(s, pid, ep, 0);
-    clear_iso_started(s, pid, ep);
-}
-
-static void urb_status_to_usb_ret(int status, USBPacket *p)
-{
-    switch (status) {
-    case -EPIPE:
-        p->status = USB_RET_STALL;
-        break;
-    case -EOVERFLOW:
-        p->status = USB_RET_BABBLE;
-        break;
-    default:
-        p->status = USB_RET_IOERROR;
-    }
-}
-
-static void usb_host_handle_iso_data(USBHostDevice *s, USBPacket *p, int in)
-{
-    AsyncURB *aurb;
-    int i, j, max_packet_size, offset, len;
-    uint8_t *buf;
-
-    max_packet_size = p->ep->max_packet_size;
-    if (max_packet_size == 0) {
-        p->status = USB_RET_NAK;
-        return;
-    }
-
-    aurb = get_iso_urb(s, p->pid, p->ep->nr);
-    if (!aurb) {
-        aurb = usb_host_alloc_iso(s, p->pid, p->ep->nr);
-    }
-
-    i = get_iso_urb_idx(s, p->pid, p->ep->nr);
-    j = aurb[i].iso_frame_idx;
-    if (j >= 0 && j < ISO_FRAME_DESC_PER_URB) {
-        if (in) {
-            /* Check urb status  */
-            if (aurb[i].urb.status) {
-                urb_status_to_usb_ret(aurb[i].urb.status, p);
-                /* Move to the next urb */
-                aurb[i].iso_frame_idx = ISO_FRAME_DESC_PER_URB - 1;
-            /* Check frame status */
-            } else if (aurb[i].urb.iso_frame_desc[j].status) {
-                urb_status_to_usb_ret(aurb[i].urb.iso_frame_desc[j].status, p);
-            /* Check the frame fits */
-            } else if (aurb[i].urb.iso_frame_desc[j].actual_length
-                       > p->iov.size) {
-                printf("husb: received iso data is larger then packet\n");
-                p->status = USB_RET_BABBLE;
-            /* All good copy data over */
-            } else {
-                len = aurb[i].urb.iso_frame_desc[j].actual_length;
-                buf  = aurb[i].urb.buffer +
-                    j * aurb[i].urb.iso_frame_desc[0].length;
-                usb_packet_copy(p, buf, len);
-            }
-        } else {
-            len = p->iov.size;
-            offset = (j == 0) ? 0 : get_iso_buffer_used(s, p->pid, p->ep->nr);
-
-            /* Check the frame fits */
-            if (len > max_packet_size) {
-                printf("husb: send iso data is larger then max packet size\n");
-                p->status = USB_RET_NAK;
-                return;
-            }
-
-            /* All good copy data over */
-            usb_packet_copy(p, aurb[i].urb.buffer + offset, len);
-            aurb[i].urb.iso_frame_desc[j].length = len;
-            offset += len;
-            set_iso_buffer_used(s, p->pid, p->ep->nr, offset);
-
-            /* Start the stream once we have buffered enough data */
-            if (!is_iso_started(s, p->pid, p->ep->nr) && i == 1 && j == 8) {
-                set_iso_started(s, p->pid, p->ep->nr);
-            }
-        }
-        aurb[i].iso_frame_idx++;
-        if (aurb[i].iso_frame_idx == ISO_FRAME_DESC_PER_URB) {
-            i = (i + 1) % s->iso_urb_count;
-            set_iso_urb_idx(s, p->pid, p->ep->nr, i);
-        }
-    } else {
-        if (in) {
-            set_iso_started(s, p->pid, p->ep->nr);
-        } else {
-            DPRINTF("hubs: iso out error no free buffer, dropping packet\n");
-        }
-    }
-
-    if (is_iso_started(s, p->pid, p->ep->nr)) {
-        /* (Re)-submit all fully consumed / filled urbs */
-        for (i = 0; i < s->iso_urb_count; i++) {
-            if (aurb[i].iso_frame_idx == ISO_FRAME_DESC_PER_URB) {
-                if (ioctl(s->fd, USBDEVFS_SUBMITURB, &aurb[i]) < 0) {
-                    perror("USBDEVFS_SUBMITURB");
-                    if (!in || p->status == USB_RET_SUCCESS) {
-                        switch(errno) {
-                        case ETIMEDOUT:
-                            p->status = USB_RET_NAK;
-                            break;
-                        case EPIPE:
-                        default:
-                            p->status = USB_RET_STALL;
-                        }
-                    }
-                    break;
-                }
-                aurb[i].iso_frame_idx = -1;
-                change_iso_inflight(s, p->pid, p->ep->nr, 1);
-            }
-        }
-    }
-}
-
-static void usb_host_handle_data(USBDevice *dev, USBPacket *p)
-{
-    USBHostDevice *s = DO_UPCAST(USBHostDevice, dev, dev);
-    struct usbdevfs_urb *urb;
-    AsyncURB *aurb;
-    int ret, rem, prem, v;
-    uint8_t *pbuf;
-    uint8_t ep;
-
-    trace_usb_host_req_data(s->bus_num, s->addr, p,
-                            p->pid == USB_TOKEN_IN,
-                            p->ep->nr, p->iov.size);
-
-    if (!is_valid(s, p->pid, p->ep->nr)) {
-        p->status = USB_RET_NAK;
-        trace_usb_host_req_complete(s->bus_num, s->addr, p,
-                                    p->status, p->actual_length);
-        return;
-    }
-
-    if (p->pid == USB_TOKEN_IN) {
-        ep = p->ep->nr | 0x80;
-    } else {
-        ep = p->ep->nr;
-    }
-
-    if (is_halted(s, p->pid, p->ep->nr)) {
-        unsigned int arg = ep;
-        ret = ioctl(s->fd, USBDEVFS_CLEAR_HALT, &arg);
-        if (ret < 0) {
-            perror("USBDEVFS_CLEAR_HALT");
-            p->status = USB_RET_NAK;
-            trace_usb_host_req_complete(s->bus_num, s->addr, p,
-                                        p->status, p->actual_length);
-            return;
-        }
-        clear_halt(s, p->pid, p->ep->nr);
-    }
-
-    if (is_isoc(s, p->pid, p->ep->nr)) {
-        usb_host_handle_iso_data(s, p, p->pid == USB_TOKEN_IN);
-        return;
-    }
-
-    v = 0;
-    prem = 0;
-    pbuf = NULL;
-    rem = p->iov.size;
-    do {
-        if (prem == 0 && rem > 0) {
-            assert(v < p->iov.niov);
-            prem = p->iov.iov[v].iov_len;
-            pbuf = p->iov.iov[v].iov_base;
-            assert(prem <= rem);
-            v++;
-        }
-        aurb = async_alloc(s);
-        aurb->packet = p;
-
-        urb = &aurb->urb;
-        urb->endpoint      = ep;
-        urb->type          = usb_host_usbfs_type(s, p);
-        urb->usercontext   = s;
-        urb->buffer        = pbuf;
-        urb->buffer_length = prem;
-
-        if (urb->buffer_length > MAX_USBFS_BUFFER_SIZE) {
-            urb->buffer_length = MAX_USBFS_BUFFER_SIZE;
-        }
-        pbuf += urb->buffer_length;
-        prem -= urb->buffer_length;
-        rem  -= urb->buffer_length;
-        if (rem) {
-            aurb->more         = 1;
-        }
-
-        trace_usb_host_urb_submit(s->bus_num, s->addr, aurb,
-                                  urb->buffer_length, aurb->more);
-        ret = ioctl(s->fd, USBDEVFS_SUBMITURB, urb);
-
-        DPRINTF("husb: data submit: ep 0x%x, len %u, more %d, packet %p, aurb %p\n",
-                urb->endpoint, urb->buffer_length, aurb->more, p, aurb);
-
-        if (ret < 0) {
-            perror("USBDEVFS_SUBMITURB");
-            async_free(aurb);
-
-            switch(errno) {
-            case ETIMEDOUT:
-                p->status = USB_RET_NAK;
-                trace_usb_host_req_complete(s->bus_num, s->addr, p,
-                                            p->status, p->actual_length);
-                break;
-            case EPIPE:
-            default:
-                p->status = USB_RET_STALL;
-                trace_usb_host_req_complete(s->bus_num, s->addr, p,
-                                            p->status, p->actual_length);
-            }
-            return;
-        }
-    } while (rem > 0);
-
-    p->status = USB_RET_ASYNC;
-}
-
-static int ctrl_error(void)
-{
-    if (errno == ETIMEDOUT) {
-        return USB_RET_NAK;
-    } else {
-        return USB_RET_STALL;
-    }
-}
-
-static void usb_host_set_address(USBHostDevice *s, int addr)
-{
-    trace_usb_host_set_address(s->bus_num, s->addr, addr);
-    s->dev.addr = addr;
-}
-
-static void usb_host_set_config(USBHostDevice *s, int config, USBPacket *p)
-{
-    int ret, first = 1;
-
-    trace_usb_host_set_config(s->bus_num, s->addr, config);
-
-    usb_host_release_interfaces(s);
-
-again:
-    ret = ioctl(s->fd, USBDEVFS_SETCONFIGURATION, &config);
-
-    DPRINTF("husb: ctrl set config %d ret %d errno %d\n", config, ret, errno);
-
-    if (ret < 0 && errno == EBUSY && first) {
-        /* happens if usb device is in use by host drivers */
-        int count = usb_linux_get_num_interfaces(s);
-        if (count > 0) {
-            DPRINTF("husb: busy -> disconnecting %d interfaces\n", count);
-            usb_host_disconnect_ifaces(s, count);
-            first = 0;
-            goto again;
-        }
-    }
-
-    if (ret < 0) {
-        p->status = ctrl_error();
-        return;
-    }
-    usb_host_claim_interfaces(s, config);
-    usb_linux_update_endp_table(s);
-}
-
-static void usb_host_set_interface(USBHostDevice *s, int iface, int alt,
-                                   USBPacket *p)
-{
-    struct usbdevfs_setinterface si;
-    int i, ret;
-
-    trace_usb_host_set_interface(s->bus_num, s->addr, iface, alt);
-
-    for (i = 1; i <= USB_MAX_ENDPOINTS; i++) {
-        if (is_isoc(s, USB_TOKEN_IN, i)) {
-            usb_host_stop_n_free_iso(s, USB_TOKEN_IN, i);
-        }
-        if (is_isoc(s, USB_TOKEN_OUT, i)) {
-            usb_host_stop_n_free_iso(s, USB_TOKEN_OUT, i);
-        }
-    }
-
-    if (iface >= USB_MAX_INTERFACES) {
-        p->status = USB_RET_STALL;
-        return;
-    }
-
-    si.interface  = iface;
-    si.altsetting = alt;
-    ret = ioctl(s->fd, USBDEVFS_SETINTERFACE, &si);
-
-    DPRINTF("husb: ctrl set iface %d altset %d ret %d errno %d\n",
-            iface, alt, ret, errno);
-
-    if (ret < 0) {
-        p->status = ctrl_error();
-        return;
-    }
-
-    s->dev.altsetting[iface] = alt;
-    usb_linux_update_endp_table(s);
-}
-
-static void usb_host_handle_control(USBDevice *dev, USBPacket *p,
-               int request, int value, int index, int length, uint8_t *data)
-{
-    USBHostDevice *s = DO_UPCAST(USBHostDevice, dev, dev);
-    struct usbdevfs_urb *urb;
-    AsyncURB *aurb;
-    int ret;
-
-    /*
-     * Process certain standard device requests.
-     * These are infrequent and are processed synchronously.
-     */
-
-    /* Note request is (bRequestType << 8) | bRequest */
-    trace_usb_host_req_control(s->bus_num, s->addr, p, request, value, index);
-
-    switch (request) {
-    case DeviceOutRequest | USB_REQ_SET_ADDRESS:
-        usb_host_set_address(s, value);
-        trace_usb_host_req_emulated(s->bus_num, s->addr, p, p->status);
-        return;
-
-    case DeviceOutRequest | USB_REQ_SET_CONFIGURATION:
-        usb_host_set_config(s, value & 0xff, p);
-        trace_usb_host_req_emulated(s->bus_num, s->addr, p, p->status);
-        return;
-
-    case InterfaceOutRequest | USB_REQ_SET_INTERFACE:
-        usb_host_set_interface(s, index, value, p);
-        trace_usb_host_req_emulated(s->bus_num, s->addr, p, p->status);
-        return;
-
-    case EndpointOutRequest | USB_REQ_CLEAR_FEATURE:
-        if (value == 0) { /* clear halt */
-            int pid = (index & USB_DIR_IN) ? USB_TOKEN_IN : USB_TOKEN_OUT;
-            ioctl(s->fd, USBDEVFS_CLEAR_HALT, &index);
-            clear_halt(s, pid, index & 0x0f);
-            trace_usb_host_req_emulated(s->bus_num, s->addr, p, 0);
-            return;
-        }
-    }
-
-    /* The rest are asynchronous */
-    if (length > sizeof(dev->data_buf)) {
-        fprintf(stderr, "husb: ctrl buffer too small (%d > %zu)\n",
-                length, sizeof(dev->data_buf));
-        p->status = USB_RET_STALL;
-        return;
-    }
-
-    aurb = async_alloc(s);
-    aurb->packet = p;
-
-    /*
-     * Setup ctrl transfer.
-     *
-     * s->ctrl is laid out such that data buffer immediately follows
-     * 'req' struct which is exactly what usbdevfs expects.
-     */
-    urb = &aurb->urb;
-
-    urb->type     = USBDEVFS_URB_TYPE_CONTROL;
-    urb->endpoint = p->ep->nr;
-
-    urb->buffer        = &dev->setup_buf;
-    urb->buffer_length = length + 8;
-
-    urb->usercontext = s;
-
-    trace_usb_host_urb_submit(s->bus_num, s->addr, aurb,
-                              urb->buffer_length, aurb->more);
-    ret = ioctl(s->fd, USBDEVFS_SUBMITURB, urb);
-
-    DPRINTF("husb: submit ctrl. len %u aurb %p\n", urb->buffer_length, aurb);
-
-    if (ret < 0) {
-        DPRINTF("husb: submit failed. errno %d\n", errno);
-        async_free(aurb);
-
-        switch(errno) {
-        case ETIMEDOUT:
-            p->status = USB_RET_NAK;
-            break;
-        case EPIPE:
-        default:
-            p->status = USB_RET_STALL;
-            break;
-        }
-        return;
-    }
-
-    p->status = USB_RET_ASYNC;
-}
-
-static void usb_linux_update_endp_table(USBHostDevice *s)
-{
-    static const char *tname[] = {
-        [USB_ENDPOINT_XFER_CONTROL] = "control",
-        [USB_ENDPOINT_XFER_ISOC]    = "isoc",
-        [USB_ENDPOINT_XFER_BULK]    = "bulk",
-        [USB_ENDPOINT_XFER_INT]     = "int",
-    };
-    uint8_t devep, type;
-    uint16_t mps, v, p;
-    int ep, pid;
-    unsigned int i, configuration = -1, interface = -1, altsetting = -1;
-    struct endp_data *epd;
-    USBDescriptor *d;
-    bool active = false;
-
-    usb_ep_reset(&s->dev);
-
-    for (i = 0;; i += d->bLength) {
-        if (i+2 >= s->descr_len) {
-            break;
-        }
-        d = (void *)(s->descr + i);
-        if (d->bLength < 2) {
-            trace_usb_host_parse_error(s->bus_num, s->addr,
-                                       "descriptor too short");
-            return;
-        }
-        if (i + d->bLength > s->descr_len) {
-            trace_usb_host_parse_error(s->bus_num, s->addr,
-                                       "descriptor too long");
-            return;
-        }
-        switch (d->bDescriptorType) {
-        case 0:
-            trace_usb_host_parse_error(s->bus_num, s->addr,
-                                       "invalid descriptor type");
-            return;
-        case USB_DT_DEVICE:
-            if (d->bLength < 0x12) {
-                trace_usb_host_parse_error(s->bus_num, s->addr,
-                                           "device descriptor too short");
-                return;
-            }
-            v = (d->u.device.idVendor_hi << 8) | d->u.device.idVendor_lo;
-            p = (d->u.device.idProduct_hi << 8) | d->u.device.idProduct_lo;
-            trace_usb_host_parse_device(s->bus_num, s->addr, v, p);
-            break;
-        case USB_DT_CONFIG:
-            if (d->bLength < 0x09) {
-                trace_usb_host_parse_error(s->bus_num, s->addr,
-                                           "config descriptor too short");
-                return;
-            }
-            configuration = d->u.config.bConfigurationValue;
-            active = (configuration == s->dev.configuration);
-            trace_usb_host_parse_config(s->bus_num, s->addr,
-                                        configuration, active);
-            break;
-        case USB_DT_INTERFACE:
-            if (d->bLength < 0x09) {
-                trace_usb_host_parse_error(s->bus_num, s->addr,
-                                           "interface descriptor too short");
-                return;
-            }
-            interface = d->u.interface.bInterfaceNumber;
-            altsetting = d->u.interface.bAlternateSetting;
-            active = (configuration == s->dev.configuration) &&
-                (altsetting == s->dev.altsetting[interface]);
-            trace_usb_host_parse_interface(s->bus_num, s->addr,
-                                           interface, altsetting, active);
-            break;
-        case USB_DT_ENDPOINT:
-            if (d->bLength < 0x07) {
-                trace_usb_host_parse_error(s->bus_num, s->addr,
-                                           "endpoint descriptor too short");
-                return;
-            }
-            devep = d->u.endpoint.bEndpointAddress;
-            pid = (devep & USB_DIR_IN) ? USB_TOKEN_IN : USB_TOKEN_OUT;
-            ep = devep & 0xf;
-            if (ep == 0) {
-                trace_usb_host_parse_error(s->bus_num, s->addr,
-                                           "invalid endpoint address");
-                return;
-            }
-
-            type = d->u.endpoint.bmAttributes & 0x3;
-            mps = d->u.endpoint.wMaxPacketSize_lo |
-                (d->u.endpoint.wMaxPacketSize_hi << 8);
-            trace_usb_host_parse_endpoint(s->bus_num, s->addr, ep,
-                                          (devep & USB_DIR_IN) ? "in" : "out",
-                                          tname[type], active);
-
-            if (active) {
-                usb_ep_set_max_packet_size(&s->dev, pid, ep, mps);
-                assert(usb_ep_get_type(&s->dev, pid, ep) ==
-                       USB_ENDPOINT_XFER_INVALID);
-                usb_ep_set_type(&s->dev, pid, ep, type);
-                usb_ep_set_ifnum(&s->dev, pid, ep, interface);
-                if ((s->options & (1 << USB_HOST_OPT_PIPELINE)) &&
-                    (type == USB_ENDPOINT_XFER_BULK) &&
-                    (pid == USB_TOKEN_OUT)) {
-                    usb_ep_set_pipeline(&s->dev, pid, ep, true);
-                }
-
-                epd = get_endp(s, pid, ep);
-                epd->halted = 0;
-            }
-
-            break;
-        default:
-            trace_usb_host_parse_unknown(s->bus_num, s->addr,
-                                         d->bLength, d->bDescriptorType);
-            break;
-        }
-    }
-}
-
-/*
- * Check if we can safely redirect a usb2 device to a usb1 virtual controller,
- * this function assumes this is safe, if:
- * 1) There are no isoc endpoints
- * 2) There are no interrupt endpoints with a max_packet_size > 64
- * Note bulk endpoints with a max_packet_size > 64 in theory also are not
- * usb1 compatible, but in practice this seems to work fine.
- */
-static int usb_linux_full_speed_compat(USBHostDevice *dev)
-{
-    int i, packet_size;
-
-    /*
-     * usb_linux_update_endp_table only registers info about ep in the current
-     * interface altsettings, so we need to parse the descriptors again.
-     */
-    for (i = 0; (i + 5) < dev->descr_len; i += dev->descr[i]) {
-        if (dev->descr[i + 1] == USB_DT_ENDPOINT) {
-            switch (dev->descr[i + 3] & 0x3) {
-            case 0x00: /* CONTROL */
-                break;
-            case 0x01: /* ISO */
-                return 0;
-            case 0x02: /* BULK */
-                break;
-            case 0x03: /* INTERRUPT */
-                packet_size = dev->descr[i + 4] + (dev->descr[i + 5] << 8);
-                if (packet_size > 64)
-                    return 0;
-                break;
-            }
-        }
-    }
-    return 1;
-}
-
-static int usb_host_open(USBHostDevice *dev, int bus_num,
-                         int addr, const char *port,
-                         const char *prod_name, int speed)
-{
-    int fd = -1, ret;
-
-    trace_usb_host_open_started(bus_num, addr);
-
-    if (dev->fd != -1) {
-        goto fail;
-    }
-
-    fd = usb_host_open_device(bus_num, addr);
-    if (fd < 0) {
-        goto fail;
-    }
-    DPRINTF("husb: opened %s\n", buf);
-
-    dev->bus_num = bus_num;
-    dev->addr = addr;
-    pstrcpy(dev->port, sizeof(dev->port), port);
-    dev->fd = fd;
-
-    /* read the device description */
-    dev->descr_len = read(fd, dev->descr, sizeof(dev->descr));
-    if (dev->descr_len <= 0) {
-        perror("husb: reading device data failed");
-        goto fail;
-    }
-
-#ifdef DEBUG
-    {
-        int x;
-        printf("=== begin dumping device descriptor data ===\n");
-        for (x = 0; x < dev->descr_len; x++) {
-            printf("%02x ", dev->descr[x]);
-        }
-        printf("\n=== end dumping device descriptor data ===\n");
-    }
-#endif
-
-
-    /* start unconfigured -- we'll wait for the guest to set a configuration */
-    if (!usb_host_claim_interfaces(dev, 0)) {
-        goto fail;
-    }
-
-    usb_ep_init(&dev->dev);
-    usb_linux_update_endp_table(dev);
-
-    if (speed == -1) {
-        struct usbdevfs_connectinfo ci;
-
-        ret = ioctl(fd, USBDEVFS_CONNECTINFO, &ci);
-        if (ret < 0) {
-            perror("usb_host_device_open: USBDEVFS_CONNECTINFO");
-            goto fail;
-        }
-
-        if (ci.slow) {
-            speed = USB_SPEED_LOW;
-        } else {
-            speed = USB_SPEED_HIGH;
-        }
-    }
-    dev->dev.speed = speed;
-    dev->dev.speedmask = (1 << speed);
-    if (dev->dev.speed == USB_SPEED_HIGH && usb_linux_full_speed_compat(dev)) {
-        dev->dev.speedmask |= USB_SPEED_MASK_FULL;
-    }
-
-    trace_usb_host_open_success(bus_num, addr);
-
-    if (!prod_name || prod_name[0] == '\0') {
-        snprintf(dev->dev.product_desc, sizeof(dev->dev.product_desc),
-                 "host:%d.%d", bus_num, addr);
-    } else {
-        pstrcpy(dev->dev.product_desc, sizeof(dev->dev.product_desc),
-                prod_name);
-    }
-
-    ret = usb_device_attach(&dev->dev);
-    if (ret) {
-        goto fail;
-    }
-
-    /* USB devio uses 'write' flag to check for async completions */
-    qemu_set_fd_handler(dev->fd, NULL, async_complete, dev);
-
-    return 0;
-
-fail:
-    trace_usb_host_open_failure(bus_num, addr);
-    if (dev->fd != -1) {
-        close(dev->fd);
-        dev->fd = -1;
-    }
-    return -1;
-}
-
-static int usb_host_close(USBHostDevice *dev)
-{
-    int i;
-
-    if (dev->fd == -1) {
-        return -1;
-    }
-
-    trace_usb_host_close(dev->bus_num, dev->addr);
-
-    qemu_set_fd_handler(dev->fd, NULL, NULL, NULL);
-    dev->closing = 1;
-    for (i = 1; i <= USB_MAX_ENDPOINTS; i++) {
-        if (is_isoc(dev, USB_TOKEN_IN, i)) {
-            usb_host_stop_n_free_iso(dev, USB_TOKEN_IN, i);
-        }
-        if (is_isoc(dev, USB_TOKEN_OUT, i)) {
-            usb_host_stop_n_free_iso(dev, USB_TOKEN_OUT, i);
-        }
-    }
-    async_complete(dev);
-    dev->closing = 0;
-    if (dev->dev.attached) {
-        usb_device_detach(&dev->dev);
-    }
-    usb_host_do_reset(dev);
-    close(dev->fd);
-    dev->fd = -1;
-    return 0;
-}
-
-static void usb_host_exit_notifier(struct Notifier *n, void *data)
-{
-    USBHostDevice *s = container_of(n, USBHostDevice, exit);
-
-    usb_host_release_port(s);
-    if (s->fd != -1) {
-        usb_host_do_reset(s);
-    }
-}
-
-/*
- * This is *NOT* about restoring state.  We have absolutely no idea
- * what state the host device is in at the moment and whenever it is
- * still present in the first place.  Attemping to contine where we
- * left off is impossible.
- *
- * What we are going to to to here is emulate a surprise removal of
- * the usb device passed through, then kick host scan so the device
- * will get re-attached (and re-initialized by the guest) in case it
- * is still present.
- *
- * As the device removal will change the state of other devices (usb
- * host controller, most likely interrupt controller too) we have to
- * wait with it until *all* vmstate is loaded.  Thus post_load just
- * kicks a bottom half which then does the actual work.
- */
-static void usb_host_post_load_bh(void *opaque)
-{
-    USBHostDevice *dev = opaque;
-
-    if (dev->fd != -1) {
-        usb_host_close(dev);
-    }
-    if (dev->dev.attached) {
-        usb_device_detach(&dev->dev);
-    }
-    usb_host_auto_check(NULL);
-}
-
-static int usb_host_post_load(void *opaque, int version_id)
-{
-    USBHostDevice *dev = opaque;
-
-    qemu_bh_schedule(dev->bh);
-    return 0;
-}
-
-static int usb_host_initfn(USBDevice *dev)
-{
-    USBHostDevice *s = DO_UPCAST(USBHostDevice, dev, dev);
-
-    dev->flags |= (1 << USB_DEV_FLAG_IS_HOST);
-    dev->auto_attach = 0;
-    s->fd = -1;
-    s->hub_fd = -1;
-
-    QTAILQ_INSERT_TAIL(&hostdevs, s, next);
-    s->exit.notify = usb_host_exit_notifier;
-    qemu_add_exit_notifier(&s->exit);
-    s->bh = qemu_bh_new(usb_host_post_load_bh, s);
-    usb_host_auto_check(NULL);
-
-    if (s->match.bus_num != 0 && s->match.port != NULL) {
-        usb_host_claim_port(s);
-    }
-    add_boot_device_path(s->bootindex, &dev->qdev, NULL);
-    return 0;
-}
-
-static const VMStateDescription vmstate_usb_host = {
-    .name = DEVNAME,
-    .version_id = 1,
-    .minimum_version_id = 1,
-    .post_load = usb_host_post_load,
-    .fields = (VMStateField[]) {
-        VMSTATE_USB_DEVICE(dev, USBHostDevice),
-        VMSTATE_END_OF_LIST()
-    }
-};
-
-static Property usb_host_dev_properties[] = {
-    DEFINE_PROP_UINT32("hostbus",  USBHostDevice, match.bus_num,    0),
-    DEFINE_PROP_UINT32("hostaddr", USBHostDevice, match.addr,       0),
-    DEFINE_PROP_STRING("hostport", USBHostDevice, match.port),
-    DEFINE_PROP_HEX32("vendorid",  USBHostDevice, match.vendor_id,  0),
-    DEFINE_PROP_HEX32("productid", USBHostDevice, match.product_id, 0),
-    DEFINE_PROP_UINT32("isobufs",  USBHostDevice, iso_urb_count,    4),
-    DEFINE_PROP_INT32("bootindex", USBHostDevice, bootindex,        -1),
-    DEFINE_PROP_BIT("pipeline",    USBHostDevice, options,
-                    USB_HOST_OPT_PIPELINE, true),
-    DEFINE_PROP_END_OF_LIST(),
-};
-
-static void usb_host_class_initfn(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
-
-    uc->init           = usb_host_initfn;
-    uc->product_desc   = "USB Host Device";
-    uc->cancel_packet  = usb_host_async_cancel;
-    uc->handle_data    = usb_host_handle_data;
-    uc->handle_control = usb_host_handle_control;
-    uc->handle_reset   = usb_host_handle_reset;
-    uc->handle_destroy = usb_host_handle_destroy;
-    dc->vmsd = &vmstate_usb_host;
-    dc->props = usb_host_dev_properties;
-    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
-}
-
-static const TypeInfo usb_host_dev_info = {
-    .name          = DEVNAME,
-    .parent        = TYPE_USB_DEVICE,
-    .instance_size = sizeof(USBHostDevice),
-    .class_init    = usb_host_class_initfn,
-};
-
-static void usb_host_register_types(void)
-{
-    type_register_static(&usb_host_dev_info);
-}
-
-type_init(usb_host_register_types)
-
-/*
- * Read sys file-system device file
- *
- * @line address of buffer to put file contents in
- * @line_size size of line
- * @device_file path to device file (printf format string)
- * @device_name device being opened (inserted into device_file)
- *
- * @return 0 failed, 1 succeeded ('line' contains data)
- */
-static int usb_host_read_file(char *line, size_t line_size,
-                              const char *device_file, const char *device_name)
-{
-    FILE *f;
-    int ret = 0;
-    char filename[PATH_MAX];
-
-    snprintf(filename, PATH_MAX, "/sys/bus/usb/devices/%s/%s", device_name,
-             device_file);
-    f = fopen(filename, "r");
-    if (f) {
-        ret = fgets(line, line_size, f) != NULL;
-        fclose(f);
-    }
-
-    return ret;
-}
-
-/*
- * Use /sys/bus/usb/devices/ directory to determine host's USB
- * devices.
- *
- * This code is based on Robert Schiele's original patches posted to
- * the Novell bug-tracker https://bugzilla.novell.com/show_bug.cgi?id=241950
- */
-static int usb_host_scan(void *opaque, USBScanFunc *func)
-{
-    DIR *dir = NULL;
-    char line[1024];
-    int bus_num, addr, speed, class_id, product_id, vendor_id;
-    int ret = 0;
-    char port[MAX_PORTLEN];
-    char product_name[512];
-    struct dirent *de;
-
-    dir = opendir("/sys/bus/usb/devices");
-    if (!dir) {
-        perror("husb: opendir /sys/bus/usb/devices");
-        fprintf(stderr, "husb: please make sure sysfs is mounted at /sys\n");
-        goto the_end;
-    }
-
-    while ((de = readdir(dir))) {
-        if (de->d_name[0] != '.' && !strchr(de->d_name, ':')) {
-            if (sscanf(de->d_name, "%d-%7[0-9.]", &bus_num, port) < 2) {
-                continue;
-            }
-
-            if (!usb_host_read_file(line, sizeof(line), "devnum", de->d_name)) {
-                goto the_end;
-            }
-            if (sscanf(line, "%d", &addr) != 1) {
-                goto the_end;
-            }
-            if (!usb_host_read_file(line, sizeof(line), "bDeviceClass",
-                                    de->d_name)) {
-                goto the_end;
-            }
-            if (sscanf(line, "%x", &class_id) != 1) {
-                goto the_end;
-            }
-
-            if (!usb_host_read_file(line, sizeof(line), "idVendor",
-                                    de->d_name)) {
-                goto the_end;
-            }
-            if (sscanf(line, "%x", &vendor_id) != 1) {
-                goto the_end;
-            }
-            if (!usb_host_read_file(line, sizeof(line), "idProduct",
-                                    de->d_name)) {
-                goto the_end;
-            }
-            if (sscanf(line, "%x", &product_id) != 1) {
-                goto the_end;
-            }
-            if (!usb_host_read_file(line, sizeof(line), "product",
-                                    de->d_name)) {
-                *product_name = 0;
-            } else {
-                if (strlen(line) > 0) {
-                    line[strlen(line) - 1] = '\0';
-                }
-                pstrcpy(product_name, sizeof(product_name), line);
-            }
-
-            if (!usb_host_read_file(line, sizeof(line), "speed", de->d_name)) {
-                goto the_end;
-            }
-            if (!strcmp(line, "5000\n")) {
-                speed = USB_SPEED_SUPER;
-            } else if (!strcmp(line, "480\n")) {
-                speed = USB_SPEED_HIGH;
-            } else if (!strcmp(line, "1.5\n")) {
-                speed = USB_SPEED_LOW;
-            } else {
-                speed = USB_SPEED_FULL;
-            }
-
-            ret = func(opaque, bus_num, addr, port, class_id, vendor_id,
-                       product_id, product_name, speed);
-            if (ret) {
-                goto the_end;
-            }
-        }
-    }
- the_end:
-    if (dir) {
-        closedir(dir);
-    }
-    return ret;
-}
-
-static QEMUTimer *usb_auto_timer;
-static VMChangeStateEntry *usb_vmstate;
-
-static int usb_host_auto_scan(void *opaque, int bus_num,
-                              int addr, const char *port,
-                              int class_id, int vendor_id, int product_id,
-                              const char *product_name, int speed)
-{
-    struct USBAutoFilter *f;
-    struct USBHostDevice *s;
-
-    /* Ignore hubs */
-    if (class_id == 9)
-        return 0;
-
-    QTAILQ_FOREACH(s, &hostdevs, next) {
-        f = &s->match;
-
-        if (f->bus_num > 0 && f->bus_num != bus_num) {
-            continue;
-        }
-        if (f->addr > 0 && f->addr != addr) {
-            continue;
-        }
-        if (f->port != NULL && strcmp(f->port, port) != 0) {
-            continue;
-        }
-
-        if (f->vendor_id > 0 && f->vendor_id != vendor_id) {
-            continue;
-        }
-
-        if (f->product_id > 0 && f->product_id != product_id) {
-            continue;
-        }
-        /* We got a match */
-        s->seen++;
-        if (s->errcount >= 3) {
-            return 0;
-        }
-
-        /* Already attached ? */
-        if (s->fd != -1) {
-            return 0;
-        }
-        DPRINTF("husb: auto open: bus_num %d addr %d\n", bus_num, addr);
-
-        if (usb_host_open(s, bus_num, addr, port, product_name, speed) < 0) {
-            s->errcount++;
-        }
-        break;
-    }
-
-    return 0;
-}
-
-static void usb_host_vm_state(void *unused, int running, RunState state)
-{
-    if (running) {
-        usb_host_auto_check(unused);
-    }
-}
-
-static void usb_host_auto_check(void *unused)
-{
-    struct USBHostDevice *s;
-    int unconnected = 0;
-
-    if (runstate_is_running()) {
-        usb_host_scan(NULL, usb_host_auto_scan);
-
-        QTAILQ_FOREACH(s, &hostdevs, next) {
-            if (s->fd == -1) {
-                unconnected++;
-            }
-            if (s->seen == 0) {
-                s->errcount = 0;
-            }
-            s->seen = 0;
-        }
-
-        if (unconnected == 0) {
-            /* nothing to watch */
-            if (usb_auto_timer) {
-                timer_del(usb_auto_timer);
-                trace_usb_host_auto_scan_disabled();
-            }
-            return;
-        }
-    }
-
-    if (!usb_vmstate) {
-        usb_vmstate = qemu_add_vm_change_state_handler(usb_host_vm_state, NULL);
-    }
-    if (!usb_auto_timer) {
-        usb_auto_timer = timer_new_ms(QEMU_CLOCK_REALTIME, usb_host_auto_check, NULL);
-        if (!usb_auto_timer) {
-            return;
-        }
-        trace_usb_host_auto_scan_enabled();
-    }
-    timer_mod(usb_auto_timer, qemu_clock_get_ms(QEMU_CLOCK_REALTIME) + 2000);
-}
-
-#ifndef CONFIG_USB_LIBUSB
-
-/**********************/
-/* USB host device info */
-
-struct usb_class_info {
-    int class;
-    const char *class_name;
-};
-
-static const struct usb_class_info usb_class_info[] = {
-    { USB_CLASS_AUDIO, "Audio"},
-    { USB_CLASS_COMM, "Communication"},
-    { USB_CLASS_HID, "HID"},
-    { USB_CLASS_HUB, "Hub" },
-    { USB_CLASS_PHYSICAL, "Physical" },
-    { USB_CLASS_PRINTER, "Printer" },
-    { USB_CLASS_MASS_STORAGE, "Storage" },
-    { USB_CLASS_CDC_DATA, "Data" },
-    { USB_CLASS_APP_SPEC, "Application Specific" },
-    { USB_CLASS_VENDOR_SPEC, "Vendor Specific" },
-    { USB_CLASS_STILL_IMAGE, "Still Image" },
-    { USB_CLASS_CSCID, "Smart Card" },
-    { USB_CLASS_CONTENT_SEC, "Content Security" },
-    { -1, NULL }
-};
-
-static const char *usb_class_str(uint8_t class)
-{
-    const struct usb_class_info *p;
-    for(p = usb_class_info; p->class != -1; p++) {
-        if (p->class == class) {
-            break;
-        }
-    }
-    return p->class_name;
-}
-
-static void usb_info_device(Monitor *mon, int bus_num,
-                            int addr, const char *port,
-                            int class_id, int vendor_id, int product_id,
-                            const char *product_name,
-                            int speed)
-{
-    const char *class_str, *speed_str;
-
-    switch(speed) {
-    case USB_SPEED_LOW:
-        speed_str = "1.5";
-        break;
-    case USB_SPEED_FULL:
-        speed_str = "12";
-        break;
-    case USB_SPEED_HIGH:
-        speed_str = "480";
-        break;
-    case USB_SPEED_SUPER:
-        speed_str = "5000";
-        break;
-    default:
-        speed_str = "?";
-        break;
-    }
-
-    monitor_printf(mon, "  Bus %d, Addr %d, Port %s, Speed %s Mb/s\n",
-                   bus_num, addr, port, speed_str);
-    class_str = usb_class_str(class_id);
-    if (class_str) {
-        monitor_printf(mon, "    %s:", class_str);
-    } else {
-        monitor_printf(mon, "    Class %02x:", class_id);
-    }
-    monitor_printf(mon, " USB device %04x:%04x", vendor_id, product_id);
-    if (product_name[0] != '\0') {
-        monitor_printf(mon, ", %s", product_name);
-    }
-    monitor_printf(mon, "\n");
-}
-
-static int usb_host_info_device(void *opaque, int bus_num, int addr,
-                                const char *path, int class_id,
-                                int vendor_id, int product_id,
-                                const char *product_name,
-                                int speed)
-{
-    Monitor *mon = opaque;
-
-    usb_info_device(mon, bus_num, addr, path, class_id, vendor_id, product_id,
-                    product_name, speed);
-    return 0;
-}
-
-static void dec2str(int val, char *str, size_t size)
-{
-    if (val == 0) {
-        snprintf(str, size, "*");
-    } else {
-        snprintf(str, size, "%d", val);
-    }
-}
-
-static void hex2str(int val, char *str, size_t size)
-{
-    if (val == 0) {
-        snprintf(str, size, "*");
-    } else {
-        snprintf(str, size, "%04x", val);
-    }
-}
-
-void usb_host_info(Monitor *mon, const QDict *qdict)
-{
-    struct USBAutoFilter *f;
-    struct USBHostDevice *s;
-
-    usb_host_scan(mon, usb_host_info_device);
-
-    if (QTAILQ_EMPTY(&hostdevs)) {
-        return;
-    }
-
-    monitor_printf(mon, "  Auto filters:\n");
-    QTAILQ_FOREACH(s, &hostdevs, next) {
-        char bus[10], addr[10], vid[10], pid[10];
-        f = &s->match;
-        dec2str(f->bus_num, bus, sizeof(bus));
-        dec2str(f->addr, addr, sizeof(addr));
-        hex2str(f->vendor_id, vid, sizeof(vid));
-        hex2str(f->product_id, pid, sizeof(pid));
-        monitor_printf(mon, "    Bus %s, Addr %s, Port %s, ID %s:%s\n",
-                       bus, addr, f->port ? f->port : "*", vid, pid);
-    }
-}
-
-#endif
diff --git a/hw/xen/xen_backend.c b/hw/xen/xen_backend.c
index d82ce5d8a6..197795ffe1 100644
--- a/hw/xen/xen_backend.c
+++ b/hw/xen/xen_backend.c
@@ -205,7 +205,6 @@ static struct XenDevice *xen_be_get_xendev(const char *type, int dom, int dev,
                                            struct XenDevOps *ops)
 {
     struct XenDevice *xendev;
-    char *dom0;
 
     xendev = xen_be_find_xendev(type, dom, dev);
     if (xendev) {
@@ -219,12 +218,10 @@ static struct XenDevice *xen_be_get_xendev(const char *type, int dom, int dev,
     xendev->dev   = dev;
     xendev->ops   = ops;
 
-    dom0 = xs_get_domain_path(xenstore, 0);
-    snprintf(xendev->be, sizeof(xendev->be), "%s/backend/%s/%d/%d",
-             dom0, xendev->type, xendev->dom, xendev->dev);
+    snprintf(xendev->be, sizeof(xendev->be), "backend/%s/%d/%d",
+             xendev->type, xendev->dom, xendev->dev);
     snprintf(xendev->name, sizeof(xendev->name), "%s-%d",
              xendev->type, xendev->dev);
-    free(dom0);
 
     xendev->debug      = debug;
     xendev->local_port = -1;
@@ -570,14 +567,12 @@ static int xenstore_scan(const char *type, int dom, struct XenDevOps *ops)
 {
     struct XenDevice *xendev;
     char path[XEN_BUFSIZE], token[XEN_BUFSIZE];
-    char **dev = NULL, *dom0;
+    char **dev = NULL;
     unsigned int cdev, j;
 
     /* setup watch */
-    dom0 = xs_get_domain_path(xenstore, 0);
     snprintf(token, sizeof(token), "be:%p:%d:%p", type, dom, ops);
-    snprintf(path, sizeof(path), "%s/backend/%s/%d", dom0, type, dom);
-    free(dom0);
+    snprintf(path, sizeof(path), "backend/%s/%d", type, dom);
     if (!xs_watch(xenstore, path, token)) {
         xen_be_printf(NULL, 0, "xen be: watching backend path (%s) failed\n", path);
         return -1;
@@ -603,12 +598,10 @@ static void xenstore_update_be(char *watch, char *type, int dom,
                                struct XenDevOps *ops)
 {
     struct XenDevice *xendev;
-    char path[XEN_BUFSIZE], *dom0, *bepath;
+    char path[XEN_BUFSIZE], *bepath;
     unsigned int len, dev;
 
-    dom0 = xs_get_domain_path(xenstore, 0);
-    len = snprintf(path, sizeof(path), "%s/backend/%s/%d", dom0, type, dom);
-    free(dom0);
+    len = snprintf(path, sizeof(path), "backend/%s/%d", type, dom);
     if (strncmp(path, watch, len) != 0) {
         return;
     }
diff --git a/include/block/block.h b/include/block/block.h
index 728ec1aebf..3560deb883 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -84,6 +84,9 @@ typedef struct BlockDevOps {
 /* BDRV_BLOCK_DATA: data is read from bs->file or another file
  * BDRV_BLOCK_ZERO: sectors read as zero
  * BDRV_BLOCK_OFFSET_VALID: sector stored in bs->file as raw data
+ * BDRV_BLOCK_RAW: used internally to indicate that the request
+ *                 was answered by the raw driver and that one
+ *                 should look in bs->file directly.
  *
  * If BDRV_BLOCK_OFFSET_VALID is set, bits 9-62 represent the offset in
  * bs->file where sector data can be read from as raw data.
@@ -105,6 +108,7 @@ typedef struct BlockDevOps {
 #define BDRV_BLOCK_DATA         1
 #define BDRV_BLOCK_ZERO         2
 #define BDRV_BLOCK_OFFSET_VALID 4
+#define BDRV_BLOCK_RAW          8
 #define BDRV_BLOCK_OFFSET_MASK  BDRV_SECTOR_MASK
 
 typedef enum {
@@ -142,8 +146,9 @@ BlockDriver *bdrv_find_format(const char *format_name);
 BlockDriver *bdrv_find_whitelisted_format(const char *format_name,
                                           bool readonly);
 int bdrv_create(BlockDriver *drv, const char* filename,
-    QEMUOptionParameter *options);
-int bdrv_create_file(const char* filename, QEMUOptionParameter *options);
+    QEMUOptionParameter *options, Error **errp);
+int bdrv_create_file(const char* filename, QEMUOptionParameter *options,
+                     Error **errp);
 BlockDriverState *bdrv_new(const char *device_name);
 void bdrv_make_anon(BlockDriverState *bs);
 void bdrv_swap(BlockDriverState *bs_new, BlockDriverState *bs_old);
@@ -151,10 +156,10 @@ void bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top);
 int bdrv_parse_cache_flags(const char *mode, int *flags);
 int bdrv_parse_discard_flags(const char *mode, int *flags);
 int bdrv_file_open(BlockDriverState **pbs, const char *filename,
-                   QDict *options, int flags);
-int bdrv_open_backing_file(BlockDriverState *bs, QDict *options);
+                   QDict *options, int flags, Error **errp);
+int bdrv_open_backing_file(BlockDriverState *bs, QDict *options, Error **errp);
 int bdrv_open(BlockDriverState *bs, const char *filename, QDict *options,
-              int flags, BlockDriver *drv);
+              int flags, BlockDriver *drv, Error **errp);
 BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue,
                                     BlockDriverState *bs, int flags);
 int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, Error **errp);
@@ -241,6 +246,22 @@ typedef enum {
 
 int bdrv_check(BlockDriverState *bs, BdrvCheckResult *res, BdrvCheckMode fix);
 
+int bdrv_amend_options(BlockDriverState *bs_new, QEMUOptionParameter *options);
+
+/* external snapshots */
+
+typedef enum {
+    EXT_SNAPSHOT_ALLOWED,
+    EXT_SNAPSHOT_FORBIDDEN,
+} ExtSnapshotPerm;
+
+/* return EXT_SNAPSHOT_ALLOWED if external snapshot is allowed
+ * return EXT_SNAPSHOT_FORBIDDEN if external snapshot is forbidden
+ */
+ExtSnapshotPerm bdrv_check_ext_snapshot(BlockDriverState *bs);
+/* helper used to forbid external snapshots like in blkverify */
+ExtSnapshotPerm bdrv_check_ext_snapshot_forbidden(BlockDriverState *bs);
+
 /* async block I/O */
 typedef void BlockDriverDirtyHandler(BlockDriverState *bs, int64_t sector,
                                      int sector_num);
@@ -332,6 +353,7 @@ int bdrv_get_flags(BlockDriverState *bs);
 int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num,
                           const uint8_t *buf, int nb_sectors);
 int bdrv_get_info(BlockDriverState *bs, BlockDriverInfo *bdi);
+ImageInfoSpecific *bdrv_get_specific_info(BlockDriverState *bs);
 void bdrv_round_to_clusters(BlockDriverState *bs,
                             int64_t sector_num, int nb_sectors,
                             int64_t *cluster_sector_num,
diff --git a/include/block/block_int.h b/include/block/block_int.h
index 7c35198ad7..a48731d539 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -67,12 +67,24 @@ typedef struct BdrvTrackedRequest {
 struct BlockDriver {
     const char *format_name;
     int instance_size;
+
+    /* if not defined external snapshots are allowed
+     * future block filters will query their children to build the response
+     */
+    ExtSnapshotPerm (*bdrv_check_ext_snapshot)(BlockDriverState *bs);
+
     int (*bdrv_probe)(const uint8_t *buf, int buf_size, const char *filename);
     int (*bdrv_probe_device)(const char *filename);
 
     /* Any driver implementing this callback is expected to be able to handle
      * NULL file names in its .bdrv_open() implementation */
     void (*bdrv_parse_filename)(const char *filename, QDict *options, Error **errp);
+    /* Drivers not implementing bdrv_parse_filename nor bdrv_open should have
+     * this field set to true, except ones that are defined only by their
+     * child's bs.
+     * An example of the last type will be the quorum block driver.
+     */
+    bool bdrv_needs_filename;
 
     /* For handling image reopen for split or non-split files */
     int (*bdrv_reopen_prepare)(BDRVReopenState *reopen_state,
@@ -80,15 +92,18 @@ struct BlockDriver {
     void (*bdrv_reopen_commit)(BDRVReopenState *reopen_state);
     void (*bdrv_reopen_abort)(BDRVReopenState *reopen_state);
 
-    int (*bdrv_open)(BlockDriverState *bs, QDict *options, int flags);
-    int (*bdrv_file_open)(BlockDriverState *bs, QDict *options, int flags);
+    int (*bdrv_open)(BlockDriverState *bs, QDict *options, int flags,
+                     Error **errp);
+    int (*bdrv_file_open)(BlockDriverState *bs, QDict *options, int flags,
+                          Error **errp);
     int (*bdrv_read)(BlockDriverState *bs, int64_t sector_num,
                      uint8_t *buf, int nb_sectors);
     int (*bdrv_write)(BlockDriverState *bs, int64_t sector_num,
                       const uint8_t *buf, int nb_sectors);
     void (*bdrv_close)(BlockDriverState *bs);
     void (*bdrv_rebind)(BlockDriverState *bs);
-    int (*bdrv_create)(const char *filename, QEMUOptionParameter *options);
+    int (*bdrv_create)(const char *filename, QEMUOptionParameter *options,
+                       Error **errp);
     int (*bdrv_set_key)(BlockDriverState *bs, const char *key);
     int (*bdrv_make_empty)(BlockDriverState *bs);
     /* aio */
@@ -150,12 +165,16 @@ struct BlockDriver {
                                 QEMUSnapshotInfo *sn_info);
     int (*bdrv_snapshot_goto)(BlockDriverState *bs,
                               const char *snapshot_id);
-    int (*bdrv_snapshot_delete)(BlockDriverState *bs, const char *snapshot_id);
+    int (*bdrv_snapshot_delete)(BlockDriverState *bs,
+                                const char *snapshot_id,
+                                const char *name,
+                                Error **errp);
     int (*bdrv_snapshot_list)(BlockDriverState *bs,
                               QEMUSnapshotInfo **psn_info);
     int (*bdrv_snapshot_load_tmp)(BlockDriverState *bs,
                                   const char *snapshot_name);
     int (*bdrv_get_info)(BlockDriverState *bs, BlockDriverInfo *bdi);
+    ImageInfoSpecific *(*bdrv_get_specific_info)(BlockDriverState *bs);
 
     int (*bdrv_save_vmstate)(BlockDriverState *bs, QEMUIOVector *qiov,
                              int64_t pos);
@@ -188,6 +207,9 @@ struct BlockDriver {
     int (*bdrv_check)(BlockDriverState* bs, BdrvCheckResult *result,
         BdrvCheckMode fix);
 
+    int (*bdrv_amend_options)(BlockDriverState *bs,
+        QEMUOptionParameter *options);
+
     void (*bdrv_debug_event)(BlockDriverState *bs, BlkDebugEvent event);
 
     /* TODO Better pass a option string/QDict/QemuOpts to add any rule? */
diff --git a/include/block/blockjob.h b/include/block/blockjob.h
index d530409ff5..d76de62a46 100644
--- a/include/block/blockjob.h
+++ b/include/block/blockjob.h
@@ -28,16 +28,16 @@
 #include "block/block.h"
 
 /**
- * BlockJobType:
+ * BlockJobDriver:
  *
- * A class type for block job objects.
+ * A class type for block job driver.
  */
-typedef struct BlockJobType {
+typedef struct BlockJobDriver {
     /** Derived BlockJob struct size */
     size_t instance_size;
 
     /** String describing the operation, part of query-block-jobs QMP API */
-    const char *job_type;
+    BlockJobType job_type;
 
     /** Optional callback for job types that support setting a speed limit */
     void (*set_speed)(BlockJob *job, int64_t speed, Error **errp);
@@ -50,7 +50,7 @@ typedef struct BlockJobType {
      * manually.
      */
     void (*complete)(BlockJob *job, Error **errp);
-} BlockJobType;
+} BlockJobDriver;
 
 /**
  * BlockJob:
@@ -59,7 +59,7 @@ typedef struct BlockJobType {
  */
 struct BlockJob {
     /** The job type, including the job vtable.  */
-    const BlockJobType *job_type;
+    const BlockJobDriver *driver;
 
     /** The block device on which the job is operating.  */
     BlockDriverState *bs;
@@ -128,7 +128,7 @@ struct BlockJob {
  * This function is not part of the public job interface; it should be
  * called from a wrapper that is specific to the job type.
  */
-void *block_job_create(const BlockJobType *job_type, BlockDriverState *bs,
+void *block_job_create(const BlockJobDriver *driver, BlockDriverState *bs,
                        int64_t speed, BlockDriverCompletionFunc *cb,
                        void *opaque, Error **errp);
 
diff --git a/include/block/qapi.h b/include/block/qapi.h
index 0496cc9282..9518ee4001 100644
--- a/include/block/qapi.h
+++ b/include/block/qapi.h
@@ -42,6 +42,8 @@ BlockStats *bdrv_query_stats(const BlockDriverState *bs);
 
 void bdrv_snapshot_dump(fprintf_function func_fprintf, void *f,
                         QEMUSnapshotInfo *sn);
+void bdrv_image_info_specific_dump(fprintf_function func_fprintf, void *f,
+                                   ImageInfoSpecific *info_spec);
 void bdrv_image_info_dump(fprintf_function func_fprintf, void *f,
                           ImageInfo *info);
 #endif
diff --git a/include/block/snapshot.h b/include/block/snapshot.h
index eaf61f0326..012bf226d3 100644
--- a/include/block/snapshot.h
+++ b/include/block/snapshot.h
@@ -26,6 +26,7 @@
 #define SNAPSHOT_H
 
 #include "qemu-common.h"
+#include "qapi/error.h"
 
 typedef struct QEMUSnapshotInfo {
     char id_str[128]; /* unique snapshot id */
@@ -40,12 +41,23 @@ typedef struct QEMUSnapshotInfo {
 
 int bdrv_snapshot_find(BlockDriverState *bs, QEMUSnapshotInfo *sn_info,
                        const char *name);
+bool bdrv_snapshot_find_by_id_and_name(BlockDriverState *bs,
+                                       const char *id,
+                                       const char *name,
+                                       QEMUSnapshotInfo *sn_info,
+                                       Error **errp);
 int bdrv_can_snapshot(BlockDriverState *bs);
 int bdrv_snapshot_create(BlockDriverState *bs,
                          QEMUSnapshotInfo *sn_info);
 int bdrv_snapshot_goto(BlockDriverState *bs,
                        const char *snapshot_id);
-int bdrv_snapshot_delete(BlockDriverState *bs, const char *snapshot_id);
+int bdrv_snapshot_delete(BlockDriverState *bs,
+                         const char *snapshot_id,
+                         const char *name,
+                         Error **errp);
+void bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs,
+                                        const char *id_or_name,
+                                        Error **errp);
 int bdrv_snapshot_list(BlockDriverState *bs,
                        QEMUSnapshotInfo **psn_info);
 int bdrv_snapshot_load_tmp(BlockDriverState *bs,
diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h
index a407b50f4a..b6998f055a 100644
--- a/include/exec/cpu-all.h
+++ b/include/exec/cpu-all.h
@@ -453,9 +453,7 @@ typedef struct RAMBlock {
      * Writes must take both locks.
      */
     QTAILQ_ENTRY(RAMBlock) next;
-#if defined(__linux__) && !defined(TARGET_S390X)
     int fd;
-#endif
 } RAMBlock;
 
 typedef struct RAMList {
diff --git a/include/exec/cpu-defs.h b/include/exec/cpu-defs.h
index a5c028c536..01cd8c7a2b 100644
--- a/include/exec/cpu-defs.h
+++ b/include/exec/cpu-defs.h
@@ -178,7 +178,5 @@ typedef struct CPUWatchpoint {
                                                                         \
     /* user data */                                                     \
     void *opaque;                                                       \
-                                                                        \
-    const char *cpu_model_str;
 
 #endif
diff --git a/include/exec/def-helper.h b/include/exec/def-helper.h
index 022a9ceb6a..73d51f9cf5 100644
--- a/include/exec/def-helper.h
+++ b/include/exec/def-helper.h
@@ -240,8 +240,7 @@ static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \
 #elif GEN_HELPER == 2
 /* Register helpers.  */
 
-#define DEF_HELPER_FLAGS_0(name, flags, ret) \
-tcg_register_helper(HELPER(name), #name);
+#define DEF_HELPER_FLAGS_0(name, flags, ret)  { HELPER(name), #name },
 
 #define DEF_HELPER_FLAGS_1(name, flags, ret, t1) \
 DEF_HELPER_FLAGS_0(name, flags, ret)
diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h
index beb41491b4..ea90b649d4 100644
--- a/include/exec/exec-all.h
+++ b/include/exec/exec-all.h
@@ -320,54 +320,10 @@ extern uintptr_t tci_tb_ptr;
 
 #define GETPC()  (GETRA() - GETPC_ADJ)
 
-/* The LDST optimizations splits code generation into fast and slow path.
-   In some implementations, we pass the "logical" return address manually;
-   in others, we must infer the logical return from the true return.  */
-#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
-# if defined (_ARCH_PPC) && !defined (_ARCH_PPC64)
-#  define GETRA_LDST(RA)   (*(int32_t *)((RA) - 4))
-# elif defined(__arm__)
-/* We define two insns between the return address and the branch back to
-   straight-line.  Find and decode that branch insn.  */
-#  define GETRA_LDST(RA)   tcg_getra_ldst(RA)
-static inline uintptr_t tcg_getra_ldst(uintptr_t ra)
-{
-    int32_t b;
-    ra += 8;                    /* skip the two insns */
-    b = *(int32_t *)ra;         /* load the branch insn */
-    b = (b << 8) >> (8 - 2);    /* extract the displacement */
-    ra += 8;                    /* branches are relative to pc+8 */
-    ra += b;                    /* apply the displacement */
-    return ra;
-}
-# elif defined(__aarch64__)
-#  define GETRA_LDST(RA)  tcg_getra_ldst(RA)
-static inline uintptr_t tcg_getra_ldst(uintptr_t ra)
-{
-    int32_t b;
-    ra += 4;                    /* skip one instruction */
-    b = *(int32_t *)ra;         /* load the branch insn */
-    b = (b << 6) >> (6 - 2);    /* extract the displacement */
-    ra += b;                    /* apply the displacement  */
-    return ra;
-}
-# endif
-#endif /* CONFIG_QEMU_LDST_OPTIMIZATION */
-
-/* ??? Delete these once they are no longer used.  */
-bool is_tcg_gen_code(uintptr_t pc_ptr);
-#ifdef GETRA_LDST
-# define GETRA_EXT()  tcg_getra_ext(GETRA())
-static inline uintptr_t tcg_getra_ext(uintptr_t ra)
-{
-    return is_tcg_gen_code(ra) ? GETRA_LDST(ra) : ra;
-}
-#else
-# define GETRA_EXT()  GETRA()
-#endif
-
 #if !defined(CONFIG_USER_ONLY)
 
+void phys_mem_set_alloc(void *(*alloc)(size_t));
+
 struct MemoryRegion *iotlb_to_region(hwaddr index);
 bool io_mem_read(struct MemoryRegion *mr, hwaddr addr,
                  uint64_t *pvalue, unsigned size);
diff --git a/include/exec/ioport.h b/include/exec/ioport.h
index b3848be684..3bd6722627 100644
--- a/include/exec/ioport.h
+++ b/include/exec/ioport.h
@@ -64,11 +64,13 @@ typedef struct PortioList {
     struct MemoryRegion **regions;
     void *opaque;
     const char *name;
+    bool flush_coalesced_mmio;
 } PortioList;
 
 void portio_list_init(PortioList *piolist, Object *owner,
                       const struct MemoryRegionPortio *callbacks,
                       void *opaque, const char *name);
+void portio_list_set_flush_coalesced(PortioList *piolist);
 void portio_list_destroy(PortioList *piolist);
 void portio_list_add(PortioList *piolist,
                      struct MemoryRegion *address_space,
diff --git a/include/exec/softmmu_template.h b/include/exec/softmmu_template.h
index 5bbc56afd5..c6a544069c 100644
--- a/include/exec/softmmu_template.h
+++ b/include/exec/softmmu_template.h
@@ -70,6 +70,48 @@
 #define ADDR_READ addr_read
 #endif
 
+#if DATA_SIZE == 8
+# define BSWAP(X)  bswap64(X)
+#elif DATA_SIZE == 4
+# define BSWAP(X)  bswap32(X)
+#elif DATA_SIZE == 2
+# define BSWAP(X)  bswap16(X)
+#else
+# define BSWAP(X)  (X)
+#endif
+
+#ifdef TARGET_WORDS_BIGENDIAN
+# define TGT_BE(X)  (X)
+# define TGT_LE(X)  BSWAP(X)
+#else
+# define TGT_BE(X)  BSWAP(X)
+# define TGT_LE(X)  (X)
+#endif
+
+#if DATA_SIZE == 1
+# define helper_le_ld_name  glue(glue(helper_ret_ld, USUFFIX), MMUSUFFIX)
+# define helper_be_ld_name  helper_le_ld_name
+# define helper_le_lds_name glue(glue(helper_ret_ld, SSUFFIX), MMUSUFFIX)
+# define helper_be_lds_name helper_le_lds_name
+# define helper_le_st_name  glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)
+# define helper_be_st_name  helper_le_st_name
+#else
+# define helper_le_ld_name  glue(glue(helper_le_ld, USUFFIX), MMUSUFFIX)
+# define helper_be_ld_name  glue(glue(helper_be_ld, USUFFIX), MMUSUFFIX)
+# define helper_le_lds_name glue(glue(helper_le_ld, SSUFFIX), MMUSUFFIX)
+# define helper_be_lds_name glue(glue(helper_be_ld, SSUFFIX), MMUSUFFIX)
+# define helper_le_st_name  glue(glue(helper_le_st, SUFFIX), MMUSUFFIX)
+# define helper_be_st_name  glue(glue(helper_be_st, SUFFIX), MMUSUFFIX)
+#endif
+
+#ifdef TARGET_WORDS_BIGENDIAN
+# define helper_te_ld_name  helper_be_ld_name
+# define helper_te_st_name  helper_be_st_name
+#else
+# define helper_te_ld_name  helper_le_ld_name
+# define helper_te_st_name  helper_le_st_name
+#endif
+
 static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,
                                               hwaddr physaddr,
                                               target_ulong addr,
@@ -89,18 +131,16 @@ static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,
     return val;
 }
 
-/* handle all cases except unaligned access which span two pages */
 #ifdef SOFTMMU_CODE_ACCESS
-static
+static __attribute__((unused))
 #endif
-WORD_TYPE
-glue(glue(helper_ret_ld, USUFFIX), MMUSUFFIX)(CPUArchState *env,
-                                              target_ulong addr, int mmu_idx,
-                                              uintptr_t retaddr)
+WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr, int mmu_idx,
+                            uintptr_t retaddr)
 {
     int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
     target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
     uintptr_t haddr;
+    DATA_TYPE res;
 
     /* Adjust the given return address.  */
     retaddr -= GETPC_ADJ;
@@ -124,7 +164,12 @@ glue(glue(helper_ret_ld, USUFFIX), MMUSUFFIX)(CPUArchState *env,
             goto do_unaligned_access;
         }
         ioaddr = env->iotlb[mmu_idx][index];
-        return glue(io_read, SUFFIX)(env, ioaddr, addr, retaddr);
+
+        /* ??? Note that the io helpers always read data in the target
+           byte ordering.  We should push the LE/BE request down into io.  */
+        res = glue(io_read, SUFFIX)(env, ioaddr, addr, retaddr);
+        res = TGT_LE(res);
+        return res;
     }
 
     /* Handle slow unaligned access (it spans two pages or IO).  */
@@ -132,7 +177,7 @@ glue(glue(helper_ret_ld, USUFFIX), MMUSUFFIX)(CPUArchState *env,
         && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
                     >= TARGET_PAGE_SIZE)) {
         target_ulong addr1, addr2;
-        DATA_TYPE res1, res2, res;
+        DATA_TYPE res1, res2;
         unsigned shift;
     do_unaligned_access:
 #ifdef ALIGNED_ONLY
@@ -142,16 +187,94 @@ glue(glue(helper_ret_ld, USUFFIX), MMUSUFFIX)(CPUArchState *env,
         addr2 = addr1 + DATA_SIZE;
         /* Note the adjustment at the beginning of the function.
            Undo that for the recursion.  */
-        res1 = glue(glue(helper_ret_ld, USUFFIX), MMUSUFFIX)
-            (env, addr1, mmu_idx, retaddr + GETPC_ADJ);
-        res2 = glue(glue(helper_ret_ld, USUFFIX), MMUSUFFIX)
-            (env, addr2, mmu_idx, retaddr + GETPC_ADJ);
+        res1 = helper_le_ld_name(env, addr1, mmu_idx, retaddr + GETPC_ADJ);
+        res2 = helper_le_ld_name(env, addr2, mmu_idx, retaddr + GETPC_ADJ);
         shift = (addr & (DATA_SIZE - 1)) * 8;
-#ifdef TARGET_WORDS_BIGENDIAN
-        res = (res1 << shift) | (res2 >> ((DATA_SIZE * 8) - shift));
-#else
+
+        /* Little-endian combine.  */
         res = (res1 >> shift) | (res2 << ((DATA_SIZE * 8) - shift));
+        return res;
+    }
+
+    /* Handle aligned access or unaligned access in the same page.  */
+#ifdef ALIGNED_ONLY
+    if ((addr & (DATA_SIZE - 1)) != 0) {
+        do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
+    }
+#endif
+
+    haddr = addr + env->tlb_table[mmu_idx][index].addend;
+#if DATA_SIZE == 1
+    res = glue(glue(ld, LSUFFIX), _p)((uint8_t *)haddr);
+#else
+    res = glue(glue(ld, LSUFFIX), _le_p)((uint8_t *)haddr);
+#endif
+    return res;
+}
+
+#if DATA_SIZE > 1
+#ifdef SOFTMMU_CODE_ACCESS
+static __attribute__((unused))
+#endif
+WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr, int mmu_idx,
+                            uintptr_t retaddr)
+{
+    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
+    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
+    uintptr_t haddr;
+    DATA_TYPE res;
+
+    /* Adjust the given return address.  */
+    retaddr -= GETPC_ADJ;
+
+    /* If the TLB entry is for a different page, reload and try again.  */
+    if ((addr & TARGET_PAGE_MASK)
+         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
+#ifdef ALIGNED_ONLY
+        if ((addr & (DATA_SIZE - 1)) != 0) {
+            do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
+        }
+#endif
+        tlb_fill(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
+        tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
+    }
+
+    /* Handle an IO access.  */
+    if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
+        hwaddr ioaddr;
+        if ((addr & (DATA_SIZE - 1)) != 0) {
+            goto do_unaligned_access;
+        }
+        ioaddr = env->iotlb[mmu_idx][index];
+
+        /* ??? Note that the io helpers always read data in the target
+           byte ordering.  We should push the LE/BE request down into io.  */
+        res = glue(io_read, SUFFIX)(env, ioaddr, addr, retaddr);
+        res = TGT_BE(res);
+        return res;
+    }
+
+    /* Handle slow unaligned access (it spans two pages or IO).  */
+    if (DATA_SIZE > 1
+        && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
+                    >= TARGET_PAGE_SIZE)) {
+        target_ulong addr1, addr2;
+        DATA_TYPE res1, res2;
+        unsigned shift;
+    do_unaligned_access:
+#ifdef ALIGNED_ONLY
+        do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
 #endif
+        addr1 = addr & ~(DATA_SIZE - 1);
+        addr2 = addr1 + DATA_SIZE;
+        /* Note the adjustment at the beginning of the function.
+           Undo that for the recursion.  */
+        res1 = helper_be_ld_name(env, addr1, mmu_idx, retaddr + GETPC_ADJ);
+        res2 = helper_be_ld_name(env, addr2, mmu_idx, retaddr + GETPC_ADJ);
+        shift = (addr & (DATA_SIZE - 1)) * 8;
+
+        /* Big-endian combine.  */
+        res = (res1 << shift) | (res2 >> ((DATA_SIZE * 8) - shift));
         return res;
     }
 
@@ -163,16 +286,16 @@ glue(glue(helper_ret_ld, USUFFIX), MMUSUFFIX)(CPUArchState *env,
 #endif
 
     haddr = addr + env->tlb_table[mmu_idx][index].addend;
-    /* Note that ldl_raw is defined with type "int".  */
-    return (DATA_TYPE) glue(glue(ld, LSUFFIX), _raw)((uint8_t *)haddr);
+    res = glue(glue(ld, LSUFFIX), _be_p)((uint8_t *)haddr);
+    return res;
 }
+#endif /* DATA_SIZE > 1 */
 
 DATA_TYPE
 glue(glue(helper_ld, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,
                                          int mmu_idx)
 {
-    return glue(glue(helper_ret_ld, USUFFIX), MMUSUFFIX)(env, addr, mmu_idx,
-                                                        GETRA_EXT());
+    return helper_te_ld_name (env, addr, mmu_idx, GETRA());
 }
 
 #ifndef SOFTMMU_CODE_ACCESS
@@ -180,14 +303,19 @@ glue(glue(helper_ld, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,
 /* Provide signed versions of the load routines as well.  We can of course
    avoid this for 64-bit data, or for 32-bit data on 32-bit host.  */
 #if DATA_SIZE * 8 < TCG_TARGET_REG_BITS
-WORD_TYPE
-glue(glue(helper_ret_ld, SSUFFIX), MMUSUFFIX)(CPUArchState *env,
-                                              target_ulong addr, int mmu_idx,
-                                              uintptr_t retaddr)
+WORD_TYPE helper_le_lds_name(CPUArchState *env, target_ulong addr,
+                             int mmu_idx, uintptr_t retaddr)
+{
+    return (SDATA_TYPE)helper_le_ld_name(env, addr, mmu_idx, retaddr);
+}
+
+# if DATA_SIZE > 1
+WORD_TYPE helper_be_lds_name(CPUArchState *env, target_ulong addr,
+                             int mmu_idx, uintptr_t retaddr)
 {
-    return (SDATA_TYPE) glue(glue(helper_ret_ld, USUFFIX), MMUSUFFIX)
-        (env, addr, mmu_idx, retaddr);
+    return (SDATA_TYPE)helper_be_ld_name(env, addr, mmu_idx, retaddr);
 }
+# endif
 #endif
 
 static inline void glue(io_write, SUFFIX)(CPUArchState *env,
@@ -208,10 +336,8 @@ static inline void glue(io_write, SUFFIX)(CPUArchState *env,
     io_mem_write(mr, physaddr, val, 1 << SHIFT);
 }
 
-void
-glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)(CPUArchState *env,
-                                             target_ulong addr, DATA_TYPE val,
-                                             int mmu_idx, uintptr_t retaddr)
+void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
+                       int mmu_idx, uintptr_t retaddr)
 {
     int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
     target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
@@ -239,6 +365,10 @@ glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)(CPUArchState *env,
             goto do_unaligned_access;
         }
         ioaddr = env->iotlb[mmu_idx][index];
+
+        /* ??? Note that the io helpers always read data in the target
+           byte ordering.  We should push the LE/BE request down into io.  */
+        val = TGT_LE(val);
         glue(io_write, SUFFIX)(env, ioaddr, val, addr, retaddr);
         return;
     }
@@ -256,11 +386,84 @@ glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)(CPUArchState *env,
         /* Note: relies on the fact that tlb_fill() does not remove the
          * previous page from the TLB cache.  */
         for (i = DATA_SIZE - 1; i >= 0; i--) {
-#ifdef TARGET_WORDS_BIGENDIAN
-            uint8_t val8 = val >> (((DATA_SIZE - 1) * 8) - (i * 8));
-#else
+            /* Little-endian extract.  */
             uint8_t val8 = val >> (i * 8);
+            /* Note the adjustment at the beginning of the function.
+               Undo that for the recursion.  */
+            glue(helper_ret_stb, MMUSUFFIX)(env, addr + i, val8,
+                                            mmu_idx, retaddr + GETPC_ADJ);
+        }
+        return;
+    }
+
+    /* Handle aligned access or unaligned access in the same page.  */
+#ifdef ALIGNED_ONLY
+    if ((addr & (DATA_SIZE - 1)) != 0) {
+        do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
+    }
+#endif
+
+    haddr = addr + env->tlb_table[mmu_idx][index].addend;
+#if DATA_SIZE == 1
+    glue(glue(st, SUFFIX), _p)((uint8_t *)haddr, val);
+#else
+    glue(glue(st, SUFFIX), _le_p)((uint8_t *)haddr, val);
 #endif
+}
+
+#if DATA_SIZE > 1
+void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
+                       int mmu_idx, uintptr_t retaddr)
+{
+    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
+    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
+    uintptr_t haddr;
+
+    /* Adjust the given return address.  */
+    retaddr -= GETPC_ADJ;
+
+    /* If the TLB entry is for a different page, reload and try again.  */
+    if ((addr & TARGET_PAGE_MASK)
+        != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
+#ifdef ALIGNED_ONLY
+        if ((addr & (DATA_SIZE - 1)) != 0) {
+            do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
+        }
+#endif
+        tlb_fill(env, addr, 1, mmu_idx, retaddr);
+        tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
+    }
+
+    /* Handle an IO access.  */
+    if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
+        hwaddr ioaddr;
+        if ((addr & (DATA_SIZE - 1)) != 0) {
+            goto do_unaligned_access;
+        }
+        ioaddr = env->iotlb[mmu_idx][index];
+
+        /* ??? Note that the io helpers always read data in the target
+           byte ordering.  We should push the LE/BE request down into io.  */
+        val = TGT_BE(val);
+        glue(io_write, SUFFIX)(env, ioaddr, val, addr, retaddr);
+        return;
+    }
+
+    /* Handle slow unaligned access (it spans two pages or IO).  */
+    if (DATA_SIZE > 1
+        && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
+                     >= TARGET_PAGE_SIZE)) {
+        int i;
+    do_unaligned_access:
+#ifdef ALIGNED_ONLY
+        do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
+#endif
+        /* XXX: not efficient, but simple */
+        /* Note: relies on the fact that tlb_fill() does not remove the
+         * previous page from the TLB cache.  */
+        for (i = DATA_SIZE - 1; i >= 0; i--) {
+            /* Big-endian extract.  */
+            uint8_t val8 = val >> (((DATA_SIZE - 1) * 8) - (i * 8));
             /* Note the adjustment at the beginning of the function.
                Undo that for the recursion.  */
             glue(helper_ret_stb, MMUSUFFIX)(env, addr + i, val8,
@@ -277,15 +480,15 @@ glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)(CPUArchState *env,
 #endif
 
     haddr = addr + env->tlb_table[mmu_idx][index].addend;
-    glue(glue(st, SUFFIX), _raw)((uint8_t *)haddr, val);
+    glue(glue(st, SUFFIX), _be_p)((uint8_t *)haddr, val);
 }
+#endif /* DATA_SIZE > 1 */
 
 void
 glue(glue(helper_st, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,
                                          DATA_TYPE val, int mmu_idx)
 {
-    glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)(env, addr, val, mmu_idx,
-                                                 GETRA_EXT());
+    helper_te_st_name(env, addr, val, mmu_idx, GETRA());
 }
 
 #endif /* !defined(SOFTMMU_CODE_ACCESS) */
@@ -301,3 +504,16 @@ glue(glue(helper_st, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,
 #undef SDATA_TYPE
 #undef USUFFIX
 #undef SSUFFIX
+#undef BSWAP
+#undef TGT_BE
+#undef TGT_LE
+#undef CPU_BE
+#undef CPU_LE
+#undef helper_le_ld_name
+#undef helper_be_ld_name
+#undef helper_le_lds_name
+#undef helper_be_lds_name
+#undef helper_le_st_name
+#undef helper_be_st_name
+#undef helper_te_ld_name
+#undef helper_te_st_name
diff --git a/include/hw/i386/pc.h b/include/hw/i386/pc.h
index 39db8cbbff..03cc0ba0f7 100644
--- a/include/hw/i386/pc.h
+++ b/include/hw/i386/pc.h
@@ -252,6 +252,14 @@ int e820_add_entry(uint64_t, uint64_t, uint32_t);
             .driver   = "e1000",\
             .property = "mitigation",\
             .value    = "off",\
+        },{\
+            .driver   = "qemu64-" TYPE_X86_CPU,\
+            .property = "model",\
+            .value    = stringify(2),\
+        },{\
+            .driver   = "qemu32-" TYPE_X86_CPU,\
+            .property = "model",\
+            .value    = stringify(3),\
         }
 
 #define PC_COMPAT_1_5 \
diff --git a/include/hw/misc/arm_integrator_debug.h b/include/hw/misc/arm_integrator_debug.h
new file mode 100644
index 0000000000..37789b69d9
--- /dev/null
+++ b/include/hw/misc/arm_integrator_debug.h
@@ -0,0 +1,18 @@
+/*
+ * ARM Integrator Board Debug, switch and LED section
+ *
+ * Browse the data sheet:
+ *
+ *    http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.dui0159b/Babbfijf.html
+ *
+ * Copyright (c) 2013 Alex Bennée <alex@bennee.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#ifndef QEMU_INTEGRATOR_DEBUG_H
+#define QEMU_INTEGRATOR_DEBUG_H
+
+#define TYPE_INTEGRATOR_DEBUG "integrator_debug"
+
+#endif
diff --git a/include/hw/pci-host/q35.h b/include/hw/pci-host/q35.h
index f9db770104..aee91aa038 100644
--- a/include/hw/pci-host/q35.h
+++ b/include/hw/pci-host/q35.h
@@ -85,7 +85,7 @@ typedef struct Q35PCIHost {
 #define MCH_HOST_BRIDGE_CONFIG_DATA            0xcfc
 
 /* D0:F0 configuration space */
-#define MCH_HOST_BRIDGE_REVISION_DEFUALT       0x0
+#define MCH_HOST_BRIDGE_REVISION_DEFAULT       0x0
 
 #define MCH_HOST_BRIDGE_PCIEXBAR               0x60    /* 64bit register */
 #define MCH_HOST_BRIDGE_PCIEXBAR_SIZE          8       /* 64bit register */
diff --git a/include/hw/pci/pci_ids.h b/include/hw/pci/pci_ids.h
index 3ddaf6aad5..4c0002beca 100644
--- a/include/hw/pci/pci_ids.h
+++ b/include/hw/pci/pci_ids.h
@@ -53,6 +53,7 @@
 /* Vendors and devices.  Sort key: vendor first, device next. */
 
 #define PCI_VENDOR_ID_LSI_LOGIC          0x1000
+#define PCI_DEVICE_ID_LSI_53C810         0x0001
 #define PCI_DEVICE_ID_LSI_53C895A        0x0012
 #define PCI_DEVICE_ID_LSI_SAS1078        0x0060
 
diff --git a/include/hw/qdev-core.h b/include/hw/qdev-core.h
index a62f231eb9..e191ca0bd2 100644
--- a/include/hw/qdev-core.h
+++ b/include/hw/qdev-core.h
@@ -30,22 +30,6 @@ typedef enum DeviceCategory {
     DEVICE_CATEGORY_MAX
 } DeviceCategory;
 
-static inline const char *qdev_category_get_name(DeviceCategory category)
-{
-    static const char *category_names[DEVICE_CATEGORY_MAX] = {
-        [DEVICE_CATEGORY_BRIDGE]  = "Controller/Bridge/Hub",
-        [DEVICE_CATEGORY_USB]     = "USB",
-        [DEVICE_CATEGORY_STORAGE] = "Storage",
-        [DEVICE_CATEGORY_NETWORK] = "Network",
-        [DEVICE_CATEGORY_INPUT]   = "Input",
-        [DEVICE_CATEGORY_DISPLAY] = "Display",
-        [DEVICE_CATEGORY_SOUND]   = "Sound",
-        [DEVICE_CATEGORY_MISC]    = "Misc",
-    };
-
-    return category_names[category];
-};
-
 typedef int (*qdev_initfn)(DeviceState *dev);
 typedef int (*qdev_event)(DeviceState *dev);
 typedef void (*qdev_resetfn)(DeviceState *dev);
diff --git a/include/hw/s390x/ebcdic.h b/include/hw/s390x/ebcdic.h
new file mode 100644
index 0000000000..1d6fde9c12
--- /dev/null
+++ b/include/hw/s390x/ebcdic.h
@@ -0,0 +1,104 @@
+/*
+ * EBCDIC/ASCII conversion Support
+ *
+ * Copyright (c) 2011 Alexander Graf
+ * Copyright IBM, Corp. 2013
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version.  See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef EBCDIC_H_
+#define EBCDIC_H_
+
+/* EBCDIC handling */
+static const uint8_t ebcdic2ascii[] = {
+    0x00, 0x01, 0x02, 0x03, 0x07, 0x09, 0x07, 0x7F,
+    0x07, 0x07, 0x07, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
+    0x10, 0x11, 0x12, 0x13, 0x07, 0x0A, 0x08, 0x07,
+    0x18, 0x19, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07,
+    0x07, 0x07, 0x1C, 0x07, 0x07, 0x0A, 0x17, 0x1B,
+    0x07, 0x07, 0x07, 0x07, 0x07, 0x05, 0x06, 0x07,
+    0x07, 0x07, 0x16, 0x07, 0x07, 0x07, 0x07, 0x04,
+    0x07, 0x07, 0x07, 0x07, 0x14, 0x15, 0x07, 0x1A,
+    0x20, 0xFF, 0x83, 0x84, 0x85, 0xA0, 0x07, 0x86,
+    0x87, 0xA4, 0x5B, 0x2E, 0x3C, 0x28, 0x2B, 0x21,
+    0x26, 0x82, 0x88, 0x89, 0x8A, 0xA1, 0x8C, 0x07,
+    0x8D, 0xE1, 0x5D, 0x24, 0x2A, 0x29, 0x3B, 0x5E,
+    0x2D, 0x2F, 0x07, 0x8E, 0x07, 0x07, 0x07, 0x8F,
+    0x80, 0xA5, 0x07, 0x2C, 0x25, 0x5F, 0x3E, 0x3F,
+    0x07, 0x90, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07,
+    0x70, 0x60, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22,
+    0x07, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
+    0x68, 0x69, 0xAE, 0xAF, 0x07, 0x07, 0x07, 0xF1,
+    0xF8, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70,
+    0x71, 0x72, 0xA6, 0xA7, 0x91, 0x07, 0x92, 0x07,
+    0xE6, 0x7E, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
+    0x79, 0x7A, 0xAD, 0xAB, 0x07, 0x07, 0x07, 0x07,
+    0x9B, 0x9C, 0x9D, 0xFA, 0x07, 0x07, 0x07, 0xAC,
+    0xAB, 0x07, 0xAA, 0x7C, 0x07, 0x07, 0x07, 0x07,
+    0x7B, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
+    0x48, 0x49, 0x07, 0x93, 0x94, 0x95, 0xA2, 0x07,
+    0x7D, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50,
+    0x51, 0x52, 0x07, 0x96, 0x81, 0x97, 0xA3, 0x98,
+    0x5C, 0xF6, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58,
+    0x59, 0x5A, 0xFD, 0x07, 0x99, 0x07, 0x07, 0x07,
+    0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
+    0x38, 0x39, 0x07, 0x07, 0x9A, 0x07, 0x07, 0x07,
+};
+
+static const uint8_t ascii2ebcdic[] = {
+    0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F,
+    0x16, 0x05, 0x15, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
+    0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26,
+    0x18, 0x19, 0x3F, 0x27, 0x22, 0x1D, 0x1E, 0x1F,
+    0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D,
+    0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61,
+    0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
+    0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F,
+    0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
+    0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6,
+    0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6,
+    0xE7, 0xE8, 0xE9, 0xBA, 0xE0, 0xBB, 0xB0, 0x6D,
+    0x79, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+    0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96,
+    0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6,
+    0xA7, 0xA8, 0xA9, 0xC0, 0x4F, 0xD0, 0xA1, 0x07,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x59, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x90, 0x3F, 0x3F, 0x3F, 0x3F, 0xEA, 0x3F, 0xFF
+};
+
+static inline void ebcdic_put(uint8_t *p, const char *ascii, int len)
+{
+    int i;
+
+    for (i = 0; i < len; i++) {
+        p[i] = ascii2ebcdic[(uint8_t)ascii[i]];
+    }
+}
+
+static inline void ascii_put(uint8_t *p, const char *ebcdic, int len)
+{
+    int i;
+
+    for (i = 0; i < len; i++) {
+        p[i] = ebcdic2ascii[(uint8_t)ebcdic[i]];
+    }
+}
+
+#endif /* EBCDIC_H_ */
diff --git a/include/hw/s390x/event-facility.h b/include/hw/s390x/event-facility.h
index 791ab2a6de..7ce7079f9f 100644
--- a/include/hw/s390x/event-facility.h
+++ b/include/hw/s390x/event-facility.h
@@ -19,12 +19,18 @@
 #include "qemu/thread.h"
 
 /* SCLP event types */
+#define SCLP_EVENT_OPRTNS_COMMAND               0x01
+#define SCLP_EVENT_MESSAGE                      0x02
+#define SCLP_EVENT_PMSGCMD                      0x09
 #define SCLP_EVENT_ASCII_CONSOLE_DATA           0x1a
 #define SCLP_EVENT_SIGNAL_QUIESCE               0x1d
 
 /* SCLP event masks */
 #define SCLP_EVENT_MASK_SIGNAL_QUIESCE          0x00000008
 #define SCLP_EVENT_MASK_MSG_ASCII               0x00000040
+#define SCLP_EVENT_MASK_OP_CMD                  0x80000000
+#define SCLP_EVENT_MASK_MSG                     0x40000000
+#define SCLP_EVENT_MASK_PMSGCMD                 0x00800000
 
 #define SCLP_UNCONDITIONAL_READ                 0x00
 #define SCLP_SELECTIVE_READ                     0x01
@@ -43,8 +49,8 @@ typedef struct WriteEventMask {
     uint16_t mask_length;
     uint32_t cp_receive_mask;
     uint32_t cp_send_mask;
-    uint32_t send_mask;
     uint32_t receive_mask;
+    uint32_t send_mask;
 } QEMU_PACKED WriteEventMask;
 
 typedef struct EventBufferHeader {
@@ -54,6 +60,80 @@ typedef struct EventBufferHeader {
     uint16_t _reserved;
 } QEMU_PACKED EventBufferHeader;
 
+typedef struct MdbHeader {
+    uint16_t length;
+    uint16_t type;
+    uint32_t tag;
+    uint32_t revision_code;
+} QEMU_PACKED MdbHeader;
+
+typedef struct MTO {
+    uint16_t line_type_flags;
+    uint8_t  alarm_control;
+    uint8_t  _reserved[3];
+    char     message[];
+} QEMU_PACKED MTO;
+
+typedef struct GO {
+    uint32_t domid;
+    uint8_t  hhmmss_time[8];
+    uint8_t  th_time[3];
+    uint8_t  _reserved_0;
+    uint8_t  dddyyyy_date[7];
+    uint8_t  _reserved_1;
+    uint16_t general_msg_flags;
+    uint8_t  _reserved_2[10];
+    uint8_t  originating_system_name[8];
+    uint8_t  job_guest_name[8];
+} QEMU_PACKED GO;
+
+#define MESSAGE_TEXT 0x0004
+
+typedef struct MDBO {
+    uint16_t length;
+    uint16_t type;
+    union {
+        GO go;
+        MTO mto;
+    };
+} QEMU_PACKED MDBO;
+
+typedef struct MDB {
+    MdbHeader header;
+    MDBO mdbo[0];
+} QEMU_PACKED MDB;
+
+typedef struct SclpMsg {
+    EventBufferHeader header;
+    MDB mdb;
+} QEMU_PACKED SclpMsg;
+
+#define GDS_ID_MDSMU                            0x1310
+#define GDS_ID_CPMSU                            0x1212
+#define GDS_ID_TEXTCMD                          0x1320
+
+typedef struct GdsVector {
+    uint16_t length;
+    uint16_t gds_id;
+} QEMU_PACKED GdsVector;
+
+#define GDS_KEY_SELFDEFTEXTMSG                  0x31
+#define GDS_KEY_TEXTMSG                         0x30
+
+typedef struct GdsSubvector {
+    uint8_t length;
+    uint8_t key;
+} QEMU_PACKED GdsSubvector;
+
+/* MDS Message Unit */
+typedef struct MDMSU {
+    GdsVector mdmsu;
+    GdsVector cpmsu;
+    GdsVector text_command;
+    GdsSubvector self_def_text_message;
+    GdsSubvector text_message;
+} QEMU_PACKED MDMSU;
+
 typedef struct WriteEventData {
     SCCBHeader h;
     EventBufferHeader ebh;
@@ -68,7 +148,6 @@ typedef struct ReadEventData {
 typedef struct SCLPEvent {
     DeviceState qdev;
     bool event_pending;
-    uint32_t event_type;
     char *name;
 } SCLPEvent;
 
@@ -88,9 +167,8 @@ typedef struct SCLPEventClass {
 
     int (*write_event_data)(SCLPEvent *event, EventBufferHeader *evt_buf_hdr);
 
-    /* returns the supported event type */
-    int (*event_type)(void);
-
+    /* can we handle this event type? */
+    bool (*can_handle_event)(uint8_t type);
 } SCLPEventClass;
 
 #endif
diff --git a/include/hw/scsi/scsi.h b/include/hw/scsi/scsi.h
index 1b6651054a..76f6ac24a7 100644
--- a/include/hw/scsi/scsi.h
+++ b/include/hw/scsi/scsi.h
@@ -9,6 +9,8 @@
 #define MAX_SCSI_DEVS	255
 
 #define SCSI_CMD_BUF_SIZE     16
+#define SCSI_SENSE_LEN      18
+#define SCSI_INQUIRY_LEN    36
 
 typedef struct SCSIBus SCSIBus;
 typedef struct SCSIBusInfo SCSIBusInfo;
diff --git a/include/hw/virtio/virtio-blk.h b/include/hw/virtio/virtio-blk.h
index b87cf490b1..41885da1a0 100644
--- a/include/hw/virtio/virtio-blk.h
+++ b/include/hw/virtio/virtio-blk.h
@@ -123,6 +123,7 @@ typedef struct VirtIOBlock {
     BlockConf *conf;
     VirtIOBlkConf blk;
     unsigned short sector_mask;
+    bool original_wce;
     VMChangeStateEntry *change;
 #ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
     Notifier migration_state_notifier;
diff --git a/include/migration/vmstate.h b/include/migration/vmstate.h
index 1c31b5d6fb..9d09e60419 100644
--- a/include/migration/vmstate.h
+++ b/include/migration/vmstate.h
@@ -310,8 +310,18 @@ extern const VMStateInfo vmstate_info_bitmap;
     .offset       = vmstate_offset_value(_state, _field, _type),     \
 }
 
-#define VMSTATE_STRUCT_POINTER_TEST(_field, _state, _test, _vmsd, _type) { \
+#define VMSTATE_STRUCT_POINTER_V(_field, _state, _version, _vmsd, _type) { \
     .name         = (stringify(_field)),                             \
+    .version_id   = (_version),                                        \
+    .vmsd         = &(_vmsd),                                        \
+    .size         = sizeof(_type),                                   \
+    .flags        = VMS_STRUCT|VMS_POINTER,                          \
+    .offset       = vmstate_offset_value(_state, _field, _type),     \
+}
+
+#define VMSTATE_STRUCT_POINTER_TEST_V(_field, _state, _test, _version, _vmsd, _type) { \
+    .name         = (stringify(_field)),                             \
+    .version_id   = (_version),                                        \
     .field_exists = (_test),                                         \
     .vmsd         = &(_vmsd),                                        \
     .size         = sizeof(_type),                                   \
@@ -497,7 +507,10 @@ extern const VMStateInfo vmstate_info_bitmap;
     VMSTATE_STRUCT_TEST(_field, _state, NULL, _version, _vmsd, _type)
 
 #define VMSTATE_STRUCT_POINTER(_field, _state, _vmsd, _type)          \
-    VMSTATE_STRUCT_POINTER_TEST(_field, _state, NULL, _vmsd, _type)
+    VMSTATE_STRUCT_POINTER_V(_field, _state, 0, _vmsd, _type)
+
+#define VMSTATE_STRUCT_POINTER_TEST(_field, _state, _test, _vmsd, _type)     \
+    VMSTATE_STRUCT_POINTER_TEST_V(_field, _state, _test, 0, _vmsd, _type)
 
 #define VMSTATE_STRUCT_ARRAY(_field, _state, _num, _version, _vmsd, _type) \
     VMSTATE_STRUCT_ARRAY_TEST(_field, _state, _num, NULL, _version,   \
diff --git a/include/qapi/qmp/dispatch.h b/include/qapi/qmp/dispatch.h
index 1ce11f5df0..cea38181bf 100644
--- a/include/qapi/qmp/dispatch.h
+++ b/include/qapi/qmp/dispatch.h
@@ -47,9 +47,12 @@ QmpCommand *qmp_find_command(const char *name);
 QObject *qmp_dispatch(QObject *request);
 void qmp_disable_command(const char *name);
 void qmp_enable_command(const char *name);
-bool qmp_command_is_enabled(const char *name);
-char **qmp_get_command_list(void);
+bool qmp_command_is_enabled(const QmpCommand *cmd);
+const char *qmp_command_name(const QmpCommand *cmd);
+bool qmp_has_success_response(const QmpCommand *cmd);
 QObject *qmp_build_error_object(Error *errp);
+typedef void (*qmp_cmd_callback_fn)(QmpCommand *cmd, void *opaque);
+void qmp_for_each_command(qmp_cmd_callback_fn fn, void *opaque);
 
 #endif
 
diff --git a/include/qapi/qmp/qdict.h b/include/qapi/qmp/qdict.h
index d6855d112e..5cefd8022a 100644
--- a/include/qapi/qmp/qdict.h
+++ b/include/qapi/qmp/qdict.h
@@ -67,4 +67,6 @@ const char *qdict_get_try_str(const QDict *qdict, const char *key);
 QDict *qdict_clone_shallow(const QDict *src);
 void qdict_flatten(QDict *qdict);
 
+void qdict_extract_subqdict(QDict *src, QDict **dst, const char *start);
+
 #endif /* QDICT_H */
diff --git a/include/qemu-common.h b/include/qemu-common.h
index 6948bb9177..50548361d0 100644
--- a/include/qemu-common.h
+++ b/include/qemu-common.h
@@ -191,6 +191,9 @@ int64_t strtosz_suffix(const char *nptr, char **end, const char default_suffix);
 int64_t strtosz_suffix_unit(const char *nptr, char **end,
                             const char default_suffix, int64_t unit);
 
+/* used to print char* safely */
+#define STR_OR_NULL(str) ((str) ? (str) : "null")
+
 /* path.c */
 void init_paths(const char *prefix);
 const char *path(const char *pathname);
diff --git a/include/qemu/bitops.h b/include/qemu/bitops.h
index 06e2e6f0ee..304c90c2b4 100644
--- a/include/qemu/bitops.h
+++ b/include/qemu/bitops.h
@@ -184,6 +184,86 @@ static inline unsigned long hweight_long(unsigned long w)
 }
 
 /**
+ * rol8 - rotate an 8-bit value left
+ * @word: value to rotate
+ * @shift: bits to roll
+ */
+static inline uint8_t rol8(uint8_t word, unsigned int shift)
+{
+    return (word << shift) | (word >> (8 - shift));
+}
+
+/**
+ * ror8 - rotate an 8-bit value right
+ * @word: value to rotate
+ * @shift: bits to roll
+ */
+static inline uint8_t ror8(uint8_t word, unsigned int shift)
+{
+    return (word >> shift) | (word << (8 - shift));
+}
+
+/**
+ * rol16 - rotate a 16-bit value left
+ * @word: value to rotate
+ * @shift: bits to roll
+ */
+static inline uint16_t rol16(uint16_t word, unsigned int shift)
+{
+    return (word << shift) | (word >> (16 - shift));
+}
+
+/**
+ * ror16 - rotate a 16-bit value right
+ * @word: value to rotate
+ * @shift: bits to roll
+ */
+static inline uint16_t ror16(uint16_t word, unsigned int shift)
+{
+    return (word >> shift) | (word << (16 - shift));
+}
+
+/**
+ * rol32 - rotate a 32-bit value left
+ * @word: value to rotate
+ * @shift: bits to roll
+ */
+static inline uint32_t rol32(uint32_t word, unsigned int shift)
+{
+    return (word << shift) | (word >> (32 - shift));
+}
+
+/**
+ * ror32 - rotate a 32-bit value right
+ * @word: value to rotate
+ * @shift: bits to roll
+ */
+static inline uint32_t ror32(uint32_t word, unsigned int shift)
+{
+    return (word >> shift) | (word << (32 - shift));
+}
+
+/**
+ * rol64 - rotate a 64-bit value left
+ * @word: value to rotate
+ * @shift: bits to roll
+ */
+static inline uint64_t rol64(uint64_t word, unsigned int shift)
+{
+    return (word << shift) | (word >> (64 - shift));
+}
+
+/**
+ * ror64 - rotate a 64-bit value right
+ * @word: value to rotate
+ * @shift: bits to roll
+ */
+static inline uint64_t ror64(uint64_t word, unsigned int shift)
+{
+    return (word >> shift) | (word << (64 - shift));
+}
+
+/**
  * extract32:
  * @value: the value to extract the bit field from
  * @start: the lowest bit in the bit field (numbered from 0)
diff --git a/include/qemu/option.h b/include/qemu/option.h
index 63db4ccb9a..5c0c6dd294 100644
--- a/include/qemu/option.h
+++ b/include/qemu/option.h
@@ -142,6 +142,7 @@ void qemu_opts_loc_restore(QemuOpts *opts);
 int qemu_opts_set(QemuOptsList *list, const char *id,
                   const char *name, const char *value);
 const char *qemu_opts_id(QemuOpts *opts);
+void qemu_opts_set_id(QemuOpts *opts, char *id);
 void qemu_opts_del(QemuOpts *opts);
 void qemu_opts_validate(QemuOpts *opts, const QemuOptDesc *desc, Error **errp);
 int qemu_opts_do_parse(QemuOpts *opts, const char *params, const char *firstname);
diff --git a/include/qemu/seqlock.h b/include/qemu/seqlock.h
new file mode 100644
index 0000000000..3ff118a1a1
--- /dev/null
+++ b/include/qemu/seqlock.h
@@ -0,0 +1,72 @@
+/*
+ * Seqlock implementation for QEMU
+ *
+ * Copyright Red Hat, Inc. 2013
+ *
+ * Author:
+ *  Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+#ifndef QEMU_SEQLOCK_H
+#define QEMU_SEQLOCK_H 1
+
+#include <qemu/atomic.h>
+#include <qemu/thread.h>
+
+typedef struct QemuSeqLock QemuSeqLock;
+
+struct QemuSeqLock {
+    QemuMutex *mutex;
+    unsigned sequence;
+};
+
+static inline void seqlock_init(QemuSeqLock *sl, QemuMutex *mutex)
+{
+    sl->mutex = mutex;
+    sl->sequence = 0;
+}
+
+/* Lock out other writers and update the count.  */
+static inline void seqlock_write_lock(QemuSeqLock *sl)
+{
+    if (sl->mutex) {
+        qemu_mutex_lock(sl->mutex);
+    }
+    ++sl->sequence;
+
+    /* Write sequence before updating other fields.  */
+    smp_wmb();
+}
+
+static inline void seqlock_write_unlock(QemuSeqLock *sl)
+{
+    /* Write other fields before finalizing sequence.  */
+    smp_wmb();
+
+    ++sl->sequence;
+    if (sl->mutex) {
+        qemu_mutex_unlock(sl->mutex);
+    }
+}
+
+static inline unsigned seqlock_read_begin(QemuSeqLock *sl)
+{
+    /* Always fail if a write is in progress.  */
+    unsigned ret = sl->sequence & ~1;
+
+    /* Read sequence before reading other fields.  */
+    smp_rmb();
+    return ret;
+}
+
+static int seqlock_read_retry(const QemuSeqLock *sl, unsigned start)
+{
+    /* Read other fields before reading final sequence.  */
+    smp_rmb();
+    return unlikely(sl->sequence != start);
+}
+
+#endif
diff --git a/include/qemu/sockets.h b/include/qemu/sockets.h
index c5174d76a7..45588d7d58 100644
--- a/include/qemu/sockets.h
+++ b/include/qemu/sockets.h
@@ -39,6 +39,7 @@ int socket_set_cork(int fd, int v);
 int socket_set_nodelay(int fd);
 void qemu_set_block(int fd);
 void qemu_set_nonblock(int fd);
+int socket_set_fast_reuse(int fd);
 int send_all(int fd, const void *buf, int len1);
 int recv_all(int fd, void *buf, int len1, bool single_read);
 
diff --git a/include/qemu/thread-posix.h b/include/qemu/thread-posix.h
index 361566abc4..eb5c7a1da1 100644
--- a/include/qemu/thread-posix.h
+++ b/include/qemu/thread-posix.h
@@ -21,6 +21,14 @@ struct QemuSemaphore {
 #endif
 };
 
+struct QemuEvent {
+#ifndef __linux__
+    pthread_mutex_t lock;
+    pthread_cond_t cond;
+#endif
+    unsigned value;
+};
+
 struct QemuThread {
     pthread_t thread;
 };
diff --git a/include/qemu/thread-win32.h b/include/qemu/thread-win32.h
index 13adb958f0..3d58081bed 100644
--- a/include/qemu/thread-win32.h
+++ b/include/qemu/thread-win32.h
@@ -17,6 +17,10 @@ struct QemuSemaphore {
     HANDLE sema;
 };
 
+struct QemuEvent {
+    HANDLE event;
+};
+
 typedef struct QemuThreadData QemuThreadData;
 struct QemuThread {
     QemuThreadData *data;
diff --git a/include/qemu/thread.h b/include/qemu/thread.h
index c02404b9fb..3e32c6531c 100644
--- a/include/qemu/thread.h
+++ b/include/qemu/thread.h
@@ -7,6 +7,7 @@
 typedef struct QemuMutex QemuMutex;
 typedef struct QemuCond QemuCond;
 typedef struct QemuSemaphore QemuSemaphore;
+typedef struct QemuEvent QemuEvent;
 typedef struct QemuThread QemuThread;
 
 #ifdef _WIN32
@@ -45,6 +46,12 @@ void qemu_sem_wait(QemuSemaphore *sem);
 int qemu_sem_timedwait(QemuSemaphore *sem, int ms);
 void qemu_sem_destroy(QemuSemaphore *sem);
 
+void qemu_event_init(QemuEvent *ev, bool init);
+void qemu_event_set(QemuEvent *ev);
+void qemu_event_reset(QemuEvent *ev);
+void qemu_event_wait(QemuEvent *ev);
+void qemu_event_destroy(QemuEvent *ev);
+
 void qemu_thread_create(QemuThread *thread,
                         void *(*start_routine)(void *),
                         void *arg, int mode);
diff --git a/include/qemu/timer.h b/include/qemu/timer.h
index e4934dd61b..5afcffc3f9 100644
--- a/include/qemu/timer.h
+++ b/include/qemu/timer.h
@@ -115,6 +115,10 @@ static inline int64_t qemu_clock_get_us(QEMUClockType type)
  * Determines whether a clock's default timer list
  * has timers attached
  *
+ * Note that this function should not be used when other threads also access
+ * the timer list.  The return value may be outdated by the time it is acted
+ * upon.
+ *
  * Returns: true if the clock's default timer list
  * has timers attached
  */
@@ -185,6 +189,12 @@ void qemu_clock_notify(QEMUClockType type);
  * @enabled: true to enable, false to disable
  *
  * Enable or disable a clock
+ * Disabling the clock will wait for related timerlists to stop
+ * executing qemu_run_timers.  Thus, this functions should not
+ * be used from the callback of a timer that is based on @clock.
+ * Doing so would cause a deadlock.
+ *
+ * Caller should hold BQL.
  */
 void qemu_clock_enable(QEMUClockType type, bool enabled);
 
@@ -271,6 +281,10 @@ void timerlist_free(QEMUTimerList *timer_list);
  *
  * Determine whether a timer list has active timers
  *
+ * Note that this function should not be used when other threads also access
+ * the timer list.  The return value may be outdated by the time it is acted
+ * upon.
+ *
  * Returns: true if the timer list has timers.
  */
 bool timerlist_has_timers(QEMUTimerList *timer_list);
@@ -512,6 +526,9 @@ void timer_free(QEMUTimer *ts);
  * @ts: the timer
  *
  * Delete a timer from the active list.
+ *
+ * This function is thread-safe but the timer and its timer list must not be
+ * freed while this function is running.
  */
 void timer_del(QEMUTimer *ts);
 
@@ -521,20 +538,52 @@ void timer_del(QEMUTimer *ts);
  * @expire_time: the expiry time in nanoseconds
  *
  * Modify a timer to expire at @expire_time
+ *
+ * This function is thread-safe but the timer and its timer list must not be
+ * freed while this function is running.
  */
 void timer_mod_ns(QEMUTimer *ts, int64_t expire_time);
 
 /**
+ * timer_mod_anticipate_ns:
+ * @ts: the timer
+ * @expire_time: the expiry time in nanoseconds
+ *
+ * Modify a timer to expire at @expire_time or the current time,
+ * whichever comes earlier.
+ *
+ * This function is thread-safe but the timer and its timer list must not be
+ * freed while this function is running.
+ */
+void timer_mod_anticipate_ns(QEMUTimer *ts, int64_t expire_time);
+
+/**
  * timer_mod:
  * @ts: the timer
  * @expire_time: the expire time in the units associated with the timer
  *
  * Modify a timer to expiry at @expire_time, taking into
  * account the scale associated with the timer.
+ *
+ * This function is thread-safe but the timer and its timer list must not be
+ * freed while this function is running.
  */
 void timer_mod(QEMUTimer *ts, int64_t expire_timer);
 
 /**
+ * timer_mod_anticipate:
+ * @ts: the timer
+ * @expire_time: the expiry time in nanoseconds
+ *
+ * Modify a timer to expire at @expire_time or the current time, whichever
+ * comes earlier, taking into account the scale associated with the timer.
+ *
+ * This function is thread-safe but the timer and its timer list must not be
+ * freed while this function is running.
+ */
+void timer_mod_anticipate(QEMUTimer *ts, int64_t expire_time);
+
+/**
  * timer_pending:
  * @ts: the timer
  *
@@ -636,7 +685,9 @@ static inline int64_t qemu_soonest_timeout(int64_t timeout1, int64_t timeout2)
 void init_clocks(void);
 
 int64_t cpu_get_ticks(void);
+/* Caller must hold BQL */
 void cpu_enable_ticks(void);
+/* Caller must hold BQL */
 void cpu_disable_ticks(void);
 
 static inline int64_t get_ticks_per_sec(void)
diff --git a/include/sysemu/blockdev.h b/include/sysemu/blockdev.h
index 804ec8839b..10820910d7 100644
--- a/include/sysemu/blockdev.h
+++ b/include/sysemu/blockdev.h
@@ -37,6 +37,7 @@ struct DriveInfo {
     int bus;
     int unit;
     int auto_del;               /* see blockdev_mark_auto_del() */
+    bool enable_auto_del; /* Only for legacy drive_init() */
     int media_cd;
     int cyls, heads, secs, trans;
     QemuOpts *opts;
diff --git a/include/sysemu/char.h b/include/sysemu/char.h
index 8053130a97..ad101d9258 100644
--- a/include/sysemu/char.h
+++ b/include/sysemu/char.h
@@ -78,6 +78,7 @@ struct CharDriverState {
     int explicit_be_open;
     int avail_connections;
     int is_mux;
+    guint fd_in_tag;
     QemuOpts *opts;
     QTAILQ_ENTRY(CharDriverState) next;
 };
diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h
index 8e7668524b..3b25f27a7c 100644
--- a/include/sysemu/kvm.h
+++ b/include/sysemu/kvm.h
@@ -46,6 +46,7 @@ extern bool kvm_halt_in_kernel_allowed;
 extern bool kvm_irqfds_allowed;
 extern bool kvm_msi_via_irqfd_allowed;
 extern bool kvm_gsi_routing_allowed;
+extern bool kvm_gsi_direct_mapping;
 extern bool kvm_readonly_mem_allowed;
 
 #if defined CONFIG_KVM || !defined NEED_CPU_H
@@ -108,6 +109,13 @@ extern bool kvm_readonly_mem_allowed;
 #define kvm_gsi_routing_enabled() (kvm_gsi_routing_allowed)
 
 /**
+ * kvm_gsi_direct_mapping:
+ *
+ * Returns: true if GSI direct mapping is enabled.
+ */
+#define kvm_gsi_direct_mapping() (kvm_gsi_direct_mapping)
+
+/**
  * kvm_readonly_mem_enabled:
  *
  * Returns: true if KVM readonly memory is enabled (ie the kernel
@@ -123,6 +131,7 @@ extern bool kvm_readonly_mem_allowed;
 #define kvm_irqfds_enabled() (false)
 #define kvm_msi_via_irqfd_enabled() (false)
 #define kvm_gsi_routing_allowed() (false)
+#define kvm_gsi_direct_mapping() (false)
 #define kvm_readonly_mem_enabled() (false)
 #endif
 
@@ -161,11 +170,6 @@ int kvm_cpu_exec(CPUState *cpu);
 
 #ifdef NEED_CPU_H
 
-#if !defined(CONFIG_USER_ONLY)
-void *kvm_ram_alloc(ram_addr_t size);
-void *kvm_arch_ram_alloc(ram_addr_t size);
-#endif
-
 void kvm_setup_guest_memory(void *start, size_t size);
 void kvm_flush_coalesced_mmio_buffer(void);
 
@@ -270,16 +274,6 @@ int kvm_check_extension(KVMState *s, unsigned int extension);
 
 uint32_t kvm_arch_get_supported_cpuid(KVMState *env, uint32_t function,
                                       uint32_t index, int reg);
-void kvm_cpu_synchronize_state(CPUState *cpu);
-
-/* generic hooks - to be moved/refactored once there are more users */
-
-static inline void cpu_synchronize_state(CPUState *cpu)
-{
-    if (kvm_enabled()) {
-        kvm_cpu_synchronize_state(cpu);
-    }
-}
 
 #if !defined(CONFIG_USER_ONLY)
 int kvm_physical_memory_addr_from_host(KVMState *s, void *ram_addr,
@@ -288,9 +282,19 @@ int kvm_physical_memory_addr_from_host(KVMState *s, void *ram_addr,
 
 #endif /* NEED_CPU_H */
 
+void kvm_cpu_synchronize_state(CPUState *cpu);
 void kvm_cpu_synchronize_post_reset(CPUState *cpu);
 void kvm_cpu_synchronize_post_init(CPUState *cpu);
 
+/* generic hooks - to be moved/refactored once there are more users */
+
+static inline void cpu_synchronize_state(CPUState *cpu)
+{
+    if (kvm_enabled()) {
+        kvm_cpu_synchronize_state(cpu);
+    }
+}
+
 static inline void cpu_synchronize_post_reset(CPUState *cpu)
 {
     if (kvm_enabled()) {
diff --git a/include/sysemu/sysemu.h b/include/sysemu/sysemu.h
index 42577363ca..cd5791eb74 100644
--- a/include/sysemu/sysemu.h
+++ b/include/sysemu/sysemu.h
@@ -18,7 +18,9 @@ extern const char *qemu_name;
 extern uint8_t qemu_uuid[];
 extern bool qemu_uuid_set;
 int qemu_uuid_parse(const char *str, uint8_t *uuid);
+
 #define UUID_FMT "%02hhx%02hhx%02hhx%02hhx-%02hhx%02hhx-%02hhx%02hhx-%02hhx%02hhx-%02hhx%02hhx%02hhx%02hhx%02hhx%02hhx"
+#define UUID_NONE "00000000-0000-0000-0000-000000000000"
 
 bool runstate_check(RunState state);
 void runstate_set(RunState new_state);
@@ -40,9 +42,11 @@ int vm_stop(RunState state);
 int vm_stop_force_state(RunState state);
 
 typedef enum WakeupReason {
-    QEMU_WAKEUP_REASON_OTHER = 0,
+    /* Always keep QEMU_WAKEUP_REASON_NONE = 0 */
+    QEMU_WAKEUP_REASON_NONE = 0,
     QEMU_WAKEUP_REASON_RTC,
     QEMU_WAKEUP_REASON_PMTIMER,
+    QEMU_WAKEUP_REASON_OTHER,
 } WakeupReason;
 
 void qemu_system_reset_request(void);
diff --git a/include/ui/qemu-spice.h b/include/ui/qemu-spice.h
index c6c756b23d..86c75c7a71 100644
--- a/include/ui/qemu-spice.h
+++ b/include/ui/qemu-spice.h
@@ -27,14 +27,15 @@
 #include "monitor/monitor.h"
 
 extern int using_spice;
-extern int spice_displays;
 
 void qemu_spice_init(void);
 void qemu_spice_input_init(void);
 void qemu_spice_audio_init(void);
-void qemu_spice_display_init(DisplayState *ds);
+void qemu_spice_display_init(void);
 int qemu_spice_display_add_client(int csock, int skipauth, int tls);
 int qemu_spice_add_interface(SpiceBaseInstance *sin);
+bool qemu_spice_have_display_interface(QemuConsole *con);
+int qemu_spice_add_display_interface(QXLInstance *qxlin, QemuConsole *con);
 int qemu_spice_set_passwd(const char *passwd,
                           bool fail_if_connected, bool disconnect_if_connected);
 int qemu_spice_set_pw_expire(time_t expires);
diff --git a/ioport.c b/ioport.c
index 707cce88ab..3d91e79edc 100644
--- a/ioport.c
+++ b/ioport.c
@@ -139,6 +139,12 @@ void portio_list_init(PortioList *piolist,
     piolist->opaque = opaque;
     piolist->owner = owner;
     piolist->name = name;
+    piolist->flush_coalesced_mmio = false;
+}
+
+void portio_list_set_flush_coalesced(PortioList *piolist)
+{
+    piolist->flush_coalesced_mmio = true;
 }
 
 void portio_list_destroy(PortioList *piolist)
@@ -231,6 +237,9 @@ static void portio_list_add_1(PortioList *piolist,
      */
     memory_region_init_io(&mrpio->mr, piolist->owner, &portio_ops, mrpio,
                           piolist->name, off_high - off_low);
+    if (piolist->flush_coalesced_mmio) {
+        memory_region_set_flush_coalesced(&mrpio->mr);
+    }
     memory_region_add_subregion(piolist->address_space,
                                 start + off_low, &mrpio->mr);
     piolist->regions[piolist->nr] = &mrpio->mr;
diff --git a/kvm-all.c b/kvm-all.c
index c29a015cca..4478969ed2 100644
--- a/kvm-all.c
+++ b/kvm-all.c
@@ -111,6 +111,7 @@ bool kvm_halt_in_kernel_allowed;
 bool kvm_irqfds_allowed;
 bool kvm_msi_via_irqfd_allowed;
 bool kvm_gsi_routing_allowed;
+bool kvm_gsi_direct_mapping;
 bool kvm_allowed;
 bool kvm_readonly_mem_allowed;
 
@@ -1069,6 +1070,10 @@ void kvm_irqchip_release_virq(KVMState *s, int virq)
     struct kvm_irq_routing_entry *e;
     int i;
 
+    if (kvm_gsi_direct_mapping()) {
+        return;
+    }
+
     for (i = 0; i < s->irq_routes->nr; i++) {
         e = &s->irq_routes->entries[i];
         if (e->gsi == virq) {
@@ -1190,6 +1195,10 @@ int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
     struct kvm_irq_routing_entry kroute = {};
     int virq;
 
+    if (kvm_gsi_direct_mapping()) {
+        return msg.data & 0xffff;
+    }
+
     if (!kvm_gsi_routing_enabled()) {
         return -ENOSYS;
     }
@@ -1216,6 +1225,10 @@ int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
 {
     struct kvm_irq_routing_entry kroute = {};
 
+    if (kvm_gsi_direct_mapping()) {
+        return 0;
+    }
+
     if (!kvm_irqchip_in_kernel()) {
         return -ENOSYS;
     }
@@ -1322,24 +1335,20 @@ static int kvm_irqchip_create(KVMState *s)
     return 0;
 }
 
-static int kvm_max_vcpus(KVMState *s)
+/* Find number of supported CPUs using the recommended
+ * procedure from the kernel API documentation to cope with
+ * older kernels that may be missing capabilities.
+ */
+static int kvm_recommended_vcpus(KVMState *s)
 {
-    int ret;
-
-    /* Find number of supported CPUs using the recommended
-     * procedure from the kernel API documentation to cope with
-     * older kernels that may be missing capabilities.
-     */
-    ret = kvm_check_extension(s, KVM_CAP_MAX_VCPUS);
-    if (ret) {
-        return ret;
-    }
-    ret = kvm_check_extension(s, KVM_CAP_NR_VCPUS);
-    if (ret) {
-        return ret;
-    }
+    int ret = kvm_check_extension(s, KVM_CAP_NR_VCPUS);
+    return (ret) ? ret : 4;
+}
 
-    return 4;
+static int kvm_max_vcpus(KVMState *s)
+{
+    int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPUS);
+    return (ret) ? ret : kvm_recommended_vcpus(s);
 }
 
 int kvm_init(void)
@@ -1347,11 +1356,19 @@ int kvm_init(void)
     static const char upgrade_note[] =
         "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
         "(see http://sourceforge.net/projects/kvm).\n";
+    struct {
+        const char *name;
+        int num;
+    } num_cpus[] = {
+        { "SMP",          smp_cpus },
+        { "hotpluggable", max_cpus },
+        { NULL, }
+    }, *nc = num_cpus;
+    int soft_vcpus_limit, hard_vcpus_limit;
     KVMState *s;
     const KVMCapabilityInfo *missing_cap;
     int ret;
     int i;
-    int max_vcpus;
 
     s = g_malloc0(sizeof(KVMState));
 
@@ -1392,19 +1409,26 @@ int kvm_init(void)
         goto err;
     }
 
-    max_vcpus = kvm_max_vcpus(s);
-    if (smp_cpus > max_vcpus) {
-        ret = -EINVAL;
-        fprintf(stderr, "Number of SMP cpus requested (%d) exceeds max cpus "
-                "supported by KVM (%d)\n", smp_cpus, max_vcpus);
-        goto err;
-    }
+    /* check the vcpu limits */
+    soft_vcpus_limit = kvm_recommended_vcpus(s);
+    hard_vcpus_limit = kvm_max_vcpus(s);
 
-    if (max_cpus > max_vcpus) {
-        ret = -EINVAL;
-        fprintf(stderr, "Number of hotpluggable cpus requested (%d) exceeds max cpus "
-                "supported by KVM (%d)\n", max_cpus, max_vcpus);
-        goto err;
+    while (nc->name) {
+        if (nc->num > soft_vcpus_limit) {
+            fprintf(stderr,
+                    "Warning: Number of %s cpus requested (%d) exceeds "
+                    "the recommended cpus supported by KVM (%d)\n",
+                    nc->name, nc->num, soft_vcpus_limit);
+
+            if (nc->num > hard_vcpus_limit) {
+                ret = -EINVAL;
+                fprintf(stderr, "Number of %s cpus requested (%d) exceeds "
+                        "the maximum cpus supported by KVM (%d)\n",
+                        nc->name, nc->num, hard_vcpus_limit);
+                goto err;
+            }
+        }
+        nc++;
     }
 
     s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
@@ -1812,19 +1836,6 @@ int kvm_has_intx_set_mask(void)
     return kvm_state->intx_set_mask;
 }
 
-void *kvm_ram_alloc(ram_addr_t size)
-{
-#ifdef TARGET_S390X
-    void *mem;
-
-    mem = kvm_arch_ram_alloc(size);
-    if (mem) {
-        return mem;
-    }
-#endif
-    return qemu_anon_ram_alloc(size);
-}
-
 void kvm_setup_guest_memory(void *start, size_t size)
 {
 #ifdef CONFIG_VALGRIND_H
diff --git a/kvm-stub.c b/kvm-stub.c
index 548f471c17..e979f76d07 100644
--- a/kvm-stub.c
+++ b/kvm-stub.c
@@ -25,6 +25,7 @@ bool kvm_async_interrupts_allowed;
 bool kvm_irqfds_allowed;
 bool kvm_msi_via_irqfd_allowed;
 bool kvm_gsi_routing_allowed;
+bool kvm_gsi_direct_mapping;
 bool kvm_allowed;
 bool kvm_readonly_mem_allowed;
 
diff --git a/libcacard/Makefile b/libcacard/Makefile
index 47827a0eb8..4d15da49b8 100644
--- a/libcacard/Makefile
+++ b/libcacard/Makefile
@@ -4,7 +4,8 @@ TOOLS += vscclient$(EXESUF)
 
 # objects linked into a shared library, built with libtool with -fPIC if required
 libcacard-obj-y = $(stub-obj-y) $(libcacard-y)
-libcacard-obj-y += util/osdep.o util/cutils.o util/qemu-timer-common.o util/error.o
+libcacard-obj-y += util/osdep.o util/cutils.o util/qemu-timer-common.o
+libcacard-obj-y += util/error.o util/qemu-error.o
 libcacard-obj-$(CONFIG_WIN32) += util/oslib-win32.o util/qemu-thread-win32.o
 libcacard-obj-$(CONFIG_POSIX) += util/oslib-posix.o util/qemu-thread-posix.o
 libcacard-obj-y += $(filter trace/%, $(util-obj-y))
diff --git a/libcacard/vscclient.c b/libcacard/vscclient.c
index 5180d2945a..a3cb7762b5 100644
--- a/libcacard/vscclient.c
+++ b/libcacard/vscclient.c
@@ -645,7 +645,6 @@ main(
     GIOChannel *channel_stdin;
     char *qemu_host;
     char *qemu_port;
-    VSCMsgHeader mhHeader;
 
     VCardEmulOptions *command_line_options = NULL;
 
@@ -754,7 +753,7 @@ main(
         .magic = VSCARD_MAGIC,
         .capabilities = {0}
     };
-    send_msg(VSC_Init, mhHeader.reader_id, &init, sizeof(init));
+    send_msg(VSC_Init, 0, &init, sizeof(init));
 
     g_main_loop_run(loop);
     g_main_loop_unref(loop);
diff --git a/linux-headers/asm-arm64/kvm.h b/linux-headers/asm-arm64/kvm.h
new file mode 100644
index 0000000000..5031f42639
--- /dev/null
+++ b/linux-headers/asm-arm64/kvm.h
@@ -0,0 +1,168 @@
+/*
+ * Copyright (C) 2012,2013 - ARM Ltd
+ * Author: Marc Zyngier <marc.zyngier@arm.com>
+ *
+ * Derived from arch/arm/include/uapi/asm/kvm.h:
+ * Copyright (C) 2012 - Virtual Open Systems and Columbia University
+ * Author: Christoffer Dall <c.dall@virtualopensystems.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __ARM_KVM_H__
+#define __ARM_KVM_H__
+
+#define KVM_SPSR_EL1	0
+#define KVM_SPSR_SVC	KVM_SPSR_EL1
+#define KVM_SPSR_ABT	1
+#define KVM_SPSR_UND	2
+#define KVM_SPSR_IRQ	3
+#define KVM_SPSR_FIQ	4
+#define KVM_NR_SPSR	5
+
+#ifndef __ASSEMBLY__
+#include <asm/types.h>
+#include <asm/ptrace.h>
+
+#define __KVM_HAVE_GUEST_DEBUG
+#define __KVM_HAVE_IRQ_LINE
+
+#define KVM_REG_SIZE(id)						\
+	(1U << (((id) & KVM_REG_SIZE_MASK) >> KVM_REG_SIZE_SHIFT))
+
+struct kvm_regs {
+	struct user_pt_regs regs;	/* sp = sp_el0 */
+
+	__u64	sp_el1;
+	__u64	elr_el1;
+
+	__u64	spsr[KVM_NR_SPSR];
+
+	struct user_fpsimd_state fp_regs;
+};
+
+/* Supported Processor Types */
+#define KVM_ARM_TARGET_AEM_V8		0
+#define KVM_ARM_TARGET_FOUNDATION_V8	1
+#define KVM_ARM_TARGET_CORTEX_A57	2
+
+#define KVM_ARM_NUM_TARGETS		3
+
+/* KVM_ARM_SET_DEVICE_ADDR ioctl id encoding */
+#define KVM_ARM_DEVICE_TYPE_SHIFT	0
+#define KVM_ARM_DEVICE_TYPE_MASK	(0xffff << KVM_ARM_DEVICE_TYPE_SHIFT)
+#define KVM_ARM_DEVICE_ID_SHIFT		16
+#define KVM_ARM_DEVICE_ID_MASK		(0xffff << KVM_ARM_DEVICE_ID_SHIFT)
+
+/* Supported device IDs */
+#define KVM_ARM_DEVICE_VGIC_V2		0
+
+/* Supported VGIC address types  */
+#define KVM_VGIC_V2_ADDR_TYPE_DIST	0
+#define KVM_VGIC_V2_ADDR_TYPE_CPU	1
+
+#define KVM_VGIC_V2_DIST_SIZE		0x1000
+#define KVM_VGIC_V2_CPU_SIZE		0x2000
+
+#define KVM_ARM_VCPU_POWER_OFF		0 /* CPU is started in OFF state */
+#define KVM_ARM_VCPU_EL1_32BIT		1 /* CPU running a 32bit VM */
+
+struct kvm_vcpu_init {
+	__u32 target;
+	__u32 features[7];
+};
+
+struct kvm_sregs {
+};
+
+struct kvm_fpu {
+};
+
+struct kvm_guest_debug_arch {
+};
+
+struct kvm_debug_exit_arch {
+};
+
+struct kvm_sync_regs {
+};
+
+struct kvm_arch_memory_slot {
+};
+
+/* If you need to interpret the index values, here is the key: */
+#define KVM_REG_ARM_COPROC_MASK		0x000000000FFF0000
+#define KVM_REG_ARM_COPROC_SHIFT	16
+
+/* Normal registers are mapped as coprocessor 16. */
+#define KVM_REG_ARM_CORE		(0x0010 << KVM_REG_ARM_COPROC_SHIFT)
+#define KVM_REG_ARM_CORE_REG(name)	(offsetof(struct kvm_regs, name) / sizeof(__u32))
+
+/* Some registers need more space to represent values. */
+#define KVM_REG_ARM_DEMUX		(0x0011 << KVM_REG_ARM_COPROC_SHIFT)
+#define KVM_REG_ARM_DEMUX_ID_MASK	0x000000000000FF00
+#define KVM_REG_ARM_DEMUX_ID_SHIFT	8
+#define KVM_REG_ARM_DEMUX_ID_CCSIDR	(0x00 << KVM_REG_ARM_DEMUX_ID_SHIFT)
+#define KVM_REG_ARM_DEMUX_VAL_MASK	0x00000000000000FF
+#define KVM_REG_ARM_DEMUX_VAL_SHIFT	0
+
+/* AArch64 system registers */
+#define KVM_REG_ARM64_SYSREG		(0x0013 << KVM_REG_ARM_COPROC_SHIFT)
+#define KVM_REG_ARM64_SYSREG_OP0_MASK	0x000000000000c000
+#define KVM_REG_ARM64_SYSREG_OP0_SHIFT	14
+#define KVM_REG_ARM64_SYSREG_OP1_MASK	0x0000000000003800
+#define KVM_REG_ARM64_SYSREG_OP1_SHIFT	11
+#define KVM_REG_ARM64_SYSREG_CRN_MASK	0x0000000000000780
+#define KVM_REG_ARM64_SYSREG_CRN_SHIFT	7
+#define KVM_REG_ARM64_SYSREG_CRM_MASK	0x0000000000000078
+#define KVM_REG_ARM64_SYSREG_CRM_SHIFT	3
+#define KVM_REG_ARM64_SYSREG_OP2_MASK	0x0000000000000007
+#define KVM_REG_ARM64_SYSREG_OP2_SHIFT	0
+
+/* KVM_IRQ_LINE irq field index values */
+#define KVM_ARM_IRQ_TYPE_SHIFT		24
+#define KVM_ARM_IRQ_TYPE_MASK		0xff
+#define KVM_ARM_IRQ_VCPU_SHIFT		16
+#define KVM_ARM_IRQ_VCPU_MASK		0xff
+#define KVM_ARM_IRQ_NUM_SHIFT		0
+#define KVM_ARM_IRQ_NUM_MASK		0xffff
+
+/* irq_type field */
+#define KVM_ARM_IRQ_TYPE_CPU		0
+#define KVM_ARM_IRQ_TYPE_SPI		1
+#define KVM_ARM_IRQ_TYPE_PPI		2
+
+/* out-of-kernel GIC cpu interrupt injection irq_number field */
+#define KVM_ARM_IRQ_CPU_IRQ		0
+#define KVM_ARM_IRQ_CPU_FIQ		1
+
+/* Highest supported SPI, from VGIC_NR_IRQS */
+#define KVM_ARM_IRQ_GIC_MAX		127
+
+/* PSCI interface */
+#define KVM_PSCI_FN_BASE		0x95c1ba5e
+#define KVM_PSCI_FN(n)			(KVM_PSCI_FN_BASE + (n))
+
+#define KVM_PSCI_FN_CPU_SUSPEND		KVM_PSCI_FN(0)
+#define KVM_PSCI_FN_CPU_OFF		KVM_PSCI_FN(1)
+#define KVM_PSCI_FN_CPU_ON		KVM_PSCI_FN(2)
+#define KVM_PSCI_FN_MIGRATE		KVM_PSCI_FN(3)
+
+#define KVM_PSCI_RET_SUCCESS		0
+#define KVM_PSCI_RET_NI			((unsigned long)-1)
+#define KVM_PSCI_RET_INVAL		((unsigned long)-2)
+#define KVM_PSCI_RET_DENIED		((unsigned long)-3)
+
+#endif
+
+#endif /* __ARM_KVM_H__ */
diff --git a/linux-headers/asm-arm64/kvm_para.h b/linux-headers/asm-arm64/kvm_para.h
new file mode 100644
index 0000000000..14fab8f0b9
--- /dev/null
+++ b/linux-headers/asm-arm64/kvm_para.h
@@ -0,0 +1 @@
+#include <asm-generic/kvm_para.h>
diff --git a/linux-headers/asm-mips/kvm.h b/linux-headers/asm-mips/kvm.h
index 3f424f5217..f09ff5ae20 100644
--- a/linux-headers/asm-mips/kvm.h
+++ b/linux-headers/asm-mips/kvm.h
@@ -58,56 +58,53 @@ struct kvm_fpu {
  *  bits[2..0]   - Register 'sel' index.
  *  bits[7..3]   - Register 'rd'  index.
  *  bits[15..8]  - Must be zero.
- *  bits[63..16] - 1 -> CP0 registers.
+ *  bits[31..16] - 1 -> CP0 registers.
+ *  bits[51..32] - Must be zero.
+ *  bits[63..52] - As per linux/kvm.h
  *
  * Other sets registers may be added in the future.  Each set would
- * have its own identifier in bits[63..16].
- *
- * The addr field of struct kvm_one_reg must point to an aligned
- * 64-bit wide location.  For registers that are narrower than
- * 64-bits, the value is stored in the low order bits of the location,
- * and sign extended to 64-bits.
+ * have its own identifier in bits[31..16].
  *
  * The registers defined in struct kvm_regs are also accessible, the
  * id values for these are below.
  */
 
-#define KVM_REG_MIPS_R0 0
-#define KVM_REG_MIPS_R1 1
-#define KVM_REG_MIPS_R2 2
-#define KVM_REG_MIPS_R3 3
-#define KVM_REG_MIPS_R4 4
-#define KVM_REG_MIPS_R5 5
-#define KVM_REG_MIPS_R6 6
-#define KVM_REG_MIPS_R7 7
-#define KVM_REG_MIPS_R8 8
-#define KVM_REG_MIPS_R9 9
-#define KVM_REG_MIPS_R10 10
-#define KVM_REG_MIPS_R11 11
-#define KVM_REG_MIPS_R12 12
-#define KVM_REG_MIPS_R13 13
-#define KVM_REG_MIPS_R14 14
-#define KVM_REG_MIPS_R15 15
-#define KVM_REG_MIPS_R16 16
-#define KVM_REG_MIPS_R17 17
-#define KVM_REG_MIPS_R18 18
-#define KVM_REG_MIPS_R19 19
-#define KVM_REG_MIPS_R20 20
-#define KVM_REG_MIPS_R21 21
-#define KVM_REG_MIPS_R22 22
-#define KVM_REG_MIPS_R23 23
-#define KVM_REG_MIPS_R24 24
-#define KVM_REG_MIPS_R25 25
-#define KVM_REG_MIPS_R26 26
-#define KVM_REG_MIPS_R27 27
-#define KVM_REG_MIPS_R28 28
-#define KVM_REG_MIPS_R29 29
-#define KVM_REG_MIPS_R30 30
-#define KVM_REG_MIPS_R31 31
+#define KVM_REG_MIPS_R0 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 0)
+#define KVM_REG_MIPS_R1 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 1)
+#define KVM_REG_MIPS_R2 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 2)
+#define KVM_REG_MIPS_R3 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 3)
+#define KVM_REG_MIPS_R4 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 4)
+#define KVM_REG_MIPS_R5 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 5)
+#define KVM_REG_MIPS_R6 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 6)
+#define KVM_REG_MIPS_R7 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 7)
+#define KVM_REG_MIPS_R8 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 8)
+#define KVM_REG_MIPS_R9 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 9)
+#define KVM_REG_MIPS_R10 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 10)
+#define KVM_REG_MIPS_R11 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 11)
+#define KVM_REG_MIPS_R12 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 12)
+#define KVM_REG_MIPS_R13 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 13)
+#define KVM_REG_MIPS_R14 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 14)
+#define KVM_REG_MIPS_R15 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 15)
+#define KVM_REG_MIPS_R16 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 16)
+#define KVM_REG_MIPS_R17 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 17)
+#define KVM_REG_MIPS_R18 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 18)
+#define KVM_REG_MIPS_R19 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 19)
+#define KVM_REG_MIPS_R20 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 20)
+#define KVM_REG_MIPS_R21 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 21)
+#define KVM_REG_MIPS_R22 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 22)
+#define KVM_REG_MIPS_R23 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 23)
+#define KVM_REG_MIPS_R24 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 24)
+#define KVM_REG_MIPS_R25 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 25)
+#define KVM_REG_MIPS_R26 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 26)
+#define KVM_REG_MIPS_R27 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 27)
+#define KVM_REG_MIPS_R28 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 28)
+#define KVM_REG_MIPS_R29 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 29)
+#define KVM_REG_MIPS_R30 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 30)
+#define KVM_REG_MIPS_R31 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 31)
 
-#define KVM_REG_MIPS_HI 32
-#define KVM_REG_MIPS_LO 33
-#define KVM_REG_MIPS_PC 34
+#define KVM_REG_MIPS_HI (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 32)
+#define KVM_REG_MIPS_LO (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 33)
+#define KVM_REG_MIPS_PC (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 34)
 
 /*
  * KVM MIPS specific structures and definitions
diff --git a/linux-headers/asm-x86/kvm_para.h b/linux-headers/asm-x86/kvm_para.h
index a1c3d72acd..e41c5c1a28 100644
--- a/linux-headers/asm-x86/kvm_para.h
+++ b/linux-headers/asm-x86/kvm_para.h
@@ -23,6 +23,7 @@
 #define KVM_FEATURE_ASYNC_PF		4
 #define KVM_FEATURE_STEAL_TIME		5
 #define KVM_FEATURE_PV_EOI		6
+#define KVM_FEATURE_PV_UNHALT		7
 
 /* The last 8 bits are used to indicate how to interpret the flags field
  * in pvclock structure. If no bits are set, all flags are ignored.
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index c614070662..13e890c53b 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -666,6 +666,8 @@ struct kvm_ppc_smmu_info {
 #define KVM_CAP_IRQ_MPIC 90
 #define KVM_CAP_PPC_RTAS 91
 #define KVM_CAP_IRQ_XICS 92
+#define KVM_CAP_ARM_EL1_32BIT 93
+#define KVM_CAP_SPAPR_MULTITCE 94
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -783,6 +785,8 @@ struct kvm_dirty_tlb {
 #define KVM_REG_IA64		0x3000000000000000ULL
 #define KVM_REG_ARM		0x4000000000000000ULL
 #define KVM_REG_S390		0x5000000000000000ULL
+#define KVM_REG_ARM64		0x6000000000000000ULL
+#define KVM_REG_MIPS		0x7000000000000000ULL
 
 #define KVM_REG_SIZE_SHIFT	52
 #define KVM_REG_SIZE_MASK	0x00f0000000000000ULL
diff --git a/linux-headers/linux/kvm_para.h b/linux-headers/linux/kvm_para.h
index 7bdcf93c1d..2dff7838b4 100644
--- a/linux-headers/linux/kvm_para.h
+++ b/linux-headers/linux/kvm_para.h
@@ -19,6 +19,7 @@
 #define KVM_HC_MMU_OP			2
 #define KVM_HC_FEATURES			3
 #define KVM_HC_PPC_MAP_MAGIC_PAGE	4
+#define KVM_HC_KICK_CPU			5
 
 /*
  * hypercalls use architecture specific
diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h
index 7ec1864765..17c58e0ede 100644
--- a/linux-headers/linux/vfio.h
+++ b/linux-headers/linux/vfio.h
@@ -22,6 +22,7 @@
 /* Extensions */
 
 #define VFIO_TYPE1_IOMMU		1
+#define VFIO_SPAPR_TCE_IOMMU		2
 
 /*
  * The IOCTL interface is designed for extensibility by embedding the
@@ -323,6 +324,44 @@ enum {
 	VFIO_PCI_NUM_IRQS
 };
 
+/**
+ * VFIO_DEVICE_GET_PCI_HOT_RESET_INFO - _IORW(VFIO_TYPE, VFIO_BASE + 12,
+ *					      struct vfio_pci_hot_reset_info)
+ *
+ * Return: 0 on success, -errno on failure:
+ *	-enospc = insufficient buffer, -enodev = unsupported for device.
+ */
+struct vfio_pci_dependent_device {
+	__u32	group_id;
+	__u16	segment;
+	__u8	bus;
+	__u8	devfn; /* Use PCI_SLOT/PCI_FUNC */
+};
+
+struct vfio_pci_hot_reset_info {
+	__u32	argsz;
+	__u32	flags;
+	__u32	count;
+	struct vfio_pci_dependent_device	devices[];
+};
+
+#define VFIO_DEVICE_GET_PCI_HOT_RESET_INFO	_IO(VFIO_TYPE, VFIO_BASE + 12)
+
+/**
+ * VFIO_DEVICE_PCI_HOT_RESET - _IOW(VFIO_TYPE, VFIO_BASE + 13,
+ *				    struct vfio_pci_hot_reset)
+ *
+ * Return: 0 on success, -errno on failure.
+ */
+struct vfio_pci_hot_reset {
+	__u32	argsz;
+	__u32	flags;
+	__u32	count;
+	__s32	group_fds[];
+};
+
+#define VFIO_DEVICE_PCI_HOT_RESET	_IO(VFIO_TYPE, VFIO_BASE + 13)
+
 /* -------- API for Type1 VFIO IOMMU -------- */
 
 /**
@@ -361,10 +400,14 @@ struct vfio_iommu_type1_dma_map {
 #define VFIO_IOMMU_MAP_DMA _IO(VFIO_TYPE, VFIO_BASE + 13)
 
 /**
- * VFIO_IOMMU_UNMAP_DMA - _IOW(VFIO_TYPE, VFIO_BASE + 14, struct vfio_dma_unmap)
+ * VFIO_IOMMU_UNMAP_DMA - _IOWR(VFIO_TYPE, VFIO_BASE + 14,
+ *							struct vfio_dma_unmap)
  *
  * Unmap IO virtual addresses using the provided struct vfio_dma_unmap.
- * Caller sets argsz.
+ * Caller sets argsz.  The actual unmapped size is returned in the size
+ * field.  No guarantee is made to the user that arbitrary unmaps of iova
+ * or size different from those used in the original mapping call will
+ * succeed.
  */
 struct vfio_iommu_type1_dma_unmap {
 	__u32	argsz;
@@ -375,4 +418,37 @@ struct vfio_iommu_type1_dma_unmap {
 
 #define VFIO_IOMMU_UNMAP_DMA _IO(VFIO_TYPE, VFIO_BASE + 14)
 
+/*
+ * IOCTLs to enable/disable IOMMU container usage.
+ * No parameters are supported.
+ */
+#define VFIO_IOMMU_ENABLE	_IO(VFIO_TYPE, VFIO_BASE + 15)
+#define VFIO_IOMMU_DISABLE	_IO(VFIO_TYPE, VFIO_BASE + 16)
+
+/* -------- Additional API for SPAPR TCE (Server POWERPC) IOMMU -------- */
+
+/*
+ * The SPAPR TCE info struct provides the information about the PCI bus
+ * address ranges available for DMA, these values are programmed into
+ * the hardware so the guest has to know that information.
+ *
+ * The DMA 32 bit window start is an absolute PCI bus address.
+ * The IOVA address passed via map/unmap ioctls are absolute PCI bus
+ * addresses too so the window works as a filter rather than an offset
+ * for IOVA addresses.
+ *
+ * A flag will need to be added if other page sizes are supported,
+ * so as defined here, it is always 4k.
+ */
+struct vfio_iommu_spapr_tce_info {
+	__u32 argsz;
+	__u32 flags;			/* reserved for future use */
+	__u32 dma32_window_start;	/* 32 bit window start (bytes) */
+	__u32 dma32_window_size;	/* 32 bit window size (bytes) */
+};
+
+#define VFIO_IOMMU_SPAPR_TCE_GET_INFO	_IO(VFIO_TYPE, VFIO_BASE + 12)
+
+/* ***************************************************************** */
+
 #endif /* VFIO_H */
diff --git a/linux-headers/linux/virtio_config.h b/linux-headers/linux/virtio_config.h
index 4f51d8f3af..75dc20ba6f 100644
--- a/linux-headers/linux/virtio_config.h
+++ b/linux-headers/linux/virtio_config.h
@@ -51,4 +51,7 @@
  * suppressed them? */
 #define VIRTIO_F_NOTIFY_ON_EMPTY	24
 
+/* Can the device handle any descriptor layout? */
+#define VIRTIO_F_ANY_LAYOUT		27
+
 #endif /* _LINUX_VIRTIO_CONFIG_H */
diff --git a/linux-user/alpha/syscall_nr.h b/linux-user/alpha/syscall_nr.h
index ac2b6e2c65..d52d76e08e 100644
--- a/linux-user/alpha/syscall_nr.h
+++ b/linux-user/alpha/syscall_nr.h
@@ -20,7 +20,7 @@
 #define TARGET_NR_lseek		 19
 #define TARGET_NR_getxpid		 20
 #define TARGET_NR_osf_mount		 21
-#define TARGET_NR_umount		 22
+#define TARGET_NR_umount2		 22
 #define TARGET_NR_setuid		 23
 #define TARGET_NR_getxuid		 24
 #define TARGET_NR_exec_with_loader	 25	/* not implemented */
@@ -255,7 +255,7 @@
 #define TARGET_NR_sysinfo		318
 #define TARGET_NR__sysctl		319
 /* 320 was sys_idle.  */
-#define TARGET_NR_oldumount		321
+#define TARGET_NR_umount		321
 #define TARGET_NR_swapon		322
 #define TARGET_NR_times		323
 #define TARGET_NR_personality	324
diff --git a/linux-user/ioctls.h b/linux-user/ioctls.h
index 439c2a9e32..7381012d3d 100644
--- a/linux-user/ioctls.h
+++ b/linux-user/ioctls.h
@@ -20,6 +20,7 @@
      IOCTL(TIOCSCTTY, 0, TYPE_INT)
      IOCTL(TIOCGPGRP, IOC_R, MK_PTR(TYPE_INT))
      IOCTL(TIOCSPGRP, IOC_W, MK_PTR(TYPE_INT))
+     IOCTL(TIOCGSID, IOC_W, MK_PTR(TYPE_INT))
      IOCTL(TIOCOUTQ, IOC_R, MK_PTR(TYPE_INT))
      IOCTL(TIOCSTI, IOC_W, MK_PTR(TYPE_INT))
      IOCTL(TIOCMGET, IOC_R, MK_PTR(TYPE_INT))
diff --git a/linux-user/linuxload.c b/linux-user/linuxload.c
index 5cd6d91554..a1fe5ed9ae 100644
--- a/linux-user/linuxload.c
+++ b/linux-user/linuxload.c
@@ -131,7 +131,7 @@ abi_ulong loader_build_argptr(int envc, int argc, abi_ulong sp,
     return sp;
 }
 
-int loader_exec(const char * filename, char ** argv, char ** envp,
+int loader_exec(int fdexec, const char *filename, char **argv, char **envp,
              struct target_pt_regs * regs, struct image_info *infop,
              struct linux_binprm *bprm)
 {
@@ -140,11 +140,7 @@ int loader_exec(const char * filename, char ** argv, char ** envp,
 
     bprm->p = TARGET_PAGE_SIZE*MAX_ARG_PAGES-sizeof(unsigned int);
     memset(bprm->page, 0, sizeof(bprm->page));
-    retval = open(filename, O_RDONLY);
-    if (retval < 0) {
-        return -errno;
-    }
-    bprm->fd = retval;
+    bprm->fd = fdexec;
     bprm->filename = (char *)filename;
     bprm->argc = count(argv);
     bprm->argv = argv;
diff --git a/linux-user/main.c b/linux-user/main.c
index 01e3cd4cc1..6b4ab0930e 100644
--- a/linux-user/main.c
+++ b/linux-user/main.c
@@ -42,7 +42,7 @@ const char *filename;
 const char *argv0;
 int gdbstub_port;
 envlist_t *envlist;
-const char *cpu_model;
+static const char *cpu_model;
 unsigned long mmap_min_addr;
 #if defined(CONFIG_USE_GUEST_BASE)
 unsigned long guest_base;
@@ -1861,7 +1861,7 @@ static const uint8_t mips_syscall_args[] = {
 	MIPS_SYS(sys_lseek	, 3)
 	MIPS_SYS(sys_getpid	, 0)	/* 4020 */
 	MIPS_SYS(sys_mount	, 5)
-	MIPS_SYS(sys_oldumount	, 1)
+	MIPS_SYS(sys_umount	, 1)
 	MIPS_SYS(sys_setuid	, 1)
 	MIPS_SYS(sys_getuid	, 0)
 	MIPS_SYS(sys_stime	, 1)	/* 4025 */
@@ -1891,7 +1891,7 @@ static const uint8_t mips_syscall_args[] = {
 	MIPS_SYS(sys_geteuid	, 0)
 	MIPS_SYS(sys_getegid	, 0)	/* 4050 */
 	MIPS_SYS(sys_acct	, 0)
-	MIPS_SYS(sys_umount	, 2)
+	MIPS_SYS(sys_umount2	, 2)
 	MIPS_SYS(sys_ni_syscall	, 0)
 	MIPS_SYS(sys_ioctl	, 3)
 	MIPS_SYS(sys_fcntl	, 3)	/* 4055 */
@@ -2400,12 +2400,31 @@ done_syscall:
                 if (env->hflags & MIPS_HFLAG_M16) {
                     if (env->insn_flags & ASE_MICROMIPS) {
                         /* microMIPS mode */
-                        abi_ulong instr[2];
-
-                        ret = get_user_u16(instr[0], env->active_tc.PC) ||
-                              get_user_u16(instr[1], env->active_tc.PC + 2);
+                        ret = get_user_u16(trap_instr, env->active_tc.PC);
+                        if (ret != 0) {
+                            goto error;
+                        }
 
-                        trap_instr = (instr[0] << 16) | instr[1];
+                        if ((trap_instr >> 10) == 0x11) {
+                            /* 16-bit instruction */
+                            code = trap_instr & 0xf;
+                        } else {
+                            /* 32-bit instruction */
+                            abi_ulong instr_lo;
+
+                            ret = get_user_u16(instr_lo,
+                                               env->active_tc.PC + 2);
+                            if (ret != 0) {
+                                goto error;
+                            }
+                            trap_instr = (trap_instr << 16) | instr_lo;
+                            code = ((trap_instr >> 6) & ((1 << 20) - 1));
+                            /* Unfortunately, microMIPS also suffers from
+                               the old assembler bug...  */
+                            if (code >= (1 << 10)) {
+                                code >>= 10;
+                            }
+                        }
                     } else {
                         /* MIPS16e mode */
                         ret = get_user_u16(trap_instr, env->active_tc.PC);
@@ -2413,26 +2432,21 @@ done_syscall:
                             goto error;
                         }
                         code = (trap_instr >> 6) & 0x3f;
-                        if (do_break(env, &info, code) != 0) {
-                            goto error;
-                        }
-                        break;
                     }
                 } else {
                     ret = get_user_ual(trap_instr, env->active_tc.PC);
-                }
-
-                if (ret != 0) {
-                    goto error;
-                }
+                    if (ret != 0) {
+                        goto error;
+                    }
 
-                /* As described in the original Linux kernel code, the
-                 * below checks on 'code' are to work around an old
-                 * assembly bug.
-                 */
-                code = ((trap_instr >> 6) & ((1 << 20) - 1));
-                if (code >= (1 << 10)) {
-                    code >>= 10;
+                    /* As described in the original Linux kernel code, the
+                     * below checks on 'code' are to work around an old
+                     * assembly bug.
+                     */
+                    code = ((trap_instr >> 6) & ((1 << 20) - 1));
+                    if (code >= (1 << 10)) {
+                        code >>= 10;
+                    }
                 }
 
                 if (do_break(env, &info, code) != 0) {
@@ -3271,6 +3285,37 @@ void init_task_state(TaskState *ts)
     ts->sigqueue_table[i].next = NULL;
 }
 
+CPUArchState *cpu_copy(CPUArchState *env)
+{
+    CPUArchState *new_env = cpu_init(cpu_model);
+#if defined(TARGET_HAS_ICE)
+    CPUBreakpoint *bp;
+    CPUWatchpoint *wp;
+#endif
+
+    /* Reset non arch specific state */
+    cpu_reset(ENV_GET_CPU(new_env));
+
+    memcpy(new_env, env, sizeof(CPUArchState));
+
+    /* Clone all break/watchpoints.
+       Note: Once we support ptrace with hw-debug register access, make sure
+       BP_CPU break/watchpoints are handled correctly on clone. */
+    QTAILQ_INIT(&env->breakpoints);
+    QTAILQ_INIT(&env->watchpoints);
+#if defined(TARGET_HAS_ICE)
+    QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
+        cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
+    }
+    QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
+        cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
+                              wp->flags, NULL);
+    }
+#endif
+
+    return new_env;
+}
+
 static void handle_arg_help(const char *arg)
 {
     usage();
@@ -3618,6 +3663,26 @@ static int parse_args(int argc, char **argv)
     return optind;
 }
 
+static int get_execfd(char **envp)
+{
+    typedef struct {
+        long a_type;
+        long a_val;
+    } auxv_t;
+    auxv_t *auxv;
+
+    while (*envp++ != NULL) {
+        ;
+    }
+
+    for (auxv = (auxv_t *)envp; auxv->a_type != AT_NULL; auxv++) {
+        if (auxv->a_type == AT_EXECFD) {
+            return auxv->a_val;
+        }
+    }
+    return -1;
+}
+
 int main(int argc, char **argv, char **envp)
 {
     struct target_pt_regs regs1, *regs = &regs1;
@@ -3632,6 +3697,7 @@ int main(int argc, char **argv, char **envp)
     int target_argc;
     int i;
     int ret;
+    int execfd;
 
     module_call_init(MODULE_INIT_QOM);
 
@@ -3809,7 +3875,16 @@ int main(int argc, char **argv, char **envp)
     env->opaque = ts;
     task_settid(ts);
 
-    ret = loader_exec(filename, target_argv, target_environ, regs,
+    execfd = get_execfd(envp);
+    if (execfd < 0) {
+        execfd = open(filename, O_RDONLY);
+    }
+    if (execfd < 0) {
+        printf("Error while loading %s: %s\n", filename, strerror(-execfd));
+        _exit(1);
+    }
+
+    ret = loader_exec(execfd, filename, target_argv, target_environ, regs,
         info, &bprm);
     if (ret != 0) {
         printf("Error while loading %s: %s\n", filename, strerror(-ret));
diff --git a/linux-user/qemu.h b/linux-user/qemu.h
index 6ffe5a2dec..da64e877c7 100644
--- a/linux-user/qemu.h
+++ b/linux-user/qemu.h
@@ -174,7 +174,7 @@ struct linux_binprm {
 void do_init_thread(struct target_pt_regs *regs, struct image_info *infop);
 abi_ulong loader_build_argptr(int envc, int argc, abi_ulong sp,
                               abi_ulong stringp, int push_ptr);
-int loader_exec(const char * filename, char ** argv, char ** envp,
+int loader_exec(int fdexec, const char *filename, char **argv, char **envp,
              struct target_pt_regs * regs, struct image_info *infop,
              struct linux_binprm *);
 
@@ -380,9 +380,9 @@ abi_long copy_from_user(void *hptr, abi_ulong gaddr, size_t len);
 abi_long copy_to_user(abi_ulong gaddr, void *hptr, size_t len);
 
 /* Functions for accessing guest memory.  The tget and tput functions
-   read/write single values, byteswapping as necessary.  The lock_user
+   read/write single values, byteswapping as necessary.  The lock_user function
    gets a pointer to a contiguous area of guest memory, but does not perform
-   and byteswapping.  lock_user may return either a pointer to the guest
+   any byteswapping.  lock_user may return either a pointer to the guest
    memory, or a temporary buffer.  */
 
 /* Lock an area of guest memory into the host.  If copy is true then the
@@ -438,7 +438,7 @@ static inline void *lock_user_string(abi_ulong guest_addr)
     return lock_user(VERIFY_READ, guest_addr, (long)(len + 1), 1);
 }
 
-/* Helper macros for locking/ulocking a target struct.  */
+/* Helper macros for locking/unlocking a target struct.  */
 #define lock_user_struct(type, host_ptr, guest_addr, copy)	\
     (host_ptr = lock_user(type, guest_addr, sizeof(*host_ptr), copy))
 #define unlock_user_struct(host_ptr, guest_addr, copy)		\
diff --git a/linux-user/strace.list b/linux-user/strace.list
index 08f115d843..cf5841af60 100644
--- a/linux-user/strace.list
+++ b/linux-user/strace.list
@@ -612,9 +612,6 @@
 #ifdef TARGET_NR_oldstat
 { TARGET_NR_oldstat, "oldstat" , NULL, NULL, NULL },
 #endif
-#ifdef TARGET_NR_oldumount
-{ TARGET_NR_oldumount, "oldumount" , NULL, NULL, NULL },
-#endif
 #ifdef TARGET_NR_olduname
 { TARGET_NR_olduname, "olduname" , NULL, NULL, NULL },
 #endif
@@ -1524,3 +1521,9 @@
 #ifdef TARGET_NR_pipe2
 { TARGET_NR_pipe2, "pipe2", NULL, NULL, NULL },
 #endif
+#ifdef TARGET_NR_atomic_cmpxchg_32
+{ TARGET_NR_atomic_cmpxchg_32, "atomic_cmpxchg_32", NULL, NULL, NULL },
+#endif
+#ifdef TARGET_NR_atomic_barrier
+{ TARGET_NR_atomic_barrier, "atomic_barrier", NULL, NULL, NULL },
+#endif
diff --git a/linux-user/syscall.c b/linux-user/syscall.c
index c62d8754f0..4a14a43037 100644
--- a/linux-user/syscall.c
+++ b/linux-user/syscall.c
@@ -106,6 +106,7 @@ int __clone2(int (*fn)(void *), void *child_stack_base,
 #include <linux/dm-ioctl.h>
 #include <linux/reboot.h>
 #include <linux/route.h>
+#include <linux/filter.h>
 #include "linux_loop.h"
 #include "cpu-uname.h"
 
@@ -1149,11 +1150,15 @@ static inline abi_long target_to_host_cmsg(struct msghdr *msgh,
             break;
         }
 
-        cmsg->cmsg_level = tswap32(target_cmsg->cmsg_level);
+        if (tswap32(target_cmsg->cmsg_level) == TARGET_SOL_SOCKET) {
+            cmsg->cmsg_level = SOL_SOCKET;
+        } else {
+            cmsg->cmsg_level = tswap32(target_cmsg->cmsg_level);
+        }
         cmsg->cmsg_type = tswap32(target_cmsg->cmsg_type);
         cmsg->cmsg_len = CMSG_LEN(len);
 
-        if (cmsg->cmsg_level != TARGET_SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS) {
+        if (cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS) {
             gemu_log("Unsupported ancillary data: %d/%d\n", cmsg->cmsg_level, cmsg->cmsg_type);
             memcpy(data, target_data, len);
         } else {
@@ -1204,11 +1209,15 @@ static inline abi_long host_to_target_cmsg(struct target_msghdr *target_msgh,
             break;
         }
 
-        target_cmsg->cmsg_level = tswap32(cmsg->cmsg_level);
+        if (cmsg->cmsg_level == SOL_SOCKET) {
+            target_cmsg->cmsg_level = tswap32(TARGET_SOL_SOCKET);
+        } else {
+            target_cmsg->cmsg_level = tswap32(cmsg->cmsg_level);
+        }
         target_cmsg->cmsg_type = tswap32(cmsg->cmsg_type);
         target_cmsg->cmsg_len = tswapal(TARGET_CMSG_LEN(len));
 
-        if ((cmsg->cmsg_level == TARGET_SOL_SOCKET) &&
+        if ((cmsg->cmsg_level == SOL_SOCKET) &&
                                 (cmsg->cmsg_type == SCM_RIGHTS)) {
             int *fd = (int *)data;
             int *target_fd = (int *)target_data;
@@ -1216,7 +1225,7 @@ static inline abi_long host_to_target_cmsg(struct target_msghdr *target_msgh,
 
             for (i = 0; i < numfds; i++)
                 target_fd[i] = tswap32(fd[i]);
-        } else if ((cmsg->cmsg_level == TARGET_SOL_SOCKET) &&
+        } else if ((cmsg->cmsg_level == SOL_SOCKET) &&
                                 (cmsg->cmsg_type == SO_TIMESTAMP) &&
                                 (len == sizeof(struct timeval))) {
             /* copy struct timeval to target */
@@ -1314,6 +1323,26 @@ static abi_long do_setsockopt(int sockfd, int level, int optname,
             goto unimplemented;
         }
         break;
+    case SOL_IPV6:
+        switch (optname) {
+        case IPV6_MTU_DISCOVER:
+        case IPV6_MTU:
+        case IPV6_V6ONLY:
+        case IPV6_RECVPKTINFO:
+            val = 0;
+            if (optlen < sizeof(uint32_t)) {
+                return -TARGET_EINVAL;
+            }
+            if (get_user_u32(val, optval_addr)) {
+                return -TARGET_EFAULT;
+            }
+            ret = get_errno(setsockopt(sockfd, level, optname,
+                                       &val, sizeof(val)));
+            break;
+        default:
+            goto unimplemented;
+        }
+        break;
     case SOL_RAW:
         switch (optname) {
         case ICMP_FILTER:
@@ -1357,6 +1386,49 @@ set_timeout:
         case TARGET_SO_SNDTIMEO:
                 optname = SO_SNDTIMEO;
                 goto set_timeout;
+        case TARGET_SO_ATTACH_FILTER:
+        {
+                struct target_sock_fprog *tfprog;
+                struct target_sock_filter *tfilter;
+                struct sock_fprog fprog;
+                struct sock_filter *filter;
+                int i;
+
+                if (optlen != sizeof(*tfprog)) {
+                    return -TARGET_EINVAL;
+                }
+                if (!lock_user_struct(VERIFY_READ, tfprog, optval_addr, 0)) {
+                    return -TARGET_EFAULT;
+                }
+                if (!lock_user_struct(VERIFY_READ, tfilter,
+                                      tswapal(tfprog->filter), 0)) {
+                    unlock_user_struct(tfprog, optval_addr, 1);
+                    return -TARGET_EFAULT;
+                }
+
+                fprog.len = tswap16(tfprog->len);
+                filter = malloc(fprog.len * sizeof(*filter));
+                if (filter == NULL) {
+                    unlock_user_struct(tfilter, tfprog->filter, 1);
+                    unlock_user_struct(tfprog, optval_addr, 1);
+                    return -TARGET_ENOMEM;
+                }
+                for (i = 0; i < fprog.len; i++) {
+                    filter[i].code = tswap16(tfilter[i].code);
+                    filter[i].jt = tfilter[i].jt;
+                    filter[i].jf = tfilter[i].jf;
+                    filter[i].k = tswap32(tfilter[i].k);
+                }
+                fprog.filter = filter;
+
+                ret = get_errno(setsockopt(sockfd, SOL_SOCKET,
+                                SO_ATTACH_FILTER, &fprog, sizeof(fprog)));
+                free(filter);
+
+                unlock_user_struct(tfilter, tfprog->filter, 1);
+                unlock_user_struct(tfprog, optval_addr, 1);
+                return ret;
+        }
             /* Options with 'int' argument.  */
         case TARGET_SO_DEBUG:
 		optname = SO_DEBUG;
@@ -1701,7 +1773,7 @@ static void unlock_iovec(struct iovec *vec, abi_ulong target_addr,
     free(vec);
 }
 
-static inline void target_to_host_sock_type(int *type)
+static inline int target_to_host_sock_type(int *type)
 {
     int host_type = 0;
     int target_type = *type;
@@ -1718,22 +1790,56 @@ static inline void target_to_host_sock_type(int *type)
         break;
     }
     if (target_type & TARGET_SOCK_CLOEXEC) {
+#if defined(SOCK_CLOEXEC)
         host_type |= SOCK_CLOEXEC;
+#else
+        return -TARGET_EINVAL;
+#endif
     }
     if (target_type & TARGET_SOCK_NONBLOCK) {
+#if defined(SOCK_NONBLOCK)
         host_type |= SOCK_NONBLOCK;
+#elif !defined(O_NONBLOCK)
+        return -TARGET_EINVAL;
+#endif
     }
     *type = host_type;
+    return 0;
+}
+
+/* Try to emulate socket type flags after socket creation.  */
+static int sock_flags_fixup(int fd, int target_type)
+{
+#if !defined(SOCK_NONBLOCK) && defined(O_NONBLOCK)
+    if (target_type & TARGET_SOCK_NONBLOCK) {
+        int flags = fcntl(fd, F_GETFL);
+        if (fcntl(fd, F_SETFL, O_NONBLOCK | flags) == -1) {
+            close(fd);
+            return -TARGET_EINVAL;
+        }
+    }
+#endif
+    return fd;
 }
 
 /* do_socket() Must return target values and target errnos. */
 static abi_long do_socket(int domain, int type, int protocol)
 {
-    target_to_host_sock_type(&type);
+    int target_type = type;
+    int ret;
+
+    ret = target_to_host_sock_type(&type);
+    if (ret) {
+        return ret;
+    }
 
     if (domain == PF_NETLINK)
         return -EAFNOSUPPORT; /* do not NETLINK socket connections possible */
-    return get_errno(socket(domain, type, protocol));
+    ret = get_errno(socket(domain, type, protocol));
+    if (ret >= 0) {
+        ret = sock_flags_fixup(ret, target_type);
+    }
+    return ret;
 }
 
 /* do_bind() Must return target values and target errnos. */
@@ -5071,22 +5177,70 @@ static int is_proc_myself(const char *filename, const char *entry)
     return 0;
 }
 
+#if defined(HOST_WORDS_BIGENDIAN) != defined(TARGET_WORDS_BIGENDIAN)
+static int is_proc(const char *filename, const char *entry)
+{
+    return strcmp(filename, entry) == 0;
+}
+
+static int open_net_route(void *cpu_env, int fd)
+{
+    FILE *fp;
+    char *line = NULL;
+    size_t len = 0;
+    ssize_t read;
+
+    fp = fopen("/proc/net/route", "r");
+    if (fp == NULL) {
+        return -EACCES;
+    }
+
+    /* read header */
+
+    read = getline(&line, &len, fp);
+    dprintf(fd, "%s", line);
+
+    /* read routes */
+
+    while ((read = getline(&line, &len, fp)) != -1) {
+        char iface[16];
+        uint32_t dest, gw, mask;
+        unsigned int flags, refcnt, use, metric, mtu, window, irtt;
+        sscanf(line, "%s\t%08x\t%08x\t%04x\t%d\t%d\t%d\t%08x\t%d\t%u\t%u\n",
+                     iface, &dest, &gw, &flags, &refcnt, &use, &metric,
+                     &mask, &mtu, &window, &irtt);
+        dprintf(fd, "%s\t%08x\t%08x\t%04x\t%d\t%d\t%d\t%08x\t%d\t%u\t%u\n",
+                iface, tswap32(dest), tswap32(gw), flags, refcnt, use,
+                metric, tswap32(mask), mtu, window, irtt);
+    }
+
+    free(line);
+    fclose(fp);
+
+    return 0;
+}
+#endif
+
 static int do_open(void *cpu_env, const char *pathname, int flags, mode_t mode)
 {
     struct fake_open {
         const char *filename;
         int (*fill)(void *cpu_env, int fd);
+        int (*cmp)(const char *s1, const char *s2);
     };
     const struct fake_open *fake_open;
     static const struct fake_open fakes[] = {
-        { "maps", open_self_maps },
-        { "stat", open_self_stat },
-        { "auxv", open_self_auxv },
-        { NULL, NULL }
+        { "maps", open_self_maps, is_proc_myself },
+        { "stat", open_self_stat, is_proc_myself },
+        { "auxv", open_self_auxv, is_proc_myself },
+#if defined(HOST_WORDS_BIGENDIAN) != defined(TARGET_WORDS_BIGENDIAN)
+        { "/proc/net/route", open_net_route, is_proc },
+#endif
+        { NULL, NULL, NULL }
     };
 
     for (fake_open = fakes; fake_open->filename; fake_open++) {
-        if (is_proc_myself(pathname, fake_open->filename)) {
+        if (fake_open->cmp(pathname, fake_open->filename)) {
             break;
         }
     }
@@ -5697,7 +5851,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             unlock_user(p, arg1, 0);
         }
         break;
-#ifdef TARGET_NR_umount2 /* not on alpha */
+#ifdef TARGET_NR_umount2
     case TARGET_NR_umount2:
         if (!(p = lock_user_string(arg1)))
             goto efault;
@@ -9013,6 +9167,34 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         break;
     }
 #endif
+#ifdef TARGET_NR_atomic_cmpxchg_32
+    case TARGET_NR_atomic_cmpxchg_32:
+    {
+        /* should use start_exclusive from main.c */
+        abi_ulong mem_value;
+        if (get_user_u32(mem_value, arg6)) {
+            target_siginfo_t info;
+            info.si_signo = SIGSEGV;
+            info.si_errno = 0;
+            info.si_code = TARGET_SEGV_MAPERR;
+            info._sifields._sigfault._addr = arg6;
+            queue_signal((CPUArchState *)cpu_env, info.si_signo, &info);
+            ret = 0xdeadbeef;
+
+        }
+        if (mem_value == arg2)
+            put_user_u32(arg1, arg6);
+        ret = mem_value;
+        break;
+    }
+#endif
+#ifdef TARGET_NR_atomic_barrier
+    case TARGET_NR_atomic_barrier:
+    {
+        /* Like the kernel implementation and the qemu arm barrier, no-op this? */
+        break;
+    }
+#endif
     default:
     unimplemented:
         gemu_log("qemu: Unsupported syscall: %d\n", num);
diff --git a/linux-user/syscall_defs.h b/linux-user/syscall_defs.h
index 2ebe3560d7..5f53a28d1b 100644
--- a/linux-user/syscall_defs.h
+++ b/linux-user/syscall_defs.h
@@ -119,6 +119,18 @@ struct target_sockaddr {
     uint8_t sa_data[14];
 };
 
+struct target_sock_filter {
+    abi_ushort code;
+    uint8_t jt;
+    uint8_t jf;
+    abi_uint k;
+};
+
+struct target_sock_fprog {
+    abi_ushort len;
+    abi_ulong filter;
+};
+
 struct target_in_addr {
     uint32_t s_addr; /* big endian */
 };
diff --git a/main-loop.c b/main-loop.c
index 1c38ea2b93..c3c9c28fba 100644
--- a/main-loop.c
+++ b/main-loop.c
@@ -466,8 +466,7 @@ int main_loop_wait(int nonblocking)
     g_array_set_size(gpollfds, 0); /* reset for new iteration */
     /* XXX: separate device handlers from system ones */
 #ifdef CONFIG_SLIRP
-    slirp_update_timeout(&timeout);
-    slirp_pollfds_fill(gpollfds);
+    slirp_pollfds_fill(gpollfds, &timeout);
 #endif
     qemu_iohandler_fill(gpollfds);
 
diff --git a/memory.c b/memory.c
index f49d31ae72..28f64491d0 100644
--- a/memory.c
+++ b/memory.c
@@ -1809,7 +1809,9 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f,
                    mr->alias->name,
                    mr->alias_offset,
                    mr->alias_offset
-                   + (hwaddr)int128_get64(mr->size) - 1);
+                   + (int128_nz(mr->size) ?
+                      (hwaddr)int128_get64(int128_sub(mr->size,
+                                                      int128_one())) : 0));
     } else {
         mon_printf(f,
                    TARGET_FMT_plx "-" TARGET_FMT_plx " (prio %d, %c%c): %s\n",
diff --git a/migration-rdma.c b/migration-rdma.c
index 05a155b93d..f94f3b4e3a 100644
--- a/migration-rdma.c
+++ b/migration-rdma.c
@@ -356,6 +356,7 @@ typedef struct RDMAContext {
      */
     struct rdma_cm_id *cm_id;               /* connection manager ID */
     struct rdma_cm_id *listen_id;
+    bool connected;
 
     struct ibv_context          *verbs;
     struct rdma_event_channel   *channel;
@@ -510,19 +511,21 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, RDMAControlHeader *head,
                                    int *resp_idx,
                                    int (*callback)(RDMAContext *rdma));
 
-static inline uint64_t ram_chunk_index(uint8_t *start, uint8_t *host)
+static inline uint64_t ram_chunk_index(const uint8_t *start,
+                                       const uint8_t *host)
 {
     return ((uintptr_t) host - (uintptr_t) start) >> RDMA_REG_CHUNK_SHIFT;
 }
 
-static inline uint8_t *ram_chunk_start(RDMALocalBlock *rdma_ram_block,
+static inline uint8_t *ram_chunk_start(const RDMALocalBlock *rdma_ram_block,
                                        uint64_t i)
 {
     return (uint8_t *) (((uintptr_t) rdma_ram_block->local_host_addr)
                                     + (i << RDMA_REG_CHUNK_SHIFT));
 }
 
-static inline uint8_t *ram_chunk_end(RDMALocalBlock *rdma_ram_block, uint64_t i)
+static inline uint8_t *ram_chunk_end(const RDMALocalBlock *rdma_ram_block,
+                                     uint64_t i)
 {
     uint8_t *result = ram_chunk_start(rdma_ram_block, i) +
                                          (1UL << RDMA_REG_CHUNK_SHIFT);
@@ -2194,7 +2197,7 @@ static void qemu_rdma_cleanup(RDMAContext *rdma)
     struct rdma_cm_event *cm_event;
     int ret, idx;
 
-    if (rdma->cm_id) {
+    if (rdma->cm_id && rdma->connected) {
         if (rdma->error_state) {
             RDMAControlHeader head = { .len = 0,
                                        .type = RDMA_CONTROL_ERROR,
@@ -2213,7 +2216,7 @@ static void qemu_rdma_cleanup(RDMAContext *rdma)
             }
         }
         DDPRINTF("Disconnected.\n");
-        rdma->cm_id = NULL;
+        rdma->connected = false;
     }
 
     g_free(rdma->block);
@@ -2235,7 +2238,7 @@ static void qemu_rdma_cleanup(RDMAContext *rdma)
     }
 
     if (rdma->qp) {
-        ibv_destroy_qp(rdma->qp);
+        rdma_destroy_qp(rdma->cm_id);
         rdma->qp = NULL;
     }
     if (rdma->cq) {
@@ -2372,6 +2375,7 @@ static int qemu_rdma_connect(RDMAContext *rdma, Error **errp)
         rdma->cm_id = NULL;
         goto err_rdma_source_connect;
     }
+    rdma->connected = true;
 
     memcpy(&cap, cm_event->param.conn.private_data, sizeof(cap));
     network_to_caps(&cap);
@@ -2906,6 +2910,7 @@ static int qemu_rdma_accept(RDMAContext *rdma)
     }
 
     rdma_ack_cm_event(cm_event);
+    rdma->connected = true;
 
     ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY);
     if (ret) {
diff --git a/migration.c b/migration.c
index 200d404547..2b1ab20c54 100644
--- a/migration.c
+++ b/migration.c
@@ -150,6 +150,7 @@ MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp)
     MigrationState *s = migrate_get_current();
     int i;
 
+    caps = NULL; /* silence compiler warning */
     for (i = 0; i < MIGRATION_CAPABILITY_MAX; i++) {
         if (head == NULL) {
             head = g_malloc0(sizeof(*caps));
@@ -567,7 +568,8 @@ static void *migration_thread(void *opaque)
         if (!qemu_file_rate_limit(s->file)) {
             DPRINTF("iterate\n");
             pending_size = qemu_savevm_state_pending(s->file, max_size);
-            DPRINTF("pending size %lu max %lu\n", pending_size, max_size);
+            DPRINTF("pending size %" PRIu64 " max %" PRIu64 "\n",
+                    pending_size, max_size);
             if (pending_size && pending_size >= max_size) {
                 qemu_savevm_state_iterate(s->file);
             } else {
diff --git a/net/socket.c b/net/socket.c
index e61309d8d5..fb21e20a54 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -262,6 +262,11 @@ static int net_socket_mcast_create(struct sockaddr_in *mcastaddr, struct in_addr
         return -1;
     }
 
+    /* Allow multiple sockets to bind the same multicast ip and port by setting
+     * SO_REUSEADDR. This is the only situation where SO_REUSEADDR should be set
+     * on windows. Use socket_set_fast_reuse otherwise as it sets SO_REUSEADDR
+     * only on posix systems.
+     */
     val = 1;
     ret = qemu_setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val));
     if (ret < 0) {
@@ -510,7 +515,7 @@ static int net_socket_listen_init(NetClientState *peer,
     NetClientState *nc;
     NetSocketState *s;
     struct sockaddr_in saddr;
-    int fd, val, ret;
+    int fd, ret;
 
     if (parse_host_port(&saddr, host_str) < 0)
         return -1;
@@ -522,9 +527,7 @@ static int net_socket_listen_init(NetClientState *peer,
     }
     qemu_set_nonblock(fd);
 
-    /* allow fast reuse */
-    val = 1;
-    qemu_setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val));
+    socket_set_fast_reuse(fd);
 
     ret = bind(fd, (struct sockaddr *)&saddr, sizeof(saddr));
     if (ret < 0) {
@@ -645,7 +648,7 @@ static int net_socket_udp_init(NetClientState *peer,
                                  const char *lhost)
 {
     NetSocketState *s;
-    int fd, val, ret;
+    int fd, ret;
     struct sockaddr_in laddr, raddr;
 
     if (parse_host_port(&laddr, lhost) < 0) {
@@ -661,11 +664,9 @@ static int net_socket_udp_init(NetClientState *peer,
         perror("socket(PF_INET, SOCK_DGRAM)");
         return -1;
     }
-    val = 1;
-    ret = qemu_setsockopt(fd, SOL_SOCKET, SO_REUSEADDR,
-                          &val, sizeof(val));
+
+    ret = socket_set_fast_reuse(fd);
     if (ret < 0) {
-        perror("setsockopt(SOL_SOCKET, SO_REUSEADDR)");
         closesocket(fd);
         return -1;
     }
diff --git a/pc-bios/README b/pc-bios/README
index e404a228a4..be8dae0aa9 100644
--- a/pc-bios/README
+++ b/pc-bios/README
@@ -12,7 +12,7 @@
   1275-1994 (referred to as Open Firmware) compliant firmware.
   The included images for PowerPC (for 32 and 64 bit PPC CPUs),
   Sparc32 and Sparc64 are built from OpenBIOS SVN revision
-  1198.
+  1229.
 
 - SLOF (Slimline Open Firmware) is a free IEEE 1275 Open Firmware
   implementation for certain IBM POWER hardware.  The sources are at
@@ -23,7 +23,7 @@
   legacy x86 software to communicate with an attached serial console as
   if a video card were attached.  The master sources reside in a subversion
   repository at http://sgabios.googlecode.com/svn/trunk.  A git mirror is
-  available at git://git.qemu.org/sgabios.git.
+  available at git://git.qemu-project.org/sgabios.git.
 
 - The PXE roms come from the iPXE project. Built with BANNER_TIME 0.
   Sources available at http://ipxe.org.  Vendor:Device ID -> ROM mapping:
diff --git a/pc-bios/acpi-dsdt.aml b/pc-bios/acpi-dsdt.aml
index 48dbe3242c..528372b41e 100644
--- a/pc-bios/acpi-dsdt.aml
+++ b/pc-bios/acpi-dsdt.aml
Binary files differdiff --git a/pc-bios/bios.bin b/pc-bios/bios.bin
index cccc487814..697440c26c 100644
--- a/pc-bios/bios.bin
+++ b/pc-bios/bios.bin
Binary files differdiff --git a/pc-bios/openbios-ppc b/pc-bios/openbios-ppc
index c6b3319fab..550273a5ef 100644
--- a/pc-bios/openbios-ppc
+++ b/pc-bios/openbios-ppc
Binary files differdiff --git a/pc-bios/openbios-sparc32 b/pc-bios/openbios-sparc32
index 2aa400cfd9..01105fc904 100644
--- a/pc-bios/openbios-sparc32
+++ b/pc-bios/openbios-sparc32
Binary files differdiff --git a/pc-bios/openbios-sparc64 b/pc-bios/openbios-sparc64
index f6ee286034..62c9e77983 100644
--- a/pc-bios/openbios-sparc64
+++ b/pc-bios/openbios-sparc64
Binary files differdiff --git a/pc-bios/q35-acpi-dsdt.aml b/pc-bios/q35-acpi-dsdt.aml
index 91ab67cada..4d23746ebf 100644
--- a/pc-bios/q35-acpi-dsdt.aml
+++ b/pc-bios/q35-acpi-dsdt.aml
Binary files differdiff --git a/pc-bios/s390-ccw.img b/pc-bios/s390-ccw.img
index 05fc7c2fae..6727f0ca39 100644
--- a/pc-bios/s390-ccw.img
+++ b/pc-bios/s390-ccw.img
Binary files differdiff --git a/pc-bios/s390-ccw/virtio.c b/pc-bios/s390-ccw/virtio.c
index 49f2d291fc..4d6e48fcbe 100644
--- a/pc-bios/s390-ccw/virtio.c
+++ b/pc-bios/s390-ccw/virtio.c
@@ -123,6 +123,7 @@ static void vring_init(struct vring *vr, unsigned int num, void *p,
     /* We're running with interrupts off anyways, so don't bother */
     vr->used->flags = VRING_USED_F_NO_NOTIFY;
     vr->used->idx = 0;
+    vr->used_idx = 0;
 
     debug_print_addr("init vr", vr);
 }
@@ -150,8 +151,6 @@ static void vring_send_buf(struct vring *vr, void *p, int len, int flags)
     if (!(flags & VRING_DESC_F_NEXT)) {
         vr->avail->idx++;
     }
-
-    vr->used->idx = vr->next_idx;
 }
 
 static u64 get_clock(void)
@@ -180,7 +179,8 @@ static int vring_wait_reply(struct vring *vr, int timeout)
     struct subchannel_id schid = vr->schid;
     int r = 0;
 
-    while (vr->used->idx == vr->next_idx) {
+    /* Wait until the used index has moved. */
+    while (vr->used->idx == vr->used_idx) {
         vring_notify(schid);
         if (timeout && (get_second() >= target_second)) {
             r = 1;
@@ -189,6 +189,7 @@ static int vring_wait_reply(struct vring *vr, int timeout)
         yield();
     }
 
+    vr->used_idx = vr->used->idx;
     vr->next_idx = 0;
     vr->desc[0].len = 0;
     vr->desc[0].flags = 0;
diff --git a/pc-bios/s390-ccw/virtio.h b/pc-bios/s390-ccw/virtio.h
index 86fdd579b4..772a63f152 100644
--- a/pc-bios/s390-ccw/virtio.h
+++ b/pc-bios/s390-ccw/virtio.h
@@ -115,6 +115,7 @@ struct vring_used {
 struct vring {
     unsigned int num;
     int next_idx;
+    int used_idx;
     struct vring_desc *desc;
     struct vring_avail *avail;
     struct vring_used *used;
diff --git a/qapi-schema.json b/qapi-schema.json
index 2b2c8bce07..60f3fd1db6 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -210,6 +210,34 @@
             'vm-clock-sec': 'int', 'vm-clock-nsec': 'int' } }
 
 ##
+# @ImageInfoSpecificQCow2:
+#
+# @compat: compatibility level
+#
+# @lazy-refcounts: #optional on or off; only valid for compat >= 1.1
+#
+# Since: 1.7
+##
+{ 'type': 'ImageInfoSpecificQCow2',
+  'data': {
+      'compat': 'str',
+      '*lazy-refcounts': 'bool'
+  } }
+
+##
+# @ImageInfoSpecific:
+#
+# A discriminated record of image format specific information structures.
+#
+# Since: 1.7
+##
+
+{ 'union': 'ImageInfoSpecific',
+  'data': {
+      'qcow2': 'ImageInfoSpecificQCow2'
+  } }
+
+##
 # @ImageInfo:
 #
 # Information about a QEMU image file
@@ -238,6 +266,9 @@
 #
 # @backing-image: #optional info of the backing image (since 1.6)
 #
+# @format-specific: #optional structure supplying additional format-specific
+# information (since 1.7)
+#
 # Since: 1.3
 #
 ##
@@ -248,7 +279,8 @@
            '*cluster-size': 'int', '*encrypted': 'bool',
            '*backing-filename': 'str', '*full-backing-filename': 'str',
            '*backing-filename-format': 'str', '*snapshots': ['SnapshotInfo'],
-           '*backing-image': 'ImageInfo' } }
+           '*backing-image': 'ImageInfo',
+           '*format-specific': 'ImageInfoSpecific' } }
 
 ##
 # @ImageCheck:
@@ -1366,6 +1398,24 @@
   'data': ['top', 'full', 'none'] }
 
 ##
+# @BlockJobType:
+#
+# Type of a block job.
+#
+# @commit: block commit job type, see "block-commit"
+#
+# @stream: block stream job type, see "block-stream"
+#
+# @mirror: drive mirror job type, see "drive-mirror"
+#
+# @backup: drive backup job type, see "drive-backup"
+#
+# Since: 1.7
+##
+{ 'enum': 'BlockJobType',
+  'data': ['commit', 'stream', 'mirror', 'backup'] }
+
+##
 # @BlockJobInfo:
 #
 # Information about a long-running block device operation.
@@ -1686,6 +1736,22 @@
             '*mode': 'NewImageMode' } }
 
 ##
+# @BlockdevSnapshotInternal
+#
+# @device: the name of the device to generate the snapshot from
+#
+# @name: the name of the internal snapshot to be created
+#
+# Notes: In transaction, if @name is empty, or any snapshot matching @name
+#        exists, the operation will fail. Only some image formats support it,
+#        for example, qcow2, rbd, and sheepdog.
+#
+# Since: 1.7
+##
+{ 'type': 'BlockdevSnapshotInternal',
+  'data': { 'device': 'str', 'name': 'str' } }
+
+##
 # @DriveBackup
 #
 # @device: the name of the device which should be copied.
@@ -1747,7 +1813,8 @@
   'data': {
        'blockdev-snapshot-sync': 'BlockdevSnapshot',
        'drive-backup': 'DriveBackup',
-       'abort': 'Abort'
+       'abort': 'Abort',
+       'blockdev-snapshot-internal-sync': 'BlockdevSnapshotInternal'
    } }
 
 ##
@@ -1788,6 +1855,53 @@
   'data': 'BlockdevSnapshot' }
 
 ##
+# @blockdev-snapshot-internal-sync
+#
+# Synchronously take an internal snapshot of a block device, when the format
+# of the image used supports it.
+#
+# For the arguments, see the documentation of BlockdevSnapshotInternal.
+#
+# Returns: nothing on success
+#          If @device is not a valid block device, DeviceNotFound
+#          If any snapshot matching @name exists, or @name is empty,
+#          GenericError
+#          If the format of the image used does not support it,
+#          BlockFormatFeatureNotSupported
+#
+# Since 1.7
+##
+{ 'command': 'blockdev-snapshot-internal-sync',
+  'data': 'BlockdevSnapshotInternal' }
+
+##
+# @blockdev-snapshot-delete-internal-sync
+#
+# Synchronously delete an internal snapshot of a block device, when the format
+# of the image used support it. The snapshot is identified by name or id or
+# both. One of the name or id is required. Return SnapshotInfo for the
+# successfully deleted snapshot.
+#
+# @device: the name of the device to delete the snapshot from
+#
+# @id: optional the snapshot's ID to be deleted
+#
+# @name: optional the snapshot's name to be deleted
+#
+# Returns: SnapshotInfo on success
+#          If @device is not a valid block device, DeviceNotFound
+#          If snapshot not found, GenericError
+#          If the format of the image used does not support it,
+#          BlockFormatFeatureNotSupported
+#          If @id and @name are both not specified, GenericError
+#
+# Since 1.7
+##
+{ 'command': 'blockdev-snapshot-delete-internal-sync',
+  'data': { 'device': 'str', '*id': 'str', '*name': 'str'},
+  'returns': 'SnapshotInfo' }
+
+##
 # @human-monitor-command:
 #
 # Execute a command on the human monitor and return the output.
@@ -3838,3 +3952,239 @@
 ##
 { 'command': 'query-rx-filter', 'data': { '*name': 'str' },
   'returns': ['RxFilterInfo'] }
+
+
+##
+# @BlockdevDiscardOptions
+#
+# Determines how to handle discard requests.
+#
+# @ignore:      Ignore the request
+# @unmap:       Forward as an unmap request
+#
+# Since: 1.7
+##
+{ 'enum': 'BlockdevDiscardOptions',
+  'data': [ 'ignore', 'unmap' ] }
+
+##
+# @BlockdevAioOptions
+#
+# Selects the AIO backend to handle I/O requests
+#
+# @threads:     Use qemu's thread pool
+# @native:      Use native AIO backend (only Linux and Windows)
+#
+# Since: 1.7
+##
+{ 'enum': 'BlockdevAioOptions',
+  'data': [ 'threads', 'native' ] }
+
+##
+# @BlockdevCacheOptions
+#
+# Includes cache-related options for block devices
+#
+# @writeback:   #optional enables writeback mode for any caches (default: true)
+# @direct:      #optional enables use of O_DIRECT (bypass the host page cache;
+#               default: false)
+# @no-flush:    #optional ignore any flush requests for the device (default:
+#               false)
+#
+# Since: 1.7
+##
+{ 'type': 'BlockdevCacheOptions',
+  'data': { '*writeback': 'bool',
+            '*direct': 'bool',
+            '*no-flush': 'bool' } }
+
+##
+# @BlockdevOptionsBase
+#
+# Options that are available for all block devices, independent of the block
+# driver.
+#
+# @driver:      block driver name
+# @id:          #optional id by which the new block device can be referred to.
+#               This is a required option on the top level of blockdev-add, and
+#               currently not allowed on any other level.
+# @discard:     #optional discard-related options (default: ignore)
+# @cache:       #optional cache-related options
+# @aio:         #optional AIO backend (default: threads)
+# @rerror:      #optional how to handle read errors on the device
+#               (default: report)
+# @werror:      #optional how to handle write errors on the device
+#               (default: enospc)
+# @read-only:   #optional whether the block device should be read-only
+#               (default: false)
+#
+# Since: 1.7
+##
+{ 'type': 'BlockdevOptionsBase',
+  'data': { 'driver': 'str',
+            '*id': 'str',
+            '*discard': 'BlockdevDiscardOptions',
+            '*cache': 'BlockdevCacheOptions',
+            '*aio': 'BlockdevAioOptions',
+            '*rerror': 'BlockdevOnError',
+            '*werror': 'BlockdevOnError',
+            '*read-only': 'bool' } }
+
+##
+# @BlockdevOptionsFile
+#
+# Driver specific block device options for the file backend and similar
+# protocols.
+#
+# @filename:    path to the image file
+#
+# Since: 1.7
+##
+{ 'type': 'BlockdevOptionsFile',
+  'data': { 'filename': 'str' } }
+
+##
+# @BlockdevOptionsVVFAT
+#
+# Driver specific block device options for the vvfat protocol.
+#
+# @dir:         directory to be exported as FAT image
+# @fat-type:    #optional FAT type: 12, 16 or 32
+# @floppy:      #optional whether to export a floppy image (true) or
+#               partitioned hard disk (false; default)
+# @rw:          #optional whether to allow write operations (default: false)
+#
+# Since: 1.7
+##
+{ 'type': 'BlockdevOptionsVVFAT',
+  'data': { 'dir': 'str', '*fat-type': 'int', '*floppy': 'bool',
+            '*rw': 'bool' } }
+
+##
+# @BlockdevOptionsGenericFormat
+#
+# Driver specific block device options for image format that have no option
+# besides their data source.
+#
+# @file:        reference to or definition of the data source block device
+#
+# Since: 1.7
+##
+{ 'type': 'BlockdevOptionsGenericFormat',
+  'data': { 'file': 'BlockdevRef' } }
+
+##
+# @BlockdevOptionsGenericCOWFormat
+#
+# Driver specific block device options for image format that have no option
+# besides their data source and an optional backing file.
+#
+# @backing:     #optional reference to or definition of the backing file block
+#               device (if missing, taken from the image file content). It is
+#               allowed to pass an empty string here in order to disable the
+#               default backing file.
+#
+# Since: 1.7
+##
+{ 'type': 'BlockdevOptionsGenericCOWFormat',
+  'base': 'BlockdevOptionsGenericFormat',
+  'data': { '*backing': 'BlockdevRef' } }
+
+##
+# @BlockdevOptionsQcow2
+#
+# Driver specific block device options for qcow2.
+#
+# @lazy-refcounts:        #optional whether to enable the lazy refcounts
+#                         feature (default is taken from the image file)
+#
+# @pass-discard-request:  #optional whether discard requests to the qcow2
+#                         device should be forwarded to the data source
+#
+# @pass-discard-snapshot: #optional whether discard requests for the data source
+#                         should be issued when a snapshot operation (e.g.
+#                         deleting a snapshot) frees clusters in the qcow2 file
+#
+# @pass-discard-other:    #optional whether discard requests for the data source
+#                         should be issued on other occasions where a cluster
+#                         gets freed
+#
+# Since: 1.7
+##
+{ 'type': 'BlockdevOptionsQcow2',
+  'base': 'BlockdevOptionsGenericCOWFormat',
+  'data': { '*lazy-refcounts': 'bool',
+            '*pass-discard-request': 'bool',
+            '*pass-discard-snapshot': 'bool',
+            '*pass-discard-other': 'bool' } }
+
+##
+# @BlockdevOptions
+#
+# Options for creating a block device.
+#
+# Since: 1.7
+##
+{ 'union': 'BlockdevOptions',
+  'base': 'BlockdevOptionsBase',
+  'discriminator': 'driver',
+  'data': {
+      'file':       'BlockdevOptionsFile',
+      'http':       'BlockdevOptionsFile',
+      'https':      'BlockdevOptionsFile',
+      'ftp':        'BlockdevOptionsFile',
+      'ftps':       'BlockdevOptionsFile',
+      'tftp':       'BlockdevOptionsFile',
+# TODO gluster: Wait for structured options
+# TODO iscsi: Wait for structured options
+# TODO nbd: Should take InetSocketAddress for 'host'?
+# TODO rbd: Wait for structured options
+# TODO sheepdog: Wait for structured options
+# TODO ssh: Should take InetSocketAddress for 'host'?
+      'vvfat':      'BlockdevOptionsVVFAT',
+
+# TODO blkdebug: Wait for structured options
+# TODO blkverify: Wait for structured options
+
+      'bochs':      'BlockdevOptionsGenericFormat',
+      'cloop':      'BlockdevOptionsGenericFormat',
+      'cow':        'BlockdevOptionsGenericCOWFormat',
+      'dmg':        'BlockdevOptionsGenericFormat',
+      'parallels':  'BlockdevOptionsGenericFormat',
+      'qcow':       'BlockdevOptionsGenericCOWFormat',
+      'qcow2':      'BlockdevOptionsQcow2',
+      'qed':        'BlockdevOptionsGenericCOWFormat',
+      'raw':        'BlockdevOptionsGenericFormat',
+      'vdi':        'BlockdevOptionsGenericFormat',
+      'vhdx':       'BlockdevOptionsGenericFormat',
+      'vmdk':       'BlockdevOptionsGenericCOWFormat',
+      'vpc':        'BlockdevOptionsGenericFormat'
+  } }
+
+##
+# @BlockdevRef
+#
+# Reference to a block device.
+#
+# @definition:      defines a new block device inline
+# @reference:       references the ID of an existing block device. An
+#                   empty string means that no block device should be
+#                   referenced.
+#
+# Since: 1.7
+##
+{ 'union': 'BlockdevRef',
+  'discriminator': {},
+  'data': { 'definition': 'BlockdevOptions',
+            'reference': 'str' } }
+
+##
+# @blockdev-add:
+#
+# Creates a new block device.
+#
+# @options: block device options for the new device
+#
+# Since: 1.7
+##
+{ 'command': 'blockdev-add', 'data': { 'options': 'BlockdevOptions' } }
diff --git a/qapi/qmp-registry.c b/qapi/qmp-registry.c
index 28bbbe849e..3e4498a3f6 100644
--- a/qapi/qmp-registry.c
+++ b/qapi/qmp-registry.c
@@ -66,35 +66,26 @@ void qmp_enable_command(const char *name)
     qmp_toggle_command(name, true);
 }
 
-bool qmp_command_is_enabled(const char *name)
+bool qmp_command_is_enabled(const QmpCommand *cmd)
 {
-    QmpCommand *cmd;
+    return cmd->enabled;
+}
 
-    QTAILQ_FOREACH(cmd, &qmp_commands, node) {
-        if (strcmp(cmd->name, name) == 0) {
-            return cmd->enabled;
-        }
-    }
+const char *qmp_command_name(const QmpCommand *cmd)
+{
+    return cmd->name;
+}
 
-    return false;
+bool qmp_has_success_response(const QmpCommand *cmd)
+{
+    return !(cmd->options & QCO_NO_SUCCESS_RESP);
 }
 
-char **qmp_get_command_list(void)
+void qmp_for_each_command(qmp_cmd_callback_fn fn, void *opaque)
 {
     QmpCommand *cmd;
-    int count = 1;
-    char **list_head, **list;
 
     QTAILQ_FOREACH(cmd, &qmp_commands, node) {
-        count++;
+        fn(cmd, opaque);
     }
-
-    list_head = list = g_malloc0(count * sizeof(char *));
-
-    QTAILQ_FOREACH(cmd, &qmp_commands, node) {
-        *list = g_strdup(cmd->name);
-        list++;
-    }
-
-    return list_head;
 }
diff --git a/qdev-monitor.c b/qdev-monitor.c
index 410cdcbe97..a02c925cb3 100644
--- a/qdev-monitor.c
+++ b/qdev-monitor.c
@@ -75,14 +75,8 @@ static bool qdev_class_has_alias(DeviceClass *dc)
     return (qdev_class_get_alias(dc) != NULL);
 }
 
-static void qdev_print_class_devinfo(DeviceClass *dc)
+static void qdev_print_devinfo(DeviceClass *dc)
 {
-    DeviceCategory category;
-
-    if (!dc) {
-        return;
-    }
-
     error_printf("name \"%s\"", object_class_get_name(OBJECT_CLASS(dc)));
     if (dc->bus_type) {
         error_printf(", bus %s", dc->bus_type);
@@ -90,12 +84,6 @@ static void qdev_print_class_devinfo(DeviceClass *dc)
     if (qdev_class_has_alias(dc)) {
         error_printf(", alias \"%s\"", qdev_class_get_alias(dc));
     }
-    error_printf(", categories");
-    for (category = 0; category < DEVICE_CATEGORY_MAX; ++category) {
-        if (test_bit(category, dc->categories)) {
-            error_printf(" \"%s\"", qdev_category_get_name(category));
-        }
-    }
     if (dc->desc) {
         error_printf(", desc \"%s\"", dc->desc);
     }
@@ -105,13 +93,53 @@ static void qdev_print_class_devinfo(DeviceClass *dc)
     error_printf("\n");
 }
 
-static void qdev_print_devinfo(ObjectClass *klass, void *opaque)
+static gint devinfo_cmp(gconstpointer a, gconstpointer b)
 {
-    DeviceClass *dc;
+    return strcasecmp(object_class_get_name((ObjectClass *)a),
+                      object_class_get_name((ObjectClass *)b));
+}
 
-    dc = (DeviceClass *)object_class_dynamic_cast(klass, TYPE_DEVICE);
+static void qdev_print_devinfos(bool show_no_user)
+{
+    static const char *cat_name[DEVICE_CATEGORY_MAX + 1] = {
+        [DEVICE_CATEGORY_BRIDGE]  = "Controller/Bridge/Hub",
+        [DEVICE_CATEGORY_USB]     = "USB",
+        [DEVICE_CATEGORY_STORAGE] = "Storage",
+        [DEVICE_CATEGORY_NETWORK] = "Network",
+        [DEVICE_CATEGORY_INPUT]   = "Input",
+        [DEVICE_CATEGORY_DISPLAY] = "Display",
+        [DEVICE_CATEGORY_SOUND]   = "Sound",
+        [DEVICE_CATEGORY_MISC]    = "Misc",
+        [DEVICE_CATEGORY_MAX]     = "Uncategorized",
+    };
+    GSList *list, *elt;
+    int i;
+    bool cat_printed;
+
+    list = g_slist_sort(object_class_get_list(TYPE_DEVICE, false),
+                        devinfo_cmp);
+
+    for (i = 0; i <= DEVICE_CATEGORY_MAX; i++) {
+        cat_printed = false;
+        for (elt = list; elt; elt = elt->next) {
+            DeviceClass *dc = OBJECT_CLASS_CHECK(DeviceClass, elt->data,
+                                                 TYPE_DEVICE);
+            if ((i < DEVICE_CATEGORY_MAX
+                 ? !test_bit(i, dc->categories)
+                 : !bitmap_empty(dc->categories, DEVICE_CATEGORY_MAX))
+                || (!show_no_user && dc->no_user)) {
+                continue;
+            }
+            if (!cat_printed) {
+                error_printf("%s%s devices:\n", i ? "\n" : "",
+                             cat_name[i]);
+                cat_printed = true;
+            }
+            qdev_print_devinfo(dc);
+        }
+    }
 
-    qdev_print_class_devinfo(dc);
+    g_slist_free(list);
 }
 
 static int set_property(const char *name, const char *value, void *opaque)
@@ -151,21 +179,6 @@ static const char *find_typename_by_alias(const char *alias)
     return NULL;
 }
 
-static void qdev_print_category_devices(DeviceCategory category)
-{
-    DeviceClass *dc;
-    GSList *list, *curr;
-
-    list = object_class_get_list(TYPE_DEVICE, false);
-    for (curr = list; curr; curr = g_slist_next(curr)) {
-        dc = (DeviceClass *)object_class_dynamic_cast(curr->data, TYPE_DEVICE);
-        if (!dc->no_user && test_bit(category, dc->categories)) {
-            qdev_print_class_devinfo(dc);
-        }
-    }
-    g_slist_free(list);
-}
-
 int qdev_device_help(QemuOpts *opts)
 {
     const char *driver;
@@ -174,11 +187,7 @@ int qdev_device_help(QemuOpts *opts)
 
     driver = qemu_opt_get(opts, "driver");
     if (driver && is_help_option(driver)) {
-        DeviceCategory category;
-        for (category = 0; category < DEVICE_CATEGORY_MAX; ++category) {
-            qdev_print_category_devices(category);
-        }
-
+        qdev_print_devinfos(false);
         return 1;
     }
 
@@ -617,7 +626,7 @@ void do_info_qtree(Monitor *mon, const QDict *qdict)
 
 void do_info_qdm(Monitor *mon, const QDict *qdict)
 {
-    object_class_foreach(qdev_print_devinfo, TYPE_DEVICE, false, NULL);
+    qdev_print_devinfos(true);
 }
 
 int do_device_add(Monitor *mon, const QDict *qdict, QObject **ret_data)
diff --git a/qemu-char.c b/qemu-char.c
index 62594965bd..e00f84c8e9 100644
--- a/qemu-char.c
+++ b/qemu-char.c
@@ -193,6 +193,8 @@ void qemu_chr_fe_printf(CharDriverState *s, const char *fmt, ...)
     va_end(ap);
 }
 
+static void remove_fd_in_watch(CharDriverState *chr);
+
 void qemu_chr_add_handlers(CharDriverState *s,
                            IOCanReadHandler *fd_can_read,
                            IOReadHandler *fd_read,
@@ -203,6 +205,7 @@ void qemu_chr_add_handlers(CharDriverState *s,
 
     if (!opaque && !fd_can_read && !fd_read && !fd_event) {
         fe_open = 0;
+        remove_fd_in_watch(s);
     } else {
         fe_open = 1;
     }
@@ -725,6 +728,14 @@ static void io_remove_watch_poll(guint tag)
     g_source_destroy(&iwp->parent);
 }
 
+static void remove_fd_in_watch(CharDriverState *chr)
+{
+    if (chr->fd_in_tag) {
+        io_remove_watch_poll(chr->fd_in_tag);
+        chr->fd_in_tag = 0;
+    }
+}
+
 #ifndef _WIN32
 static GIOChannel *io_channel_from_fd(int fd)
 {
@@ -798,7 +809,6 @@ static int io_channel_send(GIOChannel *fd, const void *buf, size_t len)
 typedef struct FDCharDriver {
     CharDriverState *chr;
     GIOChannel *fd_in, *fd_out;
-    guint fd_in_tag;
     int max_size;
     QTAILQ_ENTRY(FDCharDriver) node;
 } FDCharDriver;
@@ -830,10 +840,7 @@ static gboolean fd_chr_read(GIOChannel *chan, GIOCondition cond, void *opaque)
     status = g_io_channel_read_chars(chan, (gchar *)buf,
                                      len, &bytes_read, NULL);
     if (status == G_IO_STATUS_EOF) {
-        if (s->fd_in_tag) {
-            io_remove_watch_poll(s->fd_in_tag);
-            s->fd_in_tag = 0;
-        }
+        remove_fd_in_watch(chr);
         qemu_chr_be_event(chr, CHR_EVENT_CLOSED);
         return FALSE;
     }
@@ -863,13 +870,10 @@ static void fd_chr_update_read_handler(CharDriverState *chr)
 {
     FDCharDriver *s = chr->opaque;
 
-    if (s->fd_in_tag) {
-        io_remove_watch_poll(s->fd_in_tag);
-        s->fd_in_tag = 0;
-    }
-
+    remove_fd_in_watch(chr);
     if (s->fd_in) {
-        s->fd_in_tag = io_add_watch_poll(s->fd_in, fd_chr_read_poll, fd_chr_read, chr);
+        chr->fd_in_tag = io_add_watch_poll(s->fd_in, fd_chr_read_poll,
+                                           fd_chr_read, chr);
     }
 }
 
@@ -877,11 +881,7 @@ static void fd_chr_close(struct CharDriverState *chr)
 {
     FDCharDriver *s = chr->opaque;
 
-    if (s->fd_in_tag) {
-        io_remove_watch_poll(s->fd_in_tag);
-        s->fd_in_tag = 0;
-    }
-
+    remove_fd_in_watch(chr);
     if (s->fd_in) {
         g_io_channel_unref(s->fd_in);
     }
@@ -1012,7 +1012,6 @@ static CharDriverState *qemu_chr_open_stdio(ChardevStdio *opts)
 
 typedef struct {
     GIOChannel *fd;
-    guint fd_tag;
     int connected;
     int read_bytes;
     guint timer_tag;
@@ -1026,15 +1025,11 @@ static gboolean pty_chr_timer(gpointer opaque)
     struct CharDriverState *chr = opaque;
     PtyCharDriver *s = chr->opaque;
 
-    if (s->connected) {
-        goto out;
-    }
-
-    /* Next poll ... */
-    pty_chr_update_read_handler(chr);
-
-out:
     s->timer_tag = 0;
+    if (!s->connected) {
+        /* Next poll ... */
+        pty_chr_update_read_handler(chr);
+    }
     return FALSE;
 }
 
@@ -1127,10 +1122,7 @@ static void pty_chr_state(CharDriverState *chr, int connected)
     PtyCharDriver *s = chr->opaque;
 
     if (!connected) {
-        if (s->fd_tag) {
-            io_remove_watch_poll(s->fd_tag);
-            s->fd_tag = 0;
-        }
+        remove_fd_in_watch(chr);
         s->connected = 0;
         /* (re-)connect poll interval for idle guests: once per second.
          * We check more frequently in case the guests sends data to
@@ -1144,7 +1136,8 @@ static void pty_chr_state(CharDriverState *chr, int connected)
         if (!s->connected) {
             s->connected = 1;
             qemu_chr_be_generic_open(chr);
-            s->fd_tag = io_add_watch_poll(s->fd, pty_chr_read_poll, pty_chr_read, chr);
+            chr->fd_in_tag = io_add_watch_poll(s->fd, pty_chr_read_poll,
+                                               pty_chr_read, chr);
         }
     }
 }
@@ -1155,10 +1148,7 @@ static void pty_chr_close(struct CharDriverState *chr)
     PtyCharDriver *s = chr->opaque;
     int fd;
 
-    if (s->fd_tag) {
-        io_remove_watch_poll(s->fd_tag);
-        s->fd_tag = 0;
-    }
+    remove_fd_in_watch(chr);
     fd = g_io_channel_unix_get_fd(s->fd);
     g_io_channel_unref(s->fd);
     close(fd);
@@ -2165,7 +2155,6 @@ static CharDriverState *qemu_chr_open_stdio(ChardevStdio *opts)
 typedef struct {
     int fd;
     GIOChannel *chan;
-    guint tag;
     uint8_t buf[READ_BUF_LEN];
     int bufcnt;
     int bufptr;
@@ -2221,10 +2210,7 @@ static gboolean udp_chr_read(GIOChannel *chan, GIOCondition cond, void *opaque)
     s->bufcnt = bytes_read;
     s->bufptr = s->bufcnt;
     if (status != G_IO_STATUS_NORMAL) {
-        if (s->tag) {
-            io_remove_watch_poll(s->tag);
-            s->tag = 0;
-        }
+        remove_fd_in_watch(chr);
         return FALSE;
     }
 
@@ -2242,23 +2228,18 @@ static void udp_chr_update_read_handler(CharDriverState *chr)
 {
     NetCharDriver *s = chr->opaque;
 
-    if (s->tag) {
-        io_remove_watch_poll(s->tag);
-        s->tag = 0;
-    }
-
+    remove_fd_in_watch(chr);
     if (s->chan) {
-        s->tag = io_add_watch_poll(s->chan, udp_chr_read_poll, udp_chr_read, chr);
+        chr->fd_in_tag = io_add_watch_poll(s->chan, udp_chr_read_poll,
+                                           udp_chr_read, chr);
     }
 }
 
 static void udp_chr_close(CharDriverState *chr)
 {
     NetCharDriver *s = chr->opaque;
-    if (s->tag) {
-        io_remove_watch_poll(s->tag);
-        s->tag = 0;
-    }
+
+    remove_fd_in_watch(chr);
     if (s->chan) {
         g_io_channel_unref(s->chan);
         closesocket(s->fd);
@@ -2308,7 +2289,7 @@ static CharDriverState *qemu_chr_open_udp(QemuOpts *opts)
 typedef struct {
 
     GIOChannel *chan, *listen_chan;
-    guint tag, listen_tag;
+    guint listen_tag;
     int fd, listen_fd;
     int connected;
     int max_size;
@@ -2493,10 +2474,7 @@ static gboolean tcp_chr_read(GIOChannel *chan, GIOCondition cond, void *opaque)
         if (s->listen_chan) {
             s->listen_tag = g_io_add_watch(s->listen_chan, G_IO_IN, tcp_chr_accept, chr);
         }
-        if (s->tag) {
-            io_remove_watch_poll(s->tag);
-            s->tag = 0;
-        }
+        remove_fd_in_watch(chr);
         g_io_channel_unref(s->chan);
         s->chan = NULL;
         closesocket(s->fd);
@@ -2526,7 +2504,8 @@ static void tcp_chr_connect(void *opaque)
 
     s->connected = 1;
     if (s->chan) {
-        s->tag = io_add_watch_poll(s->chan, tcp_chr_read_poll, tcp_chr_read, chr);
+        chr->fd_in_tag = io_add_watch_poll(s->chan, tcp_chr_read_poll,
+                                           tcp_chr_read, chr);
     }
     qemu_chr_be_generic_open(chr);
 }
@@ -2609,10 +2588,7 @@ static void tcp_chr_close(CharDriverState *chr)
 {
     TCPCharDriver *s = chr->opaque;
     if (s->fd >= 0) {
-        if (s->tag) {
-            io_remove_watch_poll(s->tag);
-            s->tag = 0;
-        }
+        remove_fd_in_watch(chr);
         if (s->chan) {
             g_io_channel_unref(s->chan);
         }
@@ -2993,11 +2969,11 @@ QemuOpts *qemu_chr_parse_compat(const char *label, const char *filename)
     if (strstart(filename, "vc", &p)) {
         qemu_opt_set(opts, "backend", "vc");
         if (*p == ':') {
-            if (sscanf(p+1, "%8[0-9]x%8[0-9]", width, height) == 2) {
+            if (sscanf(p+1, "%7[0-9]x%7[0-9]", width, height) == 2) {
                 /* pixels */
                 qemu_opt_set(opts, "width", width);
                 qemu_opt_set(opts, "height", height);
-            } else if (sscanf(p+1, "%8[0-9]Cx%8[0-9]C", width, height) == 2) {
+            } else if (sscanf(p+1, "%7[0-9]Cx%7[0-9]C", width, height) == 2) {
                 /* chars */
                 qemu_opt_set(opts, "cols", width);
                 qemu_opt_set(opts, "rows", height);
@@ -3275,7 +3251,12 @@ CharDriverState *qemu_chr_new_from_opts(QemuOpts *opts,
             backend->kind = CHARDEV_BACKEND_KIND_MUX;
             backend->mux->chardev = g_strdup(bid);
             ret = qmp_chardev_add(id, backend, errp);
-            assert(!error_is_set(errp));
+            if (error_is_set(errp)) {
+                chr = qemu_chr_find(bid);
+                qemu_chr_delete(chr);
+                chr = NULL;
+                goto qapi_out;
+            }
         }
 
         chr = qemu_chr_find(id);
diff --git a/qemu-coroutine.c b/qemu-coroutine.c
index 423430d3a0..470852100a 100644
--- a/qemu-coroutine.c
+++ b/qemu-coroutine.c
@@ -30,15 +30,17 @@ static unsigned int pool_size;
 
 Coroutine *qemu_coroutine_create(CoroutineEntry *entry)
 {
-    Coroutine *co;
-
-    qemu_mutex_lock(&pool_lock);
-    co = QSLIST_FIRST(&pool);
-    if (co) {
-        QSLIST_REMOVE_HEAD(&pool, pool_next);
-        pool_size--;
+    Coroutine *co = NULL;
+
+    if (CONFIG_COROUTINE_POOL) {
+        qemu_mutex_lock(&pool_lock);
+        co = QSLIST_FIRST(&pool);
+        if (co) {
+            QSLIST_REMOVE_HEAD(&pool, pool_next);
+            pool_size--;
+        }
+        qemu_mutex_unlock(&pool_lock);
     }
-    qemu_mutex_unlock(&pool_lock);
 
     if (!co) {
         co = qemu_coroutine_new();
@@ -51,15 +53,17 @@ Coroutine *qemu_coroutine_create(CoroutineEntry *entry)
 
 static void coroutine_delete(Coroutine *co)
 {
-    qemu_mutex_lock(&pool_lock);
-    if (pool_size < POOL_MAX_SIZE) {
-        QSLIST_INSERT_HEAD(&pool, co, pool_next);
-        co->caller = NULL;
-        pool_size++;
+    if (CONFIG_COROUTINE_POOL) {
+        qemu_mutex_lock(&pool_lock);
+        if (pool_size < POOL_MAX_SIZE) {
+            QSLIST_INSERT_HEAD(&pool, co, pool_next);
+            co->caller = NULL;
+            pool_size++;
+            qemu_mutex_unlock(&pool_lock);
+            return;
+        }
         qemu_mutex_unlock(&pool_lock);
-        return;
     }
-    qemu_mutex_unlock(&pool_lock);
 
     qemu_coroutine_delete(co);
 }
diff --git a/qemu-img-cmds.hx b/qemu-img-cmds.hx
index 0c36e5968f..da1d965f3e 100644
--- a/qemu-img-cmds.hx
+++ b/qemu-img-cmds.hx
@@ -67,5 +67,11 @@ DEF("resize", img_resize,
     "resize [-q] filename [+ | -]size")
 STEXI
 @item resize [-q] @var{filename} [+ | -]@var{size}
+ETEXI
+
+DEF("amend", img_amend,
+    "amend [-q] [-f fmt] -o options filename")
+STEXI
+@item amend [-q] [-f @var{fmt}] -o @var{options} @var{filename}
 @end table
 ETEXI
diff --git a/qemu-img.c b/qemu-img.c
index 3e5e388d1c..926f0a0feb 100644
--- a/qemu-img.c
+++ b/qemu-img.c
@@ -266,6 +266,7 @@ static BlockDriverState *bdrv_new_open(const char *filename,
     BlockDriverState *bs;
     BlockDriver *drv;
     char password[256];
+    Error *local_err = NULL;
     int ret;
 
     bs = bdrv_new("image");
@@ -280,9 +281,11 @@ static BlockDriverState *bdrv_new_open(const char *filename,
         drv = NULL;
     }
 
-    ret = bdrv_open(bs, filename, NULL, flags, drv);
+    ret = bdrv_open(bs, filename, NULL, flags, drv, &local_err);
     if (ret < 0) {
-        error_report("Could not open '%s': %s", filename, strerror(-ret));
+        error_report("Could not open '%s': %s", filename,
+                     error_get_pretty(local_err));
+        error_free(local_err);
         goto fail;
     }
 
@@ -409,7 +412,7 @@ static int img_create(int argc, char **argv)
     bdrv_img_create(filename, fmt, base_filename, base_fmt,
                     options, img_size, BDRV_O_FLAGS, &local_err, quiet);
     if (error_is_set(&local_err)) {
-        error_report("%s", error_get_pretty(local_err));
+        error_report("%s: %s", filename, error_get_pretty(local_err));
         error_free(local_err);
         return 1;
     }
@@ -1136,6 +1139,7 @@ static int img_convert(int argc, char **argv)
     float local_progress = 0;
     int min_sparse = 8; /* Need at least 4k of zeros for sparse detection */
     bool quiet = false;
+    Error *local_err = NULL;
 
     fmt = NULL;
     out_fmt = "raw";
@@ -1338,18 +1342,11 @@ static int img_convert(int argc, char **argv)
 
     if (!skip_create) {
         /* Create the new image */
-        ret = bdrv_create(drv, out_filename, param);
+        ret = bdrv_create(drv, out_filename, param, &local_err);
         if (ret < 0) {
-            if (ret == -ENOTSUP) {
-                error_report("Formatting not supported for file format '%s'",
-                             out_fmt);
-            } else if (ret == -EFBIG) {
-                error_report("The image size is too large for file format '%s'",
-                             out_fmt);
-            } else {
-                error_report("%s: error while converting %s: %s",
-                             out_filename, out_fmt, strerror(-ret));
-            }
+            error_report("%s: error while converting %s: %s",
+                         out_filename, out_fmt, error_get_pretty(local_err));
+            error_free(local_err);
             goto out;
         }
     }
@@ -1842,7 +1839,7 @@ static void dump_map_entry(OutputFormat output_format, MapEntry *e,
                (e->flags & BDRV_BLOCK_ZERO) ? "true" : "false",
                (e->flags & BDRV_BLOCK_DATA) ? "true" : "false");
         if (e->flags & BDRV_BLOCK_OFFSET_VALID) {
-            printf(", 'offset': %"PRId64"", e->offset);
+            printf(", \"offset\": %"PRId64"", e->offset);
         }
         putchar('}');
 
@@ -2006,6 +2003,7 @@ static int img_snapshot(int argc, char **argv)
     int action = 0;
     qemu_timeval tv;
     bool quiet = false;
+    Error *err = NULL;
 
     bdrv_oflags = BDRV_O_FLAGS | BDRV_O_RDWR;
     /* Parse commandline parameters */
@@ -2098,10 +2096,12 @@ static int img_snapshot(int argc, char **argv)
         break;
 
     case SNAPSHOT_DELETE:
-        ret = bdrv_snapshot_delete(bs, snapshot_name);
-        if (ret) {
-            error_report("Could not delete snapshot '%s': %d (%s)",
-                snapshot_name, ret, strerror(-ret));
+        bdrv_snapshot_delete_by_id_or_name(bs, snapshot_name, &err);
+        if (error_is_set(&err)) {
+            error_report("Could not delete snapshot '%s': (%s)",
+                         snapshot_name, error_get_pretty(err));
+            error_free(err);
+            ret = 1;
         }
         break;
     }
@@ -2124,6 +2124,7 @@ static int img_rebase(int argc, char **argv)
     int unsafe = 0;
     int progress = 0;
     bool quiet = false;
+    Error *local_err = NULL;
 
     /* Parse commandline parameters */
     fmt = NULL;
@@ -2227,18 +2228,21 @@ static int img_rebase(int argc, char **argv)
         bs_old_backing = bdrv_new("old_backing");
         bdrv_get_backing_filename(bs, backing_name, sizeof(backing_name));
         ret = bdrv_open(bs_old_backing, backing_name, NULL, BDRV_O_FLAGS,
-                        old_backing_drv);
+                        old_backing_drv, &local_err);
         if (ret) {
-            error_report("Could not open old backing file '%s'", backing_name);
+            error_report("Could not open old backing file '%s': %s",
+                         backing_name, error_get_pretty(local_err));
+            error_free(local_err);
             goto out;
         }
         if (out_baseimg[0]) {
             bs_new_backing = bdrv_new("new_backing");
             ret = bdrv_open(bs_new_backing, out_baseimg, NULL, BDRV_O_FLAGS,
-                        new_backing_drv);
+                        new_backing_drv, &local_err);
             if (ret) {
-                error_report("Could not open new backing file '%s'",
-                             out_baseimg);
+                error_report("Could not open new backing file '%s': %s",
+                             out_baseimg, error_get_pretty(local_err));
+                error_free(local_err);
                 goto out;
             }
         }
@@ -2525,6 +2529,90 @@ out:
     return 0;
 }
 
+static int img_amend(int argc, char **argv)
+{
+    int c, ret = 0;
+    char *options = NULL;
+    QEMUOptionParameter *create_options = NULL, *options_param = NULL;
+    const char *fmt = NULL, *filename;
+    bool quiet = false;
+    BlockDriverState *bs = NULL;
+
+    for (;;) {
+        c = getopt(argc, argv, "hqf:o:");
+        if (c == -1) {
+            break;
+        }
+
+        switch (c) {
+            case 'h':
+            case '?':
+                help();
+                break;
+            case 'o':
+                options = optarg;
+                break;
+            case 'f':
+                fmt = optarg;
+                break;
+            case 'q':
+                quiet = true;
+                break;
+        }
+    }
+
+    if (optind != argc - 1) {
+        help();
+    }
+
+    if (!options) {
+        help();
+    }
+
+    filename = argv[argc - 1];
+
+    bs = bdrv_new_open(filename, fmt, BDRV_O_FLAGS | BDRV_O_RDWR, true, quiet);
+    if (!bs) {
+        error_report("Could not open image '%s'", filename);
+        ret = -1;
+        goto out;
+    }
+
+    fmt = bs->drv->format_name;
+
+    if (is_help_option(options)) {
+        ret = print_block_option_help(filename, fmt);
+        goto out;
+    }
+
+    create_options = append_option_parameters(create_options,
+            bs->drv->create_options);
+    options_param = parse_option_parameters(options, create_options,
+            options_param);
+    if (options_param == NULL) {
+        error_report("Invalid options for file format '%s'", fmt);
+        ret = -1;
+        goto out;
+    }
+
+    ret = bdrv_amend_options(bs, options_param);
+    if (ret < 0) {
+        error_report("Error while amending options: %s", strerror(-ret));
+        goto out;
+    }
+
+out:
+    if (bs) {
+        bdrv_unref(bs);
+    }
+    free_option_parameters(create_options);
+    free_option_parameters(options_param);
+    if (ret) {
+        return 1;
+    }
+    return 0;
+}
+
 static const img_cmd_t img_cmds[] = {
 #define DEF(option, callback, arg_string)        \
     { option, callback },
diff --git a/qemu-img.texi b/qemu-img.texi
index 43ee4eb5c4..768054e900 100644
--- a/qemu-img.texi
+++ b/qemu-img.texi
@@ -350,6 +350,11 @@ sizes accordingly.  Failure to do so will result in data loss!
 After using this command to grow a disk image, you must use file system and
 partitioning tools inside the VM to actually begin using the new space on the
 device.
+
+@item amend [-f @var{fmt}] -o @var{options} @var{filename}
+
+Amends the image format specific @var{options} for the image file
+@var{filename}. Not all file formats support this operation.
 @end table
 @c man end
 
diff --git a/qemu-io-cmds.c b/qemu-io-cmds.c
index 8565d49336..667f4e4f3a 100644
--- a/qemu-io-cmds.c
+++ b/qemu-io-cmds.c
@@ -10,6 +10,7 @@
 
 #include "qemu-io.h"
 #include "block/block_int.h"
+#include "block/qapi.h"
 #include "qemu/main-loop.h"
 
 #define CMD_NOFILE_OK   0x01
@@ -1678,6 +1679,7 @@ static const cmdinfo_t length_cmd = {
 static int info_f(BlockDriverState *bs, int argc, char **argv)
 {
     BlockDriverInfo bdi;
+    ImageInfoSpecific *spec_info;
     char s1[64], s2[64];
     int ret;
 
@@ -1699,6 +1701,13 @@ static int info_f(BlockDriverState *bs, int argc, char **argv)
     printf("cluster size: %s\n", s1);
     printf("vm state offset: %s\n", s2);
 
+    spec_info = bdrv_get_specific_info(bs);
+    if (spec_info) {
+        printf("Format specific information:\n");
+        bdrv_image_info_specific_dump(fprintf, stdout, spec_info);
+        qapi_free_ImageInfoSpecific(spec_info);
+    }
+
     return 0;
 }
 
diff --git a/qemu-io.c b/qemu-io.c
index 71f4ff1302..3b3340ab1b 100644
--- a/qemu-io.c
+++ b/qemu-io.c
@@ -16,6 +16,8 @@
 
 #include "qemu-io.h"
 #include "qemu/main-loop.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
 #include "block/block_int.h"
 #include "trace/control.h"
 
@@ -44,23 +46,29 @@ static const cmdinfo_t close_cmd = {
     .oneline    = "close the current open file",
 };
 
-static int openfile(char *name, int flags, int growable)
+static int openfile(char *name, int flags, int growable, QDict *opts)
 {
+    Error *local_err = NULL;
+
     if (qemuio_bs) {
         fprintf(stderr, "file open already, try 'help close'\n");
         return 1;
     }
 
     if (growable) {
-        if (bdrv_file_open(&qemuio_bs, name, NULL, flags)) {
-            fprintf(stderr, "%s: can't open device %s\n", progname, name);
+        if (bdrv_file_open(&qemuio_bs, name, opts, flags, &local_err)) {
+            fprintf(stderr, "%s: can't open device %s: %s\n", progname, name,
+                    error_get_pretty(local_err));
+            error_free(local_err);
             return 1;
         }
     } else {
         qemuio_bs = bdrv_new("hda");
 
-        if (bdrv_open(qemuio_bs, name, NULL, flags, NULL) < 0) {
-            fprintf(stderr, "%s: can't open device %s\n", progname, name);
+        if (bdrv_open(qemuio_bs, name, opts, flags, NULL, &local_err) < 0) {
+            fprintf(stderr, "%s: can't open device %s: %s\n", progname, name,
+                    error_get_pretty(local_err));
+            error_free(local_err);
             bdrv_unref(qemuio_bs);
             qemuio_bs = NULL;
             return 1;
@@ -83,7 +91,8 @@ static void open_help(void)
 " -r, -- open file read-only\n"
 " -s, -- use snapshot file\n"
 " -n, -- disable host cache\n"
-" -g, -- allow file to grow (only applies to protocols)"
+" -g, -- allow file to grow (only applies to protocols)\n"
+" -o, -- options to be given to the block driver"
 "\n");
 }
 
@@ -96,19 +105,30 @@ static const cmdinfo_t open_cmd = {
     .argmin     = 1,
     .argmax     = -1,
     .flags      = CMD_NOFILE_OK,
-    .args       = "[-Crsn] [path]",
+    .args       = "[-Crsn] [-o options] [path]",
     .oneline    = "open the file specified by path",
     .help       = open_help,
 };
 
+static QemuOptsList empty_opts = {
+    .name = "drive",
+    .head = QTAILQ_HEAD_INITIALIZER(empty_opts.head),
+    .desc = {
+        /* no elements => accept any params */
+        { /* end of list */ }
+    },
+};
+
 static int open_f(BlockDriverState *bs, int argc, char **argv)
 {
     int flags = 0;
     int readonly = 0;
     int growable = 0;
     int c;
+    QemuOpts *qopts;
+    QDict *opts = NULL;
 
-    while ((c = getopt(argc, argv, "snrg")) != EOF) {
+    while ((c = getopt(argc, argv, "snrgo:")) != EOF) {
         switch (c) {
         case 's':
             flags |= BDRV_O_SNAPSHOT;
@@ -122,6 +142,15 @@ static int open_f(BlockDriverState *bs, int argc, char **argv)
         case 'g':
             growable = 1;
             break;
+        case 'o':
+            qopts = qemu_opts_parse(&empty_opts, optarg, 0);
+            if (qopts == NULL) {
+                printf("could not parse option list -- %s\n", optarg);
+                return 0;
+            }
+            opts = qemu_opts_to_qdict(qopts, opts);
+            qemu_opts_del(qopts);
+            break;
         default:
             return qemuio_command_usage(&open_cmd);
         }
@@ -135,7 +164,7 @@ static int open_f(BlockDriverState *bs, int argc, char **argv)
         return qemuio_command_usage(&open_cmd);
     }
 
-    return openfile(argv[optind], flags, growable);
+    return openfile(argv[optind], flags, growable, opts);
 }
 
 static int quit_f(BlockDriverState *bs, int argc, char **argv)
@@ -412,7 +441,7 @@ int main(int argc, char **argv)
     }
 
     if ((argc - optind) == 1) {
-        openfile(argv[optind], flags, growable);
+        openfile(argv[optind], flags, growable, NULL);
     }
     command_loop();
 
diff --git a/qemu-nbd.c b/qemu-nbd.c
index f044546c28..c26c98ef1d 100644
--- a/qemu-nbd.c
+++ b/qemu-nbd.c
@@ -355,6 +355,7 @@ int main(int argc, char **argv)
 #endif
     pthread_t client_thread;
     const char *fmt = NULL;
+    Error *local_err = NULL;
 
     /* The client thread uses SIGTERM to interrupt the server.  A signal
      * handler ensures that "qemu-nbd -v -c" exits with a nice status code.
@@ -573,10 +574,11 @@ int main(int argc, char **argv)
 
     bs = bdrv_new("hda");
     srcpath = argv[optind];
-    ret = bdrv_open(bs, srcpath, NULL, flags, drv);
+    ret = bdrv_open(bs, srcpath, NULL, flags, drv, &local_err);
     if (ret < 0) {
         errno = -ret;
-        err(EXIT_FAILURE, "Failed to bdrv_open '%s'", argv[optind]);
+        err(EXIT_FAILURE, "Failed to bdrv_open '%s': %s", argv[optind],
+            error_get_pretty(local_err));
     }
 
     fd_size = bdrv_getlength(bs);
diff --git a/qemu-seccomp.c b/qemu-seccomp.c
index 37d38f881c..69cee443af 100644
--- a/qemu-seccomp.c
+++ b/qemu-seccomp.c
@@ -90,6 +90,7 @@ static const struct QemuSeccompSyscall seccomp_whitelist[] = {
     { SCMP_SYS(getuid), 245 },
     { SCMP_SYS(geteuid), 245 },
     { SCMP_SYS(timer_create), 245 },
+    { SCMP_SYS(times), 245 },
     { SCMP_SYS(exit), 245 },
     { SCMP_SYS(clock_gettime), 245 },
     { SCMP_SYS(time), 245 },
diff --git a/qemu-timer.c b/qemu-timer.c
index 95ff47fef3..e15ce477cc 100644
--- a/qemu-timer.c
+++ b/qemu-timer.c
@@ -45,6 +45,7 @@
 /* timers */
 
 typedef struct QEMUClock {
+    /* We rely on BQL to protect the timerlists */
     QLIST_HEAD(, QEMUTimerList) timerlists;
 
     NotifierList reset_notifiers;
@@ -66,10 +67,14 @@ QEMUClock qemu_clocks[QEMU_CLOCK_MAX];
 
 struct QEMUTimerList {
     QEMUClock *clock;
+    QemuMutex active_timers_lock;
     QEMUTimer *active_timers;
     QLIST_ENTRY(QEMUTimerList) list;
     QEMUTimerListNotifyCB *notify_cb;
     void *notify_opaque;
+
+    /* lightweight method to mark the end of timerlist's running */
+    QemuEvent timers_done_ev;
 };
 
 /**
@@ -98,9 +103,11 @@ QEMUTimerList *timerlist_new(QEMUClockType type,
     QEMUClock *clock = qemu_clock_ptr(type);
 
     timer_list = g_malloc0(sizeof(QEMUTimerList));
+    qemu_event_init(&timer_list->timers_done_ev, false);
     timer_list->clock = clock;
     timer_list->notify_cb = cb;
     timer_list->notify_opaque = opaque;
+    qemu_mutex_init(&timer_list->active_timers_lock);
     QLIST_INSERT_HEAD(&clock->timerlists, timer_list, list);
     return timer_list;
 }
@@ -111,6 +118,7 @@ void timerlist_free(QEMUTimerList *timer_list)
     if (timer_list->clock) {
         QLIST_REMOVE(timer_list, list);
     }
+    qemu_mutex_destroy(&timer_list->active_timers_lock);
     g_free(timer_list);
 }
 
@@ -140,13 +148,25 @@ void qemu_clock_notify(QEMUClockType type)
     }
 }
 
+/* Disabling the clock will wait for related timerlists to stop
+ * executing qemu_run_timers.  Thus, this functions should not
+ * be used from the callback of a timer that is based on @clock.
+ * Doing so would cause a deadlock.
+ *
+ * Caller should hold BQL.
+ */
 void qemu_clock_enable(QEMUClockType type, bool enabled)
 {
     QEMUClock *clock = qemu_clock_ptr(type);
+    QEMUTimerList *tl;
     bool old = clock->enabled;
     clock->enabled = enabled;
     if (enabled && !old) {
         qemu_clock_notify(type);
+    } else if (!enabled && old) {
+        QLIST_FOREACH(tl, &clock->timerlists, list) {
+            qemu_event_wait(&tl->timers_done_ev);
+        }
     }
 }
 
@@ -163,9 +183,17 @@ bool qemu_clock_has_timers(QEMUClockType type)
 
 bool timerlist_expired(QEMUTimerList *timer_list)
 {
-    return (timer_list->active_timers &&
-            timer_list->active_timers->expire_time <
-            qemu_clock_get_ns(timer_list->clock->type));
+    int64_t expire_time;
+
+    qemu_mutex_lock(&timer_list->active_timers_lock);
+    if (!timer_list->active_timers) {
+        qemu_mutex_unlock(&timer_list->active_timers_lock);
+        return false;
+    }
+    expire_time = timer_list->active_timers->expire_time;
+    qemu_mutex_unlock(&timer_list->active_timers_lock);
+
+    return expire_time < qemu_clock_get_ns(timer_list->clock->type);
 }
 
 bool qemu_clock_expired(QEMUClockType type)
@@ -182,13 +210,25 @@ bool qemu_clock_expired(QEMUClockType type)
 int64_t timerlist_deadline_ns(QEMUTimerList *timer_list)
 {
     int64_t delta;
+    int64_t expire_time;
 
-    if (!timer_list->clock->enabled || !timer_list->active_timers) {
+    if (!timer_list->clock->enabled) {
         return -1;
     }
 
-    delta = timer_list->active_timers->expire_time -
-        qemu_clock_get_ns(timer_list->clock->type);
+    /* The active timers list may be modified before the caller uses our return
+     * value but ->notify_cb() is called when the deadline changes.  Therefore
+     * the caller should notice the change and there is no race condition.
+     */
+    qemu_mutex_lock(&timer_list->active_timers_lock);
+    if (!timer_list->active_timers) {
+        qemu_mutex_unlock(&timer_list->active_timers_lock);
+        return -1;
+    }
+    expire_time = timer_list->active_timers->expire_time;
+    qemu_mutex_unlock(&timer_list->active_timers_lock);
+
+    delta = expire_time - qemu_clock_get_ns(timer_list->clock->type);
 
     if (delta <= 0) {
         return 0;
@@ -289,6 +329,7 @@ void timer_init(QEMUTimer *ts,
     ts->cb = cb;
     ts->opaque = opaque;
     ts->scale = scale;
+    ts->expire_time = -1;
 }
 
 void timer_free(QEMUTimer *ts)
@@ -296,14 +337,12 @@ void timer_free(QEMUTimer *ts)
     g_free(ts);
 }
 
-/* stop a timer, but do not dealloc it */
-void timer_del(QEMUTimer *ts)
+static void timer_del_locked(QEMUTimerList *timer_list, QEMUTimer *ts)
 {
     QEMUTimer **pt, *t;
 
-    /* NOTE: this code must be signal safe because
-       timer_expired() can be called from a signal. */
-    pt = &ts->timer_list->active_timers;
+    ts->expire_time = -1;
+    pt = &timer_list->active_timers;
     for(;;) {
         t = *pt;
         if (!t)
@@ -316,34 +355,82 @@ void timer_del(QEMUTimer *ts)
     }
 }
 
-/* modify the current timer so that it will be fired when current_time
-   >= expire_time. The corresponding callback will be called. */
-void timer_mod_ns(QEMUTimer *ts, int64_t expire_time)
+static bool timer_mod_ns_locked(QEMUTimerList *timer_list,
+                                QEMUTimer *ts, int64_t expire_time)
 {
     QEMUTimer **pt, *t;
 
-    timer_del(ts);
-
     /* add the timer in the sorted list */
-    /* NOTE: this code must be signal safe because
-       timer_expired() can be called from a signal. */
-    pt = &ts->timer_list->active_timers;
-    for(;;) {
+    pt = &timer_list->active_timers;
+    for (;;) {
         t = *pt;
         if (!timer_expired_ns(t, expire_time)) {
             break;
         }
         pt = &t->next;
     }
-    ts->expire_time = expire_time;
+    ts->expire_time = MAX(expire_time, 0);
     ts->next = *pt;
     *pt = ts;
 
-    /* Rearm if necessary  */
-    if (pt == &ts->timer_list->active_timers) {
-        /* Interrupt execution to force deadline recalculation.  */
-        qemu_clock_warp(ts->timer_list->clock->type);
-        timerlist_notify(ts->timer_list);
+    return pt == &timer_list->active_timers;
+}
+
+static void timerlist_rearm(QEMUTimerList *timer_list)
+{
+    /* Interrupt execution to force deadline recalculation.  */
+    qemu_clock_warp(timer_list->clock->type);
+    timerlist_notify(timer_list);
+}
+
+/* stop a timer, but do not dealloc it */
+void timer_del(QEMUTimer *ts)
+{
+    QEMUTimerList *timer_list = ts->timer_list;
+
+    qemu_mutex_lock(&timer_list->active_timers_lock);
+    timer_del_locked(timer_list, ts);
+    qemu_mutex_unlock(&timer_list->active_timers_lock);
+}
+
+/* modify the current timer so that it will be fired when current_time
+   >= expire_time. The corresponding callback will be called. */
+void timer_mod_ns(QEMUTimer *ts, int64_t expire_time)
+{
+    QEMUTimerList *timer_list = ts->timer_list;
+    bool rearm;
+
+    qemu_mutex_lock(&timer_list->active_timers_lock);
+    timer_del_locked(timer_list, ts);
+    rearm = timer_mod_ns_locked(timer_list, ts, expire_time);
+    qemu_mutex_unlock(&timer_list->active_timers_lock);
+
+    if (rearm) {
+        timerlist_rearm(timer_list);
+    }
+}
+
+/* modify the current timer so that it will be fired when current_time
+   >= expire_time or the current deadline, whichever comes earlier.
+   The corresponding callback will be called. */
+void timer_mod_anticipate_ns(QEMUTimer *ts, int64_t expire_time)
+{
+    QEMUTimerList *timer_list = ts->timer_list;
+    bool rearm;
+
+    qemu_mutex_lock(&timer_list->active_timers_lock);
+    if (ts->expire_time == -1 || ts->expire_time > expire_time) {
+        if (ts->expire_time != -1) {
+            timer_del_locked(timer_list, ts);
+        }
+        rearm = timer_mod_ns_locked(timer_list, ts, expire_time);
+    } else {
+        rearm = false;
+    }
+    qemu_mutex_unlock(&timer_list->active_timers_lock);
+
+    if (rearm) {
+        timerlist_rearm(timer_list);
     }
 }
 
@@ -352,15 +439,14 @@ void timer_mod(QEMUTimer *ts, int64_t expire_time)
     timer_mod_ns(ts, expire_time * ts->scale);
 }
 
+void timer_mod_anticipate(QEMUTimer *ts, int64_t expire_time)
+{
+    timer_mod_anticipate_ns(ts, expire_time * ts->scale);
+}
+
 bool timer_pending(QEMUTimer *ts)
 {
-    QEMUTimer *t;
-    for (t = ts->timer_list->active_timers; t != NULL; t = t->next) {
-        if (t == ts) {
-            return true;
-        }
-    }
-    return false;
+    return ts->expire_time >= 0;
 }
 
 bool timer_expired(QEMUTimer *timer_head, int64_t current_time)
@@ -373,25 +459,38 @@ bool timerlist_run_timers(QEMUTimerList *timer_list)
     QEMUTimer *ts;
     int64_t current_time;
     bool progress = false;
-   
+    QEMUTimerCB *cb;
+    void *opaque;
+
+    qemu_event_reset(&timer_list->timers_done_ev);
     if (!timer_list->clock->enabled) {
-        return progress;
+        goto out;
     }
 
     current_time = qemu_clock_get_ns(timer_list->clock->type);
     for(;;) {
+        qemu_mutex_lock(&timer_list->active_timers_lock);
         ts = timer_list->active_timers;
         if (!timer_expired_ns(ts, current_time)) {
+            qemu_mutex_unlock(&timer_list->active_timers_lock);
             break;
         }
+
         /* remove timer from the list before calling the callback */
         timer_list->active_timers = ts->next;
         ts->next = NULL;
+        ts->expire_time = -1;
+        cb = ts->cb;
+        opaque = ts->opaque;
+        qemu_mutex_unlock(&timer_list->active_timers_lock);
 
         /* run the callback (the timer list can be modified) */
-        ts->cb(ts->opaque);
+        cb(opaque);
         progress = true;
     }
+
+out:
+    qemu_event_set(&timer_list->timers_done_ev);
     return progress;
 }
 
diff --git a/qemu.nsi b/qemu.nsi
index 1d57455956..0dc1f52693 100644
--- a/qemu.nsi
+++ b/qemu.nsi
@@ -20,7 +20,7 @@
 ; NSIS_WIN32_MAKENSIS
 
 !define PRODUCT "QEMU"
-!define URL     "http://www.qemu.org/"
+!define URL     "http://www.qemu-project.org/"
 
 !define UNINST_EXE "$INSTDIR\qemu-uninstall.exe"
 !define UNINST_KEY "Software\Microsoft\Windows\CurrentVersion\Uninstall\${PRODUCT}"
diff --git a/qga/commands-posix.c b/qga/commands-posix.c
index e199738c71..f453132b92 100644
--- a/qga/commands-posix.c
+++ b/qga/commands-posix.c
@@ -566,7 +566,7 @@ typedef struct FsMount {
     QTAILQ_ENTRY(FsMount) next;
 } FsMount;
 
-typedef QTAILQ_HEAD(, FsMount) FsMountList;
+typedef QTAILQ_HEAD(FsMountList, FsMount) FsMountList;
 
 static void free_fs_mount_list(FsMountList *mounts)
 {
@@ -728,7 +728,7 @@ int64_t qmp_guest_fsfreeze_freeze(Error **err)
     /* cannot risk guest agent blocking itself on a write in this state */
     ga_set_frozen(ga_state);
 
-    QTAILQ_FOREACH(mount, &mounts, next) {
+    QTAILQ_FOREACH_REVERSE(mount, &mounts, FsMountList, next) {
         fd = qemu_open(mount->dirname, O_RDONLY);
         if (fd == -1) {
             error_setg_errno(err, errno, "failed to open %s", mount->dirname);
diff --git a/qga/commands.c b/qga/commands.c
index 528b082fa8..a0c2de07ec 100644
--- a/qga/commands.c
+++ b/qga/commands.c
@@ -45,35 +45,28 @@ void qmp_guest_ping(Error **err)
     slog("guest-ping called");
 }
 
-struct GuestAgentInfo *qmp_guest_info(Error **err)
+static void qmp_command_info(QmpCommand *cmd, void *opaque)
 {
-    GuestAgentInfo *info = g_malloc0(sizeof(GuestAgentInfo));
+    GuestAgentInfo *info = opaque;
     GuestAgentCommandInfo *cmd_info;
     GuestAgentCommandInfoList *cmd_info_list;
-    char **cmd_list_head, **cmd_list;
-
-    info->version = g_strdup(QEMU_VERSION);
-
-    cmd_list_head = cmd_list = qmp_get_command_list();
-    if (*cmd_list_head == NULL) {
-        goto out;
-    }
 
-    while (*cmd_list) {
-        cmd_info = g_malloc0(sizeof(GuestAgentCommandInfo));
-        cmd_info->name = g_strdup(*cmd_list);
-        cmd_info->enabled = qmp_command_is_enabled(cmd_info->name);
+    cmd_info = g_malloc0(sizeof(GuestAgentCommandInfo));
+    cmd_info->name = g_strdup(qmp_command_name(cmd));
+    cmd_info->enabled = qmp_command_is_enabled(cmd);
+    cmd_info->success_response = qmp_has_success_response(cmd);
 
-        cmd_info_list = g_malloc0(sizeof(GuestAgentCommandInfoList));
-        cmd_info_list->value = cmd_info;
-        cmd_info_list->next = info->supported_commands;
-        info->supported_commands = cmd_info_list;
+    cmd_info_list = g_malloc0(sizeof(GuestAgentCommandInfoList));
+    cmd_info_list->value = cmd_info;
+    cmd_info_list->next = info->supported_commands;
+    info->supported_commands = cmd_info_list;
+}
 
-        g_free(*cmd_list);
-        cmd_list++;
-    }
+struct GuestAgentInfo *qmp_guest_info(Error **err)
+{
+    GuestAgentInfo *info = g_malloc0(sizeof(GuestAgentInfo));
 
-out:
-    g_free(cmd_list_head);
+    info->version = g_strdup(QEMU_VERSION);
+    qmp_for_each_command(qmp_command_info, info);
     return info;
 }
diff --git a/qga/main.c b/qga/main.c
index 6c746c8f3a..c58b26a9a0 100644
--- a/qga/main.c
+++ b/qga/main.c
@@ -347,48 +347,35 @@ static gint ga_strcmp(gconstpointer str1, gconstpointer str2)
 }
 
 /* disable commands that aren't safe for fsfreeze */
-static void ga_disable_non_whitelisted(void)
+static void ga_disable_non_whitelisted(QmpCommand *cmd, void *opaque)
 {
-    char **list_head, **list;
-    bool whitelisted;
-    int i;
-
-    list_head = list = qmp_get_command_list();
-    while (*list != NULL) {
-        whitelisted = false;
-        i = 0;
-        while (ga_freeze_whitelist[i] != NULL) {
-            if (strcmp(*list, ga_freeze_whitelist[i]) == 0) {
-                whitelisted = true;
-            }
-            i++;
-        }
-        if (!whitelisted) {
-            g_debug("disabling command: %s", *list);
-            qmp_disable_command(*list);
+    bool whitelisted = false;
+    int i = 0;
+    const char *name = qmp_command_name(cmd);
+
+    while (ga_freeze_whitelist[i] != NULL) {
+        if (strcmp(name, ga_freeze_whitelist[i]) == 0) {
+            whitelisted = true;
         }
-        g_free(*list);
-        list++;
+        i++;
+    }
+    if (!whitelisted) {
+        g_debug("disabling command: %s", name);
+        qmp_disable_command(name);
     }
-    g_free(list_head);
 }
 
 /* [re-]enable all commands, except those explicitly blacklisted by user */
-static void ga_enable_non_blacklisted(GList *blacklist)
+static void ga_enable_non_blacklisted(QmpCommand *cmd, void *opaque)
 {
-    char **list_head, **list;
-
-    list_head = list = qmp_get_command_list();
-    while (*list != NULL) {
-        if (g_list_find_custom(blacklist, *list, ga_strcmp) == NULL &&
-            !qmp_command_is_enabled(*list)) {
-            g_debug("enabling command: %s", *list);
-            qmp_enable_command(*list);
-        }
-        g_free(*list);
-        list++;
+    GList *blacklist = opaque;
+    const char *name = qmp_command_name(cmd);
+
+    if (g_list_find_custom(blacklist, name, ga_strcmp) == NULL &&
+        !qmp_command_is_enabled(cmd)) {
+        g_debug("enabling command: %s", name);
+        qmp_enable_command(name);
     }
-    g_free(list_head);
 }
 
 static bool ga_create_file(const char *path)
@@ -424,7 +411,7 @@ void ga_set_frozen(GAState *s)
         return;
     }
     /* disable all non-whitelisted (for frozen state) commands */
-    ga_disable_non_whitelisted();
+    qmp_for_each_command(ga_disable_non_whitelisted, NULL);
     g_warning("disabling logging due to filesystem freeze");
     ga_disable_logging(s);
     s->frozen = true;
@@ -460,7 +447,7 @@ void ga_unset_frozen(GAState *s)
     }
 
     /* enable all disabled, non-blacklisted commands */
-    ga_enable_non_blacklisted(s->blacklist);
+    qmp_for_each_command(ga_enable_non_blacklisted, s->blacklist);
     s->frozen = false;
     if (!ga_delete_file(s->state_filepath_isfrozen)) {
         g_warning("unable to delete %s, fsfreeze may not function properly",
@@ -920,6 +907,11 @@ int64_t ga_get_fd_handle(GAState *s, Error **errp)
     return handle;
 }
 
+static void ga_print_cmd(QmpCommand *cmd, void *opaque)
+{
+    printf("%s\n", qmp_command_name(cmd));
+}
+
 int main(int argc, char **argv)
 {
     const char *sopt = "hVvdm:p:l:f:F::b:s:t:";
@@ -996,15 +988,8 @@ int main(int argc, char **argv)
             daemonize = 1;
             break;
         case 'b': {
-            char **list_head, **list;
             if (is_help_option(optarg)) {
-                list_head = list = qmp_get_command_list();
-                while (*list != NULL) {
-                    printf("%s\n", *list);
-                    g_free(*list);
-                    list++;
-                }
-                g_free(list_head);
+                qmp_for_each_command(ga_print_cmd, NULL);
                 return 0;
             }
             for (j = 0, i = 0, len = strlen(optarg); i < len; i++) {
@@ -1126,7 +1111,7 @@ int main(int argc, char **argv)
             s->deferred_options.log_filepath = log_filepath;
         }
         ga_disable_logging(s);
-        ga_disable_non_whitelisted();
+        qmp_for_each_command(ga_disable_non_whitelisted, NULL);
     } else {
         if (daemonize) {
             become_daemon(pid_filepath);
diff --git a/qga/qapi-schema.json b/qga/qapi-schema.json
index 7155b7ab55..245f968bc2 100644
--- a/qga/qapi-schema.json
+++ b/qga/qapi-schema.json
@@ -141,10 +141,13 @@
 #
 # @enabled: whether command is currently enabled by guest admin
 #
+# @success-response: whether command returns a response on success
+#                    (since 1.7)
+#
 # Since 1.1.0
 ##
 { 'type': 'GuestAgentCommandInfo',
-  'data': { 'name': 'str', 'enabled': 'bool' } }
+  'data': { 'name': 'str', 'enabled': 'bool', 'success-response': 'bool' } }
 
 ##
 # @GuestAgentInfo
diff --git a/qmp-commands.hx b/qmp-commands.hx
index 008cad95a2..fba15cdc3b 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -1001,14 +1001,15 @@ SQMP
 transaction
 -----------
 
-Atomically operate on one or more block devices.  The only supported
-operation for now is snapshotting.  If there is any failure performing
-any of the operations, all snapshots for the group are abandoned, and
-the original disks pre-snapshot attempt are used.
+Atomically operate on one or more block devices.  The only supported operations
+for now are drive-backup, internal and external snapshotting.  A list of
+dictionaries is accepted, that contains the actions to be performed.
+If there is any failure performing any of the operations, all operations
+for the group are abandoned.
 
-A list of dictionaries is accepted, that contains the actions to be performed.
-For snapshots this is the device, the file to use for the new snapshot,
-and the format.  The default format, if not specified, is qcow2.
+For external snapshots, the dictionary contains the device, the file to use for
+the new snapshot, and the format.  The default format, if not specified, is
+qcow2.
 
 Each new snapshot defaults to being created by QEMU (wiping any
 contents if the file already exists), but it is also possible to reuse
@@ -1017,6 +1018,17 @@ the new image file has the same contents as the current one; QEMU cannot
 perform any meaningful check.  Typically this is achieved by using the
 current image file as the backing file for the new image.
 
+On failure, the original disks pre-snapshot attempt will be used.
+
+For internal snapshots, the dictionary contains the device and the snapshot's
+name.  If an internal snapshot matching name already exists, the request will
+be rejected.  Only some image formats support it, for example, qcow2, rbd,
+and sheepdog.
+
+On failure, qemu will try delete the newly created internal snapshot in the
+transaction.  When an I/O error occurs during deletion, the user needs to fix
+it later with qemu-img or other command.
+
 Arguments:
 
 actions array:
@@ -1029,6 +1041,9 @@ actions array:
       - "format": format of new image (json-string, optional)
       - "mode": whether and how QEMU should create the snapshot file
         (NewImageMode, optional, default "absolute-paths")
+      When "type" is "blockdev-snapshot-internal-sync":
+      - "device": device name to snapshot (json-string)
+      - "name": name of the new snapshot (json-string)
 
 Example:
 
@@ -1040,7 +1055,10 @@ Example:
          { 'type': 'blockdev-snapshot-sync', 'data' : { "device": "ide-hd1",
                                          "snapshot-file": "/some/place/my-image2",
                                          "mode": "existing",
-                                         "format": "qcow2" } } ] } }
+                                         "format": "qcow2" } },
+         { 'type': 'blockdev-snapshot-internal-sync', 'data' : {
+                                         "device": "ide-hd2",
+                                         "name": "snapshot0" } } ] } }
 <- { "return": {} }
 
 EQMP
@@ -1080,6 +1098,76 @@ Example:
 EQMP
 
     {
+        .name       = "blockdev-snapshot-internal-sync",
+        .args_type  = "device:B,name:s",
+        .mhandler.cmd_new = qmp_marshal_input_blockdev_snapshot_internal_sync,
+    },
+
+SQMP
+blockdev-snapshot-internal-sync
+-------------------------------
+
+Synchronously take an internal snapshot of a block device when the format of
+image used supports it.  If the name is an empty string, or a snapshot with
+name already exists, the operation will fail.
+
+Arguments:
+
+- "device": device name to snapshot (json-string)
+- "name": name of the new snapshot (json-string)
+
+Example:
+
+-> { "execute": "blockdev-snapshot-internal-sync",
+                "arguments": { "device": "ide-hd0",
+                               "name": "snapshot0" }
+   }
+<- { "return": {} }
+
+EQMP
+
+    {
+        .name       = "blockdev-snapshot-delete-internal-sync",
+        .args_type  = "device:B,id:s?,name:s?",
+        .mhandler.cmd_new =
+                      qmp_marshal_input_blockdev_snapshot_delete_internal_sync,
+    },
+
+SQMP
+blockdev-snapshot-delete-internal-sync
+--------------------------------------
+
+Synchronously delete an internal snapshot of a block device when the format of
+image used supports it.  The snapshot is identified by name or id or both.  One
+of name or id is required.  If the snapshot is not found, the operation will
+fail.
+
+Arguments:
+
+- "device": device name (json-string)
+- "id": ID of the snapshot (json-string, optional)
+- "name": name of the snapshot (json-string, optional)
+
+Example:
+
+-> { "execute": "blockdev-snapshot-delete-internal-sync",
+                "arguments": { "device": "ide-hd0",
+                               "name": "snapshot0" }
+   }
+<- { "return": {
+                   "id": "1",
+                   "name": "snapshot0",
+                   "vm-state-size": 0,
+                   "date-sec": 1000012,
+                   "date-nsec": 10,
+                   "vm-clock-sec": 100,
+                   "vm-clock-nsec": 20
+     }
+   }
+
+EQMP
+
+    {
         .name       = "drive-mirror",
         .args_type  = "sync:s,device:B,target:s,speed:i?,mode:s?,format:s?,"
                       "on-source-error:s?,on-target-error:s?,"
@@ -3152,3 +3240,58 @@ Example:
    }
 
 EQMP
+
+    {
+        .name       = "blockdev-add",
+        .args_type  = "options:q",
+        .mhandler.cmd_new = qmp_marshal_input_blockdev_add,
+    },
+
+SQMP
+blockdev-add
+------------
+
+Add a block device.
+
+Arguments:
+
+- "options": block driver options
+
+Example (1):
+
+-> { "execute": "blockdev-add",
+    "arguments": { "options" : { "driver": "qcow2",
+                                 "file": { "driver": "file",
+                                           "filename": "test.qcow2" } } } }
+<- { "return": {} }
+
+Example (2):
+
+-> { "execute": "blockdev-add",
+     "arguments": {
+         "options": {
+           "driver": "qcow2",
+           "id": "my_disk",
+           "discard": "unmap",
+           "cache": {
+               "direct": true,
+               "writeback": true
+           },
+           "file": {
+               "driver": "file",
+               "filename": "/tmp/test.qcow2"
+           },
+           "backing": {
+               "driver": "raw",
+               "file": {
+                   "driver": "file",
+                   "filename": "/dev/fdset/4"
+               }
+           }
+         }
+       }
+     }
+
+<- { "return": {} }
+
+EQMP
diff --git a/qobject/qdict.c b/qobject/qdict.c
index 472f106e27..0f3e0a6c81 100644
--- a/qobject/qdict.c
+++ b/qobject/qdict.c
@@ -527,3 +527,24 @@ void qdict_flatten(QDict *qdict)
 {
     qdict_do_flatten(qdict, qdict, NULL);
 }
+
+/* extract all the src QDict entries starting by start into dst */
+void qdict_extract_subqdict(QDict *src, QDict **dst, const char *start)
+
+{
+    const QDictEntry *entry, *next;
+    const char *p;
+
+    *dst = qdict_new();
+    entry = qdict_first(src);
+
+    while (entry != NULL) {
+        next = qdict_next(src, entry);
+        if (strstart(entry->key, start, &p)) {
+            qobject_incref(entry->value);
+            qdict_put_obj(*dst, p, entry->value);
+            qdict_del(src, entry->key);
+        }
+        entry = next;
+    }
+}
diff --git a/qom/cpu.c b/qom/cpu.c
index fa7ec6b199..818fb26dd4 100644
--- a/qom/cpu.c
+++ b/qom/cpu.c
@@ -162,6 +162,7 @@ void cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
     CPUClass *cc = CPU_GET_CLASS(cpu);
 
     if (cc->dump_state) {
+        cpu_synchronize_state(cpu);
         cc->dump_state(cpu, f, cpu_fprintf, flags);
     }
 }
diff --git a/roms/Makefile b/roms/Makefile
index 7a228aed8f..10d5a65d61 100644
--- a/roms/Makefile
+++ b/roms/Makefile
@@ -1,6 +1,8 @@
 
-vgabios_variants := stdvga cirrus vmware qxl
+vgabios_variants := stdvga cirrus vmware qxl isavga
+vgabios_targets  := $(subst -isavga,,$(patsubst %,vgabios-%.bin,$(vgabios_variants)))
 pxerom_variants  := e1000 eepro100 ne2k_pci pcnet rtl8139 virtio
+pxerom_targets   := 8086100e 80861209 10500940 10222000 10ec8139 1af41000
 
 pxe-rom-e1000    efi-rom-e1000    : VID := 8086
 pxe-rom-e1000    efi-rom-e1000    : DID := 100e
@@ -16,6 +18,22 @@ pxe-rom-virtio   efi-rom-virtio   : VID := 1af4
 pxe-rom-virtio   efi-rom-virtio   : DID := 1000
 
 #
+# cross compiler auto detection
+#
+path := $(subst :, ,$(PATH))
+system := $(shell uname -s | tr "A-Z" "a-z")
+
+# first find cross binutils in path
+find-cross-ld = $(firstword $(wildcard $(patsubst %,%/$(1)-*$(system)*-ld,$(path))))
+# then check we have cross gcc too
+find-cross-gcc = $(firstword $(wildcard $(patsubst %ld,%gcc,$(call find-cross-ld,$(1)))))
+# finally strip off path + toolname so we get the prefix
+find-cross-prefix = $(subst gcc,,$(notdir $(call find-cross-gcc,$(1))))
+
+powerpc64_cross_prefix := $(call find-cross-prefix,powerpc64)
+x86_64_cross_prefix := $(call find-cross-prefix,x86_64)
+
+#
 # EfiRom utility is shipped with edk2 / tianocore, in BaseTools/
 #
 # We need that to combine multiple images (legacy bios,
@@ -32,46 +50,91 @@ default:
 	@echo "  bios           -- update bios.bin (seabios)"
 	@echo "  seavgabios     -- update vgabios binaries (seabios)"
 	@echo "  lgplvgabios    -- update vgabios binaries (lgpl)"
+	@echo "  sgabios        -- update sgabios binaries"
 	@echo "  pxerom         -- update nic roms (bios only)"
 	@echo "  efirom         -- update nic roms (bios+efi, this needs"
 	@echo "                    the EfiRom utility from edk2 / tianocore)"
+	@echo "  slof           -- update slof.bin"
 
-bios: config.seabios
-	sh configure-seabios.sh $<
-	make -C seabios out/bios.bin
-	cp seabios/out/bios.bin ../pc-bios/bios.bin
-	cp seabios/out/*dsdt.aml ../pc-bios/
+bios: build-seabios-config-seabios
+	cp seabios/builds/seabios/bios.bin ../pc-bios/bios.bin
+	cp seabios/builds/seabios/*dsdt.aml ../pc-bios/
 
 seavgabios: $(patsubst %,seavgabios-%,$(vgabios_variants))
 
-seavgabios-%: config.vga.%
-	sh configure-seabios.sh $<
-	make -C seabios out/vgabios.bin
-	cp seabios/out/vgabios.bin ../pc-bios/vgabios-$*.bin
+seavgabios-isavga: build-seabios-config-vga-isavga
+	cp seabios/builds/vga-isavga/vgabios.bin ../pc-bios/vgabios.bin
+
+seavgabios-%: build-seabios-config-vga-%
+	cp seabios/builds/vga-$*/vgabios.bin ../pc-bios/vgabios-$*.bin
+
+build-seabios-config-%: config.%
+	mkdir -p seabios/builds/$*
+	cp $< seabios/builds/$*/.config
+	$(MAKE) $(MAKEFLAGS) -C seabios \
+		KCONFIG_CONFIG=$(CURDIR)/seabios/builds/$*/.config \
+		OUT=$(CURDIR)/seabios/builds/$*/ oldnoconfig
+	$(MAKE) $(MAKEFLAGS) -C seabios \
+		KCONFIG_CONFIG=$(CURDIR)/seabios/builds/$*/.config \
+		OUT=$(CURDIR)/seabios/builds/$*/ all
+
 
 lgplvgabios: $(patsubst %,lgplvgabios-%,$(vgabios_variants))
 
-lgplvgabios-%:
-	make -C vgabios vgabios-$*.bin
+lgplvgabios-isavga: build-lgplvgabios
+	cp vgabios/VGABIOS-lgpl-latest.bin ../pc-bios/vgabios.bin
+lgplvgabios-%: build-lgplvgabios
 	cp vgabios/VGABIOS-lgpl-latest.$*.bin ../pc-bios/vgabios-$*.bin
 
+build-lgplvgabios:
+	$(MAKE) $(MAKEFLAGS) -C vgabios $(vgabios_targets)
+
+
+.PHONY: sgabios
+sgabios:
+	$(MAKE) $(MAKEFLAGS) -C sgabios
+	cp sgabios/sgabios.bin ../pc-bios
+
+
 pxerom: $(patsubst %,pxe-rom-%,$(pxerom_variants))
 
-pxe-rom-%: ipxe/src/config/local/general.h
-	make -C ipxe/src bin/$(VID)$(DID).rom
+pxe-rom-%: build-pxe-roms
 	cp ipxe/src/bin/$(VID)$(DID).rom ../pc-bios/pxe-$*.rom
 
 efirom: $(patsubst %,efi-rom-%,$(pxerom_variants))
 
-efi-rom-%: ipxe/src/config/local/general.h
-	make -C ipxe/src bin/$(VID)$(DID).rom
-	make -C ipxe/src bin-i386-efi/$(VID)$(DID).efidrv
-	make -C ipxe/src bin-x86_64-efi/$(VID)$(DID).efidrv
+efi-rom-%: build-pxe-roms build-efi-roms
 	$(EFIROM) -f "0x$(VID)" -i "0x$(DID)" -l 0x02 \
 		-b ipxe/src/bin/$(VID)$(DID).rom \
 		-ec ipxe/src/bin-i386-efi/$(VID)$(DID).efidrv \
 		-ec ipxe/src/bin-x86_64-efi/$(VID)$(DID).efidrv \
 		-o ../pc-bios/efi-$*.rom
 
+build-pxe-roms: ipxe/src/config/local/general.h
+	$(MAKE) $(MAKEFLAGS) -C ipxe/src GITVERSION="" \
+		CROSS_COMPILE=$(x86_64_cross_prefix) \
+		$(patsubst %,bin/%.rom,$(pxerom_targets))
+
+build-efi-roms: build-pxe-roms ipxe/src/config/local/general.h
+	$(MAKE) $(MAKEFLAGS) -C ipxe/src GITVERSION="" \
+		CROSS_COMPILE=$(x86_64_cross_prefix) \
+		$(patsubst %,bin-i386-efi/%.efidrv,$(pxerom_targets)) \
+		$(patsubst %,bin-x86_64-efi/%.efidrv,$(pxerom_targets))
+
 ipxe/src/config/local/%: config.ipxe.%
 	cp $< $@
+
+
+slof:
+	$(MAKE) $(MAKEFLAGS) -C SLOF CROSS=$(powerpc64_cross_prefix) qemu
+	cp SLOF/boot_rom.bin ../pc-bios/slof.bin
+
+
+clean:
+	rm -rf seabios/.config seabios/out seabios/builds
+	$(MAKE) $(MAKEFLAGS) -C vgabios clean
+	rm -f vgabios/VGABIOS-lgpl-latest*
+	$(MAKE) $(MAKEFLAGS) -C sgabios clean
+	rm -f sgabios/.depend
+	$(MAKE) $(MAKEFLAGS) -C ipxe/src veryclean
+	$(MAKE) $(MAKEFLAGS) -C SLOF clean
diff --git a/roms/config.vga.cirrus b/roms/config.vga-cirrus
index c8fe58239f..c8fe58239f 100644
--- a/roms/config.vga.cirrus
+++ b/roms/config.vga-cirrus
diff --git a/roms/config.vga.isavga b/roms/config.vga-isavga
index e55e294a0c..e55e294a0c 100644
--- a/roms/config.vga.isavga
+++ b/roms/config.vga-isavga
diff --git a/roms/config.vga.qxl b/roms/config.vga-qxl
index d393f0c34f..d393f0c34f 100644
--- a/roms/config.vga.qxl
+++ b/roms/config.vga-qxl
diff --git a/roms/config.vga.stdvga b/roms/config.vga-stdvga
index 7d063b787c..7d063b787c 100644
--- a/roms/config.vga.stdvga
+++ b/roms/config.vga-stdvga
diff --git a/roms/config.vga.vmware b/roms/config.vga-vmware
index eb10427afd..eb10427afd 100644
--- a/roms/config.vga.vmware
+++ b/roms/config.vga-vmware
diff --git a/roms/openbios b/roms/openbios
-Subproject 0f3d51ef22ec9166beb3ed434d253029ed7cfe8
+Subproject d363cf50c50c268da7e6d0bf707adde1893d1ab
diff --git a/roms/seabios b/roms/seabios
-Subproject d4f7d90f47462b4e8836899adc5060fbde5253e
+Subproject ece025f5980bae88fa677bc9c0d24d2e580e205
diff --git a/rules.mak b/rules.mak
index abc2e846b3..49edb9bf07 100644
--- a/rules.mak
+++ b/rules.mak
@@ -89,6 +89,34 @@ find-in-path = $(if $(find-string /, $1), \
         $(wildcard $1), \
         $(wildcard $(patsubst %, %/$1, $(subst :, ,$(PATH)))))
 
+# Logical functions (for operating on y/n values like CONFIG_FOO vars)
+# Inputs to these must be either "y" (true) or "n" or "" (both false)
+# Output is always either "y" or "n".
+# Usage: $(call land,$(CONFIG_FOO),$(CONFIG_BAR))
+# Logical NOT
+lnot = $(if $(subst n,,$1),n,y)
+# Logical AND
+land = $(if $(findstring yy,$1$2),y,n)
+# Logical OR
+lor = $(if $(findstring y,$1$2),y,n)
+# Logical XOR (note that this is the inverse of leqv)
+lxor = $(if $(filter $(call lnot,$1),$(call lnot,$2)),n,y)
+# Logical equivalence (note that leqv "","n" is true)
+leqv = $(if $(filter $(call lnot,$1),$(call lnot,$2)),y,n)
+# Logical if: like make's $(if) but with an leqv-like test
+lif = $(if $(subst n,,$1),$2,$3)
+
+# String testing functions: inputs to these can be any string;
+# the output is always either "y" or "n". Leading and trailing whitespace
+# is ignored when comparing strings.
+# String equality
+eq = $(if $(subst $2,,$1)$(subst $1,,$2),n,y)
+# String inequality
+ne = $(if $(subst $2,,$1)$(subst $1,,$2),y,n)
+# Emptiness/non-emptiness tests:
+isempty = $(if $1,n,y)
+notempty = $(if $1,y,n)
+
 # Generate files with tracetool
 TRACETOOL=$(PYTHON) $(SRC_PATH)/scripts/tracetool.py
 
diff --git a/savevm.c b/savevm.c
index c536aa4986..2f631d4045 100644
--- a/savevm.c
+++ b/savevm.c
@@ -566,6 +566,13 @@ QEMUFile *qemu_fopen_ops(void *opaque, const QEMUFileOps *ops)
     return f;
 }
 
+/*
+ * Get last error for stream f
+ *
+ * Return negative error value if there has been an error on previous
+ * operations, return 0 if no error happened.
+ *
+ */
 int qemu_file_get_error(QEMUFile *f)
 {
     return f->last_error;
@@ -642,7 +649,7 @@ void ram_control_after_iterate(QEMUFile *f, uint64_t flags)
 
 void ram_control_load_hook(QEMUFile *f, uint64_t flags)
 {
-    int ret = 0;
+    int ret = -EINVAL;
 
     if (f->ops->hook_ram_load) {
         ret = f->ops->hook_ram_load(f, f->opaque, flags);
@@ -2325,18 +2332,21 @@ static int del_existing_snapshots(Monitor *mon, const char *name)
 {
     BlockDriverState *bs;
     QEMUSnapshotInfo sn1, *snapshot = &sn1;
-    int ret;
+    Error *err = NULL;
 
     bs = NULL;
     while ((bs = bdrv_next(bs))) {
         if (bdrv_can_snapshot(bs) &&
             bdrv_snapshot_find(bs, snapshot, name) >= 0)
         {
-            ret = bdrv_snapshot_delete(bs, name);
-            if (ret < 0) {
+            bdrv_snapshot_delete_by_id_or_name(bs, name, &err);
+            if (error_is_set(&err)) {
                 monitor_printf(mon,
-                               "Error while deleting snapshot on '%s'\n",
-                               bdrv_get_device_name(bs));
+                               "Error while deleting snapshot on device '%s':"
+                               " %s\n",
+                               bdrv_get_device_name(bs),
+                               error_get_pretty(err));
+                error_free(err);
                 return -1;
             }
         }
@@ -2550,7 +2560,7 @@ int load_vmstate(const char *name)
 void do_delvm(Monitor *mon, const QDict *qdict)
 {
     BlockDriverState *bs, *bs1;
-    int ret;
+    Error *err = NULL;
     const char *name = qdict_get_str(qdict, "name");
 
     bs = find_vmstate_bs();
@@ -2562,15 +2572,14 @@ void do_delvm(Monitor *mon, const QDict *qdict)
     bs1 = NULL;
     while ((bs1 = bdrv_next(bs1))) {
         if (bdrv_can_snapshot(bs1)) {
-            ret = bdrv_snapshot_delete(bs1, name);
-            if (ret < 0) {
-                if (ret == -ENOTSUP)
-                    monitor_printf(mon,
-                                   "Snapshots not supported on device '%s'\n",
-                                   bdrv_get_device_name(bs1));
-                else
-                    monitor_printf(mon, "Error %d while deleting snapshot on "
-                                   "'%s'\n", ret, bdrv_get_device_name(bs1));
+            bdrv_snapshot_delete_by_id_or_name(bs, name, &err);
+            if (error_is_set(&err)) {
+                monitor_printf(mon,
+                               "Error while deleting snapshot on device '%s':"
+                               " %s\n",
+                               bdrv_get_device_name(bs),
+                               error_get_pretty(err));
+                error_free(err);
             }
         }
     }
diff --git a/scripts/get_maintainer.pl b/scripts/get_maintainer.pl
index bf5342a08d..38334de875 100755
--- a/scripts/get_maintainer.pl
+++ b/scripts/get_maintainer.pl
@@ -1385,7 +1385,7 @@ sub vcs_exists {
 	warn("$P: No supported VCS found.  Add --nogit to options?\n");
 	warn("Using a git repository produces better results.\n");
 	warn("Try latest git repository using:\n");
-	warn("git clone git://git.qemu.org/qemu.git\n");
+	warn("git clone git://git.qemu-project.org/qemu.git\n");
 	$printed_novcs = 1;
     }
     return 0;
diff --git a/scripts/qapi-types.py b/scripts/qapi-types.py
index 5222463893..4a1652b56f 100644
--- a/scripts/qapi-types.py
+++ b/scripts/qapi-types.py
@@ -71,7 +71,7 @@ def generate_struct_fields(members):
                          c_name=c_var(argname))
         if structured:
             push_indent()
-            ret += generate_struct("", argname, argentry)
+            ret += generate_struct({ "field": argname, "data": argentry})
             pop_indent()
         else:
             ret += mcgen('''
@@ -81,13 +81,22 @@ def generate_struct_fields(members):
 
     return ret
 
-def generate_struct(structname, fieldname, members):
+def generate_struct(expr):
+
+    structname = expr.get('type', "")
+    fieldname = expr.get('field', "")
+    members = expr['data']
+    base = expr.get('base')
+
     ret = mcgen('''
 struct %(name)s
 {
 ''',
           name=structname)
 
+    if base:
+        ret += generate_struct_fields({'base': base})
+
     ret += generate_struct_fields(members)
 
     if len(fieldname):
@@ -417,7 +426,7 @@ if do_builtins:
 for expr in exprs:
     ret = "\n"
     if expr.has_key('type'):
-        ret += generate_struct(expr['type'], "", expr['data']) + "\n"
+        ret += generate_struct(expr) + "\n"
         ret += generate_type_cleanup_decl(expr['type'] + "List")
         fdef.write(generate_type_cleanup(expr['type'] + "List") + "\n")
         ret += generate_type_cleanup_decl(expr['type'])
diff --git a/scripts/qapi-visit.py b/scripts/qapi-visit.py
index 597cca4b66..c39e6284b8 100644
--- a/scripts/qapi-visit.py
+++ b/scripts/qapi-visit.py
@@ -17,7 +17,7 @@ import os
 import getopt
 import errno
 
-def generate_visit_struct_fields(name, field_prefix, fn_prefix, members):
+def generate_visit_struct_fields(name, field_prefix, fn_prefix, members, base = None):
     substructs = []
     ret = ''
     full_name = name if not fn_prefix else "%s_%s" % (name, fn_prefix)
@@ -42,6 +42,19 @@ static void visit_type_%(full_name)s_fields(Visitor *m, %(name)s ** obj, Error *
         name=name, full_name=full_name)
     push_indent()
 
+    if base:
+        ret += mcgen('''
+visit_start_implicit_struct(m, obj ? (void**) &(*obj)->%(c_name)s : NULL, sizeof(%(type)s), &err);
+if (!err) {
+    visit_type_%(type)s_fields(m, obj ? &(*obj)->%(c_prefix)s%(c_name)s : NULL, &err);
+    error_propagate(errp, err);
+    err = NULL;
+    visit_end_implicit_struct(m, &err);
+}
+''',
+                     c_prefix=c_var(field_prefix),
+                     type=type_name(base), c_name=c_var('base'))
+
     for argname, argentry, optional, structured in parse_args(members):
         if optional:
             ret += mcgen('''
@@ -120,8 +133,13 @@ if (!err) {
 ''')
     return ret
 
-def generate_visit_struct(name, members):
-    ret = generate_visit_struct_fields(name, "", "", members)
+def generate_visit_struct(expr):
+
+    name = expr['type']
+    members = expr['data']
+    base = expr.get('base')
+
+    ret = generate_visit_struct_fields(name, "", "", members, base)
 
     ret += mcgen('''
 
@@ -472,7 +490,7 @@ if do_builtins:
 
 for expr in exprs:
     if expr.has_key('type'):
-        ret = generate_visit_struct(expr['type'], expr['data'])
+        ret = generate_visit_struct(expr)
         ret += generate_visit_list(expr['type'], expr['data'])
         fdef.write(ret)
 
diff --git a/QMP/qemu-ga-client b/scripts/qmp/qemu-ga-client
index b5f7e7c5ff..9908f21093 100755
--- a/QMP/qemu-ga-client
+++ b/scripts/qmp/qemu-ga-client
@@ -33,7 +33,7 @@
 # $ qemu-ga-client fsfreeze freeze
 # 2 filesystems frozen
 #
-# See also: http://wiki.qemu.org/Features/QAPI/GuestAgent
+# See also: http://wiki.qemu-project.org/Features/QAPI/GuestAgent
 #
 
 import base64
diff --git a/QMP/qmp b/scripts/qmp/qmp
index 1db3c7ffeb..1db3c7ffeb 100755
--- a/QMP/qmp
+++ b/scripts/qmp/qmp
diff --git a/QMP/qmp-shell b/scripts/qmp/qmp-shell
index 73cb3b6cef..d6b420f18a 100755
--- a/QMP/qmp-shell
+++ b/scripts/qmp/qmp-shell
@@ -91,7 +91,7 @@ class QMPShell(qmp.QEMUMonitorProtocol):
         """
         Build a QMP input object from a user provided command-line in the
         following format:
-    
+
             < command-name > [ arg-name1=arg1 ] ... [ arg-nameN=argN ]
         """
         cmdargs = cmdline.split()
diff --git a/QMP/qmp.py b/scripts/qmp/qmp.py
index c551df1ed7..5c9717594f 100644
--- a/QMP/qmp.py
+++ b/scripts/qmp/qmp.py
@@ -1,5 +1,5 @@
 # QEMU Monitor Protocol Python class
-# 
+#
 # Copyright (C) 2009, 2010 Red Hat Inc.
 #
 # Authors:
@@ -188,3 +188,9 @@ class QEMUMonitorProtocol:
 
     def settimeout(self, timeout):
         self.__sock.settimeout(timeout)
+
+    def get_sock_fd(self):
+        return self.__sock.fileno()
+
+    def is_scm_available(self):
+        return self.__sock.family == socket.AF_UNIX
diff --git a/QMP/qom-fuse b/scripts/qmp/qom-fuse
index 5c6754aa63..5c6754aa63 100755
--- a/QMP/qom-fuse
+++ b/scripts/qmp/qom-fuse
diff --git a/QMP/qom-get b/scripts/qmp/qom-get
index 0172c69441..0172c69441 100755
--- a/QMP/qom-get
+++ b/scripts/qmp/qom-get
diff --git a/QMP/qom-list b/scripts/qmp/qom-list
index 1e7cc6cb2d..1e7cc6cb2d 100755
--- a/QMP/qom-list
+++ b/scripts/qmp/qom-list
diff --git a/QMP/qom-set b/scripts/qmp/qom-set
index 54ecfecc53..54ecfecc53 100755
--- a/QMP/qom-set
+++ b/scripts/qmp/qom-set
diff --git a/scripts/refresh-pxe-roms.sh b/scripts/refresh-pxe-roms.sh
index 14d586070f..90fc0b374d 100755
--- a/scripts/refresh-pxe-roms.sh
+++ b/scripts/refresh-pxe-roms.sh
@@ -21,79 +21,11 @@
 # Usage: Run from root of qemu tree
 # ./scripts/refresh-pxe-roms.sh
 
-QEMU_DIR=$PWD
-ROM_DIR="pc-bios"
-BUILD_DIR="roms/ipxe"
-LOCAL_CONFIG="src/config/local/general.h"
-
-function cleanup ()
-{
-    if [ -n "$SAVED_CONFIG" ]; then
-        cp "$SAVED_CONFIG" "$BUILD_DIR"/"$LOCAL_CONFIG"
-        rm "$SAVED_CONFIG"
-    fi
-    cd "$QEMU_DIR"
-}
-
-function make_rom ()
-{
-    cd "$BUILD_DIR"/src
-
-    BUILD_LOG=$(mktemp)
-
-    echo Building "$2"...
-    make bin/"$1".rom > "$BUILD_LOG" 2>&1
-    if [ $? -ne 0 ]; then
-        echo Build failed
-        tail --lines=100 "$BUILD_LOG"
-        rm "$BUILD_LOG"
-        cleanup
-        exit 1
-    fi
-    rm "$BUILD_LOG"
-
-    cp bin/"$1".rom "$QEMU_DIR"/"$ROM_DIR"/"$2"
-
-    cd "$QEMU_DIR"
-}
-
-if [ ! -d "$QEMU_DIR"/"$ROM_DIR" ]; then
-    echo "error: can't find $ROM_DIR directory," \
-         "run me from the root of the qemu tree"
-    exit 1
-fi
-
-if [ ! -d "$BUILD_DIR"/src ]; then
-    echo "error: $BUILD_DIR not populated, try:"
-    echo "  git submodule init $BUILD_DIR"
-    echo "  git submodule update $BUILD_DIR"
-    exit 1
-fi
-
-if [ -e "$BUILD_DIR"/"$LOCAL_CONFIG" ]; then
-    SAVED_CONFIG=$(mktemp)
-    cp "$BUILD_DIR"/"$LOCAL_CONFIG" "$SAVED_CONFIG"
-fi
-
-echo "#undef BANNER_TIMEOUT" > "$BUILD_DIR"/"$LOCAL_CONFIG"
-echo "#define BANNER_TIMEOUT 0" >> "$BUILD_DIR"/"$LOCAL_CONFIG"
-
-IPXE_VERSION=$(cd "$BUILD_DIR" && git describe --tags)
-if [ -z "$IPXE_VERSION" ]; then
-    echo "error: unable to retrieve git version"
-    cleanup
-    exit 1
+targets="pxerom"
+if test -x "$(which EfiRom 2>/dev/null)"; then
+    targets="$targets efirom"
 fi
 
-echo "#undef PRODUCT_NAME" >> "$BUILD_DIR"/"$LOCAL_CONFIG"
-echo "#define PRODUCT_NAME \"iPXE $IPXE_VERSION\"" >> "$BUILD_DIR"/"$LOCAL_CONFIG"
-
-make_rom 8086100e pxe-e1000.rom
-make_rom 80861209 pxe-eepro100.rom
-make_rom 10500940 pxe-ne2k_pci.rom
-make_rom 10222000 pxe-pcnet.rom
-make_rom 10ec8139 pxe-rtl8139.rom
-make_rom 1af41000 pxe-virtio.rom
-
-echo done
-cleanup
+cd roms
+make -j4 $targets || exit 1
+make clean
diff --git a/slirp/libslirp.h b/slirp/libslirp.h
index ceabff81b2..5bdcbd50f7 100644
--- a/slirp/libslirp.h
+++ b/slirp/libslirp.h
@@ -16,8 +16,7 @@ Slirp *slirp_init(int restricted, struct in_addr vnetwork,
                   void *opaque);
 void slirp_cleanup(Slirp *slirp);
 
-void slirp_update_timeout(uint32_t *timeout);
-void slirp_pollfds_fill(GArray *pollfds);
+void slirp_pollfds_fill(GArray *pollfds, uint32_t *timeout);
 
 void slirp_pollfds_poll(GArray *pollfds, int select_error);
 
diff --git a/slirp/misc.c b/slirp/misc.c
index c0d489950a..6c1636f7b6 100644
--- a/slirp/misc.c
+++ b/slirp/misc.c
@@ -212,8 +212,7 @@ fork_exec(struct socket *so, const char *ex, int do_pty)
                     so->s = accept(s, (struct sockaddr *)&addr, &addrlen);
                 } while (so->s < 0 && errno == EINTR);
                 closesocket(s);
-                opt = 1;
-                qemu_setsockopt(so->s, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(int));
+                socket_set_fast_reuse(so->s);
                 opt = 1;
                 qemu_setsockopt(so->s, SOL_SOCKET, SO_OOBINLINE, &opt, sizeof(int));
 		qemu_set_nonblock(so->s);
diff --git a/slirp/slirp.c b/slirp/slirp.c
index 5c3dabba93..bad8dad02e 100644
--- a/slirp/slirp.c
+++ b/slirp/slirp.c
@@ -40,8 +40,6 @@ static const uint8_t special_ethaddr[ETH_ALEN] = {
 static const uint8_t zero_ethaddr[ETH_ALEN] = { 0, 0, 0, 0, 0, 0 };
 
 u_int curtime;
-static u_int time_fasttimo, last_slowtimo;
-static int do_slowtimo;
 
 static QTAILQ_HEAD(slirp_instances, Slirp) slirp_instances =
     QTAILQ_HEAD_INITIALIZER(slirp_instances);
@@ -49,6 +47,11 @@ static QTAILQ_HEAD(slirp_instances, Slirp) slirp_instances =
 static struct in_addr dns_addr;
 static u_int dns_addr_time;
 
+#define TIMEOUT_FAST 2  /* milliseconds */
+#define TIMEOUT_SLOW 499  /* milliseconds */
+/* for the aging of certain requests like DNS */
+#define TIMEOUT_DEFAULT 1000  /* milliseconds */
+
 #ifdef _WIN32
 
 int get_dns_addr(struct in_addr *pdns_addr)
@@ -59,7 +62,7 @@ int get_dns_addr(struct in_addr *pdns_addr)
     IP_ADDR_STRING *pIPAddr;
     struct in_addr tmp_addr;
 
-    if (dns_addr.s_addr != 0 && (curtime - dns_addr_time) < 1000) {
+    if (dns_addr.s_addr != 0 && (curtime - dns_addr_time) < TIMEOUT_DEFAULT) {
         *pdns_addr = dns_addr;
         return 0;
     }
@@ -115,7 +118,7 @@ int get_dns_addr(struct in_addr *pdns_addr)
 
     if (dns_addr.s_addr != 0) {
         struct stat old_stat;
-        if ((curtime - dns_addr_time) < 1000) {
+        if ((curtime - dns_addr_time) < TIMEOUT_DEFAULT) {
             *pdns_addr = dns_addr;
             return 0;
         }
@@ -259,14 +262,33 @@ void slirp_cleanup(Slirp *slirp)
 #define CONN_CANFSEND(so) (((so)->so_state & (SS_FCANTSENDMORE|SS_ISFCONNECTED)) == SS_ISFCONNECTED)
 #define CONN_CANFRCV(so) (((so)->so_state & (SS_FCANTRCVMORE|SS_ISFCONNECTED)) == SS_ISFCONNECTED)
 
-void slirp_update_timeout(uint32_t *timeout)
+static void slirp_update_timeout(uint32_t *timeout)
 {
-    if (!QTAILQ_EMPTY(&slirp_instances)) {
-        *timeout = MIN(1000, *timeout);
+    Slirp *slirp;
+    uint32_t t;
+
+    if (*timeout <= TIMEOUT_FAST) {
+        return;
+    }
+
+    t = MIN(1000, *timeout);
+
+    /* If we have tcp timeout with slirp, then we will fill @timeout with
+     * more precise value.
+     */
+    QTAILQ_FOREACH(slirp, &slirp_instances, entry) {
+        if (slirp->time_fasttimo) {
+            *timeout = TIMEOUT_FAST;
+            return;
+        }
+        if (slirp->do_slowtimo) {
+            t = MIN(TIMEOUT_SLOW, t);
+        }
     }
+    *timeout = t;
 }
 
-void slirp_pollfds_fill(GArray *pollfds)
+void slirp_pollfds_fill(GArray *pollfds, uint32_t *timeout)
 {
     Slirp *slirp;
     struct socket *so, *so_next;
@@ -278,14 +300,13 @@ void slirp_pollfds_fill(GArray *pollfds)
     /*
      * First, TCP sockets
      */
-    do_slowtimo = 0;
 
     QTAILQ_FOREACH(slirp, &slirp_instances, entry) {
         /*
          * *_slowtimo needs calling if there are IP fragments
          * in the fragment queue, or there are TCP connections active
          */
-        do_slowtimo |= ((slirp->tcb.so_next != &slirp->tcb) ||
+        slirp->do_slowtimo = ((slirp->tcb.so_next != &slirp->tcb) ||
                 (&slirp->ipq.ip_link != slirp->ipq.ip_link.next));
 
         for (so = slirp->tcb.so_next; so != &slirp->tcb;
@@ -299,8 +320,9 @@ void slirp_pollfds_fill(GArray *pollfds)
             /*
              * See if we need a tcp_fasttimo
              */
-            if (time_fasttimo == 0 && so->so_tcpcb->t_flags & TF_DELACK) {
-                time_fasttimo = curtime; /* Flag when we want a fasttimo */
+            if (slirp->time_fasttimo == 0 &&
+                so->so_tcpcb->t_flags & TF_DELACK) {
+                slirp->time_fasttimo = curtime; /* Flag when want a fasttimo */
             }
 
             /*
@@ -381,7 +403,7 @@ void slirp_pollfds_fill(GArray *pollfds)
                     udp_detach(so);
                     continue;
                 } else {
-                    do_slowtimo = 1; /* Let socket expire */
+                    slirp->do_slowtimo = true; /* Let socket expire */
                 }
             }
 
@@ -422,7 +444,7 @@ void slirp_pollfds_fill(GArray *pollfds)
                     icmp_detach(so);
                     continue;
                 } else {
-                    do_slowtimo = 1; /* Let socket expire */
+                    slirp->do_slowtimo = true; /* Let socket expire */
                 }
             }
 
@@ -436,6 +458,7 @@ void slirp_pollfds_fill(GArray *pollfds)
             }
         }
     }
+    slirp_update_timeout(timeout);
 }
 
 void slirp_pollfds_poll(GArray *pollfds, int select_error)
@@ -454,14 +477,16 @@ void slirp_pollfds_poll(GArray *pollfds, int select_error)
         /*
          * See if anything has timed out
          */
-        if (time_fasttimo && ((curtime - time_fasttimo) >= 2)) {
+        if (slirp->time_fasttimo &&
+            ((curtime - slirp->time_fasttimo) >= TIMEOUT_FAST)) {
             tcp_fasttimo(slirp);
-            time_fasttimo = 0;
+            slirp->time_fasttimo = 0;
         }
-        if (do_slowtimo && ((curtime - last_slowtimo) >= 499)) {
+        if (slirp->do_slowtimo &&
+            ((curtime - slirp->last_slowtimo) >= TIMEOUT_SLOW)) {
             ip_slowtimo(slirp);
             tcp_slowtimo(slirp);
-            last_slowtimo = curtime;
+            slirp->last_slowtimo = curtime;
         }
 
         /*
diff --git a/slirp/slirp.h b/slirp/slirp.h
index fe0e65d0ee..e4a1bd4abb 100644
--- a/slirp/slirp.h
+++ b/slirp/slirp.h
@@ -203,6 +203,9 @@ bool arp_table_search(Slirp *slirp, uint32_t ip_addr,
 
 struct Slirp {
     QTAILQ_ENTRY(Slirp) entry;
+    u_int time_fasttimo;
+    u_int last_slowtimo;
+    bool do_slowtimo;
 
     /* virtual network configuration */
     struct in_addr vnetwork_addr;
diff --git a/slirp/socket.c b/slirp/socket.c
index 25d60e7a89..37ac5cf2fb 100644
--- a/slirp/socket.c
+++ b/slirp/socket.c
@@ -627,9 +627,7 @@ tcp_listen(Slirp *slirp, uint32_t haddr, u_int hport, uint32_t laddr,
 	addr.sin_port = hport;
 
 	if (((s = qemu_socket(AF_INET,SOCK_STREAM,0)) < 0) ||
-#ifndef _WIN32
-	    (qemu_setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(int)) < 0) ||
-#endif
+	    (socket_set_fast_reuse(s) < 0) ||
 	    (bind(s,(struct sockaddr *)&addr, sizeof(addr)) < 0) ||
 	    (listen(s,1) < 0)) {
 		int tmperrno = errno; /* Don't clobber the real reason we failed */
diff --git a/slirp/tcp_subr.c b/slirp/tcp_subr.c
index 043f28fcae..7571c5a282 100644
--- a/slirp/tcp_subr.c
+++ b/slirp/tcp_subr.c
@@ -337,8 +337,7 @@ int tcp_fconnect(struct socket *so)
     struct sockaddr_in addr;
 
     qemu_set_nonblock(s);
-    opt = 1;
-    qemu_setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt));
+    socket_set_fast_reuse(s);
     opt = 1;
     qemu_setsockopt(s, SOL_SOCKET, SO_OOBINLINE, &opt, sizeof(opt));
 
@@ -426,8 +425,7 @@ void tcp_connect(struct socket *inso)
         return;
     }
     qemu_set_nonblock(s);
-    opt = 1;
-    qemu_setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(int));
+    socket_set_fast_reuse(s);
     opt = 1;
     qemu_setsockopt(s, SOL_SOCKET, SO_OOBINLINE, &opt, sizeof(int));
     socket_set_nodelay(s);
diff --git a/slirp/udp.c b/slirp/udp.c
index b105f871f3..8cc6cb66da 100644
--- a/slirp/udp.c
+++ b/slirp/udp.c
@@ -354,7 +354,7 @@ udp_listen(Slirp *slirp, uint32_t haddr, u_int hport, uint32_t laddr,
 {
 	struct sockaddr_in addr;
 	struct socket *so;
-	socklen_t addrlen = sizeof(struct sockaddr_in), opt = 1;
+	socklen_t addrlen = sizeof(struct sockaddr_in);
 
 	so = socreate(slirp);
 	if (!so) {
@@ -372,7 +372,7 @@ udp_listen(Slirp *slirp, uint32_t haddr, u_int hport, uint32_t laddr,
 		udp_detach(so);
 		return NULL;
 	}
-	qemu_setsockopt(so->s, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(int));
+	socket_set_fast_reuse(so->s);
 
 	getsockname(so->s,(struct sockaddr *)&addr,&addrlen);
 	so->so_fport = addr.sin_port;
diff --git a/stubs/Makefile.objs b/stubs/Makefile.objs
index f306cbada3..df92fe5b15 100644
--- a/stubs/Makefile.objs
+++ b/stubs/Makefile.objs
@@ -22,6 +22,7 @@ stub-obj-y += reset.o
 stub-obj-y += set-fd-handler.o
 stub-obj-y += slirp.o
 stub-obj-y += sysbus.o
+stub-obj-y += uuid.o
 stub-obj-y += vm-stop.o
 stub-obj-y += vmstate.o
 stub-obj-$(CONFIG_WIN32) += fd-register.o
diff --git a/stubs/slirp.c b/stubs/slirp.c
index f1fc833f7a..bd0ac7f27d 100644
--- a/stubs/slirp.c
+++ b/stubs/slirp.c
@@ -1,11 +1,7 @@
 #include "qemu-common.h"
 #include "slirp/slirp.h"
 
-void slirp_update_timeout(uint32_t *timeout)
-{
-}
-
-void slirp_pollfds_fill(GArray *pollfds)
+void slirp_pollfds_fill(GArray *pollfds, uint32_t *timeout)
 {
 }
 
diff --git a/stubs/uuid.c b/stubs/uuid.c
new file mode 100644
index 0000000000..ffc0ed40ae
--- /dev/null
+++ b/stubs/uuid.c
@@ -0,0 +1,12 @@
+#include "qemu-common.h"
+#include "sysemu/sysemu.h"
+#include "qmp-commands.h"
+
+UuidInfo *qmp_query_uuid(Error **errp)
+{
+    UuidInfo *info = g_malloc0(sizeof(*info));
+
+    info->UUID = g_strdup(UUID_NONE);
+    return info;
+}
+
diff --git a/target-alpha/cpu.c b/target-alpha/cpu.c
index cfad2ea121..a0d5d5bd93 100644
--- a/target-alpha/cpu.c
+++ b/target-alpha/cpu.c
@@ -131,7 +131,6 @@ static ObjectClass *alpha_cpu_class_by_name(const char *cpu_model)
 AlphaCPU *cpu_alpha_init(const char *cpu_model)
 {
     AlphaCPU *cpu;
-    CPUAlphaState *env;
     ObjectClass *cpu_class;
 
     cpu_class = alpha_cpu_class_by_name(cpu_model);
@@ -140,9 +139,6 @@ AlphaCPU *cpu_alpha_init(const char *cpu_model)
         cpu_class = object_class_by_name(TYPE("ev67"));
     }
     cpu = ALPHA_CPU(object_new(object_class_get_name(cpu_class)));
-    env = &cpu->env;
-
-    env->cpu_model_str = cpu_model;
 
     object_property_set_bool(OBJECT(cpu), true, "realized", NULL);
 
diff --git a/target-alpha/helper.h b/target-alpha/helper.h
index 732b701d53..5a0e78cefb 100644
--- a/target-alpha/helper.h
+++ b/target-alpha/helper.h
@@ -114,7 +114,7 @@ DEF_HELPER_FLAGS_1(tbia, TCG_CALL_NO_RWG, void, env)
 DEF_HELPER_FLAGS_2(tbis, TCG_CALL_NO_RWG, void, env, i64)
 DEF_HELPER_FLAGS_1(tb_flush, TCG_CALL_NO_RWG, void, env)
 
-DEF_HELPER_1(halt, void, i64);
+DEF_HELPER_1(halt, void, i64)
 
 DEF_HELPER_FLAGS_0(get_vmtime, TCG_CALL_NO_RWG, i64)
 DEF_HELPER_FLAGS_0(get_walltime, TCG_CALL_NO_RWG, i64)
diff --git a/target-alpha/translate.c b/target-alpha/translate.c
index 28ce4363f1..c24910f6a1 100644
--- a/target-alpha/translate.c
+++ b/target-alpha/translate.c
@@ -140,10 +140,6 @@ void alpha_translate_init(void)
                                      offsetof(CPUAlphaState, usp), "usp");
 #endif
 
-    /* register helpers */
-#define GEN_HELPER 2
-#include "helper.h"
-
     done_init = 1;
 }
 
@@ -172,44 +168,38 @@ static inline ExitStatus gen_invalid(DisasContext *ctx)
 
 static inline void gen_qemu_ldf(TCGv t0, TCGv t1, int flags)
 {
-    TCGv tmp = tcg_temp_new();
     TCGv_i32 tmp32 = tcg_temp_new_i32();
-    tcg_gen_qemu_ld32u(tmp, t1, flags);
-    tcg_gen_trunc_i64_i32(tmp32, tmp);
+    tcg_gen_qemu_ld_i32(tmp32, t1, flags, MO_LEUL);
     gen_helper_memory_to_f(t0, tmp32);
     tcg_temp_free_i32(tmp32);
-    tcg_temp_free(tmp);
 }
 
 static inline void gen_qemu_ldg(TCGv t0, TCGv t1, int flags)
 {
     TCGv tmp = tcg_temp_new();
-    tcg_gen_qemu_ld64(tmp, t1, flags);
+    tcg_gen_qemu_ld_i64(tmp, t1, flags, MO_LEQ);
     gen_helper_memory_to_g(t0, tmp);
     tcg_temp_free(tmp);
 }
 
 static inline void gen_qemu_lds(TCGv t0, TCGv t1, int flags)
 {
-    TCGv tmp = tcg_temp_new();
     TCGv_i32 tmp32 = tcg_temp_new_i32();
-    tcg_gen_qemu_ld32u(tmp, t1, flags);
-    tcg_gen_trunc_i64_i32(tmp32, tmp);
+    tcg_gen_qemu_ld_i32(tmp32, t1, flags, MO_LEUL);
     gen_helper_memory_to_s(t0, tmp32);
     tcg_temp_free_i32(tmp32);
-    tcg_temp_free(tmp);
 }
 
 static inline void gen_qemu_ldl_l(TCGv t0, TCGv t1, int flags)
 {
-    tcg_gen_qemu_ld32s(t0, t1, flags);
+    tcg_gen_qemu_ld_i64(t0, t1, flags, MO_LESL);
     tcg_gen_mov_i64(cpu_lock_addr, t1);
     tcg_gen_mov_i64(cpu_lock_value, t0);
 }
 
 static inline void gen_qemu_ldq_l(TCGv t0, TCGv t1, int flags)
 {
-    tcg_gen_qemu_ld64(t0, t1, flags);
+    tcg_gen_qemu_ld_i64(t0, t1, flags, MO_LEQ);
     tcg_gen_mov_i64(cpu_lock_addr, t1);
     tcg_gen_mov_i64(cpu_lock_value, t0);
 }
@@ -251,11 +241,8 @@ static inline void gen_load_mem(DisasContext *ctx,
 static inline void gen_qemu_stf(TCGv t0, TCGv t1, int flags)
 {
     TCGv_i32 tmp32 = tcg_temp_new_i32();
-    TCGv tmp = tcg_temp_new();
     gen_helper_f_to_memory(tmp32, t0);
-    tcg_gen_extu_i32_i64(tmp, tmp32);
-    tcg_gen_qemu_st32(tmp, t1, flags);
-    tcg_temp_free(tmp);
+    tcg_gen_qemu_st_i32(tmp32, t1, flags, MO_LEUL);
     tcg_temp_free_i32(tmp32);
 }
 
@@ -263,18 +250,15 @@ static inline void gen_qemu_stg(TCGv t0, TCGv t1, int flags)
 {
     TCGv tmp = tcg_temp_new();
     gen_helper_g_to_memory(tmp, t0);
-    tcg_gen_qemu_st64(tmp, t1, flags);
+    tcg_gen_qemu_st_i64(tmp, t1, flags, MO_LEQ);
     tcg_temp_free(tmp);
 }
 
 static inline void gen_qemu_sts(TCGv t0, TCGv t1, int flags)
 {
     TCGv_i32 tmp32 = tcg_temp_new_i32();
-    TCGv tmp = tcg_temp_new();
     gen_helper_s_to_memory(tmp32, t0);
-    tcg_gen_extu_i32_i64(tmp, tmp32);
-    tcg_gen_qemu_st32(tmp, t1, flags);
-    tcg_temp_free(tmp);
+    tcg_gen_qemu_st_i32(tmp32, t1, flags, MO_LEUL);
     tcg_temp_free_i32(tmp32);
 }
 
@@ -352,18 +336,11 @@ static ExitStatus gen_store_conditional(DisasContext *ctx, int ra, int rb,
         tcg_gen_brcond_i64(TCG_COND_NE, addr, cpu_lock_addr, lab_fail);
 
         val = tcg_temp_new();
-        if (quad) {
-            tcg_gen_qemu_ld64(val, addr, ctx->mem_idx);
-        } else {
-            tcg_gen_qemu_ld32s(val, addr, ctx->mem_idx);
-        }
+        tcg_gen_qemu_ld_i64(val, addr, ctx->mem_idx, quad ? MO_LEQ : MO_LESL);
         tcg_gen_brcond_i64(TCG_COND_NE, val, cpu_lock_value, lab_fail);
 
-        if (quad) {
-            tcg_gen_qemu_st64(cpu_ir[ra], addr, ctx->mem_idx);
-        } else {
-            tcg_gen_qemu_st32(cpu_ir[ra], addr, ctx->mem_idx);
-        }
+        tcg_gen_qemu_st_i64(cpu_ir[ra], addr, ctx->mem_idx,
+                            quad ? MO_LEQ : MO_LEUL);
         tcg_gen_movi_i64(cpu_ir[ra], 1);
         tcg_gen_br(lab_done);
 
@@ -2970,11 +2947,11 @@ static ExitStatus translate_one(DisasContext *ctx, uint32_t insn)
                 goto invalid_opc;
             case 0xA:
                 /* Longword virtual access with protection check (hw_ldl/w) */
-                tcg_gen_qemu_ld32s(cpu_ir[ra], addr, MMU_KERNEL_IDX);
+                tcg_gen_qemu_ld_i64(cpu_ir[ra], addr, MMU_KERNEL_IDX, MO_LESL);
                 break;
             case 0xB:
                 /* Quadword virtual access with protection check (hw_ldq/w) */
-                tcg_gen_qemu_ld64(cpu_ir[ra], addr, MMU_KERNEL_IDX);
+                tcg_gen_qemu_ld_i64(cpu_ir[ra], addr, MMU_KERNEL_IDX, MO_LEQ);
                 break;
             case 0xC:
                 /* Longword virtual access with alt access mode (hw_ldl/a)*/
@@ -2985,12 +2962,12 @@ static ExitStatus translate_one(DisasContext *ctx, uint32_t insn)
             case 0xE:
                 /* Longword virtual access with alternate access mode and
                    protection checks (hw_ldl/wa) */
-                tcg_gen_qemu_ld32s(cpu_ir[ra], addr, MMU_USER_IDX);
+                tcg_gen_qemu_ld_i64(cpu_ir[ra], addr, MMU_USER_IDX, MO_LESL);
                 break;
             case 0xF:
                 /* Quadword virtual access with alternate access mode and
                    protection checks (hw_ldq/wa) */
-                tcg_gen_qemu_ld64(cpu_ir[ra], addr, MMU_USER_IDX);
+                tcg_gen_qemu_ld_i64(cpu_ir[ra], addr, MMU_USER_IDX, MO_LEQ);
                 break;
             }
             tcg_temp_free(addr);
diff --git a/target-arm/Makefile.objs b/target-arm/Makefile.objs
index 6453f5c011..356fbfcdfd 100644
--- a/target-arm/Makefile.objs
+++ b/target-arm/Makefile.objs
@@ -1,7 +1,7 @@
 obj-y += arm-semi.o
 obj-$(CONFIG_SOFTMMU) += machine.o
 obj-$(CONFIG_KVM) += kvm.o
-obj-$(CONFIG_NO_KVM) += kvm-stub.o
+obj-$(call lnot,$(CONFIG_KVM)) += kvm-stub.o
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-y += neon_helper.o iwmmxt_helper.o
 obj-y += gdbstub.o
diff --git a/target-arm/cpu.h b/target-arm/cpu.h
index 2c56740bf6..9f110f15b6 100644
--- a/target-arm/cpu.h
+++ b/target-arm/cpu.h
@@ -176,6 +176,7 @@ typedef struct CPUARMState {
         uint32_t c9_pmxevtyper; /* perf monitor event type */
         uint32_t c9_pmuserenr; /* perf monitor user enable */
         uint32_t c9_pminten; /* perf monitor interrupt enables */
+        uint32_t c12_vbar; /* vector base address register */
         uint32_t c13_fcse; /* FCSE PID.  */
         uint32_t c13_context; /* Context ID.  */
         uint32_t c13_tls1; /* User RW Thread register.  */
diff --git a/target-arm/helper.c b/target-arm/helper.c
index 2a98be7436..3445813465 100644
--- a/target-arm/helper.c
+++ b/target-arm/helper.c
@@ -225,10 +225,16 @@ static void count_cpreg(gpointer key, gpointer opaque)
 
 static gint cpreg_key_compare(gconstpointer a, gconstpointer b)
 {
-    uint32_t aidx = *(uint32_t *)a;
-    uint32_t bidx = *(uint32_t *)b;
+    uint64_t aidx = cpreg_to_kvm_id(*(uint32_t *)a);
+    uint64_t bidx = cpreg_to_kvm_id(*(uint32_t *)b);
 
-    return aidx - bidx;
+    if (aidx > bidx) {
+        return 1;
+    }
+    if (aidx < bidx) {
+        return -1;
+    }
+    return 0;
 }
 
 static void cpreg_make_keylist(gpointer key, gpointer value, gpointer udata)
@@ -537,6 +543,13 @@ static int pmintenclr_write(CPUARMState *env, const ARMCPRegInfo *ri,
     return 0;
 }
 
+static int vbar_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                      uint64_t value)
+{
+    env->cp15.c12_vbar = value & ~0x1Ful;
+    return 0;
+}
+
 static int ccsidr_read(CPUARMState *env, const ARMCPRegInfo *ri,
                        uint64_t *value)
 {
@@ -622,6 +635,10 @@ static const ARMCPRegInfo v7_cp_reginfo[] = {
       .access = PL1_RW, .type = ARM_CP_NO_MIGRATE,
       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
       .resetvalue = 0, .writefn = pmintenclr_write, },
+    { .name = "VBAR", .cp = 15, .crn = 12, .crm = 0, .opc1 = 0, .opc2 = 0,
+      .access = PL1_RW, .writefn = vbar_write,
+      .fieldoffset = offsetof(CPUARMState, cp15.c12_vbar),
+      .resetvalue = 0 },
     { .name = "SCR", .cp = 15, .crn = 1, .crm = 1, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c1_scr),
       .resetvalue = 0, },
@@ -1749,7 +1766,6 @@ void register_cp_regs_for_features(ARMCPU *cpu)
 ARMCPU *cpu_arm_init(const char *cpu_model)
 {
     ARMCPU *cpu;
-    CPUARMState *env;
     ObjectClass *oc;
 
     oc = cpu_class_by_name(TYPE_ARM_CPU, cpu_model);
@@ -1757,8 +1773,6 @@ ARMCPU *cpu_arm_init(const char *cpu_model)
         return NULL;
     }
     cpu = ARM_CPU(object_new(object_class_get_name(oc)));
-    env = &cpu->env;
-    env->cpu_model_str = cpu_model;
 
     /* TODO this should be set centrally, once possible */
     object_property_set_bool(OBJECT(cpu), true, "realized", NULL);
@@ -2473,7 +2487,17 @@ void arm_cpu_do_interrupt(CPUState *cs)
     }
     /* High vectors.  */
     if (env->cp15.c1_sys & (1 << 13)) {
+        /* when enabled, base address cannot be remapped.  */
         addr += 0xffff0000;
+    } else {
+        /* ARM v7 architectures provide a vector base address register to remap
+         * the interrupt vector table.
+         * This register is only followed in non-monitor mode, and has a secure
+         * and un-secure copy. Since the cpu is always in a un-secure operation
+         * and is never in monitor mode this feature is always active.
+         * Note: only bits 31:5 are valid.
+         */
+        addr += env->cp15.c12_vbar;
     }
     switch_mode (env, new_mode);
     env->spsr = cpsr_read(env);
diff --git a/target-arm/helper.h b/target-arm/helper.h
index 63ae13acff..cac9564f5f 100644
--- a/target-arm/helper.h
+++ b/target-arm/helper.h
@@ -247,10 +247,10 @@ DEF_HELPER_3(neon_qshl_u32, i32, env, i32, i32)
 DEF_HELPER_3(neon_qshl_s32, i32, env, i32, i32)
 DEF_HELPER_3(neon_qshl_u64, i64, env, i64, i64)
 DEF_HELPER_3(neon_qshl_s64, i64, env, i64, i64)
-DEF_HELPER_3(neon_qshlu_s8, i32, env, i32, i32);
-DEF_HELPER_3(neon_qshlu_s16, i32, env, i32, i32);
-DEF_HELPER_3(neon_qshlu_s32, i32, env, i32, i32);
-DEF_HELPER_3(neon_qshlu_s64, i64, env, i64, i64);
+DEF_HELPER_3(neon_qshlu_s8, i32, env, i32, i32)
+DEF_HELPER_3(neon_qshlu_s16, i32, env, i32, i32)
+DEF_HELPER_3(neon_qshlu_s32, i32, env, i32, i32)
+DEF_HELPER_3(neon_qshlu_s64, i64, env, i64, i64)
 DEF_HELPER_3(neon_qrshl_u8, i32, env, i32, i32)
 DEF_HELPER_3(neon_qrshl_s8, i32, env, i32, i32)
 DEF_HELPER_3(neon_qrshl_u16, i32, env, i32, i32)
diff --git a/target-arm/iwmmxt_helper.c b/target-arm/iwmmxt_helper.c
index 7953b53f7e..e6cfa62da8 100644
--- a/target-arm/iwmmxt_helper.c
+++ b/target-arm/iwmmxt_helper.c
@@ -577,7 +577,7 @@ uint64_t HELPER(iwmmxt_rorl)(CPUARMState *env, uint64_t x, uint32_t n)
 
 uint64_t HELPER(iwmmxt_rorq)(CPUARMState *env, uint64_t x, uint32_t n)
 {
-    x = (x >> n) | (x << (64 - n));
+    x = ror64(x, n);
     env->iwmmxt.cregs[ARM_IWMMXT_wCASF] = NZBIT64(x);
     return x;
 }
diff --git a/target-arm/kvm.c b/target-arm/kvm.c
index b92e00dae0..6e5cd36fae 100644
--- a/target-arm/kvm.c
+++ b/target-arm/kvm.c
@@ -67,7 +67,13 @@ static bool reg_syncs_via_tuple_list(uint64_t regidx)
 
 static int compare_u64(const void *a, const void *b)
 {
-    return *(uint64_t *)a - *(uint64_t *)b;
+    if (*(uint64_t *)a > *(uint64_t *)b) {
+        return 1;
+    }
+    if (*(uint64_t *)a < *(uint64_t *)b) {
+        return -1;
+    }
+    return 0;
 }
 
 int kvm_arch_init_vcpu(CPUState *cs)
diff --git a/target-arm/translate.c b/target-arm/translate.c
index 998bde268d..5f003e785e 100644
--- a/target-arm/translate.c
+++ b/target-arm/translate.c
@@ -115,9 +115,6 @@ void arm_translate_init(void)
 #endif
 
     a64_translate_init();
-
-#define GEN_HELPER 2
-#include "helper.h"
 }
 
 static inline TCGv_i32 load_cpu_offset(int offset)
diff --git a/target-cris/helper.h b/target-cris/helper.h
index 8e8365cf69..0ac31f5670 100644
--- a/target-cris/helper.h
+++ b/target-cris/helper.h
@@ -4,14 +4,14 @@ DEF_HELPER_2(raise_exception, void, env, i32)
 DEF_HELPER_2(tlb_flush_pid, void, env, i32)
 DEF_HELPER_2(spc_write, void, env, i32)
 DEF_HELPER_3(dump, void, i32, i32, i32)
-DEF_HELPER_1(rfe, void, env);
-DEF_HELPER_1(rfn, void, env);
+DEF_HELPER_1(rfe, void, env)
+DEF_HELPER_1(rfn, void, env)
 
 DEF_HELPER_3(movl_sreg_reg, void, env, i32, i32)
 DEF_HELPER_3(movl_reg_sreg, void, env, i32, i32)
 
-DEF_HELPER_FLAGS_1(lz, TCG_CALL_NO_SE, i32, i32);
-DEF_HELPER_FLAGS_4(btst, TCG_CALL_NO_SE, i32, env, i32, i32, i32);
+DEF_HELPER_FLAGS_1(lz, TCG_CALL_NO_SE, i32, i32)
+DEF_HELPER_FLAGS_4(btst, TCG_CALL_NO_SE, i32, env, i32, i32, i32)
 
 DEF_HELPER_FLAGS_4(evaluate_flags_muls, TCG_CALL_NO_SE, i32, env, i32, i32, i32)
 DEF_HELPER_FLAGS_4(evaluate_flags_mulu, TCG_CALL_NO_SE, i32, env, i32, i32, i32)
diff --git a/target-cris/translate.c b/target-cris/translate.c
index 617e1b4242..5faa44c1ea 100644
--- a/target-cris/translate.c
+++ b/target-cris/translate.c
@@ -3480,9 +3480,6 @@ void cris_initialize_tcg(void)
 {
     int i;
 
-#define GEN_HELPER 2
-#include "helper.h"
-
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
     cc_x = tcg_global_mem_new(TCG_AREG0,
                               offsetof(CPUCRISState, cc_x), "cc_x");
diff --git a/target-i386/Makefile.objs b/target-i386/Makefile.objs
index da1fc404c7..027b94e1d3 100644
--- a/target-i386/Makefile.objs
+++ b/target-i386/Makefile.objs
@@ -4,6 +4,6 @@ obj-y += smm_helper.o misc_helper.o mem_helper.o seg_helper.o
 obj-y += gdbstub.o
 obj-$(CONFIG_SOFTMMU) += machine.o arch_memory_mapping.o arch_dump.o
 obj-$(CONFIG_KVM) += kvm.o
-obj-$(CONFIG_NO_KVM) += kvm-stub.o
+obj-$(call lnot,$(CONFIG_KVM)) += kvm-stub.o
 obj-$(CONFIG_LINUX_USER) += ioport-user.o
 obj-$(CONFIG_BSD_USER) += ioport-user.o
diff --git a/target-i386/arch_memory_mapping.c b/target-i386/arch_memory_mapping.c
index 2566a040a6..462f984a26 100644
--- a/target-i386/arch_memory_mapping.c
+++ b/target-i386/arch_memory_mapping.c
@@ -75,7 +75,7 @@ static void walk_pte2(MemoryMappingList *list,
 }
 
 /* PAE Paging or IA-32e Paging */
-#define PLM4_ADDR_MASK 0xffffffffff000 /* selects bits 51:12 */
+#define PLM4_ADDR_MASK 0xffffffffff000ULL /* selects bits 51:12 */
 
 static void walk_pde(MemoryMappingList *list, hwaddr pde_start_addr,
                      int32_t a20_mask, target_ulong start_line_addr)
diff --git a/target-i386/cpu-qom.h b/target-i386/cpu-qom.h
index c4447c2b6e..f4fab155bd 100644
--- a/target-i386/cpu-qom.h
+++ b/target-i386/cpu-qom.h
@@ -70,6 +70,9 @@ typedef struct X86CPU {
     bool hyperv_relaxed_timing;
     int hyperv_spinlock_attempts;
 
+    /* if true the CPUID code directly forward host cache leaves to the guest */
+    bool cache_info_passthrough;
+
     /* Features that were filtered out because of missing host capabilities */
     uint32_t filtered_features[FEATURE_WORDS];
 
diff --git a/target-i386/cpu.c b/target-i386/cpu.c
index c36345e426..864c80eb47 100644
--- a/target-i386/cpu.c
+++ b/target-i386/cpu.c
@@ -235,7 +235,7 @@ static const char *ext4_feature_name[] = {
 
 static const char *kvm_feature_name[] = {
     "kvmclock", "kvm_nopiodelay", "kvm_mmu", "kvmclock",
-    "kvm_asyncpf", "kvm_steal_time", "kvm_pv_eoi", NULL,
+    "kvm_asyncpf", "kvm_steal_time", "kvm_pv_eoi", "kvm_pv_unhalt",
     NULL, NULL, NULL, NULL,
     NULL, NULL, NULL, NULL,
     NULL, NULL, NULL, NULL,
@@ -328,6 +328,15 @@ X86RegisterInfo32 x86_reg_info_32[CPU_NB_REGS32] = {
 };
 #undef REGISTER
 
+typedef struct ExtSaveArea {
+    uint32_t feature, bits;
+    uint32_t offset, size;
+} ExtSaveArea;
+
+static const ExtSaveArea ext_save_areas[] = {
+    [2] = { .feature = FEAT_1_ECX, .bits = CPUID_EXT_AVX,
+            .offset = 0x100, .size = 0x240 },
+};
 
 const char *get_register_name_32(unsigned int reg)
 {
@@ -486,6 +495,7 @@ typedef struct x86_def_t {
     int stepping;
     FeatureWordArray features;
     char model_id[48];
+    bool cache_info_passthrough;
 } x86_def_t;
 
 #define I486_FEATURES (CPUID_FP87 | CPUID_VME | CPUID_PSE)
@@ -544,7 +554,7 @@ static x86_def_t builtin_x86_defs[] = {
         .level = 4,
         .vendor = CPUID_VENDOR_AMD,
         .family = 6,
-        .model = 2,
+        .model = 6,
         .stepping = 3,
         .features[FEAT_1_EDX] =
             PPRO_FEATURES |
@@ -647,7 +657,7 @@ static x86_def_t builtin_x86_defs[] = {
         .level = 4,
         .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
-        .model = 3,
+        .model = 6,
         .stepping = 3,
         .features[FEAT_1_EDX] =
             PPRO_FEATURES,
@@ -1139,6 +1149,7 @@ static void kvm_cpu_fill_host(x86_def_t *x86_cpu_def)
     assert(kvm_enabled());
 
     x86_cpu_def->name = "host";
+    x86_cpu_def->cache_info_passthrough = true;
     host_cpuid(0x0, 0, &eax, &ebx, &ecx, &edx);
     x86_cpu_vendor_words2str(x86_cpu_def->vendor, ebx, edx, ecx);
 
@@ -1888,6 +1899,7 @@ static void cpu_x86_register(X86CPU *cpu, const char *name, Error **errp)
     env->features[FEAT_C000_0001_EDX] = def->features[FEAT_C000_0001_EDX];
     env->features[FEAT_7_0_EBX] = def->features[FEAT_7_0_EBX];
     env->cpuid_xlevel2 = def->xlevel2;
+    cpu->cache_info_passthrough = def->cache_info_passthrough;
 
     object_property_set_str(OBJECT(cpu), def->model_id, "model-id", errp);
 }
@@ -1896,7 +1908,6 @@ X86CPU *cpu_x86_create(const char *cpu_model, DeviceState *icc_bridge,
                        Error **errp)
 {
     X86CPU *cpu = NULL;
-    CPUX86State *env;
     gchar **model_pieces;
     char *name, *features;
     char *typename;
@@ -1919,8 +1930,6 @@ X86CPU *cpu_x86_create(const char *cpu_model, DeviceState *icc_bridge,
     qdev_set_parent_bus(DEVICE(cpu), qdev_get_child_bus(icc_bridge, "icc"));
     object_unref(OBJECT(cpu));
 #endif
-    env = &cpu->env;
-    env->cpu_model_str = cpu_model;
 
     cpu_x86_register(cpu, name, &error);
     if (error) {
@@ -2062,6 +2071,10 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
         break;
     case 2:
         /* cache info: needed for Pentium Pro compatibility */
+        if (cpu->cache_info_passthrough) {
+            host_cpuid(index, 0, eax, ebx, ecx, edx);
+            break;
+        }
         *eax = 1; /* Number of CPUID[EAX=2] calls required */
         *ebx = 0;
         *ecx = 0;
@@ -2071,6 +2084,10 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
         break;
     case 4:
         /* cache info: needed for Core compatibility */
+        if (cpu->cache_info_passthrough) {
+            host_cpuid(index, count, eax, ebx, ecx, edx);
+            break;
+        }
         if (cs->nr_cores > 1) {
             *eax = (cs->nr_cores - 1) << 26;
         } else {
@@ -2169,29 +2186,51 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
             *edx = 0;
         }
         break;
-    case 0xD:
+    case 0xD: {
+        KVMState *s = cs->kvm_state;
+        uint64_t kvm_mask;
+        int i;
+
         /* Processor Extended State */
-        if (!(env->features[FEAT_1_ECX] & CPUID_EXT_XSAVE)) {
-            *eax = 0;
-            *ebx = 0;
-            *ecx = 0;
-            *edx = 0;
+        *eax = 0;
+        *ebx = 0;
+        *ecx = 0;
+        *edx = 0;
+        if (!(env->features[FEAT_1_ECX] & CPUID_EXT_XSAVE) || !kvm_enabled()) {
             break;
         }
-        if (kvm_enabled()) {
-            KVMState *s = cs->kvm_state;
+        kvm_mask =
+            kvm_arch_get_supported_cpuid(s, 0xd, 0, R_EAX) |
+            ((uint64_t)kvm_arch_get_supported_cpuid(s, 0xd, 0, R_EDX) << 32);
 
-            *eax = kvm_arch_get_supported_cpuid(s, 0xd, count, R_EAX);
-            *ebx = kvm_arch_get_supported_cpuid(s, 0xd, count, R_EBX);
-            *ecx = kvm_arch_get_supported_cpuid(s, 0xd, count, R_ECX);
-            *edx = kvm_arch_get_supported_cpuid(s, 0xd, count, R_EDX);
-        } else {
-            *eax = 0;
-            *ebx = 0;
-            *ecx = 0;
-            *edx = 0;
+        if (count == 0) {
+            *ecx = 0x240;
+            for (i = 2; i < ARRAY_SIZE(ext_save_areas); i++) {
+                const ExtSaveArea *esa = &ext_save_areas[i];
+                if ((env->features[esa->feature] & esa->bits) == esa->bits &&
+                    (kvm_mask & (1 << i)) != 0) {
+                    if (i < 32) {
+                        *eax |= 1 << i;
+                    } else {
+                        *edx |= 1 << (i - 32);
+                    }
+                    *ecx = MAX(*ecx, esa->offset + esa->size);
+                }
+            }
+            *eax |= kvm_mask & (XSTATE_FP | XSTATE_SSE);
+            *ebx = *ecx;
+        } else if (count == 1) {
+            *eax = kvm_arch_get_supported_cpuid(s, 0xd, 1, R_EAX);
+        } else if (count < ARRAY_SIZE(ext_save_areas)) {
+            const ExtSaveArea *esa = &ext_save_areas[count];
+            if ((env->features[esa->feature] & esa->bits) == esa->bits &&
+                (kvm_mask & (1 << count)) != 0) {
+                *eax = esa->offset;
+                *ebx = esa->size;
+            }
         }
         break;
+    }
     case 0x80000000:
         *eax = env->cpuid_xlevel;
         *ebx = env->cpuid_vendor1;
@@ -2228,6 +2267,10 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
         break;
     case 0x80000005:
         /* cache info (L1 cache) */
+        if (cpu->cache_info_passthrough) {
+            host_cpuid(index, 0, eax, ebx, ecx, edx);
+            break;
+        }
         *eax = (L1_DTLB_2M_ASSOC << 24) | (L1_DTLB_2M_ENTRIES << 16) | \
                (L1_ITLB_2M_ASSOC <<  8) | (L1_ITLB_2M_ENTRIES);
         *ebx = (L1_DTLB_4K_ASSOC << 24) | (L1_DTLB_4K_ENTRIES << 16) | \
@@ -2239,6 +2282,10 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
         break;
     case 0x80000006:
         /* cache info (L2 cache) */
+        if (cpu->cache_info_passthrough) {
+            host_cpuid(index, 0, eax, ebx, ecx, edx);
+            break;
+        }
         *eax = (AMD_ENC_ASSOC(L2_DTLB_2M_ASSOC) << 28) | \
                (L2_DTLB_2M_ENTRIES << 16) | \
                (AMD_ENC_ASSOC(L2_ITLB_2M_ASSOC) << 12) | \
@@ -2386,6 +2433,7 @@ static void x86_cpu_reset(CPUState *s)
     env->fpuc = 0x37f;
 
     env->mxcsr = 0x1f80;
+    env->xstate_bv = XSTATE_FP | XSTATE_SSE;
 
     env->pat = 0x0007040600070406ULL;
     env->msr_ia32_misc_enable = MSR_IA32_MISC_ENABLE_DEFAULT;
diff --git a/target-i386/cpu.h b/target-i386/cpu.h
index 5723eff9a8..ea373e82dc 100644
--- a/target-i386/cpu.h
+++ b/target-i386/cpu.h
@@ -380,6 +380,10 @@
 
 #define MSR_VM_HSAVE_PA                 0xc0010117
 
+#define XSTATE_FP                       1
+#define XSTATE_SSE                      2
+#define XSTATE_YMM                      4
+
 /* CPUID feature words */
 typedef enum FeatureWord {
     FEAT_1_EDX,         /* CPUID[1].EDX */
diff --git a/target-i386/helper.c b/target-i386/helper.c
index 7c58e274d9..7c196ffc42 100644
--- a/target-i386/helper.c
+++ b/target-i386/helper.c
@@ -147,7 +147,9 @@ cpu_x86_dump_seg_cache(CPUX86State *env, FILE *f, fprintf_function cpu_fprintf,
             cpu_fprintf(f, " [%c%c", (sc->flags & DESC_C_MASK) ? 'C' : '-',
                         (sc->flags & DESC_R_MASK) ? 'R' : '-');
         } else {
-            cpu_fprintf(f, (sc->flags & DESC_B_MASK) ? "DS  " : "DS16");
+            cpu_fprintf(f,
+                        (sc->flags & DESC_B_MASK || env->hflags & HF_LMA_MASK)
+                        ? "DS  " : "DS16");
             cpu_fprintf(f, " [%c%c", (sc->flags & DESC_E_MASK) ? 'E' : '-',
                         (sc->flags & DESC_W_MASK) ? 'W' : '-');
         }
@@ -188,8 +190,6 @@ void x86_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
     char cc_op_name[32];
     static const char *seg_name[6] = { "ES", "CS", "SS", "DS", "FS", "GS" };
 
-    cpu_synchronize_state(cs);
-
     eflags = cpu_compute_eflags(env);
 #ifdef TARGET_X86_64
     if (env->hflags & HF_CS64_MASK) {
@@ -894,7 +894,10 @@ hwaddr x86_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
     uint32_t page_offset;
     int page_size;
 
-    if (env->cr[4] & CR4_PAE_MASK) {
+    if (!(env->cr[0] & CR0_PG_MASK)) {
+        pte = addr & env->a20_mask;
+        page_size = 4096;
+    } else if (env->cr[4] & CR4_PAE_MASK) {
         target_ulong pdpe_addr;
         uint64_t pde, pdpe;
 
@@ -952,26 +955,21 @@ hwaddr x86_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
     } else {
         uint32_t pde;
 
-        if (!(env->cr[0] & CR0_PG_MASK)) {
-            pte = addr;
-            page_size = 4096;
+        /* page directory entry */
+        pde_addr = ((env->cr[3] & ~0xfff) + ((addr >> 20) & 0xffc)) & env->a20_mask;
+        pde = ldl_phys(pde_addr);
+        if (!(pde & PG_PRESENT_MASK))
+            return -1;
+        if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
+            pte = pde & ~0x003ff000; /* align to 4MB */
+            page_size = 4096 * 1024;
         } else {
             /* page directory entry */
-            pde_addr = ((env->cr[3] & ~0xfff) + ((addr >> 20) & 0xffc)) & env->a20_mask;
-            pde = ldl_phys(pde_addr);
-            if (!(pde & PG_PRESENT_MASK))
+            pte_addr = ((pde & ~0xfff) + ((addr >> 10) & 0xffc)) & env->a20_mask;
+            pte = ldl_phys(pte_addr);
+            if (!(pte & PG_PRESENT_MASK))
                 return -1;
-            if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
-                pte = pde & ~0x003ff000; /* align to 4MB */
-                page_size = 4096 * 1024;
-            } else {
-                /* page directory entry */
-                pte_addr = ((pde & ~0xfff) + ((addr >> 10) & 0xffc)) & env->a20_mask;
-                pte = ldl_phys(pte_addr);
-                if (!(pte & PG_PRESENT_MASK))
-                    return -1;
-                page_size = 4096;
-            }
+            page_size = 4096;
         }
         pte = pte & env->a20_mask;
     }
diff --git a/target-i386/machine.c b/target-i386/machine.c
index dc81cde535..e568da2ba4 100644
--- a/target-i386/machine.c
+++ b/target-i386/machine.c
@@ -330,9 +330,9 @@ static bool pv_eoi_msr_needed(void *opaque)
 
 static bool steal_time_msr_needed(void *opaque)
 {
-    CPUX86State *cpu = opaque;
+    X86CPU *cpu = opaque;
 
-    return cpu->steal_time_msr != 0;
+    return cpu->env.steal_time_msr != 0;
 }
 
 static const VMStateDescription vmstate_steal_time_msr = {
@@ -341,7 +341,7 @@ static const VMStateDescription vmstate_steal_time_msr = {
     .minimum_version_id = 1,
     .minimum_version_id_old = 1,
     .fields      = (VMStateField []) {
-        VMSTATE_UINT64(steal_time_msr, CPUX86State),
+        VMSTATE_UINT64(env.steal_time_msr, X86CPU),
         VMSTATE_END_OF_LIST()
     }
 };
diff --git a/target-i386/translate.c b/target-i386/translate.c
index 6d879003b3..eb0ea93dbb 100644
--- a/target-i386/translate.c
+++ b/target-i386/translate.c
@@ -6434,12 +6434,18 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 }
                 break;
             case 0x1d: /* fucomi */
+                if (!(s->cpuid_features & CPUID_CMOV)) {
+                    goto illegal_op;
+                }
                 gen_update_cc_op(s);
                 gen_helper_fmov_FT0_STN(cpu_env, tcg_const_i32(opreg));
                 gen_helper_fucomi_ST0_FT0(cpu_env);
                 set_cc_op(s, CC_OP_EFLAGS);
                 break;
             case 0x1e: /* fcomi */
+                if (!(s->cpuid_features & CPUID_CMOV)) {
+                    goto illegal_op;
+                }
                 gen_update_cc_op(s);
                 gen_helper_fmov_FT0_STN(cpu_env, tcg_const_i32(opreg));
                 gen_helper_fcomi_ST0_FT0(cpu_env);
@@ -6495,6 +6501,9 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 }
                 break;
             case 0x3d: /* fucomip */
+                if (!(s->cpuid_features & CPUID_CMOV)) {
+                    goto illegal_op;
+                }
                 gen_update_cc_op(s);
                 gen_helper_fmov_FT0_STN(cpu_env, tcg_const_i32(opreg));
                 gen_helper_fucomi_ST0_FT0(cpu_env);
@@ -6502,6 +6511,9 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 set_cc_op(s, CC_OP_EFLAGS);
                 break;
             case 0x3e: /* fcomip */
+                if (!(s->cpuid_features & CPUID_CMOV)) {
+                    goto illegal_op;
+                }
                 gen_update_cc_op(s);
                 gen_helper_fmov_FT0_STN(cpu_env, tcg_const_i32(opreg));
                 gen_helper_fcomi_ST0_FT0(cpu_env);
@@ -6518,6 +6530,10 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                         (JCC_BE << 1),
                         (JCC_P << 1),
                     };
+
+                    if (!(s->cpuid_features & CPUID_CMOV)) {
+                        goto illegal_op;
+                    }
                     op1 = fcmov_cc[op & 3] | (((op >> 3) & 1) ^ 1);
                     l1 = gen_new_label();
                     gen_jcc1_noeob(s, op1, l1);
@@ -6889,6 +6905,9 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         gen_ldst_modrm(env, s, modrm, OT_BYTE, OR_TMP0, 1);
         break;
     case 0x140 ... 0x14f: /* cmov Gv, Ev */
+        if (!(s->cpuid_features & CPUID_CMOV)) {
+            goto illegal_op;
+        }
         ot = dflag + OT_WORD;
         modrm = cpu_ldub_code(env, s->pc++);
         reg = ((modrm >> 3) & 7) | rex_r;
@@ -8242,10 +8261,6 @@ void optimize_flags_init(void)
     cpu_regs[R_EDI] = tcg_global_mem_new_i32(TCG_AREG0,
                                              offsetof(CPUX86State, regs[R_EDI]), "edi");
 #endif
-
-    /* register helpers */
-#define GEN_HELPER 2
-#include "helper.h"
 }
 
 /* generate intermediate code in gen_opc_buf and gen_opparam_buf for
diff --git a/target-m68k/helper.c b/target-m68k/helper.c
index 00a7a08e83..a364eb1e5c 100644
--- a/target-m68k/helper.c
+++ b/target-m68k/helper.c
@@ -21,7 +21,7 @@
 #include "cpu.h"
 #include "exec/gdbstub.h"
 
-#include "helpers.h"
+#include "helper.h"
 
 #define SIGNBIT (1u << 31)
 
@@ -110,7 +110,6 @@ M68kCPU *cpu_m68k_init(const char *cpu_model)
     }
     cpu = M68K_CPU(object_new(object_class_get_name(oc)));
     env = &cpu->env;
-    env->cpu_model_str = cpu_model;
 
     register_m68k_insns(env);
 
diff --git a/target-m68k/helpers.h b/target-m68k/helper.h
index 2b024502ba..2b024502ba 100644
--- a/target-m68k/helpers.h
+++ b/target-m68k/helper.h
diff --git a/target-m68k/op_helper.c b/target-m68k/op_helper.c
index 30f7d8b1ab..bbbfd7f130 100644
--- a/target-m68k/op_helper.c
+++ b/target-m68k/op_helper.c
@@ -17,7 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "cpu.h"
-#include "helpers.h"
+#include "helper.h"
 
 #if defined(CONFIG_USER_ONLY)
 
diff --git a/target-m68k/translate.c b/target-m68k/translate.c
index 0be0a96732..f54b94a53f 100644
--- a/target-m68k/translate.c
+++ b/target-m68k/translate.c
@@ -23,9 +23,9 @@
 #include "tcg-op.h"
 #include "qemu/log.h"
 
-#include "helpers.h"
+#include "helper.h"
 #define GEN_HELPER 1
-#include "helpers.h"
+#include "helper.h"
 
 //#define DEBUG_DISPATCH 1
 
@@ -108,9 +108,6 @@ void m68k_tcg_init(void)
 
     NULL_QREG = tcg_global_mem_new(TCG_AREG0, -4, "NULL");
     store_dummy = tcg_global_mem_new(TCG_AREG0, -8, "NULL");
-
-#define GEN_HELPER 2
-#include "helpers.h"
 }
 
 static inline void qemu_assert(int cond, const char *msg)
diff --git a/target-microblaze/cpu.h b/target-microblaze/cpu.h
index 7508cf5a06..e1415f043c 100644
--- a/target-microblaze/cpu.h
+++ b/target-microblaze/cpu.h
@@ -246,6 +246,7 @@ struct CPUMBState {
     /* lwx/swx reserved address */
 #define RES_ADDR_NONE 0xffffffff /* Use 0xffffffff to indicate no reservation */
     uint32_t res_addr;
+    uint32_t res_val;
 
     /* Internal flags.  */
 #define IMM_FLAG	4
diff --git a/target-microblaze/translate.c b/target-microblaze/translate.c
index 0673176957..9edcb67e66 100644
--- a/target-microblaze/translate.c
+++ b/target-microblaze/translate.c
@@ -49,6 +49,8 @@ static TCGv env_imm;
 static TCGv env_btaken;
 static TCGv env_btarget;
 static TCGv env_iflags;
+static TCGv env_res_addr;
+static TCGv env_res_val;
 
 #include "exec/gen-icount.h"
 
@@ -150,6 +152,10 @@ static void read_carry(DisasContext *dc, TCGv d)
     tcg_gen_shri_tl(d, cpu_SR[SR_MSR], 31);
 }
 
+/*
+ * write_carry sets the carry bits in MSR based on bit 0 of v.
+ * v[31:1] are ignored.
+ */
 static void write_carry(DisasContext *dc, TCGv v)
 {
     TCGv t0 = tcg_temp_new();
@@ -162,10 +168,10 @@ static void write_carry(DisasContext *dc, TCGv v)
     tcg_temp_free(t0);
 }
 
-static void write_carryi(DisasContext *dc, int carry)
+static void write_carryi(DisasContext *dc, bool carry)
 {
     TCGv t0 = tcg_temp_new();
-    tcg_gen_movi_tl(t0, carry ? 1 : 0);
+    tcg_gen_movi_tl(t0, carry);
     write_carry(dc, t0);
     tcg_temp_free(t0);
 }
@@ -386,10 +392,7 @@ static void dec_and(DisasContext *dc)
         return;
 
     if (not) {
-        TCGv t = tcg_temp_new();
-        tcg_gen_not_tl(t, *(dec_alu_op_b(dc)));
-        tcg_gen_and_tl(cpu_R[dc->rd], cpu_R[dc->ra], t);
-        tcg_temp_free(t);
+        tcg_gen_andc_tl(cpu_R[dc->rd], cpu_R[dc->ra], *(dec_alu_op_b(dc)));
     } else
         tcg_gen_and_tl(cpu_R[dc->rd], cpu_R[dc->ra], *(dec_alu_op_b(dc)));
 }
@@ -749,7 +752,7 @@ static void dec_barrel(DisasContext *dc)
 
 static void dec_bit(DisasContext *dc)
 {
-    TCGv t0, t1;
+    TCGv t0;
     unsigned int op;
     int mem_index = cpu_mmu_index(dc->env);
 
@@ -760,32 +763,22 @@ static void dec_bit(DisasContext *dc)
             t0 = tcg_temp_new();
 
             LOG_DIS("src r%d r%d\n", dc->rd, dc->ra);
-            tcg_gen_andi_tl(t0, cpu_R[dc->ra], 1);
+            tcg_gen_andi_tl(t0, cpu_SR[SR_MSR], MSR_CC);
+            write_carry(dc, cpu_R[dc->ra]);
             if (dc->rd) {
-                t1 = tcg_temp_new();
-                read_carry(dc, t1);
-                tcg_gen_shli_tl(t1, t1, 31);
-
                 tcg_gen_shri_tl(cpu_R[dc->rd], cpu_R[dc->ra], 1);
-                tcg_gen_or_tl(cpu_R[dc->rd], cpu_R[dc->rd], t1);
-                tcg_temp_free(t1);
+                tcg_gen_or_tl(cpu_R[dc->rd], cpu_R[dc->rd], t0);
             }
-
-            /* Update carry.  */
-            write_carry(dc, t0);
             tcg_temp_free(t0);
             break;
 
         case 0x1:
         case 0x41:
             /* srl.  */
-            t0 = tcg_temp_new();
             LOG_DIS("srl r%d r%d\n", dc->rd, dc->ra);
 
-            /* Update carry.  */
-            tcg_gen_andi_tl(t0, cpu_R[dc->ra], 1);
-            write_carry(dc, t0);
-            tcg_temp_free(t0);
+            /* Update carry. Note that write carry only looks at the LSB.  */
+            write_carry(dc, cpu_R[dc->ra]);
             if (dc->rd) {
                 if (op == 0x41)
                     tcg_gen_shri_tl(cpu_R[dc->rd], cpu_R[dc->ra], 1);
@@ -872,7 +865,7 @@ static void dec_imm(DisasContext *dc)
 }
 
 static inline void gen_load(DisasContext *dc, TCGv dst, TCGv addr,
-                            unsigned int size)
+                            unsigned int size, bool exclusive)
 {
     int mem_index = cpu_mmu_index(dc->env);
 
@@ -884,6 +877,11 @@ static inline void gen_load(DisasContext *dc, TCGv dst, TCGv addr,
         tcg_gen_qemu_ld32u(dst, addr, mem_index);
     } else
         cpu_abort(dc->env, "Incorrect load size %d\n", size);
+
+    if (exclusive) {
+        tcg_gen_mov_tl(env_res_addr, addr);
+        tcg_gen_mov_tl(env_res_val, dst);
+    }
 }
 
 static inline TCGv *compute_ldst_addr(DisasContext *dc, TCGv *t)
@@ -1055,7 +1053,7 @@ static void dec_load(DisasContext *dc)
          * into v. If the load succeeds, we verify alignment of the
          * address and if that succeeds we write into the destination reg.
          */
-        gen_load(dc, v, *addr, size);
+        gen_load(dc, v, *addr, size, ex);
 
         tcg_gen_movi_tl(cpu_SR[SR_PC], dc->pc);
         gen_helper_memalign(cpu_env, *addr, tcg_const_tl(dc->rd),
@@ -1070,20 +1068,19 @@ static void dec_load(DisasContext *dc)
         tcg_temp_free(v);
     } else {
         if (dc->rd) {
-            gen_load(dc, cpu_R[dc->rd], *addr, size);
+            gen_load(dc, cpu_R[dc->rd], *addr, size, ex);
             if (rev) {
                 dec_byteswap(dc, cpu_R[dc->rd], cpu_R[dc->rd], size);
             }
         } else {
             /* We are loading into r0, no need to reverse.  */
-            gen_load(dc, env_imm, *addr, size);
+            gen_load(dc, env_imm, *addr, size, ex);
         }
     }
 
     if (ex) { /* lwx */
         /* no support for for AXI exclusive so always clear C */
         write_carryi(dc, 0);
-        tcg_gen_st_tl(*addr, cpu_env, offsetof(CPUMBState, res_addr));
     }
 
     if (addr == &t)
@@ -1107,7 +1104,7 @@ static void gen_store(DisasContext *dc, TCGv addr, TCGv val,
 
 static void dec_store(DisasContext *dc)
 {
-    TCGv t, *addr, swx_addr, r_check;
+    TCGv t, *addr, swx_addr;
     int swx_skip = 0;
     unsigned int size, rev = 0, ex = 0;
 
@@ -1131,9 +1128,9 @@ static void dec_store(DisasContext *dc)
     sync_jmpstate(dc);
     addr = compute_ldst_addr(dc, &t);
 
-    r_check = tcg_temp_new();
     swx_addr = tcg_temp_local_new();
     if (ex) { /* swx */
+        TCGv tval;
 
         /* Force addr into the swx_addr. */
         tcg_gen_mov_tl(swx_addr, *addr);
@@ -1141,11 +1138,20 @@ static void dec_store(DisasContext *dc)
         /* swx does not throw unaligned access errors, so force alignment */
         tcg_gen_andi_tl(swx_addr, swx_addr, ~3);
 
-        tcg_gen_ld_tl(r_check, cpu_env, offsetof(CPUMBState, res_addr));
         write_carryi(dc, 1);
         swx_skip = gen_new_label();
-        tcg_gen_brcond_tl(TCG_COND_NE, r_check, swx_addr, swx_skip);
+        tcg_gen_brcond_tl(TCG_COND_NE, env_res_addr, swx_addr, swx_skip);
+
+        /* Compare the value loaded at lwx with current contents of
+           the reserved location.
+           FIXME: This only works for system emulation where we can expect
+           this compare and the following write to be atomic. For user
+           emulation we need to add atomicity between threads.  */
+        tval = tcg_temp_new();
+        gen_load(dc, tval, swx_addr, 4, false);
+        tcg_gen_brcond_tl(TCG_COND_NE, env_res_val, tval, swx_skip);
         write_carryi(dc, 0);
+        tcg_temp_free(tval);
     }
 
     if (rev && size != 4) {
@@ -1227,7 +1233,6 @@ static void dec_store(DisasContext *dc)
     if (ex) {
         gen_set_label(swx_skip);
     }
-    tcg_temp_free(r_check);
     tcg_temp_free(swx_addr);
 
     if (addr == &t)
@@ -2014,6 +2019,12 @@ void mb_tcg_init(void)
     env_btaken = tcg_global_mem_new(TCG_AREG0,
                      offsetof(CPUMBState, btaken),
                      "btaken");
+    env_res_addr = tcg_global_mem_new(TCG_AREG0,
+                     offsetof(CPUMBState, res_addr),
+                     "res_addr");
+    env_res_val = tcg_global_mem_new(TCG_AREG0,
+                     offsetof(CPUMBState, res_val),
+                     "res_val");
     for (i = 0; i < ARRAY_SIZE(cpu_R); i++) {
         cpu_R[i] = tcg_global_mem_new(TCG_AREG0,
                           offsetof(CPUMBState, regs[i]),
@@ -2024,8 +2035,6 @@ void mb_tcg_init(void)
                           offsetof(CPUMBState, sregs[i]),
                           special_regnames[i]);
     }
-#define GEN_HELPER 2
-#include "helper.h"
 }
 
 void restore_state_to_opc(CPUMBState *env, TranslationBlock *tb, int pc_pos)
diff --git a/target-mips/helper.h b/target-mips/helper.h
index ed75e2c9f2..1a8b86dea5 100644
--- a/target-mips/helper.h
+++ b/target-mips/helper.h
@@ -148,7 +148,7 @@ DEF_HELPER_2(mtc0_taghi, void, env, tl)
 DEF_HELPER_2(mtc0_datahi, void, env, tl)
 
 /* MIPS MT functions */
-DEF_HELPER_2(mftgpr, tl, env, i32);
+DEF_HELPER_2(mftgpr, tl, env, i32)
 DEF_HELPER_2(mftlo, tl, env, i32)
 DEF_HELPER_2(mfthi, tl, env, i32)
 DEF_HELPER_2(mftacx, tl, env, i32)
@@ -165,11 +165,11 @@ DEF_HELPER_1(evpe, tl, env)
 #endif /* !CONFIG_USER_ONLY */
 
 /* microMIPS functions */
-DEF_HELPER_4(lwm, void, env, tl, tl, i32);
-DEF_HELPER_4(swm, void, env, tl, tl, i32);
+DEF_HELPER_4(lwm, void, env, tl, tl, i32)
+DEF_HELPER_4(swm, void, env, tl, tl, i32)
 #ifdef TARGET_MIPS64
-DEF_HELPER_4(ldm, void, env, tl, tl, i32);
-DEF_HELPER_4(sdm, void, env, tl, tl, i32);
+DEF_HELPER_4(ldm, void, env, tl, tl, i32)
+DEF_HELPER_4(sdm, void, env, tl, tl, i32)
 #endif
 
 DEF_HELPER_2(fork, void, tl, tl)
@@ -615,7 +615,7 @@ DEF_HELPER_FLAGS_4(dmsubu, 0, void, tl, tl, i32, env)
 DEF_HELPER_FLAGS_1(bitrev, TCG_CALL_NO_RWG_SE, tl, tl)
 DEF_HELPER_FLAGS_3(insv, 0, tl, env, tl, tl)
 #if defined(TARGET_MIPS64)
-DEF_HELPER_FLAGS_3(dinsv, 0, tl, env, tl, tl);
+DEF_HELPER_FLAGS_3(dinsv, 0, tl, env, tl, tl)
 #endif
 
 /* DSP Compare-Pick Sub-class insns */
diff --git a/target-mips/translate.c b/target-mips/translate.c
index ad43d59103..67f326b205 100644
--- a/target-mips/translate.c
+++ b/target-mips/translate.c
@@ -15886,10 +15886,6 @@ void mips_tcg_init(void)
                                        offsetof(CPUMIPSState, active_fpu.fcr31),
                                        "fcr31");
 
-    /* register helpers */
-#define GEN_HELPER 2
-#include "helper.h"
-
     inited = 1;
 }
 
@@ -15907,7 +15903,6 @@ MIPSCPU *cpu_mips_init(const char *cpu_model)
     cpu = MIPS_CPU(object_new(TYPE_MIPS_CPU));
     env = &cpu->env;
     env->cpu_model = def;
-    env->cpu_model_str = cpu_model;
 
 #ifndef CONFIG_USER_ONLY
     mmu_init(env, def);
diff --git a/target-moxie/cpu.c b/target-moxie/cpu.c
index d97a091eb4..484ecc2124 100644
--- a/target-moxie/cpu.c
+++ b/target-moxie/cpu.c
@@ -138,7 +138,6 @@ MoxieCPU *cpu_moxie_init(const char *cpu_model)
         return NULL;
     }
     cpu = MOXIE_CPU(object_new(object_class_get_name(oc)));
-    cpu->env.cpu_model_str = cpu_model;
 
     object_property_set_bool(OBJECT(cpu), true, "realized", NULL);
 
diff --git a/target-openrisc/cpu.c b/target-openrisc/cpu.c
index 075f00a897..813794300b 100644
--- a/target-openrisc/cpu.c
+++ b/target-openrisc/cpu.c
@@ -209,7 +209,6 @@ OpenRISCCPU *cpu_openrisc_init(const char *cpu_model)
         return NULL;
     }
     cpu = OPENRISC_CPU(object_new(object_class_get_name(oc)));
-    cpu->env.cpu_model_str = cpu_model;
 
     object_property_set_bool(OBJECT(cpu), true, "realized", NULL);
 
diff --git a/target-openrisc/mmu.c b/target-openrisc/mmu.c
index 57f5616e9c..22d7cbec18 100644
--- a/target-openrisc/mmu.c
+++ b/target-openrisc/mmu.c
@@ -102,7 +102,7 @@ int cpu_openrisc_get_phys_data(OpenRISCCPU *cpu,
         }
     }
 
-    if ((rw & 0) && ((right & PAGE_READ) == 0)) {
+    if (!(rw & 1) && ((right & PAGE_READ) == 0)) {
         return TLBRET_BADADDR;
     }
     if ((rw & 1) && ((right & PAGE_WRITE) == 0)) {
@@ -122,13 +122,6 @@ static int cpu_openrisc_get_phys_addr(OpenRISCCPU *cpu,
 {
     int ret = TLBRET_MATCH;
 
-    /* [0x0000--0x2000]: unmapped */
-    if (address < 0x2000 && (cpu->env.sr & SR_SM)) {
-        *physical = address;
-        *prot = PAGE_READ | PAGE_WRITE;
-        return ret;
-    }
-
     if (rw == 2) {    /* ITLB */
        *physical = 0;
         ret = cpu->env.tlb->cpu_openrisc_map_address_code(cpu, physical,
diff --git a/target-openrisc/translate.c b/target-openrisc/translate.c
index 723b77d3b4..8908a2e32b 100644
--- a/target-openrisc/translate.c
+++ b/target-openrisc/translate.c
@@ -110,8 +110,6 @@ void openrisc_translate_init(void)
                                       offsetof(CPUOpenRISCState, gpr[i]),
                                       regnames[i]);
     }
-#define GEN_HELPER 2
-#include "helper.h"
 }
 
 /* Writeback SR_F transaltion-space to execution-space.  */
diff --git a/target-ppc/Makefile.objs b/target-ppc/Makefile.objs
index f72e3993f7..94d6d0c43b 100644
--- a/target-ppc/Makefile.objs
+++ b/target-ppc/Makefile.objs
@@ -5,7 +5,7 @@ obj-y += machine.o mmu_helper.o mmu-hash32.o
 obj-$(TARGET_PPC64) += mmu-hash64.o
 endif
 obj-$(CONFIG_KVM) += kvm.o kvm_ppc.o
-obj-$(CONFIG_NO_KVM) += kvm-stub.o
+obj-$(call lnot,$(CONFIG_KVM)) += kvm-stub.o
 obj-y += excp_helper.o
 obj-y += fpu_helper.o
 obj-y += int_helper.o
diff --git a/target-ppc/helper.h b/target-ppc/helper.h
index 56814b501f..6d282bb32d 100644
--- a/target-ppc/helper.h
+++ b/target-ppc/helper.h
@@ -168,8 +168,8 @@ DEF_HELPER_3(vslo, void, avr, avr, avr)
 DEF_HELPER_3(vsro, void, avr, avr, avr)
 DEF_HELPER_3(vaddcuw, void, avr, avr, avr)
 DEF_HELPER_3(vsubcuw, void, avr, avr, avr)
-DEF_HELPER_2(lvsl, void, avr, tl);
-DEF_HELPER_2(lvsr, void, avr, tl);
+DEF_HELPER_2(lvsl, void, avr, tl)
+DEF_HELPER_2(lvsr, void, avr, tl)
 DEF_HELPER_4(vaddsbs, void, env, avr, avr, avr)
 DEF_HELPER_4(vaddshs, void, env, avr, avr, avr)
 DEF_HELPER_4(vaddsws, void, env, avr, avr, avr)
@@ -220,7 +220,7 @@ DEF_HELPER_5(vmsumuhs, void, env, avr, avr, avr, avr)
 DEF_HELPER_5(vmsumshm, void, env, avr, avr, avr, avr)
 DEF_HELPER_5(vmsumshs, void, env, avr, avr, avr, avr)
 DEF_HELPER_4(vmladduhm, void, avr, avr, avr, avr)
-DEF_HELPER_2(mtvscr, void, env, avr);
+DEF_HELPER_2(mtvscr, void, env, avr)
 DEF_HELPER_3(lvebx, void, env, avr, tl)
 DEF_HELPER_3(lvehx, void, env, avr, tl)
 DEF_HELPER_3(lvewx, void, env, avr, tl)
@@ -349,7 +349,7 @@ DEF_HELPER_2(load_slb_vsid, tl, env, tl)
 DEF_HELPER_FLAGS_1(slbia, TCG_CALL_NO_RWG, void, env)
 DEF_HELPER_FLAGS_2(slbie, TCG_CALL_NO_RWG, void, env, tl)
 #endif
-DEF_HELPER_FLAGS_2(load_sr, TCG_CALL_NO_RWG, tl, env, tl);
+DEF_HELPER_FLAGS_2(load_sr, TCG_CALL_NO_RWG, tl, env, tl)
 DEF_HELPER_FLAGS_3(store_sr, TCG_CALL_NO_RWG, void, env, tl, tl)
 
 DEF_HELPER_FLAGS_1(602_mfrom, TCG_CALL_NO_RWG_SE, tl, tl)
@@ -367,7 +367,7 @@ DEF_HELPER_3(divo, tl, env, tl, tl)
 DEF_HELPER_3(divs, tl, env, tl, tl)
 DEF_HELPER_3(divso, tl, env, tl, tl)
 
-DEF_HELPER_2(load_dcr, tl, env, tl);
+DEF_HELPER_2(load_dcr, tl, env, tl)
 DEF_HELPER_3(store_dcr, void, env, tl, tl)
 
 DEF_HELPER_2(load_dump_spr, void, env, i32)
diff --git a/target-ppc/translate.c b/target-ppc/translate.c
index 2da7bc740f..66c777174c 100644
--- a/target-ppc/translate.c
+++ b/target-ppc/translate.c
@@ -175,10 +175,6 @@ void ppc_translate_init(void)
     cpu_access_type = tcg_global_mem_new_i32(TCG_AREG0,
                                              offsetof(CPUPPCState, access_type), "access_type");
 
-    /* register helpers */
-#define GEN_HELPER 2
-#include "helper.h"
-
     done_init = 1;
 }
 
@@ -9536,8 +9532,6 @@ void ppc_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
     CPUPPCState *env = &cpu->env;
     int i;
 
-    cpu_synchronize_state(cs);
-
     cpu_fprintf(f, "NIP " TARGET_FMT_lx "   LR " TARGET_FMT_lx " CTR "
                 TARGET_FMT_lx " XER " TARGET_FMT_lx "\n",
                 env->nip, env->lr, env->ctr, cpu_read_xer(env));
diff --git a/target-ppc/translate_init.c b/target-ppc/translate_init.c
index d2645bad28..651da6b0d5 100644
--- a/target-ppc/translate_init.c
+++ b/target-ppc/translate_init.c
@@ -8267,7 +8267,6 @@ static ObjectClass *ppc_cpu_class_by_name(const char *name)
 PowerPCCPU *cpu_ppc_init(const char *cpu_model)
 {
     PowerPCCPU *cpu;
-    CPUPPCState *env;
     ObjectClass *oc;
     Error *err = NULL;
 
@@ -8277,8 +8276,6 @@ PowerPCCPU *cpu_ppc_init(const char *cpu_model)
     }
 
     cpu = POWERPC_CPU(object_new(object_class_get_name(oc)));
-    env = &cpu->env;
-    env->cpu_model_str = cpu_model;
 
     object_property_set_bool(OBJECT(cpu), true, "realized", &err);
     if (err != NULL) {
diff --git a/target-s390x/arch_dump.c b/target-s390x/arch_dump.c
index 9d36116242..5cbb53ca2e 100644
--- a/target-s390x/arch_dump.c
+++ b/target-s390x/arch_dump.c
@@ -151,6 +151,7 @@ static int s390x_write_all_elf64_notes(const char *note_name,
     int ret = -1;
 
     for (nf = note_func; nf->note_contents_func; nf++) {
+        memset(&note, 0, sizeof(note));
         note.hdr.n_namesz = cpu_to_be32(sizeof(note.name));
         note.hdr.n_descsz = cpu_to_be32(nf->contents_size);
         strncpy(note.name, note_name, sizeof(note.name));
diff --git a/target-s390x/cpu.h b/target-s390x/cpu.h
index 8be5648806..a2c077bdcd 100644
--- a/target-s390x/cpu.h
+++ b/target-s390x/cpu.h
@@ -148,6 +148,7 @@ typedef struct CPUS390XState {
 } CPUS390XState;
 
 #include "cpu-qom.h"
+#include <sysemu/kvm.h>
 
 /* distinguish between 24 bit and 31 bit addressing */
 #define HIGH_ORDER_BIT 0x80000000
@@ -692,6 +693,14 @@ static inline const char *cc_name(int cc_op)
     return cc_names[cc_op];
 }
 
+static inline void setcc(S390CPU *cpu, uint64_t cc)
+{
+    CPUS390XState *env = &cpu->env;
+
+    env->psw.mask &= ~(3ull << 44);
+    env->psw.mask |= (cc & 3) << 44;
+}
+
 typedef struct LowCore
 {
     /* prefix area: defined by architecture */
@@ -1058,8 +1067,6 @@ void program_interrupt(CPUS390XState *env, uint32_t code, int ilen);
 void QEMU_NORETURN runtime_exception(CPUS390XState *env, int excp,
                                      uintptr_t retaddr);
 
-#include <sysemu/kvm.h>
-
 #ifdef CONFIG_KVM
 void kvm_s390_io_interrupt(S390CPU *cpu, uint16_t subchannel_id,
                            uint16_t subchannel_nr, uint32_t io_int_parm,
diff --git a/target-s390x/helper.c b/target-s390x/helper.c
index 61abfd7d9e..da33b38009 100644
--- a/target-s390x/helper.c
+++ b/target-s390x/helper.c
@@ -73,11 +73,8 @@ void s390x_cpu_timer(void *opaque)
 S390CPU *cpu_s390x_init(const char *cpu_model)
 {
     S390CPU *cpu;
-    CPUS390XState *env;
 
     cpu = S390_CPU(object_new(TYPE_S390_CPU));
-    env = &cpu->env;
-    env->cpu_model_str = cpu_model;
 
     object_property_set_bool(OBJECT(cpu), true, "realized", NULL);
 
diff --git a/target-s390x/ioinst.c b/target-s390x/ioinst.c
index 85fd285736..8d6363df4e 100644
--- a/target-s390x/ioinst.c
+++ b/target-s390x/ioinst.c
@@ -36,7 +36,7 @@ int ioinst_disassemble_sch_ident(uint32_t value, int *m, int *cssid, int *ssid,
     return 0;
 }
 
-int ioinst_handle_xsch(CPUS390XState *env, uint64_t reg1)
+void ioinst_handle_xsch(S390CPU *cpu, uint64_t reg1)
 {
     int cssid, ssid, schid, m;
     SubchDev *sch;
@@ -44,8 +44,8 @@ int ioinst_handle_xsch(CPUS390XState *env, uint64_t reg1)
     int cc;
 
     if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid)) {
-        program_interrupt(env, PGM_OPERAND, 2);
-        return -EIO;
+        program_interrupt(&cpu->env, PGM_OPERAND, 2);
+        return;
     }
     trace_ioinst_sch_id("xsch", cssid, ssid, schid);
     sch = css_find_subch(m, cssid, ssid, schid);
@@ -66,11 +66,10 @@ int ioinst_handle_xsch(CPUS390XState *env, uint64_t reg1)
         cc = 1;
         break;
     }
-
-    return cc;
+    setcc(cpu, cc);
 }
 
-int ioinst_handle_csch(CPUS390XState *env, uint64_t reg1)
+void ioinst_handle_csch(S390CPU *cpu, uint64_t reg1)
 {
     int cssid, ssid, schid, m;
     SubchDev *sch;
@@ -78,8 +77,8 @@ int ioinst_handle_csch(CPUS390XState *env, uint64_t reg1)
     int cc;
 
     if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid)) {
-        program_interrupt(env, PGM_OPERAND, 2);
-        return -EIO;
+        program_interrupt(&cpu->env, PGM_OPERAND, 2);
+        return;
     }
     trace_ioinst_sch_id("csch", cssid, ssid, schid);
     sch = css_find_subch(m, cssid, ssid, schid);
@@ -91,10 +90,10 @@ int ioinst_handle_csch(CPUS390XState *env, uint64_t reg1)
     } else {
         cc = 0;
     }
-    return cc;
+    setcc(cpu, cc);
 }
 
-int ioinst_handle_hsch(CPUS390XState *env, uint64_t reg1)
+void ioinst_handle_hsch(S390CPU *cpu, uint64_t reg1)
 {
     int cssid, ssid, schid, m;
     SubchDev *sch;
@@ -102,8 +101,8 @@ int ioinst_handle_hsch(CPUS390XState *env, uint64_t reg1)
     int cc;
 
     if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid)) {
-        program_interrupt(env, PGM_OPERAND, 2);
-        return -EIO;
+        program_interrupt(&cpu->env, PGM_OPERAND, 2);
+        return;
     }
     trace_ioinst_sch_id("hsch", cssid, ssid, schid);
     sch = css_find_subch(m, cssid, ssid, schid);
@@ -124,8 +123,7 @@ int ioinst_handle_hsch(CPUS390XState *env, uint64_t reg1)
         cc = 1;
         break;
     }
-
-    return cc;
+    setcc(cpu, cc);
 }
 
 static int ioinst_schib_valid(SCHIB *schib)
@@ -141,7 +139,7 @@ static int ioinst_schib_valid(SCHIB *schib)
     return 1;
 }
 
-int ioinst_handle_msch(CPUS390XState *env, uint64_t reg1, uint32_t ipb)
+void ioinst_handle_msch(S390CPU *cpu, uint64_t reg1, uint32_t ipb)
 {
     int cssid, ssid, schid, m;
     SubchDev *sch;
@@ -150,22 +148,21 @@ int ioinst_handle_msch(CPUS390XState *env, uint64_t reg1, uint32_t ipb)
     int ret = -ENODEV;
     int cc;
     hwaddr len = sizeof(*schib);
+    CPUS390XState *env = &cpu->env;
 
     addr = decode_basedisp_s(env, ipb);
     if (addr & 3) {
         program_interrupt(env, PGM_SPECIFICATION, 2);
-        return -EIO;
+        return;
     }
     schib = s390_cpu_physical_memory_map(env, addr, &len, 0);
     if (!schib || len != sizeof(*schib)) {
         program_interrupt(env, PGM_ADDRESSING, 2);
-        cc = -EIO;
         goto out;
     }
     if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid) ||
         !ioinst_schib_valid(schib)) {
         program_interrupt(env, PGM_OPERAND, 2);
-        cc = -EIO;
         goto out;
     }
     trace_ioinst_sch_id("msch", cssid, ssid, schid);
@@ -187,9 +184,10 @@ int ioinst_handle_msch(CPUS390XState *env, uint64_t reg1, uint32_t ipb)
         cc = 1;
         break;
     }
+    setcc(cpu, cc);
+
 out:
     s390_cpu_physical_memory_unmap(env, schib, len, 0);
-    return cc;
 }
 
 static void copy_orb_from_guest(ORB *dest, const ORB *src)
@@ -213,7 +211,7 @@ static int ioinst_orb_valid(ORB *orb)
     return 1;
 }
 
-int ioinst_handle_ssch(CPUS390XState *env, uint64_t reg1, uint32_t ipb)
+void ioinst_handle_ssch(S390CPU *cpu, uint64_t reg1, uint32_t ipb)
 {
     int cssid, ssid, schid, m;
     SubchDev *sch;
@@ -222,23 +220,22 @@ int ioinst_handle_ssch(CPUS390XState *env, uint64_t reg1, uint32_t ipb)
     int ret = -ENODEV;
     int cc;
     hwaddr len = sizeof(*orig_orb);
+    CPUS390XState *env = &cpu->env;
 
     addr = decode_basedisp_s(env, ipb);
     if (addr & 3) {
         program_interrupt(env, PGM_SPECIFICATION, 2);
-        return -EIO;
+        return;
     }
     orig_orb = s390_cpu_physical_memory_map(env, addr, &len, 0);
     if (!orig_orb || len != sizeof(*orig_orb)) {
         program_interrupt(env, PGM_ADDRESSING, 2);
-        cc = -EIO;
         goto out;
     }
     copy_orb_from_guest(&orb, orig_orb);
     if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid) ||
         !ioinst_orb_valid(&orb)) {
         program_interrupt(env, PGM_OPERAND, 2);
-        cc = -EIO;
         goto out;
     }
     trace_ioinst_sch_id("ssch", cssid, ssid, schid);
@@ -260,38 +257,39 @@ int ioinst_handle_ssch(CPUS390XState *env, uint64_t reg1, uint32_t ipb)
         cc = 1;
         break;
     }
+    setcc(cpu, cc);
 
 out:
     s390_cpu_physical_memory_unmap(env, orig_orb, len, 0);
-    return cc;
 }
 
-int ioinst_handle_stcrw(CPUS390XState *env, uint32_t ipb)
+void ioinst_handle_stcrw(S390CPU *cpu, uint32_t ipb)
 {
     CRW *crw;
     uint64_t addr;
     int cc;
     hwaddr len = sizeof(*crw);
+    CPUS390XState *env = &cpu->env;
 
     addr = decode_basedisp_s(env, ipb);
     if (addr & 3) {
         program_interrupt(env, PGM_SPECIFICATION, 2);
-        return -EIO;
+        return;
     }
     crw = s390_cpu_physical_memory_map(env, addr, &len, 1);
     if (!crw || len != sizeof(*crw)) {
         program_interrupt(env, PGM_ADDRESSING, 2);
-        cc = -EIO;
         goto out;
     }
     cc = css_do_stcrw(crw);
     /* 0 - crw stored, 1 - zeroes stored */
+    setcc(cpu, cc);
+
 out:
     s390_cpu_physical_memory_unmap(env, crw, len, 1);
-    return cc;
 }
 
-int ioinst_handle_stsch(CPUS390XState *env, uint64_t reg1, uint32_t ipb)
+void ioinst_handle_stsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb)
 {
     int cssid, ssid, schid, m;
     SubchDev *sch;
@@ -299,22 +297,21 @@ int ioinst_handle_stsch(CPUS390XState *env, uint64_t reg1, uint32_t ipb)
     int cc;
     SCHIB *schib;
     hwaddr len = sizeof(*schib);
+    CPUS390XState *env = &cpu->env;
 
     addr = decode_basedisp_s(env, ipb);
     if (addr & 3) {
         program_interrupt(env, PGM_SPECIFICATION, 2);
-        return -EIO;
+        return;
     }
     schib = s390_cpu_physical_memory_map(env, addr, &len, 1);
     if (!schib || len != sizeof(*schib)) {
         program_interrupt(env, PGM_ADDRESSING, 2);
-        cc = -EIO;
         goto out;
     }
 
     if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid)) {
         program_interrupt(env, PGM_OPERAND, 2);
-        cc = -EIO;
         goto out;
     }
     trace_ioinst_sch_id("stsch", cssid, ssid, schid);
@@ -336,9 +333,10 @@ int ioinst_handle_stsch(CPUS390XState *env, uint64_t reg1, uint32_t ipb)
             cc = 0;
         }
     }
+    setcc(cpu, cc);
+
 out:
     s390_cpu_physical_memory_unmap(env, schib, len, 1);
-    return cc;
 }
 
 int ioinst_handle_tsch(CPUS390XState *env, uint64_t reg1, uint32_t ipb)
@@ -575,7 +573,7 @@ static void ioinst_handle_chsc_unimplemented(ChscResp *res)
     res->param = 0;
 }
 
-int ioinst_handle_chsc(CPUS390XState *env, uint32_t ipb)
+void ioinst_handle_chsc(S390CPU *cpu, uint32_t ipb)
 {
     ChscReq *req;
     ChscResp *res;
@@ -584,7 +582,7 @@ int ioinst_handle_chsc(CPUS390XState *env, uint32_t ipb)
     uint16_t len;
     uint16_t command;
     hwaddr map_size = TARGET_PAGE_SIZE;
-    int ret = 0;
+    CPUS390XState *env = &cpu->env;
 
     trace_ioinst("chsc");
     reg = (ipb >> 20) & 0x00f;
@@ -592,19 +590,17 @@ int ioinst_handle_chsc(CPUS390XState *env, uint32_t ipb)
     /* Page boundary? */
     if (addr & 0xfff) {
         program_interrupt(env, PGM_SPECIFICATION, 2);
-        return -EIO;
+        return;
     }
     req = s390_cpu_physical_memory_map(env, addr, &map_size, 1);
     if (!req || map_size != TARGET_PAGE_SIZE) {
         program_interrupt(env, PGM_ADDRESSING, 2);
-        ret = -EIO;
         goto out;
     }
     len = be16_to_cpu(req->len);
     /* Length field valid? */
     if ((len < 16) || (len > 4088) || (len & 7)) {
         program_interrupt(env, PGM_OPERAND, 2);
-        ret = -EIO;
         goto out;
     }
     memset((char *)req + len, 0, TARGET_PAGE_SIZE - len);
@@ -628,7 +624,6 @@ int ioinst_handle_chsc(CPUS390XState *env, uint32_t ipb)
 
 out:
     s390_cpu_physical_memory_unmap(env, req, map_size, 1);
-    return ret;
 }
 
 int ioinst_handle_tpi(CPUS390XState *env, uint32_t ipb)
@@ -666,18 +661,19 @@ out:
 #define SCHM_REG1_UPD(_reg) ((_reg & 0x0000000000000002) >> 1)
 #define SCHM_REG1_DCT(_reg) (_reg & 0x0000000000000001)
 
-int ioinst_handle_schm(CPUS390XState *env, uint64_t reg1, uint64_t reg2,
-                       uint32_t ipb)
+void ioinst_handle_schm(S390CPU *cpu, uint64_t reg1, uint64_t reg2,
+                        uint32_t ipb)
 {
     uint8_t mbk;
     int update;
     int dct;
+    CPUS390XState *env = &cpu->env;
 
     trace_ioinst("schm");
 
     if (SCHM_REG1_RES(reg1)) {
         program_interrupt(env, PGM_OPERAND, 2);
-        return -EIO;
+        return;
     }
 
     mbk = SCHM_REG1_MBK(reg1);
@@ -686,15 +682,13 @@ int ioinst_handle_schm(CPUS390XState *env, uint64_t reg1, uint64_t reg2,
 
     if (update && (reg2 & 0x000000000000001f)) {
         program_interrupt(env, PGM_OPERAND, 2);
-        return -EIO;
+        return;
     }
 
     css_do_schm(mbk, update, dct, update ? reg2 : 0);
-
-    return 0;
 }
 
-int ioinst_handle_rsch(CPUS390XState *env, uint64_t reg1)
+void ioinst_handle_rsch(S390CPU *cpu, uint64_t reg1)
 {
     int cssid, ssid, schid, m;
     SubchDev *sch;
@@ -702,8 +696,8 @@ int ioinst_handle_rsch(CPUS390XState *env, uint64_t reg1)
     int cc;
 
     if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid)) {
-        program_interrupt(env, PGM_OPERAND, 2);
-        return -EIO;
+        program_interrupt(&cpu->env, PGM_OPERAND, 2);
+        return;
     }
     trace_ioinst_sch_id("rsch", cssid, ssid, schid);
     sch = css_find_subch(m, cssid, ssid, schid);
@@ -724,24 +718,23 @@ int ioinst_handle_rsch(CPUS390XState *env, uint64_t reg1)
         cc = 1;
         break;
     }
-
-    return cc;
-
+    setcc(cpu, cc);
 }
 
 #define RCHP_REG1_RES(_reg) (_reg & 0x00000000ff00ff00)
 #define RCHP_REG1_CSSID(_reg) ((_reg & 0x0000000000ff0000) >> 16)
 #define RCHP_REG1_CHPID(_reg) (_reg & 0x00000000000000ff)
-int ioinst_handle_rchp(CPUS390XState *env, uint64_t reg1)
+void ioinst_handle_rchp(S390CPU *cpu, uint64_t reg1)
 {
     int cc;
     uint8_t cssid;
     uint8_t chpid;
     int ret;
+    CPUS390XState *env = &cpu->env;
 
     if (RCHP_REG1_RES(reg1)) {
         program_interrupt(env, PGM_OPERAND, 2);
-        return -EIO;
+        return;
     }
 
     cssid = RCHP_REG1_CSSID(reg1);
@@ -764,19 +757,16 @@ int ioinst_handle_rchp(CPUS390XState *env, uint64_t reg1)
     default:
         /* Invalid channel subsystem. */
         program_interrupt(env, PGM_OPERAND, 2);
-        return -EIO;
+        return;
     }
-
-    return cc;
+    setcc(cpu, cc);
 }
 
 #define SAL_REG1_INVALID(_reg) (_reg & 0x0000000080000000)
-int ioinst_handle_sal(CPUS390XState *env, uint64_t reg1)
+void ioinst_handle_sal(S390CPU *cpu, uint64_t reg1)
 {
     /* We do not provide address limit checking, so let's suppress it. */
     if (SAL_REG1_INVALID(reg1) || reg1 & 0x000000000000ffff) {
-        program_interrupt(env, PGM_OPERAND, 2);
-        return -EIO;
+        program_interrupt(&cpu->env, PGM_OPERAND, 2);
     }
-    return 0;
 }
diff --git a/target-s390x/ioinst.h b/target-s390x/ioinst.h
index 7bed2910dc..613da49b3b 100644
--- a/target-s390x/ioinst.h
+++ b/target-s390x/ioinst.h
@@ -214,20 +214,20 @@ typedef struct IOIntCode {
 
 int ioinst_disassemble_sch_ident(uint32_t value, int *m, int *cssid, int *ssid,
                                  int *schid);
-int ioinst_handle_xsch(CPUS390XState *env, uint64_t reg1);
-int ioinst_handle_csch(CPUS390XState *env, uint64_t reg1);
-int ioinst_handle_hsch(CPUS390XState *env, uint64_t reg1);
-int ioinst_handle_msch(CPUS390XState *env, uint64_t reg1, uint32_t ipb);
-int ioinst_handle_ssch(CPUS390XState *env, uint64_t reg1, uint32_t ipb);
-int ioinst_handle_stcrw(CPUS390XState *env, uint32_t ipb);
-int ioinst_handle_stsch(CPUS390XState *env, uint64_t reg1, uint32_t ipb);
+void ioinst_handle_xsch(S390CPU *cpu, uint64_t reg1);
+void ioinst_handle_csch(S390CPU *cpu, uint64_t reg1);
+void ioinst_handle_hsch(S390CPU *cpu, uint64_t reg1);
+void ioinst_handle_msch(S390CPU *cpu, uint64_t reg1, uint32_t ipb);
+void ioinst_handle_ssch(S390CPU *cpu, uint64_t reg1, uint32_t ipb);
+void ioinst_handle_stcrw(S390CPU *cpu, uint32_t ipb);
+void ioinst_handle_stsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb);
 int ioinst_handle_tsch(CPUS390XState *env, uint64_t reg1, uint32_t ipb);
-int ioinst_handle_chsc(CPUS390XState *env, uint32_t ipb);
+void ioinst_handle_chsc(S390CPU *cpu, uint32_t ipb);
 int ioinst_handle_tpi(CPUS390XState *env, uint32_t ipb);
-int ioinst_handle_schm(CPUS390XState *env, uint64_t reg1, uint64_t reg2,
-                       uint32_t ipb);
-int ioinst_handle_rsch(CPUS390XState *env, uint64_t reg1);
-int ioinst_handle_rchp(CPUS390XState *env, uint64_t reg1);
-int ioinst_handle_sal(CPUS390XState *env, uint64_t reg1);
+void ioinst_handle_schm(S390CPU *cpu, uint64_t reg1, uint64_t reg2,
+                        uint32_t ipb);
+void ioinst_handle_rsch(S390CPU *cpu, uint64_t reg1);
+void ioinst_handle_rchp(S390CPU *cpu, uint64_t reg1);
+void ioinst_handle_sal(S390CPU *cpu, uint64_t reg1);
 
 #endif
diff --git a/target-s390x/kvm.c b/target-s390x/kvm.c
index 185c8f5a45..02ac4ba995 100644
--- a/target-s390x/kvm.c
+++ b/target-s390x/kvm.c
@@ -93,9 +93,15 @@ const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
 
 static int cap_sync_regs;
 
+static void *legacy_s390_alloc(size_t size);
+
 int kvm_arch_init(KVMState *s)
 {
     cap_sync_regs = kvm_check_extension(s, KVM_CAP_SYNC_REGS);
+    if (!kvm_check_extension(s, KVM_CAP_S390_GMAP)
+        || !kvm_check_extension(s, KVM_CAP_S390_COW)) {
+        phys_mem_set_alloc(legacy_s390_alloc);
+    }
     return 0;
 }
 
@@ -319,29 +325,14 @@ int kvm_s390_get_registers_partial(CPUState *cs)
  * to grow. We also have to use MAP parameters that avoid
  * read-only mapping of guest pages.
  */
-static void *legacy_s390_alloc(ram_addr_t size)
+static void *legacy_s390_alloc(size_t size)
 {
     void *mem;
 
     mem = mmap((void *) 0x800000000ULL, size,
                PROT_EXEC|PROT_READ|PROT_WRITE,
                MAP_SHARED | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
-    if (mem == MAP_FAILED) {
-        fprintf(stderr, "Allocating RAM failed\n");
-        abort();
-    }
-    return mem;
-}
-
-void *kvm_arch_ram_alloc(ram_addr_t size)
-{
-    /* Can we use the standard allocation ? */
-    if (kvm_check_extension(kvm_state, KVM_CAP_S390_GMAP) &&
-        kvm_check_extension(kvm_state, KVM_CAP_S390_COW)) {
-        return NULL;
-    } else {
-        return legacy_s390_alloc(size);
-    }
+    return mem == MAP_FAILED ? NULL : mem;
 }
 
 int kvm_arch_insert_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
@@ -427,18 +418,6 @@ static void enter_pgmcheck(S390CPU *cpu, uint16_t code)
     kvm_s390_interrupt(cpu, KVM_S390_PROGRAM_INT, code);
 }
 
-static inline void setcc(S390CPU *cpu, uint64_t cc)
-{
-    CPUS390XState *env = &cpu->env;
-    CPUState *cs = CPU(cpu);
-
-    cs->kvm_run->psw_mask &= ~(3ull << 44);
-    cs->kvm_run->psw_mask |= (cc & 3) << 44;
-
-    env->psw.mask &= ~(3ul << 44);
-    env->psw.mask |= (cc & 3) << 44;
-}
-
 static int kvm_sclp_service_call(S390CPU *cpu, struct kvm_run *run,
                                  uint16_t ipbh0)
 {
@@ -448,6 +427,10 @@ static int kvm_sclp_service_call(S390CPU *cpu, struct kvm_run *run,
     int r = 0;
 
     cpu_synchronize_state(CPU(cpu));
+    if (env->psw.mask & PSW_MASK_PSTATE) {
+        enter_pgmcheck(cpu, PGM_PRIVILEGED);
+        return 0;
+    }
     sccb = env->regs[ipbh0 & 0xf];
     code = env->regs[(ipbh0 & 0xf0) >> 4];
 
@@ -463,8 +446,6 @@ static int kvm_sclp_service_call(S390CPU *cpu, struct kvm_run *run,
 static int kvm_handle_css_inst(S390CPU *cpu, struct kvm_run *run,
                                uint8_t ipa0, uint8_t ipa1, uint8_t ipb)
 {
-    int r = 0;
-    int no_cc = 0;
     CPUS390XState *env = &cpu->env;
     CPUState *cs = CPU(cpu);
 
@@ -478,69 +459,61 @@ static int kvm_handle_css_inst(S390CPU *cpu, struct kvm_run *run,
 
     switch (ipa1) {
     case PRIV_XSCH:
-        r = ioinst_handle_xsch(env, env->regs[1]);
+        ioinst_handle_xsch(cpu, env->regs[1]);
         break;
     case PRIV_CSCH:
-        r = ioinst_handle_csch(env, env->regs[1]);
+        ioinst_handle_csch(cpu, env->regs[1]);
         break;
     case PRIV_HSCH:
-        r = ioinst_handle_hsch(env, env->regs[1]);
+        ioinst_handle_hsch(cpu, env->regs[1]);
         break;
     case PRIV_MSCH:
-        r = ioinst_handle_msch(env, env->regs[1], run->s390_sieic.ipb);
+        ioinst_handle_msch(cpu, env->regs[1], run->s390_sieic.ipb);
         break;
     case PRIV_SSCH:
-        r = ioinst_handle_ssch(env, env->regs[1], run->s390_sieic.ipb);
+        ioinst_handle_ssch(cpu, env->regs[1], run->s390_sieic.ipb);
         break;
     case PRIV_STCRW:
-        r = ioinst_handle_stcrw(env, run->s390_sieic.ipb);
+        ioinst_handle_stcrw(cpu, run->s390_sieic.ipb);
         break;
     case PRIV_STSCH:
-        r = ioinst_handle_stsch(env, env->regs[1], run->s390_sieic.ipb);
+        ioinst_handle_stsch(cpu, env->regs[1], run->s390_sieic.ipb);
         break;
     case PRIV_TSCH:
         /* We should only get tsch via KVM_EXIT_S390_TSCH. */
         fprintf(stderr, "Spurious tsch intercept\n");
         break;
     case PRIV_CHSC:
-        r = ioinst_handle_chsc(env, run->s390_sieic.ipb);
+        ioinst_handle_chsc(cpu, run->s390_sieic.ipb);
         break;
     case PRIV_TPI:
         /* This should have been handled by kvm already. */
         fprintf(stderr, "Spurious tpi intercept\n");
         break;
     case PRIV_SCHM:
-        no_cc = 1;
-        r = ioinst_handle_schm(env, env->regs[1], env->regs[2],
-                               run->s390_sieic.ipb);
+        ioinst_handle_schm(cpu, env->regs[1], env->regs[2],
+                           run->s390_sieic.ipb);
         break;
     case PRIV_RSCH:
-        r = ioinst_handle_rsch(env, env->regs[1]);
+        ioinst_handle_rsch(cpu, env->regs[1]);
         break;
     case PRIV_RCHP:
-        r = ioinst_handle_rchp(env, env->regs[1]);
+        ioinst_handle_rchp(cpu, env->regs[1]);
         break;
     case PRIV_STCPS:
         /* We do not provide this instruction, it is suppressed. */
-        no_cc = 1;
-        r = 0;
         break;
     case PRIV_SAL:
-        no_cc = 1;
-        r = ioinst_handle_sal(env, env->regs[1]);
+        ioinst_handle_sal(cpu, env->regs[1]);
         break;
     case PRIV_SIGA:
         /* Not provided, set CC = 3 for subchannel not operational */
-        r = 3;
+        setcc(cpu, 3);
         break;
     default:
         return -1;
     }
 
-    if (r >= 0 && !no_cc) {
-        setcc(cpu, r);
-    }
-
     return 0;
 }
 
diff --git a/target-s390x/misc_helper.c b/target-s390x/misc_helper.c
index 1690907169..10d04252d5 100644
--- a/target-s390x/misc_helper.c
+++ b/target-s390x/misc_helper.c
@@ -33,6 +33,7 @@
 #include "exec/softmmu_exec.h"
 #include "sysemu/cpus.h"
 #include "sysemu/sysemu.h"
+#include "hw/s390x/ebcdic.h"
 #endif
 
 /* #define DEBUG_HELPER */
@@ -72,86 +73,6 @@ void HELPER(exception)(CPUS390XState *env, uint32_t excp)
 
 #ifndef CONFIG_USER_ONLY
 
-/* EBCDIC handling */
-static const uint8_t ebcdic2ascii[] = {
-    0x00, 0x01, 0x02, 0x03, 0x07, 0x09, 0x07, 0x7F,
-    0x07, 0x07, 0x07, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
-    0x10, 0x11, 0x12, 0x13, 0x07, 0x0A, 0x08, 0x07,
-    0x18, 0x19, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07,
-    0x07, 0x07, 0x1C, 0x07, 0x07, 0x0A, 0x17, 0x1B,
-    0x07, 0x07, 0x07, 0x07, 0x07, 0x05, 0x06, 0x07,
-    0x07, 0x07, 0x16, 0x07, 0x07, 0x07, 0x07, 0x04,
-    0x07, 0x07, 0x07, 0x07, 0x14, 0x15, 0x07, 0x1A,
-    0x20, 0xFF, 0x83, 0x84, 0x85, 0xA0, 0x07, 0x86,
-    0x87, 0xA4, 0x5B, 0x2E, 0x3C, 0x28, 0x2B, 0x21,
-    0x26, 0x82, 0x88, 0x89, 0x8A, 0xA1, 0x8C, 0x07,
-    0x8D, 0xE1, 0x5D, 0x24, 0x2A, 0x29, 0x3B, 0x5E,
-    0x2D, 0x2F, 0x07, 0x8E, 0x07, 0x07, 0x07, 0x8F,
-    0x80, 0xA5, 0x07, 0x2C, 0x25, 0x5F, 0x3E, 0x3F,
-    0x07, 0x90, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07,
-    0x70, 0x60, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22,
-    0x07, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
-    0x68, 0x69, 0xAE, 0xAF, 0x07, 0x07, 0x07, 0xF1,
-    0xF8, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70,
-    0x71, 0x72, 0xA6, 0xA7, 0x91, 0x07, 0x92, 0x07,
-    0xE6, 0x7E, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
-    0x79, 0x7A, 0xAD, 0xAB, 0x07, 0x07, 0x07, 0x07,
-    0x9B, 0x9C, 0x9D, 0xFA, 0x07, 0x07, 0x07, 0xAC,
-    0xAB, 0x07, 0xAA, 0x7C, 0x07, 0x07, 0x07, 0x07,
-    0x7B, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
-    0x48, 0x49, 0x07, 0x93, 0x94, 0x95, 0xA2, 0x07,
-    0x7D, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50,
-    0x51, 0x52, 0x07, 0x96, 0x81, 0x97, 0xA3, 0x98,
-    0x5C, 0xF6, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58,
-    0x59, 0x5A, 0xFD, 0x07, 0x99, 0x07, 0x07, 0x07,
-    0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
-    0x38, 0x39, 0x07, 0x07, 0x9A, 0x07, 0x07, 0x07,
-};
-
-static const uint8_t ascii2ebcdic[] = {
-    0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F,
-    0x16, 0x05, 0x15, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
-    0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26,
-    0x18, 0x19, 0x3F, 0x27, 0x22, 0x1D, 0x1E, 0x1F,
-    0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D,
-    0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61,
-    0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
-    0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F,
-    0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
-    0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6,
-    0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6,
-    0xE7, 0xE8, 0xE9, 0xBA, 0xE0, 0xBB, 0xB0, 0x6D,
-    0x79, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
-    0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96,
-    0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6,
-    0xA7, 0xA8, 0xA9, 0xC0, 0x4F, 0xD0, 0xA1, 0x07,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x59, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x90, 0x3F, 0x3F, 0x3F, 0x3F, 0xEA, 0x3F, 0xFF
-};
-
-static inline void ebcdic_put(uint8_t *p, const char *ascii, int len)
-{
-    int i;
-
-    for (i = 0; i < len; i++) {
-        p[i] = ascii2ebcdic[(uint8_t)ascii[i]];
-    }
-}
-
 void program_interrupt(CPUS390XState *env, uint32_t code, int ilen)
 {
     qemu_log_mask(CPU_LOG_INT, "program interrupt at %#" PRIx64 "\n",
@@ -192,6 +113,29 @@ static void cpu_reset_all(void)
     }
 }
 
+static void cpu_full_reset_all(void)
+{
+    CPUState *cpu;
+
+    CPU_FOREACH(cpu) {
+        cpu_reset(cpu);
+    }
+}
+
+static int modified_clear_reset(S390CPU *cpu)
+{
+    S390CPUClass *scc = S390_CPU_GET_CLASS(cpu);
+
+    pause_all_vcpus();
+    cpu_synchronize_all_states();
+    cpu_full_reset_all();
+    io_subsystem_reset();
+    scc->load_normal(CPU(cpu));
+    cpu_synchronize_all_post_reset();
+    resume_all_vcpus();
+    return 0;
+}
+
 static int load_normal_reset(S390CPU *cpu)
 {
     S390CPUClass *scc = S390_CPU_GET_CLASS(cpu);
@@ -225,6 +169,9 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3)
     }
 
     switch (subcode) {
+    case 0:
+        modified_clear_reset(s390_env_get_cpu(env));
+        break;
     case 1:
         load_normal_reset(s390_env_get_cpu(env));
         break;
diff --git a/target-s390x/translate.c b/target-s390x/translate.c
index afe90eb8be..bc99a378a7 100644
--- a/target-s390x/translate.c
+++ b/target-s390x/translate.c
@@ -188,10 +188,6 @@ void s390x_translate_init(void)
                                       offsetof(CPUS390XState, fregs[i].d),
                                       cpu_reg_names[i + 16]);
     }
-
-    /* register helpers */
-#define GEN_HELPER 2
-#include "helper.h"
 }
 
 static TCGv_i64 load_reg(int reg)
diff --git a/target-sh4/cpu.c b/target-sh4/cpu.c
index 34b2b57ba7..c23294d410 100644
--- a/target-sh4/cpu.c
+++ b/target-sh4/cpu.c
@@ -144,7 +144,6 @@ static ObjectClass *superh_cpu_class_by_name(const char *cpu_model)
 SuperHCPU *cpu_sh4_init(const char *cpu_model)
 {
     SuperHCPU *cpu;
-    CPUSH4State *env;
     ObjectClass *oc;
 
     oc = superh_cpu_class_by_name(cpu_model);
@@ -152,8 +151,6 @@ SuperHCPU *cpu_sh4_init(const char *cpu_model)
         return NULL;
     }
     cpu = SUPERH_CPU(object_new(object_class_get_name(oc)));
-    env = &cpu->env;
-    env->cpu_model_str = cpu_model;
 
     object_property_set_bool(OBJECT(cpu), true, "realized", NULL);
 
diff --git a/target-sh4/translate.c b/target-sh4/translate.c
index c06b29f1dc..2272eb0beb 100644
--- a/target-sh4/translate.c
+++ b/target-sh4/translate.c
@@ -143,10 +143,6 @@ void sh4_translate_init(void)
                                               offsetof(CPUSH4State, fregs[i]),
                                               fregnames[i]);
 
-    /* register helpers */
-#define GEN_HELPER 2
-#include "helper.h"
-
     done_init = 1;
 }
 
diff --git a/target-sparc/cpu.c b/target-sparc/cpu.c
index 47ce60de4a..e7f878ee81 100644
--- a/target-sparc/cpu.c
+++ b/target-sparc/cpu.c
@@ -84,7 +84,6 @@ static int cpu_sparc_register(CPUSPARCState *env, const char *cpu_model)
         env->def->features |= CPU_FEATURE_FLOAT128;
     }
 #endif
-    env->cpu_model_str = cpu_model;
     env->version = def->iu_version;
     env->fsr = def->fpu_version;
     env->nwindows = def->nwindows;
diff --git a/target-sparc/helper.h b/target-sparc/helper.h
index 15f73283fa..2a771b2093 100644
--- a/target-sparc/helper.h
+++ b/target-sparc/helper.h
@@ -103,7 +103,7 @@ DEF_HELPER_3(fmuls, f32, env, f32, f32)
 DEF_HELPER_3(fdivs, f32, env, f32, f32)
 
 DEF_HELPER_3(fsmuld, f64, env, f32, f32)
-DEF_HELPER_3(fdmulq, void, env, f64, f64);
+DEF_HELPER_3(fdmulq, void, env, f64, f64)
 
 DEF_HELPER_FLAGS_1(fnegs, TCG_CALL_NO_RWG_SE, f32, f32)
 DEF_HELPER_2(fitod, f64, env, s32)
@@ -156,22 +156,22 @@ DEF_HELPER_FLAGS_3(bshuffle, TCG_CALL_NO_RWG_SE, i64, i64, i64, i64)
     DEF_HELPER_FLAGS_2(f ## name ## 32s, TCG_CALL_NO_RWG_SE, \
                        i32, i32, i32)
 
-VIS_HELPER(padd);
-VIS_HELPER(psub);
+VIS_HELPER(padd)
+VIS_HELPER(psub)
 #define VIS_CMPHELPER(name)                                              \
     DEF_HELPER_FLAGS_2(f##name##16, TCG_CALL_NO_RWG_SE,      \
                        i64, i64, i64)                                    \
     DEF_HELPER_FLAGS_2(f##name##32, TCG_CALL_NO_RWG_SE,      \
                        i64, i64, i64)
-VIS_CMPHELPER(cmpgt);
-VIS_CMPHELPER(cmpeq);
-VIS_CMPHELPER(cmple);
-VIS_CMPHELPER(cmpne);
+VIS_CMPHELPER(cmpgt)
+VIS_CMPHELPER(cmpeq)
+VIS_CMPHELPER(cmple)
+VIS_CMPHELPER(cmpne)
 #endif
 #undef F_HELPER_0_1
 #undef VIS_HELPER
 #undef VIS_CMPHELPER
-DEF_HELPER_1(compute_psr, void, env);
-DEF_HELPER_1(compute_C_icc, i32, env);
+DEF_HELPER_1(compute_psr, void, env)
+DEF_HELPER_1(compute_C_icc, i32, env)
 
 #include "exec/def-helper.h"
diff --git a/target-sparc/translate.c b/target-sparc/translate.c
index 36615f1979..dce64c3c4a 100644
--- a/target-sparc/translate.c
+++ b/target-sparc/translate.c
@@ -5456,11 +5456,6 @@ void gen_intermediate_code_init(CPUSPARCState *env)
                                                 offsetof(CPUSPARCState, fpr[i]),
                                                 fregnames[i]);
         }
-
-        /* register helpers */
-
-#define GEN_HELPER 2
-#include "helper.h"
     }
 }
 
diff --git a/target-unicore32/helper.c b/target-unicore32/helper.c
index 61eb2c374a..9bf4fea5db 100644
--- a/target-unicore32/helper.c
+++ b/target-unicore32/helper.c
@@ -37,7 +37,6 @@ CPUUniCore32State *uc32_cpu_init(const char *cpu_model)
     }
     cpu = UNICORE32_CPU(object_new(object_class_get_name(oc)));
     env = &cpu->env;
-    env->cpu_model_str = cpu_model;
 
     object_property_set_bool(OBJECT(cpu), true, "realized", NULL);
 
diff --git a/target-unicore32/translate.c b/target-unicore32/translate.c
index 1246895f86..4572890ffa 100644
--- a/target-unicore32/translate.c
+++ b/target-unicore32/translate.c
@@ -74,9 +74,6 @@ void uc32_translate_init(void)
         cpu_R[i] = tcg_global_mem_new_i32(TCG_AREG0,
                                 offsetof(CPUUniCore32State, regs[i]), regnames[i]);
     }
-
-#define GEN_HELPER 2
-#include "helper.h"
 }
 
 static int num_temps;
diff --git a/target-xtensa/translate.c b/target-xtensa/translate.c
index 24343bdf60..2d2df33115 100644
--- a/target-xtensa/translate.c
+++ b/target-xtensa/translate.c
@@ -238,8 +238,6 @@ void xtensa_translate_init(void)
                     uregnames[i].name);
         }
     }
-#define GEN_HELPER 2
-#include "helper.h"
 }
 
 static inline bool option_bits_enabled(DisasContext *dc, uint64_t opt)
@@ -3018,6 +3016,14 @@ void gen_intermediate_code_internal(XtensaCPU *cpu,
     gen_tb_end(tb, insn_count);
     *tcg_ctx.gen_opc_ptr = INDEX_op_end;
 
+#ifdef DEBUG_DISAS
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+        qemu_log("----------------\n");
+        qemu_log("IN: %s\n", lookup_symbol(pc_start));
+        log_target_disas(env, pc_start, dc.pc - pc_start, 0);
+        qemu_log("\n");
+    }
+#endif
     if (search_pc) {
         j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         memset(tcg_ctx.gen_opc_instr_start + lj + 1, 0,
diff --git a/tcg/README b/tcg/README
index 063aeb95ea..f1782123b7 100644
--- a/tcg/README
+++ b/tcg/README
@@ -412,30 +412,25 @@ current TB was linked to this TB. Otherwise execute the next
 instructions. Only indices 0 and 1 are valid and tcg_gen_goto_tb may be issued
 at most once with each slot index per TB.
 
-* qemu_ld8u t0, t1, flags
-qemu_ld8s t0, t1, flags
-qemu_ld16u t0, t1, flags
-qemu_ld16s t0, t1, flags
-qemu_ld32 t0, t1, flags
-qemu_ld32u t0, t1, flags
-qemu_ld32s t0, t1, flags
-qemu_ld64 t0, t1, flags
-
-Load data at the QEMU CPU address t1 into t0. t1 has the QEMU CPU address
-type. 'flags' contains the QEMU memory index (selects user or kernel access)
-for example.
-
-Note that "qemu_ld32" implies a 32-bit result, while "qemu_ld32u" and
-"qemu_ld32s" imply a 64-bit result appropriately extended from 32 bits.
-
-* qemu_st8 t0, t1, flags
-qemu_st16 t0, t1, flags
-qemu_st32 t0, t1, flags
-qemu_st64 t0, t1, flags
-
-Store the data t0 at the QEMU CPU Address t1. t1 has the QEMU CPU
-address type. 'flags' contains the QEMU memory index (selects user or
-kernel access) for example.
+* qemu_ld_i32/i64 t0, t1, flags, memidx
+* qemu_st_i32/i64 t0, t1, flags, memidx
+
+Load data at the guest address t1 into t0, or store data in t0 at guest
+address t1.  The _i32/_i64 size applies to the size of the input/output
+register t0 only.  The address t1 is always sized according to the guest,
+and the width of the memory operation is controlled by flags.
+
+Both t0 and t1 may be split into little-endian ordered pairs of registers
+if dealing with 64-bit quantities on a 32-bit host.
+
+The memidx selects the qemu tlb index to use (e.g. user or kernel access).
+The flags are the TCGMemOp bits, selecting the sign, width, and endianness
+of the memory access.
+
+For a 32-bit host, qemu_ld/st_i64 is guaranteed to only be used with a
+64-bit memory access specified in flags.
+
+*********
 
 Note 1: Some shortcuts are defined when the last operand is known to be
 a constant (e.g. addi for add, movi for mov).
diff --git a/tcg/aarch64/tcg-target.c b/tcg/aarch64/tcg-target.c
index 6379df1f68..04d7ae328d 100644
--- a/tcg/aarch64/tcg-target.c
+++ b/tcg/aarch64/tcg-target.c
@@ -10,6 +10,7 @@
  * See the COPYING file in the top-level directory for details.
  */
 
+#include "tcg-be-ldst.h"
 #include "qemu/bitops.h"
 
 #ifndef NDEBUG
@@ -778,22 +779,24 @@ static inline void tcg_out_nop(TCGContext *s)
 }
 
 #ifdef CONFIG_SOFTMMU
-/* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
-   int mmu_idx) */
+/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
+ *                                     int mmu_idx, uintptr_t ra)
+ */
 static const void * const qemu_ld_helpers[4] = {
-    helper_ldb_mmu,
-    helper_ldw_mmu,
-    helper_ldl_mmu,
-    helper_ldq_mmu,
+    helper_ret_ldub_mmu,
+    helper_ret_lduw_mmu,
+    helper_ret_ldul_mmu,
+    helper_ret_ldq_mmu,
 };
 
-/* helper signature: helper_st_mmu(CPUState *env, target_ulong addr,
-   uintxx_t val, int mmu_idx) */
+/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,
+ *                                     uintxx_t val, int mmu_idx, uintptr_t ra)
+ */
 static const void * const qemu_st_helpers[4] = {
-    helper_stb_mmu,
-    helper_stw_mmu,
-    helper_stl_mmu,
-    helper_stq_mmu,
+    helper_ret_stb_mmu,
+    helper_ret_stw_mmu,
+    helper_ret_stl_mmu,
+    helper_ret_stq_mmu,
 };
 
 static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
@@ -802,6 +805,7 @@ static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
     tcg_out_movr(s, 1, TCG_REG_X0, TCG_AREG0);
     tcg_out_movr(s, (TARGET_LONG_BITS == 64), TCG_REG_X1, lb->addrlo_reg);
     tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X2, lb->mem_index);
+    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_X3, (tcg_target_long)lb->raddr);
     tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP,
                  (tcg_target_long)qemu_ld_helpers[lb->opc & 3]);
     tcg_out_callr(s, TCG_REG_TMP);
@@ -822,6 +826,7 @@ static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
     tcg_out_movr(s, (TARGET_LONG_BITS == 64), TCG_REG_X1, lb->addrlo_reg);
     tcg_out_movr(s, 1, TCG_REG_X2, lb->datalo_reg);
     tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X3, lb->mem_index);
+    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_X4, (tcg_target_long)lb->raddr);
     tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP,
                  (tcg_target_long)qemu_st_helpers[lb->opc & 3]);
     tcg_out_callr(s, TCG_REG_TMP);
@@ -830,33 +835,13 @@ static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
     tcg_out_goto(s, (tcg_target_long)lb->raddr);
 }
 
-void tcg_out_tb_finalize(TCGContext *s)
-{
-    int i;
-    for (i = 0; i < s->nb_qemu_ldst_labels; i++) {
-        TCGLabelQemuLdst *label = &s->qemu_ldst_labels[i];
-        if (label->is_ld) {
-            tcg_out_qemu_ld_slow_path(s, label);
-        } else {
-            tcg_out_qemu_st_slow_path(s, label);
-        }
-    }
-}
-
 static void add_qemu_ldst_label(TCGContext *s, int is_ld, int opc,
                                 TCGReg data_reg, TCGReg addr_reg,
                                 int mem_index,
                                 uint8_t *raddr, uint8_t *label_ptr)
 {
-    int idx;
-    TCGLabelQemuLdst *label;
-
-    if (s->nb_qemu_ldst_labels >= TCG_MAX_QEMU_LDST) {
-        tcg_abort();
-    }
+    TCGLabelQemuLdst *label = new_ldst_label(s);
 
-    idx = s->nb_qemu_ldst_labels++;
-    label = &s->qemu_ldst_labels[idx];
     label->is_ld = is_ld;
     label->opc = opc;
     label->datalo_reg = data_reg;
diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h
index d3a1bc2437..82ad919518 100644
--- a/tcg/aarch64/tcg-target.h
+++ b/tcg/aarch64/tcg-target.h
@@ -96,6 +96,8 @@ enum {
     TCG_AREG0 = TCG_REG_X19,
 };
 
+#define TCG_TARGET_HAS_new_ldst         0
+
 static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
 {
     __builtin___clear_cache((char *)start, (char *)stop);
diff --git a/tcg/arm/tcg-target.c b/tcg/arm/tcg-target.c
index eb0e84ce44..e93a4a237b 100644
--- a/tcg/arm/tcg-target.c
+++ b/tcg/arm/tcg-target.c
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "tcg-be-ldst.h"
+
 /* The __ARM_ARCH define is provided by gcc 4.8.  Construct it otherwise.  */
 #ifndef __ARM_ARCH
 # if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) \
@@ -175,24 +177,16 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
         ct->ct |= TCG_CT_REG;
         tcg_regset_set32(ct->u.regs, 0, (1 << TCG_TARGET_NB_REGS) - 1);
 #ifdef CONFIG_SOFTMMU
-        /* r0-r2 will be overwritten when reading the tlb entry,
+        /* r0-r2,lr will be overwritten when reading the tlb entry,
            so don't use these. */
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R0);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R1);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R2);
-#endif
-        break;
-    case 'L':
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, (1 << TCG_TARGET_NB_REGS) - 1);
-#ifdef CONFIG_SOFTMMU
-        /* r1 is still needed to load data_reg or data_reg2,
-           so don't use it. */
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R1);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R14);
 #endif
         break;
 
-    /* qemu_st address & data_reg */
+    /* qemu_st address & data */
     case 's':
         ct->ct |= TCG_CT_REG;
         tcg_regset_set32(ct->u.regs, 0, (1 << TCG_TARGET_NB_REGS) - 1);
@@ -207,6 +201,7 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
         /* Avoid clashes with registers being used for helper args */
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R3);
 #endif
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R14);
 #endif
         break;
 
@@ -320,6 +315,9 @@ typedef enum {
     INSN_STRB_REG  = 0x06400000,
 
     INSN_LDRD_IMM  = 0x004000d0,
+    INSN_LDRD_REG  = 0x000000d0,
+    INSN_STRD_IMM  = 0x004000f0,
+    INSN_STRD_REG  = 0x000000f0,
 } ARMInsn;
 
 #define SHIFT_IMM_LSL(im)	(((im) << 7) | 0x00)
@@ -379,13 +377,17 @@ static inline void tcg_out_b_noaddr(TCGContext *s, int cond)
     /* We pay attention here to not modify the branch target by skipping
        the corresponding bytes. This ensure that caches and memory are
        kept coherent during retranslation. */
-#ifdef HOST_WORDS_BIGENDIAN
-    tcg_out8(s, (cond << 4) | 0x0a);
-    s->code_ptr += 3;
-#else
     s->code_ptr += 3;
     tcg_out8(s, (cond << 4) | 0x0a);
-#endif
+}
+
+static inline void tcg_out_bl_noaddr(TCGContext *s, int cond)
+{
+    /* We pay attention here to not modify the branch target by skipping
+       the corresponding bytes. This ensure that caches and memory are
+       kept coherent during retranslation. */
+    s->code_ptr += 3;
+    tcg_out8(s, (cond << 4) | 0x0b);
 }
 
 static inline void tcg_out_bl(TCGContext *s, int cond, int32_t offset)
@@ -810,6 +812,30 @@ static inline void tcg_out_st32_r(TCGContext *s, int cond, TCGReg rt,
     tcg_out_memop_r(s, cond, INSN_STR_REG, rt, rn, rm, 1, 1, 0);
 }
 
+static inline void tcg_out_ldrd_8(TCGContext *s, int cond, TCGReg rt,
+                                   TCGReg rn, int imm8)
+{
+    tcg_out_memop_8(s, cond, INSN_LDRD_IMM, rt, rn, imm8, 1, 0);
+}
+
+static inline void tcg_out_ldrd_r(TCGContext *s, int cond, TCGReg rt,
+                                  TCGReg rn, TCGReg rm)
+{
+    tcg_out_memop_r(s, cond, INSN_LDRD_REG, rt, rn, rm, 1, 1, 0);
+}
+
+static inline void tcg_out_strd_8(TCGContext *s, int cond, TCGReg rt,
+                                   TCGReg rn, int imm8)
+{
+    tcg_out_memop_8(s, cond, INSN_STRD_IMM, rt, rn, imm8, 1, 0);
+}
+
+static inline void tcg_out_strd_r(TCGContext *s, int cond, TCGReg rt,
+                                  TCGReg rn, TCGReg rm)
+{
+    tcg_out_memop_r(s, cond, INSN_STRD_REG, rt, rn, rm, 1, 1, 0);
+}
+
 /* Register pre-increment with base writeback.  */
 static inline void tcg_out_ld32_rwb(TCGContext *s, int cond, TCGReg rt,
                                     TCGReg rn, TCGReg rm)
@@ -975,34 +1001,27 @@ static inline void tcg_out_st8(TCGContext *s, int cond,
         tcg_out_st8_12(s, cond, rd, rn, offset);
 }
 
-/* The _goto case is normally between TBs within the same code buffer,
- * and with the code buffer limited to 16MB we shouldn't need the long
- * case.
- *
- * .... except to the prologue that is in its own buffer.
+/* The _goto case is normally between TBs within the same code buffer, and
+ * with the code buffer limited to 16MB we wouldn't need the long case.
+ * But we also use it for the tail-call to the qemu_ld/st helpers, which does.
  */
 static inline void tcg_out_goto(TCGContext *s, int cond, uint32_t addr)
 {
-    int32_t val;
+    int32_t disp = addr - (tcg_target_long) s->code_ptr;
 
-    if (addr & 1) {
-        /* goto to a Thumb destination isn't supported */
-        tcg_abort();
+    if ((addr & 1) == 0 && disp - 8 < 0x01fffffd && disp - 8 > -0x01fffffd) {
+        tcg_out_b(s, cond, disp);
+        return;
     }
 
-    val = addr - (tcg_target_long) s->code_ptr;
-    if (val - 8 < 0x01fffffd && val - 8 > -0x01fffffd)
-        tcg_out_b(s, cond, val);
-    else {
-        if (cond == COND_AL) {
-            tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, TCG_REG_PC, -4);
-            tcg_out32(s, addr);
-        } else {
-            tcg_out_movi32(s, cond, TCG_REG_TMP, val - 8);
-            tcg_out_dat_reg(s, cond, ARITH_ADD,
-                            TCG_REG_PC, TCG_REG_PC,
-                            TCG_REG_TMP, SHIFT_IMM_LSL(0));
+    tcg_out_movi32(s, cond, TCG_REG_TMP, addr);
+    if (use_armv5t_instructions) {
+        tcg_out_bx(s, cond, TCG_REG_TMP);
+    } else {
+        if (addr & 1) {
+            tcg_abort();
         }
+        tcg_out_mov_reg(s, cond, TCG_REG_PC, TCG_REG_TMP);
     }
 }
 
@@ -1057,23 +1076,37 @@ static inline void tcg_out_goto_label(TCGContext *s, int cond, int label_index)
 }
 
 #ifdef CONFIG_SOFTMMU
-
-/* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
-   int mmu_idx) */
-static const void * const qemu_ld_helpers[4] = {
-    helper_ldb_mmu,
-    helper_ldw_mmu,
-    helper_ldl_mmu,
-    helper_ldq_mmu,
+/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
+ *                                     int mmu_idx, uintptr_t ra)
+ */
+static const void * const qemu_ld_helpers[16] = {
+    [MO_UB]   = helper_ret_ldub_mmu,
+    [MO_SB]   = helper_ret_ldsb_mmu,
+
+    [MO_LEUW] = helper_le_lduw_mmu,
+    [MO_LEUL] = helper_le_ldul_mmu,
+    [MO_LEQ]  = helper_le_ldq_mmu,
+    [MO_LESW] = helper_le_ldsw_mmu,
+    [MO_LESL] = helper_le_ldul_mmu,
+
+    [MO_BEUW] = helper_be_lduw_mmu,
+    [MO_BEUL] = helper_be_ldul_mmu,
+    [MO_BEQ]  = helper_be_ldq_mmu,
+    [MO_BESW] = helper_be_ldsw_mmu,
+    [MO_BESL] = helper_be_ldul_mmu,
 };
 
-/* helper signature: helper_st_mmu(CPUState *env, target_ulong addr,
-   uintxx_t val, int mmu_idx) */
-static const void * const qemu_st_helpers[4] = {
-    helper_stb_mmu,
-    helper_stw_mmu,
-    helper_stl_mmu,
-    helper_stq_mmu,
+/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,
+ *                                     uintxx_t val, int mmu_idx, uintptr_t ra)
+ */
+static const void * const qemu_st_helpers[16] = {
+    [MO_UB]   = helper_ret_stb_mmu,
+    [MO_LEUW] = helper_le_stw_mmu,
+    [MO_LEUL] = helper_le_stl_mmu,
+    [MO_LEQ]  = helper_le_stq_mmu,
+    [MO_BEUW] = helper_be_stw_mmu,
+    [MO_BEUL] = helper_be_stl_mmu,
+    [MO_BEQ]  = helper_be_stq_mmu,
 };
 
 /* Helper routines for marshalling helper function arguments into
@@ -1117,53 +1150,62 @@ static TCGReg tcg_out_arg_reg64(TCGContext *s, TCGReg argreg,
     if (argreg & 1) {
         argreg++;
     }
-    argreg = tcg_out_arg_reg32(s, argreg, arglo);
-    argreg = tcg_out_arg_reg32(s, argreg, arghi);
-    return argreg;
+    if (use_armv6_instructions && argreg >= 4
+        && (arglo & 1) == 0 && arghi == arglo + 1) {
+        tcg_out_strd_8(s, COND_AL, arglo,
+                       TCG_REG_CALL_STACK, (argreg - 4) * 4);
+        return argreg + 2;
+    } else {
+        argreg = tcg_out_arg_reg32(s, argreg, arglo);
+        argreg = tcg_out_arg_reg32(s, argreg, arghi);
+        return argreg;
+    }
 }
 
 #define TLB_SHIFT	(CPU_TLB_ENTRY_BITS + CPU_TLB_BITS)
 
-/* Load and compare a TLB entry, leaving the flags set.  Leaves R2 pointing
-   to the tlb entry.  Clobbers R1 and TMP.  */
+/* We're expecting to use an 8-bit immediate and to mask.  */
+QEMU_BUILD_BUG_ON(CPU_TLB_BITS > 8);
+
+/* We're expecting to use an 8-bit immediate add + 8-bit ldrd offset.
+   Using the offset of the second entry in the last tlb table ensures
+   that we can index all of the elements of the first entry.  */
+QEMU_BUILD_BUG_ON(offsetof(CPUArchState, tlb_table[NB_MMU_MODES - 1][1])
+                  > 0xffff);
+
+/* Load and compare a TLB entry, leaving the flags set.  Returns the register
+   containing the addend of the tlb entry.  Clobbers R0, R1, R2, TMP.  */
 
-static void tcg_out_tlb_read(TCGContext *s, TCGReg addrlo, TCGReg addrhi,
-                             int s_bits, int tlb_offset)
+static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addrlo, TCGReg addrhi,
+                               TCGMemOp s_bits, int mem_index, bool is_load)
 {
     TCGReg base = TCG_AREG0;
+    int cmp_off =
+        (is_load
+         ? offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)
+         : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write));
+    int add_off = offsetof(CPUArchState, tlb_table[mem_index][0].addend);
 
     /* Should generate something like the following:
-     * pre-v7:
-     *   shr    tmp, addr_reg, #TARGET_PAGE_BITS                  (1)
-     *   add    r2, env, #off & 0xff00
+     *   shr    tmp, addrlo, #TARGET_PAGE_BITS                    (1)
+     *   add    r2, env, #high
      *   and    r0, tmp, #(CPU_TLB_SIZE - 1)                      (2)
      *   add    r2, r2, r0, lsl #CPU_TLB_ENTRY_BITS               (3)
-     *   ldr    r0, [r2, #off & 0xff]!                            (4)
-     *   tst    addr_reg, #s_mask
-     *   cmpeq  r0, tmp, lsl #TARGET_PAGE_BITS                    (5)
-     *
-     * v7 (not implemented yet):
-     *   ubfx   r2, addr_reg, #TARGET_PAGE_BITS, #CPU_TLB_BITS    (1)
-     *   movw   tmp, #~TARGET_PAGE_MASK & ~s_mask
-     *   movw   r0, #off
-     *   add    r2, env, r2, lsl #CPU_TLB_ENTRY_BITS              (2)
-     *   bic    tmp, addr_reg, tmp
-     *   ldr    r0, [r2, r0]!                                     (3)
-     *   cmp    r0, tmp                                           (4)
+     *   ldr    r0, [r2, #cmp]                                    (4)
+     *   tst    addrlo, #s_mask
+     *   ldr    r2, [r2, #add]                                    (5)
+     *   cmpeq  r0, tmp, lsl #TARGET_PAGE_BITS
      */
-#  if CPU_TLB_BITS > 8
-#   error
-#  endif
     tcg_out_dat_reg(s, COND_AL, ARITH_MOV, TCG_REG_TMP,
                     0, addrlo, SHIFT_IMM_LSR(TARGET_PAGE_BITS));
 
-    /* We assume that the offset is contained within 16 bits.  */
-    assert((tlb_offset & ~0xffff) == 0);
-    if (tlb_offset > 0xff) {
+    /* We checked that the offset is contained within 16 bits above.  */
+    if (add_off > 0xfff || (use_armv6_instructions && cmp_off > 0xff)) {
         tcg_out_dat_imm(s, COND_AL, ARITH_ADD, TCG_REG_R2, base,
-                        (24 << 7) | (tlb_offset >> 8));
-        tlb_offset &= 0xff;
+                        (24 << 7) | (cmp_off >> 8));
         base = TCG_REG_R2;
+        add_off -= cmp_off & 0xff00;
+        cmp_off &= 0xff;
     }
 
     tcg_out_dat_imm(s, COND_AL, ARITH_AND,
@@ -1175,14 +1217,11 @@ static void tcg_out_tlb_read(TCGContext *s, TCGReg addrlo, TCGReg addrhi,
        but due to how the pointer needs setting up, ldm isn't useful.
        Base arm5 doesn't have ldrd, but armv5te does.  */
     if (use_armv6_instructions && TARGET_LONG_BITS == 64) {
-        tcg_out_memop_8(s, COND_AL, INSN_LDRD_IMM, TCG_REG_R0,
-                        TCG_REG_R2, tlb_offset, 1, 1);
+        tcg_out_ldrd_8(s, COND_AL, TCG_REG_R0, TCG_REG_R2, cmp_off);
     } else {
-        tcg_out_memop_12(s, COND_AL, INSN_LDR_IMM, TCG_REG_R0,
-                         TCG_REG_R2, tlb_offset, 1, 1);
+        tcg_out_ld32_12(s, COND_AL, TCG_REG_R0, TCG_REG_R2, cmp_off);
         if (TARGET_LONG_BITS == 64) {
-            tcg_out_memop_12(s, COND_AL, INSN_LDR_IMM, TCG_REG_R1,
-                             TCG_REG_R2, 4, 1, 0);
+            tcg_out_ld32_12(s, COND_AL, TCG_REG_R1, TCG_REG_R2, cmp_off + 4);
         }
     }
 
@@ -1192,6 +1231,9 @@ static void tcg_out_tlb_read(TCGContext *s, TCGReg addrlo, TCGReg addrhi,
                         0, addrlo, (1 << s_bits) - 1);
     }
 
+    /* Load the tlb addend.  */
+    tcg_out_ld32_12(s, COND_AL, TCG_REG_R2, TCG_REG_R2, add_off);
+
     tcg_out_dat_reg(s, (s_bits ? COND_EQ : COND_AL), ARITH_CMP, 0,
                     TCG_REG_R0, TCG_REG_TMP, SHIFT_IMM_LSL(TARGET_PAGE_BITS));
 
@@ -1199,31 +1241,26 @@ static void tcg_out_tlb_read(TCGContext *s, TCGReg addrlo, TCGReg addrhi,
         tcg_out_dat_reg(s, COND_EQ, ARITH_CMP, 0,
                         TCG_REG_R1, addrhi, SHIFT_IMM_LSL(0));
     }
+
+    return TCG_REG_R2;
 }
 
 /* Record the context of a call to the out of line helper code for the slow
    path for a load or store, so that we can later generate the correct
    helper code.  */
-static void add_qemu_ldst_label(TCGContext *s, int is_ld, int opc,
-                                int data_reg, int data_reg2, int addrlo_reg,
-                                int addrhi_reg, int mem_index,
+static void add_qemu_ldst_label(TCGContext *s, int is_ld, TCGMemOp opc,
+                                TCGReg datalo, TCGReg datahi, TCGReg addrlo,
+                                TCGReg addrhi, int mem_index,
                                 uint8_t *raddr, uint8_t *label_ptr)
 {
-    int idx;
-    TCGLabelQemuLdst *label;
-
-    if (s->nb_qemu_ldst_labels >= TCG_MAX_QEMU_LDST) {
-        tcg_abort();
-    }
+    TCGLabelQemuLdst *label = new_ldst_label(s);
 
-    idx = s->nb_qemu_ldst_labels++;
-    label = (TCGLabelQemuLdst *)&s->qemu_ldst_labels[idx];
     label->is_ld = is_ld;
     label->opc = opc;
-    label->datalo_reg = data_reg;
-    label->datahi_reg = data_reg2;
-    label->addrlo_reg = addrlo_reg;
-    label->addrhi_reg = addrhi_reg;
+    label->datalo_reg = datalo;
+    label->datahi_reg = datahi;
+    label->addrlo_reg = addrlo;
+    label->addrhi_reg = addrhi;
     label->mem_index = mem_index;
     label->raddr = raddr;
     label->label_ptr[0] = label_ptr;
@@ -1231,8 +1268,9 @@ static void add_qemu_ldst_label(TCGContext *s, int is_ld, int opc,
 
 static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
 {
-    TCGReg argreg, data_reg, data_reg2;
-    uint8_t *start;
+    TCGReg argreg, datalo, datahi;
+    TCGMemOp opc = lb->opc;
+    uintptr_t func;
 
     reloc_pc24(lb->label_ptr[0], (tcg_target_long)s->code_ptr);
 
@@ -1243,46 +1281,46 @@ static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
         argreg = tcg_out_arg_reg32(s, argreg, lb->addrlo_reg);
     }
     argreg = tcg_out_arg_imm32(s, argreg, lb->mem_index);
-    tcg_out_call(s, (tcg_target_long) qemu_ld_helpers[lb->opc & 3]);
+    argreg = tcg_out_arg_reg32(s, argreg, TCG_REG_R14);
 
-    data_reg = lb->datalo_reg;
-    data_reg2 = lb->datahi_reg;
+    /* For armv6 we can use the canonical unsigned helpers and minimize
+       icache usage.  For pre-armv6, use the signed helpers since we do
+       not have a single insn sign-extend.  */
+    if (use_armv6_instructions) {
+        func = (uintptr_t)qemu_ld_helpers[opc & ~MO_SIGN];
+    } else {
+        func = (uintptr_t)qemu_ld_helpers[opc];
+        if (opc & MO_SIGN) {
+            opc = MO_UL;
+        }
+    }
+    tcg_out_call(s, func);
 
-    start = s->code_ptr;
-    switch (lb->opc) {
-    case 0 | 4:
-        tcg_out_ext8s(s, COND_AL, data_reg, TCG_REG_R0);
+    datalo = lb->datalo_reg;
+    datahi = lb->datahi_reg;
+    switch (opc & MO_SSIZE) {
+    case MO_SB:
+        tcg_out_ext8s(s, COND_AL, datalo, TCG_REG_R0);
         break;
-    case 1 | 4:
-        tcg_out_ext16s(s, COND_AL, data_reg, TCG_REG_R0);
+    case MO_SW:
+        tcg_out_ext16s(s, COND_AL, datalo, TCG_REG_R0);
         break;
-    case 0:
-    case 1:
-    case 2:
     default:
-        tcg_out_mov_reg(s, COND_AL, data_reg, TCG_REG_R0);
-        break;
-    case 3:
-        tcg_out_mov_reg(s, COND_AL, data_reg, TCG_REG_R0);
-        tcg_out_mov_reg(s, COND_AL, data_reg2, TCG_REG_R1);
-        break;
-    }
-
-    /* For GETPC_LDST in exec-all.h, we architect exactly 2 insns between
-       the call and the branch back to straight-line code.  Note that the
-       moves above could be elided by register allocation, nor do we know
-       which code alternative we chose for extension.  */
-    switch (s->code_ptr - start) {
-    case 0:
-        tcg_out_nop(s);
-        /* FALLTHRU */
-    case 4:
-        tcg_out_nop(s);
-        /* FALLTHRU */
-    case 8:
+        tcg_out_mov_reg(s, COND_AL, datalo, TCG_REG_R0);
+        break;
+    case MO_Q:
+        if (datalo != TCG_REG_R1) {
+            tcg_out_mov_reg(s, COND_AL, datalo, TCG_REG_R0);
+            tcg_out_mov_reg(s, COND_AL, datahi, TCG_REG_R1);
+        } else if (datahi != TCG_REG_R0) {
+            tcg_out_mov_reg(s, COND_AL, datahi, TCG_REG_R1);
+            tcg_out_mov_reg(s, COND_AL, datalo, TCG_REG_R0);
+        } else {
+            tcg_out_mov_reg(s, COND_AL, TCG_REG_TMP, TCG_REG_R0);
+            tcg_out_mov_reg(s, COND_AL, datahi, TCG_REG_R1);
+            tcg_out_mov_reg(s, COND_AL, datalo, TCG_REG_TMP);
+        }
         break;
-    default:
-        abort();
     }
 
     tcg_out_goto(s, COND_AL, (tcg_target_long)lb->raddr);
@@ -1290,7 +1328,8 @@ static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
 
 static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
 {
-    TCGReg argreg, data_reg, data_reg2;
+    TCGReg argreg, datalo, datahi;
+    TCGMemOp opc = lb->opc;
 
     reloc_pc24(lb->label_ptr[0], (tcg_target_long)s->code_ptr);
 
@@ -1302,293 +1341,311 @@ static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
         argreg = tcg_out_arg_reg32(s, argreg, lb->addrlo_reg);
     }
 
-    data_reg = lb->datalo_reg;
-    data_reg2 = lb->datahi_reg;
-    switch (lb->opc) {
-    case 0:
-        argreg = tcg_out_arg_reg8(s, argreg, data_reg);
+    datalo = lb->datalo_reg;
+    datahi = lb->datahi_reg;
+    switch (opc & MO_SIZE) {
+    case MO_8:
+        argreg = tcg_out_arg_reg8(s, argreg, datalo);
         break;
-    case 1:
-        argreg = tcg_out_arg_reg16(s, argreg, data_reg);
+    case MO_16:
+        argreg = tcg_out_arg_reg16(s, argreg, datalo);
         break;
-    case 2:
-        argreg = tcg_out_arg_reg32(s, argreg, data_reg);
+    case MO_32:
+    default:
+        argreg = tcg_out_arg_reg32(s, argreg, datalo);
         break;
-    case 3:
-        argreg = tcg_out_arg_reg64(s, argreg, data_reg, data_reg2);
+    case MO_64:
+        argreg = tcg_out_arg_reg64(s, argreg, datalo, datahi);
         break;
     }
 
     argreg = tcg_out_arg_imm32(s, argreg, lb->mem_index);
-    tcg_out_call(s, (tcg_target_long) qemu_st_helpers[lb->opc & 3]);
+    argreg = tcg_out_arg_reg32(s, argreg, TCG_REG_R14);
 
-    /* For GETPC_LDST in exec-all.h, we architect exactly 2 insns between
-       the call and the branch back to straight-line code.  */
-    tcg_out_nop(s);
-    tcg_out_nop(s);
-    tcg_out_goto(s, COND_AL, (tcg_target_long)lb->raddr);
+    /* Tail-call to the helper, which will return to the fast path.  */
+    tcg_out_goto(s, COND_AL, (uintptr_t)qemu_st_helpers[opc]);
 }
 #endif /* SOFTMMU */
 
-static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
+static inline void tcg_out_qemu_ld_index(TCGContext *s, TCGMemOp opc,
+                                         TCGReg datalo, TCGReg datahi,
+                                         TCGReg addrlo, TCGReg addend)
 {
-    TCGReg addr_reg, data_reg, data_reg2;
-    bool bswap;
-#ifdef CONFIG_SOFTMMU
-    int mem_index, s_bits;
-    TCGReg addr_reg2;
-    uint8_t *label_ptr;
-#endif
-#ifdef TARGET_WORDS_BIGENDIAN
-    bswap = 1;
-#else
-    bswap = 0;
-#endif
-
-    data_reg = *args++;
-    data_reg2 = (opc == 3 ? *args++ : 0);
-    addr_reg = *args++;
-#ifdef CONFIG_SOFTMMU
-    addr_reg2 = (TARGET_LONG_BITS == 64 ? *args++ : 0);
-    mem_index = *args;
-    s_bits = opc & 3;
-
-    tcg_out_tlb_read(s, addr_reg, addr_reg2, s_bits,
-                     offsetof(CPUArchState, tlb_table[mem_index][0].addr_read));
-
-    label_ptr = s->code_ptr;
-    tcg_out_b_noaddr(s, COND_NE);
-
-    tcg_out_ld32_12(s, COND_AL, TCG_REG_R1, TCG_REG_R2,
-                    offsetof(CPUTLBEntry, addend)
-                    - offsetof(CPUTLBEntry, addr_read));
+    TCGMemOp bswap = opc & MO_BSWAP;
 
-    switch (opc) {
-    case 0:
-        tcg_out_ld8_r(s, COND_AL, data_reg, addr_reg, TCG_REG_R1);
+    switch (opc & MO_SSIZE) {
+    case MO_UB:
+        tcg_out_ld8_r(s, COND_AL, datalo, addrlo, addend);
         break;
-    case 0 | 4:
-        tcg_out_ld8s_r(s, COND_AL, data_reg, addr_reg, TCG_REG_R1);
+    case MO_SB:
+        tcg_out_ld8s_r(s, COND_AL, datalo, addrlo, addend);
         break;
-    case 1:
-        tcg_out_ld16u_r(s, COND_AL, data_reg, addr_reg, TCG_REG_R1);
+    case MO_UW:
+        tcg_out_ld16u_r(s, COND_AL, datalo, addrlo, addend);
         if (bswap) {
-            tcg_out_bswap16(s, COND_AL, data_reg, data_reg);
+            tcg_out_bswap16(s, COND_AL, datalo, datalo);
         }
         break;
-    case 1 | 4:
+    case MO_SW:
         if (bswap) {
-            tcg_out_ld16u_r(s, COND_AL, data_reg, addr_reg, TCG_REG_R1);
-            tcg_out_bswap16s(s, COND_AL, data_reg, data_reg);
+            tcg_out_ld16u_r(s, COND_AL, datalo, addrlo, addend);
+            tcg_out_bswap16s(s, COND_AL, datalo, datalo);
         } else {
-            tcg_out_ld16s_r(s, COND_AL, data_reg, addr_reg, TCG_REG_R1);
+            tcg_out_ld16s_r(s, COND_AL, datalo, addrlo, addend);
         }
         break;
-    case 2:
+    case MO_UL:
     default:
-        tcg_out_ld32_r(s, COND_AL, data_reg, addr_reg, TCG_REG_R1);
+        tcg_out_ld32_r(s, COND_AL, datalo, addrlo, addend);
         if (bswap) {
-            tcg_out_bswap32(s, COND_AL, data_reg, data_reg);
+            tcg_out_bswap32(s, COND_AL, datalo, datalo);
         }
         break;
-    case 3:
-        if (bswap) {
-            tcg_out_ld32_rwb(s, COND_AL, data_reg2, TCG_REG_R1, addr_reg);
-            tcg_out_ld32_12(s, COND_AL, data_reg, TCG_REG_R1, 4);
-            tcg_out_bswap32(s, COND_AL, data_reg2, data_reg2);
-            tcg_out_bswap32(s, COND_AL, data_reg, data_reg);
-        } else {
-            tcg_out_ld32_rwb(s, COND_AL, data_reg, TCG_REG_R1, addr_reg);
-            tcg_out_ld32_12(s, COND_AL, data_reg2, TCG_REG_R1, 4);
+    case MO_Q:
+        {
+            TCGReg dl = (bswap ? datahi : datalo);
+            TCGReg dh = (bswap ? datalo : datahi);
+
+            if (use_armv6_instructions && (dl & 1) == 0 && dh == dl + 1) {
+                tcg_out_ldrd_r(s, COND_AL, dl, addrlo, addend);
+            } else if (dl != addend) {
+                tcg_out_ld32_rwb(s, COND_AL, dl, addend, addrlo);
+                tcg_out_ld32_12(s, COND_AL, dh, addend, 4);
+            } else {
+                tcg_out_dat_reg(s, COND_AL, ARITH_ADD, TCG_REG_TMP,
+                                addend, addrlo, SHIFT_IMM_LSL(0));
+                tcg_out_ld32_12(s, COND_AL, dl, TCG_REG_TMP, 0);
+                tcg_out_ld32_12(s, COND_AL, dh, TCG_REG_TMP, 4);
+            }
+            if (bswap) {
+                tcg_out_bswap32(s, COND_AL, dl, dl);
+                tcg_out_bswap32(s, COND_AL, dh, dh);
+            }
         }
         break;
     }
+}
 
-    add_qemu_ldst_label(s, 1, opc, data_reg, data_reg2, addr_reg, addr_reg2,
-                        mem_index, s->code_ptr, label_ptr);
-#else /* !CONFIG_SOFTMMU */
-    if (GUEST_BASE) {
-        uint32_t offset = GUEST_BASE;
-        int i, rot;
-
-        while (offset) {
-            i = ctz32(offset) & ~1;
-            rot = ((32 - i) << 7) & 0xf00;
+static inline void tcg_out_qemu_ld_direct(TCGContext *s, TCGMemOp opc,
+                                          TCGReg datalo, TCGReg datahi,
+                                          TCGReg addrlo)
+{
+    TCGMemOp bswap = opc & MO_BSWAP;
 
-            tcg_out_dat_imm(s, COND_AL, ARITH_ADD, TCG_REG_TMP, addr_reg,
-                            ((offset >> i) & 0xff) | rot);
-            addr_reg = TCG_REG_TMP;
-            offset &= ~(0xff << i);
-        }
-    }
-    switch (opc) {
-    case 0:
-        tcg_out_ld8_12(s, COND_AL, data_reg, addr_reg, 0);
+    switch (opc & MO_SSIZE) {
+    case MO_UB:
+        tcg_out_ld8_12(s, COND_AL, datalo, addrlo, 0);
         break;
-    case 0 | 4:
-        tcg_out_ld8s_8(s, COND_AL, data_reg, addr_reg, 0);
+    case MO_SB:
+        tcg_out_ld8s_8(s, COND_AL, datalo, addrlo, 0);
         break;
-    case 1:
-        tcg_out_ld16u_8(s, COND_AL, data_reg, addr_reg, 0);
+    case MO_UW:
+        tcg_out_ld16u_8(s, COND_AL, datalo, addrlo, 0);
         if (bswap) {
-            tcg_out_bswap16(s, COND_AL, data_reg, data_reg);
+            tcg_out_bswap16(s, COND_AL, datalo, datalo);
         }
         break;
-    case 1 | 4:
+    case MO_SW:
         if (bswap) {
-            tcg_out_ld16u_8(s, COND_AL, data_reg, addr_reg, 0);
-            tcg_out_bswap16s(s, COND_AL, data_reg, data_reg);
+            tcg_out_ld16u_8(s, COND_AL, datalo, addrlo, 0);
+            tcg_out_bswap16s(s, COND_AL, datalo, datalo);
         } else {
-            tcg_out_ld16s_8(s, COND_AL, data_reg, addr_reg, 0);
+            tcg_out_ld16s_8(s, COND_AL, datalo, addrlo, 0);
         }
         break;
-    case 2:
+    case MO_UL:
     default:
-        tcg_out_ld32_12(s, COND_AL, data_reg, addr_reg, 0);
+        tcg_out_ld32_12(s, COND_AL, datalo, addrlo, 0);
         if (bswap) {
-            tcg_out_bswap32(s, COND_AL, data_reg, data_reg);
+            tcg_out_bswap32(s, COND_AL, datalo, datalo);
         }
         break;
-    case 3:
-        /* TODO: use block load -
-         * check that data_reg2 > data_reg or the other way */
-        if (data_reg == addr_reg) {
-            tcg_out_ld32_12(s, COND_AL, data_reg2, addr_reg, bswap ? 0 : 4);
-            tcg_out_ld32_12(s, COND_AL, data_reg, addr_reg, bswap ? 4 : 0);
-        } else {
-            tcg_out_ld32_12(s, COND_AL, data_reg, addr_reg, bswap ? 4 : 0);
-            tcg_out_ld32_12(s, COND_AL, data_reg2, addr_reg, bswap ? 0 : 4);
-        }
-        if (bswap) {
-            tcg_out_bswap32(s, COND_AL, data_reg, data_reg);
-            tcg_out_bswap32(s, COND_AL, data_reg2, data_reg2);
+    case MO_Q:
+        {
+            TCGReg dl = (bswap ? datahi : datalo);
+            TCGReg dh = (bswap ? datalo : datahi);
+
+            if (use_armv6_instructions && (dl & 1) == 0 && dh == dl + 1) {
+                tcg_out_ldrd_8(s, COND_AL, dl, addrlo, 0);
+            } else if (dl == addrlo) {
+                tcg_out_ld32_12(s, COND_AL, dh, addrlo, bswap ? 0 : 4);
+                tcg_out_ld32_12(s, COND_AL, dl, addrlo, bswap ? 4 : 0);
+            } else {
+                tcg_out_ld32_12(s, COND_AL, dl, addrlo, bswap ? 4 : 0);
+                tcg_out_ld32_12(s, COND_AL, dh, addrlo, bswap ? 0 : 4);
+            }
+            if (bswap) {
+                tcg_out_bswap32(s, COND_AL, dl, dl);
+                tcg_out_bswap32(s, COND_AL, dh, dh);
+            }
         }
         break;
     }
-#endif
 }
 
-static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
+static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
 {
-    TCGReg addr_reg, data_reg, data_reg2;
-    bool bswap;
+    TCGReg addrlo, datalo, datahi, addrhi __attribute__((unused));
+    TCGMemOp opc;
 #ifdef CONFIG_SOFTMMU
-    int mem_index, s_bits;
-    TCGReg addr_reg2;
+    int mem_index;
+    TCGReg addend;
     uint8_t *label_ptr;
 #endif
-#ifdef TARGET_WORDS_BIGENDIAN
-    bswap = 1;
-#else
-    bswap = 0;
-#endif
 
-    data_reg = *args++;
-    data_reg2 = (opc == 3 ? *args++ : 0);
-    addr_reg = *args++;
+    datalo = *args++;
+    datahi = (is64 ? *args++ : 0);
+    addrlo = *args++;
+    addrhi = (TARGET_LONG_BITS == 64 ? *args++ : 0);
+    opc = *args++;
+
 #ifdef CONFIG_SOFTMMU
-    addr_reg2 = (TARGET_LONG_BITS == 64 ? *args++ : 0);
     mem_index = *args;
-    s_bits = opc & 3;
-
-    tcg_out_tlb_read(s, addr_reg, addr_reg2, s_bits,
-                     offsetof(CPUArchState,
-                              tlb_table[mem_index][0].addr_write));
+    addend = tcg_out_tlb_read(s, addrlo, addrhi, opc & MO_SIZE, mem_index, 1);
 
+    /* This a conditional BL only to load a pointer within this opcode into LR
+       for the slow path.  We will not be using the value for a tail call.  */
     label_ptr = s->code_ptr;
-    tcg_out_b_noaddr(s, COND_NE);
+    tcg_out_bl_noaddr(s, COND_NE);
 
-    tcg_out_ld32_12(s, COND_AL, TCG_REG_R1, TCG_REG_R2,
-                    offsetof(CPUTLBEntry, addend)
-                    - offsetof(CPUTLBEntry, addr_write));
+    tcg_out_qemu_ld_index(s, opc, datalo, datahi, addrlo, addend);
 
-    switch (opc) {
-    case 0:
-        tcg_out_st8_r(s, COND_AL, data_reg, addr_reg, TCG_REG_R1);
+    add_qemu_ldst_label(s, 1, opc, datalo, datahi, addrlo, addrhi,
+                        mem_index, s->code_ptr, label_ptr);
+#else /* !CONFIG_SOFTMMU */
+    if (GUEST_BASE) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP, GUEST_BASE);
+        tcg_out_qemu_ld_index(s, opc, datalo, datahi, addrlo, TCG_REG_TMP);
+    } else {
+        tcg_out_qemu_ld_direct(s, opc, datalo, datahi, addrlo);
+    }
+#endif
+}
+
+static inline void tcg_out_qemu_st_index(TCGContext *s, int cond, TCGMemOp opc,
+                                         TCGReg datalo, TCGReg datahi,
+                                         TCGReg addrlo, TCGReg addend)
+{
+    TCGMemOp bswap = opc & MO_BSWAP;
+
+    switch (opc & MO_SIZE) {
+    case MO_8:
+        tcg_out_st8_r(s, cond, datalo, addrlo, addend);
         break;
-    case 1:
+    case MO_16:
         if (bswap) {
-            tcg_out_bswap16st(s, COND_AL, TCG_REG_R0, data_reg);
-            tcg_out_st16_r(s, COND_AL, TCG_REG_R0, addr_reg, TCG_REG_R1);
+            tcg_out_bswap16st(s, cond, TCG_REG_R0, datalo);
+            tcg_out_st16_r(s, cond, TCG_REG_R0, addrlo, addend);
         } else {
-            tcg_out_st16_r(s, COND_AL, data_reg, addr_reg, TCG_REG_R1);
+            tcg_out_st16_r(s, cond, datalo, addrlo, addend);
         }
         break;
-    case 2:
+    case MO_32:
     default:
         if (bswap) {
-            tcg_out_bswap32(s, COND_AL, TCG_REG_R0, data_reg);
-            tcg_out_st32_r(s, COND_AL, TCG_REG_R0, addr_reg, TCG_REG_R1);
+            tcg_out_bswap32(s, cond, TCG_REG_R0, datalo);
+            tcg_out_st32_r(s, cond, TCG_REG_R0, addrlo, addend);
         } else {
-            tcg_out_st32_r(s, COND_AL, data_reg, addr_reg, TCG_REG_R1);
+            tcg_out_st32_r(s, cond, datalo, addrlo, addend);
         }
         break;
-    case 3:
+    case MO_64:
         if (bswap) {
-            tcg_out_bswap32(s, COND_AL, TCG_REG_R0, data_reg2);
-            tcg_out_st32_rwb(s, COND_AL, TCG_REG_R0, TCG_REG_R1, addr_reg);
-            tcg_out_bswap32(s, COND_AL, TCG_REG_R0, data_reg);
-            tcg_out_st32_12(s, COND_AL, TCG_REG_R0, TCG_REG_R1, 4);
+            tcg_out_bswap32(s, cond, TCG_REG_R0, datahi);
+            tcg_out_st32_rwb(s, cond, TCG_REG_R0, addend, addrlo);
+            tcg_out_bswap32(s, cond, TCG_REG_R0, datalo);
+            tcg_out_st32_12(s, cond, TCG_REG_R0, addend, 4);
+        } else if (use_armv6_instructions
+                   && (datalo & 1) == 0 && datahi == datalo + 1) {
+            tcg_out_strd_r(s, cond, datalo, addrlo, addend);
         } else {
-            tcg_out_st32_rwb(s, COND_AL, data_reg, TCG_REG_R1, addr_reg);
-            tcg_out_st32_12(s, COND_AL, data_reg2, TCG_REG_R1, 4);
+            tcg_out_st32_rwb(s, cond, datalo, addend, addrlo);
+            tcg_out_st32_12(s, cond, datahi, addend, 4);
         }
         break;
     }
+}
 
-    add_qemu_ldst_label(s, 0, opc, data_reg, data_reg2, addr_reg, addr_reg2,
-                        mem_index, s->code_ptr, label_ptr);
-#else /* !CONFIG_SOFTMMU */
-    if (GUEST_BASE) {
-        uint32_t offset = GUEST_BASE;
-        int i;
-        int rot;
-
-        while (offset) {
-            i = ctz32(offset) & ~1;
-            rot = ((32 - i) << 7) & 0xf00;
-
-            tcg_out_dat_imm(s, COND_AL, ARITH_ADD, TCG_REG_R1, addr_reg,
-                            ((offset >> i) & 0xff) | rot);
-            addr_reg = TCG_REG_R1;
-            offset &= ~(0xff << i);
-        }
-    }
-    switch (opc) {
-    case 0:
-        tcg_out_st8_12(s, COND_AL, data_reg, addr_reg, 0);
+static inline void tcg_out_qemu_st_direct(TCGContext *s, TCGMemOp opc,
+                                          TCGReg datalo, TCGReg datahi,
+                                          TCGReg addrlo)
+{
+    TCGMemOp bswap = opc & MO_BSWAP;
+
+    switch (opc & MO_SIZE) {
+    case MO_8:
+        tcg_out_st8_12(s, COND_AL, datalo, addrlo, 0);
         break;
-    case 1:
+    case MO_16:
         if (bswap) {
-            tcg_out_bswap16st(s, COND_AL, TCG_REG_R0, data_reg);
-            tcg_out_st16_8(s, COND_AL, TCG_REG_R0, addr_reg, 0);
+            tcg_out_bswap16st(s, COND_AL, TCG_REG_R0, datalo);
+            tcg_out_st16_8(s, COND_AL, TCG_REG_R0, addrlo, 0);
         } else {
-            tcg_out_st16_8(s, COND_AL, data_reg, addr_reg, 0);
+            tcg_out_st16_8(s, COND_AL, datalo, addrlo, 0);
         }
         break;
-    case 2:
+    case MO_32:
     default:
         if (bswap) {
-            tcg_out_bswap32(s, COND_AL, TCG_REG_R0, data_reg);
-            tcg_out_st32_12(s, COND_AL, TCG_REG_R0, addr_reg, 0);
+            tcg_out_bswap32(s, COND_AL, TCG_REG_R0, datalo);
+            tcg_out_st32_12(s, COND_AL, TCG_REG_R0, addrlo, 0);
         } else {
-            tcg_out_st32_12(s, COND_AL, data_reg, addr_reg, 0);
+            tcg_out_st32_12(s, COND_AL, datalo, addrlo, 0);
         }
         break;
-    case 3:
-        /* TODO: use block store -
-         * check that data_reg2 > data_reg or the other way */
+    case MO_64:
         if (bswap) {
-            tcg_out_bswap32(s, COND_AL, TCG_REG_R0, data_reg2);
-            tcg_out_st32_12(s, COND_AL, TCG_REG_R0, addr_reg, 0);
-            tcg_out_bswap32(s, COND_AL, TCG_REG_R0, data_reg);
-            tcg_out_st32_12(s, COND_AL, TCG_REG_R0, addr_reg, 4);
+            tcg_out_bswap32(s, COND_AL, TCG_REG_R0, datahi);
+            tcg_out_st32_12(s, COND_AL, TCG_REG_R0, addrlo, 0);
+            tcg_out_bswap32(s, COND_AL, TCG_REG_R0, datalo);
+            tcg_out_st32_12(s, COND_AL, TCG_REG_R0, addrlo, 4);
+        } else if (use_armv6_instructions
+                   && (datalo & 1) == 0 && datahi == datalo + 1) {
+            tcg_out_strd_8(s, COND_AL, datalo, addrlo, 0);
         } else {
-            tcg_out_st32_12(s, COND_AL, data_reg, addr_reg, 0);
-            tcg_out_st32_12(s, COND_AL, data_reg2, addr_reg, 4);
+            tcg_out_st32_12(s, COND_AL, datalo, addrlo, 0);
+            tcg_out_st32_12(s, COND_AL, datahi, addrlo, 4);
         }
         break;
     }
+}
+
+static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
+{
+    TCGReg addrlo, datalo, datahi, addrhi __attribute__((unused));
+    TCGMemOp opc;
+#ifdef CONFIG_SOFTMMU
+    int mem_index;
+    TCGReg addend;
+    uint8_t *label_ptr;
+#endif
+
+    datalo = *args++;
+    datahi = (is64 ? *args++ : 0);
+    addrlo = *args++;
+    addrhi = (TARGET_LONG_BITS == 64 ? *args++ : 0);
+    opc = *args++;
+
+#ifdef CONFIG_SOFTMMU
+    mem_index = *args;
+    addend = tcg_out_tlb_read(s, addrlo, addrhi, opc & MO_SIZE, mem_index, 0);
+
+    tcg_out_qemu_st_index(s, COND_EQ, opc, datalo, datahi, addrlo, addend);
+
+    /* The conditional call must come last, as we're going to return here.  */
+    label_ptr = s->code_ptr;
+    tcg_out_bl_noaddr(s, COND_NE);
+
+    add_qemu_ldst_label(s, 0, opc, datalo, datahi, addrlo, addrhi,
+                        mem_index, s->code_ptr, label_ptr);
+#else /* !CONFIG_SOFTMMU */
+    if (GUEST_BASE) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP, GUEST_BASE);
+        tcg_out_qemu_st_index(s, COND_AL, opc, datalo,
+                              datahi, addrlo, TCG_REG_TMP);
+    } else {
+        tcg_out_qemu_st_direct(s, opc, datalo, datahi, addrlo);
+    }
 #endif
 }
 
@@ -1857,37 +1914,18 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
                         ARITH_MOV, args[0], 0, 0);
         break;
 
-    case INDEX_op_qemu_ld8u:
+    case INDEX_op_qemu_ld_i32:
         tcg_out_qemu_ld(s, args, 0);
         break;
-    case INDEX_op_qemu_ld8s:
-        tcg_out_qemu_ld(s, args, 0 | 4);
-        break;
-    case INDEX_op_qemu_ld16u:
+    case INDEX_op_qemu_ld_i64:
         tcg_out_qemu_ld(s, args, 1);
         break;
-    case INDEX_op_qemu_ld16s:
-        tcg_out_qemu_ld(s, args, 1 | 4);
-        break;
-    case INDEX_op_qemu_ld32:
-        tcg_out_qemu_ld(s, args, 2);
-        break;
-    case INDEX_op_qemu_ld64:
-        tcg_out_qemu_ld(s, args, 3);
-        break;
-
-    case INDEX_op_qemu_st8:
+    case INDEX_op_qemu_st_i32:
         tcg_out_qemu_st(s, args, 0);
         break;
-    case INDEX_op_qemu_st16:
+    case INDEX_op_qemu_st_i64:
         tcg_out_qemu_st(s, args, 1);
         break;
-    case INDEX_op_qemu_st32:
-        tcg_out_qemu_st(s, args, 2);
-        break;
-    case INDEX_op_qemu_st64:
-        tcg_out_qemu_st(s, args, 3);
-        break;
 
     case INDEX_op_bswap16_i32:
         tcg_out_bswap16(s, COND_AL, args[0], args[1]);
@@ -1923,22 +1961,6 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     }
 }
 
-#ifdef CONFIG_SOFTMMU
-/* Generate TB finalization at the end of block.  */
-void tcg_out_tb_finalize(TCGContext *s)
-{
-    int i;
-    for (i = 0; i < s->nb_qemu_ldst_labels; i++) {
-        TCGLabelQemuLdst *label = &s->qemu_ldst_labels[i];
-        if (label->is_ld) {
-            tcg_out_qemu_ld_slow_path(s, label);
-        } else {
-            tcg_out_qemu_st_slow_path(s, label);
-        }
-    }
-}
-#endif /* SOFTMMU */
-
 static const TCGTargetOpDef arm_op_defs[] = {
     { INDEX_op_exit_tb, { } },
     { INDEX_op_goto_tb, { } },
@@ -1986,29 +2008,15 @@ static const TCGTargetOpDef arm_op_defs[] = {
     { INDEX_op_setcond2_i32, { "r", "r", "r", "rIN", "rIN" } },
 
 #if TARGET_LONG_BITS == 32
-    { INDEX_op_qemu_ld8u, { "r", "l" } },
-    { INDEX_op_qemu_ld8s, { "r", "l" } },
-    { INDEX_op_qemu_ld16u, { "r", "l" } },
-    { INDEX_op_qemu_ld16s, { "r", "l" } },
-    { INDEX_op_qemu_ld32, { "r", "l" } },
-    { INDEX_op_qemu_ld64, { "L", "L", "l" } },
-
-    { INDEX_op_qemu_st8, { "s", "s" } },
-    { INDEX_op_qemu_st16, { "s", "s" } },
-    { INDEX_op_qemu_st32, { "s", "s" } },
-    { INDEX_op_qemu_st64, { "s", "s", "s" } },
+    { INDEX_op_qemu_ld_i32, { "r", "l" } },
+    { INDEX_op_qemu_ld_i64, { "r", "r", "l" } },
+    { INDEX_op_qemu_st_i32, { "s", "s" } },
+    { INDEX_op_qemu_st_i64, { "s", "s", "s" } },
 #else
-    { INDEX_op_qemu_ld8u, { "r", "l", "l" } },
-    { INDEX_op_qemu_ld8s, { "r", "l", "l" } },
-    { INDEX_op_qemu_ld16u, { "r", "l", "l" } },
-    { INDEX_op_qemu_ld16s, { "r", "l", "l" } },
-    { INDEX_op_qemu_ld32, { "r", "l", "l" } },
-    { INDEX_op_qemu_ld64, { "L", "L", "l", "l" } },
-
-    { INDEX_op_qemu_st8, { "s", "s", "s" } },
-    { INDEX_op_qemu_st16, { "s", "s", "s" } },
-    { INDEX_op_qemu_st32, { "s", "s", "s" } },
-    { INDEX_op_qemu_st64, { "s", "s", "s", "s" } },
+    { INDEX_op_qemu_ld_i32, { "r", "l", "l" } },
+    { INDEX_op_qemu_ld_i64, { "r", "r", "l", "l" } },
+    { INDEX_op_qemu_st_i32, { "s", "s", "s" } },
+    { INDEX_op_qemu_st_i64, { "s", "s", "s", "s" } },
 #endif
 
     { INDEX_op_bswap16_i32, { "r", "r" } },
diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h
index 9482bfa993..3746b6e298 100644
--- a/tcg/arm/tcg-target.h
+++ b/tcg/arm/tcg-target.h
@@ -85,6 +85,8 @@ extern bool use_idiv_instructions;
 #define TCG_TARGET_HAS_div_i32          use_idiv_instructions
 #define TCG_TARGET_HAS_rem_i32          0
 
+#define TCG_TARGET_HAS_new_ldst         1
+
 extern bool tcg_target_deposit_valid(int ofs, int len);
 #define TCG_TARGET_deposit_i32_valid  tcg_target_deposit_valid
 
diff --git a/tcg/hppa/tcg-target.c b/tcg/hppa/tcg-target.c
deleted file mode 100644
index 236b39c31f..0000000000
--- a/tcg/hppa/tcg-target.c
+++ /dev/null
@@ -1,1831 +0,0 @@
-/*
- * Tiny Code Generator for QEMU
- *
- * Copyright (c) 2008 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#if TCG_TARGET_REG_BITS != 32
-#error unsupported
-#endif
-
-#ifndef NDEBUG
-static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
-    "%r0", "%r1", "%rp", "%r3", "%r4", "%r5", "%r6", "%r7",
-    "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15",
-    "%r16", "%r17", "%r18", "%r19", "%r20", "%r21", "%r22", "%r23",
-    "%r24", "%r25", "%r26", "%dp", "%ret0", "%ret1", "%sp", "%r31",
-};
-#endif
-
-/* This is an 8 byte temp slot in the stack frame.  */
-#define STACK_TEMP_OFS -16
-
-#ifdef CONFIG_USE_GUEST_BASE
-#define TCG_GUEST_BASE_REG TCG_REG_R16
-#else
-#define TCG_GUEST_BASE_REG TCG_REG_R0
-#endif
-
-static const int tcg_target_reg_alloc_order[] = {
-    TCG_REG_R4,
-    TCG_REG_R5,
-    TCG_REG_R6,
-    TCG_REG_R7,
-    TCG_REG_R8,
-    TCG_REG_R9,
-    TCG_REG_R10,
-    TCG_REG_R11,
-    TCG_REG_R12,
-    TCG_REG_R13,
-
-    TCG_REG_R17,
-    TCG_REG_R14,
-    TCG_REG_R15,
-    TCG_REG_R16,
-
-    TCG_REG_R26,
-    TCG_REG_R25,
-    TCG_REG_R24,
-    TCG_REG_R23,
-
-    TCG_REG_RET0,
-    TCG_REG_RET1,
-};
-
-static const int tcg_target_call_iarg_regs[4] = {
-    TCG_REG_R26,
-    TCG_REG_R25,
-    TCG_REG_R24,
-    TCG_REG_R23,
-};
-
-static const int tcg_target_call_oarg_regs[2] = {
-    TCG_REG_RET0,
-    TCG_REG_RET1,
-};
-
-/* True iff val fits a signed field of width BITS.  */
-static inline int check_fit_tl(tcg_target_long val, unsigned int bits)
-{
-    return (val << ((sizeof(tcg_target_long) * 8 - bits))
-            >> (sizeof(tcg_target_long) * 8 - bits)) == val;
-}
-
-/* True iff depi can be used to compute (reg | MASK).
-   Accept a bit pattern like:
-      0....01....1
-      1....10....0
-      0..01..10..0
-   Copied from gcc sources.  */
-static inline int or_mask_p(tcg_target_ulong mask)
-{
-    if (mask == 0 || mask == -1) {
-        return 0;
-    }
-    mask += mask & -mask;
-    return (mask & (mask - 1)) == 0;
-}
-
-/* True iff depi or extru can be used to compute (reg & mask).
-   Accept a bit pattern like these:
-      0....01....1
-      1....10....0
-      1..10..01..1
-   Copied from gcc sources.  */
-static inline int and_mask_p(tcg_target_ulong mask)
-{
-    return or_mask_p(~mask);
-}
-
-static int low_sign_ext(int val, int len)
-{
-    return (((val << 1) & ~(-1u << len)) | ((val >> (len - 1)) & 1));
-}
-
-static int reassemble_12(int as12)
-{
-    return (((as12 & 0x800) >> 11) |
-            ((as12 & 0x400) >> 8) |
-            ((as12 & 0x3ff) << 3));
-}
-
-static int reassemble_17(int as17)
-{
-    return (((as17 & 0x10000) >> 16) |
-            ((as17 & 0x0f800) << 5) |
-            ((as17 & 0x00400) >> 8) |
-            ((as17 & 0x003ff) << 3));
-}
-
-static int reassemble_21(int as21)
-{
-    return (((as21 & 0x100000) >> 20) |
-            ((as21 & 0x0ffe00) >> 8) |
-            ((as21 & 0x000180) << 7) |
-            ((as21 & 0x00007c) << 14) |
-            ((as21 & 0x000003) << 12));
-}
-
-/* ??? Bizzarely, there is no PCREL12F relocation type.  I guess all
-   such relocations are simply fully handled by the assembler.  */
-#define R_PARISC_PCREL12F  R_PARISC_NONE
-
-static void patch_reloc(uint8_t *code_ptr, int type,
-                        intptr_t value, intptr_t addend)
-{
-    uint32_t *insn_ptr = (uint32_t *)code_ptr;
-    uint32_t insn = *insn_ptr;
-    intptr_t pcrel;
-
-    value += addend;
-    pcrel = (value - ((intptr_t)code_ptr + 8)) >> 2;
-
-    switch (type) {
-    case R_PARISC_PCREL12F:
-        assert(check_fit_tl(pcrel, 12));
-        /* ??? We assume all patches are forward.  See tcg_out_brcond
-           re setting the NUL bit on the branch and eliding the nop.  */
-        assert(pcrel >= 0);
-        insn &= ~0x1ffdu;
-        insn |= reassemble_12(pcrel);
-        break;
-    case R_PARISC_PCREL17F:
-        assert(check_fit_tl(pcrel, 17));
-        insn &= ~0x1f1ffdu;
-        insn |= reassemble_17(pcrel);
-        break;
-    default:
-        tcg_abort();
-    }
-
-    *insn_ptr = insn;
-}
-
-/* parse target specific constraints */
-static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
-{
-    const char *ct_str;
-
-    ct_str = *pct_str;
-    switch (ct_str[0]) {
-    case 'r':
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
-        break;
-    case 'L': /* qemu_ld/st constraint */
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R26);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R25);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R24);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R23);
-        break;
-    case 'Z':
-        ct->ct |= TCG_CT_CONST_0;
-        break;
-    case 'I':
-        ct->ct |= TCG_CT_CONST_S11;
-        break;
-    case 'J':
-        ct->ct |= TCG_CT_CONST_S5;
-	break;
-    case 'K':
-        ct->ct |= TCG_CT_CONST_MS11;
-        break;
-    case 'M':
-        ct->ct |= TCG_CT_CONST_AND;
-        break;
-    case 'O':
-        ct->ct |= TCG_CT_CONST_OR;
-        break;
-    default:
-        return -1;
-    }
-    ct_str++;
-    *pct_str = ct_str;
-    return 0;
-}
-
-/* test if a constant matches the constraint */
-static int tcg_target_const_match(tcg_target_long val,
-                                  const TCGArgConstraint *arg_ct)
-{
-    int ct = arg_ct->ct;
-    if (ct & TCG_CT_CONST) {
-        return 1;
-    } else if (ct & TCG_CT_CONST_0) {
-        return val == 0;
-    } else if (ct & TCG_CT_CONST_S5) {
-        return check_fit_tl(val, 5);
-    } else if (ct & TCG_CT_CONST_S11) {
-        return check_fit_tl(val, 11);
-    } else if (ct & TCG_CT_CONST_MS11) {
-        return check_fit_tl(-val, 11);
-    } else if (ct & TCG_CT_CONST_AND) {
-        return and_mask_p(val);
-    } else if (ct & TCG_CT_CONST_OR) {
-        return or_mask_p(val);
-    }
-    return 0;
-}
-
-#define INSN_OP(x)       ((x) << 26)
-#define INSN_EXT3BR(x)   ((x) << 13)
-#define INSN_EXT3SH(x)   ((x) << 10)
-#define INSN_EXT4(x)     ((x) << 6)
-#define INSN_EXT5(x)     (x)
-#define INSN_EXT6(x)     ((x) << 6)
-#define INSN_EXT7(x)     ((x) << 6)
-#define INSN_EXT8A(x)    ((x) << 6)
-#define INSN_EXT8B(x)    ((x) << 5)
-#define INSN_T(x)        (x)
-#define INSN_R1(x)       ((x) << 16)
-#define INSN_R2(x)       ((x) << 21)
-#define INSN_DEP_LEN(x)  (32 - (x))
-#define INSN_SHDEP_CP(x) ((31 - (x)) << 5)
-#define INSN_SHDEP_P(x)  ((x) << 5)
-#define INSN_COND(x)     ((x) << 13)
-#define INSN_IM11(x)     low_sign_ext(x, 11)
-#define INSN_IM14(x)     low_sign_ext(x, 14)
-#define INSN_IM5(x)      (low_sign_ext(x, 5) << 16)
-
-#define COND_NEVER   0
-#define COND_EQ      1
-#define COND_LT      2
-#define COND_LE      3
-#define COND_LTU     4
-#define COND_LEU     5
-#define COND_SV      6
-#define COND_OD      7
-#define COND_FALSE   8
-
-#define INSN_ADD	(INSN_OP(0x02) | INSN_EXT6(0x18))
-#define INSN_ADDC	(INSN_OP(0x02) | INSN_EXT6(0x1c))
-#define INSN_ADDI	(INSN_OP(0x2d))
-#define INSN_ADDIL	(INSN_OP(0x0a))
-#define INSN_ADDL	(INSN_OP(0x02) | INSN_EXT6(0x28))
-#define INSN_AND	(INSN_OP(0x02) | INSN_EXT6(0x08))
-#define INSN_ANDCM	(INSN_OP(0x02) | INSN_EXT6(0x00))
-#define INSN_COMCLR	(INSN_OP(0x02) | INSN_EXT6(0x22))
-#define INSN_COMICLR	(INSN_OP(0x24))
-#define INSN_DEP	(INSN_OP(0x35) | INSN_EXT3SH(3))
-#define INSN_DEPI	(INSN_OP(0x35) | INSN_EXT3SH(7))
-#define INSN_EXTRS	(INSN_OP(0x34) | INSN_EXT3SH(7))
-#define INSN_EXTRU	(INSN_OP(0x34) | INSN_EXT3SH(6))
-#define INSN_LDIL	(INSN_OP(0x08))
-#define INSN_LDO	(INSN_OP(0x0d))
-#define INSN_MTCTL	(INSN_OP(0x00) | INSN_EXT8B(0xc2))
-#define INSN_OR		(INSN_OP(0x02) | INSN_EXT6(0x09))
-#define INSN_SHD	(INSN_OP(0x34) | INSN_EXT3SH(2))
-#define INSN_SUB	(INSN_OP(0x02) | INSN_EXT6(0x10))
-#define INSN_SUBB	(INSN_OP(0x02) | INSN_EXT6(0x14))
-#define INSN_SUBI	(INSN_OP(0x25))
-#define INSN_VEXTRS	(INSN_OP(0x34) | INSN_EXT3SH(5))
-#define INSN_VEXTRU	(INSN_OP(0x34) | INSN_EXT3SH(4))
-#define INSN_VSHD	(INSN_OP(0x34) | INSN_EXT3SH(0))
-#define INSN_XOR	(INSN_OP(0x02) | INSN_EXT6(0x0a))
-#define INSN_ZDEP	(INSN_OP(0x35) | INSN_EXT3SH(2))
-#define INSN_ZVDEP	(INSN_OP(0x35) | INSN_EXT3SH(0))
-
-#define INSN_BL         (INSN_OP(0x3a) | INSN_EXT3BR(0))
-#define INSN_BL_N       (INSN_OP(0x3a) | INSN_EXT3BR(0) | 2)
-#define INSN_BLR        (INSN_OP(0x3a) | INSN_EXT3BR(2))
-#define INSN_BV         (INSN_OP(0x3a) | INSN_EXT3BR(6))
-#define INSN_BV_N       (INSN_OP(0x3a) | INSN_EXT3BR(6) | 2)
-#define INSN_BLE_SR4    (INSN_OP(0x39) | (1 << 13))
-
-#define INSN_LDB        (INSN_OP(0x10))
-#define INSN_LDH        (INSN_OP(0x11))
-#define INSN_LDW        (INSN_OP(0x12))
-#define INSN_LDWM       (INSN_OP(0x13))
-#define INSN_FLDDS      (INSN_OP(0x0b) | INSN_EXT4(0) | (1 << 12))
-
-#define INSN_LDBX	(INSN_OP(0x03) | INSN_EXT4(0))
-#define INSN_LDHX	(INSN_OP(0x03) | INSN_EXT4(1))
-#define INSN_LDWX       (INSN_OP(0x03) | INSN_EXT4(2))
-
-#define INSN_STB        (INSN_OP(0x18))
-#define INSN_STH        (INSN_OP(0x19))
-#define INSN_STW        (INSN_OP(0x1a))
-#define INSN_STWM       (INSN_OP(0x1b))
-#define INSN_FSTDS      (INSN_OP(0x0b) | INSN_EXT4(8) | (1 << 12))
-
-#define INSN_COMBT      (INSN_OP(0x20))
-#define INSN_COMBF      (INSN_OP(0x22))
-#define INSN_COMIBT     (INSN_OP(0x21))
-#define INSN_COMIBF     (INSN_OP(0x23))
-
-/* supplied by libgcc */
-extern void *__canonicalize_funcptr_for_compare(const void *);
-
-static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
-{
-    /* PA1.1 defines COPY as OR r,0,t; PA2.0 defines COPY as LDO 0(r),t
-       but hppa-dis.c is unaware of this definition */
-    if (ret != arg) {
-        tcg_out32(s, INSN_OR | INSN_T(ret) | INSN_R1(arg)
-                  | INSN_R2(TCG_REG_R0));
-    }
-}
-
-static void tcg_out_movi(TCGContext *s, TCGType type,
-                         TCGReg ret, tcg_target_long arg)
-{
-    if (check_fit_tl(arg, 14)) {
-        tcg_out32(s, INSN_LDO | INSN_R1(ret)
-                  | INSN_R2(TCG_REG_R0) | INSN_IM14(arg));
-    } else {
-        uint32_t hi, lo;
-        hi = arg >> 11;
-        lo = arg & 0x7ff;
-
-        tcg_out32(s, INSN_LDIL | INSN_R2(ret) | reassemble_21(hi));
-        if (lo) {
-            tcg_out32(s, INSN_LDO | INSN_R1(ret)
-                      | INSN_R2(ret) | INSN_IM14(lo));
-        }
-    }
-}
-
-static void tcg_out_ldst(TCGContext *s, int ret, int addr,
-                         tcg_target_long offset, int op)
-{
-    if (!check_fit_tl(offset, 14)) {
-        uint32_t hi, lo, op;
-
-        hi = offset >> 11;
-        lo = offset & 0x7ff;
-
-        if (addr == TCG_REG_R0) {
-            op = INSN_LDIL | INSN_R2(TCG_REG_R1);
-        } else {
-            op = INSN_ADDIL | INSN_R2(addr);
-        }
-        tcg_out32(s, op | reassemble_21(hi));
-
-        addr = TCG_REG_R1;
-	offset = lo;
-    }
-
-    if (ret != addr || offset != 0 || op != INSN_LDO) {
-        tcg_out32(s, op | INSN_R1(ret) | INSN_R2(addr) | INSN_IM14(offset));
-    }
-}
-
-/* This function is required by tcg.c.  */
-static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
-                              TCGReg arg1, intptr_t arg2)
-{
-    tcg_out_ldst(s, ret, arg1, arg2, INSN_LDW);
-}
-
-/* This function is required by tcg.c.  */
-static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg ret,
-                              TCGReg arg1, intptr_t arg2)
-{
-    tcg_out_ldst(s, ret, arg1, arg2, INSN_STW);
-}
-
-static void tcg_out_ldst_index(TCGContext *s, int data,
-                               int base, int index, int op)
-{
-    tcg_out32(s, op | INSN_T(data) | INSN_R1(index) | INSN_R2(base));
-}
-
-static inline void tcg_out_addi2(TCGContext *s, int ret, int arg1,
-                                 tcg_target_long val)
-{
-    tcg_out_ldst(s, ret, arg1, val, INSN_LDO);
-}
-
-/* This function is required by tcg.c.  */
-static inline void tcg_out_addi(TCGContext *s, int reg, tcg_target_long val)
-{
-    tcg_out_addi2(s, reg, reg, val);
-}
-
-static inline void tcg_out_arith(TCGContext *s, int t, int r1, int r2, int op)
-{
-    tcg_out32(s, op | INSN_T(t) | INSN_R1(r1) | INSN_R2(r2));
-}
-
-static inline void tcg_out_arithi(TCGContext *s, int t, int r1,
-                                  tcg_target_long val, int op)
-{
-    assert(check_fit_tl(val, 11));
-    tcg_out32(s, op | INSN_R1(t) | INSN_R2(r1) | INSN_IM11(val));
-}
-
-static inline void tcg_out_nop(TCGContext *s)
-{
-    tcg_out_arith(s, TCG_REG_R0, TCG_REG_R0, TCG_REG_R0, INSN_OR);
-}
-
-static inline void tcg_out_mtctl_sar(TCGContext *s, int arg)
-{
-    tcg_out32(s, INSN_MTCTL | INSN_R2(11) | INSN_R1(arg));
-}
-
-/* Extract LEN bits at position OFS from ARG and place in RET.
-   Note that here the bit ordering is reversed from the PA-RISC
-   standard, such that the right-most bit is 0.  */
-static inline void tcg_out_extr(TCGContext *s, int ret, int arg,
-                                unsigned ofs, unsigned len, int sign)
-{
-    assert(ofs < 32 && len <= 32 - ofs);
-    tcg_out32(s, (sign ? INSN_EXTRS : INSN_EXTRU)
-              | INSN_R1(ret) | INSN_R2(arg)
-              | INSN_SHDEP_P(31 - ofs) | INSN_DEP_LEN(len));
-}
-
-/* Likewise with OFS interpreted little-endian.  */
-static inline void tcg_out_dep(TCGContext *s, int ret, int arg,
-                               unsigned ofs, unsigned len)
-{
-    assert(ofs < 32 && len <= 32 - ofs);
-    tcg_out32(s, INSN_DEP | INSN_R2(ret) | INSN_R1(arg)
-              | INSN_SHDEP_CP(31 - ofs) | INSN_DEP_LEN(len));
-}
-
-static inline void tcg_out_depi(TCGContext *s, int ret, int arg,
-                                unsigned ofs, unsigned len)
-{
-    assert(ofs < 32 && len <= 32 - ofs);
-    tcg_out32(s, INSN_DEPI | INSN_R2(ret) | INSN_IM5(arg)
-              | INSN_SHDEP_CP(31 - ofs) | INSN_DEP_LEN(len));
-}
-
-static inline void tcg_out_shd(TCGContext *s, int ret, int hi, int lo,
-                               unsigned count)
-{
-    assert(count < 32);
-    tcg_out32(s, INSN_SHD | INSN_R1(hi) | INSN_R2(lo) | INSN_T(ret)
-              | INSN_SHDEP_CP(count));
-}
-
-static void tcg_out_vshd(TCGContext *s, int ret, int hi, int lo, int creg)
-{
-    tcg_out_mtctl_sar(s, creg);
-    tcg_out32(s, INSN_VSHD | INSN_T(ret) | INSN_R1(hi) | INSN_R2(lo));
-}
-
-static void tcg_out_ori(TCGContext *s, int ret, int arg, tcg_target_ulong m)
-{
-    int bs0, bs1;
-
-    /* Note that the argument is constrained to match or_mask_p.  */
-    for (bs0 = 0; bs0 < 32; bs0++) {
-        if ((m & (1u << bs0)) != 0) {
-            break;
-        }
-    }
-    for (bs1 = bs0; bs1 < 32; bs1++) {
-        if ((m & (1u << bs1)) == 0) {
-            break;
-        }
-    }
-    assert(bs1 == 32 || (1ul << bs1) > m);
-
-    tcg_out_mov(s, TCG_TYPE_I32, ret, arg);
-    tcg_out_depi(s, ret, -1, bs0, bs1 - bs0);
-}
-
-static void tcg_out_andi(TCGContext *s, int ret, int arg, tcg_target_ulong m)
-{
-    int ls0, ls1, ms0;
-
-    /* Note that the argument is constrained to match and_mask_p.  */
-    for (ls0 = 0; ls0 < 32; ls0++) {
-        if ((m & (1u << ls0)) == 0) {
-            break;
-        }
-    }
-    for (ls1 = ls0; ls1 < 32; ls1++) {
-        if ((m & (1u << ls1)) != 0) {
-            break;
-        }
-    }
-    for (ms0 = ls1; ms0 < 32; ms0++) {
-        if ((m & (1u << ms0)) == 0) {
-            break;
-        }
-    }
-    assert (ms0 == 32);
-
-    if (ls1 == 32) {
-        tcg_out_extr(s, ret, arg, 0, ls0, 0);
-    } else {
-        tcg_out_mov(s, TCG_TYPE_I32, ret, arg);
-        tcg_out_depi(s, ret, 0, ls0, ls1 - ls0);
-    }
-}
-
-static inline void tcg_out_ext8s(TCGContext *s, int ret, int arg)
-{
-    tcg_out_extr(s, ret, arg, 0, 8, 1);
-}
-
-static inline void tcg_out_ext16s(TCGContext *s, int ret, int arg)
-{
-    tcg_out_extr(s, ret, arg, 0, 16, 1);
-}
-
-static void tcg_out_shli(TCGContext *s, int ret, int arg, int count)
-{
-    count &= 31;
-    tcg_out32(s, INSN_ZDEP | INSN_R2(ret) | INSN_R1(arg)
-              | INSN_SHDEP_CP(31 - count) | INSN_DEP_LEN(32 - count));
-}
-
-static void tcg_out_shl(TCGContext *s, int ret, int arg, int creg)
-{
-    tcg_out_arithi(s, TCG_REG_R20, creg, 31, INSN_SUBI);
-    tcg_out_mtctl_sar(s, TCG_REG_R20);
-    tcg_out32(s, INSN_ZVDEP | INSN_R2(ret) | INSN_R1(arg) | INSN_DEP_LEN(32));
-}
-
-static void tcg_out_shri(TCGContext *s, int ret, int arg, int count)
-{
-    count &= 31;
-    tcg_out_extr(s, ret, arg, count, 32 - count, 0);
-}
-
-static void tcg_out_shr(TCGContext *s, int ret, int arg, int creg)
-{
-    tcg_out_vshd(s, ret, TCG_REG_R0, arg, creg);
-}
-
-static void tcg_out_sari(TCGContext *s, int ret, int arg, int count)
-{
-    count &= 31;
-    tcg_out_extr(s, ret, arg, count, 32 - count, 1);
-}
-
-static void tcg_out_sar(TCGContext *s, int ret, int arg, int creg)
-{
-    tcg_out_arithi(s, TCG_REG_R20, creg, 31, INSN_SUBI);
-    tcg_out_mtctl_sar(s, TCG_REG_R20);
-    tcg_out32(s, INSN_VEXTRS | INSN_R1(ret) | INSN_R2(arg) | INSN_DEP_LEN(32));
-}
-
-static void tcg_out_rotli(TCGContext *s, int ret, int arg, int count)
-{
-    count &= 31;
-    tcg_out_shd(s, ret, arg, arg, 32 - count);
-}
-
-static void tcg_out_rotl(TCGContext *s, int ret, int arg, int creg)
-{
-    tcg_out_arithi(s, TCG_REG_R20, creg, 32, INSN_SUBI);
-    tcg_out_vshd(s, ret, arg, arg, TCG_REG_R20);
-}
-
-static void tcg_out_rotri(TCGContext *s, int ret, int arg, int count)
-{
-    count &= 31;
-    tcg_out_shd(s, ret, arg, arg, count);
-}
-
-static void tcg_out_rotr(TCGContext *s, int ret, int arg, int creg)
-{
-    tcg_out_vshd(s, ret, arg, arg, creg);
-}
-
-static void tcg_out_bswap16(TCGContext *s, int ret, int arg, int sign)
-{
-    if (ret != arg) {
-        tcg_out_mov(s, TCG_TYPE_I32, ret, arg); /* arg =  xxAB */
-    }
-    tcg_out_dep(s, ret, ret, 16, 8);          /* ret =  xBAB */
-    tcg_out_extr(s, ret, ret, 8, 16, sign);   /* ret =  ..BA */
-}
-
-static void tcg_out_bswap32(TCGContext *s, int ret, int arg, int temp)
-{
-                                          /* arg =  ABCD */
-    tcg_out_rotri(s, temp, arg, 16);      /* temp = CDAB */
-    tcg_out_dep(s, temp, temp, 16, 8);    /* temp = CBAB */
-    tcg_out_shd(s, ret, arg, temp, 8);    /* ret =  DCBA */
-}
-
-static void tcg_out_call(TCGContext *s, const void *func)
-{
-    tcg_target_long val, hi, lo, disp;
-
-    val = (uint32_t)__canonicalize_funcptr_for_compare(func);
-    disp = (val - ((tcg_target_long)s->code_ptr + 8)) >> 2;
-
-    if (check_fit_tl(disp, 17)) {
-        tcg_out32(s, INSN_BL_N | INSN_R2(TCG_REG_RP) | reassemble_17(disp));
-    } else {
-        hi = val >> 11;
-        lo = val & 0x7ff;
-
-        tcg_out32(s, INSN_LDIL | INSN_R2(TCG_REG_R20) | reassemble_21(hi));
-        tcg_out32(s, INSN_BLE_SR4 | INSN_R2(TCG_REG_R20)
-                  | reassemble_17(lo >> 2));
-        tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_RP, TCG_REG_R31);
-    }
-}
-
-static void tcg_out_xmpyu(TCGContext *s, int retl, int reth,
-                          int arg1, int arg2)
-{
-    /* Store both words into the stack for copy to the FPU.  */
-    tcg_out_ldst(s, arg1, TCG_REG_CALL_STACK, STACK_TEMP_OFS, INSN_STW);
-    tcg_out_ldst(s, arg2, TCG_REG_CALL_STACK, STACK_TEMP_OFS + 4, INSN_STW);
-
-    /* Load both words into the FPU at the same time.  We get away
-       with this because we can address the left and right half of the
-       FPU registers individually once loaded.  */
-    /* fldds stack_temp(sp),fr22 */
-    tcg_out32(s, INSN_FLDDS | INSN_R2(TCG_REG_CALL_STACK)
-              | INSN_IM5(STACK_TEMP_OFS) | INSN_T(22));
-
-    /* xmpyu fr22r,fr22,fr22 */
-    tcg_out32(s, 0x3ad64796);
-
-    /* Store the 64-bit result back into the stack.  */
-    /* fstds stack_temp(sp),fr22 */
-    tcg_out32(s, INSN_FSTDS | INSN_R2(TCG_REG_CALL_STACK)
-              | INSN_IM5(STACK_TEMP_OFS) | INSN_T(22));
-
-    /* Load the pieces of the result that the caller requested.  */
-    if (reth) {
-        tcg_out_ldst(s, reth, TCG_REG_CALL_STACK, STACK_TEMP_OFS, INSN_LDW);
-    }
-    if (retl) {
-        tcg_out_ldst(s, retl, TCG_REG_CALL_STACK, STACK_TEMP_OFS + 4,
-                     INSN_LDW);
-    }
-}
-
-static void tcg_out_add2(TCGContext *s, int destl, int desth,
-                         int al, int ah, int bl, int bh, int blconst)
-{
-    int tmp = (destl == ah || destl == bh ? TCG_REG_R20 : destl);
-
-    if (blconst) {
-        tcg_out_arithi(s, tmp, al, bl, INSN_ADDI);
-    } else {
-        tcg_out_arith(s, tmp, al, bl, INSN_ADD);
-    }
-    tcg_out_arith(s, desth, ah, bh, INSN_ADDC);
-
-    tcg_out_mov(s, TCG_TYPE_I32, destl, tmp);
-}
-
-static void tcg_out_sub2(TCGContext *s, int destl, int desth, int al, int ah,
-                         int bl, int bh, int alconst, int blconst)
-{
-    int tmp = (destl == ah || destl == bh ? TCG_REG_R20 : destl);
-
-    if (alconst) {
-        if (blconst) {
-            tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R20, bl);
-            bl = TCG_REG_R20;
-        }
-        tcg_out_arithi(s, tmp, bl, al, INSN_SUBI);
-    } else if (blconst) {
-        tcg_out_arithi(s, tmp, al, -bl, INSN_ADDI);
-    } else {
-        tcg_out_arith(s, tmp, al, bl, INSN_SUB);
-    }
-    tcg_out_arith(s, desth, ah, bh, INSN_SUBB);
-
-    tcg_out_mov(s, TCG_TYPE_I32, destl, tmp);
-}
-
-static void tcg_out_branch(TCGContext *s, int label_index, int nul)
-{
-    TCGLabel *l = &s->labels[label_index];
-    uint32_t op = nul ? INSN_BL_N : INSN_BL;
-
-    if (l->has_value) {
-        tcg_target_long val = l->u.value;
-
-        val -= (tcg_target_long)s->code_ptr + 8;
-        val >>= 2;
-        assert(check_fit_tl(val, 17));
-
-        tcg_out32(s, op | reassemble_17(val));
-    } else {
-        /* We need to keep the offset unchanged for retranslation.  */
-        uint32_t old_insn = *(uint32_t *)s->code_ptr;
-
-        tcg_out_reloc(s, s->code_ptr, R_PARISC_PCREL17F, label_index, 0);
-        tcg_out32(s, op | (old_insn & 0x1f1ffdu));
-    }
-}
-
-static const uint8_t tcg_cond_to_cmp_cond[] =
-{
-    [TCG_COND_EQ] = COND_EQ,
-    [TCG_COND_NE] = COND_EQ | COND_FALSE,
-    [TCG_COND_LT] = COND_LT,
-    [TCG_COND_GE] = COND_LT | COND_FALSE,
-    [TCG_COND_LE] = COND_LE,
-    [TCG_COND_GT] = COND_LE | COND_FALSE,
-    [TCG_COND_LTU] = COND_LTU,
-    [TCG_COND_GEU] = COND_LTU | COND_FALSE,
-    [TCG_COND_LEU] = COND_LEU,
-    [TCG_COND_GTU] = COND_LEU | COND_FALSE,
-};
-
-static void tcg_out_brcond(TCGContext *s, int cond, TCGArg c1,
-                           TCGArg c2, int c2const, int label_index)
-{
-    TCGLabel *l = &s->labels[label_index];
-    int op, pacond;
-
-    /* Note that COMIB operates as if the immediate is the first
-       operand.  We model brcond with the immediate in the second
-       to better match what targets are likely to give us.  For
-       consistency, model COMB with reversed operands as well.  */
-    pacond = tcg_cond_to_cmp_cond[tcg_swap_cond(cond)];
-
-    if (c2const) {
-        op = (pacond & COND_FALSE ? INSN_COMIBF : INSN_COMIBT);
-        op |= INSN_IM5(c2);
-    } else {
-        op = (pacond & COND_FALSE ? INSN_COMBF : INSN_COMBT);
-        op |= INSN_R1(c2);
-    }
-    op |= INSN_R2(c1);
-    op |= INSN_COND(pacond & 7);
-
-    if (l->has_value) {
-        tcg_target_long val = l->u.value;
-
-        val -= (tcg_target_long)s->code_ptr + 8;
-        val >>= 2;
-        assert(check_fit_tl(val, 12));
-
-        /* ??? Assume that all branches to defined labels are backward.
-           Which means that if the nul bit is set, the delay slot is
-           executed if the branch is taken, and not executed in fallthru.  */
-        tcg_out32(s, op | reassemble_12(val));
-        tcg_out_nop(s);
-    } else {
-        /* We need to keep the offset unchanged for retranslation.  */
-        uint32_t old_insn = *(uint32_t *)s->code_ptr;
-
-        tcg_out_reloc(s, s->code_ptr, R_PARISC_PCREL12F, label_index, 0);
-        /* ??? Assume that all branches to undefined labels are forward.
-           Which means that if the nul bit is set, the delay slot is
-           not executed if the branch is taken, which is what we want.  */
-        tcg_out32(s, op | 2 | (old_insn & 0x1ffdu));
-    }
-}
-
-static void tcg_out_comclr(TCGContext *s, int cond, TCGArg ret,
-                           TCGArg c1, TCGArg c2, int c2const)
-{
-    int op, pacond;
-
-    /* Note that COMICLR operates as if the immediate is the first
-       operand.  We model setcond with the immediate in the second
-       to better match what targets are likely to give us.  For
-       consistency, model COMCLR with reversed operands as well.  */
-    pacond = tcg_cond_to_cmp_cond[tcg_swap_cond(cond)];
-
-    if (c2const) {
-        op = INSN_COMICLR | INSN_R2(c1) | INSN_R1(ret) | INSN_IM11(c2);
-    } else {
-        op = INSN_COMCLR | INSN_R2(c1) | INSN_R1(c2) | INSN_T(ret);
-    }
-    op |= INSN_COND(pacond & 7);
-    op |= pacond & COND_FALSE ? 1 << 12 : 0;
-
-    tcg_out32(s, op);
-}
-
-static void tcg_out_brcond2(TCGContext *s, int cond, TCGArg al, TCGArg ah,
-                            TCGArg bl, int blconst, TCGArg bh, int bhconst,
-                            int label_index)
-{
-    switch (cond) {
-    case TCG_COND_EQ:
-        tcg_out_comclr(s, TCG_COND_NE, TCG_REG_R0, al, bl, blconst);
-        tcg_out_brcond(s, TCG_COND_EQ, ah, bh, bhconst, label_index);
-        break;
-    case TCG_COND_NE:
-        tcg_out_brcond(s, TCG_COND_NE, al, bl, blconst, label_index);
-        tcg_out_brcond(s, TCG_COND_NE, ah, bh, bhconst, label_index);
-        break;
-    default:
-        tcg_out_brcond(s, tcg_high_cond(cond), ah, bh, bhconst, label_index);
-        tcg_out_comclr(s, TCG_COND_NE, TCG_REG_R0, ah, bh, bhconst);
-        tcg_out_brcond(s, tcg_unsigned_cond(cond),
-                       al, bl, blconst, label_index);
-        break;
-    }
-}
-
-static void tcg_out_setcond(TCGContext *s, int cond, TCGArg ret,
-                            TCGArg c1, TCGArg c2, int c2const)
-{
-    tcg_out_comclr(s, tcg_invert_cond(cond), ret, c1, c2, c2const);
-    tcg_out_movi(s, TCG_TYPE_I32, ret, 1);
-}
-
-static void tcg_out_setcond2(TCGContext *s, int cond, TCGArg ret,
-                             TCGArg al, TCGArg ah, TCGArg bl, int blconst,
-                             TCGArg bh, int bhconst)
-{
-    int scratch = TCG_REG_R20;
-
-    /* Note that the low parts are fully consumed before scratch is set.  */
-    if (ret != ah && (bhconst || ret != bh)) {
-        scratch = ret;
-    }
-
-    switch (cond) {
-    case TCG_COND_EQ:
-    case TCG_COND_NE:
-        tcg_out_setcond(s, cond, scratch, al, bl, blconst);
-        tcg_out_comclr(s, TCG_COND_EQ, TCG_REG_R0, ah, bh, bhconst);
-        tcg_out_movi(s, TCG_TYPE_I32, scratch, cond == TCG_COND_NE);
-        break;
-
-    case TCG_COND_GE:
-    case TCG_COND_GEU:
-    case TCG_COND_LT:
-    case TCG_COND_LTU:
-        /* Optimize compares with low part zero.  */
-        if (bl == 0) {
-            tcg_out_setcond(s, cond, ret, ah, bh, bhconst);
-            return;
-        }
-        /* FALLTHRU */
-
-    case TCG_COND_LE:
-    case TCG_COND_LEU:
-    case TCG_COND_GT:
-    case TCG_COND_GTU:
-        /* <= : ah < bh | (ah == bh && al <= bl) */
-        tcg_out_setcond(s, tcg_unsigned_cond(cond), scratch, al, bl, blconst);
-        tcg_out_comclr(s, TCG_COND_EQ, TCG_REG_R0, ah, bh, bhconst);
-        tcg_out_movi(s, TCG_TYPE_I32, scratch, 0);
-        tcg_out_comclr(s, tcg_invert_cond(tcg_high_cond(cond)),
-                       TCG_REG_R0, ah, bh, bhconst);
-        tcg_out_movi(s, TCG_TYPE_I32, scratch, 1);
-        break;
-
-    default:
-        tcg_abort();
-    }
-
-    tcg_out_mov(s, TCG_TYPE_I32, ret, scratch);
-}
-
-static void tcg_out_movcond(TCGContext *s, int cond, TCGArg ret,
-                            TCGArg c1, TCGArg c2, int c2const,
-                            TCGArg v1, int v1const)
-{
-    tcg_out_comclr(s, tcg_invert_cond(cond), TCG_REG_R0, c1, c2, c2const);
-    if (v1const) {
-        tcg_out_movi(s, TCG_TYPE_I32, ret, v1);
-    } else {
-        tcg_out_mov(s, TCG_TYPE_I32, ret, v1);
-    }
-}
-
-#if defined(CONFIG_SOFTMMU)
-/* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
-   int mmu_idx) */
-static const void * const qemu_ld_helpers[4] = {
-    helper_ldb_mmu,
-    helper_ldw_mmu,
-    helper_ldl_mmu,
-    helper_ldq_mmu,
-};
-
-/* helper signature: helper_st_mmu(CPUState *env, target_ulong addr,
-   uintxx_t val, int mmu_idx) */
-static const void * const qemu_st_helpers[4] = {
-    helper_stb_mmu,
-    helper_stw_mmu,
-    helper_stl_mmu,
-    helper_stq_mmu,
-};
-
-/* Load and compare a TLB entry, and branch if TLB miss.  OFFSET is set to
-   the offset of the first ADDR_READ or ADDR_WRITE member of the appropriate
-   TLB for the memory index.  The return value is the offset from ENV
-   contained in R1 afterward (to be used when loading ADDEND); if the
-   return value is 0, R1 is not used.  */
-
-static int tcg_out_tlb_read(TCGContext *s, int r0, int r1, int addrlo,
-                            int addrhi, int s_bits, int lab_miss, int offset)
-{
-    int ret;
-
-    /* Extracting the index into the TLB.  The "normal C operation" is
-          r1 = addr_reg >> TARGET_PAGE_BITS;
-          r1 &= CPU_TLB_SIZE - 1;
-          r1 <<= CPU_TLB_ENTRY_BITS;
-       What this does is extract CPU_TLB_BITS beginning at TARGET_PAGE_BITS
-       and place them at CPU_TLB_ENTRY_BITS.  We can combine the first two
-       operations with an EXTRU.  Unfortunately, the current value of
-       CPU_TLB_ENTRY_BITS is > 3, so we can't merge that shift with the
-       add that follows.  */
-    tcg_out_extr(s, r1, addrlo, TARGET_PAGE_BITS, CPU_TLB_BITS, 0);
-    tcg_out_shli(s, r1, r1, CPU_TLB_ENTRY_BITS);
-    tcg_out_arith(s, r1, r1, TCG_AREG0, INSN_ADDL);
-
-    /* Make sure that both the addr_{read,write} and addend can be
-       read with a 14-bit offset from the same base register.  */
-    if (check_fit_tl(offset + CPU_TLB_SIZE, 14)) {
-        ret = 0;
-    } else {
-        ret = (offset + 0x400) & ~0x7ff;
-        offset = ret - offset;
-        tcg_out_addi2(s, TCG_REG_R1, r1, ret);
-        r1 = TCG_REG_R1;
-    }
-
-    /* Load the entry from the computed slot.  */
-    if (TARGET_LONG_BITS == 64) {
-        tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R23, r1, offset);
-        tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R20, r1, offset + 4);
-    } else {
-        tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R20, r1, offset);
-    }
-
-    /* Compute the value that ought to appear in the TLB for a hit, namely,
-       the page of the address.  We include the low N bits of the address
-       to catch unaligned accesses and force them onto the slow path.  Do
-       this computation after having issued the load from the TLB slot to
-       give the load time to complete.  */
-    tcg_out_andi(s, r0, addrlo, TARGET_PAGE_MASK | ((1 << s_bits) - 1));
-
-    /* If not equal, jump to lab_miss. */
-    if (TARGET_LONG_BITS == 64) {
-        tcg_out_brcond2(s, TCG_COND_NE, TCG_REG_R20, TCG_REG_R23,
-                        r0, 0, addrhi, 0, lab_miss);
-    } else {
-        tcg_out_brcond(s, TCG_COND_NE, TCG_REG_R20, r0, 0, lab_miss);
-    }
-
-    return ret;
-}
-
-static int tcg_out_arg_reg32(TCGContext *s, int argno, TCGArg v, bool vconst)
-{
-    if (argno < 4) {
-        if (vconst) {
-            tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[argno], v);
-        } else {
-            tcg_out_mov(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[argno], v);
-        }
-    } else {
-        if (vconst && v != 0) {
-            tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R20, v);
-            v = TCG_REG_R20;
-        }
-        tcg_out_st(s, TCG_TYPE_I32, v, TCG_REG_CALL_STACK,
-                   TCG_TARGET_CALL_STACK_OFFSET - ((argno - 3) * 4));
-    }
-    return argno + 1;
-}
-
-static int tcg_out_arg_reg64(TCGContext *s, int argno, TCGArg vl, TCGArg vh)
-{
-    /* 64-bit arguments must go in even reg pairs and stack slots.  */
-    if (argno & 1) {
-        argno++;
-    }
-    argno = tcg_out_arg_reg32(s, argno, vl, false);
-    argno = tcg_out_arg_reg32(s, argno, vh, false);
-    return argno;
-}
-#endif
-
-static void tcg_out_qemu_ld_direct(TCGContext *s, int datalo_reg, int datahi_reg,
-                                   int addr_reg, int addend_reg, int opc)
-{
-#ifdef TARGET_WORDS_BIGENDIAN
-    const int bswap = 0;
-#else
-    const int bswap = 1;
-#endif
-
-    switch (opc) {
-    case 0:
-        tcg_out_ldst_index(s, datalo_reg, addr_reg, addend_reg, INSN_LDBX);
-        break;
-    case 0 | 4:
-        tcg_out_ldst_index(s, datalo_reg, addr_reg, addend_reg, INSN_LDBX);
-        tcg_out_ext8s(s, datalo_reg, datalo_reg);
-        break;
-    case 1:
-        tcg_out_ldst_index(s, datalo_reg, addr_reg, addend_reg, INSN_LDHX);
-        if (bswap) {
-            tcg_out_bswap16(s, datalo_reg, datalo_reg, 0);
-        }
-        break;
-    case 1 | 4:
-        tcg_out_ldst_index(s, datalo_reg, addr_reg, addend_reg, INSN_LDHX);
-        if (bswap) {
-            tcg_out_bswap16(s, datalo_reg, datalo_reg, 1);
-        } else {
-            tcg_out_ext16s(s, datalo_reg, datalo_reg);
-        }
-        break;
-    case 2:
-        tcg_out_ldst_index(s, datalo_reg, addr_reg, addend_reg, INSN_LDWX);
-        if (bswap) {
-            tcg_out_bswap32(s, datalo_reg, datalo_reg, TCG_REG_R20);
-        }
-        break;
-    case 3:
-        if (bswap) {
-            int t = datahi_reg;
-            datahi_reg = datalo_reg;
-            datalo_reg = t;
-        }
-        /* We can't access the low-part with a reg+reg addressing mode,
-           so perform the addition now and use reg_ofs addressing mode.  */
-        if (addend_reg != TCG_REG_R0) {
-            tcg_out_arith(s, TCG_REG_R20, addr_reg, addend_reg, INSN_ADD);
-            addr_reg = TCG_REG_R20;
-	}
-        /* Make sure not to clobber the base register.  */
-        if (datahi_reg == addr_reg) {
-            tcg_out_ldst(s, datalo_reg, addr_reg, 4, INSN_LDW);
-            tcg_out_ldst(s, datahi_reg, addr_reg, 0, INSN_LDW);
-        } else {
-            tcg_out_ldst(s, datahi_reg, addr_reg, 0, INSN_LDW);
-            tcg_out_ldst(s, datalo_reg, addr_reg, 4, INSN_LDW);
-        }
-        if (bswap) {
-            tcg_out_bswap32(s, datalo_reg, datalo_reg, TCG_REG_R20);
-            tcg_out_bswap32(s, datahi_reg, datahi_reg, TCG_REG_R20);
-        }
-        break;
-    default:
-        tcg_abort();
-    }
-}
-
-static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
-{
-    int datalo_reg = *args++;
-    /* Note that datahi_reg is only used for 64-bit loads.  */
-    int datahi_reg = (opc == 3 ? *args++ : TCG_REG_R0);
-    int addrlo_reg = *args++;
-
-#if defined(CONFIG_SOFTMMU)
-    /* Note that addrhi_reg is only used for 64-bit guests.  */
-    int addrhi_reg = (TARGET_LONG_BITS == 64 ? *args++ : TCG_REG_R0);
-    int mem_index = *args;
-    int lab1, lab2, argno, offset;
-
-    lab1 = gen_new_label();
-    lab2 = gen_new_label();
-
-    offset = offsetof(CPUArchState, tlb_table[mem_index][0].addr_read);
-    offset = tcg_out_tlb_read(s, TCG_REG_R26, TCG_REG_R25, addrlo_reg,
-                              addrhi_reg, opc & 3, lab1, offset);
-
-    /* TLB Hit.  */
-    tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R20,
-               (offset ? TCG_REG_R1 : TCG_REG_R25),
-               offsetof(CPUArchState, tlb_table[mem_index][0].addend) - offset);
-    tcg_out_qemu_ld_direct(s, datalo_reg, datahi_reg, addrlo_reg,
-                           TCG_REG_R20, opc);
-    tcg_out_branch(s, lab2, 1);
-
-    /* TLB Miss.  */
-    /* label1: */
-    tcg_out_label(s, lab1, s->code_ptr);
-
-    argno = 0;
-    argno = tcg_out_arg_reg32(s, argno, TCG_AREG0, false);
-    if (TARGET_LONG_BITS == 64) {
-        argno = tcg_out_arg_reg64(s, argno, addrlo_reg, addrhi_reg);
-    } else {
-        argno = tcg_out_arg_reg32(s, argno, addrlo_reg, false);
-    }
-    argno = tcg_out_arg_reg32(s, argno, mem_index, true);
-
-    tcg_out_call(s, qemu_ld_helpers[opc & 3]);
-
-    switch (opc) {
-    case 0:
-        tcg_out_andi(s, datalo_reg, TCG_REG_RET0, 0xff);
-        break;
-    case 0 | 4:
-        tcg_out_ext8s(s, datalo_reg, TCG_REG_RET0);
-        break;
-    case 1:
-        tcg_out_andi(s, datalo_reg, TCG_REG_RET0, 0xffff);
-        break;
-    case 1 | 4:
-        tcg_out_ext16s(s, datalo_reg, TCG_REG_RET0);
-        break;
-    case 2:
-    case 2 | 4:
-        tcg_out_mov(s, TCG_TYPE_I32, datalo_reg, TCG_REG_RET0);
-        break;
-    case 3:
-        tcg_out_mov(s, TCG_TYPE_I32, datahi_reg, TCG_REG_RET0);
-        tcg_out_mov(s, TCG_TYPE_I32, datalo_reg, TCG_REG_RET1);
-        break;
-    default:
-        tcg_abort();
-    }
-
-    /* label2: */
-    tcg_out_label(s, lab2, s->code_ptr);
-#else
-    tcg_out_qemu_ld_direct(s, datalo_reg, datahi_reg, addrlo_reg,
-                           (GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_R0), opc);
-#endif
-}
-
-static void tcg_out_qemu_st_direct(TCGContext *s, int datalo_reg,
-                                   int datahi_reg, int addr_reg, int opc)
-{
-#ifdef TARGET_WORDS_BIGENDIAN
-    const int bswap = 0;
-#else
-    const int bswap = 1;
-#endif
-
-    switch (opc) {
-    case 0:
-        tcg_out_ldst(s, datalo_reg, addr_reg, 0, INSN_STB);
-        break;
-    case 1:
-        if (bswap) {
-            tcg_out_bswap16(s, TCG_REG_R20, datalo_reg, 0);
-            datalo_reg = TCG_REG_R20;
-        }
-        tcg_out_ldst(s, datalo_reg, addr_reg, 0, INSN_STH);
-        break;
-    case 2:
-        if (bswap) {
-            tcg_out_bswap32(s, TCG_REG_R20, datalo_reg, TCG_REG_R20);
-            datalo_reg = TCG_REG_R20;
-        }
-        tcg_out_ldst(s, datalo_reg, addr_reg, 0, INSN_STW);
-        break;
-    case 3:
-        if (bswap) {
-            tcg_out_bswap32(s, TCG_REG_R20, datalo_reg, TCG_REG_R20);
-            tcg_out_bswap32(s, TCG_REG_R23, datahi_reg, TCG_REG_R23);
-            datahi_reg = TCG_REG_R20;
-            datalo_reg = TCG_REG_R23;
-        }
-        tcg_out_ldst(s, datahi_reg, addr_reg, 0, INSN_STW);
-        tcg_out_ldst(s, datalo_reg, addr_reg, 4, INSN_STW);
-        break;
-    default:
-        tcg_abort();
-    }
-
-}
-
-static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
-{
-    int datalo_reg = *args++;
-    /* Note that datahi_reg is only used for 64-bit loads.  */
-    int datahi_reg = (opc == 3 ? *args++ : TCG_REG_R0);
-    int addrlo_reg = *args++;
-
-#if defined(CONFIG_SOFTMMU)
-    /* Note that addrhi_reg is only used for 64-bit guests.  */
-    int addrhi_reg = (TARGET_LONG_BITS == 64 ? *args++ : TCG_REG_R0);
-    int mem_index = *args;
-    int lab1, lab2, argno, next, offset;
-
-    lab1 = gen_new_label();
-    lab2 = gen_new_label();
-
-    offset = offsetof(CPUArchState, tlb_table[mem_index][0].addr_write);
-    offset = tcg_out_tlb_read(s, TCG_REG_R26, TCG_REG_R25, addrlo_reg,
-                              addrhi_reg, opc, lab1, offset);
-
-    /* TLB Hit.  */
-    tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R20,
-               (offset ? TCG_REG_R1 : TCG_REG_R25),
-               offsetof(CPUArchState, tlb_table[mem_index][0].addend) - offset);
-
-    /* There are no indexed stores, so we must do this addition explitly.
-       Careful to avoid R20, which is used for the bswaps to follow.  */
-    tcg_out_arith(s, TCG_REG_R31, addrlo_reg, TCG_REG_R20, INSN_ADDL);
-    tcg_out_qemu_st_direct(s, datalo_reg, datahi_reg, TCG_REG_R31, opc);
-    tcg_out_branch(s, lab2, 1);
-
-    /* TLB Miss.  */
-    /* label1: */
-    tcg_out_label(s, lab1, s->code_ptr);
-
-    argno = 0;
-    argno = tcg_out_arg_reg32(s, argno, TCG_AREG0, false);
-    if (TARGET_LONG_BITS == 64) {
-        argno = tcg_out_arg_reg64(s, argno, addrlo_reg, addrhi_reg);
-    } else {
-        argno = tcg_out_arg_reg32(s, argno, addrlo_reg, false);
-    }
-
-    next = (argno < 4 ? tcg_target_call_iarg_regs[argno] : TCG_REG_R20);
-    switch(opc) {
-    case 0:
-        tcg_out_andi(s, next, datalo_reg, 0xff);
-        argno = tcg_out_arg_reg32(s, argno, next, false);
-        break;
-    case 1:
-        tcg_out_andi(s, next, datalo_reg, 0xffff);
-        argno = tcg_out_arg_reg32(s, argno, next, false);
-        break;
-    case 2:
-        argno = tcg_out_arg_reg32(s, argno, datalo_reg, false);
-        break;
-    case 3:
-        argno = tcg_out_arg_reg64(s, argno, datalo_reg, datahi_reg);
-        break;
-    default:
-        tcg_abort();
-    }
-    argno = tcg_out_arg_reg32(s, argno, mem_index, true);
-
-    tcg_out_call(s, qemu_st_helpers[opc]);
-
-    /* label2: */
-    tcg_out_label(s, lab2, s->code_ptr);
-#else
-    /* There are no indexed stores, so if GUEST_BASE is set we must do
-       the add explicitly.  Careful to avoid R20, which is used for the
-       bswaps to follow.  */
-    if (GUEST_BASE != 0) {
-        tcg_out_arith(s, TCG_REG_R31, addrlo_reg,
-                      TCG_GUEST_BASE_REG, INSN_ADDL);
-        addrlo_reg = TCG_REG_R31;
-    }
-    tcg_out_qemu_st_direct(s, datalo_reg, datahi_reg, addrlo_reg, opc);
-#endif
-}
-
-static void tcg_out_exit_tb(TCGContext *s, TCGArg arg)
-{
-    if (!check_fit_tl(arg, 14)) {
-        uint32_t hi, lo;
-        hi = arg & ~0x7ff;
-        lo = arg & 0x7ff;
-        if (lo) {
-            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_RET0, hi);
-            tcg_out32(s, INSN_BV | INSN_R2(TCG_REG_R18));
-            tcg_out_addi(s, TCG_REG_RET0, lo);
-            return;
-        }
-        arg = hi;
-    }
-    tcg_out32(s, INSN_BV | INSN_R2(TCG_REG_R18));
-    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_RET0, arg);
-}
-
-static void tcg_out_goto_tb(TCGContext *s, TCGArg arg)
-{
-    if (s->tb_jmp_offset) {
-        /* direct jump method */
-        fprintf(stderr, "goto_tb direct\n");
-        tcg_abort();
-    } else {
-        /* indirect jump method */
-        tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R20, TCG_REG_R0,
-                   (tcg_target_long)(s->tb_next + arg));
-        tcg_out32(s, INSN_BV_N | INSN_R2(TCG_REG_R20));
-    }
-    s->tb_next_offset[arg] = s->code_ptr - s->code_buf;
-}
-
-static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
-                              const int *const_args)
-{
-    switch (opc) {
-    case INDEX_op_exit_tb:
-        tcg_out_exit_tb(s, args[0]);
-        break;
-    case INDEX_op_goto_tb:
-        tcg_out_goto_tb(s, args[0]);
-        break;
-
-    case INDEX_op_call:
-        if (const_args[0]) {
-            tcg_out_call(s, (void *)args[0]);
-        } else {
-            /* ??? FIXME: the value in the register in args[0] is almost
-               certainly a procedure descriptor, not a code address.  We
-               probably need to use the millicode $$dyncall routine.  */
-            tcg_abort();
-        }
-        break;
-
-    case INDEX_op_br:
-        tcg_out_branch(s, args[0], 1);
-        break;
-
-    case INDEX_op_movi_i32:
-        tcg_out_movi(s, TCG_TYPE_I32, args[0], (uint32_t)args[1]);
-        break;
-
-    case INDEX_op_ld8u_i32:
-        tcg_out_ldst(s, args[0], args[1], args[2], INSN_LDB);
-        break;
-    case INDEX_op_ld8s_i32:
-        tcg_out_ldst(s, args[0], args[1], args[2], INSN_LDB);
-        tcg_out_ext8s(s, args[0], args[0]);
-        break;
-    case INDEX_op_ld16u_i32:
-        tcg_out_ldst(s, args[0], args[1], args[2], INSN_LDH);
-        break;
-    case INDEX_op_ld16s_i32:
-        tcg_out_ldst(s, args[0], args[1], args[2], INSN_LDH);
-        tcg_out_ext16s(s, args[0], args[0]);
-        break;
-    case INDEX_op_ld_i32:
-        tcg_out_ldst(s, args[0], args[1], args[2], INSN_LDW);
-        break;
-
-    case INDEX_op_st8_i32:
-        tcg_out_ldst(s, args[0], args[1], args[2], INSN_STB);
-        break;
-    case INDEX_op_st16_i32:
-        tcg_out_ldst(s, args[0], args[1], args[2], INSN_STH);
-        break;
-    case INDEX_op_st_i32:
-        tcg_out_ldst(s, args[0], args[1], args[2], INSN_STW);
-        break;
-
-    case INDEX_op_add_i32:
-        if (const_args[2]) {
-            tcg_out_addi2(s, args[0], args[1], args[2]);
-        } else {
-            tcg_out_arith(s, args[0], args[1], args[2], INSN_ADDL);
-        }
-        break;
-
-    case INDEX_op_sub_i32:
-        if (const_args[1]) {
-            if (const_args[2]) {
-                tcg_out_movi(s, TCG_TYPE_I32, args[0], args[1] - args[2]);
-            } else {
-                /* Recall that SUBI is a reversed subtract.  */
-                tcg_out_arithi(s, args[0], args[2], args[1], INSN_SUBI);
-            }
-        } else if (const_args[2]) {
-            tcg_out_addi2(s, args[0], args[1], -args[2]);
-        } else {
-            tcg_out_arith(s, args[0], args[1], args[2], INSN_SUB);
-        }
-        break;
-
-    case INDEX_op_and_i32:
-        if (const_args[2]) {
-            tcg_out_andi(s, args[0], args[1], args[2]);
-        } else {
-            tcg_out_arith(s, args[0], args[1], args[2], INSN_AND);
-        }
-        break;
-
-    case INDEX_op_or_i32:
-        if (const_args[2]) {
-            tcg_out_ori(s, args[0], args[1], args[2]);
-        } else {
-            tcg_out_arith(s, args[0], args[1], args[2], INSN_OR);
-        }
-        break;
-
-    case INDEX_op_xor_i32:
-        tcg_out_arith(s, args[0], args[1], args[2], INSN_XOR);
-        break;
-
-    case INDEX_op_andc_i32:
-        if (const_args[2]) {
-            tcg_out_andi(s, args[0], args[1], ~args[2]);
-        } else {
-            tcg_out_arith(s, args[0], args[1], args[2], INSN_ANDCM);
-        }
-        break;
-
-    case INDEX_op_shl_i32:
-        if (const_args[2]) {
-            tcg_out_shli(s, args[0], args[1], args[2]);
-        } else {
-            tcg_out_shl(s, args[0], args[1], args[2]);
-        }
-        break;
-
-    case INDEX_op_shr_i32:
-        if (const_args[2]) {
-            tcg_out_shri(s, args[0], args[1], args[2]);
-        } else {
-            tcg_out_shr(s, args[0], args[1], args[2]);
-        }
-        break;
-
-    case INDEX_op_sar_i32:
-        if (const_args[2]) {
-            tcg_out_sari(s, args[0], args[1], args[2]);
-        } else {
-            tcg_out_sar(s, args[0], args[1], args[2]);
-        }
-        break;
-
-    case INDEX_op_rotl_i32:
-        if (const_args[2]) {
-            tcg_out_rotli(s, args[0], args[1], args[2]);
-        } else {
-            tcg_out_rotl(s, args[0], args[1], args[2]);
-        }
-        break;
-
-    case INDEX_op_rotr_i32:
-        if (const_args[2]) {
-            tcg_out_rotri(s, args[0], args[1], args[2]);
-        } else {
-            tcg_out_rotr(s, args[0], args[1], args[2]);
-        }
-        break;
-
-    case INDEX_op_mul_i32:
-        tcg_out_xmpyu(s, args[0], TCG_REG_R0, args[1], args[2]);
-        break;
-    case INDEX_op_mulu2_i32:
-        tcg_out_xmpyu(s, args[0], args[1], args[2], args[3]);
-        break;
-
-    case INDEX_op_bswap16_i32:
-        tcg_out_bswap16(s, args[0], args[1], 0);
-        break;
-    case INDEX_op_bswap32_i32:
-        tcg_out_bswap32(s, args[0], args[1], TCG_REG_R20);
-        break;
-
-    case INDEX_op_not_i32:
-        tcg_out_arithi(s, args[0], args[1], -1, INSN_SUBI);
-        break;
-    case INDEX_op_ext8s_i32:
-        tcg_out_ext8s(s, args[0], args[1]);
-        break;
-    case INDEX_op_ext16s_i32:
-        tcg_out_ext16s(s, args[0], args[1]);
-        break;
-
-    case INDEX_op_brcond_i32:
-        tcg_out_brcond(s, args[2], args[0], args[1], const_args[1], args[3]);
-        break;
-    case INDEX_op_brcond2_i32:
-        tcg_out_brcond2(s, args[4], args[0], args[1],
-                        args[2], const_args[2],
-                        args[3], const_args[3], args[5]);
-        break;
-
-    case INDEX_op_setcond_i32:
-        tcg_out_setcond(s, args[3], args[0], args[1], args[2], const_args[2]);
-        break;
-    case INDEX_op_setcond2_i32:
-        tcg_out_setcond2(s, args[5], args[0], args[1], args[2],
-                         args[3], const_args[3], args[4], const_args[4]);
-        break;
-
-    case INDEX_op_movcond_i32:
-        tcg_out_movcond(s, args[5], args[0], args[1], args[2], const_args[2],
-                        args[3], const_args[3]);
-        break;
-
-    case INDEX_op_add2_i32:
-        tcg_out_add2(s, args[0], args[1], args[2], args[3],
-                     args[4], args[5], const_args[4]);
-        break;
-
-    case INDEX_op_sub2_i32:
-        tcg_out_sub2(s, args[0], args[1], args[2], args[3],
-                     args[4], args[5], const_args[2], const_args[4]);
-        break;
-
-    case INDEX_op_deposit_i32:
-        if (const_args[2]) {
-            tcg_out_depi(s, args[0], args[2], args[3], args[4]);
-        } else {
-            tcg_out_dep(s, args[0], args[2], args[3], args[4]);
-        }
-        break;
-
-    case INDEX_op_qemu_ld8u:
-        tcg_out_qemu_ld(s, args, 0);
-        break;
-    case INDEX_op_qemu_ld8s:
-        tcg_out_qemu_ld(s, args, 0 | 4);
-        break;
-    case INDEX_op_qemu_ld16u:
-        tcg_out_qemu_ld(s, args, 1);
-        break;
-    case INDEX_op_qemu_ld16s:
-        tcg_out_qemu_ld(s, args, 1 | 4);
-        break;
-    case INDEX_op_qemu_ld32:
-        tcg_out_qemu_ld(s, args, 2);
-        break;
-    case INDEX_op_qemu_ld64:
-        tcg_out_qemu_ld(s, args, 3);
-        break;
-
-    case INDEX_op_qemu_st8:
-        tcg_out_qemu_st(s, args, 0);
-        break;
-    case INDEX_op_qemu_st16:
-        tcg_out_qemu_st(s, args, 1);
-        break;
-    case INDEX_op_qemu_st32:
-        tcg_out_qemu_st(s, args, 2);
-        break;
-    case INDEX_op_qemu_st64:
-        tcg_out_qemu_st(s, args, 3);
-        break;
-
-    default:
-        fprintf(stderr, "unknown opcode 0x%x\n", opc);
-        tcg_abort();
-    }
-}
-
-static const TCGTargetOpDef hppa_op_defs[] = {
-    { INDEX_op_exit_tb, { } },
-    { INDEX_op_goto_tb, { } },
-
-    { INDEX_op_call, { "ri" } },
-    { INDEX_op_br, { } },
-
-    { INDEX_op_mov_i32, { "r", "r" } },
-    { INDEX_op_movi_i32, { "r" } },
-
-    { INDEX_op_ld8u_i32, { "r", "r" } },
-    { INDEX_op_ld8s_i32, { "r", "r" } },
-    { INDEX_op_ld16u_i32, { "r", "r" } },
-    { INDEX_op_ld16s_i32, { "r", "r" } },
-    { INDEX_op_ld_i32, { "r", "r" } },
-    { INDEX_op_st8_i32, { "rZ", "r" } },
-    { INDEX_op_st16_i32, { "rZ", "r" } },
-    { INDEX_op_st_i32, { "rZ", "r" } },
-
-    { INDEX_op_add_i32, { "r", "rZ", "ri" } },
-    { INDEX_op_sub_i32, { "r", "rI", "ri" } },
-    { INDEX_op_and_i32, { "r", "rZ", "rM" } },
-    { INDEX_op_or_i32, { "r", "rZ", "rO" } },
-    { INDEX_op_xor_i32, { "r", "rZ", "rZ" } },
-    /* Note that the second argument will be inverted, which means
-       we want a constant whose inversion matches M, and that O = ~M.
-       See the implementation of and_mask_p.  */
-    { INDEX_op_andc_i32, { "r", "rZ", "rO" } },
-
-    { INDEX_op_mul_i32, { "r", "r", "r" } },
-    { INDEX_op_mulu2_i32, { "r", "r", "r", "r" } },
-
-    { INDEX_op_shl_i32, { "r", "r", "ri" } },
-    { INDEX_op_shr_i32, { "r", "r", "ri" } },
-    { INDEX_op_sar_i32, { "r", "r", "ri" } },
-    { INDEX_op_rotl_i32, { "r", "r", "ri" } },
-    { INDEX_op_rotr_i32, { "r", "r", "ri" } },
-
-    { INDEX_op_bswap16_i32, { "r", "r" } },
-    { INDEX_op_bswap32_i32, { "r", "r" } },
-    { INDEX_op_not_i32, { "r", "r" } },
-
-    { INDEX_op_ext8s_i32, { "r", "r" } },
-    { INDEX_op_ext16s_i32, { "r", "r" } },
-
-    { INDEX_op_brcond_i32, { "rZ", "rJ" } },
-    { INDEX_op_brcond2_i32,  { "rZ", "rZ", "rJ", "rJ" } },
-
-    { INDEX_op_setcond_i32, { "r", "rZ", "rI" } },
-    { INDEX_op_setcond2_i32, { "r", "rZ", "rZ", "rI", "rI" } },
-
-    /* ??? We can actually support a signed 14-bit arg3, but we
-       only have existing constraints for a signed 11-bit.  */
-    { INDEX_op_movcond_i32, { "r", "rZ", "rI", "rI", "0" } },
-
-    { INDEX_op_add2_i32, { "r", "r", "rZ", "rZ", "rI", "rZ" } },
-    { INDEX_op_sub2_i32, { "r", "r", "rI", "rZ", "rK", "rZ" } },
-
-    { INDEX_op_deposit_i32, { "r", "0", "rJ" } },
-
-#if TARGET_LONG_BITS == 32
-    { INDEX_op_qemu_ld8u, { "r", "L" } },
-    { INDEX_op_qemu_ld8s, { "r", "L" } },
-    { INDEX_op_qemu_ld16u, { "r", "L" } },
-    { INDEX_op_qemu_ld16s, { "r", "L" } },
-    { INDEX_op_qemu_ld32, { "r", "L" } },
-    { INDEX_op_qemu_ld64, { "r", "r", "L" } },
-
-    { INDEX_op_qemu_st8, { "LZ", "L" } },
-    { INDEX_op_qemu_st16, { "LZ", "L" } },
-    { INDEX_op_qemu_st32, { "LZ", "L" } },
-    { INDEX_op_qemu_st64, { "LZ", "LZ", "L" } },
-#else
-    { INDEX_op_qemu_ld8u, { "r", "L", "L" } },
-    { INDEX_op_qemu_ld8s, { "r", "L", "L" } },
-    { INDEX_op_qemu_ld16u, { "r", "L", "L" } },
-    { INDEX_op_qemu_ld16s, { "r", "L", "L" } },
-    { INDEX_op_qemu_ld32, { "r", "L", "L" } },
-    { INDEX_op_qemu_ld64, { "r", "r", "L", "L" } },
-
-    { INDEX_op_qemu_st8, { "LZ", "L", "L" } },
-    { INDEX_op_qemu_st16, { "LZ", "L", "L" } },
-    { INDEX_op_qemu_st32, { "LZ", "L", "L" } },
-    { INDEX_op_qemu_st64, { "LZ", "LZ", "L", "L" } },
-#endif
-    { -1 },
-};
-
-static int tcg_target_callee_save_regs[] = {
-    /* R2, the return address register, is saved specially
-       in the caller's frame.  */
-    /* R3, the frame pointer, is not currently modified.  */
-    TCG_REG_R4,
-    TCG_REG_R5,
-    TCG_REG_R6,
-    TCG_REG_R7,
-    TCG_REG_R8,
-    TCG_REG_R9,
-    TCG_REG_R10,
-    TCG_REG_R11,
-    TCG_REG_R12,
-    TCG_REG_R13,
-    TCG_REG_R14,
-    TCG_REG_R15,
-    TCG_REG_R16,
-    TCG_REG_R17, /* R17 is the global env.  */
-    TCG_REG_R18
-};
-
-#define FRAME_SIZE ((-TCG_TARGET_CALL_STACK_OFFSET \
-                     + TCG_TARGET_STATIC_CALL_ARGS_SIZE \
-                     + ARRAY_SIZE(tcg_target_callee_save_regs) * 4 \
-                     + CPU_TEMP_BUF_NLONGS * sizeof(long) \
-                     + TCG_TARGET_STACK_ALIGN - 1) \
-                    & -TCG_TARGET_STACK_ALIGN)
-
-static void tcg_target_qemu_prologue(TCGContext *s)
-{
-    int frame_size, i;
-
-    frame_size = FRAME_SIZE;
-
-    /* The return address is stored in the caller's frame.  */
-    tcg_out_st(s, TCG_TYPE_PTR, TCG_REG_RP, TCG_REG_CALL_STACK, -20);
-
-    /* Allocate stack frame, saving the first register at the same time.  */
-    tcg_out_ldst(s, tcg_target_callee_save_regs[0],
-                 TCG_REG_CALL_STACK, frame_size, INSN_STWM);
-
-    /* Save all callee saved registers.  */
-    for (i = 1; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) {
-        tcg_out_st(s, TCG_TYPE_PTR, tcg_target_callee_save_regs[i],
-                   TCG_REG_CALL_STACK, -frame_size + i * 4);
-    }
-
-    /* Record the location of the TCG temps.  */
-    tcg_set_frame(s, TCG_REG_CALL_STACK, -frame_size + i * 4,
-                  CPU_TEMP_BUF_NLONGS * sizeof(long));
-
-#ifdef CONFIG_USE_GUEST_BASE
-    if (GUEST_BASE != 0) {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, GUEST_BASE);
-        tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
-    }
-#endif
-
-    tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
-
-    /* Jump to TB, and adjust R18 to be the return address.  */
-    tcg_out32(s, INSN_BLE_SR4 | INSN_R2(tcg_target_call_iarg_regs[1]));
-    tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_R18, TCG_REG_R31);
-
-    /* Restore callee saved registers.  */
-    tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_RP, TCG_REG_CALL_STACK,
-               -frame_size - 20);
-    for (i = 1; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) {
-        tcg_out_ld(s, TCG_TYPE_PTR, tcg_target_callee_save_regs[i],
-                   TCG_REG_CALL_STACK, -frame_size + i * 4);
-    }
-
-    /* Deallocate stack frame and return.  */
-    tcg_out32(s, INSN_BV | INSN_R2(TCG_REG_RP));
-    tcg_out_ldst(s, tcg_target_callee_save_regs[0],
-                 TCG_REG_CALL_STACK, -frame_size, INSN_LDWM);
-}
-
-static void tcg_target_init(TCGContext *s)
-{
-    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xffffffff);
-
-    tcg_regset_clear(tcg_target_call_clobber_regs);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R20);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R21);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R22);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R23);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R24);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R25);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R26);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_RET0);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_RET1);
-
-    tcg_regset_clear(s->reserved_regs);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R0);  /* hardwired to zero */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R1);  /* addil target */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_RP);  /* link register */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R3);  /* frame pointer */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R18); /* return pointer */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R19); /* clobbered w/o pic */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R20); /* reserved */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_DP);  /* data pointer */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK);  /* stack pointer */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R31); /* ble link reg */
-
-    tcg_add_target_add_op_defs(hppa_op_defs);
-}
-
-typedef struct {
-    DebugFrameCIE cie;
-    DebugFrameFDEHeader fde;
-    uint8_t fde_def_cfa[4];
-    uint8_t fde_ret_ofs[3];
-    uint8_t fde_reg_ofs[ARRAY_SIZE(tcg_target_callee_save_regs) * 2];
-} DebugFrame;
-
-#define ELF_HOST_MACHINE  EM_PARISC
-#define ELF_HOST_FLAGS    EFA_PARISC_1_1
-
-/* ??? BFD (and thus GDB) wants very much to distinguish between HPUX
-   and other extensions.  We don't really care, but if we don't set this
-   to *something* then the object file won't be properly matched.  */
-#define ELF_OSABI         ELFOSABI_LINUX
-
-static DebugFrame debug_frame = {
-    .cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
-    .cie.id = -1,
-    .cie.version = 1,
-    .cie.code_align = 1,
-    .cie.data_align = 1,
-    .cie.return_column = 2,
-
-    /* Total FDE size does not include the "len" member.  */
-    .fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, fde.cie_offset),
-
-    .fde_def_cfa = {
-        0x12, 30,                       /* DW_CFA_def_cfa_sf sp, ... */
-        (-FRAME_SIZE & 0x7f) | 0x80,     /* ... sleb128 -FRAME_SIZE */
-        (-FRAME_SIZE >> 7) & 0x7f
-    },
-    .fde_ret_ofs = {
-        0x11, 2, (-20 / 4) & 0x7f       /* DW_CFA_offset_extended_sf r2, 20 */
-    },
-    .fde_reg_ofs = {
-        /* This must match the ordering in tcg_target_callee_save_regs.  */
-        0x80 + 4, 0,                    /* DW_CFA_offset r4, 0 */
-        0x80 + 5, 4,                    /* DW_CFA_offset r5, 4 */
-        0x80 + 6, 8,                    /* DW_CFA_offset r6, 8 */
-        0x80 + 7, 12,                    /* ... */
-        0x80 + 8, 16,
-        0x80 + 9, 20,
-        0x80 + 10, 24,
-        0x80 + 11, 28,
-        0x80 + 12, 32,
-        0x80 + 13, 36,
-        0x80 + 14, 40,
-        0x80 + 15, 44,
-        0x80 + 16, 48,
-        0x80 + 17, 52,
-        0x80 + 18, 56,
-    }
-};
-
-void tcg_register_jit(void *buf, size_t buf_size)
-{
-    debug_frame.fde.func_start = (tcg_target_long) buf;
-    debug_frame.fde.func_len = buf_size;
-
-    tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
-}
diff --git a/tcg/hppa/tcg-target.h b/tcg/hppa/tcg-target.h
deleted file mode 100644
index 122edce7a7..0000000000
--- a/tcg/hppa/tcg-target.h
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Tiny Code Generator for QEMU
- *
- * Copyright (c) 2008 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#ifndef TCG_TARGET_HPPA
-#define TCG_TARGET_HPPA 1
-
-#define TCG_TARGET_WORDS_BIGENDIAN
-
-#define TCG_TARGET_NB_REGS 32
-
-typedef enum {
-    TCG_REG_R0 = 0,
-    TCG_REG_R1,
-    TCG_REG_RP,
-    TCG_REG_R3,
-    TCG_REG_R4,
-    TCG_REG_R5,
-    TCG_REG_R6,
-    TCG_REG_R7,
-    TCG_REG_R8,
-    TCG_REG_R9,
-    TCG_REG_R10,
-    TCG_REG_R11,
-    TCG_REG_R12,
-    TCG_REG_R13,
-    TCG_REG_R14,
-    TCG_REG_R15,
-    TCG_REG_R16,
-    TCG_REG_R17,
-    TCG_REG_R18,
-    TCG_REG_R19,
-    TCG_REG_R20,
-    TCG_REG_R21,
-    TCG_REG_R22,
-    TCG_REG_R23,
-    TCG_REG_R24,
-    TCG_REG_R25,
-    TCG_REG_R26,
-    TCG_REG_DP,
-    TCG_REG_RET0,
-    TCG_REG_RET1,
-    TCG_REG_SP,
-    TCG_REG_R31,
-} TCGReg;
-
-#define TCG_CT_CONST_0    0x0100
-#define TCG_CT_CONST_S5   0x0200
-#define TCG_CT_CONST_S11  0x0400
-#define TCG_CT_CONST_MS11 0x0800
-#define TCG_CT_CONST_AND  0x1000
-#define TCG_CT_CONST_OR   0x2000
-
-/* used for function call generation */
-#define TCG_REG_CALL_STACK TCG_REG_SP
-#define TCG_TARGET_STACK_ALIGN 64
-#define TCG_TARGET_CALL_STACK_OFFSET -48
-#define TCG_TARGET_STATIC_CALL_ARGS_SIZE 8*4
-#define TCG_TARGET_CALL_ALIGN_ARGS 1
-#define TCG_TARGET_STACK_GROWSUP
-
-/* optional instructions */
-#define TCG_TARGET_HAS_div_i32          0
-#define TCG_TARGET_HAS_rem_i32          0
-#define TCG_TARGET_HAS_rot_i32          1
-#define TCG_TARGET_HAS_ext8s_i32        1
-#define TCG_TARGET_HAS_ext16s_i32       1
-#define TCG_TARGET_HAS_bswap16_i32      1
-#define TCG_TARGET_HAS_bswap32_i32      1
-#define TCG_TARGET_HAS_not_i32          1
-#define TCG_TARGET_HAS_andc_i32         1
-#define TCG_TARGET_HAS_orc_i32          0
-#define TCG_TARGET_HAS_eqv_i32          0
-#define TCG_TARGET_HAS_nand_i32         0
-#define TCG_TARGET_HAS_nor_i32          0
-#define TCG_TARGET_HAS_deposit_i32      1
-#define TCG_TARGET_HAS_movcond_i32      1
-#define TCG_TARGET_HAS_muls2_i32        0
-#define TCG_TARGET_HAS_muluh_i32        0
-#define TCG_TARGET_HAS_mulsh_i32        0
-
-/* optional instructions automatically implemented */
-#define TCG_TARGET_HAS_neg_i32          0 /* sub rd, 0, rs */
-#define TCG_TARGET_HAS_ext8u_i32        0 /* and rd, rs, 0xff */
-#define TCG_TARGET_HAS_ext16u_i32       0 /* and rd, rs, 0xffff */
-
-#define TCG_AREG0 TCG_REG_R17
-
-
-static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
-{
-    start &= ~31;
-    while (start <= stop) {
-        asm volatile ("fdc 0(%0)\n\t"
-                      "sync\n\t"
-                      "fic 0(%%sr4, %0)\n\t"
-                      "sync"
-                      : : "r"(start) : "memory");
-        start += 32;
-    }
-}
-
-#endif
diff --git a/tcg/i386/tcg-target.c b/tcg/i386/tcg-target.c
index c1f07415ab..7ac8e45485 100644
--- a/tcg/i386/tcg-target.c
+++ b/tcg/i386/tcg-target.c
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "tcg-be-ldst.h"
+
 #ifndef NDEBUG
 static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
 #if TCG_TARGET_REG_BITS == 64
@@ -1024,39 +1026,33 @@ static void tcg_out_jmp(TCGContext *s, uintptr_t dest)
 /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
  *                                     int mmu_idx, uintptr_t ra)
  */
-static const void * const qemu_ld_helpers[4] = {
-    helper_ret_ldub_mmu,
-    helper_ret_lduw_mmu,
-    helper_ret_ldul_mmu,
-    helper_ret_ldq_mmu,
+static const void * const qemu_ld_helpers[16] = {
+    [MO_UB]   = helper_ret_ldub_mmu,
+    [MO_LEUW] = helper_le_lduw_mmu,
+    [MO_LEUL] = helper_le_ldul_mmu,
+    [MO_LEQ]  = helper_le_ldq_mmu,
+    [MO_BEUW] = helper_be_lduw_mmu,
+    [MO_BEUL] = helper_be_ldul_mmu,
+    [MO_BEQ]  = helper_be_ldq_mmu,
 };
 
 /* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,
  *                                     uintxx_t val, int mmu_idx, uintptr_t ra)
  */
-static const void * const qemu_st_helpers[4] = {
-    helper_ret_stb_mmu,
-    helper_ret_stw_mmu,
-    helper_ret_stl_mmu,
-    helper_ret_stq_mmu,
+static const void * const qemu_st_helpers[16] = {
+    [MO_UB]   = helper_ret_stb_mmu,
+    [MO_LEUW] = helper_le_stw_mmu,
+    [MO_LEUL] = helper_le_stl_mmu,
+    [MO_LEQ]  = helper_le_stq_mmu,
+    [MO_BEUW] = helper_be_stw_mmu,
+    [MO_BEUL] = helper_be_stl_mmu,
+    [MO_BEQ]  = helper_be_stq_mmu,
 };
 
-static void add_qemu_ldst_label(TCGContext *s,
-                                int is_ld,
-                                int opc,
-                                int data_reg,
-                                int data_reg2,
-                                int addrlo_reg,
-                                int addrhi_reg,
-                                int mem_index,
-                                uint8_t *raddr,
-                                uint8_t **label_ptr);
-
 /* Perform the TLB load and compare.
 
    Inputs:
-   ADDRLO_IDX contains the index into ARGS of the low part of the
-   address; the high part of the address is at ADDR_LOW_IDX+1.
+   ADDRLO and ADDRHI contain the low and high part of the address.
 
    MEM_INDEX and S_BITS are the memory context and log2 size of the load.
 
@@ -1074,14 +1070,12 @@ static void add_qemu_ldst_label(TCGContext *s,
 
    First argument register is clobbered.  */
 
-static inline void tcg_out_tlb_load(TCGContext *s, int addrlo_idx,
-                                    int mem_index, int s_bits,
-                                    const TCGArg *args,
+static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi,
+                                    int mem_index, TCGMemOp s_bits,
                                     uint8_t **label_ptr, int which)
 {
-    const int addrlo = args[addrlo_idx];
-    const int r0 = TCG_REG_L0;
-    const int r1 = TCG_REG_L1;
+    const TCGReg r0 = TCG_REG_L0;
+    const TCGReg r1 = TCG_REG_L1;
     TCGType ttype = TCG_TYPE_I32;
     TCGType htype = TCG_TYPE_I32;
     int trexw = 0, hrexw = 0;
@@ -1130,7 +1124,7 @@ static inline void tcg_out_tlb_load(TCGContext *s, int addrlo_idx,
 
     if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
         /* cmp 4(r0), addrhi */
-        tcg_out_modrm_offset(s, OPC_CMP_GvEv, args[addrlo_idx+1], r0, 4);
+        tcg_out_modrm_offset(s, OPC_CMP_GvEv, addrhi, r0, 4);
 
         /* jne slow_path */
         tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0);
@@ -1144,6 +1138,182 @@ static inline void tcg_out_tlb_load(TCGContext *s, int addrlo_idx,
     tcg_out_modrm_offset(s, OPC_ADD_GvEv + hrexw, r1, r0,
                          offsetof(CPUTLBEntry, addend) - which);
 }
+
+/*
+ * Record the context of a call to the out of line helper code for the slow path
+ * for a load or store, so that we can later generate the correct helper code
+ */
+static void add_qemu_ldst_label(TCGContext *s, int is_ld, TCGMemOp opc,
+                                TCGReg datalo, TCGReg datahi,
+                                TCGReg addrlo, TCGReg addrhi,
+                                int mem_index, uint8_t *raddr,
+                                uint8_t **label_ptr)
+{
+    TCGLabelQemuLdst *label = new_ldst_label(s);
+
+    label->is_ld = is_ld;
+    label->opc = opc;
+    label->datalo_reg = datalo;
+    label->datahi_reg = datahi;
+    label->addrlo_reg = addrlo;
+    label->addrhi_reg = addrhi;
+    label->mem_index = mem_index;
+    label->raddr = raddr;
+    label->label_ptr[0] = label_ptr[0];
+    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+        label->label_ptr[1] = label_ptr[1];
+    }
+}
+
+/*
+ * Generate code for the slow path for a load at the end of block
+ */
+static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
+{
+    TCGMemOp opc = l->opc;
+    TCGReg data_reg;
+    uint8_t **label_ptr = &l->label_ptr[0];
+
+    /* resolve label address */
+    *(uint32_t *)label_ptr[0] = (uint32_t)(s->code_ptr - label_ptr[0] - 4);
+    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+        *(uint32_t *)label_ptr[1] = (uint32_t)(s->code_ptr - label_ptr[1] - 4);
+    }
+
+    if (TCG_TARGET_REG_BITS == 32) {
+        int ofs = 0;
+
+        tcg_out_st(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP, ofs);
+        ofs += 4;
+
+        tcg_out_st(s, TCG_TYPE_I32, l->addrlo_reg, TCG_REG_ESP, ofs);
+        ofs += 4;
+
+        if (TARGET_LONG_BITS == 64) {
+            tcg_out_st(s, TCG_TYPE_I32, l->addrhi_reg, TCG_REG_ESP, ofs);
+            ofs += 4;
+        }
+
+        tcg_out_sti(s, TCG_TYPE_I32, TCG_REG_ESP, ofs, l->mem_index);
+        ofs += 4;
+
+        tcg_out_sti(s, TCG_TYPE_I32, TCG_REG_ESP, ofs, (uintptr_t)l->raddr);
+    } else {
+        tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
+        /* The second argument is already loaded with addrlo.  */
+        tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[2],
+                     l->mem_index);
+        tcg_out_movi(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[3],
+                     (uintptr_t)l->raddr);
+    }
+
+    tcg_out_calli(s, (uintptr_t)qemu_ld_helpers[opc & ~MO_SIGN]);
+
+    data_reg = l->datalo_reg;
+    switch (opc & MO_SSIZE) {
+    case MO_SB:
+        tcg_out_ext8s(s, data_reg, TCG_REG_EAX, P_REXW);
+        break;
+    case MO_SW:
+        tcg_out_ext16s(s, data_reg, TCG_REG_EAX, P_REXW);
+        break;
+#if TCG_TARGET_REG_BITS == 64
+    case MO_SL:
+        tcg_out_ext32s(s, data_reg, TCG_REG_EAX);
+        break;
+#endif
+    case MO_UB:
+    case MO_UW:
+        /* Note that the helpers have zero-extended to tcg_target_long.  */
+    case MO_UL:
+        tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
+        break;
+    case MO_Q:
+        if (TCG_TARGET_REG_BITS == 64) {
+            tcg_out_mov(s, TCG_TYPE_I64, data_reg, TCG_REG_RAX);
+        } else if (data_reg == TCG_REG_EDX) {
+            /* xchg %edx, %eax */
+            tcg_out_opc(s, OPC_XCHG_ax_r32 + TCG_REG_EDX, 0, 0, 0);
+            tcg_out_mov(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_EAX);
+        } else {
+            tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
+            tcg_out_mov(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_EDX);
+        }
+        break;
+    default:
+        tcg_abort();
+    }
+
+    /* Jump to the code corresponding to next IR of qemu_st */
+    tcg_out_jmp(s, (uintptr_t)l->raddr);
+}
+
+/*
+ * Generate code for the slow path for a store at the end of block
+ */
+static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
+{
+    TCGMemOp opc = l->opc;
+    TCGMemOp s_bits = opc & MO_SIZE;
+    uint8_t **label_ptr = &l->label_ptr[0];
+    TCGReg retaddr;
+
+    /* resolve label address */
+    *(uint32_t *)label_ptr[0] = (uint32_t)(s->code_ptr - label_ptr[0] - 4);
+    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+        *(uint32_t *)label_ptr[1] = (uint32_t)(s->code_ptr - label_ptr[1] - 4);
+    }
+
+    if (TCG_TARGET_REG_BITS == 32) {
+        int ofs = 0;
+
+        tcg_out_st(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP, ofs);
+        ofs += 4;
+
+        tcg_out_st(s, TCG_TYPE_I32, l->addrlo_reg, TCG_REG_ESP, ofs);
+        ofs += 4;
+
+        if (TARGET_LONG_BITS == 64) {
+            tcg_out_st(s, TCG_TYPE_I32, l->addrhi_reg, TCG_REG_ESP, ofs);
+            ofs += 4;
+        }
+
+        tcg_out_st(s, TCG_TYPE_I32, l->datalo_reg, TCG_REG_ESP, ofs);
+        ofs += 4;
+
+        if (s_bits == MO_64) {
+            tcg_out_st(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_ESP, ofs);
+            ofs += 4;
+        }
+
+        tcg_out_sti(s, TCG_TYPE_I32, TCG_REG_ESP, ofs, l->mem_index);
+        ofs += 4;
+
+        retaddr = TCG_REG_EAX;
+        tcg_out_movi(s, TCG_TYPE_I32, retaddr, (uintptr_t)l->raddr);
+        tcg_out_st(s, TCG_TYPE_I32, retaddr, TCG_REG_ESP, ofs);
+    } else {
+        tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
+        /* The second argument is already loaded with addrlo.  */
+        tcg_out_mov(s, (s_bits == MO_64 ? TCG_TYPE_I64 : TCG_TYPE_I32),
+                    tcg_target_call_iarg_regs[2], l->datalo_reg);
+        tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[3],
+                     l->mem_index);
+
+        if (ARRAY_SIZE(tcg_target_call_iarg_regs) > 4) {
+            retaddr = tcg_target_call_iarg_regs[4];
+            tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr);
+        } else {
+            retaddr = TCG_REG_RAX;
+            tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr);
+            tcg_out_st(s, TCG_TYPE_PTR, retaddr, TCG_REG_ESP, 0);
+        }
+    }
+
+    /* "Tail call" to the helper, with the return address back inline.  */
+    tcg_out_push(s, retaddr);
+    tcg_out_jmp(s, (uintptr_t)qemu_st_helpers[opc]);
+}
 #elif defined(__x86_64__) && defined(__linux__)
 # include <asm/prctl.h>
 # include <sys/prctl.h>
@@ -1162,28 +1332,26 @@ static inline void setup_guest_base_seg(void)
 static inline void setup_guest_base_seg(void) { }
 #endif /* SOFTMMU */
 
-static void tcg_out_qemu_ld_direct(TCGContext *s, int datalo, int datahi,
-                                   int base, intptr_t ofs, int seg, int sizeop)
+static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
+                                   TCGReg base, intptr_t ofs, int seg,
+                                   TCGMemOp memop)
 {
-#ifdef TARGET_WORDS_BIGENDIAN
-    const int bswap = 1;
-#else
-    const int bswap = 0;
-#endif
-    switch (sizeop) {
-    case 0:
+    const TCGMemOp bswap = memop & MO_BSWAP;
+
+    switch (memop & MO_SSIZE) {
+    case MO_UB:
         tcg_out_modrm_offset(s, OPC_MOVZBL + seg, datalo, base, ofs);
         break;
-    case 0 | 4:
+    case MO_SB:
         tcg_out_modrm_offset(s, OPC_MOVSBL + P_REXW + seg, datalo, base, ofs);
         break;
-    case 1:
+    case MO_UW:
         tcg_out_modrm_offset(s, OPC_MOVZWL + seg, datalo, base, ofs);
         if (bswap) {
             tcg_out_rolw_8(s, datalo);
         }
         break;
-    case 1 | 4:
+    case MO_SW:
         if (bswap) {
             tcg_out_modrm_offset(s, OPC_MOVZWL + seg, datalo, base, ofs);
             tcg_out_rolw_8(s, datalo);
@@ -1193,14 +1361,14 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, int datalo, int datahi,
                                  datalo, base, ofs);
         }
         break;
-    case 2:
+    case MO_UL:
         tcg_out_modrm_offset(s, OPC_MOVL_GvEv + seg, datalo, base, ofs);
         if (bswap) {
             tcg_out_bswap32(s, datalo);
         }
         break;
 #if TCG_TARGET_REG_BITS == 64
-    case 2 | 4:
+    case MO_SL:
         if (bswap) {
             tcg_out_modrm_offset(s, OPC_MOVL_GvEv + seg, datalo, base, ofs);
             tcg_out_bswap32(s, datalo);
@@ -1210,7 +1378,7 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, int datalo, int datahi,
         }
         break;
 #endif
-    case 3:
+    case MO_Q:
         if (TCG_TARGET_REG_BITS == 64) {
             tcg_out_modrm_offset(s, OPC_MOVL_GvEv + P_REXW + seg,
                                  datalo, base, ofs);
@@ -1248,48 +1416,40 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, int datalo, int datahi,
 /* XXX: qemu_ld and qemu_st could be modified to clobber only EDX and
    EAX. It will be useful once fixed registers globals are less
    common. */
-static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args,
-                            int opc)
+static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
 {
-    int data_reg, data_reg2 = 0;
-    int addrlo_idx;
+    TCGReg datalo, datahi, addrlo;
+    TCGReg addrhi __attribute__((unused));
+    TCGMemOp opc;
 #if defined(CONFIG_SOFTMMU)
-    int mem_index, s_bits;
+    int mem_index;
+    TCGMemOp s_bits;
     uint8_t *label_ptr[2];
 #endif
 
-    data_reg = args[0];
-    addrlo_idx = 1;
-    if (TCG_TARGET_REG_BITS == 32 && opc == 3) {
-        data_reg2 = args[1];
-        addrlo_idx = 2;
-    }
+    datalo = *args++;
+    datahi = (TCG_TARGET_REG_BITS == 32 && is64 ? *args++ : 0);
+    addrlo = *args++;
+    addrhi = (TARGET_LONG_BITS > TCG_TARGET_REG_BITS ? *args++ : 0);
+    opc = *args++;
 
 #if defined(CONFIG_SOFTMMU)
-    mem_index = args[addrlo_idx + 1 + (TARGET_LONG_BITS > TCG_TARGET_REG_BITS)];
-    s_bits = opc & 3;
+    mem_index = *args++;
+    s_bits = opc & MO_SIZE;
 
-    tcg_out_tlb_load(s, addrlo_idx, mem_index, s_bits, args,
+    tcg_out_tlb_load(s, addrlo, addrhi, mem_index, s_bits,
                      label_ptr, offsetof(CPUTLBEntry, addr_read));
 
     /* TLB Hit.  */
-    tcg_out_qemu_ld_direct(s, data_reg, data_reg2, TCG_REG_L1, 0, 0, opc);
+    tcg_out_qemu_ld_direct(s, datalo, datahi, TCG_REG_L1, 0, 0, opc);
 
     /* Record the current context of a load into ldst label */
-    add_qemu_ldst_label(s,
-                        1,
-                        opc,
-                        data_reg,
-                        data_reg2,
-                        args[addrlo_idx],
-                        args[addrlo_idx + 1],
-                        mem_index,
-                        s->code_ptr,
-                        label_ptr);
+    add_qemu_ldst_label(s, 1, opc, datalo, datahi, addrlo, addrhi,
+                        mem_index, s->code_ptr, label_ptr);
 #else
     {
         int32_t offset = GUEST_BASE;
-        int base = args[addrlo_idx];
+        TCGReg base = addrlo;
         int seg = 0;
 
         /* ??? We assume all operations have left us with register contents
@@ -1307,32 +1467,35 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args,
             offset = 0;
         }
 
-        tcg_out_qemu_ld_direct(s, data_reg, data_reg2, base, offset, seg, opc);
+        tcg_out_qemu_ld_direct(s, datalo, datahi, base, offset, seg, opc);
     }
 #endif
 }
 
-static void tcg_out_qemu_st_direct(TCGContext *s, int datalo, int datahi,
-                                   int base, intptr_t ofs, int seg,
-                                   int sizeop)
+static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
+                                   TCGReg base, intptr_t ofs, int seg,
+                                   TCGMemOp memop)
 {
-#ifdef TARGET_WORDS_BIGENDIAN
-    const int bswap = 1;
-#else
-    const int bswap = 0;
-#endif
+    const TCGMemOp bswap = memop & MO_BSWAP;
+
     /* ??? Ideally we wouldn't need a scratch register.  For user-only,
        we could perform the bswap twice to restore the original value
        instead of moving to the scratch.  But as it is, the L constraint
        means that TCG_REG_L0 is definitely free here.  */
-    const int scratch = TCG_REG_L0;
+    const TCGReg scratch = TCG_REG_L0;
 
-    switch (sizeop) {
-    case 0:
+    switch (memop & MO_SIZE) {
+    case MO_8:
+        /* In 32-bit mode, 8-byte stores can only happen from [abcd]x.
+           Use the scratch register if necessary.  */
+        if (TCG_TARGET_REG_BITS == 32 && datalo >= 4) {
+            tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo);
+            datalo = scratch;
+        }
         tcg_out_modrm_offset(s, OPC_MOVB_EvGv + P_REXB_R + seg,
                              datalo, base, ofs);
         break;
-    case 1:
+    case MO_16:
         if (bswap) {
             tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo);
             tcg_out_rolw_8(s, scratch);
@@ -1341,7 +1504,7 @@ static void tcg_out_qemu_st_direct(TCGContext *s, int datalo, int datahi,
         tcg_out_modrm_offset(s, OPC_MOVL_EvGv + P_DATA16 + seg,
                              datalo, base, ofs);
         break;
-    case 2:
+    case MO_32:
         if (bswap) {
             tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo);
             tcg_out_bswap32(s, scratch);
@@ -1349,7 +1512,7 @@ static void tcg_out_qemu_st_direct(TCGContext *s, int datalo, int datahi,
         }
         tcg_out_modrm_offset(s, OPC_MOVL_EvGv + seg, datalo, base, ofs);
         break;
-    case 3:
+    case MO_64:
         if (TCG_TARGET_REG_BITS == 64) {
             if (bswap) {
                 tcg_out_mov(s, TCG_TYPE_I64, scratch, datalo);
@@ -1375,48 +1538,40 @@ static void tcg_out_qemu_st_direct(TCGContext *s, int datalo, int datahi,
     }
 }
 
-static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args,
-                            int opc)
+static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
 {
-    int data_reg, data_reg2 = 0;
-    int addrlo_idx;
+    TCGReg datalo, datahi, addrlo;
+    TCGReg addrhi __attribute__((unused));
+    TCGMemOp opc;
 #if defined(CONFIG_SOFTMMU)
-    int mem_index, s_bits;
+    int mem_index;
+    TCGMemOp s_bits;
     uint8_t *label_ptr[2];
 #endif
 
-    data_reg = args[0];
-    addrlo_idx = 1;
-    if (TCG_TARGET_REG_BITS == 32 && opc == 3) {
-        data_reg2 = args[1];
-        addrlo_idx = 2;
-    }
+    datalo = *args++;
+    datahi = (TCG_TARGET_REG_BITS == 32 && is64 ? *args++ : 0);
+    addrlo = *args++;
+    addrhi = (TARGET_LONG_BITS > TCG_TARGET_REG_BITS ? *args++ : 0);
+    opc = *args++;
 
 #if defined(CONFIG_SOFTMMU)
-    mem_index = args[addrlo_idx + 1 + (TARGET_LONG_BITS > TCG_TARGET_REG_BITS)];
-    s_bits = opc;
+    mem_index = *args++;
+    s_bits = opc & MO_SIZE;
 
-    tcg_out_tlb_load(s, addrlo_idx, mem_index, s_bits, args,
+    tcg_out_tlb_load(s, addrlo, addrhi, mem_index, s_bits,
                      label_ptr, offsetof(CPUTLBEntry, addr_write));
 
     /* TLB Hit.  */
-    tcg_out_qemu_st_direct(s, data_reg, data_reg2, TCG_REG_L1, 0, 0, opc);
+    tcg_out_qemu_st_direct(s, datalo, datahi, TCG_REG_L1, 0, 0, opc);
 
     /* Record the current context of a store into ldst label */
-    add_qemu_ldst_label(s,
-                        0,
-                        opc,
-                        data_reg,
-                        data_reg2,
-                        args[addrlo_idx],
-                        args[addrlo_idx + 1],
-                        mem_index,
-                        s->code_ptr,
-                        label_ptr);
+    add_qemu_ldst_label(s, 0, opc, datalo, datahi, addrlo, addrhi,
+                        mem_index, s->code_ptr, label_ptr);
 #else
     {
         int32_t offset = GUEST_BASE;
-        int base = args[addrlo_idx];
+        TCGReg base = addrlo;
         int seg = 0;
 
         /* ??? We assume all operations have left us with register contents
@@ -1434,221 +1589,11 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args,
             offset = 0;
         }
 
-        tcg_out_qemu_st_direct(s, data_reg, data_reg2, base, offset, seg, opc);
-    }
-#endif
-}
-
-#if defined(CONFIG_SOFTMMU)
-/*
- * Record the context of a call to the out of line helper code for the slow path
- * for a load or store, so that we can later generate the correct helper code
- */
-static void add_qemu_ldst_label(TCGContext *s,
-                                int is_ld,
-                                int opc,
-                                int data_reg,
-                                int data_reg2,
-                                int addrlo_reg,
-                                int addrhi_reg,
-                                int mem_index,
-                                uint8_t *raddr,
-                                uint8_t **label_ptr)
-{
-    int idx;
-    TCGLabelQemuLdst *label;
-
-    if (s->nb_qemu_ldst_labels >= TCG_MAX_QEMU_LDST) {
-        tcg_abort();
-    }
-
-    idx = s->nb_qemu_ldst_labels++;
-    label = (TCGLabelQemuLdst *)&s->qemu_ldst_labels[idx];
-    label->is_ld = is_ld;
-    label->opc = opc;
-    label->datalo_reg = data_reg;
-    label->datahi_reg = data_reg2;
-    label->addrlo_reg = addrlo_reg;
-    label->addrhi_reg = addrhi_reg;
-    label->mem_index = mem_index;
-    label->raddr = raddr;
-    label->label_ptr[0] = label_ptr[0];
-    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
-        label->label_ptr[1] = label_ptr[1];
-    }
-}
-
-/*
- * Generate code for the slow path for a load at the end of block
- */
-static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
-{
-    int opc = l->opc;
-    int s_bits = opc & 3;
-    TCGReg data_reg;
-    uint8_t **label_ptr = &l->label_ptr[0];
-
-    /* resolve label address */
-    *(uint32_t *)label_ptr[0] = (uint32_t)(s->code_ptr - label_ptr[0] - 4);
-    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
-        *(uint32_t *)label_ptr[1] = (uint32_t)(s->code_ptr - label_ptr[1] - 4);
-    }
-
-    if (TCG_TARGET_REG_BITS == 32) {
-        int ofs = 0;
-
-        tcg_out_st(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP, ofs);
-        ofs += 4;
-
-        tcg_out_st(s, TCG_TYPE_I32, l->addrlo_reg, TCG_REG_ESP, ofs);
-        ofs += 4;
-
-        if (TARGET_LONG_BITS == 64) {
-            tcg_out_st(s, TCG_TYPE_I32, l->addrhi_reg, TCG_REG_ESP, ofs);
-            ofs += 4;
-        }
-
-        tcg_out_sti(s, TCG_TYPE_I32, TCG_REG_ESP, ofs, l->mem_index);
-        ofs += 4;
-
-        tcg_out_sti(s, TCG_TYPE_I32, TCG_REG_ESP, ofs, (uintptr_t)l->raddr);
-    } else {
-        tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
-        /* The second argument is already loaded with addrlo.  */
-        tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[2],
-                     l->mem_index);
-        tcg_out_movi(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[3],
-                     (uintptr_t)l->raddr);
+        tcg_out_qemu_st_direct(s, datalo, datahi, base, offset, seg, opc);
     }
-
-    tcg_out_calli(s, (uintptr_t)qemu_ld_helpers[s_bits]);
-
-    data_reg = l->datalo_reg;
-    switch(opc) {
-    case 0 | 4:
-        tcg_out_ext8s(s, data_reg, TCG_REG_EAX, P_REXW);
-        break;
-    case 1 | 4:
-        tcg_out_ext16s(s, data_reg, TCG_REG_EAX, P_REXW);
-        break;
-#if TCG_TARGET_REG_BITS == 64
-    case 2 | 4:
-        tcg_out_ext32s(s, data_reg, TCG_REG_EAX);
-        break;
 #endif
-    case 0:
-    case 1:
-        /* Note that the helpers have zero-extended to tcg_target_long.  */
-    case 2:
-        tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
-        break;
-    case 3:
-        if (TCG_TARGET_REG_BITS == 64) {
-            tcg_out_mov(s, TCG_TYPE_I64, data_reg, TCG_REG_RAX);
-        } else if (data_reg == TCG_REG_EDX) {
-            /* xchg %edx, %eax */
-            tcg_out_opc(s, OPC_XCHG_ax_r32 + TCG_REG_EDX, 0, 0, 0);
-            tcg_out_mov(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_EAX);
-        } else {
-            tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
-            tcg_out_mov(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_EDX);
-        }
-        break;
-    default:
-        tcg_abort();
-    }
-
-    /* Jump to the code corresponding to next IR of qemu_st */
-    tcg_out_jmp(s, (uintptr_t)l->raddr);
 }
 
-/*
- * Generate code for the slow path for a store at the end of block
- */
-static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
-{
-    int opc = l->opc;
-    int s_bits = opc & 3;
-    uint8_t **label_ptr = &l->label_ptr[0];
-    TCGReg retaddr;
-
-    /* resolve label address */
-    *(uint32_t *)label_ptr[0] = (uint32_t)(s->code_ptr - label_ptr[0] - 4);
-    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
-        *(uint32_t *)label_ptr[1] = (uint32_t)(s->code_ptr - label_ptr[1] - 4);
-    }
-
-    if (TCG_TARGET_REG_BITS == 32) {
-        int ofs = 0;
-
-        tcg_out_st(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP, ofs);
-        ofs += 4;
-
-        tcg_out_st(s, TCG_TYPE_I32, l->addrlo_reg, TCG_REG_ESP, ofs);
-        ofs += 4;
-
-        if (TARGET_LONG_BITS == 64) {
-            tcg_out_st(s, TCG_TYPE_I32, l->addrhi_reg, TCG_REG_ESP, ofs);
-            ofs += 4;
-        }
-
-        tcg_out_st(s, TCG_TYPE_I32, l->datalo_reg, TCG_REG_ESP, ofs);
-        ofs += 4;
-
-        if (opc == 3) {
-            tcg_out_st(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_ESP, ofs);
-            ofs += 4;
-        }
-
-        tcg_out_sti(s, TCG_TYPE_I32, TCG_REG_ESP, ofs, l->mem_index);
-        ofs += 4;
-
-        retaddr = TCG_REG_EAX;
-        tcg_out_movi(s, TCG_TYPE_I32, retaddr, (uintptr_t)l->raddr);
-        tcg_out_st(s, TCG_TYPE_I32, retaddr, TCG_REG_ESP, ofs);
-    } else {
-        tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
-        /* The second argument is already loaded with addrlo.  */
-        tcg_out_mov(s, (opc == 3 ? TCG_TYPE_I64 : TCG_TYPE_I32),
-                    tcg_target_call_iarg_regs[2], l->datalo_reg);
-        tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[3],
-                     l->mem_index);
-
-        if (ARRAY_SIZE(tcg_target_call_iarg_regs) > 4) {
-            retaddr = tcg_target_call_iarg_regs[4];
-            tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr);
-        } else {
-            retaddr = TCG_REG_RAX;
-            tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr);
-            tcg_out_st(s, TCG_TYPE_PTR, retaddr, TCG_REG_ESP, 0);
-        }
-    }
-
-    /* "Tail call" to the helper, with the return address back inline.  */
-    tcg_out_push(s, retaddr);
-    tcg_out_jmp(s, (uintptr_t)qemu_st_helpers[s_bits]);
-}
-
-/*
- * Generate TB finalization at the end of block
- */
-void tcg_out_tb_finalize(TCGContext *s)
-{
-    int i;
-    TCGLabelQemuLdst *label;
-
-    /* qemu_ld/st slow paths */
-    for (i = 0; i < s->nb_qemu_ldst_labels; i++) {
-        label = (TCGLabelQemuLdst *)&s->qemu_ldst_labels[i];
-        if (label->is_ld) {
-            tcg_out_qemu_ld_slow_path(s, label);
-        } else {
-            tcg_out_qemu_st_slow_path(s, label);
-        }
-    }
-}
-#endif  /* CONFIG_SOFTMMU */
-
 static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
                               const TCGArg *args, const int *const_args)
 {
@@ -1874,40 +1819,18 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_ext16u(s, args[0], args[1]);
         break;
 
-    case INDEX_op_qemu_ld8u:
+    case INDEX_op_qemu_ld_i32:
         tcg_out_qemu_ld(s, args, 0);
         break;
-    case INDEX_op_qemu_ld8s:
-        tcg_out_qemu_ld(s, args, 0 | 4);
-        break;
-    case INDEX_op_qemu_ld16u:
+    case INDEX_op_qemu_ld_i64:
         tcg_out_qemu_ld(s, args, 1);
         break;
-    case INDEX_op_qemu_ld16s:
-        tcg_out_qemu_ld(s, args, 1 | 4);
-        break;
-#if TCG_TARGET_REG_BITS == 64
-    case INDEX_op_qemu_ld32u:
-#endif
-    case INDEX_op_qemu_ld32:
-        tcg_out_qemu_ld(s, args, 2);
-        break;
-    case INDEX_op_qemu_ld64:
-        tcg_out_qemu_ld(s, args, 3);
-        break;
-
-    case INDEX_op_qemu_st8:
+    case INDEX_op_qemu_st_i32:
         tcg_out_qemu_st(s, args, 0);
         break;
-    case INDEX_op_qemu_st16:
+    case INDEX_op_qemu_st_i64:
         tcg_out_qemu_st(s, args, 1);
         break;
-    case INDEX_op_qemu_st32:
-        tcg_out_qemu_st(s, args, 2);
-        break;
-    case INDEX_op_qemu_st64:
-        tcg_out_qemu_st(s, args, 3);
-        break;
 
     OP_32_64(mulu2):
         tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_MUL, args[3]);
@@ -1966,9 +1889,6 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
             tcg_out_st(s, TCG_TYPE_I64, args[0], args[1], args[2]);
         }
         break;
-    case INDEX_op_qemu_ld32s:
-        tcg_out_qemu_ld(s, args, 2 | 4);
-        break;
 
     case INDEX_op_brcond_i64:
         tcg_out_brcond64(s, args[2], args[0], args[1], const_args[1],
@@ -2133,43 +2053,20 @@ static const TCGTargetOpDef x86_op_defs[] = {
 #endif
 
 #if TCG_TARGET_REG_BITS == 64
-    { INDEX_op_qemu_ld8u, { "r", "L" } },
-    { INDEX_op_qemu_ld8s, { "r", "L" } },
-    { INDEX_op_qemu_ld16u, { "r", "L" } },
-    { INDEX_op_qemu_ld16s, { "r", "L" } },
-    { INDEX_op_qemu_ld32, { "r", "L" } },
-    { INDEX_op_qemu_ld32u, { "r", "L" } },
-    { INDEX_op_qemu_ld32s, { "r", "L" } },
-    { INDEX_op_qemu_ld64, { "r", "L" } },
-
-    { INDEX_op_qemu_st8, { "L", "L" } },
-    { INDEX_op_qemu_st16, { "L", "L" } },
-    { INDEX_op_qemu_st32, { "L", "L" } },
-    { INDEX_op_qemu_st64, { "L", "L" } },
+    { INDEX_op_qemu_ld_i32, { "r", "L" } },
+    { INDEX_op_qemu_st_i32, { "L", "L" } },
+    { INDEX_op_qemu_ld_i64, { "r", "L" } },
+    { INDEX_op_qemu_st_i64, { "L", "L" } },
 #elif TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
-    { INDEX_op_qemu_ld8u, { "r", "L" } },
-    { INDEX_op_qemu_ld8s, { "r", "L" } },
-    { INDEX_op_qemu_ld16u, { "r", "L" } },
-    { INDEX_op_qemu_ld16s, { "r", "L" } },
-    { INDEX_op_qemu_ld32, { "r", "L" } },
-    { INDEX_op_qemu_ld64, { "r", "r", "L" } },
-
-    { INDEX_op_qemu_st8, { "cb", "L" } },
-    { INDEX_op_qemu_st16, { "L", "L" } },
-    { INDEX_op_qemu_st32, { "L", "L" } },
-    { INDEX_op_qemu_st64, { "L", "L", "L" } },
+    { INDEX_op_qemu_ld_i32, { "r", "L" } },
+    { INDEX_op_qemu_st_i32, { "L", "L" } },
+    { INDEX_op_qemu_ld_i64, { "r", "r", "L" } },
+    { INDEX_op_qemu_st_i64, { "L", "L", "L" } },
 #else
-    { INDEX_op_qemu_ld8u, { "r", "L", "L" } },
-    { INDEX_op_qemu_ld8s, { "r", "L", "L" } },
-    { INDEX_op_qemu_ld16u, { "r", "L", "L" } },
-    { INDEX_op_qemu_ld16s, { "r", "L", "L" } },
-    { INDEX_op_qemu_ld32, { "r", "L", "L" } },
-    { INDEX_op_qemu_ld64, { "r", "r", "L", "L" } },
-
-    { INDEX_op_qemu_st8, { "cb", "L", "L" } },
-    { INDEX_op_qemu_st16, { "L", "L", "L" } },
-    { INDEX_op_qemu_st32, { "L", "L", "L" } },
-    { INDEX_op_qemu_st64, { "L", "L", "L", "L" } },
+    { INDEX_op_qemu_ld_i32, { "r", "L", "L" } },
+    { INDEX_op_qemu_st_i32, { "L", "L", "L" } },
+    { INDEX_op_qemu_ld_i64, { "r", "r", "L", "L" } },
+    { INDEX_op_qemu_st_i64, { "L", "L", "L", "L" } },
 #endif
     { -1 },
 };
diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h
index d32d7ef6f0..92c0fcd36d 100644
--- a/tcg/i386/tcg-target.h
+++ b/tcg/i386/tcg-target.h
@@ -130,6 +130,8 @@ typedef enum {
 #define TCG_TARGET_HAS_mulsh_i64        0
 #endif
 
+#define TCG_TARGET_HAS_new_ldst         1
+
 #define TCG_TARGET_deposit_i32_valid(ofs, len) \
     (((ofs) == 0 && (len) == 8) || ((ofs) == 8 && (len) == 8) || \
      ((ofs) == 0 && (len) == 16))
diff --git a/tcg/ia64/tcg-target.c b/tcg/ia64/tcg-target.c
index cd4f1ae1db..0656d3907a 100644
--- a/tcg/ia64/tcg-target.c
+++ b/tcg/ia64/tcg-target.c
@@ -23,6 +23,8 @@
  * THE SOFTWARE.
  */
 
+#include "tcg-be-null.h"
+
 /*
  * Register definitions
  */
diff --git a/tcg/ia64/tcg-target.h b/tcg/ia64/tcg-target.h
index 4330c9cdd3..c90038aae5 100644
--- a/tcg/ia64/tcg-target.h
+++ b/tcg/ia64/tcg-target.h
@@ -151,6 +151,8 @@ typedef enum {
 #define TCG_TARGET_HAS_mulsh_i32        0
 #define TCG_TARGET_HAS_mulsh_i64        0
 
+#define TCG_TARGET_HAS_new_ldst         0
+
 #define TCG_TARGET_deposit_i32_valid(ofs, len) ((len) <= 16)
 #define TCG_TARGET_deposit_i64_valid(ofs, len) ((len) <= 16)
 
diff --git a/tcg/mips/tcg-target.c b/tcg/mips/tcg-target.c
index 5f0a65b4ea..40551cdcb5 100644
--- a/tcg/mips/tcg-target.c
+++ b/tcg/mips/tcg-target.c
@@ -24,6 +24,8 @@
  * THE SOFTWARE.
  */
 
+#include "tcg-be-null.h"
+
 #if defined(TCG_TARGET_WORDS_BIGENDIAN) == defined(TARGET_WORDS_BIGENDIAN)
 # define TCG_NEED_BSWAP 0
 #else
diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h
index c37252269f..683c6af8b9 100644
--- a/tcg/mips/tcg-target.h
+++ b/tcg/mips/tcg-target.h
@@ -122,6 +122,8 @@ extern bool use_mips32r2_instructions;
 #define TCG_TARGET_HAS_ext16s_i32       use_mips32r2_instructions
 #define TCG_TARGET_HAS_rot_i32          use_mips32r2_instructions
 
+#define TCG_TARGET_HAS_new_ldst         0
+
 /* optional instructions automatically implemented */
 #define TCG_TARGET_HAS_neg_i32          0 /* sub  rd, zero, rt   */
 #define TCG_TARGET_HAS_ext8u_i32        0 /* andi rt, rs, 0xff   */
diff --git a/tcg/optimize.c b/tcg/optimize.c
index b29bf25b67..89e2d6a3b3 100644
--- a/tcg/optimize.c
+++ b/tcg/optimize.c
@@ -238,20 +238,16 @@ static TCGArg do_constant_folding_2(TCGOpcode op, TCGArg x, TCGArg y)
         return (int64_t)x >> (int64_t)y;
 
     case INDEX_op_rotr_i32:
-        x = ((uint32_t)x << (32 - y)) | ((uint32_t)x >> y);
-        return x;
+        return ror32(x, y);
 
     case INDEX_op_rotr_i64:
-        x = ((uint64_t)x << (64 - y)) | ((uint64_t)x >> y);
-        return x;
+        return ror64(x, y);
 
     case INDEX_op_rotl_i32:
-        x = ((uint32_t)x << y) | ((uint32_t)x >> (32 - y));
-        return x;
+        return rol32(x, y);
 
     case INDEX_op_rotl_i64:
-        x = ((uint64_t)x << y) | ((uint64_t)x >> (64 - y));
-        return x;
+        return rol64(x, y);
 
     CASE_OP_32_64(not):
         return ~x;
diff --git a/tcg/ppc/tcg-target.c b/tcg/ppc/tcg-target.c
index 25955563b8..dc2c2df890 100644
--- a/tcg/ppc/tcg-target.c
+++ b/tcg/ppc/tcg-target.c
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "tcg-be-ldst.h"
+
 static uint8_t *tb_ret_addr;
 
 #if defined _CALL_DARWIN || defined __APPLE__
@@ -450,7 +452,9 @@ static const uint32_t tcg_to_bc[] = {
 
 static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
 {
-    tcg_out32 (s, OR | SAB (arg, ret, arg));
+    if (ret != arg) {
+        tcg_out32(s, OR | SAB(arg, ret, arg));
+    }
 }
 
 static void tcg_out_movi(TCGContext *s, TCGType type,
@@ -490,7 +494,8 @@ static void tcg_out_b (TCGContext *s, int mask, tcg_target_long target)
     }
 }
 
-static void tcg_out_call (TCGContext *s, tcg_target_long arg, int const_arg)
+static void tcg_out_call (TCGContext *s, tcg_target_long arg, int const_arg,
+                          int lk)
 {
 #ifdef _CALL_AIX
     int reg;
@@ -504,14 +509,14 @@ static void tcg_out_call (TCGContext *s, tcg_target_long arg, int const_arg)
     tcg_out32 (s, LWZ | RT (0) | RA (reg));
     tcg_out32 (s, MTSPR | RA (0) | CTR);
     tcg_out32 (s, LWZ | RT (2) | RA (reg) | 4);
-    tcg_out32 (s, BCCTR | BO_ALWAYS | LK);
+    tcg_out32 (s, BCCTR | BO_ALWAYS | lk);
 #else
     if (const_arg) {
-        tcg_out_b (s, LK, arg);
+        tcg_out_b (s, lk, arg);
     }
     else {
         tcg_out32 (s, MTSPR | RS (arg) | LR);
-        tcg_out32 (s, BCLR | BO_ALWAYS | LK);
+        tcg_out32 (s, BCLR | BO_ALWAYS | lk);
     }
 #endif
 }
@@ -520,7 +525,7 @@ static void tcg_out_call (TCGContext *s, tcg_target_long arg, int const_arg)
 
 static void add_qemu_ldst_label (TCGContext *s,
                                  int is_ld,
-                                 int opc,
+                                 TCGMemOp opc,
                                  int data_reg,
                                  int data_reg2,
                                  int addrlo_reg,
@@ -529,15 +534,8 @@ static void add_qemu_ldst_label (TCGContext *s,
                                  uint8_t *raddr,
                                  uint8_t *label_ptr)
 {
-    int idx;
-    TCGLabelQemuLdst *label;
-
-    if (s->nb_qemu_ldst_labels >= TCG_MAX_QEMU_LDST) {
-        tcg_abort();
-    }
+    TCGLabelQemuLdst *label = new_ldst_label(s);
 
-    idx = s->nb_qemu_ldst_labels++;
-    label = (TCGLabelQemuLdst *)&s->qemu_ldst_labels[idx];
     label->is_ld = is_ld;
     label->opc = opc;
     label->datalo_reg = data_reg;
@@ -549,442 +547,347 @@ static void add_qemu_ldst_label (TCGContext *s,
     label->label_ptr[0] = label_ptr;
 }
 
-/* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
-   int mmu_idx) */
-static const void * const qemu_ld_helpers[4] = {
-    helper_ldb_mmu,
-    helper_ldw_mmu,
-    helper_ldl_mmu,
-    helper_ldq_mmu,
+/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
+ *                                     int mmu_idx, uintptr_t ra)
+ */
+static const void * const qemu_ld_helpers[16] = {
+    [MO_UB]   = helper_ret_ldub_mmu,
+    [MO_LEUW] = helper_le_lduw_mmu,
+    [MO_LEUL] = helper_le_ldul_mmu,
+    [MO_LEQ]  = helper_le_ldq_mmu,
+    [MO_BEUW] = helper_be_lduw_mmu,
+    [MO_BEUL] = helper_be_ldul_mmu,
+    [MO_BEQ]  = helper_be_ldq_mmu,
 };
 
-/* helper signature: helper_st_mmu(CPUState *env, target_ulong addr,
-   uintxx_t val, int mmu_idx) */
-static const void * const qemu_st_helpers[4] = {
-    helper_stb_mmu,
-    helper_stw_mmu,
-    helper_stl_mmu,
-    helper_stq_mmu,
+/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,
+ *                                     uintxx_t val, int mmu_idx, uintptr_t ra)
+ */
+static const void * const qemu_st_helpers[16] = {
+    [MO_UB]   = helper_ret_stb_mmu,
+    [MO_LEUW] = helper_le_stw_mmu,
+    [MO_LEUL] = helper_le_stl_mmu,
+    [MO_LEQ]  = helper_le_stq_mmu,
+    [MO_BEUW] = helper_be_stw_mmu,
+    [MO_BEUL] = helper_be_stl_mmu,
+    [MO_BEQ]  = helper_be_stq_mmu,
 };
 
-static void *ld_trampolines[4];
-static void *st_trampolines[4];
+static void *ld_trampolines[16];
+static void *st_trampolines[16];
+
+/* Perform the TLB load and compare.  Branches to the slow path, placing the
+   address of the branch in *LABEL_PTR.  Loads the addend of the TLB into R0.
+   Clobbers R1 and R2.  */
 
-static void tcg_out_tlb_check (TCGContext *s, int r0, int r1, int r2,
-                               int addr_reg, int addr_reg2, int s_bits,
-                               int offset1, int offset2, uint8_t **label_ptr)
+static void tcg_out_tlb_check(TCGContext *s, TCGReg r0, TCGReg r1, TCGReg r2,
+                              TCGReg addrlo, TCGReg addrhi, TCGMemOp s_bits,
+                              int mem_index, int is_load, uint8_t **label_ptr)
 {
+    int cmp_off =
+        (is_load
+         ? offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)
+         : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write));
+    int add_off = offsetof(CPUArchState, tlb_table[mem_index][0].addend);
     uint16_t retranst;
+    TCGReg base = TCG_AREG0;
+
+    /* Extract the page index, shifted into place for tlb index.  */
+    tcg_out32(s, (RLWINM
+                  | RA(r0)
+                  | RS(addrlo)
+                  | SH(32 - (TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS))
+                  | MB(32 - (CPU_TLB_BITS + CPU_TLB_ENTRY_BITS))
+                  | ME(31 - CPU_TLB_ENTRY_BITS)));
+
+    /* Compensate for very large offsets.  */
+    if (add_off >= 0x8000) {
+        /* Most target env are smaller than 32k; none are larger than 64k.
+           Simplify the logic here merely to offset by 0x7ff0, giving us a
+           range just shy of 64k.  Check this assumption.  */
+        QEMU_BUILD_BUG_ON(offsetof(CPUArchState,
+                                   tlb_table[NB_MMU_MODES - 1][1])
+                          > 0x7ff0 + 0x7fff);
+        tcg_out32(s, ADDI | RT(r1) | RA(base) | 0x7ff0);
+        base = r1;
+        cmp_off -= 0x7ff0;
+        add_off -= 0x7ff0;
+    }
 
-    tcg_out32 (s, (RLWINM
-                   | RA (r0)
-                   | RS (addr_reg)
-                   | SH (32 - (TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS))
-                   | MB (32 - (CPU_TLB_BITS + CPU_TLB_ENTRY_BITS))
-                   | ME (31 - CPU_TLB_ENTRY_BITS)
-                   )
-        );
-    tcg_out32 (s, ADD | RT (r0) | RA (r0) | RB (TCG_AREG0));
-    tcg_out32 (s, (LWZU
-                   | RT (r1)
-                   | RA (r0)
-                   | offset1
-                   )
-        );
-    tcg_out32 (s, (RLWINM
-                   | RA (r2)
-                   | RS (addr_reg)
-                   | SH (0)
-                   | MB ((32 - s_bits) & 31)
-                   | ME (31 - TARGET_PAGE_BITS)
-                   )
-        );
+    /* Clear the non-page, non-alignment bits from the address.  */
+    tcg_out32(s, (RLWINM
+                  | RA(r2)
+                  | RS(addrlo)
+                  | SH(0)
+                  | MB((32 - s_bits) & 31)
+                  | ME(31 - TARGET_PAGE_BITS)));
 
-    tcg_out32 (s, CMP | BF (7) | RA (r2) | RB (r1));
-#if TARGET_LONG_BITS == 64
-    tcg_out32 (s, LWZ | RT (r1) | RA (r0) | 4);
-    tcg_out32 (s, CMP | BF (6) | RA (addr_reg2) | RB (r1));
-    tcg_out32 (s, CRAND | BT (7, CR_EQ) | BA (6, CR_EQ) | BB (7, CR_EQ));
-#endif
-    *label_ptr = s->code_ptr;
-    retranst = ((uint16_t *) s->code_ptr)[1] & ~3;
-    tcg_out32 (s, BC | BI (7, CR_EQ) | retranst | BO_COND_FALSE);
+    tcg_out32(s, ADD | RT(r0) | RA(r0) | RB(base));
+    base = r0;
 
-    /* r0 now contains &env->tlb_table[mem_index][index].addr_x */
-    tcg_out32 (s, (LWZ
-                   | RT (r0)
-                   | RA (r0)
-                   | offset2
-                   )
-        );
-    /* r0 = env->tlb_table[mem_index][index].addend */
-    tcg_out32 (s, ADD | RT (r0) | RA (r0) | RB (addr_reg));
-    /* r0 = env->tlb_table[mem_index][index].addend + addr */
+    /* Load the tlb comparator.  */
+    tcg_out32(s, LWZ | RT(r1) | RA(base) | (cmp_off & 0xffff));
+
+    tcg_out32(s, CMP | BF(7) | RA(r2) | RB(r1));
+
+    if (TARGET_LONG_BITS == 64) {
+        tcg_out32(s, LWZ | RT(r1) | RA(base) | ((cmp_off + 4) & 0xffff));
+    }
+
+    /* Load the tlb addend for use on the fast path.
+       Do this asap to minimize load delay.  */
+    tcg_out32(s, LWZ | RT(r0) | RA(base) | (add_off & 0xffff));
+
+    if (TARGET_LONG_BITS == 64) {
+        tcg_out32(s, CMP | BF(6) | RA(addrhi) | RB(r1));
+        tcg_out32(s, CRAND | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, CR_EQ));
+    }
 
+    /* Use a conditional branch-and-link so that we load a pointer to
+       somewhere within the current opcode, for passing on to the helper.
+       This address cannot be used for a tail call, but it's shorter
+       than forming an address from scratch.  */
+    *label_ptr = s->code_ptr;
+    retranst = ((uint16_t *) s->code_ptr)[1] & ~3;
+    tcg_out32(s, BC | BI(7, CR_EQ) | retranst | BO_COND_FALSE | LK);
 }
 #endif
 
-static void tcg_out_qemu_ld (TCGContext *s, const TCGArg *args, int opc)
+static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
 {
-    int addr_reg, data_reg, data_reg2, r0, r1, rbase, bswap;
+    TCGReg addrlo, datalo, datahi, rbase, addrhi __attribute__((unused));
+    TCGMemOp opc, bswap;
 #ifdef CONFIG_SOFTMMU
-    int mem_index, s_bits, r2, addr_reg2;
+    int mem_index;
     uint8_t *label_ptr;
 #endif
 
-    data_reg = *args++;
-    if (opc == 3)
-        data_reg2 = *args++;
-    else
-        data_reg2 = 0;
-    addr_reg = *args++;
+    datalo = *args++;
+    datahi = (is64 ? *args++ : 0);
+    addrlo = *args++;
+    addrhi = (TARGET_LONG_BITS == 64 ? *args++ : 0);
+    opc = *args++;
+    bswap = opc & MO_BSWAP;
 
 #ifdef CONFIG_SOFTMMU
-#if TARGET_LONG_BITS == 64
-    addr_reg2 = *args++;
-#else
-    addr_reg2 = 0;
-#endif
     mem_index = *args;
-    s_bits = opc & 3;
-    r0 = 3;
-    r1 = 4;
-    r2 = 0;
-    rbase = 0;
-
-    tcg_out_tlb_check (
-        s, r0, r1, r2, addr_reg, addr_reg2, s_bits,
-        offsetof (CPUArchState, tlb_table[mem_index][0].addr_read),
-        offsetof (CPUTLBEntry, addend) - offsetof (CPUTLBEntry, addr_read),
-        &label_ptr
-        );
+    tcg_out_tlb_check(s, TCG_REG_R3, TCG_REG_R4, TCG_REG_R0, addrlo,
+                      addrhi, opc & MO_SIZE, mem_index, 0, &label_ptr);
+    rbase = TCG_REG_R3;
 #else  /* !CONFIG_SOFTMMU */
-    r0 = addr_reg;
-    r1 = 3;
     rbase = GUEST_BASE ? TCG_GUEST_BASE_REG : 0;
 #endif
 
-#ifdef TARGET_WORDS_BIGENDIAN
-    bswap = 0;
-#else
-    bswap = 1;
-#endif
-
-    switch (opc) {
+    switch (opc & MO_SSIZE) {
     default:
-    case 0:
-        tcg_out32 (s, LBZX | TAB (data_reg, rbase, r0));
+    case MO_UB:
+        tcg_out32(s, LBZX | TAB(datalo, rbase, addrlo));
         break;
-    case 0|4:
-        tcg_out32 (s, LBZX | TAB (data_reg, rbase, r0));
-        tcg_out32 (s, EXTSB | RA (data_reg) | RS (data_reg));
+    case MO_SB:
+        tcg_out32(s, LBZX | TAB(datalo, rbase, addrlo));
+        tcg_out32(s, EXTSB | RA(datalo) | RS(datalo));
         break;
-    case 1:
-        if (bswap)
-            tcg_out32 (s, LHBRX | TAB (data_reg, rbase, r0));
-        else
-            tcg_out32 (s, LHZX | TAB (data_reg, rbase, r0));
+    case MO_UW:
+        tcg_out32(s, (bswap ? LHBRX : LHZX) | TAB(datalo, rbase, addrlo));
         break;
-    case 1|4:
+    case MO_SW:
         if (bswap) {
-            tcg_out32 (s, LHBRX | TAB (data_reg, rbase, r0));
-            tcg_out32 (s, EXTSH | RA (data_reg) | RS (data_reg));
+            tcg_out32(s, LHBRX | TAB(datalo, rbase, addrlo));
+            tcg_out32(s, EXTSH | RA(datalo) | RS(datalo));
+        } else {
+            tcg_out32(s, LHAX | TAB(datalo, rbase, addrlo));
         }
-        else tcg_out32 (s, LHAX | TAB (data_reg, rbase, r0));
         break;
-    case 2:
-        if (bswap)
-            tcg_out32 (s, LWBRX | TAB (data_reg, rbase, r0));
-        else
-            tcg_out32 (s, LWZX | TAB (data_reg, rbase, r0));
+    case MO_UL:
+        tcg_out32(s, (bswap ? LWBRX : LWZX) | TAB(datalo, rbase, addrlo));
         break;
-    case 3:
+    case MO_Q:
         if (bswap) {
-            tcg_out32 (s, ADDI | RT (r1) | RA (r0) | 4);
-            tcg_out32 (s, LWBRX | TAB (data_reg, rbase, r0));
-            tcg_out32 (s, LWBRX | TAB (data_reg2, rbase, r1));
-        }
-        else {
-#ifdef CONFIG_USE_GUEST_BASE
-            tcg_out32 (s, ADDI | RT (r1) | RA (r0) | 4);
-            tcg_out32 (s, LWZX | TAB (data_reg2, rbase, r0));
-            tcg_out32 (s, LWZX | TAB (data_reg, rbase, r1));
-#else
-            if (r0 == data_reg2) {
-                tcg_out32 (s, LWZ | RT (0) | RA (r0));
-                tcg_out32 (s, LWZ | RT (data_reg) | RA (r0) | 4);
-                tcg_out_mov (s, TCG_TYPE_I32, data_reg2, 0);
-            }
-            else {
-                tcg_out32 (s, LWZ | RT (data_reg2) | RA (r0));
-                tcg_out32 (s, LWZ | RT (data_reg) | RA (r0) | 4);
-            }
-#endif
+            tcg_out32(s, ADDI | RT(TCG_REG_R0) | RA(addrlo) | 4);
+            tcg_out32(s, LWBRX | TAB(datalo, rbase, addrlo));
+            tcg_out32(s, LWBRX | TAB(datahi, rbase, TCG_REG_R0));
+        } else if (rbase != 0) {
+            tcg_out32(s, ADDI | RT(TCG_REG_R0) | RA(addrlo) | 4);
+            tcg_out32(s, LWZX | TAB(datahi, rbase, addrlo));
+            tcg_out32(s, LWZX | TAB(datalo, rbase, TCG_REG_R0));
+        } else if (addrlo == datahi) {
+            tcg_out32(s, LWZ | RT(datalo) | RA(addrlo) | 4);
+            tcg_out32(s, LWZ | RT(datahi) | RA(addrlo));
+        } else {
+            tcg_out32(s, LWZ | RT(datahi) | RA(addrlo));
+            tcg_out32(s, LWZ | RT(datalo) | RA(addrlo) | 4);
         }
         break;
     }
 #ifdef CONFIG_SOFTMMU
-    add_qemu_ldst_label (s,
-                         1,
-                         opc,
-                         data_reg,
-                         data_reg2,
-                         addr_reg,
-                         addr_reg2,
-                         mem_index,
-                         s->code_ptr,
-                         label_ptr);
+    add_qemu_ldst_label(s, 1, opc, datalo, datahi, addrlo,
+                        addrhi, mem_index, s->code_ptr, label_ptr);
 #endif
 }
 
-static void tcg_out_qemu_st (TCGContext *s, const TCGArg *args, int opc)
+static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
 {
-    int addr_reg, r0, r1, data_reg, data_reg2, bswap, rbase;
+    TCGReg addrlo, datalo, datahi, rbase, addrhi __attribute__((unused));
+    TCGMemOp opc, bswap, s_bits;
 #ifdef CONFIG_SOFTMMU
-    int mem_index, r2, addr_reg2;
+    int mem_index;
     uint8_t *label_ptr;
 #endif
 
-    data_reg = *args++;
-    if (opc == 3)
-        data_reg2 = *args++;
-    else
-        data_reg2 = 0;
-    addr_reg = *args++;
+    datalo = *args++;
+    datahi = (is64 ? *args++ : 0);
+    addrlo = *args++;
+    addrhi = (TARGET_LONG_BITS == 64 ? *args++ : 0);
+    opc = *args++;
+    bswap = opc & MO_BSWAP;
+    s_bits = opc & MO_SIZE;
 
 #ifdef CONFIG_SOFTMMU
-#if TARGET_LONG_BITS == 64
-    addr_reg2 = *args++;
-#else
-    addr_reg2 = 0;
-#endif
     mem_index = *args;
-    r0 = 3;
-    r1 = 4;
-    r2 = 0;
-    rbase = 0;
-
-    tcg_out_tlb_check (
-        s, r0, r1, r2, addr_reg, addr_reg2, opc & 3,
-        offsetof (CPUArchState, tlb_table[mem_index][0].addr_write),
-        offsetof (CPUTLBEntry, addend) - offsetof (CPUTLBEntry, addr_write),
-        &label_ptr
-        );
+    tcg_out_tlb_check(s, TCG_REG_R3, TCG_REG_R4, TCG_REG_R0, addrlo,
+                      addrhi, s_bits, mem_index, 0, &label_ptr);
+    rbase = TCG_REG_R3;
 #else  /* !CONFIG_SOFTMMU */
-    r0 = addr_reg;
-    r1 = 3;
     rbase = GUEST_BASE ? TCG_GUEST_BASE_REG : 0;
 #endif
 
-#ifdef TARGET_WORDS_BIGENDIAN
-    bswap = 0;
-#else
-    bswap = 1;
-#endif
-    switch (opc) {
-    case 0:
-        tcg_out32 (s, STBX | SAB (data_reg, rbase, r0));
+    switch (s_bits) {
+    case MO_8:
+        tcg_out32(s, STBX | SAB(datalo, rbase, addrlo));
         break;
-    case 1:
-        if (bswap)
-            tcg_out32 (s, STHBRX | SAB (data_reg, rbase, r0));
-        else
-            tcg_out32 (s, STHX | SAB (data_reg, rbase, r0));
+    case MO_16:
+        tcg_out32(s, (bswap ? STHBRX : STHX) | SAB(datalo, rbase, addrlo));
         break;
-    case 2:
-        if (bswap)
-            tcg_out32 (s, STWBRX | SAB (data_reg, rbase, r0));
-        else
-            tcg_out32 (s, STWX | SAB (data_reg, rbase, r0));
+    case MO_32:
+    default:
+        tcg_out32(s, (bswap ? STWBRX : STWX) | SAB(datalo, rbase, addrlo));
         break;
-    case 3:
+    case MO_64:
         if (bswap) {
-            tcg_out32 (s, ADDI | RT (r1) | RA (r0) | 4);
-            tcg_out32 (s, STWBRX | SAB (data_reg,  rbase, r0));
-            tcg_out32 (s, STWBRX | SAB (data_reg2, rbase, r1));
-        }
-        else {
-#ifdef CONFIG_USE_GUEST_BASE
-            tcg_out32 (s, STWX | SAB (data_reg2, rbase, r0));
-            tcg_out32 (s, ADDI | RT (r1) | RA (r0) | 4);
-            tcg_out32 (s, STWX | SAB (data_reg,  rbase, r1));
-#else
-            tcg_out32 (s, STW | RS (data_reg2) | RA (r0));
-            tcg_out32 (s, STW | RS (data_reg) | RA (r0) | 4);
-#endif
+            tcg_out32(s, ADDI | RT(TCG_REG_R0) | RA(addrlo) | 4);
+            tcg_out32(s, STWBRX | SAB(datalo, rbase, addrlo));
+            tcg_out32(s, STWBRX | SAB(datahi, rbase, TCG_REG_R0));
+        } else if (rbase != 0) {
+            tcg_out32(s, ADDI | RT(TCG_REG_R0) | RA(addrlo) | 4);
+            tcg_out32(s, STWX | SAB(datahi, rbase, addrlo));
+            tcg_out32(s, STWX | SAB(datalo, rbase, TCG_REG_R0));
+        } else {
+            tcg_out32(s, STW | RS(datahi) | RA(addrlo));
+            tcg_out32(s, STW | RS(datalo) | RA(addrlo) | 4);
         }
         break;
     }
 
 #ifdef CONFIG_SOFTMMU
-    add_qemu_ldst_label (s,
-                         0,
-                         opc,
-                         data_reg,
-                         data_reg2,
-                         addr_reg,
-                         addr_reg2,
-                         mem_index,
-                         s->code_ptr,
-                         label_ptr);
+    add_qemu_ldst_label(s, 0, opc, datalo, datahi, addrlo, addrhi,
+                        mem_index, s->code_ptr, label_ptr);
 #endif
 }
 
 #if defined(CONFIG_SOFTMMU)
-static void tcg_out_qemu_ld_slow_path (TCGContext *s, TCGLabelQemuLdst *label)
+static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
 {
-    int s_bits;
-    int ir;
-    int opc = label->opc;
-    int mem_index = label->mem_index;
-    int data_reg = label->datalo_reg;
-    int data_reg2 = label->datahi_reg;
-    int addr_reg = label->addrlo_reg;
-    uint8_t *raddr = label->raddr;
-    uint8_t **label_ptr = &label->label_ptr[0];
-
-    s_bits = opc & 3;
-
-    /* resolve label address */
-    reloc_pc14 (label_ptr[0], (tcg_target_long) s->code_ptr);
-
-    /* slow path */
-    ir = 4;
-#if TARGET_LONG_BITS == 32
-    tcg_out_mov (s, TCG_TYPE_I32, ir++, addr_reg);
-#else
+    TCGReg ir, datalo, datahi;
+    TCGMemOp opc = l->opc;
+
+    reloc_pc14 (l->label_ptr[0], (uintptr_t)s->code_ptr);
+
+    ir = TCG_REG_R4;
+    if (TARGET_LONG_BITS == 32) {
+        tcg_out_mov(s, TCG_TYPE_I32, ir++, l->addrlo_reg);
+    } else {
 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
-    ir |= 1;
-#endif
-    tcg_out_mov (s, TCG_TYPE_I32, ir++, label->addrhi_reg);
-    tcg_out_mov (s, TCG_TYPE_I32, ir++, addr_reg);
+        ir |= 1;
 #endif
-    tcg_out_movi (s, TCG_TYPE_I32, ir, mem_index);
-    tcg_out_call (s, (tcg_target_long) ld_trampolines[s_bits], 1);
-    tcg_out32 (s, (tcg_target_long) raddr);
-    switch (opc) {
-    case 0|4:
-        tcg_out32 (s, EXTSB | RA (data_reg) | RS (3));
-        break;
-    case 1|4:
-        tcg_out32 (s, EXTSH | RA (data_reg) | RS (3));
-        break;
-    case 0:
-    case 1:
-    case 2:
-        if (data_reg != 3)
-            tcg_out_mov (s, TCG_TYPE_I32, data_reg, 3);
-        break;
-    case 3:
-        if (data_reg == 3) {
-            if (data_reg2 == 4) {
-                tcg_out_mov (s, TCG_TYPE_I32, 0, 4);
-                tcg_out_mov (s, TCG_TYPE_I32, 4, 3);
-                tcg_out_mov (s, TCG_TYPE_I32, 3, 0);
-            }
-            else {
-                tcg_out_mov (s, TCG_TYPE_I32, data_reg2, 3);
-                tcg_out_mov (s, TCG_TYPE_I32, 3, 4);
-            }
-        }
-        else {
-            if (data_reg != 4) tcg_out_mov (s, TCG_TYPE_I32, data_reg, 4);
-            if (data_reg2 != 3) tcg_out_mov (s, TCG_TYPE_I32, data_reg2, 3);
+        tcg_out_mov(s, TCG_TYPE_I32, ir++, l->addrhi_reg);
+        tcg_out_mov(s, TCG_TYPE_I32, ir++, l->addrlo_reg);
+    }
+    tcg_out_movi(s, TCG_TYPE_I32, ir++, l->mem_index);
+    tcg_out32(s, MFSPR | RT(ir++) | LR);
+    tcg_out_b(s, LK, (uintptr_t)ld_trampolines[opc & ~MO_SIGN]);
+
+    datalo = l->datalo_reg;
+    switch (opc & MO_SSIZE) {
+    case MO_SB:
+        tcg_out32(s, EXTSB | RA(datalo) | RS(TCG_REG_R3));
+        break;
+    case MO_SW:
+        tcg_out32(s, EXTSH | RA(datalo) | RS(TCG_REG_R3));
+        break;
+    default:
+        tcg_out_mov(s, TCG_TYPE_I32, datalo, TCG_REG_R3);
+        break;
+    case MO_Q:
+        datahi = l->datahi_reg;
+        if (datalo != TCG_REG_R3) {
+            tcg_out_mov(s, TCG_TYPE_I32, datalo, TCG_REG_R4);
+            tcg_out_mov(s, TCG_TYPE_I32, datahi, TCG_REG_R3);
+        } else if (datahi != TCG_REG_R4) {
+            tcg_out_mov(s, TCG_TYPE_I32, datahi, TCG_REG_R3);
+            tcg_out_mov(s, TCG_TYPE_I32, datalo, TCG_REG_R4);
+        } else {
+            tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_R0, TCG_REG_R4);
+            tcg_out_mov(s, TCG_TYPE_I32, datahi, TCG_REG_R3);
+            tcg_out_mov(s, TCG_TYPE_I32, datalo, TCG_REG_R0);
         }
         break;
     }
-    /* Jump to the code corresponding to next IR of qemu_st */
-    tcg_out_b (s, 0, (tcg_target_long) raddr);
+    tcg_out_b (s, 0, (uintptr_t)l->raddr);
 }
 
-static void tcg_out_qemu_st_slow_path (TCGContext *s, TCGLabelQemuLdst *label)
+static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
 {
-    int ir;
-    int opc = label->opc;
-    int mem_index = label->mem_index;
-    int data_reg = label->datalo_reg;
-    int data_reg2 = label->datahi_reg;
-    int addr_reg = label->addrlo_reg;
-    uint8_t *raddr = label->raddr;
-    uint8_t **label_ptr = &label->label_ptr[0];
-
-    /* resolve label address */
-    reloc_pc14 (label_ptr[0], (tcg_target_long) s->code_ptr);
-
-    /* slow path */
-    ir = 4;
-#if TARGET_LONG_BITS == 32
-    tcg_out_mov (s, TCG_TYPE_I32, ir++, addr_reg);
-#else
+    TCGReg ir, datalo;
+    TCGMemOp opc = l->opc;
+
+    reloc_pc14 (l->label_ptr[0], (tcg_target_long) s->code_ptr);
+
+    ir = TCG_REG_R4;
+    if (TARGET_LONG_BITS == 32) {
+        tcg_out_mov (s, TCG_TYPE_I32, ir++, l->addrlo_reg);
+    } else {
 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
-    ir |= 1;
-#endif
-    tcg_out_mov (s, TCG_TYPE_I32, ir++, label->addrhi_reg);
-    tcg_out_mov (s, TCG_TYPE_I32, ir++, addr_reg);
+        ir |= 1;
 #endif
+        tcg_out_mov (s, TCG_TYPE_I32, ir++, l->addrhi_reg);
+        tcg_out_mov (s, TCG_TYPE_I32, ir++, l->addrlo_reg);
+    }
 
-    switch (opc) {
-    case 0:
-        tcg_out32 (s, (RLWINM
-                       | RA (ir)
-                       | RS (data_reg)
-                       | SH (0)
-                       | MB (24)
-                       | ME (31)));
+    datalo = l->datalo_reg;
+    switch (opc & MO_SIZE) {
+    case MO_8:
+        tcg_out32(s, (RLWINM | RA (ir) | RS (datalo)
+                      | SH (0) | MB (24) | ME (31)));
         break;
-    case 1:
-        tcg_out32 (s, (RLWINM
-                       | RA (ir)
-                       | RS (data_reg)
-                       | SH (0)
-                       | MB (16)
-                       | ME (31)));
+    case MO_16:
+        tcg_out32(s, (RLWINM | RA (ir) | RS (datalo)
+                      | SH (0) | MB (16) | ME (31)));
         break;
-    case 2:
-        tcg_out_mov (s, TCG_TYPE_I32, ir, data_reg);
+    default:
+        tcg_out_mov(s, TCG_TYPE_I32, ir, datalo);
         break;
-    case 3:
+    case MO_64:
 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
         ir |= 1;
 #endif
-        tcg_out_mov (s, TCG_TYPE_I32, ir++, data_reg2);
-        tcg_out_mov (s, TCG_TYPE_I32, ir, data_reg);
+        tcg_out_mov(s, TCG_TYPE_I32, ir++, l->datahi_reg);
+        tcg_out_mov(s, TCG_TYPE_I32, ir, datalo);
         break;
     }
     ir++;
 
-    tcg_out_movi (s, TCG_TYPE_I32, ir, mem_index);
-    tcg_out_call (s, (tcg_target_long) st_trampolines[opc], 1);
-    tcg_out32 (s, (tcg_target_long) raddr);
-    tcg_out_b (s, 0, (tcg_target_long) raddr);
-}
-
-void tcg_out_tb_finalize(TCGContext *s)
-{
-    int i;
-    TCGLabelQemuLdst *label;
-
-    /* qemu_ld/st slow paths */
-    for (i = 0; i < s->nb_qemu_ldst_labels; i++) {
-        label = (TCGLabelQemuLdst *) &s->qemu_ldst_labels[i];
-        if (label->is_ld) {
-            tcg_out_qemu_ld_slow_path (s, label);
-        }
-        else {
-            tcg_out_qemu_st_slow_path (s, label);
-        }
-    }
+    tcg_out_movi(s, TCG_TYPE_I32, ir++, l->mem_index);
+    tcg_out32(s, MFSPR | RT(ir++) | LR);
+    tcg_out_b(s, LK, (uintptr_t)st_trampolines[opc]);
+    tcg_out_b(s, 0, (uintptr_t)l->raddr);
 }
 #endif
 
 #ifdef CONFIG_SOFTMMU
 static void emit_ldst_trampoline (TCGContext *s, const void *ptr)
 {
-    tcg_out32 (s, MFSPR | RT (3) | LR);
-    tcg_out32 (s, ADDI | RT (3) | RA (3) | 4);
-    tcg_out32 (s, MTSPR | RS (3) | LR);
     tcg_out_mov (s, TCG_TYPE_I32, 3, TCG_AREG0);
-    tcg_out_b (s, 0, (tcg_target_long) ptr);
+    tcg_out_call (s, (tcg_target_long) ptr, 1, 0);
 }
 #endif
 
@@ -1050,12 +953,15 @@ static void tcg_target_qemu_prologue (TCGContext *s)
     tcg_out32 (s, BCLR | BO_ALWAYS);
 
 #ifdef CONFIG_SOFTMMU
-    for (i = 0; i < 4; ++i) {
-        ld_trampolines[i] = s->code_ptr;
-        emit_ldst_trampoline (s, qemu_ld_helpers[i]);
-
-        st_trampolines[i] = s->code_ptr;
-        emit_ldst_trampoline (s, qemu_st_helpers[i]);
+    for (i = 0; i < 16; ++i) {
+        if (qemu_ld_helpers[i]) {
+            ld_trampolines[i] = s->code_ptr;
+            emit_ldst_trampoline(s, qemu_ld_helpers[i]);
+        }
+        if (qemu_st_helpers[i]) {
+            st_trampolines[i] = s->code_ptr;
+            emit_ldst_trampoline(s, qemu_st_helpers[i]);
+        }
     }
 #endif
 }
@@ -1493,7 +1399,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         }
         break;
     case INDEX_op_call:
-        tcg_out_call (s, args[0], const_args[0]);
+        tcg_out_call (s, args[0], const_args[0], LK);
         break;
     case INDEX_op_movi_i32:
         tcg_out_movi(s, TCG_TYPE_I32, args[0], args[1]);
@@ -1800,36 +1706,18 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         tcg_out32 (s, NOR | SAB (args[1], args[0], args[1]));
         break;
 
-    case INDEX_op_qemu_ld8u:
+    case INDEX_op_qemu_ld_i32:
         tcg_out_qemu_ld(s, args, 0);
         break;
-    case INDEX_op_qemu_ld8s:
-        tcg_out_qemu_ld(s, args, 0 | 4);
-        break;
-    case INDEX_op_qemu_ld16u:
+    case INDEX_op_qemu_ld_i64:
         tcg_out_qemu_ld(s, args, 1);
         break;
-    case INDEX_op_qemu_ld16s:
-        tcg_out_qemu_ld(s, args, 1 | 4);
-        break;
-    case INDEX_op_qemu_ld32:
-        tcg_out_qemu_ld(s, args, 2);
-        break;
-    case INDEX_op_qemu_ld64:
-        tcg_out_qemu_ld(s, args, 3);
-        break;
-    case INDEX_op_qemu_st8:
+    case INDEX_op_qemu_st_i32:
         tcg_out_qemu_st(s, args, 0);
         break;
-    case INDEX_op_qemu_st16:
+    case INDEX_op_qemu_st_i64:
         tcg_out_qemu_st(s, args, 1);
         break;
-    case INDEX_op_qemu_st32:
-        tcg_out_qemu_st(s, args, 2);
-        break;
-    case INDEX_op_qemu_st64:
-        tcg_out_qemu_st(s, args, 3);
-        break;
 
     case INDEX_op_ext8s_i32:
         tcg_out32 (s, EXTSB | RS (args[1]) | RA (args[0]));
@@ -2013,29 +1901,15 @@ static const TCGTargetOpDef ppc_op_defs[] = {
     { INDEX_op_bswap32_i32, { "r", "r" } },
 
 #if TARGET_LONG_BITS == 32
-    { INDEX_op_qemu_ld8u, { "r", "L" } },
-    { INDEX_op_qemu_ld8s, { "r", "L" } },
-    { INDEX_op_qemu_ld16u, { "r", "L" } },
-    { INDEX_op_qemu_ld16s, { "r", "L" } },
-    { INDEX_op_qemu_ld32, { "r", "L" } },
-    { INDEX_op_qemu_ld64, { "r", "r", "L" } },
-
-    { INDEX_op_qemu_st8, { "K", "K" } },
-    { INDEX_op_qemu_st16, { "K", "K" } },
-    { INDEX_op_qemu_st32, { "K", "K" } },
-    { INDEX_op_qemu_st64, { "M", "M", "M" } },
+    { INDEX_op_qemu_ld_i32, { "r", "L" } },
+    { INDEX_op_qemu_ld_i64, { "L", "L", "L" } },
+    { INDEX_op_qemu_st_i32, { "K", "K" } },
+    { INDEX_op_qemu_st_i64, { "M", "M", "M" } },
 #else
-    { INDEX_op_qemu_ld8u, { "r", "L", "L" } },
-    { INDEX_op_qemu_ld8s, { "r", "L", "L" } },
-    { INDEX_op_qemu_ld16u, { "r", "L", "L" } },
-    { INDEX_op_qemu_ld16s, { "r", "L", "L" } },
-    { INDEX_op_qemu_ld32, { "r", "L", "L" } },
-    { INDEX_op_qemu_ld64, { "r", "L", "L", "L" } },
-
-    { INDEX_op_qemu_st8, { "K", "K", "K" } },
-    { INDEX_op_qemu_st16, { "K", "K", "K" } },
-    { INDEX_op_qemu_st32, { "K", "K", "K" } },
-    { INDEX_op_qemu_st64, { "M", "M", "M", "M" } },
+    { INDEX_op_qemu_ld_i32, { "r", "L", "L" } },
+    { INDEX_op_qemu_ld_i64, { "L", "L", "L", "L" } },
+    { INDEX_op_qemu_st_i32, { "K", "K", "K" } },
+    { INDEX_op_qemu_st_i64, { "M", "M", "M", "M" } },
 #endif
 
     { INDEX_op_ext8s_i32, { "r", "r" } },
diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h
index c9f8ff5206..e3395e301c 100644
--- a/tcg/ppc/tcg-target.h
+++ b/tcg/ppc/tcg-target.h
@@ -99,6 +99,8 @@ typedef enum {
 #define TCG_TARGET_HAS_muluh_i32        0
 #define TCG_TARGET_HAS_mulsh_i32        0
 
+#define TCG_TARGET_HAS_new_ldst         1
+
 #define TCG_AREG0 TCG_REG_R27
 
 #define tcg_qemu_tb_exec(env, tb_ptr) \
diff --git a/tcg/ppc64/tcg-target.c b/tcg/ppc64/tcg-target.c
index 0bd1e0ce8c..6109d862db 100644
--- a/tcg/ppc64/tcg-target.c
+++ b/tcg/ppc64/tcg-target.c
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "tcg-be-ldst.h"
+
 #define TCG_CT_CONST_S16  0x100
 #define TCG_CT_CONST_U16  0x200
 #define TCG_CT_CONST_S32  0x400
@@ -31,13 +33,11 @@
 
 static uint8_t *tb_ret_addr;
 
-#define FAST_PATH
-
 #if TARGET_LONG_BITS == 32
-#define LD_ADDR LWZU
+#define LD_ADDR LWZ
 #define CMP_L 0
 #else
-#define LD_ADDR LDU
+#define LD_ADDR LD
 #define CMP_L (1<<21)
 #endif
 
@@ -99,7 +99,7 @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
 #endif
 
 static const int tcg_target_reg_alloc_order[] = {
-    TCG_REG_R14,
+    TCG_REG_R14,  /* call saved registers */
     TCG_REG_R15,
     TCG_REG_R16,
     TCG_REG_R17,
@@ -109,29 +109,24 @@ static const int tcg_target_reg_alloc_order[] = {
     TCG_REG_R21,
     TCG_REG_R22,
     TCG_REG_R23,
+    TCG_REG_R24,
+    TCG_REG_R25,
+    TCG_REG_R26,
+    TCG_REG_R27,
     TCG_REG_R28,
     TCG_REG_R29,
     TCG_REG_R30,
     TCG_REG_R31,
-#ifdef __APPLE__
-    TCG_REG_R2,
-#endif
-    TCG_REG_R3,
-    TCG_REG_R4,
-    TCG_REG_R5,
-    TCG_REG_R6,
-    TCG_REG_R7,
-    TCG_REG_R8,
-    TCG_REG_R9,
-    TCG_REG_R10,
-#ifndef __APPLE__
+    TCG_REG_R12,  /* call clobbered, non-arguments */
     TCG_REG_R11,
-#endif
-    TCG_REG_R12,
-    TCG_REG_R24,
-    TCG_REG_R25,
-    TCG_REG_R26,
-    TCG_REG_R27
+    TCG_REG_R10,  /* call clobbered, arguments */
+    TCG_REG_R9,
+    TCG_REG_R8,
+    TCG_REG_R7,
+    TCG_REG_R6,
+    TCG_REG_R5,
+    TCG_REG_R4,
+    TCG_REG_R3,
 };
 
 static const int tcg_target_call_iarg_regs[] = {
@@ -173,58 +168,74 @@ static const int tcg_target_callee_save_regs[] = {
     TCG_REG_R31
 };
 
-static uint32_t reloc_pc24_val (void *pc, tcg_target_long target)
+static inline bool in_range_b(tcg_target_long target)
+{
+    return target == sextract64(target, 0, 26);
+}
+
+static uint32_t reloc_pc24_val(void *pc, tcg_target_long target)
 {
     tcg_target_long disp;
 
-    disp = target - (tcg_target_long) pc;
-    if ((disp << 38) >> 38 != disp)
-        tcg_abort ();
+    disp = target - (tcg_target_long)pc;
+    assert(in_range_b(disp));
 
     return disp & 0x3fffffc;
 }
 
-static void reloc_pc24 (void *pc, tcg_target_long target)
+static void reloc_pc24(void *pc, tcg_target_long target)
 {
-    *(uint32_t *) pc = (*(uint32_t *) pc & ~0x3fffffc)
-        | reloc_pc24_val (pc, target);
+    *(uint32_t *)pc = (*(uint32_t *)pc & ~0x3fffffc)
+        | reloc_pc24_val(pc, target);
 }
 
-static uint16_t reloc_pc14_val (void *pc, tcg_target_long target)
+static uint16_t reloc_pc14_val(void *pc, tcg_target_long target)
 {
     tcg_target_long disp;
 
-    disp = target - (tcg_target_long) pc;
-    if (disp != (int16_t) disp)
-        tcg_abort ();
+    disp = target - (tcg_target_long)pc;
+    if (disp != (int16_t) disp) {
+        tcg_abort();
+    }
 
     return disp & 0xfffc;
 }
 
-static void reloc_pc14 (void *pc, tcg_target_long target)
+static void reloc_pc14(void *pc, tcg_target_long target)
 {
-    *(uint32_t *) pc = (*(uint32_t *) pc & ~0xfffc)
-        | reloc_pc14_val (pc, target);
+    *(uint32_t *)pc = (*(uint32_t *)pc & ~0xfffc) | reloc_pc14_val(pc, target);
 }
 
-static void patch_reloc (uint8_t *code_ptr, int type,
-                         intptr_t value, intptr_t addend)
+static inline void tcg_out_b_noaddr(TCGContext *s, int insn)
+{
+    unsigned retrans = *(uint32_t *)s->code_ptr & 0x3fffffc;
+    tcg_out32(s, insn | retrans);
+}
+
+static inline void tcg_out_bc_noaddr(TCGContext *s, int insn)
+{
+    unsigned retrans = *(uint32_t *)s->code_ptr & 0xfffc;
+    tcg_out32(s, insn | retrans);
+}
+
+static void patch_reloc(uint8_t *code_ptr, int type,
+                        intptr_t value, intptr_t addend)
 {
     value += addend;
     switch (type) {
     case R_PPC_REL14:
-        reloc_pc14 (code_ptr, value);
+        reloc_pc14(code_ptr, value);
         break;
     case R_PPC_REL24:
-        reloc_pc24 (code_ptr, value);
+        reloc_pc24(code_ptr, value);
         break;
     default:
-        tcg_abort ();
+        tcg_abort();
     }
 }
 
 /* parse target specific constraints */
-static int target_parse_constraint (TCGArgConstraint *ct, const char **pct_str)
+static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
 {
     const char *ct_str;
 
@@ -232,29 +243,29 @@ static int target_parse_constraint (TCGArgConstraint *ct, const char **pct_str)
     switch (ct_str[0]) {
     case 'A': case 'B': case 'C': case 'D':
         ct->ct |= TCG_CT_REG;
-        tcg_regset_set_reg (ct->u.regs, 3 + ct_str[0] - 'A');
+        tcg_regset_set_reg(ct->u.regs, 3 + ct_str[0] - 'A');
         break;
     case 'r':
         ct->ct |= TCG_CT_REG;
-        tcg_regset_set32 (ct->u.regs, 0, 0xffffffff);
+        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
         break;
     case 'L':                   /* qemu_ld constraint */
         ct->ct |= TCG_CT_REG;
-        tcg_regset_set32 (ct->u.regs, 0, 0xffffffff);
-        tcg_regset_reset_reg (ct->u.regs, TCG_REG_R3);
+        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R3);
 #ifdef CONFIG_SOFTMMU
-        tcg_regset_reset_reg (ct->u.regs, TCG_REG_R4);
-        tcg_regset_reset_reg (ct->u.regs, TCG_REG_R5);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R4);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R5);
 #endif
         break;
     case 'S':                   /* qemu_st constraint */
         ct->ct |= TCG_CT_REG;
-        tcg_regset_set32 (ct->u.regs, 0, 0xffffffff);
-        tcg_regset_reset_reg (ct->u.regs, TCG_REG_R3);
+        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R3);
 #ifdef CONFIG_SOFTMMU
-        tcg_regset_reset_reg (ct->u.regs, TCG_REG_R4);
-        tcg_regset_reset_reg (ct->u.regs, TCG_REG_R5);
-        tcg_regset_reset_reg (ct->u.regs, TCG_REG_R6);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R4);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R5);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R6);
 #endif
         break;
     case 'I':
@@ -284,8 +295,8 @@ static int target_parse_constraint (TCGArgConstraint *ct, const char **pct_str)
 }
 
 /* test if a constant matches the constraint */
-static int tcg_target_const_match (tcg_target_long val,
-                                   const TCGArgConstraint *arg_ct)
+static int tcg_target_const_match(tcg_target_long val,
+                                  const TCGArgConstraint *arg_ct)
 {
     int ct = arg_ct->ct;
     if (ct & TCG_CT_CONST) {
@@ -425,7 +436,7 @@ static int tcg_target_const_match (tcg_target_long val,
 #define STHX   XO31(407)
 #define STWX   XO31(151)
 
-#define SPR(a,b) ((((a)<<5)|(b))<<11)
+#define SPR(a, b) ((((a)<<5)|(b))<<11)
 #define LR     SPR(8, 0)
 #define CTR    SPR(9, 0)
 
@@ -439,7 +450,7 @@ static int tcg_target_const_match (tcg_target_long val,
 #define SRADI  XO31(413<<1)
 
 #define TW     XO31( 4)
-#define TRAP   (TW | TO (31))
+#define TRAP   (TW | TO(31))
 
 #define RT(r) ((r)<<21)
 #define RS(r) ((r)<<21)
@@ -467,9 +478,9 @@ static int tcg_target_const_match (tcg_target_long val,
 #define BB(n, c) (((c)+((n)*4))<<11)
 #define BC_(n, c) (((c)+((n)*4))<<6)
 
-#define BO_COND_TRUE  BO (12)
-#define BO_COND_FALSE BO ( 4)
-#define BO_ALWAYS     BO (20)
+#define BO_COND_TRUE  BO(12)
+#define BO_COND_FALSE BO( 4)
+#define BO_ALWAYS     BO(20)
 
 enum {
     CR_LT,
@@ -479,16 +490,16 @@ enum {
 };
 
 static const uint32_t tcg_to_bc[] = {
-    [TCG_COND_EQ]  = BC | BI (7, CR_EQ) | BO_COND_TRUE,
-    [TCG_COND_NE]  = BC | BI (7, CR_EQ) | BO_COND_FALSE,
-    [TCG_COND_LT]  = BC | BI (7, CR_LT) | BO_COND_TRUE,
-    [TCG_COND_GE]  = BC | BI (7, CR_LT) | BO_COND_FALSE,
-    [TCG_COND_LE]  = BC | BI (7, CR_GT) | BO_COND_FALSE,
-    [TCG_COND_GT]  = BC | BI (7, CR_GT) | BO_COND_TRUE,
-    [TCG_COND_LTU] = BC | BI (7, CR_LT) | BO_COND_TRUE,
-    [TCG_COND_GEU] = BC | BI (7, CR_LT) | BO_COND_FALSE,
-    [TCG_COND_LEU] = BC | BI (7, CR_GT) | BO_COND_FALSE,
-    [TCG_COND_GTU] = BC | BI (7, CR_GT) | BO_COND_TRUE,
+    [TCG_COND_EQ]  = BC | BI(7, CR_EQ) | BO_COND_TRUE,
+    [TCG_COND_NE]  = BC | BI(7, CR_EQ) | BO_COND_FALSE,
+    [TCG_COND_LT]  = BC | BI(7, CR_LT) | BO_COND_TRUE,
+    [TCG_COND_GE]  = BC | BI(7, CR_LT) | BO_COND_FALSE,
+    [TCG_COND_LE]  = BC | BI(7, CR_GT) | BO_COND_FALSE,
+    [TCG_COND_GT]  = BC | BI(7, CR_GT) | BO_COND_TRUE,
+    [TCG_COND_LTU] = BC | BI(7, CR_LT) | BO_COND_TRUE,
+    [TCG_COND_GEU] = BC | BI(7, CR_LT) | BO_COND_FALSE,
+    [TCG_COND_LEU] = BC | BI(7, CR_GT) | BO_COND_FALSE,
+    [TCG_COND_GTU] = BC | BI(7, CR_GT) | BO_COND_TRUE,
 };
 
 /* The low bit here is set if the RA and RB fields must be inverted.  */
@@ -508,15 +519,17 @@ static const uint32_t tcg_to_isel[] = {
 static inline void tcg_out_mov(TCGContext *s, TCGType type,
                                TCGReg ret, TCGReg arg)
 {
-    tcg_out32 (s, OR | SAB (arg, ret, arg));
+    if (ret != arg) {
+        tcg_out32(s, OR | SAB(arg, ret, arg));
+    }
 }
 
 static inline void tcg_out_rld(TCGContext *s, int op, TCGReg ra, TCGReg rs,
                                int sh, int mb)
 {
-    sh = SH (sh & 0x1f) | (((sh >> 5) & 1) << 1);
-    mb = MB64 ((mb >> 5) | ((mb << 1) & 0x3f));
-    tcg_out32 (s, op | RA (ra) | RS (rs) | sh | mb);
+    sh = SH(sh & 0x1f) | (((sh >> 5) & 1) << 1);
+    mb = MB64((mb >> 5) | ((mb << 1) & 0x3f));
+    tcg_out32(s, op | RA(ra) | RS(rs) | sh | mb);
 }
 
 static inline void tcg_out_rlw(TCGContext *s, int op, TCGReg ra, TCGReg rs,
@@ -636,8 +649,8 @@ static void tcg_out_andi32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
     } else if (mask_operand(c, &mb, &me)) {
         tcg_out_rlw(s, RLWINM, dst, src, 0, mb, me);
     } else {
-        tcg_out_movi(s, TCG_TYPE_I32, 0, c);
-        tcg_out32(s, AND | SAB(src, dst, 0));
+        tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R0, c);
+        tcg_out32(s, AND | SAB(src, dst, TCG_REG_R0));
     }
 }
 
@@ -658,8 +671,8 @@ static void tcg_out_andi64(TCGContext *s, TCGReg dst, TCGReg src, uint64_t c)
             tcg_out_rld(s, RLDICL, dst, src, 0, mb);
         }
     } else {
-        tcg_out_movi(s, TCG_TYPE_I64, 0, c);
-        tcg_out32(s, AND | SAB(src, dst, 0));
+        tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_R0, c);
+        tcg_out32(s, AND | SAB(src, dst, TCG_REG_R0));
     }
 }
 
@@ -686,405 +699,487 @@ static void tcg_out_xori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
     tcg_out_zori32(s, dst, src, c, XORI, XORIS);
 }
 
-static void tcg_out_b (TCGContext *s, int mask, tcg_target_long target)
+static void tcg_out_b(TCGContext *s, int mask, tcg_target_long target)
 {
     tcg_target_long disp;
 
-    disp = target - (tcg_target_long) s->code_ptr;
-    if ((disp << 38) >> 38 == disp)
-        tcg_out32 (s, B | (disp & 0x3fffffc) | mask);
-    else {
-        tcg_out_movi (s, TCG_TYPE_I64, 0, (tcg_target_long) target);
-        tcg_out32 (s, MTSPR | RS (0) | CTR);
-        tcg_out32 (s, BCCTR | BO_ALWAYS | mask);
+    disp = target - (tcg_target_long)s->code_ptr;
+    if (in_range_b(disp)) {
+        tcg_out32(s, B | (disp & 0x3fffffc) | mask);
+    } else {
+        tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_R0, (tcg_target_long)target);
+        tcg_out32(s, MTSPR | RS(TCG_REG_R0) | CTR);
+        tcg_out32(s, BCCTR | BO_ALWAYS | mask);
     }
 }
 
-static void tcg_out_call (TCGContext *s, tcg_target_long arg, int const_arg)
+static void tcg_out_call(TCGContext *s, tcg_target_long arg, int const_arg)
 {
 #ifdef __APPLE__
     if (const_arg) {
-        tcg_out_b (s, LK, arg);
-    }
-    else {
-        tcg_out32 (s, MTSPR | RS (arg) | LR);
-        tcg_out32 (s, BCLR | BO_ALWAYS | LK);
+        tcg_out_b(s, LK, arg);
+    } else {
+        tcg_out32(s, MTSPR | RS(arg) | LR);
+        tcg_out32(s, BCLR | BO_ALWAYS | LK);
     }
 #else
-    int reg;
+    TCGReg reg = arg;
+    int ofs = 0;
 
     if (const_arg) {
-        reg = 2;
-        tcg_out_movi (s, TCG_TYPE_I64, reg, arg);
+        /* Look through the descriptor.  If the branch is in range, and we
+           don't have to spend too much effort on building the toc.  */
+        intptr_t tgt = ((intptr_t *)arg)[0];
+        intptr_t toc = ((intptr_t *)arg)[1];
+        intptr_t diff = tgt - (intptr_t)s->code_ptr;
+
+        if (in_range_b(diff) && toc == (uint32_t)toc) {
+            tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_R2, toc);
+            tcg_out_b(s, LK, tgt);
+            return;
+        }
+
+        /* Fold the low bits of the constant into the addresses below.  */
+        ofs = (int16_t)arg;
+        if (ofs + 8 < 0x8000) {
+            arg -= ofs;
+        } else {
+            ofs = 0;
+        }
+        reg = TCG_REG_R2;
+        tcg_out_movi(s, TCG_TYPE_I64, reg, arg);
     }
-    else reg = arg;
 
-    tcg_out32 (s, LD | RT (0) | RA (reg));
-    tcg_out32 (s, MTSPR | RA (0) | CTR);
-    tcg_out32 (s, LD | RT (11) | RA (reg) | 16);
-    tcg_out32 (s, LD | RT (2) | RA (reg) | 8);
-    tcg_out32 (s, BCCTR | BO_ALWAYS | LK);
+    tcg_out32(s, LD | TAI(TCG_REG_R0, reg, ofs));
+    tcg_out32(s, MTSPR | RA(TCG_REG_R0) | CTR);
+    tcg_out32(s, LD | TAI(TCG_REG_R2, reg, ofs + 8));
+    tcg_out32(s, BCCTR | BO_ALWAYS | LK);
 #endif
 }
 
-static void tcg_out_ldst(TCGContext *s, TCGReg ret, TCGReg addr,
-                         int offset, int op1, int op2)
+static void tcg_out_mem_long(TCGContext *s, int opi, int opx, TCGReg rt,
+                             TCGReg base, tcg_target_long offset)
 {
-    if (offset == (int16_t) offset) {
-        tcg_out32(s, op1 | TAI(ret, addr, offset));
-    } else {
-        tcg_out_movi(s, TCG_TYPE_I64, 0, offset);
-        tcg_out32(s, op2 | TAB(ret, addr, 0));
+    tcg_target_long orig = offset, l0, l1, extra = 0, align = 0;
+    TCGReg rs = TCG_REG_R2;
+
+    assert(rt != TCG_REG_R2 && base != TCG_REG_R2);
+
+    switch (opi) {
+    case LD: case LWA:
+        align = 3;
+        /* FALLTHRU */
+    default:
+        if (rt != TCG_REG_R0) {
+            rs = rt;
+        }
+        break;
+    case STD:
+        align = 3;
+        break;
+    case STB: case STH: case STW:
+        break;
     }
-}
 
-static void tcg_out_ldsta(TCGContext *s, TCGReg ret, TCGReg addr,
-                          int offset, int op1, int op2)
-{
-    if (offset == (int16_t) (offset & ~3)) {
-        tcg_out32(s, op1 | TAI(ret, addr, offset));
-    } else {
-        tcg_out_movi(s, TCG_TYPE_I64, 0, offset);
-        tcg_out32(s, op2 | TAB(ret, addr, 0));
+    /* For unaligned, or very large offsets, use the indexed form.  */
+    if (offset & align || offset != (int32_t)offset) {
+        tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_R2, orig);
+        tcg_out32(s, opx | TAB(rt, base, TCG_REG_R2));
+        return;
+    }
+
+    l0 = (int16_t)offset;
+    offset = (offset - l0) >> 16;
+    l1 = (int16_t)offset;
+
+    if (l1 < 0 && orig >= 0) {
+        extra = 0x4000;
+        l1 = (int16_t)(offset - 0x4000);
+    }
+    if (l1) {
+        tcg_out32(s, ADDIS | TAI(rs, base, l1));
+        base = rs;
+    }
+    if (extra) {
+        tcg_out32(s, ADDIS | TAI(rs, base, extra));
+        base = rs;
+    }
+    if (opi != ADDI || base != rt || l0 != 0) {
+        tcg_out32(s, opi | TAI(rt, base, l0));
     }
 }
 
+static const uint32_t qemu_ldx_opc[16] = {
+    [MO_UB] = LBZX,
+    [MO_UW] = LHZX,
+    [MO_UL] = LWZX,
+    [MO_Q]  = LDX,
+    [MO_SW] = LHAX,
+    [MO_SL] = LWAX,
+    [MO_BSWAP | MO_UB] = LBZX,
+    [MO_BSWAP | MO_UW] = LHBRX,
+    [MO_BSWAP | MO_UL] = LWBRX,
+    [MO_BSWAP | MO_Q]  = LDBRX,
+};
+
+static const uint32_t qemu_stx_opc[16] = {
+    [MO_UB] = STBX,
+    [MO_UW] = STHX,
+    [MO_UL] = STWX,
+    [MO_Q]  = STDX,
+    [MO_BSWAP | MO_UB] = STBX,
+    [MO_BSWAP | MO_UW] = STHBRX,
+    [MO_BSWAP | MO_UL] = STWBRX,
+    [MO_BSWAP | MO_Q]  = STDBRX,
+};
+
+static const uint32_t qemu_exts_opc[4] = {
+    EXTSB, EXTSH, EXTSW, 0
+};
+
 #if defined (CONFIG_SOFTMMU)
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
-   int mmu_idx) */
-static const void * const qemu_ld_helpers[4] = {
-    helper_ldb_mmu,
-    helper_ldw_mmu,
-    helper_ldl_mmu,
-    helper_ldq_mmu,
+ *                                 int mmu_idx, uintptr_t ra)
+ */
+static const void * const qemu_ld_helpers[16] = {
+    [MO_UB]   = helper_ret_ldub_mmu,
+    [MO_LEUW] = helper_le_lduw_mmu,
+    [MO_LEUL] = helper_le_ldul_mmu,
+    [MO_LEQ]  = helper_le_ldq_mmu,
+    [MO_BEUW] = helper_be_lduw_mmu,
+    [MO_BEUL] = helper_be_ldul_mmu,
+    [MO_BEQ]  = helper_be_ldq_mmu,
 };
 
 /* helper signature: helper_st_mmu(CPUState *env, target_ulong addr,
-   uintxx_t val, int mmu_idx) */
-static const void * const qemu_st_helpers[4] = {
-    helper_stb_mmu,
-    helper_stw_mmu,
-    helper_stl_mmu,
-    helper_stq_mmu,
+ *                                 uintxx_t val, int mmu_idx, uintptr_t ra)
+ */
+static const void * const qemu_st_helpers[16] = {
+    [MO_UB]   = helper_ret_stb_mmu,
+    [MO_LEUW] = helper_le_stw_mmu,
+    [MO_LEUL] = helper_le_stl_mmu,
+    [MO_LEQ]  = helper_le_stq_mmu,
+    [MO_BEUW] = helper_be_stw_mmu,
+    [MO_BEUL] = helper_be_stl_mmu,
+    [MO_BEQ]  = helper_be_stq_mmu,
 };
 
-static void tcg_out_tlb_read(TCGContext *s, TCGReg r0, TCGReg r1, TCGReg r2,
-                             TCGReg addr_reg, int s_bits, int offset)
+/* Perform the TLB load and compare.  Places the result of the comparison
+   in CR7, loads the addend of the TLB into R3, and returns the register
+   containing the guest address (zero-extended into R4).  Clobbers R0 and R2. */
+
+static TCGReg tcg_out_tlb_read(TCGContext *s, TCGMemOp s_bits, TCGReg addr_reg,
+                               int mem_index, bool is_read)
 {
-#if TARGET_LONG_BITS == 32
-    tcg_out_ext32u(s, addr_reg, addr_reg);
-
-    tcg_out_rlw(s, RLWINM, r0, addr_reg,
-                32 - (TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS),
-                32 - (CPU_TLB_BITS + CPU_TLB_ENTRY_BITS),
-                31 - CPU_TLB_ENTRY_BITS);
-    tcg_out32(s, ADD | TAB(r0, r0, TCG_AREG0));
-    tcg_out32(s, LWZU | TAI(r1, r0, offset));
-    tcg_out_rlw(s, RLWINM, r2, addr_reg, 0,
-                (32 - s_bits) & 31, 31 - TARGET_PAGE_BITS);
-#else
-    tcg_out_rld (s, RLDICL, r0, addr_reg,
-                 64 - TARGET_PAGE_BITS,
-                 64 - CPU_TLB_BITS);
-    tcg_out_shli64(s, r0, r0, CPU_TLB_ENTRY_BITS);
+    int cmp_off
+        = (is_read
+           ? offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)
+           : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write));
+    int add_off = offsetof(CPUArchState, tlb_table[mem_index][0].addend);
+    TCGReg base = TCG_AREG0;
+
+    /* Extract the page index, shifted into place for tlb index.  */
+    if (TARGET_LONG_BITS == 32) {
+        /* Zero-extend the address into a place helpful for further use.  */
+        tcg_out_ext32u(s, TCG_REG_R4, addr_reg);
+        addr_reg = TCG_REG_R4;
+    } else {
+        tcg_out_rld(s, RLDICL, TCG_REG_R3, addr_reg,
+                    64 - TARGET_PAGE_BITS, 64 - CPU_TLB_BITS);
+    }
 
-    tcg_out32(s, ADD | TAB(r0, r0, TCG_AREG0));
-    tcg_out32(s, LD_ADDR | TAI(r1, r0, offset));
+    /* Compensate for very large offsets.  */
+    if (add_off >= 0x8000) {
+        /* Most target env are smaller than 32k; none are larger than 64k.
+           Simplify the logic here merely to offset by 0x7ff0, giving us a
+           range just shy of 64k.  Check this assumption.  */
+        QEMU_BUILD_BUG_ON(offsetof(CPUArchState,
+                                   tlb_table[NB_MMU_MODES - 1][1])
+                          > 0x7ff0 + 0x7fff);
+        tcg_out32(s, ADDI | TAI(TCG_REG_R2, base, 0x7ff0));
+        base = TCG_REG_R2;
+        cmp_off -= 0x7ff0;
+        add_off -= 0x7ff0;
+    }
 
-    if (!s_bits) {
-        tcg_out_rld (s, RLDICR, r2, addr_reg, 0, 63 - TARGET_PAGE_BITS);
+    /* Extraction and shifting, part 2.  */
+    if (TARGET_LONG_BITS == 32) {
+        tcg_out_rlw(s, RLWINM, TCG_REG_R3, addr_reg,
+                    32 - (TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS),
+                    32 - (CPU_TLB_BITS + CPU_TLB_ENTRY_BITS),
+                    31 - CPU_TLB_ENTRY_BITS);
+    } else {
+        tcg_out_shli64(s, TCG_REG_R3, TCG_REG_R3, CPU_TLB_ENTRY_BITS);
     }
-    else {
-        tcg_out_rld (s, RLDICL, r2, addr_reg,
-                     64 - TARGET_PAGE_BITS,
-                     TARGET_PAGE_BITS - s_bits);
-        tcg_out_rld (s, RLDICL, r2, r2, TARGET_PAGE_BITS, 0);
+
+    tcg_out32(s, ADD | TAB(TCG_REG_R3, TCG_REG_R3, base));
+
+    /* Load the tlb comparator.  */
+    tcg_out32(s, LD_ADDR | TAI(TCG_REG_R2, TCG_REG_R3, cmp_off));
+
+    /* Load the TLB addend for use on the fast path.  Do this asap
+       to minimize any load use delay.  */
+    tcg_out32(s, LD | TAI(TCG_REG_R3, TCG_REG_R3, add_off));
+
+    /* Clear the non-page, non-alignment bits from the address.  */
+    if (TARGET_LONG_BITS == 32) {
+        tcg_out_rlw(s, RLWINM, TCG_REG_R0, addr_reg, 0,
+                    (32 - s_bits) & 31, 31 - TARGET_PAGE_BITS);
+    } else if (!s_bits) {
+        tcg_out_rld(s, RLDICR, TCG_REG_R0, addr_reg, 0, 63 - TARGET_PAGE_BITS);
+    } else {
+        tcg_out_rld(s, RLDICL, TCG_REG_R0, addr_reg,
+                    64 - TARGET_PAGE_BITS, TARGET_PAGE_BITS - s_bits);
+        tcg_out_rld(s, RLDICL, TCG_REG_R0, TCG_REG_R0, TARGET_PAGE_BITS, 0);
     }
-#endif
-}
-#endif
 
-static const uint32_t qemu_ldx_opc[8] = {
-#ifdef TARGET_WORDS_BIGENDIAN
-    LBZX, LHZX, LWZX, LDX,
-    0,    LHAX, LWAX, LDX
-#else
-    LBZX, LHBRX, LWBRX, LDBRX,
-    0,    0,     0,     LDBRX,
-#endif
-};
+    tcg_out32(s, CMP | BF(7) | RA(TCG_REG_R0) | RB(TCG_REG_R2) | CMP_L);
 
-static const uint32_t qemu_stx_opc[4] = {
-#ifdef TARGET_WORDS_BIGENDIAN
-    STBX, STHX, STWX, STDX
-#else
-    STBX, STHBRX, STWBRX, STDBRX,
-#endif
-};
+    return addr_reg;
+}
 
-static const uint32_t qemu_exts_opc[4] = {
-    EXTSB, EXTSH, EXTSW, 0
-};
+/* Record the context of a call to the out of line helper code for the slow
+   path for a load or store, so that we can later generate the correct
+   helper code.  */
+static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOp opc,
+                                int data_reg, int addr_reg, int mem_index,
+                                uint8_t *raddr, uint8_t *label_ptr)
+{
+    TCGLabelQemuLdst *label = new_ldst_label(s);
+
+    label->is_ld = is_ld;
+    label->opc = opc;
+    label->datalo_reg = data_reg;
+    label->addrlo_reg = addr_reg;
+    label->mem_index = mem_index;
+    label->raddr = raddr;
+    label->label_ptr[0] = label_ptr;
+}
 
-static void tcg_out_qemu_ld (TCGContext *s, const TCGArg *args, int opc)
+static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
 {
-    TCGReg addr_reg, data_reg, r0, r1, rbase;
-    uint32_t insn, s_bits;
-#ifdef CONFIG_SOFTMMU
-    TCGReg r2, ir;
-    int mem_index;
-    void *label1_ptr, *label2_ptr;
-#endif
+    TCGMemOp opc = lb->opc;
 
-    data_reg = *args++;
-    addr_reg = *args++;
-    s_bits = opc & 3;
+    reloc_pc14(lb->label_ptr[0], (uintptr_t)s->code_ptr);
 
-#ifdef CONFIG_SOFTMMU
-    mem_index = *args;
+    tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_R3, TCG_AREG0);
 
-    r0 = 3;
-    r1 = 4;
-    r2 = 0;
-    rbase = 0;
+    /* If the address needed to be zero-extended, we'll have already
+       placed it in R4.  The only remaining case is 64-bit guest.  */
+    tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R4, lb->addrlo_reg);
 
-    tcg_out_tlb_read (s, r0, r1, r2, addr_reg, s_bits,
-                      offsetof (CPUArchState, tlb_table[mem_index][0].addr_read));
+    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R5, lb->mem_index);
+    tcg_out32(s, MFSPR | RT(TCG_REG_R6) | LR);
 
-    tcg_out32 (s, CMP | BF (7) | RA (r2) | RB (r1) | CMP_L);
+    tcg_out_call(s, (tcg_target_long)qemu_ld_helpers[opc & ~MO_SIGN], 1);
 
-    label1_ptr = s->code_ptr;
-#ifdef FAST_PATH
-    tcg_out32 (s, BC | BI (7, CR_EQ) | BO_COND_TRUE);
-#endif
+    if (opc & MO_SIGN) {
+        uint32_t insn = qemu_exts_opc[opc & MO_SIZE];
+        tcg_out32(s, insn | RA(lb->datalo_reg) | RS(TCG_REG_R3));
+    } else {
+        tcg_out_mov(s, TCG_TYPE_I64, lb->datalo_reg, TCG_REG_R3);
+    }
 
-    /* slow path */
-    ir = 3;
-    tcg_out_mov (s, TCG_TYPE_I64, ir++, TCG_AREG0);
-    tcg_out_mov (s, TCG_TYPE_I64, ir++, addr_reg);
-    tcg_out_movi (s, TCG_TYPE_I64, ir++, mem_index);
+    tcg_out_b(s, 0, (uintptr_t)lb->raddr);
+}
 
-    tcg_out_call (s, (tcg_target_long) qemu_ld_helpers[s_bits], 1);
+static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
+{
+    TCGMemOp opc = lb->opc;
+    TCGMemOp s_bits = opc & MO_SIZE;
 
-    if (opc & 4) {
-        insn = qemu_exts_opc[s_bits];
-        tcg_out32(s, insn | RA(data_reg) | RS(3));
-    } else if (data_reg != 3) {
-        tcg_out_mov(s, TCG_TYPE_I64, data_reg, 3);
-    }
-    label2_ptr = s->code_ptr;
-    tcg_out32 (s, B);
+    reloc_pc14(lb->label_ptr[0], (uintptr_t)s->code_ptr);
+
+    tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R3, TCG_AREG0);
+
+    /* If the address needed to be zero-extended, we'll have already
+       placed it in R4.  The only remaining case is 64-bit guest.  */
+    tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R4, lb->addrlo_reg);
+
+    tcg_out_rld(s, RLDICL, TCG_REG_R5, lb->datalo_reg,
+                0, 64 - (1 << (3 + s_bits)));
+    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R6, lb->mem_index);
+    tcg_out32(s, MFSPR | RT(TCG_REG_R7) | LR);
 
-    /* label1: fast path */
-#ifdef FAST_PATH
-    reloc_pc14 (label1_ptr, (tcg_target_long) s->code_ptr);
+    tcg_out_call(s, (tcg_target_long)qemu_st_helpers[opc], 1);
+
+    tcg_out_b(s, 0, (uintptr_t)lb->raddr);
+}
+#endif /* SOFTMMU */
+
+static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
+                            TCGMemOp opc, int mem_index)
+{
+    TCGReg rbase;
+    uint32_t insn;
+    TCGMemOp s_bits = opc & MO_SIZE;
+#ifdef CONFIG_SOFTMMU
+    void *label_ptr;
 #endif
 
-    /* r0 now contains &env->tlb_table[mem_index][index].addr_read */
-    tcg_out32(s, LD | TAI(r0, r0,
-                          offsetof(CPUTLBEntry, addend)
-                          - offsetof(CPUTLBEntry, addr_read)));
-    /* r0 = env->tlb_table[mem_index][index].addend */
-    tcg_out32(s, ADD | TAB(r0, r0, addr_reg));
-    /* r0 = env->tlb_table[mem_index][index].addend + addr */
+#ifdef CONFIG_SOFTMMU
+    addr_reg = tcg_out_tlb_read(s, s_bits, addr_reg, mem_index, true);
 
+    /* Load a pointer into the current opcode w/conditional branch-link. */
+    label_ptr = s->code_ptr;
+    tcg_out_bc_noaddr(s, BC | BI(7, CR_EQ) | BO_COND_FALSE | LK);
+
+    rbase = TCG_REG_R3;
 #else  /* !CONFIG_SOFTMMU */
-#if TARGET_LONG_BITS == 32
-    tcg_out_ext32u(s, addr_reg, addr_reg);
-#endif
-    r0 = addr_reg;
-    r1 = 3;
     rbase = GUEST_BASE ? TCG_GUEST_BASE_REG : 0;
+    if (TARGET_LONG_BITS == 32) {
+        tcg_out_ext32u(s, TCG_REG_R2, addr_reg);
+        addr_reg = TCG_REG_R2;
+    }
 #endif
 
     insn = qemu_ldx_opc[opc];
     if (!HAVE_ISA_2_06 && insn == LDBRX) {
-        tcg_out32(s, ADDI | TAI(r1, r0, 4));
-        tcg_out32(s, LWBRX | TAB(data_reg, rbase, r0));
-        tcg_out32(s, LWBRX | TAB(      r1, rbase, r1));
-        tcg_out_rld(s, RLDIMI, data_reg, r1, 32, 0);
+        tcg_out32(s, ADDI | TAI(TCG_REG_R0, addr_reg, 4));
+        tcg_out32(s, LWBRX | TAB(data_reg, rbase, addr_reg));
+        tcg_out32(s, LWBRX | TAB(TCG_REG_R0, rbase, TCG_REG_R0));
+        tcg_out_rld(s, RLDIMI, data_reg, TCG_REG_R0, 32, 0);
     } else if (insn) {
-        tcg_out32(s, insn | TAB(data_reg, rbase, r0));
+        tcg_out32(s, insn | TAB(data_reg, rbase, addr_reg));
     } else {
-        insn = qemu_ldx_opc[s_bits];
-        tcg_out32(s, insn | TAB(data_reg, rbase, r0));
+        insn = qemu_ldx_opc[opc & (MO_SIZE | MO_BSWAP)];
+        tcg_out32(s, insn | TAB(data_reg, rbase, addr_reg));
         insn = qemu_exts_opc[s_bits];
-        tcg_out32 (s, insn | RA(data_reg) | RS(data_reg));
+        tcg_out32(s, insn | RA(data_reg) | RS(data_reg));
     }
 
 #ifdef CONFIG_SOFTMMU
-    reloc_pc24 (label2_ptr, (tcg_target_long) s->code_ptr);
+    add_qemu_ldst_label(s, true, opc, data_reg, addr_reg, mem_index,
+                        s->code_ptr, label_ptr);
 #endif
 }
 
-static void tcg_out_qemu_st (TCGContext *s, const TCGArg *args, int opc)
+static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
+                            TCGMemOp opc, int mem_index)
 {
-    TCGReg addr_reg, r0, r1, rbase, data_reg;
+    TCGReg rbase;
     uint32_t insn;
 #ifdef CONFIG_SOFTMMU
-    TCGReg r2, ir;
-    int mem_index;
-    void *label1_ptr, *label2_ptr;
+    void *label_ptr;
 #endif
 
-    data_reg = *args++;
-    addr_reg = *args++;
-
 #ifdef CONFIG_SOFTMMU
-    mem_index = *args;
-
-    r0 = 3;
-    r1 = 4;
-    r2 = 0;
-    rbase = 0;
+    addr_reg = tcg_out_tlb_read(s, opc & MO_SIZE, addr_reg, mem_index, false);
 
-    tcg_out_tlb_read (s, r0, r1, r2, addr_reg, opc,
-                      offsetof (CPUArchState, tlb_table[mem_index][0].addr_write));
-
-    tcg_out32 (s, CMP | BF (7) | RA (r2) | RB (r1) | CMP_L);
-
-    label1_ptr = s->code_ptr;
-#ifdef FAST_PATH
-    tcg_out32 (s, BC | BI (7, CR_EQ) | BO_COND_TRUE);
-#endif
-
-    /* slow path */
-    ir = 3;
-    tcg_out_mov (s, TCG_TYPE_I64, ir++, TCG_AREG0);
-    tcg_out_mov (s, TCG_TYPE_I64, ir++, addr_reg);
-    tcg_out_rld (s, RLDICL, ir++, data_reg, 0, 64 - (1 << (3 + opc)));
-    tcg_out_movi (s, TCG_TYPE_I64, ir++, mem_index);
-
-    tcg_out_call (s, (tcg_target_long) qemu_st_helpers[opc], 1);
-
-    label2_ptr = s->code_ptr;
-    tcg_out32 (s, B);
-
-    /* label1: fast path */
-#ifdef FAST_PATH
-    reloc_pc14 (label1_ptr, (tcg_target_long) s->code_ptr);
-#endif
-
-    tcg_out32 (s, (LD
-                   | RT (r0)
-                   | RA (r0)
-                   | (offsetof (CPUTLBEntry, addend)
-                      - offsetof (CPUTLBEntry, addr_write))
-                   ));
-    /* r0 = env->tlb_table[mem_index][index].addend */
-    tcg_out32(s, ADD | TAB(r0, r0, addr_reg));
-    /* r0 = env->tlb_table[mem_index][index].addend + addr */
+    /* Load a pointer into the current opcode w/conditional branch-link. */
+    label_ptr = s->code_ptr;
+    tcg_out_bc_noaddr(s, BC | BI(7, CR_EQ) | BO_COND_FALSE | LK);
 
+    rbase = TCG_REG_R3;
 #else  /* !CONFIG_SOFTMMU */
-#if TARGET_LONG_BITS == 32
-    tcg_out_ext32u(s, addr_reg, addr_reg);
-#endif
-    r1 = 3;
-    r0 = addr_reg;
     rbase = GUEST_BASE ? TCG_GUEST_BASE_REG : 0;
+    if (TARGET_LONG_BITS == 32) {
+        tcg_out_ext32u(s, TCG_REG_R2, addr_reg);
+        addr_reg = TCG_REG_R2;
+    }
 #endif
 
     insn = qemu_stx_opc[opc];
     if (!HAVE_ISA_2_06 && insn == STDBRX) {
-        tcg_out32(s, STWBRX | SAB(data_reg, rbase, r0));
-        tcg_out32(s, ADDI | TAI(r1, r0, 4));
-        tcg_out_shri64(s, 0, data_reg, 32);
-        tcg_out32(s, STWBRX | SAB(0, rbase, r1));
+        tcg_out32(s, STWBRX | SAB(data_reg, rbase, addr_reg));
+        tcg_out32(s, ADDI | TAI(TCG_REG_R2, addr_reg, 4));
+        tcg_out_shri64(s, TCG_REG_R0, data_reg, 32);
+        tcg_out32(s, STWBRX | SAB(TCG_REG_R0, rbase, TCG_REG_R2));
     } else {
-        tcg_out32(s, insn | SAB(data_reg, rbase, r0));
+        tcg_out32(s, insn | SAB(data_reg, rbase, addr_reg));
     }
 
 #ifdef CONFIG_SOFTMMU
-    reloc_pc24 (label2_ptr, (tcg_target_long) s->code_ptr);
+    add_qemu_ldst_label(s, false, opc, data_reg, addr_reg, mem_index,
+                        s->code_ptr, label_ptr);
 #endif
 }
 
-static void tcg_target_qemu_prologue (TCGContext *s)
+#define FRAME_SIZE ((int) \
+    ((8                     /* back chain */              \
+      + 8                   /* CR */                      \
+      + 8                   /* LR */                      \
+      + 8                   /* compiler doubleword */     \
+      + 8                   /* link editor doubleword */  \
+      + 8                   /* TOC save area */           \
+      + TCG_STATIC_CALL_ARGS_SIZE                         \
+      + CPU_TEMP_BUF_NLONGS * sizeof(long)                \
+      + ARRAY_SIZE(tcg_target_callee_save_regs) * 8       \
+      + 15) & ~15))
+
+#define REG_SAVE_BOT (FRAME_SIZE - ARRAY_SIZE(tcg_target_callee_save_regs) * 8)
+
+static void tcg_target_qemu_prologue(TCGContext *s)
 {
-    int i, frame_size;
-#ifndef __APPLE__
-    uint64_t addr;
-#endif
+    int i;
 
-    frame_size = 0
-        + 8                     /* back chain */
-        + 8                     /* CR */
-        + 8                     /* LR */
-        + 8                     /* compiler doubleword */
-        + 8                     /* link editor doubleword */
-        + 8                     /* TOC save area */
-        + TCG_STATIC_CALL_ARGS_SIZE
-        + ARRAY_SIZE (tcg_target_callee_save_regs) * 8
-        + CPU_TEMP_BUF_NLONGS * sizeof(long)
-        ;
-    frame_size = (frame_size + 15) & ~15;
-
-    tcg_set_frame (s, TCG_REG_CALL_STACK, frame_size
-                   - CPU_TEMP_BUF_NLONGS * sizeof (long),
-                   CPU_TEMP_BUF_NLONGS * sizeof (long));
+    tcg_set_frame(s, TCG_REG_CALL_STACK,
+                  REG_SAVE_BOT - CPU_TEMP_BUF_NLONGS * sizeof(long),
+                  CPU_TEMP_BUF_NLONGS * sizeof(long));
 
 #ifndef __APPLE__
     /* First emit adhoc function descriptor */
-    addr = (uint64_t) s->code_ptr + 24;
-    tcg_out32 (s, addr >> 32); tcg_out32 (s, addr); /* entry point */
+    tcg_out64(s, (uint64_t)s->code_ptr + 24); /* entry point */
     s->code_ptr += 16;          /* skip TOC and environment pointer */
 #endif
 
     /* Prologue */
-    tcg_out32 (s, MFSPR | RT (0) | LR);
-    tcg_out32 (s, STDU | RS (1) | RA (1) | (-frame_size & 0xffff));
-    for (i = 0; i < ARRAY_SIZE (tcg_target_callee_save_regs); ++i)
-        tcg_out32 (s, (STD
-                       | RS (tcg_target_callee_save_regs[i])
-                       | RA (1)
-                       | (i * 8 + 48 + TCG_STATIC_CALL_ARGS_SIZE)
-                       )
-            );
-    tcg_out32 (s, STD | RS (0) | RA (1) | (frame_size + 16));
+    tcg_out32(s, MFSPR | RT(TCG_REG_R0) | LR);
+    tcg_out32(s, STDU | SAI(TCG_REG_R1, TCG_REG_R1, -FRAME_SIZE));
+    for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i) {
+        tcg_out32(s, STD | SAI(tcg_target_callee_save_regs[i], 1, 
+                               REG_SAVE_BOT + i * 8));
+    }
+    tcg_out32(s, STD | SAI(TCG_REG_R0, TCG_REG_R1, FRAME_SIZE + 16));
 
 #ifdef CONFIG_USE_GUEST_BASE
     if (GUEST_BASE) {
-        tcg_out_movi (s, TCG_TYPE_I64, TCG_GUEST_BASE_REG, GUEST_BASE);
-        tcg_regset_set_reg (s->reserved_regs, TCG_GUEST_BASE_REG);
+        tcg_out_movi(s, TCG_TYPE_I64, TCG_GUEST_BASE_REG, GUEST_BASE);
+        tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
     }
 #endif
 
-    tcg_out_mov (s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
-    tcg_out32 (s, MTSPR | RS (tcg_target_call_iarg_regs[1]) | CTR);
-    tcg_out32 (s, BCCTR | BO_ALWAYS);
+    tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
+    tcg_out32(s, MTSPR | RS(tcg_target_call_iarg_regs[1]) | CTR);
+    tcg_out32(s, BCCTR | BO_ALWAYS);
 
     /* Epilogue */
     tb_ret_addr = s->code_ptr;
 
-    for (i = 0; i < ARRAY_SIZE (tcg_target_callee_save_regs); ++i)
-        tcg_out32 (s, (LD
-                       | RT (tcg_target_callee_save_regs[i])
-                       | RA (1)
-                       | (i * 8 + 48 + TCG_STATIC_CALL_ARGS_SIZE)
-                       )
-            );
-    tcg_out32(s, LD | TAI(0, 1, frame_size + 16));
-    tcg_out32(s, MTSPR | RS(0) | LR);
-    tcg_out32(s, ADDI | TAI(1, 1, frame_size));
+    for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i) {
+        tcg_out32(s, LD | TAI(tcg_target_callee_save_regs[i], TCG_REG_R1,
+                              REG_SAVE_BOT + i * 8));
+    }
+    tcg_out32(s, LD | TAI(TCG_REG_R0, TCG_REG_R1, FRAME_SIZE + 16));
+    tcg_out32(s, MTSPR | RS(TCG_REG_R0) | LR);
+    tcg_out32(s, ADDI | TAI(TCG_REG_R1, TCG_REG_R1, FRAME_SIZE));
     tcg_out32(s, BCLR | BO_ALWAYS);
 }
 
-static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
-                       intptr_t arg2)
+static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
+                              TCGReg arg1, intptr_t arg2)
 {
-    if (type == TCG_TYPE_I32)
-        tcg_out_ldst (s, ret, arg1, arg2, LWZ, LWZX);
-    else
-        tcg_out_ldsta (s, ret, arg1, arg2, LD, LDX);
+    int opi, opx;
+
+    if (type == TCG_TYPE_I32) {
+        opi = LWZ, opx = LWZX;
+    } else {
+        opi = LD, opx = LDX;
+    }
+    tcg_out_mem_long(s, opi, opx, ret, arg1, arg2);
 }
 
-static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
-                       intptr_t arg2)
+static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
+                              TCGReg arg1, intptr_t arg2)
 {
-    if (type == TCG_TYPE_I32)
-        tcg_out_ldst (s, arg, arg1, arg2, STW, STWX);
-    else
-        tcg_out_ldsta (s, arg, arg1, arg2, STD, STDX);
+    int opi, opx;
+
+    if (type == TCG_TYPE_I32) {
+        opi = STW, opx = STWX;
+    } else {
+        opi = STD, opx = STDX;
+    }
+    tcg_out_mem_long(s, opi, opx, arg, arg1, arg2);
 }
 
 static void tcg_out_cmp(TCGContext *s, int cond, TCGArg arg1, TCGArg arg2,
@@ -1106,8 +1201,7 @@ static void tcg_out_cmp(TCGContext *s, int cond, TCGArg arg1, TCGArg arg2,
                 op = CMPI;
                 imm = 1;
                 break;
-            }
-            else if ((uint16_t) arg2 == arg2) {
+            } else if ((uint16_t) arg2 == arg2) {
                 op = CMPLI;
                 imm = 1;
                 break;
@@ -1148,7 +1242,7 @@ static void tcg_out_cmp(TCGContext *s, int cond, TCGArg arg1, TCGArg arg2,
         break;
 
     default:
-        tcg_abort ();
+        tcg_abort();
     }
     op |= BF(cr) | ((type == TCG_TYPE_I64) << 21);
 
@@ -1156,8 +1250,8 @@ static void tcg_out_cmp(TCGContext *s, int cond, TCGArg arg1, TCGArg arg2,
         tcg_out32(s, op | RA(arg1) | (arg2 & 0xffff));
     } else {
         if (const_arg2) {
-            tcg_out_movi(s, type, 0, arg2);
-            arg2 = 0;
+            tcg_out_movi(s, type, TCG_REG_R0, arg2);
+            arg2 = TCG_REG_R0;
         }
         tcg_out32(s, op | RA(arg1) | RB(arg2));
     }
@@ -1178,8 +1272,8 @@ static void tcg_out_setcond_ne0(TCGContext *s, TCGReg dst, TCGReg src)
         tcg_out32(s, ADDIC | TAI(dst, src, -1));
         tcg_out32(s, SUBFE | TAB(dst, dst, src));
     } else {
-        tcg_out32(s, ADDIC | TAI(0, src, -1));
-        tcg_out32(s, SUBFE | TAB(dst, 0, src));
+        tcg_out32(s, ADDIC | TAI(TCG_REG_R0, src, -1));
+        tcg_out32(s, SUBFE | TAB(dst, TCG_REG_R0, src));
     }
 }
 
@@ -1292,13 +1386,13 @@ static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond,
     case TCG_COND_GE:
     case TCG_COND_GEU:
         sh = 31;
-        crop = CRNOR | BT (7, CR_EQ) | BA (7, CR_LT) | BB (7, CR_LT);
+        crop = CRNOR | BT(7, CR_EQ) | BA(7, CR_LT) | BB(7, CR_LT);
         goto crtest;
 
     case TCG_COND_LE:
     case TCG_COND_LEU:
         sh = 31;
-        crop = CRNOR | BT (7, CR_EQ) | BA (7, CR_GT) | BB (7, CR_GT);
+        crop = CRNOR | BT(7, CR_EQ) | BA(7, CR_GT) | BB(7, CR_GT);
     crtest:
         tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type);
         if (crop) {
@@ -1309,22 +1403,19 @@ static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond,
         break;
 
     default:
-        tcg_abort ();
+        tcg_abort();
     }
 }
 
-static void tcg_out_bc (TCGContext *s, int bc, int label_index)
+static void tcg_out_bc(TCGContext *s, int bc, int label_index)
 {
     TCGLabel *l = &s->labels[label_index];
 
-    if (l->has_value)
-        tcg_out32 (s, bc | reloc_pc14_val (s->code_ptr, l->u.value));
-    else {
-        uint16_t val = *(uint16_t *) &s->code_ptr[2];
-
-        /* Thanks to Andrzej Zaborowski */
-        tcg_out32 (s, bc | (val & 0xfffc));
-        tcg_out_reloc (s, s->code_ptr - 4, R_PPC_REL14, label_index, 0);
+    if (l->has_value) {
+        tcg_out32(s, bc | reloc_pc14_val(s->code_ptr, l->u.value));
+    } else {
+        tcg_out_reloc(s, s->code_ptr, R_PPC_REL14, label_index, 0);
+        tcg_out_bc_noaddr(s, bc);
     }
 }
 
@@ -1360,7 +1451,7 @@ static void tcg_out_movcond(TCGContext *s, TCGType type, TCGCond cond,
         }
         /* V1 == 0 is handled by isel; V2 == 0 must be handled by hand.  */
         if (v2 == 0) {
-            tcg_out_movi(s, type, 0, 0);
+            tcg_out_movi(s, type, TCG_REG_R0, 0);
         }
         tcg_out32(s, isel | TAB(dest, v1, v2));
     } else {
@@ -1384,37 +1475,36 @@ static void tcg_out_movcond(TCGContext *s, TCGType type, TCGCond cond,
     }
 }
 
-void ppc_tb_set_jmp_target (unsigned long jmp_addr, unsigned long addr)
+void ppc_tb_set_jmp_target(unsigned long jmp_addr, unsigned long addr)
 {
     TCGContext s;
     unsigned long patch_size;
 
     s.code_ptr = (uint8_t *) jmp_addr;
-    tcg_out_b (&s, 0, addr);
+    tcg_out_b(&s, 0, addr);
     patch_size = s.code_ptr - (uint8_t *) jmp_addr;
-    flush_icache_range (jmp_addr, jmp_addr + patch_size);
+    flush_icache_range(jmp_addr, jmp_addr + patch_size);
 }
 
-static void tcg_out_op (TCGContext *s, TCGOpcode opc, const TCGArg *args,
-                        const int *const_args)
+static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
+                       const int *const_args)
 {
     TCGArg a0, a1, a2;
     int c;
 
     switch (opc) {
     case INDEX_op_exit_tb:
-        tcg_out_movi (s, TCG_TYPE_I64, TCG_REG_R3, args[0]);
-        tcg_out_b (s, 0, (tcg_target_long) tb_ret_addr);
+        tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_R3, args[0]);
+        tcg_out_b(s, 0, (tcg_target_long)tb_ret_addr);
         break;
     case INDEX_op_goto_tb:
         if (s->tb_jmp_offset) {
-            /* direct jump method */
-
+            /* Direct jump method.  */
             s->tb_jmp_offset[args[0]] = s->code_ptr - s->code_buf;
             s->code_ptr += 28;
-        }
-        else {
-            tcg_abort ();
+        } else {
+            /* Indirect jump method.  */
+            tcg_abort();
         }
         s->tb_next_offset[args[0]] = s->code_ptr - s->code_buf;
         break;
@@ -1423,83 +1513,70 @@ static void tcg_out_op (TCGContext *s, TCGOpcode opc, const TCGArg *args,
             TCGLabel *l = &s->labels[args[0]];
 
             if (l->has_value) {
-                tcg_out_b (s, 0, l->u.value);
-            }
-            else {
-                uint32_t val = *(uint32_t *) s->code_ptr;
-
-                /* Thanks to Andrzej Zaborowski */
-                tcg_out32 (s, B | (val & 0x3fffffc));
-                tcg_out_reloc (s, s->code_ptr - 4, R_PPC_REL24, args[0], 0);
+                tcg_out_b(s, 0, l->u.value);
+            } else {
+                tcg_out_reloc(s, s->code_ptr, R_PPC_REL24, args[0], 0);
+                tcg_out_b_noaddr(s, B);
             }
         }
         break;
     case INDEX_op_call:
-        tcg_out_call (s, args[0], const_args[0]);
+        tcg_out_call(s, args[0], const_args[0]);
         break;
     case INDEX_op_movi_i32:
-        tcg_out_movi (s, TCG_TYPE_I32, args[0], args[1]);
+        tcg_out_movi(s, TCG_TYPE_I32, args[0], args[1]);
         break;
     case INDEX_op_movi_i64:
-        tcg_out_movi (s, TCG_TYPE_I64, args[0], args[1]);
+        tcg_out_movi(s, TCG_TYPE_I64, args[0], args[1]);
         break;
     case INDEX_op_ld8u_i32:
     case INDEX_op_ld8u_i64:
-        tcg_out_ldst (s, args[0], args[1], args[2], LBZ, LBZX);
+        tcg_out_mem_long(s, LBZ, LBZX, args[0], args[1], args[2]);
         break;
     case INDEX_op_ld8s_i32:
     case INDEX_op_ld8s_i64:
-        tcg_out_ldst (s, args[0], args[1], args[2], LBZ, LBZX);
-        tcg_out32 (s, EXTSB | RS (args[0]) | RA (args[0]));
+        tcg_out_mem_long(s, LBZ, LBZX, args[0], args[1], args[2]);
+        tcg_out32(s, EXTSB | RS(args[0]) | RA(args[0]));
         break;
     case INDEX_op_ld16u_i32:
     case INDEX_op_ld16u_i64:
-        tcg_out_ldst (s, args[0], args[1], args[2], LHZ, LHZX);
+        tcg_out_mem_long(s, LHZ, LHZX, args[0], args[1], args[2]);
         break;
     case INDEX_op_ld16s_i32:
     case INDEX_op_ld16s_i64:
-        tcg_out_ldst (s, args[0], args[1], args[2], LHA, LHAX);
+        tcg_out_mem_long(s, LHA, LHAX, args[0], args[1], args[2]);
         break;
     case INDEX_op_ld_i32:
     case INDEX_op_ld32u_i64:
-        tcg_out_ldst (s, args[0], args[1], args[2], LWZ, LWZX);
+        tcg_out_mem_long(s, LWZ, LWZX, args[0], args[1], args[2]);
         break;
     case INDEX_op_ld32s_i64:
-        tcg_out_ldsta (s, args[0], args[1], args[2], LWA, LWAX);
+        tcg_out_mem_long(s, LWA, LWAX, args[0], args[1], args[2]);
         break;
     case INDEX_op_ld_i64:
-        tcg_out_ldsta (s, args[0], args[1], args[2], LD, LDX);
+        tcg_out_mem_long(s, LD, LDX, args[0], args[1], args[2]);
         break;
     case INDEX_op_st8_i32:
     case INDEX_op_st8_i64:
-        tcg_out_ldst (s, args[0], args[1], args[2], STB, STBX);
+        tcg_out_mem_long(s, STB, STBX, args[0], args[1], args[2]);
         break;
     case INDEX_op_st16_i32:
     case INDEX_op_st16_i64:
-        tcg_out_ldst (s, args[0], args[1], args[2], STH, STHX);
+        tcg_out_mem_long(s, STH, STHX, args[0], args[1], args[2]);
         break;
     case INDEX_op_st_i32:
     case INDEX_op_st32_i64:
-        tcg_out_ldst (s, args[0], args[1], args[2], STW, STWX);
+        tcg_out_mem_long(s, STW, STWX, args[0], args[1], args[2]);
         break;
     case INDEX_op_st_i64:
-        tcg_out_ldsta (s, args[0], args[1], args[2], STD, STDX);
+        tcg_out_mem_long(s, STD, STDX, args[0], args[1], args[2]);
         break;
 
     case INDEX_op_add_i32:
         a0 = args[0], a1 = args[1], a2 = args[2];
         if (const_args[2]) {
-            int32_t l, h;
         do_addi_32:
-            l = (int16_t)a2;
-            h = a2 - l;
-            if (h) {
-                tcg_out32(s, ADDIS | TAI(a0, a1, h >> 16));
-                a1 = a0;
-            }
-            if (l || a0 != a1) {
-                tcg_out32(s, ADDI | TAI(a0, a1, l));
-            }
+            tcg_out_mem_long(s, ADDI, ADD, a0, a1, (int32_t)a2);
         } else {
             tcg_out32(s, ADD | TAB(a0, a1, a2));
         }
@@ -1607,32 +1684,33 @@ static void tcg_out_op (TCGContext *s, TCGOpcode opc, const TCGArg *args,
         break;
 
     case INDEX_op_div_i32:
-        tcg_out32 (s, DIVW | TAB (args[0], args[1], args[2]));
+        tcg_out32(s, DIVW | TAB(args[0], args[1], args[2]));
         break;
 
     case INDEX_op_divu_i32:
-        tcg_out32 (s, DIVWU | TAB (args[0], args[1], args[2]));
+        tcg_out32(s, DIVWU | TAB(args[0], args[1], args[2]));
         break;
 
     case INDEX_op_shl_i32:
         if (const_args[2]) {
             tcg_out_rlw(s, RLWINM, args[0], args[1], args[2], 0, 31 - args[2]);
         } else {
-            tcg_out32 (s, SLW | SAB (args[1], args[0], args[2]));
+            tcg_out32(s, SLW | SAB(args[1], args[0], args[2]));
         }
         break;
     case INDEX_op_shr_i32:
         if (const_args[2]) {
             tcg_out_rlw(s, RLWINM, args[0], args[1], 32 - args[2], args[2], 31);
         } else {
-            tcg_out32 (s, SRW | SAB (args[1], args[0], args[2]));
+            tcg_out32(s, SRW | SAB(args[1], args[0], args[2]));
         }
         break;
     case INDEX_op_sar_i32:
-        if (const_args[2])
-            tcg_out32 (s, SRAWI | RS (args[1]) | RA (args[0]) | SH (args[2]));
-        else
-            tcg_out32 (s, SRAW | SAB (args[1], args[0], args[2]));
+        if (const_args[2]) {
+            tcg_out32(s, SRAWI | RS(args[1]) | RA(args[0]) | SH(args[2]));
+        } else {
+            tcg_out32(s, SRAW | SAB(args[1], args[0], args[2]));
+        }
         break;
     case INDEX_op_rotl_i32:
         if (const_args[2]) {
@@ -1646,8 +1724,8 @@ static void tcg_out_op (TCGContext *s, TCGOpcode opc, const TCGArg *args,
         if (const_args[2]) {
             tcg_out_rlw(s, RLWINM, args[0], args[1], 32 - args[2], 0, 31);
         } else {
-            tcg_out32(s, SUBFIC | TAI(0, args[2], 32));
-            tcg_out32(s, RLWNM | SAB(args[1], args[0], 0)
+            tcg_out32(s, SUBFIC | TAI(TCG_REG_R0, args[2], 32));
+            tcg_out32(s, RLWNM | SAB(args[1], args[0], TCG_REG_R0)
                          | MB(0) | ME(31));
         }
         break;
@@ -1664,43 +1742,19 @@ static void tcg_out_op (TCGContext *s, TCGOpcode opc, const TCGArg *args,
 
     case INDEX_op_neg_i32:
     case INDEX_op_neg_i64:
-        tcg_out32 (s, NEG | RT (args[0]) | RA (args[1]));
+        tcg_out32(s, NEG | RT(args[0]) | RA(args[1]));
         break;
 
     case INDEX_op_not_i32:
     case INDEX_op_not_i64:
-        tcg_out32 (s, NOR | SAB (args[1], args[0], args[1]));
+        tcg_out32(s, NOR | SAB(args[1], args[0], args[1]));
         break;
 
     case INDEX_op_add_i64:
         a0 = args[0], a1 = args[1], a2 = args[2];
         if (const_args[2]) {
-            int32_t l0, h1, h2;
         do_addi_64:
-            /* We can always split any 32-bit signed constant into 3 pieces.
-               Note the positive 0x80000000 coming from the sub_i64 path,
-               handled with the same code we need for eg 0x7fff8000.  */
-            assert(a2 == (int32_t)a2 || a2 == 0x80000000);
-            l0 = (int16_t)a2;
-            h1 = a2 - l0;
-            h2 = 0;
-            if (h1 < 0 && (int64_t)a2 > 0) {
-                h2 = 0x40000000;
-                h1 = a2 - h2 - l0;
-            }
-            assert((TCGArg)h2 + h1 + l0 == a2);
-
-            if (h2) {
-                tcg_out32(s, ADDIS | TAI(a0, a1, h2 >> 16));
-                a1 = a0;
-            }
-            if (h1) {
-                tcg_out32(s, ADDIS | TAI(a0, a1, h1 >> 16));
-                a1 = a0;
-            }
-            if (l0 || a0 != a1) {
-                tcg_out32(s, ADDI | TAI(a0, a1, l0));
-            }
+            tcg_out_mem_long(s, ADDI, ADD, a0, a1, a2);
         } else {
             tcg_out32(s, ADD | TAB(a0, a1, a2));
         }
@@ -1722,24 +1776,26 @@ static void tcg_out_op (TCGContext *s, TCGOpcode opc, const TCGArg *args,
         break;
 
     case INDEX_op_shl_i64:
-        if (const_args[2])
+        if (const_args[2]) {
             tcg_out_shli64(s, args[0], args[1], args[2]);
-        else
-            tcg_out32 (s, SLD | SAB (args[1], args[0], args[2]));
+        } else {
+            tcg_out32(s, SLD | SAB(args[1], args[0], args[2]));
+        }
         break;
     case INDEX_op_shr_i64:
-        if (const_args[2])
+        if (const_args[2]) {
             tcg_out_shri64(s, args[0], args[1], args[2]);
-        else
-            tcg_out32 (s, SRD | SAB (args[1], args[0], args[2]));
+        } else {
+            tcg_out32(s, SRD | SAB(args[1], args[0], args[2]));
+        }
         break;
     case INDEX_op_sar_i64:
         if (const_args[2]) {
-            int sh = SH (args[2] & 0x1f) | (((args[2] >> 5) & 1) << 1);
-            tcg_out32 (s, SRADI | RA (args[0]) | RS (args[1]) | sh);
+            int sh = SH(args[2] & 0x1f) | (((args[2] >> 5) & 1) << 1);
+            tcg_out32(s, SRADI | RA(args[0]) | RS(args[1]) | sh);
+        } else {
+            tcg_out32(s, SRAD | SAB(args[1], args[0], args[2]));
         }
-        else
-            tcg_out32 (s, SRAD | SAB (args[1], args[0], args[2]));
         break;
     case INDEX_op_rotl_i64:
         if (const_args[2]) {
@@ -1752,8 +1808,8 @@ static void tcg_out_op (TCGContext *s, TCGOpcode opc, const TCGArg *args,
         if (const_args[2]) {
             tcg_out_rld(s, RLDICL, args[0], args[1], 64 - args[2], 0);
         } else {
-            tcg_out32(s, SUBFIC | TAI(0, args[2], 64));
-            tcg_out32(s, RLDCL | SAB(args[1], args[0], 0) | MB64(0));
+            tcg_out32(s, SUBFIC | TAI(TCG_REG_R0, args[2], 64));
+            tcg_out32(s, RLDCL | SAB(args[1], args[0], TCG_REG_R0) | MB64(0));
         }
         break;
 
@@ -1766,45 +1822,19 @@ static void tcg_out_op (TCGContext *s, TCGOpcode opc, const TCGArg *args,
         }
         break;
     case INDEX_op_div_i64:
-        tcg_out32 (s, DIVD | TAB (args[0], args[1], args[2]));
+        tcg_out32(s, DIVD | TAB(args[0], args[1], args[2]));
         break;
     case INDEX_op_divu_i64:
-        tcg_out32 (s, DIVDU | TAB (args[0], args[1], args[2]));
+        tcg_out32(s, DIVDU | TAB(args[0], args[1], args[2]));
         break;
 
-    case INDEX_op_qemu_ld8u:
-        tcg_out_qemu_ld (s, args, 0);
+    case INDEX_op_qemu_ld_i32:
+    case INDEX_op_qemu_ld_i64:
+        tcg_out_qemu_ld(s, args[0], args[1], args[2], args[3]);
         break;
-    case INDEX_op_qemu_ld8s:
-        tcg_out_qemu_ld (s, args, 0 | 4);
-        break;
-    case INDEX_op_qemu_ld16u:
-        tcg_out_qemu_ld (s, args, 1);
-        break;
-    case INDEX_op_qemu_ld16s:
-        tcg_out_qemu_ld (s, args, 1 | 4);
-        break;
-    case INDEX_op_qemu_ld32:
-    case INDEX_op_qemu_ld32u:
-        tcg_out_qemu_ld (s, args, 2);
-        break;
-    case INDEX_op_qemu_ld32s:
-        tcg_out_qemu_ld (s, args, 2 | 4);
-        break;
-    case INDEX_op_qemu_ld64:
-        tcg_out_qemu_ld (s, args, 3);
-        break;
-    case INDEX_op_qemu_st8:
-        tcg_out_qemu_st (s, args, 0);
-        break;
-    case INDEX_op_qemu_st16:
-        tcg_out_qemu_st (s, args, 1);
-        break;
-    case INDEX_op_qemu_st32:
-        tcg_out_qemu_st (s, args, 2);
-        break;
-    case INDEX_op_qemu_st64:
-        tcg_out_qemu_st (s, args, 3);
+    case INDEX_op_qemu_st_i32:
+    case INDEX_op_qemu_st_i64:
+        tcg_out_qemu_st(s, args[0], args[1], args[2], args[3]);
         break;
 
     case INDEX_op_ext8s_i32:
@@ -1819,16 +1849,16 @@ static void tcg_out_op (TCGContext *s, TCGOpcode opc, const TCGArg *args,
         c = EXTSW;
         goto gen_ext;
     gen_ext:
-        tcg_out32 (s, c | RS (args[1]) | RA (args[0]));
+        tcg_out32(s, c | RS(args[1]) | RA(args[0]));
         break;
 
     case INDEX_op_setcond_i32:
-        tcg_out_setcond (s, TCG_TYPE_I32, args[3], args[0], args[1], args[2],
-                         const_args[2]);
+        tcg_out_setcond(s, TCG_TYPE_I32, args[3], args[0], args[1], args[2],
+                        const_args[2]);
         break;
     case INDEX_op_setcond_i64:
-        tcg_out_setcond (s, TCG_TYPE_I64, args[3], args[0], args[1], args[2],
-                         const_args[2]);
+        tcg_out_setcond(s, TCG_TYPE_I64, args[3], args[0], args[1], args[2],
+                        const_args[2]);
         break;
 
     case INDEX_op_bswap16_i32:
@@ -1870,9 +1900,9 @@ static void tcg_out_op (TCGContext *s, TCGOpcode opc, const TCGArg *args,
         break;
 
     case INDEX_op_bswap64_i64:
-        a0 = args[0], a1 = args[1], a2 = 0;
+        a0 = args[0], a1 = args[1], a2 = TCG_REG_R0;
         if (a0 == a1) {
-            a0 = 0;
+            a0 = TCG_REG_R0;
             a2 = a1;
         }
 
@@ -1980,8 +2010,8 @@ static void tcg_out_op (TCGContext *s, TCGOpcode opc, const TCGArg *args,
         break;
 
     default:
-        tcg_dump_ops (s);
-        tcg_abort ();
+        tcg_dump_ops(s);
+        tcg_abort();
     }
 }
 
@@ -2067,19 +2097,10 @@ static const TCGTargetOpDef ppc_op_defs[] = {
     { INDEX_op_neg_i64, { "r", "r" } },
     { INDEX_op_not_i64, { "r", "r" } },
 
-    { INDEX_op_qemu_ld8u, { "r", "L" } },
-    { INDEX_op_qemu_ld8s, { "r", "L" } },
-    { INDEX_op_qemu_ld16u, { "r", "L" } },
-    { INDEX_op_qemu_ld16s, { "r", "L" } },
-    { INDEX_op_qemu_ld32, { "r", "L" } },
-    { INDEX_op_qemu_ld32u, { "r", "L" } },
-    { INDEX_op_qemu_ld32s, { "r", "L" } },
-    { INDEX_op_qemu_ld64, { "r", "L" } },
-
-    { INDEX_op_qemu_st8, { "S", "S" } },
-    { INDEX_op_qemu_st16, { "S", "S" } },
-    { INDEX_op_qemu_st32, { "S", "S" } },
-    { INDEX_op_qemu_st64, { "S", "S" } },
+    { INDEX_op_qemu_ld_i32, { "r", "L" } },
+    { INDEX_op_qemu_ld_i64, { "r", "L" } },
+    { INDEX_op_qemu_st_i32, { "S", "S" } },
+    { INDEX_op_qemu_st_i64, { "S", "S" } },
 
     { INDEX_op_ext8s_i32, { "r", "r" } },
     { INDEX_op_ext16s_i32, { "r", "r" } },
@@ -2109,7 +2130,7 @@ static const TCGTargetOpDef ppc_op_defs[] = {
     { -1 },
 };
 
-static void tcg_target_init (TCGContext *s)
+static void tcg_target_init(TCGContext *s)
 {
 #ifdef CONFIG_GETAUXVAL
     unsigned long hwcap = getauxval(AT_HWCAP);
@@ -2118,13 +2139,11 @@ static void tcg_target_init (TCGContext *s)
     }
 #endif
 
-    tcg_regset_set32 (tcg_target_available_regs[TCG_TYPE_I32], 0, 0xffffffff);
-    tcg_regset_set32 (tcg_target_available_regs[TCG_TYPE_I64], 0, 0xffffffff);
-    tcg_regset_set32 (tcg_target_call_clobber_regs, 0,
+    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xffffffff);
+    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I64], 0, 0xffffffff);
+    tcg_regset_set32(tcg_target_call_clobber_regs, 0,
                      (1 << TCG_REG_R0) |
-#ifdef __APPLE__
                      (1 << TCG_REG_R2) |
-#endif
                      (1 << TCG_REG_R3) |
                      (1 << TCG_REG_R4) |
                      (1 << TCG_REG_R5) |
@@ -2134,16 +2153,65 @@ static void tcg_target_init (TCGContext *s)
                      (1 << TCG_REG_R9) |
                      (1 << TCG_REG_R10) |
                      (1 << TCG_REG_R11) |
-                     (1 << TCG_REG_R12)
-        );
+                     (1 << TCG_REG_R12));
 
-    tcg_regset_clear (s->reserved_regs);
-    tcg_regset_set_reg (s->reserved_regs, TCG_REG_R0);
-    tcg_regset_set_reg (s->reserved_regs, TCG_REG_R1);
-#ifndef __APPLE__
-    tcg_regset_set_reg (s->reserved_regs, TCG_REG_R2);
+    tcg_regset_clear(s->reserved_regs);
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R0); /* tcg temp */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R1); /* stack pointer */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R2); /* mem temp */
+#ifdef __APPLE__
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R11); /* ??? */
 #endif
-    tcg_regset_set_reg (s->reserved_regs, TCG_REG_R13);
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R13); /* thread pointer */
+
+    tcg_add_target_add_op_defs(ppc_op_defs);
+}
+
+typedef struct {
+    DebugFrameCIE cie;
+    DebugFrameFDEHeader fde;
+    uint8_t fde_def_cfa[4];
+    uint8_t fde_reg_ofs[ARRAY_SIZE(tcg_target_callee_save_regs) * 2 + 3];
+} DebugFrame;
+
+/* We're expecting a 2 byte uleb128 encoded value.  */
+QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
+
+#define ELF_HOST_MACHINE EM_PPC64
+
+static DebugFrame debug_frame = {
+    .cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
+    .cie.id = -1,
+    .cie.version = 1,
+    .cie.code_align = 1,
+    .cie.data_align = 0x78,             /* sleb128 -8 */
+    .cie.return_column = 65,
+
+    /* Total FDE size does not include the "len" member.  */
+    .fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, fde.cie_offset),
+
+    .fde_def_cfa = {
+        12, 1,                          /* DW_CFA_def_cfa r1, ... */
+        (FRAME_SIZE & 0x7f) | 0x80,     /* ... uleb128 FRAME_SIZE */
+        (FRAME_SIZE >> 7)
+    },
+    .fde_reg_ofs = {
+        0x11, 65, 0x7e,                 /* DW_CFA_offset_extended_sf, lr, 16 */
+    }
+};
+
+void tcg_register_jit(void *buf, size_t buf_size)
+{
+    uint8_t *p = &debug_frame.fde_reg_ofs[3];
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i, p += 2) {
+        p[0] = 0x80 + tcg_target_callee_save_regs[i];
+        p[1] = (FRAME_SIZE - (REG_SAVE_BOT + i * 8)) / 8;
+    }
+
+    debug_frame.fde.func_start = (tcg_target_long) buf;
+    debug_frame.fde.func_len = buf_size;
 
-    tcg_add_target_add_op_defs (ppc_op_defs);
+    tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
 }
diff --git a/tcg/ppc64/tcg-target.h b/tcg/ppc64/tcg-target.h
index fa4b9da093..7ee50b6c6c 100644
--- a/tcg/ppc64/tcg-target.h
+++ b/tcg/ppc64/tcg-target.h
@@ -123,6 +123,8 @@ typedef enum {
 #define TCG_TARGET_HAS_muluh_i64        1
 #define TCG_TARGET_HAS_mulsh_i64        1
 
+#define TCG_TARGET_HAS_new_ldst         1
+
 #define TCG_AREG0 TCG_REG_R27
 
 #define TCG_TARGET_EXTEND_ARGS 1
diff --git a/tcg/s390/tcg-target.c b/tcg/s390/tcg-target.c
index 1b44aeee96..0a4f3be0e9 100644
--- a/tcg/s390/tcg-target.c
+++ b/tcg/s390/tcg-target.c
@@ -24,6 +24,8 @@
  * THE SOFTWARE.
  */
 
+#include "tcg-be-null.h"
+
 /* We only support generating code for 64-bit mode.  */
 #if TCG_TARGET_REG_BITS != 64
 #error "unsupported code generation mode"
diff --git a/tcg/s390/tcg-target.h b/tcg/s390/tcg-target.h
index 6142fb26a2..10adb778c7 100644
--- a/tcg/s390/tcg-target.h
+++ b/tcg/s390/tcg-target.h
@@ -99,6 +99,8 @@ typedef enum TCGReg {
 #define TCG_TARGET_HAS_muluh_i64        0
 #define TCG_TARGET_HAS_mulsh_i64        0
 
+#define TCG_TARGET_HAS_new_ldst         0
+
 extern bool tcg_target_deposit_valid(int ofs, int len);
 #define TCG_TARGET_deposit_i32_valid  tcg_target_deposit_valid
 #define TCG_TARGET_deposit_i64_valid  tcg_target_deposit_valid
diff --git a/tcg/sparc/tcg-target.c b/tcg/sparc/tcg-target.c
index 9574954ac4..cbd1c91779 100644
--- a/tcg/sparc/tcg-target.c
+++ b/tcg/sparc/tcg-target.c
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "tcg-be-null.h"
+
 #ifndef NDEBUG
 static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
     "%g0",
diff --git a/tcg/sparc/tcg-target.h b/tcg/sparc/tcg-target.h
index 2edf858733..00f3a1848b 100644
--- a/tcg/sparc/tcg-target.h
+++ b/tcg/sparc/tcg-target.h
@@ -148,12 +148,14 @@ typedef enum {
 #define TCG_TARGET_HAS_mulsh_i64        0
 #endif
 
+#define TCG_TARGET_HAS_new_ldst         0
+
 #define TCG_AREG0 TCG_REG_I0
 
 static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
 {
     uintptr_t p;
-    for (p = start & -8; p < (stop + 7) & -8; p += 8) {
+    for (p = start & -8; p < ((stop + 7) & -8); p += 8) {
         __asm__ __volatile__("flush\t%0" : : "r" (p));
     }
 }
diff --git a/tcg/tcg-be-ldst.h b/tcg/tcg-be-ldst.h
new file mode 100644
index 0000000000..284db0c70d
--- /dev/null
+++ b/tcg/tcg-be-ldst.h
@@ -0,0 +1,90 @@
+/*
+ * TCG Backend Data: load-store optimization only.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifdef CONFIG_SOFTMMU
+#define TCG_MAX_QEMU_LDST       640
+
+typedef struct TCGLabelQemuLdst {
+    int is_ld:1;            /* qemu_ld: 1, qemu_st: 0 */
+    TCGMemOp opc:4;
+    TCGReg addrlo_reg;      /* reg index for low word of guest virtual addr */
+    TCGReg addrhi_reg;      /* reg index for high word of guest virtual addr */
+    TCGReg datalo_reg;      /* reg index for low word to be loaded or stored */
+    TCGReg datahi_reg;      /* reg index for high word to be loaded or stored */
+    int mem_index;          /* soft MMU memory index */
+    uint8_t *raddr;         /* gen code addr of the next IR of qemu_ld/st IR */
+    uint8_t *label_ptr[2];  /* label pointers to be updated */
+} TCGLabelQemuLdst;
+
+typedef struct TCGBackendData {
+    int nb_ldst_labels;
+    TCGLabelQemuLdst ldst_labels[TCG_MAX_QEMU_LDST];
+} TCGBackendData;
+
+
+/*
+ * Initialize TB backend data at the beginning of the TB.
+ */
+
+static inline void tcg_out_tb_init(TCGContext *s)
+{
+    s->be->nb_ldst_labels = 0;
+}
+
+/*
+ * Generate TB finalization at the end of block
+ */
+
+static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l);
+static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l);
+
+static void tcg_out_tb_finalize(TCGContext *s)
+{
+    TCGLabelQemuLdst *lb = s->be->ldst_labels;
+    int i, n = s->be->nb_ldst_labels;
+
+    /* qemu_ld/st slow paths */
+    for (i = 0; i < n; i++) {
+        if (lb[i].is_ld) {
+            tcg_out_qemu_ld_slow_path(s, lb + i);
+        } else {
+            tcg_out_qemu_st_slow_path(s, lb + i);
+        }
+    }
+}
+
+/*
+ * Allocate a new TCGLabelQemuLdst entry.
+ */
+
+static inline TCGLabelQemuLdst *new_ldst_label(TCGContext *s)
+{
+    TCGBackendData *be = s->be;
+    int n = be->nb_ldst_labels;
+
+    assert(n < TCG_MAX_QEMU_LDST);
+    be->nb_ldst_labels = n + 1;
+    return &be->ldst_labels[n];
+}
+#else
+#include "tcg-be-null.h"
+#endif /* CONFIG_SOFTMMU */
diff --git a/tcg/tcg-be-null.h b/tcg/tcg-be-null.h
new file mode 100644
index 0000000000..74c57d5a6c
--- /dev/null
+++ b/tcg/tcg-be-null.h
@@ -0,0 +1,43 @@
+/*
+ * TCG Backend Data: No backend data
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+typedef struct TCGBackendData {
+    /* Empty */
+    char dummy;
+} TCGBackendData;
+
+
+/*
+ * Initialize TB backend data at the beginning of the TB.
+ */
+
+static inline void tcg_out_tb_init(TCGContext *s)
+{
+}
+
+/*
+ * Generate TB finalization at the end of block
+ */
+
+static inline void tcg_out_tb_finalize(TCGContext *s)
+{
+}
diff --git a/tcg/tcg-op.h b/tcg/tcg-op.h
index bb30a7cf39..7eabf22f01 100644
--- a/tcg/tcg-op.h
+++ b/tcg/tcg-op.h
@@ -137,24 +137,6 @@ static inline void tcg_gen_ldst_op_i64(TCGOpcode opc, TCGv_i64 val,
     *tcg_ctx.gen_opparam_ptr++ = offset;
 }
 
-static inline void tcg_gen_qemu_ldst_op_i64_i32(TCGOpcode opc, TCGv_i64 val,
-                                                TCGv_i32 addr, TCGArg mem_index)
-{
-    *tcg_ctx.gen_opc_ptr++ = opc;
-    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(val);
-    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(addr);
-    *tcg_ctx.gen_opparam_ptr++ = mem_index;
-}
-
-static inline void tcg_gen_qemu_ldst_op_i64_i64(TCGOpcode opc, TCGv_i64 val,
-                                                TCGv_i64 addr, TCGArg mem_index)
-{
-    *tcg_ctx.gen_opc_ptr++ = opc;
-    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(val);
-    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(addr);
-    *tcg_ctx.gen_opparam_ptr++ = mem_index;
-}
-
 static inline void tcg_gen_op4_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                    TCGv_i32 arg3, TCGv_i32 arg4)
 {
@@ -361,6 +343,21 @@ static inline void tcg_gen_op6ii_i64(TCGOpcode opc, TCGv_i64 arg1,
     *tcg_ctx.gen_opparam_ptr++ = arg6;
 }
 
+static inline void tcg_add_param_i32(TCGv_i32 val)
+{
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(val);
+}
+
+static inline void tcg_add_param_i64(TCGv_i64 val)
+{
+#if TCG_TARGET_REG_BITS == 32
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(TCGV_LOW(val));
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(TCGV_HIGH(val));
+#else
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(val);
+#endif
+}
+
 static inline void gen_set_label(int n)
 {
     tcg_gen_op1i(INDEX_op_set_label, n);
@@ -2600,11 +2597,12 @@ static inline void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh,
 #define tcg_global_mem_new tcg_global_mem_new_i32
 #define tcg_temp_local_new() tcg_temp_local_new_i32()
 #define tcg_temp_free tcg_temp_free_i32
-#define tcg_gen_qemu_ldst_op tcg_gen_op3i_i32
-#define tcg_gen_qemu_ldst_op_i64 tcg_gen_qemu_ldst_op_i64_i32
 #define TCGV_UNUSED(x) TCGV_UNUSED_I32(x)
 #define TCGV_IS_UNUSED(x) TCGV_IS_UNUSED_I32(x)
 #define TCGV_EQUAL(a, b) TCGV_EQUAL_I32(a, b)
+#define tcg_add_param_tl tcg_add_param_i32
+#define tcg_gen_qemu_ld_tl tcg_gen_qemu_ld_i32
+#define tcg_gen_qemu_st_tl tcg_gen_qemu_st_i32
 #else
 #define TCGv TCGv_i64
 #define tcg_temp_new() tcg_temp_new_i64()
@@ -2612,11 +2610,12 @@ static inline void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh,
 #define tcg_global_mem_new tcg_global_mem_new_i64
 #define tcg_temp_local_new() tcg_temp_local_new_i64()
 #define tcg_temp_free tcg_temp_free_i64
-#define tcg_gen_qemu_ldst_op tcg_gen_op3i_i64
-#define tcg_gen_qemu_ldst_op_i64 tcg_gen_qemu_ldst_op_i64_i64
 #define TCGV_UNUSED(x) TCGV_UNUSED_I64(x)
 #define TCGV_IS_UNUSED(x) TCGV_IS_UNUSED_I64(x)
 #define TCGV_EQUAL(a, b) TCGV_EQUAL_I64(a, b)
+#define tcg_add_param_tl tcg_add_param_i64
+#define tcg_gen_qemu_ld_tl tcg_gen_qemu_ld_i64
+#define tcg_gen_qemu_st_tl tcg_gen_qemu_st_i64
 #endif
 
 /* debug info: write the PC of the corresponding QEMU CPU instruction */
@@ -2648,197 +2647,67 @@ static inline void tcg_gen_goto_tb(unsigned idx)
     tcg_gen_op1i(INDEX_op_goto_tb, idx);
 }
 
-#if TCG_TARGET_REG_BITS == 32
-static inline void tcg_gen_qemu_ld8u(TCGv ret, TCGv addr, int mem_index)
-{
-#if TARGET_LONG_BITS == 32
-    tcg_gen_op3i_i32(INDEX_op_qemu_ld8u, ret, addr, mem_index);
-#else
-    tcg_gen_op4i_i32(INDEX_op_qemu_ld8u, TCGV_LOW(ret), TCGV_LOW(addr),
-                     TCGV_HIGH(addr), mem_index);
-    tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
-#endif
-}
-
-static inline void tcg_gen_qemu_ld8s(TCGv ret, TCGv addr, int mem_index)
-{
-#if TARGET_LONG_BITS == 32
-    tcg_gen_op3i_i32(INDEX_op_qemu_ld8s, ret, addr, mem_index);
-#else
-    tcg_gen_op4i_i32(INDEX_op_qemu_ld8s, TCGV_LOW(ret), TCGV_LOW(addr),
-                     TCGV_HIGH(addr), mem_index);
-    tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
-#endif
-}
 
-static inline void tcg_gen_qemu_ld16u(TCGv ret, TCGv addr, int mem_index)
-{
-#if TARGET_LONG_BITS == 32
-    tcg_gen_op3i_i32(INDEX_op_qemu_ld16u, ret, addr, mem_index);
-#else
-    tcg_gen_op4i_i32(INDEX_op_qemu_ld16u, TCGV_LOW(ret), TCGV_LOW(addr),
-                     TCGV_HIGH(addr), mem_index);
-    tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
-#endif
-}
-
-static inline void tcg_gen_qemu_ld16s(TCGv ret, TCGv addr, int mem_index)
-{
-#if TARGET_LONG_BITS == 32
-    tcg_gen_op3i_i32(INDEX_op_qemu_ld16s, ret, addr, mem_index);
-#else
-    tcg_gen_op4i_i32(INDEX_op_qemu_ld16s, TCGV_LOW(ret), TCGV_LOW(addr),
-                     TCGV_HIGH(addr), mem_index);
-    tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
-#endif
-}
-
-static inline void tcg_gen_qemu_ld32u(TCGv ret, TCGv addr, int mem_index)
-{
-#if TARGET_LONG_BITS == 32
-    tcg_gen_op3i_i32(INDEX_op_qemu_ld32, ret, addr, mem_index);
-#else
-    tcg_gen_op4i_i32(INDEX_op_qemu_ld32, TCGV_LOW(ret), TCGV_LOW(addr),
-                     TCGV_HIGH(addr), mem_index);
-    tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
-#endif
-}
-
-static inline void tcg_gen_qemu_ld32s(TCGv ret, TCGv addr, int mem_index)
-{
-#if TARGET_LONG_BITS == 32
-    tcg_gen_op3i_i32(INDEX_op_qemu_ld32, ret, addr, mem_index);
-#else
-    tcg_gen_op4i_i32(INDEX_op_qemu_ld32, TCGV_LOW(ret), TCGV_LOW(addr),
-                     TCGV_HIGH(addr), mem_index);
-    tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
-#endif
-}
-
-static inline void tcg_gen_qemu_ld64(TCGv_i64 ret, TCGv addr, int mem_index)
-{
-#if TARGET_LONG_BITS == 32
-    tcg_gen_op4i_i32(INDEX_op_qemu_ld64, TCGV_LOW(ret), TCGV_HIGH(ret), addr, mem_index);
-#else
-    tcg_gen_op5i_i32(INDEX_op_qemu_ld64, TCGV_LOW(ret), TCGV_HIGH(ret),
-                     TCGV_LOW(addr), TCGV_HIGH(addr), mem_index);
-#endif
-}
-
-static inline void tcg_gen_qemu_st8(TCGv arg, TCGv addr, int mem_index)
-{
-#if TARGET_LONG_BITS == 32
-    tcg_gen_op3i_i32(INDEX_op_qemu_st8, arg, addr, mem_index);
-#else
-    tcg_gen_op4i_i32(INDEX_op_qemu_st8, TCGV_LOW(arg), TCGV_LOW(addr),
-                     TCGV_HIGH(addr), mem_index);
-#endif
-}
-
-static inline void tcg_gen_qemu_st16(TCGv arg, TCGv addr, int mem_index)
-{
-#if TARGET_LONG_BITS == 32
-    tcg_gen_op3i_i32(INDEX_op_qemu_st16, arg, addr, mem_index);
-#else
-    tcg_gen_op4i_i32(INDEX_op_qemu_st16, TCGV_LOW(arg), TCGV_LOW(addr),
-                     TCGV_HIGH(addr), mem_index);
-#endif
-}
-
-static inline void tcg_gen_qemu_st32(TCGv arg, TCGv addr, int mem_index)
-{
-#if TARGET_LONG_BITS == 32
-    tcg_gen_op3i_i32(INDEX_op_qemu_st32, arg, addr, mem_index);
-#else
-    tcg_gen_op4i_i32(INDEX_op_qemu_st32, TCGV_LOW(arg), TCGV_LOW(addr),
-                     TCGV_HIGH(addr), mem_index);
-#endif
-}
-
-static inline void tcg_gen_qemu_st64(TCGv_i64 arg, TCGv addr, int mem_index)
-{
-#if TARGET_LONG_BITS == 32
-    tcg_gen_op4i_i32(INDEX_op_qemu_st64, TCGV_LOW(arg), TCGV_HIGH(arg), addr,
-                     mem_index);
-#else
-    tcg_gen_op5i_i32(INDEX_op_qemu_st64, TCGV_LOW(arg), TCGV_HIGH(arg),
-                     TCGV_LOW(addr), TCGV_HIGH(addr), mem_index);
-#endif
-}
-
-#define tcg_gen_ld_ptr(R, A, O) tcg_gen_ld_i32(TCGV_PTR_TO_NAT(R), (A), (O))
-#define tcg_gen_discard_ptr(A) tcg_gen_discard_i32(TCGV_PTR_TO_NAT(A))
-
-#else /* TCG_TARGET_REG_BITS == 32 */
+void tcg_gen_qemu_ld_i32(TCGv_i32, TCGv, TCGArg, TCGMemOp);
+void tcg_gen_qemu_st_i32(TCGv_i32, TCGv, TCGArg, TCGMemOp);
+void tcg_gen_qemu_ld_i64(TCGv_i64, TCGv, TCGArg, TCGMemOp);
+void tcg_gen_qemu_st_i64(TCGv_i64, TCGv, TCGArg, TCGMemOp);
 
 static inline void tcg_gen_qemu_ld8u(TCGv ret, TCGv addr, int mem_index)
 {
-    tcg_gen_qemu_ldst_op(INDEX_op_qemu_ld8u, ret, addr, mem_index);
+    tcg_gen_qemu_ld_tl(ret, addr, mem_index, MO_UB);
 }
 
 static inline void tcg_gen_qemu_ld8s(TCGv ret, TCGv addr, int mem_index)
 {
-    tcg_gen_qemu_ldst_op(INDEX_op_qemu_ld8s, ret, addr, mem_index);
+    tcg_gen_qemu_ld_tl(ret, addr, mem_index, MO_SB);
 }
 
 static inline void tcg_gen_qemu_ld16u(TCGv ret, TCGv addr, int mem_index)
 {
-    tcg_gen_qemu_ldst_op(INDEX_op_qemu_ld16u, ret, addr, mem_index);
+    tcg_gen_qemu_ld_tl(ret, addr, mem_index, MO_TEUW);
 }
 
 static inline void tcg_gen_qemu_ld16s(TCGv ret, TCGv addr, int mem_index)
 {
-    tcg_gen_qemu_ldst_op(INDEX_op_qemu_ld16s, ret, addr, mem_index);
+    tcg_gen_qemu_ld_tl(ret, addr, mem_index, MO_TESW);
 }
 
 static inline void tcg_gen_qemu_ld32u(TCGv ret, TCGv addr, int mem_index)
 {
-#if TARGET_LONG_BITS == 32
-    tcg_gen_qemu_ldst_op(INDEX_op_qemu_ld32, ret, addr, mem_index);
-#else
-    tcg_gen_qemu_ldst_op(INDEX_op_qemu_ld32u, ret, addr, mem_index);
-#endif
+    tcg_gen_qemu_ld_tl(ret, addr, mem_index, MO_TEUL);
 }
 
 static inline void tcg_gen_qemu_ld32s(TCGv ret, TCGv addr, int mem_index)
 {
-#if TARGET_LONG_BITS == 32
-    tcg_gen_qemu_ldst_op(INDEX_op_qemu_ld32, ret, addr, mem_index);
-#else
-    tcg_gen_qemu_ldst_op(INDEX_op_qemu_ld32s, ret, addr, mem_index);
-#endif
+    tcg_gen_qemu_ld_tl(ret, addr, mem_index, MO_TESL);
 }
 
 static inline void tcg_gen_qemu_ld64(TCGv_i64 ret, TCGv addr, int mem_index)
 {
-    tcg_gen_qemu_ldst_op_i64(INDEX_op_qemu_ld64, ret, addr, mem_index);
+    tcg_gen_qemu_ld_i64(ret, addr, mem_index, MO_TEQ);
 }
 
 static inline void tcg_gen_qemu_st8(TCGv arg, TCGv addr, int mem_index)
 {
-    tcg_gen_qemu_ldst_op(INDEX_op_qemu_st8, arg, addr, mem_index);
+    tcg_gen_qemu_st_tl(arg, addr, mem_index, MO_UB);
 }
 
 static inline void tcg_gen_qemu_st16(TCGv arg, TCGv addr, int mem_index)
 {
-    tcg_gen_qemu_ldst_op(INDEX_op_qemu_st16, arg, addr, mem_index);
+    tcg_gen_qemu_st_tl(arg, addr, mem_index, MO_TEUW);
 }
 
 static inline void tcg_gen_qemu_st32(TCGv arg, TCGv addr, int mem_index)
 {
-    tcg_gen_qemu_ldst_op(INDEX_op_qemu_st32, arg, addr, mem_index);
+    tcg_gen_qemu_st_tl(arg, addr, mem_index, MO_TEUL);
 }
 
 static inline void tcg_gen_qemu_st64(TCGv_i64 arg, TCGv addr, int mem_index)
 {
-    tcg_gen_qemu_ldst_op_i64(INDEX_op_qemu_st64, arg, addr, mem_index);
+    tcg_gen_qemu_st_i64(arg, addr, mem_index, MO_TEQ);
 }
 
-#define tcg_gen_ld_ptr(R, A, O) tcg_gen_ld_i64(TCGV_PTR_TO_NAT(R), (A), (O))
-#define tcg_gen_discard_ptr(A) tcg_gen_discard_i64(TCGV_PTR_TO_NAT(A))
-
-#endif /* TCG_TARGET_REG_BITS != 32 */
-
 #if TARGET_LONG_BITS == 64
 #define tcg_gen_movi_tl tcg_gen_movi_i64
 #define tcg_gen_mov_tl tcg_gen_mov_i64
@@ -2997,17 +2866,25 @@ static inline void tcg_gen_qemu_st64(TCGv_i64 arg, TCGv addr, int mem_index)
 #endif
 
 #if TCG_TARGET_REG_BITS == 32
-#define tcg_gen_add_ptr(R, A, B) tcg_gen_add_i32(TCGV_PTR_TO_NAT(R), \
-                                               TCGV_PTR_TO_NAT(A), \
-                                               TCGV_PTR_TO_NAT(B))
-#define tcg_gen_addi_ptr(R, A, B) tcg_gen_addi_i32(TCGV_PTR_TO_NAT(R), \
-                                                 TCGV_PTR_TO_NAT(A), (B))
-#define tcg_gen_ext_i32_ptr(R, A) tcg_gen_mov_i32(TCGV_PTR_TO_NAT(R), (A))
-#else /* TCG_TARGET_REG_BITS == 32 */
-#define tcg_gen_add_ptr(R, A, B) tcg_gen_add_i64(TCGV_PTR_TO_NAT(R), \
-                                               TCGV_PTR_TO_NAT(A), \
-                                               TCGV_PTR_TO_NAT(B))
-#define tcg_gen_addi_ptr(R, A, B) tcg_gen_addi_i64(TCGV_PTR_TO_NAT(R),   \
-                                                 TCGV_PTR_TO_NAT(A), (B))
-#define tcg_gen_ext_i32_ptr(R, A) tcg_gen_ext_i32_i64(TCGV_PTR_TO_NAT(R), (A))
-#endif /* TCG_TARGET_REG_BITS != 32 */
+# define tcg_gen_ld_ptr(R, A, O) \
+    tcg_gen_ld_i32(TCGV_PTR_TO_NAT(R), (A), (O))
+# define tcg_gen_discard_ptr(A) \
+    tcg_gen_discard_i32(TCGV_PTR_TO_NAT(A))
+# define tcg_gen_add_ptr(R, A, B) \
+    tcg_gen_add_i32(TCGV_PTR_TO_NAT(R), TCGV_PTR_TO_NAT(A), TCGV_PTR_TO_NAT(B))
+# define tcg_gen_addi_ptr(R, A, B) \
+    tcg_gen_addi_i32(TCGV_PTR_TO_NAT(R), TCGV_PTR_TO_NAT(A), (B))
+# define tcg_gen_ext_i32_ptr(R, A) \
+    tcg_gen_mov_i32(TCGV_PTR_TO_NAT(R), (A))
+#else
+# define tcg_gen_ld_ptr(R, A, O) \
+    tcg_gen_ld_i64(TCGV_PTR_TO_NAT(R), (A), (O))
+# define tcg_gen_discard_ptr(A) \
+    tcg_gen_discard_i64(TCGV_PTR_TO_NAT(A))
+# define tcg_gen_add_ptr(R, A, B) \
+    tcg_gen_add_i64(TCGV_PTR_TO_NAT(R), TCGV_PTR_TO_NAT(A), TCGV_PTR_TO_NAT(B))
+# define tcg_gen_addi_ptr(R, A, B) \
+    tcg_gen_addi_i64(TCGV_PTR_TO_NAT(R), TCGV_PTR_TO_NAT(A), (B))
+# define tcg_gen_ext_i32_ptr(R, A) \
+    tcg_gen_ext_i32_i64(TCGV_PTR_TO_NAT(R), (A))
+#endif /* TCG_TARGET_REG_BITS == 32 */
diff --git a/tcg/tcg-opc.h b/tcg/tcg-opc.h
index a75c29d518..d71707d9bb 100644
--- a/tcg/tcg-opc.h
+++ b/tcg/tcg-opc.h
@@ -180,79 +180,107 @@ DEF(debug_insn_start, 0, 0, 1, TCG_OPF_NOT_PRESENT)
 #endif
 DEF(exit_tb, 0, 0, 1, TCG_OPF_BB_END)
 DEF(goto_tb, 0, 0, 1, TCG_OPF_BB_END)
-/* Note: even if TARGET_LONG_BITS is not defined, the INDEX_op
-   constants must be defined */
+
+#define IMPL_NEW_LDST \
+    (TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS \
+     | IMPL(TCG_TARGET_HAS_new_ldst))
+
+#if TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
+DEF(qemu_ld_i32, 1, 1, 2, IMPL_NEW_LDST)
+DEF(qemu_st_i32, 0, 2, 2, IMPL_NEW_LDST)
+# if TCG_TARGET_REG_BITS == 64
+DEF(qemu_ld_i64, 1, 1, 2, IMPL_NEW_LDST | TCG_OPF_64BIT)
+DEF(qemu_st_i64, 0, 2, 2, IMPL_NEW_LDST | TCG_OPF_64BIT)
+# else
+DEF(qemu_ld_i64, 2, 1, 2, IMPL_NEW_LDST | TCG_OPF_64BIT)
+DEF(qemu_st_i64, 0, 3, 2, IMPL_NEW_LDST | TCG_OPF_64BIT)
+# endif
+#else
+DEF(qemu_ld_i32, 1, 2, 2, IMPL_NEW_LDST)
+DEF(qemu_st_i32, 0, 3, 2, IMPL_NEW_LDST)
+DEF(qemu_ld_i64, 2, 2, 2, IMPL_NEW_LDST | TCG_OPF_64BIT)
+DEF(qemu_st_i64, 0, 4, 2, IMPL_NEW_LDST | TCG_OPF_64BIT)
+#endif
+
+#undef IMPL_NEW_LDST
+
+#define IMPL_OLD_LDST \
+    (TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS \
+     | IMPL(!TCG_TARGET_HAS_new_ldst))
+
 #if TCG_TARGET_REG_BITS == 32
 #if TARGET_LONG_BITS == 32
-DEF(qemu_ld8u, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld8u, 1, 1, 1, IMPL_OLD_LDST)
 #else
-DEF(qemu_ld8u, 1, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld8u, 1, 2, 1, IMPL_OLD_LDST)
 #endif
 #if TARGET_LONG_BITS == 32
-DEF(qemu_ld8s, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld8s, 1, 1, 1, IMPL_OLD_LDST)
 #else
-DEF(qemu_ld8s, 1, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld8s, 1, 2, 1, IMPL_OLD_LDST)
 #endif
 #if TARGET_LONG_BITS == 32
-DEF(qemu_ld16u, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld16u, 1, 1, 1, IMPL_OLD_LDST)
 #else
-DEF(qemu_ld16u, 1, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld16u, 1, 2, 1, IMPL_OLD_LDST)
 #endif
 #if TARGET_LONG_BITS == 32
-DEF(qemu_ld16s, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld16s, 1, 1, 1, IMPL_OLD_LDST)
 #else
-DEF(qemu_ld16s, 1, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld16s, 1, 2, 1, IMPL_OLD_LDST)
 #endif
 #if TARGET_LONG_BITS == 32
-DEF(qemu_ld32, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld32, 1, 1, 1, IMPL_OLD_LDST)
 #else
-DEF(qemu_ld32, 1, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld32, 1, 2, 1, IMPL_OLD_LDST)
 #endif
 #if TARGET_LONG_BITS == 32
-DEF(qemu_ld64, 2, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld64, 2, 1, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
 #else
-DEF(qemu_ld64, 2, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld64, 2, 2, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
 #endif
 
 #if TARGET_LONG_BITS == 32
-DEF(qemu_st8, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_st8, 0, 2, 1, IMPL_OLD_LDST)
 #else
-DEF(qemu_st8, 0, 3, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_st8, 0, 3, 1, IMPL_OLD_LDST)
 #endif
 #if TARGET_LONG_BITS == 32
-DEF(qemu_st16, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_st16, 0, 2, 1, IMPL_OLD_LDST)
 #else
-DEF(qemu_st16, 0, 3, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_st16, 0, 3, 1, IMPL_OLD_LDST)
 #endif
 #if TARGET_LONG_BITS == 32
-DEF(qemu_st32, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_st32, 0, 2, 1, IMPL_OLD_LDST)
 #else
-DEF(qemu_st32, 0, 3, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_st32, 0, 3, 1, IMPL_OLD_LDST)
 #endif
 #if TARGET_LONG_BITS == 32
-DEF(qemu_st64, 0, 3, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_st64, 0, 3, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
 #else
-DEF(qemu_st64, 0, 4, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_st64, 0, 4, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
 #endif
 
 #else /* TCG_TARGET_REG_BITS == 32 */
 
-DEF(qemu_ld8u, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-DEF(qemu_ld8s, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-DEF(qemu_ld16u, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-DEF(qemu_ld16s, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-DEF(qemu_ld32, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-DEF(qemu_ld32u, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-DEF(qemu_ld32s, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-DEF(qemu_ld64, 1, 1, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld8u, 1, 1, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
+DEF(qemu_ld8s, 1, 1, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
+DEF(qemu_ld16u, 1, 1, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
+DEF(qemu_ld16s, 1, 1, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
+DEF(qemu_ld32, 1, 1, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
+DEF(qemu_ld32u, 1, 1, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
+DEF(qemu_ld32s, 1, 1, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
+DEF(qemu_ld64, 1, 1, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
 
-DEF(qemu_st8, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-DEF(qemu_st16, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-DEF(qemu_st32, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
-DEF(qemu_st64, 0, 2, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_st8, 0, 2, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
+DEF(qemu_st16, 0, 2, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
+DEF(qemu_st32, 0, 2, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
+DEF(qemu_st64, 0, 2, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
 
 #endif /* TCG_TARGET_REG_BITS != 32 */
 
+#undef IMPL_OLD_LDST
+
 #undef IMPL
 #undef IMPL64
 #undef DEF
diff --git a/tcg/tcg.c b/tcg/tcg.c
index fd7fb6b85e..66d3f3de80 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -103,6 +103,9 @@ static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
                        intptr_t arg2);
 static int tcg_target_const_match(tcg_target_long val,
                                   const TCGArgConstraint *arg_ct);
+static void tcg_out_tb_init(TCGContext *s);
+static void tcg_out_tb_finalize(TCGContext *s);
+
 
 TCGOpDef tcg_op_defs[] = {
 #define DEF(s, oargs, iargs, cargs, flags) { #s, oargs, iargs, cargs, iargs + oargs + cargs, flags },
@@ -254,12 +257,41 @@ void tcg_pool_reset(TCGContext *s)
     s->pool_current = NULL;
 }
 
+#include "helper.h"
+
+typedef struct TCGHelperInfo {
+    void *func;
+    const char *name;
+} TCGHelperInfo;
+
+static const TCGHelperInfo all_helpers[] = {
+#define GEN_HELPER 2
+#include "helper.h"
+
+    /* Include tcg-runtime.c functions.  */
+    { tcg_helper_div_i32, "div_i32" },
+    { tcg_helper_rem_i32, "rem_i32" },
+    { tcg_helper_divu_i32, "divu_i32" },
+    { tcg_helper_remu_i32, "remu_i32" },
+
+    { tcg_helper_shl_i64, "shl_i64" },
+    { tcg_helper_shr_i64, "shr_i64" },
+    { tcg_helper_sar_i64, "sar_i64" },
+    { tcg_helper_div_i64, "div_i64" },
+    { tcg_helper_rem_i64, "rem_i64" },
+    { tcg_helper_divu_i64, "divu_i64" },
+    { tcg_helper_remu_i64, "remu_i64" },
+    { tcg_helper_mulsh_i64, "mulsh_i64" },
+    { tcg_helper_muluh_i64, "muluh_i64" },
+};
+
 void tcg_context_init(TCGContext *s)
 {
-    int op, total_args, n;
+    int op, total_args, n, i;
     TCGOpDef *def;
     TCGArgConstraint *args_ct;
     int *sorted_args;
+    GHashTable *helper_table;
 
     memset(s, 0, sizeof(*s));
     s->nb_globals = 0;
@@ -284,7 +316,16 @@ void tcg_context_init(TCGContext *s)
         sorted_args += n;
         args_ct += n;
     }
-    
+
+    /* Register helpers.  */
+    /* Use g_direct_hash/equal for direct pointer comparisons on func.  */
+    s->helpers = helper_table = g_hash_table_new(NULL, NULL);
+
+    for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
+        g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
+                            (gpointer)all_helpers[i].name);
+    }
+
     tcg_target_init(s);
 }
 
@@ -332,13 +373,7 @@ void tcg_func_start(TCGContext *s)
     s->gen_opc_ptr = s->gen_opc_buf;
     s->gen_opparam_ptr = s->gen_opparam_buf;
 
-#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
-    /* Initialize qemu_ld/st labels to assist code generation at the end of TB
-       for TLB miss cases at the end of TB */
-    s->qemu_ldst_labels = tcg_malloc(sizeof(TCGLabelQemuLdst) *
-                                     TCG_MAX_QEMU_LDST);
-    s->nb_qemu_ldst_labels = 0;
-#endif
+    s->be = tcg_malloc(sizeof(TCGBackendData));
 }
 
 static inline void tcg_temp_alloc(TCGContext *s, int n)
@@ -620,25 +655,6 @@ int tcg_check_temp_count(void)
 }
 #endif
 
-void tcg_register_helper(void *func, const char *name)
-{
-    TCGContext *s = &tcg_ctx;
-    int n;
-    if ((s->nb_helpers + 1) > s->allocated_helpers) {
-        n = s->allocated_helpers;
-        if (n == 0) {
-            n = 4;
-        } else {
-            n *= 2;
-        }
-        s->helpers = realloc(s->helpers, n * sizeof(TCGHelperInfo));
-        s->allocated_helpers = n;
-    }
-    s->helpers[s->nb_helpers].func = (uintptr_t)func;
-    s->helpers[s->nb_helpers].name = name;
-    s->nb_helpers++;
-}
-
 /* Note: we convert the 64 bit args to 32 bit and do some alignment
    and endian swap. Maybe it would be better to do the alignment
    and endian swap in tcg_reg_alloc_call(). */
@@ -795,6 +811,188 @@ void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
 }
 #endif
 
+static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st)
+{
+    switch (op & MO_SIZE) {
+    case MO_8:
+        op &= ~MO_BSWAP;
+        break;
+    case MO_16:
+        break;
+    case MO_32:
+        if (!is64) {
+            op &= ~MO_SIGN;
+        }
+        break;
+    case MO_64:
+        if (!is64) {
+            tcg_abort();
+        }
+        break;
+    }
+    if (st) {
+        op &= ~MO_SIGN;
+    }
+    return op;
+}
+
+static const TCGOpcode old_ld_opc[8] = {
+    [MO_UB] = INDEX_op_qemu_ld8u,
+    [MO_SB] = INDEX_op_qemu_ld8s,
+    [MO_UW] = INDEX_op_qemu_ld16u,
+    [MO_SW] = INDEX_op_qemu_ld16s,
+#if TCG_TARGET_REG_BITS == 32
+    [MO_UL] = INDEX_op_qemu_ld32,
+    [MO_SL] = INDEX_op_qemu_ld32,
+#else
+    [MO_UL] = INDEX_op_qemu_ld32u,
+    [MO_SL] = INDEX_op_qemu_ld32s,
+#endif
+    [MO_Q]  = INDEX_op_qemu_ld64,
+};
+
+static const TCGOpcode old_st_opc[4] = {
+    [MO_UB] = INDEX_op_qemu_st8,
+    [MO_UW] = INDEX_op_qemu_st16,
+    [MO_UL] = INDEX_op_qemu_st32,
+    [MO_Q]  = INDEX_op_qemu_st64,
+};
+
+void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
+{
+    memop = tcg_canonicalize_memop(memop, 0, 0);
+
+    if (TCG_TARGET_HAS_new_ldst) {
+        *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_ld_i32;
+        tcg_add_param_i32(val);
+        tcg_add_param_tl(addr);
+        *tcg_ctx.gen_opparam_ptr++ = memop;
+        *tcg_ctx.gen_opparam_ptr++ = idx;
+        return;
+    }
+
+    /* The old opcodes only support target-endian memory operations.  */
+    assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8);
+    assert(old_ld_opc[memop & MO_SSIZE] != 0);
+
+    if (TCG_TARGET_REG_BITS == 32) {
+        *tcg_ctx.gen_opc_ptr++ = old_ld_opc[memop & MO_SSIZE];
+        tcg_add_param_i32(val);
+        tcg_add_param_tl(addr);
+        *tcg_ctx.gen_opparam_ptr++ = idx;
+    } else {
+        TCGv_i64 val64 = tcg_temp_new_i64();
+
+        *tcg_ctx.gen_opc_ptr++ = old_ld_opc[memop & MO_SSIZE];
+        tcg_add_param_i64(val64);
+        tcg_add_param_tl(addr);
+        *tcg_ctx.gen_opparam_ptr++ = idx;
+
+        tcg_gen_trunc_i64_i32(val, val64);
+        tcg_temp_free_i64(val64);
+    }
+}
+
+void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
+{
+    memop = tcg_canonicalize_memop(memop, 0, 1);
+
+    if (TCG_TARGET_HAS_new_ldst) {
+        *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_st_i32;
+        tcg_add_param_i32(val);
+        tcg_add_param_tl(addr);
+        *tcg_ctx.gen_opparam_ptr++ = memop;
+        *tcg_ctx.gen_opparam_ptr++ = idx;
+        return;
+    }
+
+    /* The old opcodes only support target-endian memory operations.  */
+    assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8);
+    assert(old_st_opc[memop & MO_SIZE] != 0);
+
+    if (TCG_TARGET_REG_BITS == 32) {
+        *tcg_ctx.gen_opc_ptr++ = old_st_opc[memop & MO_SIZE];
+        tcg_add_param_i32(val);
+        tcg_add_param_tl(addr);
+        *tcg_ctx.gen_opparam_ptr++ = idx;
+    } else {
+        TCGv_i64 val64 = tcg_temp_new_i64();
+
+        tcg_gen_extu_i32_i64(val64, val);
+
+        *tcg_ctx.gen_opc_ptr++ = old_st_opc[memop & MO_SIZE];
+        tcg_add_param_i64(val64);
+        tcg_add_param_tl(addr);
+        *tcg_ctx.gen_opparam_ptr++ = idx;
+
+        tcg_temp_free_i64(val64);
+    }
+}
+
+void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
+{
+    memop = tcg_canonicalize_memop(memop, 1, 0);
+
+#if TCG_TARGET_REG_BITS == 32
+    if ((memop & MO_SIZE) < MO_64) {
+        tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);
+        if (memop & MO_SIGN) {
+            tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
+        } else {
+            tcg_gen_movi_i32(TCGV_HIGH(val), 0);
+        }
+        return;
+    }
+#endif
+
+    if (TCG_TARGET_HAS_new_ldst) {
+        *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_ld_i64;
+        tcg_add_param_i64(val);
+        tcg_add_param_tl(addr);
+        *tcg_ctx.gen_opparam_ptr++ = memop;
+        *tcg_ctx.gen_opparam_ptr++ = idx;
+        return;
+    }
+
+    /* The old opcodes only support target-endian memory operations.  */
+    assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8);
+    assert(old_ld_opc[memop & MO_SSIZE] != 0);
+
+    *tcg_ctx.gen_opc_ptr++ = old_ld_opc[memop & MO_SSIZE];
+    tcg_add_param_i64(val);
+    tcg_add_param_tl(addr);
+    *tcg_ctx.gen_opparam_ptr++ = idx;
+}
+
+void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
+{
+    memop = tcg_canonicalize_memop(memop, 1, 1);
+
+#if TCG_TARGET_REG_BITS == 32
+    if ((memop & MO_SIZE) < MO_64) {
+        tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);
+        return;
+    }
+#endif
+
+    if (TCG_TARGET_HAS_new_ldst) {
+        *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_st_i64;
+        tcg_add_param_i64(val);
+        tcg_add_param_tl(addr);
+        *tcg_ctx.gen_opparam_ptr++ = memop;
+        *tcg_ctx.gen_opparam_ptr++ = idx;
+        return;
+    }
+
+    /* The old opcodes only support target-endian memory operations.  */
+    assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8);
+    assert(old_st_opc[memop & MO_SIZE] != 0);
+
+    *tcg_ctx.gen_opc_ptr++ = old_st_opc[memop & MO_SIZE];
+    tcg_add_param_i64(val);
+    tcg_add_param_tl(addr);
+    *tcg_ctx.gen_opparam_ptr++ = idx;
+}
 
 static void tcg_reg_alloc_start(TCGContext *s)
 {
@@ -851,47 +1049,14 @@ char *tcg_get_arg_str_i64(TCGContext *s, char *buf, int buf_size, TCGv_i64 arg)
     return tcg_get_arg_str_idx(s, buf, buf_size, GET_TCGV_I64(arg));
 }
 
-static int helper_cmp(const void *p1, const void *p2)
+/* Find helper name.  */
+static inline const char *tcg_find_helper(TCGContext *s, uintptr_t val)
 {
-    const TCGHelperInfo *th1 = p1;
-    const TCGHelperInfo *th2 = p2;
-    if (th1->func < th2->func)
-        return -1;
-    else if (th1->func == th2->func)
-        return 0;
-    else
-        return 1;
-}
-
-/* find helper definition (Note: A hash table would be better) */
-static TCGHelperInfo *tcg_find_helper(TCGContext *s, uintptr_t val)
-{
-    int m, m_min, m_max;
-    TCGHelperInfo *th;
-    uintptr_t v;
-
-    if (unlikely(!s->helpers_sorted)) {
-        qsort(s->helpers, s->nb_helpers, sizeof(TCGHelperInfo), 
-              helper_cmp);
-        s->helpers_sorted = 1;
+    const char *ret = NULL;
+    if (s->helpers) {
+        ret = g_hash_table_lookup(s->helpers, (gpointer)val);
     }
-
-    /* binary search */
-    m_min = 0;
-    m_max = s->nb_helpers - 1;
-    while (m_min <= m_max) {
-        m = (m_min + m_max) >> 1;
-        th = &s->helpers[m];
-        v = th->func;
-        if (v == val)
-            return th;
-        else if (val < v) {
-            m_max = m - 1;
-        } else {
-            m_min = m + 1;
-        }
-    }
-    return NULL;
+    return ret;
 }
 
 static const char * const cond_name[] =
@@ -910,6 +1075,22 @@ static const char * const cond_name[] =
     [TCG_COND_GTU] = "gtu"
 };
 
+static const char * const ldst_name[] =
+{
+    [MO_UB]   = "ub",
+    [MO_SB]   = "sb",
+    [MO_LEUW] = "leuw",
+    [MO_LESW] = "lesw",
+    [MO_LEUL] = "leul",
+    [MO_LESL] = "lesl",
+    [MO_LEQ]  = "leq",
+    [MO_BEUW] = "beuw",
+    [MO_BESW] = "besw",
+    [MO_BEUL] = "beul",
+    [MO_BESL] = "besl",
+    [MO_BEQ]  = "beq",
+};
+
 void tcg_dump_ops(TCGContext *s)
 {
     const uint16_t *opc_ptr;
@@ -976,7 +1157,7 @@ void tcg_dump_ops(TCGContext *s)
             }
         } else if (c == INDEX_op_movi_i32 || c == INDEX_op_movi_i64) {
             tcg_target_ulong val;
-            TCGHelperInfo *th;
+            const char *name;
 
             nb_oargs = def->nb_oargs;
             nb_iargs = def->nb_iargs;
@@ -984,9 +1165,9 @@ void tcg_dump_ops(TCGContext *s)
             qemu_log(" %s %s,$", def->name,
                      tcg_get_arg_str_idx(s, buf, sizeof(buf), args[0]));
             val = args[1];
-            th = tcg_find_helper(s, val);
-            if (th) {
-                qemu_log("%s", th->name);
+            name = tcg_find_helper(s, val);
+            if (name) {
+                qemu_log("%s", name);
             } else {
                 if (c == INDEX_op_movi_i32) {
                     qemu_log("0x%x", (uint32_t)val);
@@ -1038,6 +1219,17 @@ void tcg_dump_ops(TCGContext *s)
                 }
                 i = 1;
                 break;
+            case INDEX_op_qemu_ld_i32:
+            case INDEX_op_qemu_st_i32:
+            case INDEX_op_qemu_ld_i64:
+            case INDEX_op_qemu_st_i64:
+                if (args[k] < ARRAY_SIZE(ldst_name) && ldst_name[args[k]]) {
+                    qemu_log(",%s", ldst_name[args[k++]]);
+                } else {
+                    qemu_log(",$0x%" TCG_PRIlx, args[k++]);
+                }
+                i = 1;
+                break;
             default:
                 i = 0;
                 break;
@@ -2311,6 +2503,8 @@ static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf,
     s->code_buf = gen_code_buf;
     s->code_ptr = gen_code_buf;
 
+    tcg_out_tb_init(s);
+
     args = s->gen_opparam_buf;
     op_index = 0;
 
@@ -2384,10 +2578,8 @@ static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf,
 #endif
     }
  the_end:
-#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
     /* Generate TB finalization at the end of block */
     tcg_out_tb_finalize(s);
-#endif
     return -1;
 }
 
diff --git a/tcg/tcg.h b/tcg/tcg.h
index 902c751d26..0d9bd293b5 100644
--- a/tcg/tcg.h
+++ b/tcg/tcg.h
@@ -197,6 +197,60 @@ typedef enum TCGType {
 #endif
 } TCGType;
 
+/* Constants for qemu_ld and qemu_st for the Memory Operation field.  */
+typedef enum TCGMemOp {
+    MO_8     = 0,
+    MO_16    = 1,
+    MO_32    = 2,
+    MO_64    = 3,
+    MO_SIZE  = 3,   /* Mask for the above.  */
+
+    MO_SIGN  = 4,   /* Sign-extended, otherwise zero-extended.  */
+
+    MO_BSWAP = 8,   /* Host reverse endian.  */
+#ifdef HOST_WORDS_BIGENDIAN
+    MO_LE    = MO_BSWAP,
+    MO_BE    = 0,
+#else
+    MO_LE    = 0,
+    MO_BE    = MO_BSWAP,
+#endif
+#ifdef TARGET_WORDS_BIGENDIAN
+    MO_TE    = MO_BE,
+#else
+    MO_TE    = MO_LE,
+#endif
+
+    /* Combinations of the above, for ease of use.  */
+    MO_UB    = MO_8,
+    MO_UW    = MO_16,
+    MO_UL    = MO_32,
+    MO_SB    = MO_SIGN | MO_8,
+    MO_SW    = MO_SIGN | MO_16,
+    MO_SL    = MO_SIGN | MO_32,
+    MO_Q     = MO_64,
+
+    MO_LEUW  = MO_LE | MO_UW,
+    MO_LEUL  = MO_LE | MO_UL,
+    MO_LESW  = MO_LE | MO_SW,
+    MO_LESL  = MO_LE | MO_SL,
+    MO_LEQ   = MO_LE | MO_Q,
+
+    MO_BEUW  = MO_BE | MO_UW,
+    MO_BEUL  = MO_BE | MO_UL,
+    MO_BESW  = MO_BE | MO_SW,
+    MO_BESL  = MO_BE | MO_SL,
+    MO_BEQ   = MO_BE | MO_Q,
+
+    MO_TEUW  = MO_TE | MO_UW,
+    MO_TEUL  = MO_TE | MO_UL,
+    MO_TESW  = MO_TE | MO_SW,
+    MO_TESL  = MO_TE | MO_SL,
+    MO_TEQ   = MO_TE | MO_Q,
+
+    MO_SSIZE = MO_SIZE | MO_SIGN,
+} TCGMemOp;
+
 typedef tcg_target_ulong TCGArg;
 
 /* Define a type and accessor macros for variables.  Using a struct is
@@ -211,24 +265,6 @@ typedef tcg_target_ulong TCGArg;
    are aliases for target_ulong and host pointer sized values respectively.
  */
 
-#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
-/* Macros/structures for qemu_ld/st IR code optimization:
-   TCG_MAX_HELPER_LABELS is defined as same as OPC_BUF_SIZE in exec-all.h. */
-#define TCG_MAX_QEMU_LDST       640
-
-typedef struct TCGLabelQemuLdst {
-    int is_ld:1;            /* qemu_ld: 1, qemu_st: 0 */
-    int opc:4;
-    int addrlo_reg;         /* reg index for low word of guest virtual addr */
-    int addrhi_reg;         /* reg index for high word of guest virtual addr */
-    int datalo_reg;         /* reg index for low word to be loaded or stored */
-    int datahi_reg;         /* reg index for high word to be loaded or stored */
-    int mem_index;          /* soft MMU memory index */
-    uint8_t *raddr;         /* gen code addr of the next IR of qemu_ld/st IR */
-    uint8_t *label_ptr[2];  /* label pointers to be updated */
-} TCGLabelQemuLdst;
-#endif
-
 #ifdef CONFIG_DEBUG_TCG
 #define DEBUG_TCGV 1
 #endif
@@ -405,11 +441,6 @@ typedef struct TCGTemp {
     const char *name;
 } TCGTemp;
 
-typedef struct TCGHelperInfo {
-    uintptr_t func;
-    const char *name;
-} TCGHelperInfo;
-
 typedef struct TCGContext TCGContext;
 
 struct TCGContext {
@@ -447,10 +478,7 @@ struct TCGContext {
     uint8_t *code_ptr;
     TCGTemp temps[TCG_MAX_TEMPS]; /* globals first, temps after */
 
-    TCGHelperInfo *helpers;
-    int nb_helpers;
-    int allocated_helpers;
-    int helpers_sorted;
+    GHashTable *helpers;
 
 #ifdef CONFIG_PROFILER
     /* profiling info */
@@ -496,12 +524,8 @@ struct TCGContext {
 
     TBContext tb_ctx;
 
-#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
-    /* labels info for qemu_ld/st IRs
-       The labels help to generate TLB miss case codes at the end of TB */
-    TCGLabelQemuLdst *qemu_ldst_labels;
-    int nb_qemu_ldst_labels;
-#endif
+    /* The TCGBackendData structure is private to tcg-target.c.  */
+    struct TCGBackendData *be;
 };
 
 extern TCGContext tcg_ctx;
@@ -680,8 +704,6 @@ TCGArg *tcg_optimize(TCGContext *s, uint16_t *tcg_opc_ptr, TCGArg *args,
                      TCGOpDef *tcg_op_def);
 
 /* only used for debugging purposes */
-void tcg_register_helper(void *func, const char *name);
-const char *tcg_helper_get_name(TCGContext *s, void *func);
 void tcg_dump_ops(TCGContext *s);
 
 void dump_ops(const uint16_t *opc_buf, const TCGArg *opparam_buf);
@@ -745,11 +767,6 @@ TCGv_i64 tcg_const_local_i64(int64_t val);
 
 void tcg_register_jit(void *buf, size_t buf_size);
 
-#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
-/* Generate TB finalization at the end of block */
-void tcg_out_tb_finalize(TCGContext *s);
-#endif
-
 /*
  * Memory helpers that will be used by TCG generated code.
  */
@@ -757,29 +774,66 @@ void tcg_out_tb_finalize(TCGContext *s);
 /* Value zero-extended to tcg register size.  */
 tcg_target_ulong helper_ret_ldub_mmu(CPUArchState *env, target_ulong addr,
                                      int mmu_idx, uintptr_t retaddr);
-tcg_target_ulong helper_ret_lduw_mmu(CPUArchState *env, target_ulong addr,
-                                     int mmu_idx, uintptr_t retaddr);
-tcg_target_ulong helper_ret_ldul_mmu(CPUArchState *env, target_ulong addr,
-                                     int mmu_idx, uintptr_t retaddr);
-uint64_t helper_ret_ldq_mmu(CPUArchState *env, target_ulong addr,
-                            int mmu_idx, uintptr_t retaddr);
+tcg_target_ulong helper_le_lduw_mmu(CPUArchState *env, target_ulong addr,
+                                    int mmu_idx, uintptr_t retaddr);
+tcg_target_ulong helper_le_ldul_mmu(CPUArchState *env, target_ulong addr,
+                                    int mmu_idx, uintptr_t retaddr);
+uint64_t helper_le_ldq_mmu(CPUArchState *env, target_ulong addr,
+                           int mmu_idx, uintptr_t retaddr);
+tcg_target_ulong helper_be_lduw_mmu(CPUArchState *env, target_ulong addr,
+                                    int mmu_idx, uintptr_t retaddr);
+tcg_target_ulong helper_be_ldul_mmu(CPUArchState *env, target_ulong addr,
+                                    int mmu_idx, uintptr_t retaddr);
+uint64_t helper_be_ldq_mmu(CPUArchState *env, target_ulong addr,
+                           int mmu_idx, uintptr_t retaddr);
 
 /* Value sign-extended to tcg register size.  */
 tcg_target_ulong helper_ret_ldsb_mmu(CPUArchState *env, target_ulong addr,
                                      int mmu_idx, uintptr_t retaddr);
-tcg_target_ulong helper_ret_ldsw_mmu(CPUArchState *env, target_ulong addr,
-                                     int mmu_idx, uintptr_t retaddr);
-tcg_target_ulong helper_ret_ldsl_mmu(CPUArchState *env, target_ulong addr,
-                                     int mmu_idx, uintptr_t retaddr);
+tcg_target_ulong helper_le_ldsw_mmu(CPUArchState *env, target_ulong addr,
+                                    int mmu_idx, uintptr_t retaddr);
+tcg_target_ulong helper_le_ldsl_mmu(CPUArchState *env, target_ulong addr,
+                                    int mmu_idx, uintptr_t retaddr);
+tcg_target_ulong helper_be_ldsw_mmu(CPUArchState *env, target_ulong addr,
+                                    int mmu_idx, uintptr_t retaddr);
+tcg_target_ulong helper_be_ldsl_mmu(CPUArchState *env, target_ulong addr,
+                                    int mmu_idx, uintptr_t retaddr);
 
 void helper_ret_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val,
                         int mmu_idx, uintptr_t retaddr);
-void helper_ret_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
-                        int mmu_idx, uintptr_t retaddr);
-void helper_ret_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
-                        int mmu_idx, uintptr_t retaddr);
-void helper_ret_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
-                        int mmu_idx, uintptr_t retaddr);
+void helper_le_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
+                       int mmu_idx, uintptr_t retaddr);
+void helper_le_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
+                       int mmu_idx, uintptr_t retaddr);
+void helper_le_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
+                       int mmu_idx, uintptr_t retaddr);
+void helper_be_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
+                       int mmu_idx, uintptr_t retaddr);
+void helper_be_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
+                       int mmu_idx, uintptr_t retaddr);
+void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
+                       int mmu_idx, uintptr_t retaddr);
+
+/* Temporary aliases until backends are converted.  */
+#ifdef TARGET_WORDS_BIGENDIAN
+# define helper_ret_ldsw_mmu  helper_be_ldsw_mmu
+# define helper_ret_lduw_mmu  helper_be_lduw_mmu
+# define helper_ret_ldsl_mmu  helper_be_ldsl_mmu
+# define helper_ret_ldul_mmu  helper_be_ldul_mmu
+# define helper_ret_ldq_mmu   helper_be_ldq_mmu
+# define helper_ret_stw_mmu   helper_be_stw_mmu
+# define helper_ret_stl_mmu   helper_be_stl_mmu
+# define helper_ret_stq_mmu   helper_be_stq_mmu
+#else
+# define helper_ret_ldsw_mmu  helper_le_ldsw_mmu
+# define helper_ret_lduw_mmu  helper_le_lduw_mmu
+# define helper_ret_ldsl_mmu  helper_le_ldsl_mmu
+# define helper_ret_ldul_mmu  helper_le_ldul_mmu
+# define helper_ret_ldq_mmu   helper_le_ldq_mmu
+# define helper_ret_stw_mmu   helper_le_stw_mmu
+# define helper_ret_stl_mmu   helper_le_stl_mmu
+# define helper_ret_stq_mmu   helper_le_stq_mmu
+#endif
 
 uint8_t helper_ldb_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
 uint16_t helper_ldw_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
diff --git a/tcg/tci/tcg-target.c b/tcg/tci/tcg-target.c
index 233ab3bf35..fc80704de8 100644
--- a/tcg/tci/tcg-target.c
+++ b/tcg/tci/tcg-target.c
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "tcg-be-null.h"
+
 /* TODO list:
  * - See TODO comments in code.
  */
@@ -670,7 +672,6 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
     case INDEX_op_shl_i64:
     case INDEX_op_shr_i64:
     case INDEX_op_sar_i64:
-        /* TODO: Implementation of rotl_i64, rotr_i64 missing in tci.c. */
     case INDEX_op_rotl_i64:     /* Optional (TCG_TARGET_HAS_rot_i64). */
     case INDEX_op_rotr_i64:     /* Optional (TCG_TARGET_HAS_rot_i64). */
         tcg_out_r(s, args[0]);
diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h
index c2ecfbe047..6e1da8c007 100644
--- a/tcg/tci/tcg-target.h
+++ b/tcg/tci/tcg-target.h
@@ -120,6 +120,8 @@
 #define TCG_TARGET_HAS_mulsh_i64        0
 #endif /* TCG_TARGET_REG_BITS == 64 */
 
+#define TCG_TARGET_HAS_new_ldst         0
+
 /* Number of registers available.
    For 32 bit hosts, we need more than 8 registers (call arguments). */
 /* #define TCG_TARGET_NB_REGS 8 */
diff --git a/tci.c b/tci.c
index 18c888e54d..0202ed97d1 100644
--- a/tci.c
+++ b/tci.c
@@ -688,13 +688,13 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
             t0 = *tb_ptr++;
             t1 = tci_read_ri32(&tb_ptr);
             t2 = tci_read_ri32(&tb_ptr);
-            tci_write_reg32(t0, (t1 << t2) | (t1 >> (32 - t2)));
+            tci_write_reg32(t0, rol32(t1, t2));
             break;
         case INDEX_op_rotr_i32:
             t0 = *tb_ptr++;
             t1 = tci_read_ri32(&tb_ptr);
             t2 = tci_read_ri32(&tb_ptr);
-            tci_write_reg32(t0, (t1 >> t2) | (t1 << (32 - t2)));
+            tci_write_reg32(t0, ror32(t1, t2));
             break;
 #endif
 #if TCG_TARGET_HAS_deposit_i32
@@ -952,8 +952,16 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
             break;
 #if TCG_TARGET_HAS_rot_i64
         case INDEX_op_rotl_i64:
+            t0 = *tb_ptr++;
+            t1 = tci_read_ri64(&tb_ptr);
+            t2 = tci_read_ri64(&tb_ptr);
+            tci_write_reg64(t0, rol64(t1, t2));
+            break;
         case INDEX_op_rotr_i64:
-            TODO();
+            t0 = *tb_ptr++;
+            t1 = tci_read_ri64(&tb_ptr);
+            t2 = tci_read_ri64(&tb_ptr);
+            tci_write_reg64(t0, ror64(t1, t2));
             break;
 #endif
 #if TCG_TARGET_HAS_deposit_i64
@@ -1085,7 +1093,6 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
             tmp8 = helper_ldb_mmu(env, taddr, tci_read_i(&tb_ptr));
 #else
             host_addr = (tcg_target_ulong)taddr;
-            assert(taddr == host_addr);
             tmp8 = *(uint8_t *)(host_addr + GUEST_BASE);
 #endif
             tci_write_reg8(t0, tmp8);
@@ -1097,7 +1104,6 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
             tmp8 = helper_ldb_mmu(env, taddr, tci_read_i(&tb_ptr));
 #else
             host_addr = (tcg_target_ulong)taddr;
-            assert(taddr == host_addr);
             tmp8 = *(uint8_t *)(host_addr + GUEST_BASE);
 #endif
             tci_write_reg8s(t0, tmp8);
@@ -1109,7 +1115,6 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
             tmp16 = helper_ldw_mmu(env, taddr, tci_read_i(&tb_ptr));
 #else
             host_addr = (tcg_target_ulong)taddr;
-            assert(taddr == host_addr);
             tmp16 = tswap16(*(uint16_t *)(host_addr + GUEST_BASE));
 #endif
             tci_write_reg16(t0, tmp16);
@@ -1121,7 +1126,6 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
             tmp16 = helper_ldw_mmu(env, taddr, tci_read_i(&tb_ptr));
 #else
             host_addr = (tcg_target_ulong)taddr;
-            assert(taddr == host_addr);
             tmp16 = tswap16(*(uint16_t *)(host_addr + GUEST_BASE));
 #endif
             tci_write_reg16s(t0, tmp16);
@@ -1134,7 +1138,6 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
             tmp32 = helper_ldl_mmu(env, taddr, tci_read_i(&tb_ptr));
 #else
             host_addr = (tcg_target_ulong)taddr;
-            assert(taddr == host_addr);
             tmp32 = tswap32(*(uint32_t *)(host_addr + GUEST_BASE));
 #endif
             tci_write_reg32(t0, tmp32);
@@ -1146,7 +1149,6 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
             tmp32 = helper_ldl_mmu(env, taddr, tci_read_i(&tb_ptr));
 #else
             host_addr = (tcg_target_ulong)taddr;
-            assert(taddr == host_addr);
             tmp32 = tswap32(*(uint32_t *)(host_addr + GUEST_BASE));
 #endif
             tci_write_reg32s(t0, tmp32);
@@ -1159,7 +1161,6 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
             tmp32 = helper_ldl_mmu(env, taddr, tci_read_i(&tb_ptr));
 #else
             host_addr = (tcg_target_ulong)taddr;
-            assert(taddr == host_addr);
             tmp32 = tswap32(*(uint32_t *)(host_addr + GUEST_BASE));
 #endif
             tci_write_reg32(t0, tmp32);
@@ -1174,7 +1175,6 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
             tmp64 = helper_ldq_mmu(env, taddr, tci_read_i(&tb_ptr));
 #else
             host_addr = (tcg_target_ulong)taddr;
-            assert(taddr == host_addr);
             tmp64 = tswap64(*(uint64_t *)(host_addr + GUEST_BASE));
 #endif
             tci_write_reg(t0, tmp64);
@@ -1190,7 +1190,6 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
             helper_stb_mmu(env, taddr, t0, t2);
 #else
             host_addr = (tcg_target_ulong)taddr;
-            assert(taddr == host_addr);
             *(uint8_t *)(host_addr + GUEST_BASE) = t0;
 #endif
             break;
@@ -1202,7 +1201,6 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
             helper_stw_mmu(env, taddr, t0, t2);
 #else
             host_addr = (tcg_target_ulong)taddr;
-            assert(taddr == host_addr);
             *(uint16_t *)(host_addr + GUEST_BASE) = tswap16(t0);
 #endif
             break;
@@ -1214,7 +1212,6 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
             helper_stl_mmu(env, taddr, t0, t2);
 #else
             host_addr = (tcg_target_ulong)taddr;
-            assert(taddr == host_addr);
             *(uint32_t *)(host_addr + GUEST_BASE) = tswap32(t0);
 #endif
             break;
@@ -1226,7 +1223,6 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
             helper_stq_mmu(env, taddr, tmp64, t2);
 #else
             host_addr = (tcg_target_ulong)taddr;
-            assert(taddr == host_addr);
             *(uint64_t *)(host_addr + GUEST_BASE) = tswap64(tmp64);
 #endif
             break;
diff --git a/tests/.gitignore b/tests/.gitignore
index d11cc22373..425757cfe1 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -5,8 +5,11 @@ check-qjson
 check-qlist
 check-qstring
 test-aio
+test-bitops
+test-throttle
 test-cutils
 test-hbitmap
+test-int128
 test-iov
 test-mul64
 test-qapi-types.[ch]
@@ -20,3 +23,4 @@ test-thread-pool
 test-x86-cpuid
 test-xbzrle
 *-test
+qapi-schema/*.test.*
diff --git a/tests/Makefile b/tests/Makefile
index c13fefc314..fa4c9f0cbb 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -174,6 +174,7 @@ tests/boot-order-test$(EXESUF): tests/boot-order-test.o $(libqos-obj-y)
 tests/tmp105-test$(EXESUF): tests/tmp105-test.o $(libqos-omap-obj-y)
 tests/i440fx-test$(EXESUF): tests/i440fx-test.o $(libqos-pc-obj-y)
 tests/fw_cfg-test$(EXESUF): tests/fw_cfg-test.o $(libqos-pc-obj-y)
+tests/qemu-iotests/socket_scm_helper$(EXESUF): tests/qemu-iotests/socket_scm_helper.o
 
 # QTest rules
 
@@ -195,6 +196,7 @@ check-help:
 	@echo " make check-qapi-schema    Run QAPI schema tests"
 	@echo " make check-block          Run block tests"
 	@echo " make check-report.html    Generates an HTML test report"
+	@echo " make check-clean          Clean the tests"
 	@echo
 	@echo "Please note that HTML reports do not regenerate if the unit tests"
 	@echo "has not changed."
@@ -251,8 +253,10 @@ check-report.html: check-report.xml
 
 # Other tests
 
+QEMU_IOTESTS_HELPERS-$(CONFIG_LINUX) = tests/qemu-iotests/socket_scm_helper$(EXESUF)
+
 .PHONY: check-tests/qemu-iotests-quick.sh
-check-tests/qemu-iotests-quick.sh: tests/qemu-iotests-quick.sh qemu-img$(EXESUF) qemu-io$(EXESUF)
+check-tests/qemu-iotests-quick.sh: tests/qemu-iotests-quick.sh qemu-img$(EXESUF) qemu-io$(EXESUF) $(QEMU_IOTESTS_HELPERS-y)
 	$<
 
 .PHONY: check-tests/test-qapi.py
@@ -260,19 +264,28 @@ check-tests/test-qapi.py: tests/test-qapi.py
 
 .PHONY: $(patsubst %, check-%, $(check-qapi-schema-y))
 $(patsubst %, check-%, $(check-qapi-schema-y)): check-%.json: $(SRC_PATH)/%.json
-	$(call quiet-command, PYTHONPATH=$(SRC_PATH)/scripts $(PYTHON) $(SRC_PATH)/tests/qapi-schema/test-qapi.py <$^ >$*.out 2>$*.err; echo $$? >$*.exit, "  TEST  $*.out")
-	@diff -q $(SRC_PATH)/$*.out $*.out
-	@diff -q $(SRC_PATH)/$*.err $*.err
-	@diff -q $(SRC_PATH)/$*.exit $*.exit
+	$(call quiet-command, PYTHONPATH=$(SRC_PATH)/scripts $(PYTHON) $(SRC_PATH)/tests/qapi-schema/test-qapi.py <$^ >$*.test.out 2>$*.test.err; echo $$? >$*.test.exit, "  TEST  $*.out")
+	@diff -q $(SRC_PATH)/$*.out $*.test.out
+	@diff -q $(SRC_PATH)/$*.err $*.test.err
+	@diff -q $(SRC_PATH)/$*.exit $*.test.exit
 
 # Consolidated targets
 
-.PHONY: check-qapi-schema check-qtest check-unit check
+.PHONY: check-qapi-schema check-qtest check-unit check check-clean
 check-qapi-schema: $(patsubst %,check-%, $(check-qapi-schema-y))
 check-qtest: $(patsubst %,check-qtest-%, $(QTEST_TARGETS))
 check-unit: $(patsubst %,check-%, $(check-unit-y))
 check-block: $(patsubst %,check-%, $(check-block-y))
 check: check-qapi-schema check-unit check-qtest
+check-clean:
+	$(MAKE) -C tests/tcg clean
+	rm -rf $(check-unit-y) $(check-qtest-i386-y) $(check-qtest-x86_64-y) $(check-qtest-sparc64-y) $(check-qtest-sparc-y) tests/*.o $(QEMU_IOTESTS_HELPERS-y)
+
+clean: check-clean
+
+# Build the help program automatically
+
+all: $(QEMU_IOTESTS_HELPERS-y)
 
 -include $(wildcard tests/*.d)
 -include $(wildcard tests/libqos/*.d)
diff --git a/tests/qemu-iotests/.gitignore b/tests/qemu-iotests/.gitignore
index 62b4002995..0541f80daa 100644
--- a/tests/qemu-iotests/.gitignore
+++ b/tests/qemu-iotests/.gitignore
@@ -2,6 +2,7 @@ check.log
 check.time
 *.out.bad
 *.notrun
+socket_scm_helper
 
 # ignore everything in the scratch directory
 scratch/
diff --git a/tests/qemu-iotests/001 b/tests/qemu-iotests/001
index bd88dde879..4e1646941b 100755
--- a/tests/qemu-iotests/001
+++ b/tests/qemu-iotests/001
@@ -48,15 +48,15 @@ _make_test_img $size
 
 echo
 echo "== reading whole image =="
-$QEMU_IO -c "read 0 $size" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "read 0 $size" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== rewriting whole image =="
-$QEMU_IO -c "write -P 0xa 0 $size" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "write -P 0xa 0 $size" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== verify pattern =="
-$QEMU_IO -c "read -P 0xa 0 $size" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "read -P 0xa 0 $size" "$TEST_IMG" | _filter_qemu_io
 
 
 # success, all done
diff --git a/tests/qemu-iotests/002 b/tests/qemu-iotests/002
index 51d0a8f4ad..6a865aac73 100755
--- a/tests/qemu-iotests/002
+++ b/tests/qemu-iotests/002
@@ -48,36 +48,36 @@ _make_test_img $size
 
 echo
 echo "== reading whole image =="
-$QEMU_IO -c "read -p 0 $size" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "read -p 0 $size" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== rewriting whole image =="
-$QEMU_IO -c "write -pP 0xa 0 $size" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "write -pP 0xa 0 $size" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== verify pattern =="
-$QEMU_IO -c "read -pP 0xa 0 $size" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "read -pP 0xa 0 $size" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "unaligned pwrite"
-$QEMU_IO -c 'write -pP 0xab 66 42' $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c 'write -pP 0xac 512 288' $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c 'write -pP 0xad 800 224' $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c 'write -pP 0xae 66000 128k' $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c 'write -pP 0xaf 256k 42' $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c 'write -pP 0xab 66 42' "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c 'write -pP 0xac 512 288' "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c 'write -pP 0xad 800 224' "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c 'write -pP 0xae 66000 128k' "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c 'write -pP 0xaf 256k 42' "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "verify pattern"
-$QEMU_IO -c 'read -pP 0xa 0 66' $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c 'read -pP 0xab 66 42' $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c 'read -pP 0xa 108 404' $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c 'read -pP 0xac 512 288' $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c 'read -pP 0xad 800 224' $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c 'read -pP 0xa 1k 64976' $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c 'read -pP 0xae 66000 128k' $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c 'read -pP 0xa 197072 65072' $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c 'read -pP 0xaf 256k 42' $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c 'read -pP 0xa 262186 470' $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c 'read -pP 0xa 0 66' "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c 'read -pP 0xab 66 42' "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c 'read -pP 0xa 108 404' "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c 'read -pP 0xac 512 288' "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c 'read -pP 0xad 800 224' "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c 'read -pP 0xa 1k 64976' "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c 'read -pP 0xae 66000 128k' "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c 'read -pP 0xa 197072 65072' "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c 'read -pP 0xaf 256k 42' "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c 'read -pP 0xa 262186 470' "$TEST_IMG" | _filter_qemu_io
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/003 b/tests/qemu-iotests/003
index ee25fb8078..98638d4ce7 100755
--- a/tests/qemu-iotests/003
+++ b/tests/qemu-iotests/003
@@ -50,27 +50,27 @@ _make_test_img $size
 
 echo
 echo "== reading whole image =="
-$QEMU_IO -c "readv 0 $size" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "readv 0 $size" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== rewriting whole image =="
-$QEMU_IO -c "writev -P 0xa 0 $size" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "writev -P 0xa 0 $size" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== verify pattern =="
-$QEMU_IO -c "readv -P 0xa 0 $size" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "readv -P 0xa 0 $size" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== vectored write =="
 $QEMU_IO -c "writev -P 0xb $offset $chunksize $chunksize \
 	$chunksize $chunksize $chunksize $chunksize $chunksize" \
-	$TEST_IMG | _filter_qemu_io
+	"$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== verify pattern =="
 $QEMU_IO -c "readv -P 0xb $offset $chunksize $chunksize \
 	$chunksize $chunksize $chunksize $chunksize $chunksize" \
-	$TEST_IMG | _filter_qemu_io
+	"$TEST_IMG" | _filter_qemu_io
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/004 b/tests/qemu-iotests/004
index c76451c5a7..651072ef89 100755
--- a/tests/qemu-iotests/004
+++ b/tests/qemu-iotests/004
@@ -51,51 +51,51 @@ _make_test_img $size
 
 echo
 echo "write before image boundary"
-$QEMU_IO -c "write $pre_offset 1M" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "write $pre_offset 1M" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "write into image boundary"
-$QEMU_IO -c "write $pre_offset 4M" $TEST_IMG
+$QEMU_IO -c "write $pre_offset 4M" "$TEST_IMG"
 
 echo
 echo "write at image boundary"
-$QEMU_IO -c "write $size 4096" $TEST_IMG
+$QEMU_IO -c "write $size 4096" "$TEST_IMG"
 
 echo
 echo "write past image boundary"
-$QEMU_IO -c "write $past_offset 4096" $TEST_IMG
+$QEMU_IO -c "write $past_offset 4096" "$TEST_IMG"
 
 echo
 echo "pwrite past image boundary"
-$QEMU_IO -c "write -p $past_offset 4096" $TEST_IMG
+$QEMU_IO -c "write -p $past_offset 4096" "$TEST_IMG"
 
 echo
 echo "writev past image boundary"
-$QEMU_IO -c "writev $past_offset 4096" $TEST_IMG
+$QEMU_IO -c "writev $past_offset 4096" "$TEST_IMG"
 
 echo
 echo "read before image boundary"
-$QEMU_IO -c "read $pre_offset 1M" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "read $pre_offset 1M" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "read into image boundary"
-$QEMU_IO -c "read $pre_offset 4M" $TEST_IMG
+$QEMU_IO -c "read $pre_offset 4M" "$TEST_IMG"
 
 echo
 echo "read at image boundary"
-$QEMU_IO -c "read $size 4096" $TEST_IMG
+$QEMU_IO -c "read $size 4096" "$TEST_IMG"
 
 echo
 echo "read past image boundary"
-$QEMU_IO -c "read $past_offset 4096" $TEST_IMG
+$QEMU_IO -c "read $past_offset 4096" "$TEST_IMG"
 
 echo
 echo "pread past image boundary"
-$QEMU_IO -c "read -p $past_offset 4096" $TEST_IMG
+$QEMU_IO -c "read -p $past_offset 4096" "$TEST_IMG"
 
 echo
 echo "readv past image boundary"
-$QEMU_IO -c "readv $past_offset 4096" $TEST_IMG
+$QEMU_IO -c "readv $past_offset 4096" "$TEST_IMG"
 
 
 # success, all done
diff --git a/tests/qemu-iotests/005 b/tests/qemu-iotests/005
index b7970e3b58..9abcb84e4b 100755
--- a/tests/qemu-iotests/005
+++ b/tests/qemu-iotests/005
@@ -61,11 +61,11 @@ _make_test_img 5000G
 
 echo
 echo "small read"
-$QEMU_IO -c "read 1024 4096" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "read 1024 4096" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "small write"
-$QEMU_IO -c "write 8192 4096" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "write 8192 4096" "$TEST_IMG" | _filter_qemu_io
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/007 b/tests/qemu-iotests/007
index c454f2c8ec..fe1a743806 100755
--- a/tests/qemu-iotests/007
+++ b/tests/qemu-iotests/007
@@ -30,7 +30,7 @@ status=1	# failure is the default!
 
 _cleanup()
 {
-#	_cleanup_test_img
+	_cleanup_test_img
 	true
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
@@ -50,7 +50,7 @@ _make_test_img 1M
 
 for i in `seq 1 10`; do
     echo "savevm $i"
-    $QEMU -nographic -hda $TEST_IMG -serial none -monitor stdio >/dev/null 2>&1 <<EOF
+    $QEMU -nographic -hda "$TEST_IMG" -serial none -monitor stdio >/dev/null 2>&1 <<EOF
 savevm test-$i
 quit
 EOF
diff --git a/tests/qemu-iotests/008 b/tests/qemu-iotests/008
index 2c53bac925..2d28efd428 100755
--- a/tests/qemu-iotests/008
+++ b/tests/qemu-iotests/008
@@ -48,15 +48,15 @@ _make_test_img $size
 
 echo
 echo "== reading whole image =="
-$QEMU_IO -c "aio_read 0 $size" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "aio_read 0 $size" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== rewriting whole image =="
-$QEMU_IO -c "aio_write -P 0xa 0 $size" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "aio_write -P 0xa 0 $size" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== verify pattern =="
-$QEMU_IO -c "aio_read -P 0xa 0 $size" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "aio_read -P 0xa 0 $size" "$TEST_IMG" | _filter_qemu_io
 
 
 # success, all done
diff --git a/tests/qemu-iotests/009 b/tests/qemu-iotests/009
index 25368c819b..57a43f5a16 100755
--- a/tests/qemu-iotests/009
+++ b/tests/qemu-iotests/009
@@ -57,7 +57,7 @@ $QEMU_IO \
 	-c "write 4k 4k" \
 	-c "write 9M 4k" \
 	-c "read -P 65 -s 4k -l 4k 2044k 8k" \
-$TEST_IMG | _filter_qemu_io
+"$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "checking image for errors"
diff --git a/tests/qemu-iotests/010 b/tests/qemu-iotests/010
index 7b5792934a..896a0058ff 100755
--- a/tests/qemu-iotests/010
+++ b/tests/qemu-iotests/010
@@ -59,7 +59,7 @@ $QEMU_IO \
 	-c "write -P 165 2044k 4k" \
 	-c "write -P 99 8M 4k" \
 	-c "read -P 165 2044k 8k" \
-$TEST_IMG | _filter_qemu_io
+"$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "checking image for errors"
diff --git a/tests/qemu-iotests/011 b/tests/qemu-iotests/011
index b03df6887d..1c5158af43 100755
--- a/tests/qemu-iotests/011
+++ b/tests/qemu-iotests/011
@@ -60,7 +60,7 @@ for i in `seq 1 10`; do
     # Note that we filter away the actual offset.  That's because qemu
     # may re-order the two aio requests.  We only want to make sure the
     # filesystem isn't corrupted afterwards anyway.
-    $QEMU_IO -c "aio_write $off1 1M" -c "aio_write $off2 1M" $TEST_IMG | \
+    $QEMU_IO -c "aio_write $off1 1M" -c "aio_write $off2 1M" "$TEST_IMG" | \
     	_filter_qemu_io | \
 	sed -e 's/bytes at offset [0-9]*/bytes at offset XXX/g'
 done
diff --git a/tests/qemu-iotests/012 b/tests/qemu-iotests/012
index 4052956cd9..7c5b6892d3 100755
--- a/tests/qemu-iotests/012
+++ b/tests/qemu-iotests/012
@@ -50,11 +50,11 @@ _make_test_img $size
 
 echo
 echo "== mark image read-only"
-chmod a-w $TEST_IMG
+chmod a-w "$TEST_IMG"
 
 echo
 echo "== read from read-only image"
-$QEMU_IO -r -c "read 0 512" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -r -c "read 0 512" "$TEST_IMG" | _filter_qemu_io
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/013 b/tests/qemu-iotests/013
index ce40d5c5b6..389f4b8156 100755
--- a/tests/qemu-iotests/013
+++ b/tests/qemu-iotests/013
@@ -65,8 +65,8 @@ done
 echo "Compressing image"
 echo
 
-mv $TEST_IMG $TEST_IMG.orig
-$QEMU_IMG convert -f $IMGFMT -O $IMGFMT -c $TEST_IMG.orig $TEST_IMG
+mv "$TEST_IMG" "$TEST_IMG.orig"
+$QEMU_IMG convert -f $IMGFMT -O $IMGFMT -c "$TEST_IMG.orig" "$TEST_IMG"
 
 echo "Testing compressed image"
 echo
diff --git a/tests/qemu-iotests/014 b/tests/qemu-iotests/014
index a6d0aea7c0..0edeb4b6f5 100755
--- a/tests/qemu-iotests/014
+++ b/tests/qemu-iotests/014
@@ -61,7 +61,7 @@ done
 
 # With snapshots
 for i in `seq 1 3`; do
-    $QEMU_IMG snapshot -c test$i $TEST_IMG
+    $QEMU_IMG snapshot -c test$i "$TEST_IMG"
     for offset in $TEST_OFFSETS; do
         echo With snapshot test$i, offset $offset
         for op in $TEST_OPS; do
diff --git a/tests/qemu-iotests/015 b/tests/qemu-iotests/015
index 44c134f948..099d75723c 100755
--- a/tests/qemu-iotests/015
+++ b/tests/qemu-iotests/015
@@ -61,19 +61,19 @@ _make_test_img $size
 
 # Create two snapshots which fill the image with two different patterns
 echo "creating first snapshot"
-$QEMU_IO -c "aio_write -P 123 0 $size" $TEST_IMG | _filter_qemu_io
-$QEMU_IMG snapshot -c snap1 $TEST_IMG
+$QEMU_IO -c "aio_write -P 123 0 $size" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IMG snapshot -c snap1 "$TEST_IMG"
 echo "creating second snapshot"
-$QEMU_IO -c "aio_write -P 165 0 $size" $TEST_IMG | _filter_qemu_io
-$QEMU_IMG snapshot -c snap2 $TEST_IMG
+$QEMU_IO -c "aio_write -P 165 0 $size" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IMG snapshot -c snap2 "$TEST_IMG"
 
 # Now check the pattern
 echo "checking first snapshot"
-$QEMU_IMG snapshot -a snap1 $TEST_IMG
-$QEMU_IO -c "aio_read -P 123 0 $size" $TEST_IMG | _filter_qemu_io
+$QEMU_IMG snapshot -a snap1 "$TEST_IMG"
+$QEMU_IO -c "aio_read -P 123 0 $size" "$TEST_IMG" | _filter_qemu_io
 echo "checking second snapshot"
-$QEMU_IMG snapshot -a snap2 $TEST_IMG
-$QEMU_IO -c "aio_read -P 165 0 $size" $TEST_IMG | _filter_qemu_io
+$QEMU_IMG snapshot -a snap2 "$TEST_IMG"
+$QEMU_IO -c "aio_read -P 165 0 $size" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "checking image for errors"
diff --git a/tests/qemu-iotests/016 b/tests/qemu-iotests/016
index a1467b8a3c..b87a32bc27 100755
--- a/tests/qemu-iotests/016
+++ b/tests/qemu-iotests/016
@@ -48,21 +48,21 @@ _make_test_img $size
 
 echo
 echo "== reading at EOF =="
-$QEMU_IO -g -c "read -P 0 $size 512" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -g -c "read -P 0 $size 512" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== reading far past EOF =="
-$QEMU_IO -g -c "read -P 0 256M 512" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -g -c "read -P 0 256M 512" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== writing at EOF =="
-$QEMU_IO -g -c "write -P 66 $size 512" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 66 $size 512" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -g -c "write -P 66 $size 512" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 66 $size 512" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== writing far past EOF =="
-$QEMU_IO -g -c "write -P 66 256M 512" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 66 256M 512" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -g -c "write -P 66 256M 512" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 66 256M 512" "$TEST_IMG" | _filter_qemu_io
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/018 b/tests/qemu-iotests/018
index 453ce61e75..15fcfe5670 100755
--- a/tests/qemu-iotests/018
+++ b/tests/qemu-iotests/018
@@ -66,7 +66,7 @@ echo "Creating test image with backing file"
 echo
 
 TEST_IMG=$TEST_IMG_SAVE
-_make_test_img -b $TEST_IMG.base 6G
+_make_test_img -b "$TEST_IMG.base" 6G
 
 echo "Filling test image"
 echo
@@ -80,8 +80,8 @@ for offset in $TEST_OFFSETS; do
 done
 _check_test_img
 
-mv $TEST_IMG $TEST_IMG.orig
-$QEMU_IMG convert -O $IMGFMT $TEST_IMG.orig $TEST_IMG
+mv "$TEST_IMG" "$TEST_IMG.orig"
+$QEMU_IMG convert -O $IMGFMT "$TEST_IMG.orig" "$TEST_IMG"
 
 echo "Reading"
 echo
diff --git a/tests/qemu-iotests/019 b/tests/qemu-iotests/019
index 8872b30350..cd3582cf6f 100755
--- a/tests/qemu-iotests/019
+++ b/tests/qemu-iotests/019
@@ -33,8 +33,8 @@ status=1	# failure is the default!
 _cleanup()
 {
 	_cleanup_test_img
-    rm -f $TEST_IMG.base
-    rm -f $TEST_IMG.orig
+    rm -f "$TEST_IMG.base"
+    rm -f "$TEST_IMG.orig"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -68,8 +68,8 @@ _check_test_img
 echo "Creating test image with backing file"
 echo
 
-mv $TEST_IMG $TEST_IMG.base
-_make_test_img -b $TEST_IMG.base 6G
+mv "$TEST_IMG" "$TEST_IMG.base"
+_make_test_img -b "$TEST_IMG.base" 6G
 
 echo "Filling test image"
 echo
@@ -83,7 +83,7 @@ for offset in $TEST_OFFSETS; do
 done
 _check_test_img
 
-mv $TEST_IMG $TEST_IMG.orig
+mv "$TEST_IMG" "$TEST_IMG.orig"
 
 
 
@@ -95,7 +95,7 @@ for backing_option in "-B $TEST_IMG.base" "-o backing_file=$TEST_IMG.base"; do
     echo
     echo Testing conversion with $backing_option | _filter_testdir | _filter_imgfmt
     echo
-    $QEMU_IMG convert -O $IMGFMT $backing_option $TEST_IMG.orig $TEST_IMG
+    $QEMU_IMG convert -O $IMGFMT $backing_option "$TEST_IMG.orig" "$TEST_IMG"
 
     echo "Checking if backing clusters are allocated when they shouldn't"
     echo
diff --git a/tests/qemu-iotests/020 b/tests/qemu-iotests/020
index 2fb0ff87f2..b3c86d844e 100755
--- a/tests/qemu-iotests/020
+++ b/tests/qemu-iotests/020
@@ -31,8 +31,8 @@ status=1	# failure is the default!
 _cleanup()
 {
 	_cleanup_test_img
-    rm -f $TEST_IMG.base
-    rm -f $TEST_IMG.orig
+    rm -f "$TEST_IMG.base"
+    rm -f "$TEST_IMG.orig"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -65,8 +65,8 @@ _check_test_img
 echo "Creating test image with backing file"
 echo
 
-mv $TEST_IMG $TEST_IMG.base
-_make_test_img -b $TEST_IMG.base 6G
+mv "$TEST_IMG" "$TEST_IMG.base"
+_make_test_img -b "$TEST_IMG.base" 6G
 
 echo "Filling test image"
 echo
@@ -80,8 +80,8 @@ for offset in $TEST_OFFSETS; do
 done
 _check_test_img
 
-$QEMU_IMG commit $TEST_IMG
-mv $TEST_IMG.base $TEST_IMG
+$QEMU_IMG commit "$TEST_IMG"
+mv "$TEST_IMG.base" "$TEST_IMG"
 
 echo "Reading from the backing file"
 echo
diff --git a/tests/qemu-iotests/021 b/tests/qemu-iotests/021
index 6da79ebbbe..1c69024ccb 100755
--- a/tests/qemu-iotests/021
+++ b/tests/qemu-iotests/021
@@ -53,7 +53,7 @@ for pattern in $INVALID_PATTERNS; do
     for op in $TEST_OPS; do
         echo
 	echo "== testing $op -P $pattern =="
-	$QEMU_IO -c "$op -P $pattern 0 4096" $TEST_IMG | _filter_qemu_io
+	$QEMU_IO -c "$op -P $pattern 0 4096" "$TEST_IMG" | _filter_qemu_io
     done
 done
 
diff --git a/tests/qemu-iotests/023 b/tests/qemu-iotests/023
index 4f31b56589..090ed23dec 100755
--- a/tests/qemu-iotests/023
+++ b/tests/qemu-iotests/023
@@ -71,8 +71,8 @@ for CLUSTER_SIZE in $CLUSTER_SIZES; do
     echo "Compressing image"
     echo
 
-    mv $TEST_IMG $TEST_IMG.orig
-    $QEMU_IMG convert -f $IMGFMT -O $IMGFMT -c $TEST_IMG.orig $TEST_IMG
+    mv "$TEST_IMG" "$TEST_IMG.orig"
+    $QEMU_IMG convert -f $IMGFMT -O $IMGFMT -c "$TEST_IMG.orig" "$TEST_IMG"
 
     echo "Testing compressed image"
     echo
diff --git a/tests/qemu-iotests/024 b/tests/qemu-iotests/024
index 554b74b2d3..be974f02a2 100755
--- a/tests/qemu-iotests/024
+++ b/tests/qemu-iotests/024
@@ -31,8 +31,8 @@ status=1	# failure is the default!
 _cleanup()
 {
 	_cleanup_test_img
-	rm -f $TEST_DIR/t.$IMGFMT.base_old
-	rm -f $TEST_DIR/t.$IMGFMT.base_new
+	rm -f "$TEST_DIR/t.$IMGFMT.base_old"
+	rm -f "$TEST_DIR/t.$IMGFMT.base_new"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -62,19 +62,19 @@ echo
 
 _make_test_img 1G
 io_pattern writev 0 $CLUSTER_SIZE $((2 * CLUSTER_SIZE)) 8 0x11
-mv $TEST_IMG $TEST_IMG.base_old
+mv "$TEST_IMG" "$TEST_IMG.base_old"
 
 echo "Creating new backing file"
 echo
 
 _make_test_img 1G
 io_pattern writev 0 $((2 * CLUSTER_SIZE)) $((4 * CLUSTER_SIZE)) 4 0x22
-mv $TEST_IMG $TEST_IMG.base_new
+mv "$TEST_IMG" "$TEST_IMG.base_new"
 
 echo "Creating COW image"
 echo
 
-_make_test_img -b $TEST_IMG.base_old 1G
+_make_test_img -b "$TEST_IMG.base_old" 1G
 io_pattern writev 0 $((4 * CLUSTER_SIZE)) 0 1 0x33
 io_pattern writev $((8 * CLUSTER_SIZE)) $((4 * CLUSTER_SIZE)) 0 1 0x33
 
@@ -100,7 +100,7 @@ io_pattern readv $((15 * CLUSTER_SIZE)) $CLUSTER_SIZE 0 1 0x00
 echo
 echo Rebase and test again
 echo
-$QEMU_IMG rebase -b $TEST_IMG.base_new $TEST_IMG
+$QEMU_IMG rebase -b "$TEST_IMG.base_new" "$TEST_IMG"
 io_pattern readv $((0 * CLUSTER_SIZE)) $CLUSTER_SIZE 0 1 0x33
 io_pattern readv $((1 * CLUSTER_SIZE)) $CLUSTER_SIZE 0 1 0x33
 io_pattern readv $((2 * CLUSTER_SIZE)) $CLUSTER_SIZE 0 1 0x33
diff --git a/tests/qemu-iotests/025 b/tests/qemu-iotests/025
index 7062aa6f36..a7241ccc95 100755
--- a/tests/qemu-iotests/025
+++ b/tests/qemu-iotests/025
@@ -56,7 +56,7 @@ _check_test_img
 
 echo
 echo "=== Resizing image"
-$QEMU_IO $TEST_IMG <<EOF
+$QEMU_IO "$TEST_IMG" <<EOF
 length
 truncate $big_size
 length
@@ -65,7 +65,7 @@ _check_test_img
 
 echo
 echo "=== Verifying image size after reopen"
-$QEMU_IO -c "length" $TEST_IMG
+$QEMU_IO -c "length" "$TEST_IMG"
 
 echo
 echo "=== Verifying resized image"
diff --git a/tests/qemu-iotests/026 b/tests/qemu-iotests/026
index 107a3ff2f6..ebe29d0168 100755
--- a/tests/qemu-iotests/026
+++ b/tests/qemu-iotests/026
@@ -31,7 +31,7 @@ status=1	# failure is the default!
 _cleanup()
 {
 	_cleanup_test_img
-    rm $TEST_DIR/blkdebug.conf
+    rm "$TEST_DIR/blkdebug.conf"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -75,7 +75,7 @@ for imm in off; do
 for once in on off; do
 for vmstate in "" "-b"; do
 
-cat > $TEST_DIR/blkdebug.conf <<EOF
+cat > "$TEST_DIR/blkdebug.conf" <<EOF
 [inject-error]
 event = "$event"
 errno = "$errno"
@@ -90,16 +90,16 @@ echo "Event: $event; errno: $errno; imm: $imm; once: $once; write $vmstate"
 
 # We want to catch a simple L2 update, not the allocation of the first L2 table
 if [ "$event" == "l2_update" ]; then
-    $QEMU_IO -c "write $vmstate 0 512" $TEST_IMG > /dev/null 2>&1
+    $QEMU_IO -c "write $vmstate 0 512" "$TEST_IMG" > /dev/null 2>&1
 fi
 
-$QEMU_IO -c "write $vmstate 0 128k " $BLKDBG_TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "write $vmstate 0 128k " "$BLKDBG_TEST_IMG" | _filter_qemu_io
 
 # l2_load is not called on allocation, so issue a second write
 # Reads are another path to trigger l2_load, so do a read, too
 if [ "$event" == "l2_load" ]; then
-    $QEMU_IO -c "write $vmstate 0 128k " $BLKDBG_TEST_IMG | _filter_qemu_io
-    $QEMU_IO -c "read $vmstate 0 128k " $BLKDBG_TEST_IMG | _filter_qemu_io
+    $QEMU_IO -c "write $vmstate 0 128k " "$BLKDBG_TEST_IMG" | _filter_qemu_io
+    $QEMU_IO -c "read $vmstate 0 128k " "$BLKDBG_TEST_IMG" | _filter_qemu_io
 fi
 
 _check_test_img 2>&1 | grep -v "refcount=1 reference=0"
@@ -133,7 +133,7 @@ for imm in off; do
 for once in on off; do
 for vmstate in "" "-b"; do
 
-cat > $TEST_DIR/blkdebug.conf <<EOF
+cat > "$TEST_DIR/blkdebug.conf" <<EOF
 [inject-error]
 event = "$event"
 errno = "$errno"
@@ -145,7 +145,7 @@ _make_test_img 1G
 
 echo
 echo "Event: $event; errno: $errno; imm: $imm; once: $once; write $vmstate"
-$QEMU_IO -c "write $vmstate 0 64M" $BLKDBG_TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "write $vmstate 0 64M" "$BLKDBG_TEST_IMG" | _filter_qemu_io
 
 _check_test_img 2>&1 | grep -v "refcount=1 reference=0"
 
@@ -172,7 +172,7 @@ for errno in 5 28; do
 for imm in off; do
 for once in on off; do
 
-cat > $TEST_DIR/blkdebug.conf <<EOF
+cat > "$TEST_DIR/blkdebug.conf" <<EOF
 [inject-error]
 event = "$event"
 errno = "$errno"
@@ -184,7 +184,7 @@ _make_test_img 1G
 
 echo
 echo "Event: $event; errno: $errno; imm: $imm; once: $once"
-$QEMU_IO -c "write -b 0 64k" $BLKDBG_TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "write -b 0 64k" "$BLKDBG_TEST_IMG" | _filter_qemu_io
 
 _check_test_img 2>&1 | grep -v "refcount=1 reference=0"
 
diff --git a/tests/qemu-iotests/026.out b/tests/qemu-iotests/026.out
index 0764389f8e..15045799a2 100644
--- a/tests/qemu-iotests/026.out
+++ b/tests/qemu-iotests/026.out
@@ -5,16 +5,12 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l1_update; errno: 5; imm: off; once: on; write 
 write failed: Input/output error
-
-1 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
+No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_update; errno: 5; imm: off; once: on; write -b
 write failed: Input/output error
-
-1 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
+No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_update; errno: 5; imm: off; once: off; write 
@@ -33,16 +29,12 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l1_update; errno: 28; imm: off; once: on; write 
 write failed: No space left on device
-
-1 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
+No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_update; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
-
-1 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
+No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_update; errno: 28; imm: off; once: off; write 
@@ -181,16 +173,12 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l2_alloc.write; errno: 5; imm: off; once: on; write 
 write failed: Input/output error
-
-1 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
+No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l2_alloc.write; errno: 5; imm: off; once: on; write -b
 write failed: Input/output error
-
-1 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
+No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l2_alloc.write; errno: 5; imm: off; once: off; write 
@@ -207,16 +195,12 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l2_alloc.write; errno: 28; imm: off; once: on; write 
 write failed: No space left on device
-
-1 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
+No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l2_alloc.write; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
-
-1 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
+No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l2_alloc.write; errno: 28; imm: off; once: off; write 
diff --git a/tests/qemu-iotests/026.out.nocache b/tests/qemu-iotests/026.out.nocache
index 33bad0d6ae..c9d242e9ec 100644
--- a/tests/qemu-iotests/026.out.nocache
+++ b/tests/qemu-iotests/026.out.nocache
@@ -5,16 +5,12 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l1_update; errno: 5; imm: off; once: on; write 
 write failed: Input/output error
-
-1 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
+No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_update; errno: 5; imm: off; once: on; write -b
 write failed: Input/output error
-
-1 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
+No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_update; errno: 5; imm: off; once: off; write 
@@ -33,16 +29,12 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l1_update; errno: 28; imm: off; once: on; write 
 write failed: No space left on device
-
-1 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
+No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_update; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
-
-1 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
+No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_update; errno: 28; imm: off; once: off; write 
@@ -189,16 +181,12 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l2_alloc.write; errno: 5; imm: off; once: on; write 
 write failed: Input/output error
-
-1 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
+No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l2_alloc.write; errno: 5; imm: off; once: on; write -b
 write failed: Input/output error
-
-1 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
+No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l2_alloc.write; errno: 5; imm: off; once: off; write 
@@ -215,16 +203,12 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l2_alloc.write; errno: 28; imm: off; once: on; write 
 write failed: No space left on device
-
-1 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
+No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l2_alloc.write; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
-
-1 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
+No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l2_alloc.write; errno: 28; imm: off; once: off; write 
diff --git a/tests/qemu-iotests/027 b/tests/qemu-iotests/027
index 7d90481832..3fa81b83bb 100755
--- a/tests/qemu-iotests/027
+++ b/tests/qemu-iotests/027
@@ -54,23 +54,23 @@ _make_test_img $size
 # Otherwise an L2 table could get in the way after the data cluster.
 echo
 echo "== writing first cluster to populate metadata =="
-$QEMU_IO -c "write -pP 0xde $cluster_size $cluster_size" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "write -pP 0xde $cluster_size $cluster_size" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== writing at sub-cluster granularity =="
-$QEMU_IO -c "write -pP 0xa $subcluster_offset $subcluster_size" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "write -pP 0xa $subcluster_offset $subcluster_size" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== verify pattern =="
-$QEMU_IO -c "read -pP 0xa $subcluster_offset $subcluster_size" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "read -pP 0xa $subcluster_offset $subcluster_size" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== verify zeroes before sub-cluster pattern =="
-$QEMU_IO -c "read -pP 0 -l $subcluster_offset 0 $subcluster_size" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "read -pP 0 -l $subcluster_offset 0 $subcluster_size" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== verify zeroes after sub-cluster pattern =="
-$QEMU_IO -c "read -pP 0 -l 512 -s $subcluster_size $subcluster_offset $(( subcluster_size + 512 ))" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "read -pP 0 -l 512 -s $subcluster_size $subcluster_offset $(( subcluster_size + 512 ))" "$TEST_IMG" | _filter_qemu_io
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/028 b/tests/qemu-iotests/028
index b091ba9f07..93a9fa6e83 100755
--- a/tests/qemu-iotests/028
+++ b/tests/qemu-iotests/028
@@ -71,8 +71,8 @@ _check_test_img
 echo "Creating test image with backing file"
 echo
 
-mv $TEST_IMG $TEST_IMG.base
-_make_test_img -b $TEST_IMG.base $image_size
+mv "$TEST_IMG" "$TEST_IMG.base"
+_make_test_img -b "$TEST_IMG.base" $image_size
 
 echo "Filling test image"
 echo
@@ -97,7 +97,7 @@ io_zero readv $(( offset + 32 * 1024 )) 512 1024 32
 _check_test_img
 
 # Rebase it on top of its base image
-$QEMU_IMG rebase -b $TEST_IMG.base $TEST_IMG
+$QEMU_IMG rebase -b "$TEST_IMG.base" "$TEST_IMG"
 
 _check_test_img
 
diff --git a/tests/qemu-iotests/029 b/tests/qemu-iotests/029
index 0ad5e45f88..b424726fc4 100755
--- a/tests/qemu-iotests/029
+++ b/tests/qemu-iotests/029
@@ -47,16 +47,16 @@ _supported_os Linux
 
 CLUSTER_SIZE=65536
 _make_test_img 64M
-$QEMU_IMG snapshot -c foo $TEST_IMG
-$QEMU_IO -c 'write -b 0 4k' $TEST_IMG | _filter_qemu_io
-$QEMU_IMG snapshot -a foo $TEST_IMG
+$QEMU_IMG snapshot -c foo "$TEST_IMG"
+$QEMU_IO -c 'write -b 0 4k' "$TEST_IMG" | _filter_qemu_io
+$QEMU_IMG snapshot -a foo "$TEST_IMG"
 _check_test_img
 
 CLUSTER_SIZE=1024
 _make_test_img 16M
-$QEMU_IMG snapshot -c foo $TEST_IMG
-$QEMU_IO -c 'write -b 0 4M' $TEST_IMG | _filter_qemu_io
-$QEMU_IMG snapshot -a foo $TEST_IMG
+$QEMU_IMG snapshot -c foo "$TEST_IMG"
+$QEMU_IO -c 'write -b 0 4M' "$TEST_IMG" | _filter_qemu_io
+$QEMU_IMG snapshot -a foo "$TEST_IMG"
 _check_test_img
 
 # success, all done
diff --git a/tests/qemu-iotests/031 b/tests/qemu-iotests/031
index 2d5e3b12d1..c9070b0513 100755
--- a/tests/qemu-iotests/031
+++ b/tests/qemu-iotests/031
@@ -56,22 +56,22 @@ for IMGOPTS in "compat=0.10" "compat=1.1"; do
     echo === Create image with unknown header extension ===
     echo
     _make_test_img 64M
-    ./qcow2.py $TEST_IMG add-header-ext 0x12345678 "This is a test header extension"
-    ./qcow2.py $TEST_IMG dump-header
+    ./qcow2.py "$TEST_IMG" add-header-ext 0x12345678 "This is a test header extension"
+    ./qcow2.py "$TEST_IMG" dump-header
     _check_test_img
 
     echo
     echo === Rewrite header with no backing file ===
     echo
-    $QEMU_IMG rebase -u -b "" $TEST_IMG
-    ./qcow2.py $TEST_IMG dump-header
+    $QEMU_IMG rebase -u -b "" "$TEST_IMG"
+    ./qcow2.py "$TEST_IMG" dump-header
     _check_test_img
 
     echo
     echo === Add a backing file and format ===
     echo
-    $QEMU_IMG rebase -u -b "/some/backing/file/path" -F host_device $TEST_IMG
-    ./qcow2.py $TEST_IMG dump-header
+    $QEMU_IMG rebase -u -b "/some/backing/file/path" -F host_device "$TEST_IMG"
+    ./qcow2.py "$TEST_IMG" dump-header
 done
 
 # success, all done
diff --git a/tests/qemu-iotests/032 b/tests/qemu-iotests/032
index 7155568a4f..b1ba5c3218 100755
--- a/tests/qemu-iotests/032
+++ b/tests/qemu-iotests/032
@@ -55,12 +55,12 @@ _make_test_img 64M
 
 # Allocate every other cluster so that afterwards a big write request will
 # actually loop a while and issue many I/O requests for the lower layer
-for i in $(seq 0 128 4096); do echo "write ${i}k 64k"; done | $QEMU_IO $TEST_IMG | _filter_qemu_io
+for i in $(seq 0 128 4096); do echo "write ${i}k 64k"; done | $QEMU_IO "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo === AIO request during close ===
 echo
-$QEMU_IO -c "aio_write 0 4M" -c "close" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "aio_write 0 4M" -c "close" "$TEST_IMG" | _filter_qemu_io
 _check_test_img
 
 # success, all done
diff --git a/tests/qemu-iotests/033 b/tests/qemu-iotests/033
index 9aee0784f6..ea3351c3e7 100755
--- a/tests/qemu-iotests/033
+++ b/tests/qemu-iotests/033
@@ -48,24 +48,24 @@ _make_test_img $size
 
 echo
 echo "== preparing image =="
-$QEMU_IO -c "write -P 0xa 0x200 0x400" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "write -P 0xa 0x20000 0x600" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "write -z 0x400 0x20000" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "write -P 0xa 0x200 0x400" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -P 0xa 0x20000 0x600" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -z 0x400 0x20000" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== verifying patterns (1) =="
-$QEMU_IO -c "read -P 0xa 0x200 0x200" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0x0 0x400 0x20000" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0xa 0x20400 0x200" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "read -P 0xa 0x200 0x200" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x0 0x400 0x20000" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0xa 0x20400 0x200" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== rewriting zeroes =="
-$QEMU_IO -c "write -P 0xb 0x10000 0x10000" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "write -z 0x10000 0x10000" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "write -P 0xb 0x10000 0x10000" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -z 0x10000 0x10000" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== verifying patterns (2) =="
-$QEMU_IO -c "read -P 0x0 0x400 0x20000" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "read -P 0x0 0x400 0x20000" "$TEST_IMG" | _filter_qemu_io
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/034 b/tests/qemu-iotests/034
index 8254df82ba..67f1959690 100755
--- a/tests/qemu-iotests/034
+++ b/tests/qemu-iotests/034
@@ -49,63 +49,63 @@ echo
 echo "== creating backing file for COW tests =="
 
 _make_test_img $size
-$QEMU_IO -c "write -P 0x55 0 1M" $TEST_IMG | _filter_qemu_io
-mv $TEST_IMG $TEST_IMG.base
+$QEMU_IO -c "write -P 0x55 0 1M" "$TEST_IMG" | _filter_qemu_io
+mv "$TEST_IMG" "$TEST_IMG.base"
 
-_make_test_img -b $TEST_IMG.base 6G
+_make_test_img -b "$TEST_IMG.base" 6G
 
 echo
 echo "== zero write with backing file =="
-$QEMU_IO -c "write -z 64k 192k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "write -z 513k 13k" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "write -z 64k 192k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -z 513k 13k" "$TEST_IMG" | _filter_qemu_io
 
 _check_test_img
 
 echo
 echo "== verifying patterns (3) =="
-$QEMU_IO -c "read -P 0x55 0 64k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0x0 64k 192k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0x55 256k 257k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0x0 513k 13k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0x55 526k 498k" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "read -P 0x55 0 64k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x0 64k 192k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x55 256k 257k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x0 513k 13k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x55 526k 498k" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== overwriting zero cluster =="
-$QEMU_IO -c "write -P 0xa 60k 8k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "write -P 0xb 64k 8k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "write -P 0xc 76k 4k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "write -P 0xd 252k 8k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "write -P 0xe 248k 8k" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "write -P 0xa 60k 8k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -P 0xb 64k 8k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -P 0xc 76k 4k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -P 0xd 252k 8k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -P 0xe 248k 8k" "$TEST_IMG" | _filter_qemu_io
 
 _check_test_img
 
 echo
 echo "== verifying patterns (4) =="
-$QEMU_IO -c "read -P 0x55 0 60k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0xa 60k 4k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0xb 64k 8k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0x0 72k 4k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0xc 76k 4k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0x0 80k 168k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0xe 248k 8k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0xd 256k 4k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0x55 260k 64k" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "read -P 0x55 0 60k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0xa 60k 4k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0xb 64k 8k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x0 72k 4k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0xc 76k 4k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x0 80k 168k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0xe 248k 8k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0xd 256k 4k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x55 260k 64k" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== re-zeroing overwritten area =="
-$QEMU_IO -c "write -z 64k 192k" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "write -z 64k 192k" "$TEST_IMG" | _filter_qemu_io
 
 _check_test_img
 
 echo
 echo "== verifying patterns (5) =="
-$QEMU_IO -c "read -P 0x55 0 60k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0xa 60k 4k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0x0 64k 192k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0xd 256k 4k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0x55 260k 253k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0x0 513k 13k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0x55 526k 498k" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "read -P 0x55 0 60k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0xa 60k 4k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x0 64k 192k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0xd 256k 4k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x55 260k 253k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x0 513k 13k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x55 526k 498k" "$TEST_IMG" | _filter_qemu_io
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/035 b/tests/qemu-iotests/035
index 9d2d3472e7..ebe9b8c925 100755
--- a/tests/qemu-iotests/035
+++ b/tests/qemu-iotests/035
@@ -59,7 +59,7 @@ function generate_requests() {
     done
 }
 
-generate_requests | $QEMU_IO $TEST_IMG | _filter_qemu_io |\
+generate_requests | $QEMU_IO "$TEST_IMG" | _filter_qemu_io |\
 	sed -e 's/bytes at offset [0-9]*/bytes at offset XXX/g'
 
 echo
diff --git a/tests/qemu-iotests/036 b/tests/qemu-iotests/036
index 4dbfc5724c..e049a645e7 100755
--- a/tests/qemu-iotests/036
+++ b/tests/qemu-iotests/036
@@ -53,15 +53,15 @@ IMGOPTS="compat=1.1"
 echo === Create image with unknown autoclear feature bit ===
 echo
 _make_test_img 64M
-./qcow2.py $TEST_IMG set-feature-bit autoclear 63
-./qcow2.py $TEST_IMG dump-header
+./qcow2.py "$TEST_IMG" set-feature-bit autoclear 63
+./qcow2.py "$TEST_IMG" dump-header
 
 echo
 echo === Repair image ===
 echo
 _check_test_img -r all
 
-./qcow2.py $TEST_IMG dump-header
+./qcow2.py "$TEST_IMG" dump-header
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/037 b/tests/qemu-iotests/037
index c11460b92f..743bae33d3 100755
--- a/tests/qemu-iotests/037
+++ b/tests/qemu-iotests/037
@@ -66,50 +66,50 @@ function backing_io()
     done
 }
 
-backing_io 0 256 write | $QEMU_IO $TEST_IMG | _filter_qemu_io
+backing_io 0 256 write | $QEMU_IO "$TEST_IMG" | _filter_qemu_io
 
-mv $TEST_IMG $TEST_IMG.base
+mv "$TEST_IMG" "$TEST_IMG.base"
 
-_make_test_img -b $TEST_IMG.base 6G
+_make_test_img -b "$TEST_IMG.base" 6G
 
 echo
 echo "== COW in a single cluster =="
-$QEMU_IO -c "write -P 0x77 0 2k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "write -P 0x88 6k 2k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "write -P 0x99 9k 2k" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "write -P 0x77 0 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -P 0x88 6k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -P 0x99 9k 2k" "$TEST_IMG" | _filter_qemu_io
 
-$QEMU_IO -c "read -P 0x77 0 2k" $TEST_IMG | _filter_qemu_io
-backing_io $((2 * 1024)) 8 read | $QEMU_IO $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0x88 6k 2k" $TEST_IMG | _filter_qemu_io
-backing_io $((8 * 1024)) 2 read | $QEMU_IO $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0x99 9k 2k" $TEST_IMG | _filter_qemu_io
-backing_io $((11 * 1024)) 2 read | $QEMU_IO $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "read -P 0x77 0 2k" "$TEST_IMG" | _filter_qemu_io
+backing_io $((2 * 1024)) 8 read | $QEMU_IO "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x88 6k 2k" "$TEST_IMG" | _filter_qemu_io
+backing_io $((8 * 1024)) 2 read | $QEMU_IO "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x99 9k 2k" "$TEST_IMG" | _filter_qemu_io
+backing_io $((11 * 1024)) 2 read | $QEMU_IO "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== COW in two-cluster allocations =="
-$QEMU_IO -c "write -P 0x77 16k 6k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "write -P 0x88 26k 6k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "write -P 0x99 33k 5k" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "write -P 0x77 16k 6k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -P 0x88 26k 6k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -P 0x99 33k 5k" "$TEST_IMG" | _filter_qemu_io
 
-$QEMU_IO -c "read -P 0x77 16k 6k" $TEST_IMG | _filter_qemu_io
-backing_io $((22 * 1024)) 8 read | $QEMU_IO $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0x88 26k 6k" $TEST_IMG | _filter_qemu_io
-backing_io $((32 * 1024)) 2 read | $QEMU_IO $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0x99 33k 5k" $TEST_IMG | _filter_qemu_io
-backing_io $((38 * 1024)) 4 read | $QEMU_IO $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "read -P 0x77 16k 6k" "$TEST_IMG" | _filter_qemu_io
+backing_io $((22 * 1024)) 8 read | $QEMU_IO "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x88 26k 6k" "$TEST_IMG" | _filter_qemu_io
+backing_io $((32 * 1024)) 2 read | $QEMU_IO "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x99 33k 5k" "$TEST_IMG" | _filter_qemu_io
+backing_io $((38 * 1024)) 4 read | $QEMU_IO "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== COW in multi-cluster allocations =="
-$QEMU_IO -c "write -P 0x77 48k 15k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "write -P 0x88 66k 14k" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "write -P 0x99 83k 15k" $TEST_IMG | _filter_qemu_io
-
-$QEMU_IO -c "read -P 0x77 48k 15k" $TEST_IMG | _filter_qemu_io
-backing_io $((63 * 1024)) 6 read | $QEMU_IO $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0x88 66k 14k" $TEST_IMG | _filter_qemu_io
-backing_io $((80 * 1024)) 6 read | $QEMU_IO $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0x99 83k 15k" $TEST_IMG | _filter_qemu_io
-backing_io $((98 * 1024)) 4 read | $QEMU_IO $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "write -P 0x77 48k 15k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -P 0x88 66k 14k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -P 0x99 83k 15k" "$TEST_IMG" | _filter_qemu_io
+
+$QEMU_IO -c "read -P 0x77 48k 15k" "$TEST_IMG" | _filter_qemu_io
+backing_io $((63 * 1024)) 6 read | $QEMU_IO "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x88 66k 14k" "$TEST_IMG" | _filter_qemu_io
+backing_io $((80 * 1024)) 6 read | $QEMU_IO "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x99 83k 15k" "$TEST_IMG" | _filter_qemu_io
+backing_io $((98 * 1024)) 4 read | $QEMU_IO "$TEST_IMG" | _filter_qemu_io
 
 _check_test_img
 
diff --git a/tests/qemu-iotests/038 b/tests/qemu-iotests/038
index 36125eab1e..7bb7906e7f 100755
--- a/tests/qemu-iotests/038
+++ b/tests/qemu-iotests/038
@@ -66,11 +66,11 @@ function backing_io()
     done
 }
 
-backing_io 0 256 write | $QEMU_IO $TEST_IMG | _filter_qemu_io
+backing_io 0 256 write | $QEMU_IO "$TEST_IMG" | _filter_qemu_io
 
-mv $TEST_IMG $TEST_IMG.base
+mv "$TEST_IMG" "$TEST_IMG.base"
 
-_make_test_img -b $TEST_IMG.base 6G
+_make_test_img -b "$TEST_IMG.base" 6G
 
 echo
 echo "== Some concurrent requests touching the same cluster =="
@@ -94,8 +94,9 @@ function overlay_io()
     echo aio_write -P 0x90 4080k 80k
 }
 
-overlay_io | $QEMU_IO $TEST_IMG | _filter_qemu_io |\
-	sed -e 's/bytes at offset [0-9]*/bytes at offset XXX/g'
+overlay_io | $QEMU_IO "$TEST_IMG" | _filter_qemu_io |\
+    sed -e 's/bytes at offset [0-9]*/bytes at offset XXX/g' \
+    -e 's/qemu-io> //g' | paste - - | sort | tr '\t' '\n'
 
 echo
 echo "== Verify image content =="
@@ -123,7 +124,7 @@ function verify_io()
     done
 }
 
-verify_io | $QEMU_IO $TEST_IMG | _filter_qemu_io
+verify_io | $QEMU_IO "$TEST_IMG" | _filter_qemu_io
 
 _check_test_img
 
diff --git a/tests/qemu-iotests/038.out b/tests/qemu-iotests/038.out
index 9cd0cd8771..96c2f849bb 100644
--- a/tests/qemu-iotests/038.out
+++ b/tests/qemu-iotests/038.out
@@ -517,7 +517,7 @@ qemu-io> wrote 65536/65536 bytes at offset 16711680
 qemu-io> Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base' 
 
 == Some concurrent requests touching the same cluster ==
-qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> wrote 65536/65536 bytes at offset XXX
+wrote 65536/65536 bytes at offset XXX
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset XXX
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -577,8 +577,6 @@ wrote 65536/65536 bytes at offset XXX
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset XXX
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-wrote 81920/81920 bytes at offset XXX
-80 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset XXX
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset XXX
@@ -645,8 +643,6 @@ wrote 65536/65536 bytes at offset XXX
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset XXX
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-wrote 81920/81920 bytes at offset XXX
-80 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset XXX
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset XXX
@@ -705,6 +701,10 @@ wrote 65536/65536 bytes at offset XXX
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset XXX
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 81920/81920 bytes at offset XXX
+80 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 81920/81920 bytes at offset XXX
+80 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 == Verify image content ==
 qemu-io> read 4096/4096 bytes at offset 2064384
diff --git a/tests/qemu-iotests/039 b/tests/qemu-iotests/039
index ae3517575c..f85b4ce63f 100755
--- a/tests/qemu-iotests/039
+++ b/tests/qemu-iotests/039
@@ -54,10 +54,10 @@ echo "== Checking that image is clean on shutdown =="
 IMGOPTS="compat=1.1,lazy_refcounts=on"
 _make_test_img $size
 
-$QEMU_IO -c "write -P 0x5a 0 512" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "write -P 0x5a 0 512" ""$TEST_IMG"" | _filter_qemu_io
 
 # The dirty bit must not be set
-./qcow2.py $TEST_IMG dump-header | grep incompatible_features
+./qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
 _check_test_img
 
 echo
@@ -68,20 +68,20 @@ _make_test_img $size
 
 old_ulimit=$(ulimit -c)
 ulimit -c 0 # do not produce a core dump on abort(3)
-$QEMU_IO -c "write -P 0x5a 0 512" -c "abort" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "write -P 0x5a 0 512" -c "abort" "$TEST_IMG" | _filter_qemu_io
 ulimit -c "$old_ulimit"
 
 # The dirty bit must be set
-./qcow2.py $TEST_IMG dump-header | grep incompatible_features
+./qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
 _check_test_img
 
 echo
 echo "== Read-only access must still work =="
 
-$QEMU_IO -r -c "read -P 0x5a 0 512" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -r -c "read -P 0x5a 0 512" "$TEST_IMG" | _filter_qemu_io
 
 # The dirty bit must be set
-./qcow2.py $TEST_IMG dump-header | grep incompatible_features
+./qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
 
 echo
 echo "== Repairing the image file must succeed =="
@@ -89,12 +89,12 @@ echo "== Repairing the image file must succeed =="
 _check_test_img -r all
 
 # The dirty bit must not be set
-./qcow2.py $TEST_IMG dump-header | grep incompatible_features
+./qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
 
 echo
 echo "== Data should still be accessible after repair =="
 
-$QEMU_IO -c "read -P 0x5a 0 512" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "read -P 0x5a 0 512" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== Opening a dirty image read/write should repair it =="
@@ -104,16 +104,16 @@ _make_test_img $size
 
 old_ulimit=$(ulimit -c)
 ulimit -c 0 # do not produce a core dump on abort(3)
-$QEMU_IO -c "write -P 0x5a 0 512" -c "abort" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "write -P 0x5a 0 512" -c "abort" "$TEST_IMG" | _filter_qemu_io
 ulimit -c "$old_ulimit"
 
 # The dirty bit must be set
-./qcow2.py $TEST_IMG dump-header | grep incompatible_features
+./qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
 
-$QEMU_IO -c "write 0 512" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "write 0 512" "$TEST_IMG" | _filter_qemu_io
 
 # The dirty bit must not be set
-./qcow2.py $TEST_IMG dump-header | grep incompatible_features
+./qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
 
 echo
 echo "== Creating an image file with lazy_refcounts=off =="
@@ -123,11 +123,11 @@ _make_test_img $size
 
 old_ulimit=$(ulimit -c)
 ulimit -c 0 # do not produce a core dump on abort(3)
-$QEMU_IO -c "write -P 0x5a 0 512" -c "abort" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "write -P 0x5a 0 512" -c "abort" "$TEST_IMG" | _filter_qemu_io
 ulimit -c "$old_ulimit"
 
 # The dirty bit must not be set since lazy_refcounts=off
-./qcow2.py $TEST_IMG dump-header | grep incompatible_features
+./qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
 _check_test_img
 
 # success, all done
diff --git a/tests/qemu-iotests/042 b/tests/qemu-iotests/042
index 16b2fdbd5e..94ce3a9cc3 100755
--- a/tests/qemu-iotests/042
+++ b/tests/qemu-iotests/042
@@ -48,27 +48,27 @@ echo "== Creating zero size image =="
 _make_test_img 0
 _check_test_img
 
-mv $TEST_IMG $TEST_IMG.orig
+mv "$TEST_IMG" "$TEST_IMG.orig"
 
 echo
 echo "== Converting the image =="
 
-$QEMU_IMG convert -O $IMGFMT $TEST_IMG.orig $TEST_IMG
+$QEMU_IMG convert -O $IMGFMT "$TEST_IMG.orig" "$TEST_IMG"
 _check_test_img
 
 echo
 echo "== Converting the image, compressed =="
 
 if [ "$IMGFMT" == "qcow2" ]; then
-    $QEMU_IMG convert -c -O $IMGFMT $TEST_IMG.orig $TEST_IMG
+    $QEMU_IMG convert -c -O $IMGFMT "$TEST_IMG.orig" "$TEST_IMG"
 fi
 _check_test_img
 
 echo
 echo "== Rebasing the image =="
 
-$QEMU_IMG rebase -u -b $TEST_IMG.orig $TEST_IMG
-$QEMU_IMG rebase -b $TEST_IMG.orig $TEST_IMG
+$QEMU_IMG rebase -u -b "$TEST_IMG.orig" "$TEST_IMG"
+$QEMU_IMG rebase -b "$TEST_IMG.orig" "$TEST_IMG"
 _check_test_img
 
 # success, all done
diff --git a/tests/qemu-iotests/043 b/tests/qemu-iotests/043
index 478773d102..d7f12319b3 100755
--- a/tests/qemu-iotests/043
+++ b/tests/qemu-iotests/043
@@ -31,7 +31,7 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_test_img
-    rm -f $TEST_IMG.[123].base
+    rm -f "$TEST_IMG".[123].base
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -47,39 +47,39 @@ _supported_os Linux
 
 size=128M
 _make_test_img $size
-$QEMU_IMG rebase -u -b $TEST_IMG $TEST_IMG
+$QEMU_IMG rebase -u -b "$TEST_IMG" "$TEST_IMG"
 
 echo
 echo "== backing file references self =="
 _img_info --backing-chain
 
 _make_test_img $size
-mv $TEST_IMG $TEST_IMG.base
-_make_test_img -b $TEST_IMG.base $size
-$QEMU_IMG rebase -u -b $TEST_IMG $TEST_IMG.base
+mv "$TEST_IMG" "$TEST_IMG.base"
+_make_test_img -b "$TEST_IMG.base" $size
+$QEMU_IMG rebase -u -b "$TEST_IMG" "$TEST_IMG.base"
 
 echo
 echo "== parent references self =="
 _img_info --backing-chain
 
 _make_test_img $size
-mv $TEST_IMG $TEST_IMG.1.base
-_make_test_img -b $TEST_IMG.1.base $size
-mv $TEST_IMG $TEST_IMG.2.base
-_make_test_img -b $TEST_IMG.2.base $size
-mv $TEST_IMG $TEST_IMG.3.base
-_make_test_img -b $TEST_IMG.3.base $size
-$QEMU_IMG rebase -u -b $TEST_IMG.2.base $TEST_IMG.1.base
+mv "$TEST_IMG" "$TEST_IMG.1.base"
+_make_test_img -b "$TEST_IMG.1.base" $size
+mv "$TEST_IMG" "$TEST_IMG.2.base"
+_make_test_img -b "$TEST_IMG.2.base" $size
+mv "$TEST_IMG" "$TEST_IMG.3.base"
+_make_test_img -b "$TEST_IMG.3.base" $size
+$QEMU_IMG rebase -u -b "$TEST_IMG.2.base" "$TEST_IMG.1.base"
 
 echo
 echo "== ancestor references another ancestor =="
 _img_info --backing-chain
 
 _make_test_img $size
-mv $TEST_IMG $TEST_IMG.1.base
-_make_test_img -b $TEST_IMG.1.base $size
-mv $TEST_IMG $TEST_IMG.2.base
-_make_test_img -b $TEST_IMG.2.base $size
+mv "$TEST_IMG" "$TEST_IMG.1.base"
+_make_test_img -b "$TEST_IMG.1.base" $size
+mv "$TEST_IMG" "$TEST_IMG.2.base"
+_make_test_img -b "$TEST_IMG.2.base" $size
 
 echo
 echo "== finite chain of length 3 (human) =="
diff --git a/tests/qemu-iotests/045 b/tests/qemu-iotests/045
index 2b6f1af27a..6be8fc4912 100755
--- a/tests/qemu-iotests/045
+++ b/tests/qemu-iotests/045
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# Tests for fdsets.
+# Tests for fdsets and getfd.
 #
 # Copyright (C) 2012 IBM Corp.
 #
@@ -125,5 +125,54 @@ class TestFdSets(iotests.QMPTestCase):
                 'No file descriptor supplied via SCM_RIGHTS')
         self.vm.shutdown()
 
+# Add fd at runtime, there are two ways: monitor related or fdset related
+class TestSCMFd(iotests.QMPTestCase):
+    def setUp(self):
+        self.vm = iotests.VM()
+        qemu_img('create', '-f', iotests.imgfmt, image0, '128K')
+        # Add an unused monitor, to verify it works fine when two monitor
+        # instances present
+        self.vm.add_monitor_telnet("0",4445)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(image0)
+
+    def _send_fd_by_SCM(self):
+        ret = self.vm.send_fd_scm(image0)
+        self.assertEqual(ret, 0, 'Failed to send fd with UNIX SCM')
+
+    def test_add_fd(self):
+        self._send_fd_by_SCM()
+        result = self.vm.qmp('add-fd', fdset_id=2, opaque='image0:r')
+        self.assert_qmp(result, 'return/fdset-id', 2)
+
+    def test_getfd(self):
+        self._send_fd_by_SCM()
+        result = self.vm.qmp('getfd', fdname='image0:r')
+        self.assert_qmp(result, 'return', {})
+
+    def test_getfd_invalid_fdname(self):
+        self._send_fd_by_SCM()
+        result = self.vm.qmp('getfd', fdname='0image0:r')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc',
+            "Parameter 'fdname' expects a name not starting with a digit")
+
+    def test_closefd(self):
+        self._send_fd_by_SCM()
+        result = self.vm.qmp('getfd', fdname='image0:r')
+        self.assert_qmp(result, 'return', {})
+        result = self.vm.qmp('closefd', fdname='image0:r')
+        self.assert_qmp(result, 'return', {})
+
+    def test_closefd_fd_not_found(self):
+        fdname = 'image0:r'
+        result = self.vm.qmp('closefd', fdname=fdname)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc',
+            "File descriptor named '%s' not found" % fdname)
+
 if __name__ == '__main__':
     iotests.main(supported_fmts=['raw'])
diff --git a/tests/qemu-iotests/045.out b/tests/qemu-iotests/045.out
index 3f8a935a08..e56cae021b 100644
--- a/tests/qemu-iotests/045.out
+++ b/tests/qemu-iotests/045.out
@@ -1,5 +1,5 @@
-......
+...........
 ----------------------------------------------------------------------
-Ran 6 tests
+Ran 11 tests
 
 OK
diff --git a/tests/qemu-iotests/046 b/tests/qemu-iotests/046
index 987bfff8fa..3f17ceb1b9 100755
--- a/tests/qemu-iotests/046
+++ b/tests/qemu-iotests/046
@@ -66,11 +66,11 @@ function backing_io()
     done
 }
 
-backing_io 0 32 write | $QEMU_IO $TEST_IMG | _filter_qemu_io
+backing_io 0 32 write | $QEMU_IO "$TEST_IMG" | _filter_qemu_io
 
-mv $TEST_IMG $TEST_IMG.base
+mv "$TEST_IMG" "$TEST_IMG.base"
 
-_make_test_img -b $TEST_IMG.base 6G
+_make_test_img -b "$TEST_IMG.base" 6G
 
 echo
 echo "== Some concurrent requests touching the same cluster =="
@@ -185,7 +185,7 @@ aio_flush
 EOF
 }
 
-overlay_io | $QEMU_IO blkdebug::$TEST_IMG | _filter_qemu_io |\
+overlay_io | $QEMU_IO blkdebug::"$TEST_IMG" | _filter_qemu_io |\
 	sed -e 's/bytes at offset [0-9]*/bytes at offset XXX/g'
 
 echo
@@ -252,7 +252,7 @@ function verify_io()
     echo read -P 17  0x11c000 0x4000
 }
 
-verify_io | $QEMU_IO $TEST_IMG | _filter_qemu_io
+verify_io | $QEMU_IO "$TEST_IMG" | _filter_qemu_io
 
 _check_test_img
 
diff --git a/tests/qemu-iotests/047 b/tests/qemu-iotests/047
index 0cf36b434f..c35cd096b8 100755
--- a/tests/qemu-iotests/047
+++ b/tests/qemu-iotests/047
@@ -66,7 +66,7 @@ read -P 0x55 1M 128k
 EOF
 }
 
-qemu_io_cmds | $QEMU_IO $TEST_IMG | _filter_qemu_io
+qemu_io_cmds | $QEMU_IO "$TEST_IMG" | _filter_qemu_io
 _check_test_img
 
 # success, all done
diff --git a/tests/qemu-iotests/048 b/tests/qemu-iotests/048
index 7cce049d2d..9b9d118ef3 100755
--- a/tests/qemu-iotests/048
+++ b/tests/qemu-iotests/048
@@ -31,13 +31,13 @@ _cleanup()
 {
     echo "Cleanup"
     _cleanup_test_img
-    rm ${TEST_IMG2}
+    rm "${TEST_IMG2}"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
 _compare()
 {
-    $QEMU_IMG compare "$@" $TEST_IMG ${TEST_IMG2}
+    $QEMU_IMG compare "$@" "$TEST_IMG" "${TEST_IMG2}"
     echo $?
 }
 
@@ -59,12 +59,12 @@ _make_test_img $size
 io_pattern write 524288 $CLUSTER_SIZE $CLUSTER_SIZE 4 45
 
 # Compare identical images
-cp $TEST_IMG ${TEST_IMG2}
+cp "$TEST_IMG" "${TEST_IMG2}"
 _compare
 _compare -q
 
 # Compare images with different size
-$QEMU_IMG resize $TEST_IMG +512M
+$QEMU_IMG resize "$TEST_IMG" +512M
 _compare
 _compare -s
 
diff --git a/tests/qemu-iotests/049 b/tests/qemu-iotests/049
index 6c6017e2d2..93aa0ea55f 100755
--- a/tests/qemu-iotests/049
+++ b/tests/qemu-iotests/049
@@ -63,13 +63,13 @@ sizes+="1024.0 1024.0b 1.5k 1.5K 1.5M 1.5G 1.5T"
 echo "== 1. Traditional size parameter =="
 echo
 for s in $sizes; do
-    test_qemu_img create -f $IMGFMT $TEST_IMG $s
+    test_qemu_img create -f $IMGFMT "$TEST_IMG" $s
 done
 
 echo "== 2. Specifying size via -o =="
 echo
 for s in $sizes; do
-    test_qemu_img create -f $IMGFMT -o size=$s $TEST_IMG
+    test_qemu_img create -f $IMGFMT -o size=$s "$TEST_IMG"
 done
 
 echo "== 3. Invalid sizes =="
@@ -77,8 +77,8 @@ echo
 sizes="-1024 -1k 1kilobyte foobar"
 
 for s in $sizes; do
-    test_qemu_img create -f $IMGFMT $TEST_IMG -- $s
-    test_qemu_img create -f $IMGFMT -o size=$s $TEST_IMG
+    test_qemu_img create -f $IMGFMT "$TEST_IMG" -- $s
+    test_qemu_img create -f $IMGFMT -o size=$s "$TEST_IMG"
 done
 
 echo "== Check correct interpretation of suffixes for cluster size =="
@@ -87,35 +87,35 @@ sizes="1024 1024b 1k 1K 1M "
 sizes+="1024.0 1024.0b 0.5k 0.5K 0.5M"
 
 for s in $sizes; do
-    test_qemu_img create -f $IMGFMT -o cluster_size=$s $TEST_IMG 64M
+    test_qemu_img create -f $IMGFMT -o cluster_size=$s "$TEST_IMG" 64M
 done
 
 echo "== Check compat level option =="
 echo
-test_qemu_img create -f $IMGFMT -o compat=0.10 $TEST_IMG 64M
-test_qemu_img create -f $IMGFMT -o compat=1.1 $TEST_IMG 64M
+test_qemu_img create -f $IMGFMT -o compat=0.10 "$TEST_IMG" 64M
+test_qemu_img create -f $IMGFMT -o compat=1.1 "$TEST_IMG" 64M
 
-test_qemu_img create -f $IMGFMT -o compat=0.42 $TEST_IMG 64M
-test_qemu_img create -f $IMGFMT -o compat=foobar $TEST_IMG 64M
+test_qemu_img create -f $IMGFMT -o compat=0.42 "$TEST_IMG" 64M
+test_qemu_img create -f $IMGFMT -o compat=foobar "$TEST_IMG" 64M
 
 echo "== Check preallocation option =="
 echo
-test_qemu_img create -f $IMGFMT -o preallocation=off $TEST_IMG 64M
-test_qemu_img create -f $IMGFMT -o preallocation=metadata $TEST_IMG 64M
-test_qemu_img create -f $IMGFMT -o preallocation=1234 $TEST_IMG 64M
+test_qemu_img create -f $IMGFMT -o preallocation=off "$TEST_IMG" 64M
+test_qemu_img create -f $IMGFMT -o preallocation=metadata "$TEST_IMG" 64M
+test_qemu_img create -f $IMGFMT -o preallocation=1234 "$TEST_IMG" 64M
 
 echo "== Check encryption option =="
 echo
-test_qemu_img create -f $IMGFMT -o encryption=off $TEST_IMG 64M
-test_qemu_img create -f $IMGFMT -o encryption=on $TEST_IMG 64M
+test_qemu_img create -f $IMGFMT -o encryption=off "$TEST_IMG" 64M
+test_qemu_img create -f $IMGFMT -o encryption=on "$TEST_IMG" 64M
 
 echo "== Check lazy_refcounts option (only with v3) =="
 echo
-test_qemu_img create -f $IMGFMT -o compat=1.1,lazy_refcounts=off $TEST_IMG 64M
-test_qemu_img create -f $IMGFMT -o compat=1.1,lazy_refcounts=on $TEST_IMG 64M
+test_qemu_img create -f $IMGFMT -o compat=1.1,lazy_refcounts=off "$TEST_IMG" 64M
+test_qemu_img create -f $IMGFMT -o compat=1.1,lazy_refcounts=on "$TEST_IMG" 64M
 
-test_qemu_img create -f $IMGFMT -o compat=0.10,lazy_refcounts=off $TEST_IMG 64M
-test_qemu_img create -f $IMGFMT -o compat=0.10,lazy_refcounts=on $TEST_IMG 64M
+test_qemu_img create -f $IMGFMT -o compat=0.10,lazy_refcounts=off "$TEST_IMG" 64M
+test_qemu_img create -f $IMGFMT -o compat=0.10,lazy_refcounts=on "$TEST_IMG" 64M
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/049.out b/tests/qemu-iotests/049.out
index d2f0efe16d..ceb23289fd 100644
--- a/tests/qemu-iotests/049.out
+++ b/tests/qemu-iotests/049.out
@@ -96,7 +96,7 @@ qemu-img: Image size must be less than 8 EiB!
 
 qemu-img create -f qcow2 -o size=-1024 TEST_DIR/t.qcow2
 qemu-img: qcow2 doesn't support shrinking images yet
-qemu-img: Formatting or formatting option not supported for file format 'qcow2'
+qemu-img: TEST_DIR/t.qcow2: Could not resize image: Operation not supported
 Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=-1024 encryption=off cluster_size=65536 lazy_refcounts=off 
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 -- -1k
@@ -104,7 +104,7 @@ qemu-img: Image size must be less than 8 EiB!
 
 qemu-img create -f qcow2 -o size=-1k TEST_DIR/t.qcow2
 qemu-img: qcow2 doesn't support shrinking images yet
-qemu-img: Formatting or formatting option not supported for file format 'qcow2'
+qemu-img: TEST_DIR/t.qcow2: Could not resize image: Operation not supported
 Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=-1024 encryption=off cluster_size=65536 lazy_refcounts=off 
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 -- 1kilobyte
@@ -120,7 +120,7 @@ qemu-img: kilobytes, megabytes, gigabytes, terabytes, petabytes and exabytes.
 
 qemu-img create -f qcow2 -o size=foobar TEST_DIR/t.qcow2
 qemu-img: Parameter 'size' expects a size
-qemu-img: Invalid options for file format 'qcow2'.
+qemu-img: TEST_DIR/t.qcow2: Invalid options for file format 'qcow2'.
 
 == Check correct interpretation of suffixes for cluster size ==
 
@@ -163,13 +163,11 @@ qemu-img create -f qcow2 -o compat=1.1 TEST_DIR/t.qcow2 64M
 Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='1.1' encryption=off cluster_size=65536 lazy_refcounts=off 
 
 qemu-img create -f qcow2 -o compat=0.42 TEST_DIR/t.qcow2 64M
-Invalid compatibility level: '0.42'
-qemu-img: TEST_DIR/t.qcow2: error while creating qcow2: Invalid argument
+qemu-img: TEST_DIR/t.qcow2: Invalid compatibility level: '0.42'
 Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='0.42' encryption=off cluster_size=65536 lazy_refcounts=off 
 
 qemu-img create -f qcow2 -o compat=foobar TEST_DIR/t.qcow2 64M
-Invalid compatibility level: 'foobar'
-qemu-img: TEST_DIR/t.qcow2: error while creating qcow2: Invalid argument
+qemu-img: TEST_DIR/t.qcow2: Invalid compatibility level: 'foobar'
 Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='foobar' encryption=off cluster_size=65536 lazy_refcounts=off 
 
 == Check preallocation option ==
@@ -181,8 +179,7 @@ qemu-img create -f qcow2 -o preallocation=metadata TEST_DIR/t.qcow2 64M
 Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 preallocation='metadata' lazy_refcounts=off 
 
 qemu-img create -f qcow2 -o preallocation=1234 TEST_DIR/t.qcow2 64M
-Invalid preallocation mode: '1234'
-qemu-img: TEST_DIR/t.qcow2: error while creating qcow2: Invalid argument
+qemu-img: TEST_DIR/t.qcow2: Invalid preallocation mode: '1234'
 Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 preallocation='1234' lazy_refcounts=off 
 
 == Check encryption option ==
@@ -205,8 +202,7 @@ qemu-img create -f qcow2 -o compat=0.10,lazy_refcounts=off TEST_DIR/t.qcow2 64M
 Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='0.10' encryption=off cluster_size=65536 lazy_refcounts=off 
 
 qemu-img create -f qcow2 -o compat=0.10,lazy_refcounts=on TEST_DIR/t.qcow2 64M
-Lazy refcounts only supported with compatibility level 1.1 and above (use compat=1.1 or greater)
-qemu-img: TEST_DIR/t.qcow2: error while creating qcow2: Invalid argument
+qemu-img: TEST_DIR/t.qcow2: Lazy refcounts only supported with compatibility level 1.1 and above (use compat=1.1 or greater)
 Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='0.10' encryption=off cluster_size=65536 lazy_refcounts=on 
 
 *** done
diff --git a/tests/qemu-iotests/050 b/tests/qemu-iotests/050
index 05793e2d4b..07802bc49c 100755
--- a/tests/qemu-iotests/050
+++ b/tests/qemu-iotests/050
@@ -31,8 +31,8 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_test_img
-    rm -f $TEST_IMG.old
-    rm -f $TEST_IMG.new
+    rm -f "$TEST_IMG.old"
+    rm -f "$TEST_IMG.new"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -53,21 +53,21 @@ echo "== Creating images =="
 
 size=10M
 _make_test_img $size
-$QEMU_IO -c "write -P 0x40 0 1048576" $TEST_IMG | _filter_qemu_io
-mv $TEST_IMG $TEST_IMG.old
+$QEMU_IO -c "write -P 0x40 0 1048576" "$TEST_IMG" | _filter_qemu_io
+mv "$TEST_IMG" "$TEST_IMG.old"
 
 _make_test_img $size
-$QEMU_IO -c "write -P 0x5a 0 1048576" $TEST_IMG | _filter_qemu_io
-mv $TEST_IMG $TEST_IMG.new
+$QEMU_IO -c "write -P 0x5a 0 1048576" "$TEST_IMG" | _filter_qemu_io
+mv "$TEST_IMG" "$TEST_IMG.new"
 
-_make_test_img -b $TEST_IMG.old $size
-$QEMU_IO -c "write -z 0 1048576" $TEST_IMG | _filter_qemu_io
+_make_test_img -b "$TEST_IMG.old" $size
+$QEMU_IO -c "write -z 0 1048576" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== Rebasing the image =="
 
-$QEMU_IMG rebase -b $TEST_IMG.new $TEST_IMG
-$QEMU_IO -c "read -P 0x00 0 1048576" $TEST_IMG | _filter_qemu_io
+$QEMU_IMG rebase -b "$TEST_IMG.new" "$TEST_IMG"
+$QEMU_IO -c "read -P 0x00 0 1048576" "$TEST_IMG" | _filter_qemu_io
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/051 b/tests/qemu-iotests/051
index 1f39c6ad21..356c3756f4 100755
--- a/tests/qemu-iotests/051
+++ b/tests/qemu-iotests/051
@@ -45,7 +45,14 @@ _supported_os Linux
 function do_run_qemu()
 {
     echo Testing: "$@"
-    echo quit | $QEMU -nographic -monitor stdio -serial none "$@"
+    (
+        if ! test -t 0; then
+            while read cmd; do
+                echo $cmd
+            done
+        fi
+        echo quit
+    ) | $QEMU -nographic -monitor stdio -serial none "$@"
     echo
 }
 
@@ -57,26 +64,34 @@ function run_qemu()
 size=128M
 
 _make_test_img $size
+cp $TEST_IMG $TEST_IMG.orig
+mv $TEST_IMG $TEST_IMG.base
+_make_test_img -b $TEST_IMG.base $size
 
 echo
 echo === Unknown option ===
 echo
 
-run_qemu -drive file=$TEST_IMG,format=qcow2,unknown_opt=
-run_qemu -drive file=$TEST_IMG,format=qcow2,unknown_opt=on
-run_qemu -drive file=$TEST_IMG,format=qcow2,unknown_opt=1234
-run_qemu -drive file=$TEST_IMG,format=qcow2,unknown_opt=foo
+run_qemu -drive file="$TEST_IMG",format=qcow2,unknown_opt=
+run_qemu -drive file="$TEST_IMG",format=qcow2,unknown_opt=on
+run_qemu -drive file="$TEST_IMG",format=qcow2,unknown_opt=1234
+run_qemu -drive file="$TEST_IMG",format=qcow2,unknown_opt=foo
 
+echo
+echo === Overriding backing file ===
+echo
+
+echo "info block" | run_qemu -drive file=$TEST_IMG,driver=qcow2,backing.file.filename=$TEST_IMG.orig -nodefaults
 
 echo
 echo === Enable and disable lazy refcounting on the command line, plus some invalid values ===
 echo
 
-run_qemu -drive file=$TEST_IMG,format=qcow2,lazy-refcounts=on
-run_qemu -drive file=$TEST_IMG,format=qcow2,lazy-refcounts=off
-run_qemu -drive file=$TEST_IMG,format=qcow2,lazy-refcounts=
-run_qemu -drive file=$TEST_IMG,format=qcow2,lazy-refcounts=42
-run_qemu -drive file=$TEST_IMG,format=qcow2,lazy-refcounts=foo
+run_qemu -drive file="$TEST_IMG",format=qcow2,lazy-refcounts=on
+run_qemu -drive file="$TEST_IMG",format=qcow2,lazy-refcounts=off
+run_qemu -drive file="$TEST_IMG",format=qcow2,lazy-refcounts=
+run_qemu -drive file="$TEST_IMG",format=qcow2,lazy-refcounts=42
+run_qemu -drive file="$TEST_IMG",format=qcow2,lazy-refcounts=foo
 
 
 echo
@@ -85,8 +100,8 @@ echo
 
 _make_test_img -ocompat=0.10 $size
 
-run_qemu -drive file=$TEST_IMG,format=qcow2,lazy-refcounts=on
-run_qemu -drive file=$TEST_IMG,format=qcow2,lazy-refcounts=off
+run_qemu -drive file="$TEST_IMG",format=qcow2,lazy-refcounts=on
+run_qemu -drive file="$TEST_IMG",format=qcow2,lazy-refcounts=off
 
 echo
 echo === No medium ===
@@ -112,21 +127,21 @@ echo
 echo === Read-only ===
 echo
 
-run_qemu -drive file=$TEST_IMG,if=floppy,readonly=on
-run_qemu -drive file=$TEST_IMG,if=ide,media=cdrom,readonly=on
-run_qemu -drive file=$TEST_IMG,if=scsi,media=cdrom,readonly=on
+run_qemu -drive file="$TEST_IMG",if=floppy,readonly=on
+run_qemu -drive file="$TEST_IMG",if=ide,media=cdrom,readonly=on
+run_qemu -drive file="$TEST_IMG",if=scsi,media=cdrom,readonly=on
 
-run_qemu -drive file=$TEST_IMG,if=ide,readonly=on
-run_qemu -drive file=$TEST_IMG,if=virtio,readonly=on
-run_qemu -drive file=$TEST_IMG,if=scsi,readonly=on
+run_qemu -drive file="$TEST_IMG",if=ide,readonly=on
+run_qemu -drive file="$TEST_IMG",if=virtio,readonly=on
+run_qemu -drive file="$TEST_IMG",if=scsi,readonly=on
 
-run_qemu -drive file=$TEST_IMG,if=none,id=disk,readonly=on -device ide-cd,drive=disk
-run_qemu -drive file=$TEST_IMG,if=none,id=disk,readonly=on -device lsi53c895a -device scsi-cd,drive=disk
+run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device ide-cd,drive=disk
+run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device lsi53c895a -device scsi-cd,drive=disk
 
-run_qemu -drive file=$TEST_IMG,if=none,id=disk,readonly=on -device ide-drive,drive=disk
-run_qemu -drive file=$TEST_IMG,if=none,id=disk,readonly=on -device ide-hd,drive=disk
-run_qemu -drive file=$TEST_IMG,if=none,id=disk,readonly=on -device lsi53c895a -device scsi-disk,drive=disk
-run_qemu -drive file=$TEST_IMG,if=none,id=disk,readonly=on -device lsi53c895a -device scsi-hd,drive=disk
+run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device ide-drive,drive=disk
+run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device ide-hd,drive=disk
+run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device lsi53c895a -device scsi-disk,drive=disk
+run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device lsi53c895a -device scsi-hd,drive=disk
 
 echo
 echo === Cache modes ===
@@ -146,8 +161,8 @@ echo
 echo === Specifying the protocol layer ===
 echo
 
-run_qemu -drive file=$TEST_IMG,file.driver=file
-run_qemu -drive file=$TEST_IMG,file.driver=qcow2
+run_qemu -drive file="$TEST_IMG",file.driver=file
+run_qemu -drive file="$TEST_IMG",file.driver=qcow2
 
 echo
 echo === Parsing protocol from file name ===
diff --git a/tests/qemu-iotests/051.out b/tests/qemu-iotests/051.out
index 86e989cc6a..2839e32807 100644
--- a/tests/qemu-iotests/051.out
+++ b/tests/qemu-iotests/051.out
@@ -1,23 +1,30 @@
 QA output created by 051
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.base' 
 
 === Unknown option ===
 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=: Block format 'qcow2' used by device 'ide0-hd0' doesn't support the option 'unknown_opt'
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=: could not open disk image TEST_DIR/t.qcow2: Invalid argument
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=: could not open disk image TEST_DIR/t.qcow2: Block format 'qcow2' used by device 'ide0-hd0' doesn't support the option 'unknown_opt'
 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=on
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=on: Block format 'qcow2' used by device 'ide0-hd0' doesn't support the option 'unknown_opt'
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=on: could not open disk image TEST_DIR/t.qcow2: Invalid argument
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=on: could not open disk image TEST_DIR/t.qcow2: Block format 'qcow2' used by device 'ide0-hd0' doesn't support the option 'unknown_opt'
 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=1234
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=1234: Block format 'qcow2' used by device 'ide0-hd0' doesn't support the option 'unknown_opt'
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=1234: could not open disk image TEST_DIR/t.qcow2: Invalid argument
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=1234: could not open disk image TEST_DIR/t.qcow2: Block format 'qcow2' used by device 'ide0-hd0' doesn't support the option 'unknown_opt'
 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=foo
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=foo: Block format 'qcow2' used by device 'ide0-hd0' doesn't support the option 'unknown_opt'
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=foo: could not open disk image TEST_DIR/t.qcow2: Invalid argument
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=foo: could not open disk image TEST_DIR/t.qcow2: Block format 'qcow2' used by device 'ide0-hd0' doesn't support the option 'unknown_opt'
+
+
+=== Overriding backing file ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,driver=qcow2,backing.file.filename=TEST_DIR/t.qcow2.orig -nodefaults
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo block
+ide0-hd0: TEST_DIR/t.qcow2 (qcow2)
+    Backing file:     TEST_DIR/t.qcow2.orig (chain depth: 1)
+ [not inserted](qemu) qququiquit
 
 
 === Enable and disable lazy refcounting on the command line, plus some invalid values ===
@@ -31,24 +38,20 @@ QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) qququiquit

 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=: Parameter 'lazy-refcounts' expects 'on' or 'off'
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=: could not open disk image TEST_DIR/t.qcow2: Invalid argument
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=: could not open disk image TEST_DIR/t.qcow2: Parameter 'lazy-refcounts' expects 'on' or 'off'
 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=42
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=42: Parameter 'lazy-refcounts' expects 'on' or 'off'
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=42: could not open disk image TEST_DIR/t.qcow2: Invalid argument
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=42: could not open disk image TEST_DIR/t.qcow2: Parameter 'lazy-refcounts' expects 'on' or 'off'
 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=foo
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=foo: Parameter 'lazy-refcounts' expects 'on' or 'off'
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=foo: could not open disk image TEST_DIR/t.qcow2: Invalid argument
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=foo: could not open disk image TEST_DIR/t.qcow2: Parameter 'lazy-refcounts' expects 'on' or 'off'
 
 
 === With version 2 images enabling lazy refcounts must fail ===
 
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=on
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=on: Lazy refcounts require a qcow2 image with at least qemu 1.1 compatibility level
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=on: could not open disk image TEST_DIR/t.qcow2: Invalid argument
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=on: could not open disk image TEST_DIR/t.qcow2: Lazy refcounts require a qcow2 image with at least qemu 1.1 compatibility level
 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=off
 QEMU X.Y.Z monitor - type 'help' for more information

@@ -136,7 +139,10 @@ QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) qququiquit

 
 Testing: -drive file=TEST_DIR/t.qcow2,if=ide,readonly=on
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,if=ide,readonly=on: read-only not supported by this bus type
+QEMU X.Y.Z monitor - type 'help' for more information

+(qemu) QEMU_PROG: Can't use a read-only drive
+QEMU_PROG: Device initialization failed.
+QEMU_PROG: Initialization of device ide-hd failed
 
 Testing: -drive file=TEST_DIR/t.qcow2,if=virtio,readonly=on
 QEMU X.Y.Z monitor - type 'help' for more information

@@ -208,21 +214,18 @@ QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) qququiquit

 
 Testing: -drive file=TEST_DIR/t.qcow2,file.driver=qcow2
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,file.driver=qcow2: Can't use 'qcow2' as a block driver for the protocol level
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,file.driver=qcow2: could not open disk image TEST_DIR/t.qcow2: Invalid argument
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,file.driver=qcow2: could not open disk image TEST_DIR/t.qcow2: Can't use 'qcow2' as a block driver for the protocol level
 
 
 === Parsing protocol from file name ===
 
 Testing: -hda foo:bar
-QEMU_PROG: -hda foo:bar: Unknown protocol
-QEMU_PROG: -hda foo:bar: could not open disk image foo:bar: No such file or directory
+QEMU_PROG: -hda foo:bar: could not open disk image foo:bar: Unknown protocol
 
 Testing: -drive file=foo:bar
-QEMU_PROG: -drive file=foo:bar: Unknown protocol
-QEMU_PROG: -drive file=foo:bar: could not open disk image foo:bar: No such file or directory
+QEMU_PROG: -drive file=foo:bar: could not open disk image foo:bar: Unknown protocol
 
 Testing: -drive file.filename=foo:bar
-QEMU_PROG: -drive file.filename=foo:bar: could not open disk image ide0-hd0: No such file or directory
+QEMU_PROG: -drive file.filename=foo:bar: could not open disk image ide0-hd0: Could not open file: No such file or directory
 
 *** done
diff --git a/tests/qemu-iotests/052 b/tests/qemu-iotests/052
index 14a5126635..f5f9683e68 100755
--- a/tests/qemu-iotests/052
+++ b/tests/qemu-iotests/052
@@ -41,6 +41,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 _supported_fmt generic
 _supported_proto generic
 _supported_os Linux
+_unsupported_qemu_io_options --nocache
 
 
 size=128M
@@ -48,12 +49,12 @@ _make_test_img $size
 
 echo
 echo "== reading whole image =="
-$QEMU_IO -s -c "read 0 $size" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -s -c "read 0 $size" "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "== writing whole image does not modify image =="
-$QEMU_IO -s -c "write -P 0xa 0 $size" $TEST_IMG | _filter_qemu_io
-$QEMU_IO -c "read -P 0 0 $size" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -s -c "write -P 0xa 0 $size" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 0 $size" "$TEST_IMG" | _filter_qemu_io
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/053 b/tests/qemu-iotests/053
index bc56992582..e589e5f126 100755
--- a/tests/qemu-iotests/053
+++ b/tests/qemu-iotests/053
@@ -30,7 +30,7 @@ status=1	# failure is the default!
 
 _cleanup()
 {
-	rm -f $TEST_IMG.orig
+	rm -f "$TEST_IMG.orig"
 	_cleanup_test_img
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
@@ -47,13 +47,13 @@ echo
 echo "== Creating single sector image =="
 
 _make_test_img 512
-$QEMU_IO -c "write -P0xa 0 512" $TEST_IMG | _filter_qemu_io
-mv $TEST_IMG $TEST_IMG.orig
+$QEMU_IO -c "write -P0xa 0 512" "$TEST_IMG" | _filter_qemu_io
+mv "$TEST_IMG" "$TEST_IMG.orig"
 
 echo
 echo "== Converting the image, compressed =="
 
-$QEMU_IMG convert -c -O $IMGFMT $TEST_IMG.orig $TEST_IMG
+$QEMU_IMG convert -c -O $IMGFMT "$TEST_IMG.orig" "$TEST_IMG"
 _check_test_img
 
 echo
@@ -64,7 +64,7 @@ _img_info | grep '^virtual size:'
 echo
 echo "== Verifying the compressed image =="
 
-$QEMU_IO -c "read -P0xa 0 512" $TEST_IMG | _filter_qemu_io
+$QEMU_IO -c "read -P0xa 0 512" "$TEST_IMG" | _filter_qemu_io
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/054 b/tests/qemu-iotests/054
index b36042958c..5a0d1b16c2 100755
--- a/tests/qemu-iotests/054
+++ b/tests/qemu-iotests/054
@@ -49,7 +49,7 @@ _make_test_img $((1024*1024))T
 echo
 echo "creating too large image (1 EB) using qcow2.py"
 _make_test_img 4G
-./qcow2.py $TEST_IMG set-header size $((1024 ** 6))
+./qcow2.py "$TEST_IMG" set-header size $((1024 ** 6))
 _check_test_img
 
 # success, all done
diff --git a/tests/qemu-iotests/054.out b/tests/qemu-iotests/054.out
index 2f357c271d..7161d6e50b 100644
--- a/tests/qemu-iotests/054.out
+++ b/tests/qemu-iotests/054.out
@@ -1,10 +1,10 @@
 QA output created by 054
 
 creating too large image (1 EB)
-qemu-img: The image size is too large for file format 'qcow2' (try using a larger cluster size)
+qemu-img: TEST_DIR/t.IMGFMT: The image size is too large for file format 'IMGFMT' (try using a larger cluster size)
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1152921504606846976 
 
 creating too large image (1 EB) using qcow2.py
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 
-qemu-img: Could not open 'TEST_DIR/t.qcow2': File too large
+qemu-img: Could not open 'TEST_DIR/t.qcow2': Image is too big
 *** done
diff --git a/tests/qemu-iotests/057 b/tests/qemu-iotests/057
new file mode 100755
index 0000000000..9cdd582e39
--- /dev/null
+++ b/tests/qemu-iotests/057
@@ -0,0 +1,259 @@
+#!/usr/bin/env python
+#
+# Tests for internal snapshot.
+#
+# Copyright (C) 2013 IBM, Inc.
+#
+# Based on 055.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import time
+import os
+import iotests
+from iotests import qemu_img, qemu_io
+
+test_drv_base_name = 'drive'
+
+class ImageSnapshotTestCase(iotests.QMPTestCase):
+    image_len = 120 * 1024 * 1024 # MB
+
+    def __init__(self, *args):
+        self.expect = []
+        super(ImageSnapshotTestCase, self).__init__(*args)
+
+    def _setUp(self, test_img_base_name, image_num):
+        self.vm = iotests.VM()
+        for i in range(0, image_num):
+            filename = '%s%d' % (test_img_base_name, i)
+            img = os.path.join(iotests.test_dir, filename)
+            device = '%s%d' % (test_drv_base_name, i)
+            qemu_img('create', '-f', iotests.imgfmt, img, str(self.image_len))
+            self.vm.add_drive(img)
+            self.expect.append({'image': img, 'device': device,
+                                'snapshots': [],
+                                'snapshots_name_counter': 0})
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        for dev_expect in self.expect:
+            os.remove(dev_expect['image'])
+
+    def createSnapshotInTransaction(self, snapshot_num, abort = False):
+        actions = []
+        for dev_expect in self.expect:
+            num = dev_expect['snapshots_name_counter']
+            for j in range(0, snapshot_num):
+                name = '%s_sn%d' % (dev_expect['device'], num)
+                num = num + 1
+                if abort == False:
+                    dev_expect['snapshots'].append({'name': name})
+                    dev_expect['snapshots_name_counter'] = num
+                actions.append({
+                    'type': 'blockdev-snapshot-internal-sync',
+                    'data': { 'device': dev_expect['device'],
+                              'name': name },
+                })
+
+        if abort == True:
+            actions.append({
+                'type': 'abort',
+                'data': {},
+            })
+
+        result = self.vm.qmp('transaction', actions = actions)
+
+        if abort == True:
+            self.assert_qmp(result, 'error/class', 'GenericError')
+        else:
+            self.assert_qmp(result, 'return', {})
+
+    def verifySnapshotInfo(self):
+        result = self.vm.qmp('query-block')
+
+        # Verify each expected result
+        for dev_expect in self.expect:
+            # 1. Find the returned image value and snapshot info
+            image_result = None
+            for device in result['return']:
+                if device['device'] == dev_expect['device']:
+                    image_result = device['inserted']['image']
+                    break
+            self.assertTrue(image_result != None)
+            # Do not consider zero snapshot case now
+            sn_list_result = image_result['snapshots']
+            sn_list_expect = dev_expect['snapshots']
+
+            # 2. Verify it with expect
+            self.assertTrue(len(sn_list_result) == len(sn_list_expect))
+
+            for sn_expect in sn_list_expect:
+                sn_result = None
+                for sn in sn_list_result:
+                    if sn_expect['name'] == sn['name']:
+                        sn_result = sn
+                        break
+                self.assertTrue(sn_result != None)
+                # Fill in the detail info
+                sn_expect.update(sn_result)
+
+    def deleteSnapshot(self, device, id = None, name = None):
+        sn_list_expect = None
+        sn_expect = None
+
+        self.assertTrue(id != None or name != None)
+
+        # Fill in the detail info include ID
+        self.verifySnapshotInfo()
+
+        #find the expected snapshot list
+        for dev_expect in self.expect:
+            if dev_expect['device'] == device:
+                sn_list_expect = dev_expect['snapshots']
+                break
+        self.assertTrue(sn_list_expect != None)
+
+        if id != None and name != None:
+            for sn in sn_list_expect:
+                if sn['id'] == id and sn['name'] == name:
+                    sn_expect = sn
+                    result = \
+                          self.vm.qmp('blockdev-snapshot-delete-internal-sync',
+                                      device = device,
+                                      id = id,
+                                      name = name)
+                    break
+        elif id != None:
+            for sn in sn_list_expect:
+                if sn['id'] == id:
+                    sn_expect = sn
+                    result = \
+                          self.vm.qmp('blockdev-snapshot-delete-internal-sync',
+                                      device = device,
+                                      id = id)
+                    break
+        else:
+            for sn in sn_list_expect:
+                if sn['name'] == name:
+                    sn_expect = sn
+                    result = \
+                          self.vm.qmp('blockdev-snapshot-delete-internal-sync',
+                                      device = device,
+                                      name = name)
+                    break
+
+        self.assertTrue(sn_expect != None)
+
+        self.assert_qmp(result, 'return', sn_expect)
+        sn_list_expect.remove(sn_expect)
+
+class TestSingleTransaction(ImageSnapshotTestCase):
+    def setUp(self):
+        self._setUp('test_a.img', 1)
+
+    def test_create(self):
+        self.createSnapshotInTransaction(1)
+        self.verifySnapshotInfo()
+
+    def test_error_name_empty(self):
+        actions = [{'type': 'blockdev-snapshot-internal-sync',
+                    'data': { 'device': self.expect[0]['device'],
+                              'name': '' },
+                  }]
+        result = self.vm.qmp('transaction', actions = actions)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+    def test_error_device(self):
+        actions = [{'type': 'blockdev-snapshot-internal-sync',
+                    'data': { 'device': 'drive_error',
+                              'name': 'a' },
+                  }]
+        result = self.vm.qmp('transaction', actions = actions)
+        self.assert_qmp(result, 'error/class', 'DeviceNotFound')
+
+    def test_error_exist(self):
+        self.createSnapshotInTransaction(1)
+        self.verifySnapshotInfo()
+        actions = [{'type': 'blockdev-snapshot-internal-sync',
+                    'data': { 'device': self.expect[0]['device'],
+                              'name': self.expect[0]['snapshots'][0] },
+                  }]
+        result = self.vm.qmp('transaction', actions = actions)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+class TestMultipleTransaction(ImageSnapshotTestCase):
+    def setUp(self):
+        self._setUp('test_b.img', 2)
+
+    def test_create(self):
+        self.createSnapshotInTransaction(3)
+        self.verifySnapshotInfo()
+
+    def test_abort(self):
+        self.createSnapshotInTransaction(2)
+        self.verifySnapshotInfo()
+        self.createSnapshotInTransaction(3, abort = True)
+        self.verifySnapshotInfo()
+
+class TestSnapshotDelete(ImageSnapshotTestCase):
+    def setUp(self):
+        self._setUp('test_c.img', 1)
+
+    def test_delete_with_id(self):
+        self.createSnapshotInTransaction(2)
+        self.verifySnapshotInfo()
+        self.deleteSnapshot(self.expect[0]['device'],
+                            id = self.expect[0]['snapshots'][0]['id'])
+        self.verifySnapshotInfo()
+
+    def test_delete_with_name(self):
+        self.createSnapshotInTransaction(3)
+        self.verifySnapshotInfo()
+        self.deleteSnapshot(self.expect[0]['device'],
+                            name = self.expect[0]['snapshots'][1]['name'])
+        self.verifySnapshotInfo()
+
+    def test_delete_with_id_and_name(self):
+        self.createSnapshotInTransaction(4)
+        self.verifySnapshotInfo()
+        self.deleteSnapshot(self.expect[0]['device'],
+                            id = self.expect[0]['snapshots'][2]['id'],
+                            name = self.expect[0]['snapshots'][2]['name'])
+        self.verifySnapshotInfo()
+
+
+    def test_error_device(self):
+        result = self.vm.qmp('blockdev-snapshot-delete-internal-sync',
+                              device = 'drive_error',
+                              id = '0')
+        self.assert_qmp(result, 'error/class', 'DeviceNotFound')
+
+    def test_error_no_id_and_name(self):
+        result = self.vm.qmp('blockdev-snapshot-delete-internal-sync',
+                              device = self.expect[0]['device'])
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+    def test_error_snapshot_not_exist(self):
+        self.createSnapshotInTransaction(2)
+        self.verifySnapshotInfo()
+        result = self.vm.qmp('blockdev-snapshot-delete-internal-sync',
+                              device = self.expect[0]['device'],
+                              id = self.expect[0]['snapshots'][0]['id'],
+                              name = self.expect[0]['snapshots'][1]['name'])
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=['qcow2'])
diff --git a/tests/qemu-iotests/057.out b/tests/qemu-iotests/057.out
new file mode 100644
index 0000000000..281b69efea
--- /dev/null
+++ b/tests/qemu-iotests/057.out
@@ -0,0 +1,5 @@
+............
+----------------------------------------------------------------------
+Ran 12 tests
+
+OK
diff --git a/tests/qemu-iotests/059 b/tests/qemu-iotests/059
index b03429dd01..b81c575d94 100755
--- a/tests/qemu-iotests/059
+++ b/tests/qemu-iotests/059
@@ -47,24 +47,33 @@ capacity_offset=16
 granularity_offset=20
 grain_table_size_offset=44
 
-echo "=== Testing invalid granularity ==="
 echo
+echo "=== Testing invalid granularity ==="
 _make_test_img 64M
 poke_file "$TEST_IMG" "$granularity_offset" "\xff\xff\xff\xff\xff\xff\xff\xff"
-{ $QEMU_IO -c "read 0 512" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
+{ $QEMU_IO -c "read 0 512" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir
 
-echo "=== Testing too big L2 table size ==="
 echo
+echo "=== Testing too big L2 table size ==="
 _make_test_img 64M
 poke_file "$TEST_IMG" "$grain_table_size_offset" "\xff\xff\xff\xff"
-{ $QEMU_IO -c "read 0 512" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
+{ $QEMU_IO -c "read 0 512" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir
 
-echo "=== Testing too big L1 table size ==="
 echo
+echo "=== Testing too big L1 table size ==="
 _make_test_img 64M
 poke_file "$TEST_IMG" "$capacity_offset" "\xff\xff\xff\xff"
 poke_file "$TEST_IMG" "$grain_table_size_offset" "\x01\x00\x00\x00"
-{ $QEMU_IO -c "read 0 512" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
+{ $QEMU_IO -c "read 0 512" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir
+
+echo
+echo "=== Testing monolithicFlat creation and opening ==="
+IMGOPTS="subformat=monolithicFlat" _make_test_img 2G
+$QEMU_IMG info $TEST_IMG | _filter_testdir
+
+echo
+echo "=== Testing monolithicFlat with zeroed_grain ==="
+IMGOPTS="subformat=monolithicFlat,zeroed_grain=on" _make_test_img 2G
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out
index 9e715e5a95..9b12efb466 100644
--- a/tests/qemu-iotests/059.out
+++ b/tests/qemu-iotests/059.out
@@ -1,20 +1,29 @@
 QA output created by 059
-=== Testing invalid granularity ===
 
+=== Testing invalid granularity ===
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-invalid granularity, image may be corrupt
-qemu-io: can't open device TEST_DIR/t.vmdk
+qemu-io: can't open device TEST_DIR/t.vmdk: Invalid granularity, image may be corrupt
 no file open, try 'help open'
-=== Testing too big L2 table size ===
 
+=== Testing too big L2 table size ===
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 L2 table size too big
-qemu-io: can't open device TEST_DIR/t.vmdk
+qemu-io: can't open device TEST_DIR/t.vmdk: Could not open 'TEST_DIR/t.vmdk': Wrong medium type
 no file open, try 'help open'
-=== Testing too big L1 table size ===
 
+=== Testing too big L1 table size ===
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-L1 size too big
-qemu-io: can't open device TEST_DIR/t.vmdk
+qemu-io: can't open device TEST_DIR/t.vmdk: L1 size too big
 no file open, try 'help open'
+
+=== Testing monolithicFlat creation and opening ===
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648
+image: TEST_DIR/t.vmdk
+file format: vmdk
+virtual size: 2.0G (2147483648 bytes)
+disk size: 4.0K
+
+=== Testing monolithicFlat with zeroed_grain ===
+qemu-img: TEST_DIR/t.IMGFMT: Flat image can't enable zeroed grain
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648
 *** done
diff --git a/tests/qemu-iotests/060 b/tests/qemu-iotests/060
index 65bb09f023..bbb19090a1 100755
--- a/tests/qemu-iotests/060
+++ b/tests/qemu-iotests/060
@@ -21,10 +21,10 @@
 # creator
 owner=mreitz@redhat.com
 
-seq=`basename $0`
+seq="$(basename $0)"
 echo "QA output created by $seq"
 
-here=`pwd`
+here="$PWD"
 tmp=/tmp/$$
 status=1	# failure is the default!
 
@@ -47,9 +47,15 @@ rt_offset=65536  # 0x10000 (XXX: just an assumption)
 rb_offset=131072 # 0x20000 (XXX: just an assumption)
 l1_offset=196608 # 0x30000 (XXX: just an assumption)
 l2_offset=262144 # 0x40000 (XXX: just an assumption)
+l2_offset_after_snapshot=524288 # 0x80000 (XXX: just an assumption)
 
 IMGOPTS="compat=1.1"
 
+OPEN_RW="open -o overlap-check=all $TEST_IMG"
+# Overlap checks are done before write operations only, therefore opening an
+# image read-only makes the overlap-check option irrelevant
+OPEN_RO="open -r $TEST_IMG"
+
 echo
 echo "=== Testing L2 reference into L1 ==="
 echo
@@ -65,16 +71,18 @@ _check_test_img
 ./qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
 
 # Try to write something, thereby forcing the corrupt bit to be set
-$QEMU_IO -c "write -P 0x2a 0 512" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "$OPEN_RW" -c "write -P 0x2a 0 512" | _filter_qemu_io
 
 # The corrupt bit must now be set
 ./qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
 
 # Try to open the image R/W (which should fail)
-$QEMU_IO -c "read 0 512" "$TEST_IMG" 2>&1 | _filter_qemu_io | sed -e "s/can't open device .*$/can't open device/"
+$QEMU_IO -c "$OPEN_RW" -c "read 0 512" 2>&1 | _filter_qemu_io \
+                                            | _filter_testdir \
+                                            | _filter_imgfmt
 
 # Try to open it RO (which should succeed)
-$QEMU_IO -c "read 0 512" -r "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "$OPEN_RO" -c "read 0 512" | _filter_qemu_io
 
 # We could now try to fix the image, but this would probably fail (how should an
 # L2 table linked onto the L1 table be fixed?)
@@ -92,7 +100,7 @@ poke_file "$TEST_IMG" "$(($rb_offset+8))" "\x00\x01"
 poke_file "$TEST_IMG" "$l2_offset" "\x80\x00\x00\x00\x00\x02\x00\x00"
 _check_test_img
 ./qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
-$QEMU_IO -c "write -P 0x2a 0 512" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "$OPEN_RW" -c "write -P 0x2a 0 512" | _filter_qemu_io
 ./qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
 
 # Try to fix it
@@ -102,8 +110,33 @@ _check_test_img -r all
 ./qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
 
 # Look if it's really really fixed
-$QEMU_IO -c "write -P 0x2a 0 512" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "$OPEN_RW" -c "write -P 0x2a 0 512" | _filter_qemu_io
+./qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
+
+echo
+echo "=== Testing cluster data reference into inactive L2 table ==="
+echo
+_make_test_img 64M
+$QEMU_IO -c "$OPEN_RW" -c "write -P 1 0 512" | _filter_qemu_io
+$QEMU_IMG snapshot -c foo "$TEST_IMG"
+$QEMU_IO -c "$OPEN_RW" -c "write -P 2 0 512" | _filter_qemu_io
+# The inactive L2 table remains at its old offset
+poke_file "$TEST_IMG" "$l2_offset_after_snapshot" \
+                      "\x80\x00\x00\x00\x00\x04\x00\x00"
+_check_test_img
 ./qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
+$QEMU_IO -c "$OPEN_RW" -c "write -P 3 0 512" | _filter_qemu_io
+./qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
+_check_test_img -r all
+./qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
+$QEMU_IO -c "$OPEN_RW" -c "write -P 4 0 512" | _filter_qemu_io
+./qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
+
+# Check data
+$QEMU_IO -c "$OPEN_RO" -c "read -P 4 0 512" | _filter_qemu_io
+$QEMU_IMG snapshot -a foo "$TEST_IMG"
+_check_test_img
+$QEMU_IO -c "$OPEN_RO" -c "read -P 1 0 512" | _filter_qemu_io
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/060.out b/tests/qemu-iotests/060.out
index ca4583a4a4..6c7bdbb2f2 100644
--- a/tests/qemu-iotests/060.out
+++ b/tests/qemu-iotests/060.out
@@ -11,9 +11,7 @@ incompatible_features     0x0
 qcow2: Preventing invalid write on metadata (overlaps with active L1 table); image marked as corrupt.
 write failed: Input/output error
 incompatible_features     0x2
-qcow2: Image is corrupt; cannot be opened read/write.
-qemu-io: can't open device
-no file open, try 'help open'
+qemu-io: can't open device TEST_DIR/t.IMGFMT: IMGFMT: Image is corrupt; cannot be opened read/write
 read 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
@@ -41,4 +39,43 @@ incompatible_features     0x0
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 incompatible_features     0x0
+
+=== Testing cluster data reference into inactive L2 table ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+ERROR cluster 4 refcount=1 reference=2
+Leaked cluster 9 refcount=1 reference=0
+
+1 errors were found on the image.
+Data may be corrupted, or further writes to the image may corrupt it.
+
+1 leaked clusters were found on the image.
+This means waste of disk space, but no harm to data.
+incompatible_features     0x0
+qcow2: Preventing invalid write on metadata (overlaps with inactive L2 table); image marked as corrupt.
+write failed: Input/output error
+incompatible_features     0x2
+Repairing cluster 4 refcount=1 reference=2
+Repairing cluster 9 refcount=1 reference=0
+Repairing OFLAG_COPIED data cluster: l2_entry=8000000000040000 refcount=2
+The following inconsistencies were found and repaired:
+
+    1 leaked clusters
+    2 corruptions
+
+Double checking the fixed image now...
+No errors were found on the image.
+incompatible_features     0x0
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+incompatible_features     0x0
+read 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
+read 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 *** done
diff --git a/tests/qemu-iotests/061 b/tests/qemu-iotests/061
new file mode 100755
index 0000000000..fa9319da26
--- /dev/null
+++ b/tests/qemu-iotests/061
@@ -0,0 +1,215 @@
+#!/bin/bash
+#
+# Test case for image option amendment in qcow2.
+#
+# Copyright (C) 2013 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+# This tests qocw2-specific low-level functionality
+_supported_fmt qcow2
+_supported_proto generic
+_supported_os Linux
+
+echo
+echo "=== Testing version downgrade with zero expansion ==="
+echo
+IMGOPTS="compat=1.1,lazy_refcounts=on" _make_test_img 64M
+$QEMU_IO -c "write -z 0 128k" "$TEST_IMG" | _filter_qemu_io
+./qcow2.py "$TEST_IMG" dump-header
+$QEMU_IMG amend -o "compat=0.10" "$TEST_IMG"
+./qcow2.py "$TEST_IMG" dump-header
+$QEMU_IO -c "read -P 0 0 128k" "$TEST_IMG" | _filter_qemu_io
+_check_test_img
+
+echo
+echo "=== Testing dirty version downgrade ==="
+echo
+IMGOPTS="compat=1.1,lazy_refcounts=on" _make_test_img 64M
+$QEMU_IO -c "write -P 0x2a 0 128k" -c flush -c abort "$TEST_IMG" | _filter_qemu_io
+./qcow2.py "$TEST_IMG" dump-header
+$QEMU_IMG amend -o "compat=0.10" "$TEST_IMG"
+./qcow2.py "$TEST_IMG" dump-header
+$QEMU_IO -c "read -P 0x2a 0 128k" "$TEST_IMG" | _filter_qemu_io
+_check_test_img
+
+echo
+echo "=== Testing version downgrade with unknown compat/autoclear flags ==="
+echo
+IMGOPTS="compat=1.1" _make_test_img 64M
+./qcow2.py "$TEST_IMG" set-feature-bit compatible 42
+./qcow2.py "$TEST_IMG" set-feature-bit autoclear 42
+./qcow2.py "$TEST_IMG" dump-header
+$QEMU_IMG amend -o "compat=0.10" "$TEST_IMG"
+./qcow2.py "$TEST_IMG" dump-header
+_check_test_img
+
+echo
+echo "=== Testing version upgrade and resize ==="
+echo
+IMGOPTS="compat=0.10" _make_test_img 64M
+$QEMU_IO -c "write -P 0x2a 42M 64k" "$TEST_IMG" | _filter_qemu_io
+./qcow2.py "$TEST_IMG" dump-header
+$QEMU_IMG amend -o "compat=1.1,lazy_refcounts=on,size=128M" "$TEST_IMG"
+./qcow2.py "$TEST_IMG" dump-header
+$QEMU_IO -c "read -P 0x2a 42M 64k" "$TEST_IMG" | _filter_qemu_io
+_check_test_img
+
+echo
+echo "=== Testing dirty lazy_refcounts=off ==="
+echo
+IMGOPTS="compat=1.1,lazy_refcounts=on" _make_test_img 64M
+$QEMU_IO -c "write -P 0x2a 0 128k" -c flush -c abort "$TEST_IMG" | _filter_qemu_io
+./qcow2.py "$TEST_IMG" dump-header
+$QEMU_IMG amend -o "lazy_refcounts=off" "$TEST_IMG"
+./qcow2.py "$TEST_IMG" dump-header
+$QEMU_IO -c "read -P 0x2a 0 128k" "$TEST_IMG" | _filter_qemu_io
+_check_test_img
+
+echo
+echo "=== Testing backing file ==="
+echo
+IMGOPTS="compat=1.1" _make_test_img 64M
+IMGOPTS="compat=1.1" TEST_IMG="$TEST_IMG.base" _make_test_img 64M
+$QEMU_IO -c "write -P 0x2a 0 128k" "$TEST_IMG.base" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 0 128k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IMG amend -o "backing_file=$TEST_IMG.base,backing_fmt=qcow2" "$TEST_IMG"
+$QEMU_IO -c "read -P 0x2a 0 128k" "$TEST_IMG" | _filter_qemu_io
+_check_test_img
+
+echo
+echo "=== Testing invalid configurations ==="
+echo
+IMGOPTS="compat=0.10" _make_test_img 64M
+$QEMU_IMG amend -o "lazy_refcounts=on" "$TEST_IMG"
+$QEMU_IMG amend -o "compat=1.1" "$TEST_IMG" # actually valid
+$QEMU_IMG amend -o "compat=0.10,lazy_refcounts=on" "$TEST_IMG"
+$QEMU_IMG amend -o "compat=0.42" "$TEST_IMG"
+$QEMU_IMG amend -o "foo=bar" "$TEST_IMG"
+$QEMU_IMG amend -o "cluster_size=1k" "$TEST_IMG"
+$QEMU_IMG amend -o "encryption=on" "$TEST_IMG"
+$QEMU_IMG amend -o "preallocation=on" "$TEST_IMG"
+
+echo
+echo "=== Testing correct handling of unset value ==="
+echo
+IMGOPTS="compat=1.1,cluster_size=1k" _make_test_img 64M
+echo "Should work:"
+$QEMU_IMG amend -o "lazy_refcounts=on" "$TEST_IMG"
+echo "Should not work:" # Just to know which of these tests actually fails
+$QEMU_IMG amend -o "cluster_size=64k" "$TEST_IMG"
+
+echo
+echo "=== Testing zero expansion on inactive clusters ==="
+echo
+IMGOPTS="compat=1.1" _make_test_img 64M
+$QEMU_IO -c "write -z 0 128k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IMG snapshot -c foo "$TEST_IMG"
+$QEMU_IO -c "write -P 0x2a 0 128k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IMG amend -o "compat=0.10" "$TEST_IMG"
+_check_test_img
+$QEMU_IO -c "read -P 0x2a 0 128k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IMG snapshot -a foo "$TEST_IMG"
+_check_test_img
+$QEMU_IO -c "read -P 0 0 128k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "=== Testing zero expansion on shared L2 table ==="
+echo
+IMGOPTS="compat=1.1" _make_test_img 64M
+$QEMU_IO -c "write -z 0 128k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IMG snapshot -c foo "$TEST_IMG"
+$QEMU_IMG amend -o "compat=0.10" "$TEST_IMG"
+_check_test_img
+$QEMU_IO -c "read -P 0 0 128k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IMG snapshot -a foo "$TEST_IMG"
+_check_test_img
+$QEMU_IO -c "read -P 0 0 128k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "=== Testing zero expansion on backed image ==="
+echo
+IMGOPTS="compat=1.1" TEST_IMG="$TEST_IMG.base" _make_test_img 64M
+$QEMU_IO -c "write -P 0x2a 0 128k" "$TEST_IMG.base" | _filter_qemu_io
+IMGOPTS="compat=1.1,backing_file=$TEST_IMG.base" _make_test_img 64M
+$QEMU_IO -c "read -P 0x2a 0 128k" -c "write -z 0 64k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IMG amend -o "compat=0.10" "$TEST_IMG"
+_check_test_img
+$QEMU_IO -c "read -P 0 0 64k" -c "read -P 0x2a 64k 64k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "=== Testing zero expansion on backed inactive clusters ==="
+echo
+IMGOPTS="compat=1.1" TEST_IMG="$TEST_IMG.base" _make_test_img 64M
+$QEMU_IO -c "write -P 0x2a 0 128k" "$TEST_IMG.base" | _filter_qemu_io
+IMGOPTS="compat=1.1,backing_file=$TEST_IMG.base" _make_test_img 64M
+$QEMU_IO -c "write -z 0 64k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IMG snapshot -c foo "$TEST_IMG"
+$QEMU_IO -c "write -P 0x42 0 128k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IMG amend -o "compat=0.10" "$TEST_IMG"
+_check_test_img
+$QEMU_IO -c "read -P 0x42 0 128k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IMG snapshot -a foo "$TEST_IMG"
+_check_test_img
+$QEMU_IO -c "read -P 0 0 64k" -c "read -P 0x2a 64k 64k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "=== Testing zero expansion on backed image with shared L2 table ==="
+echo
+IMGOPTS="compat=1.1" TEST_IMG="$TEST_IMG.base" _make_test_img 64M
+$QEMU_IO -c "write -P 0x2a 0 128k" "$TEST_IMG.base" | _filter_qemu_io
+IMGOPTS="compat=1.1,backing_file=$TEST_IMG.base" _make_test_img 64M
+$QEMU_IO -c "write -z 0 128k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IMG snapshot -c foo "$TEST_IMG"
+$QEMU_IMG amend -o "compat=0.10" "$TEST_IMG"
+_check_test_img
+$QEMU_IO -c "read -P 0 0 128k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IMG snapshot -a foo "$TEST_IMG"
+_check_test_img
+$QEMU_IO -c "read -P 0 0 128k" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "=== Testing preallocated zero expansion on full image ==="
+echo
+IMGOPTS="compat=1.1" TEST_IMG="$TEST_IMG" _make_test_img 64M
+$QEMU_IO -c "write -P 0x2a 0 64M" "$TEST_IMG" -c "write -z 0 64M" | _filter_qemu_io
+$QEMU_IMG amend -o "compat=0.10" "$TEST_IMG"
+_check_test_img
+$QEMU_IO -c "read -P 0 0 64M" "$TEST_IMG" | _filter_qemu_io
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/061.out b/tests/qemu-iotests/061.out
new file mode 100644
index 0000000000..4027e0077e
--- /dev/null
+++ b/tests/qemu-iotests/061.out
@@ -0,0 +1,387 @@
+QA output created by 061
+
+=== Testing version downgrade with zero expansion ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+wrote 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+magic                     0x514649fb
+version                   3
+backing_file_offset       0x0
+backing_file_size         0x0
+cluster_bits              16
+size                      67108864
+crypt_method              0
+l1_size                   1
+l1_table_offset           0x30000
+refcount_table_offset     0x10000
+refcount_table_clusters   1
+nb_snapshots              0
+snapshot_offset           0x0
+incompatible_features     0x0
+compatible_features       0x1
+autoclear_features        0x0
+refcount_order            4
+header_length             104
+
+magic                     0x514649fb
+version                   2
+backing_file_offset       0x0
+backing_file_size         0x0
+cluster_bits              16
+size                      67108864
+crypt_method              0
+l1_size                   1
+l1_table_offset           0x30000
+refcount_table_offset     0x10000
+refcount_table_clusters   1
+nb_snapshots              0
+snapshot_offset           0x0
+incompatible_features     0x0
+compatible_features       0x0
+autoclear_features        0x0
+refcount_order            4
+header_length             72
+
+Header extension:
+magic                     0x6803f857
+length                    144
+data                      <binary>
+
+read 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
+
+=== Testing dirty version downgrade ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+wrote 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+magic                     0x514649fb
+version                   3
+backing_file_offset       0x0
+backing_file_size         0x0
+cluster_bits              16
+size                      67108864
+crypt_method              0
+l1_size                   1
+l1_table_offset           0x30000
+refcount_table_offset     0x10000
+refcount_table_clusters   1
+nb_snapshots              0
+snapshot_offset           0x0
+incompatible_features     0x1
+compatible_features       0x1
+autoclear_features        0x0
+refcount_order            4
+header_length             104
+
+Repairing cluster 5 refcount=0 reference=1
+Repairing cluster 6 refcount=0 reference=1
+magic                     0x514649fb
+version                   2
+backing_file_offset       0x0
+backing_file_size         0x0
+cluster_bits              16
+size                      67108864
+crypt_method              0
+l1_size                   1
+l1_table_offset           0x30000
+refcount_table_offset     0x10000
+refcount_table_clusters   1
+nb_snapshots              0
+snapshot_offset           0x0
+incompatible_features     0x0
+compatible_features       0x0
+autoclear_features        0x0
+refcount_order            4
+header_length             72
+
+Header extension:
+magic                     0x6803f857
+length                    144
+data                      <binary>
+
+read 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
+
+=== Testing version downgrade with unknown compat/autoclear flags ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+magic                     0x514649fb
+version                   3
+backing_file_offset       0x0
+backing_file_size         0x0
+cluster_bits              16
+size                      67108864
+crypt_method              0
+l1_size                   1
+l1_table_offset           0x30000
+refcount_table_offset     0x10000
+refcount_table_clusters   1
+nb_snapshots              0
+snapshot_offset           0x0
+incompatible_features     0x0
+compatible_features       0x40000000000
+autoclear_features        0x40000000000
+refcount_order            4
+header_length             104
+
+magic                     0x514649fb
+version                   2
+backing_file_offset       0x0
+backing_file_size         0x0
+cluster_bits              16
+size                      67108864
+crypt_method              0
+l1_size                   1
+l1_table_offset           0x30000
+refcount_table_offset     0x10000
+refcount_table_clusters   1
+nb_snapshots              0
+snapshot_offset           0x0
+incompatible_features     0x0
+compatible_features       0x0
+autoclear_features        0x0
+refcount_order            4
+header_length             72
+
+Header extension:
+magic                     0x6803f857
+length                    144
+data                      <binary>
+
+No errors were found on the image.
+
+=== Testing version upgrade and resize ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+wrote 65536/65536 bytes at offset 44040192
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+magic                     0x514649fb
+version                   2
+backing_file_offset       0x0
+backing_file_size         0x0
+cluster_bits              16
+size                      67108864
+crypt_method              0
+l1_size                   1
+l1_table_offset           0x30000
+refcount_table_offset     0x10000
+refcount_table_clusters   1
+nb_snapshots              0
+snapshot_offset           0x0
+incompatible_features     0x0
+compatible_features       0x0
+autoclear_features        0x0
+refcount_order            4
+header_length             72
+
+magic                     0x514649fb
+version                   3
+backing_file_offset       0x0
+backing_file_size         0x0
+cluster_bits              16
+size                      134217728
+crypt_method              0
+l1_size                   1
+l1_table_offset           0x30000
+refcount_table_offset     0x10000
+refcount_table_clusters   1
+nb_snapshots              0
+snapshot_offset           0x0
+incompatible_features     0x0
+compatible_features       0x1
+autoclear_features        0x0
+refcount_order            4
+header_length             104
+
+Header extension:
+magic                     0x6803f857
+length                    144
+data                      <binary>
+
+read 65536/65536 bytes at offset 44040192
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
+
+=== Testing dirty lazy_refcounts=off ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+wrote 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+magic                     0x514649fb
+version                   3
+backing_file_offset       0x0
+backing_file_size         0x0
+cluster_bits              16
+size                      67108864
+crypt_method              0
+l1_size                   1
+l1_table_offset           0x30000
+refcount_table_offset     0x10000
+refcount_table_clusters   1
+nb_snapshots              0
+snapshot_offset           0x0
+incompatible_features     0x1
+compatible_features       0x1
+autoclear_features        0x0
+refcount_order            4
+header_length             104
+
+Repairing cluster 5 refcount=0 reference=1
+Repairing cluster 6 refcount=0 reference=1
+magic                     0x514649fb
+version                   3
+backing_file_offset       0x0
+backing_file_size         0x0
+cluster_bits              16
+size                      67108864
+crypt_method              0
+l1_size                   1
+l1_table_offset           0x30000
+refcount_table_offset     0x10000
+refcount_table_clusters   1
+nb_snapshots              0
+snapshot_offset           0x0
+incompatible_features     0x0
+compatible_features       0x0
+autoclear_features        0x0
+refcount_order            4
+header_length             104
+
+Header extension:
+magic                     0x6803f857
+length                    144
+data                      <binary>
+
+read 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
+
+=== Testing backing file ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
+wrote 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
+
+=== Testing invalid configurations ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Lazy refcounts only supported with compatibility level 1.1 and above (use compat=1.1 or greater)
+qemu-img: Error while amending options: Invalid argument
+Lazy refcounts only supported with compatibility level 1.1 and above (use compat=1.1 or greater)
+qemu-img: Error while amending options: Invalid argument
+Unknown compatibility level 0.42.
+qemu-img: Error while amending options: Invalid argument
+Unknown option 'foo'
+qemu-img: Invalid options for file format 'qcow2'
+Changing the cluster size is not supported.
+qemu-img: Error while amending options: Operation not supported
+Changing the encryption flag is not supported.
+qemu-img: Error while amending options: Operation not supported
+Cannot change preallocation mode.
+qemu-img: Error while amending options: Operation not supported
+
+=== Testing correct handling of unset value ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Should work:
+Should not work:
+Changing the cluster size is not supported.
+qemu-img: Error while amending options: Operation not supported
+
+=== Testing zero expansion on inactive clusters ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+wrote 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
+read 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
+read 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Testing zero expansion on shared L2 table ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+wrote 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
+read 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
+read 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Testing zero expansion on backed image ===
+
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
+wrote 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base' 
+read 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Testing zero expansion on backed inactive clusters ===
+
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
+wrote 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base' 
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
+read 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Testing zero expansion on backed image with shared L2 table ===
+
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
+wrote 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base' 
+wrote 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
+read 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
+read 131072/131072 bytes at offset 0
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Testing preallocated zero expansion on full image ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+wrote 67108864/67108864 bytes at offset 0
+64 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 67108864/67108864 bytes at offset 0
+64 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
+read 67108864/67108864 bytes at offset 0
+64 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+*** done
diff --git a/tests/qemu-iotests/063 b/tests/qemu-iotests/063
index de0cbbd8bb..2ab8f20e02 100755
--- a/tests/qemu-iotests/063
+++ b/tests/qemu-iotests/063
@@ -32,7 +32,7 @@ status=1	# failure is the default!
 _cleanup()
 {
 	_cleanup_test_img
-	rm -f $TEST_IMG.orig $TEST_IMG.raw $TEST_IMG.raw2
+	rm -f "$TEST_IMG.orig" "$TEST_IMG.raw" "$TEST_IMG.raw2"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -49,47 +49,47 @@ _make_test_img 4M
 
 echo "== Testing conversion with -n fails with no target file =="
 # check .orig file does not exist
-rm -f $TEST_IMG.orig
-if $QEMU_IMG convert -f $IMGFMT -O $IMGFMT -n $TEST_IMG $TEST_IMG.orig >/dev/null 2>&1; then
+rm -f "$TEST_IMG.orig"
+if $QEMU_IMG convert -f $IMGFMT -O $IMGFMT -n "$TEST_IMG" "$TEST_IMG.orig" >/dev/null 2>&1; then
     exit 1
 fi
 
 echo "== Testing conversion with -n succeeds with a target file =="
-rm -f $TEST_IMG.orig
-cp $TEST_IMG $TEST_IMG.orig
-if ! $QEMU_IMG convert -f $IMGFMT -O $IMGFMT -n $TEST_IMG $TEST_IMG.orig ; then
+rm -f "$TEST_IMG.orig"
+cp "$TEST_IMG" "$TEST_IMG.orig"
+if ! $QEMU_IMG convert -f $IMGFMT -O $IMGFMT -n "$TEST_IMG" "$TEST_IMG.orig" ; then
     exit 1
 fi
 
 echo "== Testing conversion to raw is the same after conversion with -n =="
 # compare the raw files
-if ! $QEMU_IMG convert -f $IMGFMT -O raw $TEST_IMG $TEST_IMG.raw1 ; then
+if ! $QEMU_IMG convert -f $IMGFMT -O raw "$TEST_IMG" "$TEST_IMG.raw1" ; then
     exit 1
 fi
 
-if ! $QEMU_IMG convert -f $IMGFMT -O raw $TEST_IMG.orig $TEST_IMG.raw2 ; then
+if ! $QEMU_IMG convert -f $IMGFMT -O raw "$TEST_IMG.orig" "$TEST_IMG.raw2" ; then
     exit 1
 fi
 
-if ! cmp $TEST_IMG.raw1 $TEST_IMG.raw2 ; then
+if ! cmp "$TEST_IMG.raw1" "$TEST_IMG.raw2" ; then
     exit 1
 fi
 
 echo "== Testing conversion back to original format =="
-if ! $QEMU_IMG convert -f raw -O $IMGFMT -n $TEST_IMG.raw2 $TEST_IMG ; then
+if ! $QEMU_IMG convert -f raw -O $IMGFMT -n "$TEST_IMG.raw2" "$TEST_IMG" ; then
     exit 1
 fi
 _check_test_img
 
 echo "== Testing conversion to a smaller file fails =="
-rm -f $TEST_IMG.orig
-mv $TEST_IMG $TEST_IMG.orig
+rm -f "$TEST_IMG.orig"
+mv "$TEST_IMG" "$TEST_IMG.orig"
 _make_test_img 2M
-if $QEMU_IMG convert -f $IMGFMT -O $IMGFMT -n $TEST_IMG.orig $TEST_IMG >/dev/null 2>&1; then
+if $QEMU_IMG convert -f $IMGFMT -O $IMGFMT -n "$TEST_IMG.orig" "$TEST_IMG" >/dev/null 2>&1; then
     exit 1
 fi
 
-rm -f $TEST_IMG.orig $TEST_IMG.raw $TEST_IMG.raw2
+rm -f "$TEST_IMG.orig" "$TEST_IMG.raw" "$TEST_IMG.raw2"
 
 echo "*** done"
 rm -f $seq.full
diff --git a/tests/qemu-iotests/064 b/tests/qemu-iotests/064
new file mode 100755
index 0000000000..6789aa6ee4
--- /dev/null
+++ b/tests/qemu-iotests/064
@@ -0,0 +1,62 @@
+#!/bin/bash
+#
+# Test VHDX read/write from a sample image created with Hyper-V
+#
+# Copyright (C) 2013 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=jcody@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt vhdx
+_supported_proto generic
+_supported_os Linux
+
+_use_sample_img iotest-dynamic-1G.vhdx.bz2
+
+echo
+echo "=== Verify pattern 0xa5, 0 - 33MB ==="
+$QEMU_IO -r -c "read -pP 0xa5 0 33M" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "=== Verify pattern 0x96, 33M - 66M ==="
+$QEMU_IO -r -c "read -pP 0x96 33M 33M" "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo "=== Verify pattern 0x00, 66M - 1024M ==="
+$QEMU_IO -r -c "read -pP 0x00 66M 958M" "$TEST_IMG" | _filter_qemu_io
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/064.out b/tests/qemu-iotests/064.out
new file mode 100644
index 0000000000..b9e8e4a873
--- /dev/null
+++ b/tests/qemu-iotests/064.out
@@ -0,0 +1,14 @@
+QA output created by 064
+
+=== Verify pattern 0xa5, 0 - 33MB ===
+read 34603008/34603008 bytes at offset 0
+33 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Verify pattern 0x96, 33M - 66M ===
+read 34603008/34603008 bytes at offset 34603008
+33 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Verify pattern 0x00, 66M - 1024M ===
+read 1004535808/1004535808 bytes at offset 69206016
+958 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+*** done
diff --git a/tests/qemu-iotests/065 b/tests/qemu-iotests/065
new file mode 100755
index 0000000000..ab5445f62d
--- /dev/null
+++ b/tests/qemu-iotests/065
@@ -0,0 +1,125 @@
+#!/usr/bin/env python2
+#
+# Test for additional information emitted by qemu-img info on qcow2
+# images
+#
+# Copyright (C) 2013 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import re
+import json
+import iotests
+from iotests import qemu_img, qemu_img_pipe
+import unittest
+
+test_img = os.path.join(iotests.test_dir, 'test.img')
+
+class TestImageInfoSpecific(iotests.QMPTestCase):
+    '''Abstract base class for ImageInfoSpecific tests'''
+
+    def setUp(self):
+        if self.img_options is None:
+            self.skipTest('Skipping abstract test class')
+        qemu_img('create', '-f', iotests.imgfmt, '-o', self.img_options,
+                 test_img, '128K')
+
+    def tearDown(self):
+        os.remove(test_img)
+
+class TestQemuImgInfo(TestImageInfoSpecific):
+    '''Abstract base class for qemu-img info tests'''
+
+    img_options = None
+    json_compare = None
+    human_compare = None
+
+    def test_json(self):
+        data = json.loads(qemu_img_pipe('info', '--output=json', test_img))
+        data = data['format-specific']
+        self.assertEqual(data['type'], iotests.imgfmt)
+        self.assertEqual(data['data'], self.json_compare)
+
+    def test_human(self):
+        data = qemu_img_pipe('info', '--output=human', test_img).split('\n')
+        data = data[(data.index('Format specific information:') + 1)
+                    :data.index('')]
+        for field in data:
+            self.assertTrue(re.match('^ {4}[^ ]', field) is not None)
+        data = map(lambda line: line.strip(), data)
+        self.assertEqual(data, self.human_compare)
+
+class TestQMP(TestImageInfoSpecific):
+    '''Abstract base class for qemu QMP tests'''
+
+    img_options = None
+    qemu_options = ''
+    TestImageInfoSpecific = TestImageInfoSpecific
+
+    def setUp(self):
+        self.TestImageInfoSpecific.setUp(self)
+        self.vm = iotests.VM().add_drive(test_img, self.qemu_options)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        self.TestImageInfoSpecific.tearDown(self)
+
+    def test_qmp(self):
+        result = self.vm.qmp('query-block')['return']
+        drive = filter(lambda drive: drive['device'] == 'drive0', result)[0]
+        data = drive['inserted']['image']['format-specific']
+        self.assertEqual(data['type'], iotests.imgfmt)
+        self.assertEqual(data['data'], self.compare)
+
+class TestQCow2(TestQemuImgInfo):
+    '''Testing a qcow2 version 2 image'''
+    img_options = 'compat=0.10'
+    json_compare = { 'compat': '0.10' }
+    human_compare = [ 'compat: 0.10' ]
+
+class TestQCow3NotLazy(TestQemuImgInfo):
+    '''Testing a qcow2 version 3 image with lazy refcounts disabled'''
+    img_options = 'compat=1.1,lazy_refcounts=off'
+    json_compare = { 'compat': '1.1', 'lazy-refcounts': False }
+    human_compare = [ 'compat: 1.1', 'lazy refcounts: false' ]
+
+class TestQCow3Lazy(TestQemuImgInfo):
+    '''Testing a qcow2 version 3 image with lazy refcounts enabled'''
+    img_options = 'compat=1.1,lazy_refcounts=on'
+    json_compare = { 'compat': '1.1', 'lazy-refcounts': True }
+    human_compare = [ 'compat: 1.1', 'lazy refcounts: true' ]
+
+class TestQCow3NotLazyQMP(TestQMP):
+    '''Testing a qcow2 version 3 image with lazy refcounts disabled, opening
+       with lazy refcounts enabled'''
+    img_options = 'compat=1.1,lazy_refcounts=off'
+    qemu_options = 'lazy-refcounts=on'
+    compare = { 'compat': '1.1', 'lazy-refcounts': False }
+
+class TestQCow3LazyQMP(TestQMP):
+    '''Testing a qcow2 version 3 image with lazy refcounts enabled, opening
+       with lazy refcounts disabled'''
+    img_options = 'compat=1.1,lazy_refcounts=on'
+    qemu_options = 'lazy-refcounts=off'
+    compare = { 'compat': '1.1', 'lazy-refcounts': True }
+
+TestImageInfoSpecific = None
+TestQemuImgInfo = None
+TestQMP = None
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=['qcow2'])
diff --git a/tests/qemu-iotests/065.out b/tests/qemu-iotests/065.out
new file mode 100644
index 0000000000..594c16f49f
--- /dev/null
+++ b/tests/qemu-iotests/065.out
@@ -0,0 +1,5 @@
+........
+----------------------------------------------------------------------
+Ran 8 tests
+
+OK
diff --git a/tests/qemu-iotests/066 b/tests/qemu-iotests/066
new file mode 100755
index 0000000000..1c2452b0c5
--- /dev/null
+++ b/tests/qemu-iotests/066
@@ -0,0 +1,63 @@
+#!/bin/bash
+#
+# Test case for discarding preallocated zero clusters in qcow2
+#
+# Copyright (C) 2013 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+# This tests qocw2-specific low-level functionality
+_supported_fmt qcow2
+_supported_proto generic
+_supported_os Linux
+
+IMGOPTS="compat=1.1"
+IMG_SIZE=64M
+
+echo
+echo "=== Testing snapshotting an image with zero clusters ==="
+echo
+_make_test_img $IMG_SIZE
+# Write some normal clusters, zero them (creating preallocated zero clusters)
+# and discard those
+$QEMU_IO -c "write 0 256k" -c "write -z 0 256k" -c "discard 0 256k" "$TEST_IMG" \
+         | _filter_qemu_io
+# Check the image (there shouldn't be any leaks)
+_check_test_img
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/066.out b/tests/qemu-iotests/066.out
new file mode 100644
index 0000000000..9139780f49
--- /dev/null
+++ b/tests/qemu-iotests/066.out
@@ -0,0 +1,13 @@
+QA output created by 066
+
+=== Testing snapshotting an image with zero clusters ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+wrote 262144/262144 bytes at offset 0
+256 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 262144/262144 bytes at offset 0
+256 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+discard 262144/262144 bytes at offset 0
+256 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
+*** done
diff --git a/tests/qemu-iotests/067 b/tests/qemu-iotests/067
new file mode 100755
index 0000000000..79dc38bc04
--- /dev/null
+++ b/tests/qemu-iotests/067
@@ -0,0 +1,133 @@
+#!/bin/bash
+#
+# Test automatic deletion of BDSes created by -drive/drive_add
+#
+# Copyright (C) 2013 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+function do_run_qemu()
+{
+    echo Testing: "$@"
+    $QEMU -nographic -qmp stdio -serial none "$@"
+    echo
+}
+
+function run_qemu()
+{
+    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qmp
+}
+
+size=128M
+
+_make_test_img $size
+
+echo
+echo === -drive/-device and device_del ===
+echo
+
+run_qemu -drive file=$TEST_IMG,format=$IMGFMT,if=none,id=disk -device virtio-blk-pci,drive=disk,id=virtio0 <<EOF
+{ "execute": "qmp_capabilities" }
+{ "execute": "query-block" }
+{ "execute": "device_del", "arguments": { "id": "virtio0" } }
+{ "execute": "system_reset" }
+{ "execute": "query-block" }
+{ "execute": "quit" }
+EOF
+
+echo
+echo === -drive/device_add and device_del ===
+echo
+
+run_qemu -drive file=$TEST_IMG,format=$IMGFMT,if=none,id=disk <<EOF
+{ "execute": "qmp_capabilities" }
+{ "execute": "query-block" }
+{ "execute": "device_add",
+   "arguments": { "driver": "virtio-blk-pci", "drive": "disk",
+                  "id": "virtio0" } }
+{ "execute": "device_del", "arguments": { "id": "virtio0" } }
+{ "execute": "system_reset" }
+{ "execute": "query-block" }
+{ "execute": "quit" }
+EOF
+
+echo
+echo === drive_add/device_add and device_del ===
+echo
+
+run_qemu <<EOF
+{ "execute": "qmp_capabilities" }
+{ "execute": "human-monitor-command",
+  "arguments": { "command-line": "drive_add 0 file=$TEST_IMG,format=$IMGFMT,if=none,id=disk" } }
+{ "execute": "query-block" }
+{ "execute": "device_add",
+   "arguments": { "driver": "virtio-blk-pci", "drive": "disk",
+                  "id": "virtio0" } }
+{ "execute": "device_del", "arguments": { "id": "virtio0" } }
+{ "execute": "system_reset" }
+{ "execute": "query-block" }
+{ "execute": "quit" }
+EOF
+
+echo
+echo === blockdev_add/device_add and device_del ===
+echo
+
+run_qemu <<EOF
+{ "execute": "qmp_capabilities" }
+{ "execute": "blockdev-add",
+  "arguments": {
+      "options": {
+        "driver": "$IMGFMT",
+        "id": "disk",
+        "file": {
+            "driver": "file",
+            "filename": "$TEST_IMG"
+        }
+      }
+    }
+  }
+{ "execute": "query-block" }
+{ "execute": "device_add",
+   "arguments": { "driver": "virtio-blk-pci", "drive": "disk",
+                  "id": "virtio0" } }
+{ "execute": "device_del", "arguments": { "id": "virtio0" } }
+{ "execute": "system_reset" }
+{ "execute": "query-block" }
+{ "execute": "quit" }
+EOF
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/067.out b/tests/qemu-iotests/067.out
new file mode 100644
index 0000000000..4bb9ff9652
--- /dev/null
+++ b/tests/qemu-iotests/067.out
@@ -0,0 +1,80 @@
+QA output created by 067
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+
+=== -drive/-device and device_del ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk -device virtio-blk-pci,drive=disk,id=virtio0
+QMP_VERSION
+{"return": {}}
+{"return": [{"io-status": "ok", "device": "disk", "locked": false, "removable": false, "inserted": {"iops_rd": 0, "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": 139264, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "type": "unknown"}, {"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}]}
+{"return": {}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_DELETED", "data": {"path": "/machine/peripheral/virtio0/virtio-backend"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_DELETED", "data": {"device": "virtio0", "path": "/machine/peripheral/virtio0"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "RESET"}
+{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}]}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
+
+
+=== -drive/device_add and device_del ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk
+QMP_VERSION
+{"return": {}}
+{"return": [{"device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": 139264, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}, {"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}]}
+{"return": {}}
+{"return": {}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_DELETED", "data": {"path": "/machine/peripheral/virtio0/virtio-backend"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_DELETED", "data": {"device": "virtio0", "path": "/machine/peripheral/virtio0"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "RESET"}
+{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}]}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
+
+
+=== drive_add/device_add and device_del ===
+
+Testing:
+QMP_VERSION
+{"return": {}}
+{"return": "OK\r\n"}
+{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": 139264, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}]}
+{"return": {}}
+{"return": {}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_DELETED", "data": {"path": "/machine/peripheral/virtio0/virtio-backend"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_DELETED", "data": {"device": "virtio0", "path": "/machine/peripheral/virtio0"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "RESET"}
+{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}]}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
+
+
+=== blockdev_add/device_add and device_del ===
+
+Testing:
+QMP_VERSION
+{"return": {}}
+{"return": {}}
+{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": 139264, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}]}
+{"return": {}}
+{"return": {}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_DELETED", "data": {"path": "/machine/peripheral/virtio0/virtio-backend"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_DELETED", "data": {"device": "virtio0", "path": "/machine/peripheral/virtio0"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "RESET"}
+{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"io-status": "ok", "device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": 139264, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}]}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
+
+*** done
diff --git a/tests/qemu-iotests/check b/tests/qemu-iotests/check
index 4ecf497d8e..f5f328f5f5 100755
--- a/tests/qemu-iotests/check
+++ b/tests/qemu-iotests/check
@@ -164,6 +164,7 @@ QEMU_IO       -- $QEMU_IO
 IMGFMT        -- $FULL_IMGFMT_DETAILS
 IMGPROTO      -- $FULL_IMGPROTO_DETAILS
 PLATFORM      -- $FULL_HOST_DETAILS
+SOCKET_SCM_HELPER -- $SOCKET_SCM_HELPER
 
 EOF
 #MKFS_OPTIONS  -- $FULL_MKFS_OPTIONS
diff --git a/tests/qemu-iotests/common b/tests/qemu-iotests/common
index fecaf85074..2932e14e73 100644
--- a/tests/qemu-iotests/common
+++ b/tests/qemu-iotests/common
@@ -45,6 +45,7 @@ valgrind=false
 rm -f $tmp.list $tmp.tmp $tmp.sed
 
 export IMGFMT=raw
+export IMGFMT_GENERIC=true
 export IMGPROTO=file
 export IMGOPTS=""
 export QEMU_IO_OPTIONS=""
@@ -133,6 +134,7 @@ check options
     -qed                test qed
     -vdi                test vdi
     -vpc                test vpc
+    -vhdx               test vhdx
     -vmdk               test vmdk
     -rbd                test rbd
     -sheepdog           test sheepdog
@@ -195,6 +197,12 @@ testlist options
             xpand=false
             ;;
 
+        -vhdx)
+            IMGFMT=vhdx
+            xpand=false
+            IMGFMT_GENERIC=false
+            ;;
+
         -rbd)
             IMGPROTO=rbd
             xpand=false
diff --git a/tests/qemu-iotests/common.config b/tests/qemu-iotests/common.config
index d794e624e7..d90a8bca8b 100644
--- a/tests/qemu-iotests/common.config
+++ b/tests/qemu-iotests/common.config
@@ -125,6 +125,17 @@ fi
 
 export TEST_DIR
 
+if [ -z "$SAMPLE_IMG_DIR" ]; then
+        SAMPLE_IMG_DIR=`pwd`/sample_images
+fi
+
+if [ ! -d "$SAMPLE_IMG_DIR" ]; then
+    echo "common.config: Error: \$SAMPLE_IMG_DIR ($SAMPLE_IMG_DIR) is not a directory"
+    exit 1
+fi
+
+export SAMPLE_IMG_DIR
+
 _readlink()
 {
     if [ $# -ne 1 ]; then
diff --git a/tests/qemu-iotests/common.filter b/tests/qemu-iotests/common.filter
index 5dfda63e59..8e7b1a4195 100644
--- a/tests/qemu-iotests/common.filter
+++ b/tests/qemu-iotests/common.filter
@@ -159,5 +159,13 @@ _filter_qemu()
         -e 's#^QEMU [0-9]\+\.[0-9]\+\.[0-9]\+ monitor#QEMU X.Y.Z monitor#'
 }
 
+# replace problematic QMP output like timestamps
+_filter_qmp()
+{
+    _filter_win32 | \
+    sed -e 's#\("\(micro\)\?seconds": \)[0-9]\+#\1 TIMESTAMP#g' \
+        -e 's#^{"QMP":.*}$#QMP_VERSION#'
+}
+
 # make sure this script returns success
 /bin/true
diff --git a/tests/qemu-iotests/common.rc b/tests/qemu-iotests/common.rc
index 88fecf7870..4e826040d4 100644
--- a/tests/qemu-iotests/common.rc
+++ b/tests/qemu-iotests/common.rc
@@ -91,6 +91,18 @@ _set_default_imgopts()
     fi
 }
 
+_use_sample_img()
+{
+    SAMPLE_IMG_FILE="${1%\.bz2}"
+    TEST_IMG="$TEST_DIR/$SAMPLE_IMG_FILE"
+    bzcat "$SAMPLE_IMG_DIR/$1" > "$TEST_IMG"
+    if [ $? -ne 0 ]
+    then
+        echo "_use_sample_img error, cannot extract '$SAMPLE_IMG_DIR/$1'"
+        exit 1
+    fi
+}
+
 _make_test_img()
 {
     # extra qemu-img options can be added by tests
@@ -123,7 +135,7 @@ _make_test_img()
     fi
 
     # XXX(hch): have global image options?
-    $QEMU_IMG create -f $IMGFMT $extra_img_options $img_name $image_size | \
+    $QEMU_IMG create -f $IMGFMT $extra_img_options $img_name $image_size 2>&1 | \
         sed -e "s#$IMGPROTO:$TEST_DIR#TEST_DIR#g" \
             -e "s#$TEST_DIR#TEST_DIR#g" \
             -e "s#$IMGFMT#IMGFMT#g" \
@@ -152,20 +164,24 @@ _cleanup_test_img()
 
         nbd)
             kill $QEMU_NBD_PID
-            rm -f $TEST_IMG_FILE
+            rm -f "$TEST_IMG_FILE"
             ;;
         file)
-            rm -f $TEST_DIR/t.$IMGFMT
-            rm -f $TEST_DIR/t.$IMGFMT.orig
-            rm -f $TEST_DIR/t.$IMGFMT.base
+            rm -f "$TEST_DIR/t.$IMGFMT"
+            rm -f "$TEST_DIR/t.$IMGFMT.orig"
+            rm -f "$TEST_DIR/t.$IMGFMT.base"
+            if [ -n "$SAMPLE_IMG_FILE" ]
+            then
+                rm -f "$TEST_DIR/$SAMPLE_IMG_FILE"
+            fi
             ;;
 
         rbd)
-            rbd rm $TEST_DIR/t.$IMGFMT > /dev/null
+            rbd rm "$TEST_DIR/t.$IMGFMT" > /dev/null
             ;;
 
         sheepdog)
-            collie vdi delete $TEST_DIR/t.$IMGFMT
+            collie vdi delete "$TEST_DIR/t.$IMGFMT"
             ;;
 
     esac
@@ -173,7 +189,7 @@ _cleanup_test_img()
 
 _check_test_img()
 {
-    $QEMU_IMG check "$@" -f $IMGFMT $TEST_IMG 2>&1 | _filter_testdir | \
+    $QEMU_IMG check "$@" -f $IMGFMT "$TEST_IMG" 2>&1 | _filter_testdir | \
         sed -e '/allocated.*fragmented.*compressed clusters/d' \
             -e 's/qemu-img: This image format does not support checks/No errors were found on the image./' \
             -e '/Image end offset: [0-9]\+/d'
@@ -181,12 +197,30 @@ _check_test_img()
 
 _img_info()
 {
-    $QEMU_IMG info "$@" $TEST_IMG 2>&1 | \
+    discard=0
+    regex_json_spec_start='^ *"format-specific": \{'
+    $QEMU_IMG info "$@" "$TEST_IMG" 2>&1 | \
         sed -e "s#$IMGPROTO:$TEST_DIR#TEST_DIR#g" \
             -e "s#$TEST_DIR#TEST_DIR#g" \
             -e "s#$IMGFMT#IMGFMT#g" \
             -e "/^disk size:/ D" \
-            -e "/actual-size/ D"
+            -e "/actual-size/ D" | \
+        while IFS='' read line; do
+            if [[ $line == "Format specific information:" ]]; then
+                discard=1
+            elif [[ $line =~ $regex_json_spec_start ]]; then
+                discard=2
+                regex_json_spec_end="^${line%%[^ ]*}\\},? *$"
+            fi
+            if [[ $discard == 0 ]]; then
+                echo "$line"
+            elif [[ $discard == 1 && ! $line ]]; then
+                echo
+                discard=0
+            elif [[ $discard == 2 && $line =~ $regex_json_spec_end ]]; then
+                discard=0
+            fi
+        done
 }
 
 _get_pids_by_name()
@@ -305,7 +339,7 @@ _fail()
 _supported_fmt()
 {
     for f; do
-        if [ "$f" = "$IMGFMT" -o "$f" = "generic" ]; then
+        if [ "$f" = "$IMGFMT" -o "$f" = "generic" -a "$IMGFMT_GENERIC" = "true" ]; then
             return
         fi
     done
diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group
index 316b1dd75c..13c5500f54 100644
--- a/tests/qemu-iotests/group
+++ b/tests/qemu-iotests/group
@@ -63,7 +63,13 @@
 054 rw auto
 055 rw auto
 056 rw auto backing
+057 rw auto
 059 rw auto
 060 rw auto
+061 rw auto
 062 rw auto
 063 rw auto
+064 rw auto
+065 rw auto
+066 rw auto
+067 rw auto
diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index 33ad0ecb92..fb10ff43a7 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -21,7 +21,7 @@ import re
 import subprocess
 import string
 import unittest
-import sys; sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'QMP'))
+import sys; sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'scripts', 'qmp'))
 import qmp
 import struct
 
@@ -38,6 +38,8 @@ imgfmt = os.environ.get('IMGFMT', 'raw')
 imgproto = os.environ.get('IMGPROTO', 'file')
 test_dir = os.environ.get('TEST_DIR', '/var/tmp')
 
+socket_scm_helper = os.environ.get('SOCKET_SCM_HELPER', 'socket_scm_helper')
+
 def qemu_img(*args):
     '''Run qemu-img and return the exit code'''
     devnull = open('/dev/null', 'r+')
@@ -47,6 +49,10 @@ def qemu_img_verbose(*args):
     '''Run qemu-img without suppressing its output and return the exit code'''
     return subprocess.call(qemu_img_args + list(args))
 
+def qemu_img_pipe(*args):
+    '''Run qemu-img and return its output'''
+    return subprocess.Popen(qemu_img_args + list(args), stdout=subprocess.PIPE).communicate()[0]
+
 def qemu_io(*args):
     '''Run qemu-io and return the stdout data'''
     args = qemu_io_args + list(args)
@@ -80,6 +86,12 @@ class VM(object):
                      '-display', 'none', '-vga', 'none']
         self._num_drives = 0
 
+    # This can be used to add an unused monitor instance.
+    def add_monitor_telnet(self, ip, port):
+        args = 'tcp:%s:%d,server,nowait,telnet' % (ip, port)
+        self._args.append('-monitor')
+        self._args.append(args)
+
     def add_drive(self, path, opts=''):
         '''Add a virtio-blk drive to the VM'''
         options = ['if=virtio',
@@ -112,6 +124,21 @@ class VM(object):
         self._args.append(','.join(options))
         return self
 
+    def send_fd_scm(self, fd_file_path):
+        # In iotest.py, the qmp should always use unix socket.
+        assert self._qmp.is_scm_available()
+        bin = socket_scm_helper
+        if os.path.exists(bin) == False:
+            print "Scm help program does not present, path '%s'." % bin
+            return -1
+        fd_param = ["%s" % bin,
+                    "%d" % self._qmp.get_sock_fd(),
+                    "%s" % fd_file_path]
+        devnull = open('/dev/null', 'rb')
+        p = subprocess.Popen(fd_param, stdin=devnull, stdout=sys.stdout,
+                             stderr=sys.stderr)
+        return p.wait()
+
     def launch(self):
         '''Launch the VM and establish a QMP connection'''
         devnull = open('/dev/null', 'rb')
diff --git a/tests/qemu-iotests/sample_images/README b/tests/qemu-iotests/sample_images/README
new file mode 100644
index 0000000000..507af5f5ff
--- /dev/null
+++ b/tests/qemu-iotests/sample_images/README
@@ -0,0 +1,8 @@
+This is for small sample images to be used with qemu-iotests, intended for
+non-native formats that QEMU supports for compatibility.  The idea is to use
+the native tool to create the sample image.
+
+For instance, a VHDX image in this directory would be an image created not by
+QEMU itself, but rather created by Hyper-V.
+
+Sample images added here must be compressed with bzip2.
diff --git a/tests/qemu-iotests/sample_images/iotest-dynamic-1G.vhdx.bz2 b/tests/qemu-iotests/sample_images/iotest-dynamic-1G.vhdx.bz2
new file mode 100644
index 0000000000..77d97a0bae
--- /dev/null
+++ b/tests/qemu-iotests/sample_images/iotest-dynamic-1G.vhdx.bz2
Binary files differdiff --git a/tests/qemu-iotests/socket_scm_helper.c b/tests/qemu-iotests/socket_scm_helper.c
new file mode 100644
index 0000000000..0e2b2859af
--- /dev/null
+++ b/tests/qemu-iotests/socket_scm_helper.c
@@ -0,0 +1,135 @@
+/*
+ * SCM_RIGHTS with unix socket help program for test
+ *
+ * Copyright IBM, Inc. 2013
+ *
+ * Authors:
+ *  Wenchao Xia    <xiawenc@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#include <stdio.h>
+#include <errno.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+/* #define SOCKET_SCM_DEBUG */
+
+/*
+ * @fd and @fd_to_send will not be checked for validation in this function,
+ * a blank will be sent as iov data to notify qemu.
+ */
+static int send_fd(int fd, int fd_to_send)
+{
+    struct msghdr msg;
+    struct iovec iov[1];
+    int ret;
+    char control[CMSG_SPACE(sizeof(int))];
+    struct cmsghdr *cmsg;
+
+    memset(&msg, 0, sizeof(msg));
+    memset(control, 0, sizeof(control));
+
+    /* Send a blank to notify qemu */
+    iov[0].iov_base = (void *)" ";
+    iov[0].iov_len = 1;
+
+    msg.msg_iov = iov;
+    msg.msg_iovlen = 1;
+
+    msg.msg_control = control;
+    msg.msg_controllen = sizeof(control);
+
+    cmsg = CMSG_FIRSTHDR(&msg);
+
+    cmsg->cmsg_len = CMSG_LEN(sizeof(int));
+    cmsg->cmsg_level = SOL_SOCKET;
+    cmsg->cmsg_type = SCM_RIGHTS;
+    memcpy(CMSG_DATA(cmsg), &fd, sizeof(int));
+
+    do {
+        ret = sendmsg(fd, &msg, 0);
+    } while (ret < 0 && errno == EINTR);
+
+    if (ret < 0) {
+        fprintf(stderr, "Failed to send msg, reason: %s\n", strerror(errno));
+    }
+
+    return ret;
+}
+
+/* Convert string to fd number. */
+static int get_fd_num(const char *fd_str)
+{
+    int sock;
+    char *err;
+
+    errno = 0;
+    sock = strtol(fd_str, &err, 10);
+    if (errno) {
+        fprintf(stderr, "Failed in strtol for socket fd, reason: %s\n",
+                strerror(errno));
+        return -1;
+    }
+    if (!*fd_str || *err || sock < 0) {
+        fprintf(stderr, "bad numerical value for socket fd '%s'\n", fd_str);
+        return -1;
+    }
+
+    return sock;
+}
+
+/*
+ * To make things simple, the caller needs to specify:
+ * 1. socket fd.
+ * 2. path of the file to be sent.
+ */
+int main(int argc, char **argv, char **envp)
+{
+    int sock, fd, ret;
+
+#ifdef SOCKET_SCM_DEBUG
+    int i;
+    for (i = 0; i < argc; i++) {
+        fprintf(stderr, "Parameter %d: %s\n", i, argv[i]);
+    }
+#endif
+
+    if (argc != 3) {
+        fprintf(stderr,
+                "Usage: %s < socket-fd > < file-path >\n",
+                argv[0]);
+        return EXIT_FAILURE;
+    }
+
+
+    sock = get_fd_num(argv[1]);
+    if (sock < 0) {
+        return EXIT_FAILURE;
+    }
+
+    /* Now only open a file in readonly mode for test purpose. If more precise
+       control is needed, use python script in file operation, which is
+       supposed to fork and exec this program. */
+    fd = open(argv[2], O_RDONLY);
+    if (fd < 0) {
+        fprintf(stderr, "Failed to open file '%s'\n", argv[2]);
+        return EXIT_FAILURE;
+    }
+
+    ret = send_fd(sock, fd);
+    if (ret < 0) {
+        close(fd);
+        return EXIT_FAILURE;
+    }
+
+    close(fd);
+    return EXIT_SUCCESS;
+}
diff --git a/tests/test-coroutine.c b/tests/test-coroutine.c
index 39be046ec7..15a885e882 100644
--- a/tests/test-coroutine.c
+++ b/tests/test-coroutine.c
@@ -182,17 +182,17 @@ static void perf_nesting(void)
     unsigned int i, maxcycles, maxnesting;
     double duration;
 
-    maxcycles = 100000000;
+    maxcycles = 10000;
     maxnesting = 1000;
     Coroutine *root;
-    NestData nd = {
-        .n_enter  = 0,
-        .n_return = 0,
-        .max      = maxnesting,
-    };
 
     g_test_timer_start();
     for (i = 0; i < maxcycles; i++) {
+        NestData nd = {
+            .n_enter  = 0,
+            .n_return = 0,
+            .max      = maxnesting,
+        };
         root = qemu_coroutine_create(nest);
         qemu_coroutine_enter(root, &nd);
     }
@@ -202,6 +202,38 @@ static void perf_nesting(void)
         maxcycles, maxnesting, duration);
 }
 
+/*
+ * Yield benchmark
+ */
+
+static void coroutine_fn yield_loop(void *opaque)
+{
+    unsigned int *counter = opaque;
+
+    while ((*counter) > 0) {
+        (*counter)--;
+        qemu_coroutine_yield();
+    }
+}
+
+static void perf_yield(void)
+{
+    unsigned int i, maxcycles;
+    double duration;
+
+    maxcycles = 100000000;
+    i = maxcycles;
+    Coroutine *coroutine = qemu_coroutine_create(yield_loop);
+
+    g_test_timer_start();
+    while (i > 0) {
+        qemu_coroutine_enter(coroutine, &i);
+    }
+    duration = g_test_timer_elapsed();
+
+    g_test_message("Yield %u iterations: %f s\n",
+        maxcycles, duration);
+}
 
 int main(int argc, char **argv)
 {
@@ -214,6 +246,7 @@ int main(int argc, char **argv)
     if (g_test_perf()) {
         g_test_add_func("/perf/lifecycle", perf_lifecycle);
         g_test_add_func("/perf/nesting", perf_nesting);
+        g_test_add_func("/perf/yield", perf_yield);
     }
     return g_test_run();
 }
diff --git a/trace-events b/trace-events
index d4dba24551..8695e9e5b7 100644
--- a/trace-events
+++ b/trace-events
@@ -1109,7 +1109,6 @@ qemu_spice_wakeup(uint32_t qid) "%d"
 qemu_spice_create_update(uint32_t left, uint32_t right, uint32_t top, uint32_t bottom) "lr %d -> %d,  tb -> %d -> %d"
 
 # hw/display/qxl-render.c
-qxl_render_blit_guest_primary_initialized(void) ""
 qxl_render_blit(int32_t stride, int32_t left, int32_t right, int32_t top, int32_t bottom) "stride=%d [%d, %d, %d, %d]"
 qxl_render_guest_primary_resized(int32_t width, int32_t height, int32_t stride, int32_t bytes_pp, int32_t bits_pp) "%dx%d, stride %d, bpp %d, depth %d"
 qxl_render_update_area_done(void *cookie) "%p"
@@ -1122,7 +1121,7 @@ spapr_pci_rtas_ibm_query_interrupt_source_number(unsigned ioa, unsigned intr) "q
 spapr_pci_msi_write(uint64_t addr, uint64_t data, uint32_t dt_irq) "@%"PRIx64"<=%"PRIx64" IRQ %u"
 spapr_pci_lsi_set(const char *busname, int pin, uint32_t irq) "%s PIN%d IRQ %u"
 
-# hw/ppc/xics.c
+# hw/intc/xics.c
 xics_icp_check_ipi(int server, uint8_t mfrr) "CPU %d can take IPI mfrr=%#x"
 xics_icp_accept(uint32_t old_xirr, uint32_t new_xirr) "icp_accept: XIRR %#"PRIx32"->%#"PRIx32
 xics_icp_eoi(int server, uint32_t xirr, uint32_t new_xirr) "icp_eoi: server %d given XIRR %#"PRIx32" new XIRR %#"PRIx32
@@ -1167,9 +1166,9 @@ virtio_ccw_new_device(int cssid, int ssid, int schid, int devno, const char *dev
 migrate_set_state(int new_state) "new state %d"
 
 # kvm-all.c
-kvm_ioctl(int type, void *arg) "type %d, arg %p"
-kvm_vm_ioctl(int type, void *arg) "type %d, arg %p"
-kvm_vcpu_ioctl(int cpu_index, int type, void *arg) "cpu_index %d, type %d, arg %p"
+kvm_ioctl(int type, void *arg) "type 0x%x, arg %p"
+kvm_vm_ioctl(int type, void *arg) "type 0x%x, arg %p"
+kvm_vcpu_ioctl(int cpu_index, int type, void *arg) "cpu_index %d, type 0x%x, arg %p"
 kvm_run_exit(int cpu_index, uint32_t reason) "cpu_index %d, reason %d"
 
 # memory.c
diff --git a/translate-all.c b/translate-all.c
index 2c923c644b..aeda54dfbd 100644
--- a/translate-all.c
+++ b/translate-all.c
@@ -1318,18 +1318,6 @@ static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
     mmap_unlock();
 }
 
-#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
-/* check whether the given addr is in TCG generated code buffer or not */
-bool is_tcg_gen_code(uintptr_t tc_ptr)
-{
-    /* This can be called during code generation, code_gen_buffer_size
-       is used instead of code_gen_ptr for upper boundary checking */
-    return (tc_ptr >= (uintptr_t)tcg_ctx.code_gen_buffer &&
-            tc_ptr < (uintptr_t)(tcg_ctx.code_gen_buffer +
-                    tcg_ctx.code_gen_buffer_size));
-}
-#endif
-
 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
    tb[1].tc_ptr. Return NULL if not found */
 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
@@ -1679,8 +1667,9 @@ static int dump_region(void *priv, abi_ulong start,
 /* dump memory mappings */
 void page_dump(FILE *f)
 {
-    (void) fprintf(f, "%-8s %-8s %-8s %s\n",
-            "start", "end", "size", "prot");
+    const int length = sizeof(abi_ulong) * 2;
+    (void) fprintf(f, "%-*s %-*s %-*s %s\n",
+            length, "start", length, "end", length, "size", "prot");
     walk_memory_regions(f, dump_region);
 }
 
diff --git a/ui/Makefile.objs b/ui/Makefile.objs
index 6ddc0def6d..f33be47576 100644
--- a/ui/Makefile.objs
+++ b/ui/Makefile.objs
@@ -17,6 +17,4 @@ common-obj-$(CONFIG_GTK) += gtk.o x_keymap.o
 
 $(obj)/sdl.o $(obj)/sdl_zoom.o: QEMU_CFLAGS += $(SDL_CFLAGS) 
 
-$(obj)/cocoa.o: $(SRC_PATH)/$(obj)/cocoa.m
-
 $(obj)/gtk.o: QEMU_CFLAGS += $(GTK_CFLAGS) $(VTE_CFLAGS)
diff --git a/ui/spice-core.c b/ui/spice-core.c
index 33ef83731a..e4d533d4c4 100644
--- a/ui/spice-core.c
+++ b/ui/spice-core.c
@@ -48,7 +48,6 @@ static char *auth_passwd;
 static time_t auth_expires = TIME_MAX;
 static int spice_migration_completed;
 int using_spice = 0;
-int spice_displays;
 
 static QemuThread me;
 
@@ -383,17 +382,16 @@ static SpiceChannelList *qmp_query_spice_channels(void)
         struct sockaddr *paddr;
         socklen_t plen;
 
+        if (!(item->info->flags & SPICE_CHANNEL_EVENT_FLAG_ADDR_EXT)) {
+            error_report("invalid channel event");
+            return NULL;
+        }
+
         chan = g_malloc0(sizeof(*chan));
         chan->value = g_malloc0(sizeof(*chan->value));
 
-        if (item->info->flags & SPICE_CHANNEL_EVENT_FLAG_ADDR_EXT) {
-            paddr = (struct sockaddr *)&item->info->paddr_ext;
-            plen = item->info->plen_ext;
-        } else {
-            paddr = &item->info->paddr;
-            plen = item->info->plen;
-        }
-
+        paddr = (struct sockaddr *)&item->info->paddr_ext;
+        plen = item->info->plen_ext;
         getnameinfo(paddr, plen,
                     host, sizeof(host), port, sizeof(port),
                     NI_NUMERICHOST | NI_NUMERICSERV);
@@ -833,15 +831,33 @@ int qemu_spice_add_interface(SpiceBaseInstance *sin)
          * With a command line like '-vnc :0 -vga qxl' you'll end up here.
          */
         spice_server = spice_server_new();
+        spice_server_set_sasl_appname(spice_server, "qemu");
         spice_server_init(spice_server, &core_interface);
         qemu_add_vm_change_state_handler(vm_change_state_handler, NULL);
     }
 
-    if (strcmp(sin->sif->type, SPICE_INTERFACE_QXL) == 0) {
-        spice_displays++;
+    return spice_server_add_interface(spice_server, sin);
+}
+
+static GSList *spice_consoles;
+static int display_id;
+
+bool qemu_spice_have_display_interface(QemuConsole *con)
+{
+    if (g_slist_find(spice_consoles, con)) {
+        return true;
     }
+    return false;
+}
 
-    return spice_server_add_interface(spice_server, sin);
+int qemu_spice_add_display_interface(QXLInstance *qxlin, QemuConsole *con)
+{
+    if (g_slist_find(spice_consoles, con)) {
+        return -1;
+    }
+    qxlin->id = display_id++;
+    spice_consoles = g_slist_append(spice_consoles, con);
+    return qemu_spice_add_interface(&qxlin->base);
 }
 
 static int qemu_spice_set_ticket(bool fail_if_conn, bool disconnect_if_conn)
diff --git a/ui/spice-display.c b/ui/spice-display.c
index 82d8b9f9a5..f23a31854d 100644
--- a/ui/spice-display.c
+++ b/ui/spice-display.c
@@ -83,14 +83,14 @@ void qemu_spice_add_memslot(SimpleSpiceDisplay *ssd, QXLDevMemSlot *memslot,
                 (uintptr_t)qxl_cookie_new(QXL_COOKIE_TYPE_IO,
                                           QXL_IO_MEMSLOT_ADD_ASYNC));
     } else {
-        ssd->worker->add_memslot(ssd->worker, memslot);
+        spice_qxl_add_memslot(&ssd->qxl, memslot);
     }
 }
 
 void qemu_spice_del_memslot(SimpleSpiceDisplay *ssd, uint32_t gid, uint32_t sid)
 {
     trace_qemu_spice_del_memslot(ssd->qxl.id, gid, sid);
-    ssd->worker->del_memslot(ssd->worker, gid, sid);
+    spice_qxl_del_memslot(&ssd->qxl, gid, sid);
 }
 
 void qemu_spice_create_primary_surface(SimpleSpiceDisplay *ssd, uint32_t id,
@@ -103,7 +103,7 @@ void qemu_spice_create_primary_surface(SimpleSpiceDisplay *ssd, uint32_t id,
                 (uintptr_t)qxl_cookie_new(QXL_COOKIE_TYPE_IO,
                                           QXL_IO_CREATE_PRIMARY_ASYNC));
     } else {
-        ssd->worker->create_primary_surface(ssd->worker, id, surface);
+        spice_qxl_create_primary_surface(&ssd->qxl, id, surface);
     }
 }
 
@@ -116,14 +116,14 @@ void qemu_spice_destroy_primary_surface(SimpleSpiceDisplay *ssd,
                 (uintptr_t)qxl_cookie_new(QXL_COOKIE_TYPE_IO,
                                           QXL_IO_DESTROY_PRIMARY_ASYNC));
     } else {
-        ssd->worker->destroy_primary_surface(ssd->worker, id);
+        spice_qxl_destroy_primary_surface(&ssd->qxl, id);
     }
 }
 
 void qemu_spice_wakeup(SimpleSpiceDisplay *ssd)
 {
     trace_qemu_spice_wakeup(ssd->qxl.id);
-    ssd->worker->wakeup(ssd->worker);
+    spice_qxl_wakeup(&ssd->qxl);
 }
 
 static int spice_display_is_running;
@@ -297,7 +297,7 @@ void qemu_spice_create_host_memslot(SimpleSpiceDisplay *ssd)
 {
     QXLDevMemSlot memslot;
 
-    dprint(1, "%s:\n", __FUNCTION__);
+    dprint(1, "%s/%d:\n", __func__, ssd->qxl.id);
 
     memset(&memslot, 0, sizeof(memslot));
     memslot.slot_group_id = MEMSLOT_GROUP_HOST;
@@ -311,7 +311,7 @@ void qemu_spice_create_host_primary(SimpleSpiceDisplay *ssd)
 
     memset(&surface, 0, sizeof(surface));
 
-    dprint(1, "%s: %dx%d\n", __FUNCTION__,
+    dprint(1, "%s/%d: %dx%d\n", __func__, ssd->qxl.id,
            surface_width(ssd->ds), surface_height(ssd->ds));
 
     surface.format     = SPICE_SURFACE_FMT_32_xRGB;
@@ -329,7 +329,7 @@ void qemu_spice_create_host_primary(SimpleSpiceDisplay *ssd)
 
 void qemu_spice_destroy_host_primary(SimpleSpiceDisplay *ssd)
 {
-    dprint(1, "%s:\n", __FUNCTION__);
+    dprint(1, "%s/%d:\n", __func__, ssd->qxl.id);
 
     qemu_spice_destroy_primary_surface(ssd, 0, QXL_SYNC);
 }
@@ -354,7 +354,8 @@ void qemu_spice_display_update(SimpleSpiceDisplay *ssd,
 {
     QXLRect update_area;
 
-    dprint(2, "%s: x %d y %d w %d h %d\n", __FUNCTION__, x, y, w, h);
+    dprint(2, "%s/%d: x %d y %d w %d h %d\n", __func__,
+           ssd->qxl.id, x, y, w, h);
     update_area.left = x,
     update_area.right = x + w;
     update_area.top = y;
@@ -371,7 +372,7 @@ void qemu_spice_display_switch(SimpleSpiceDisplay *ssd,
 {
     SimpleSpiceUpdate *update;
 
-    dprint(1, "%s:\n", __FUNCTION__);
+    dprint(1, "%s/%d:\n", __func__, ssd->qxl.id);
 
     memset(&ssd->dirty, 0, sizeof(ssd->dirty));
     if (ssd->surface) {
@@ -413,7 +414,7 @@ void qemu_spice_cursor_refresh_unlocked(SimpleSpiceDisplay *ssd)
 
 void qemu_spice_display_refresh(SimpleSpiceDisplay *ssd)
 {
-    dprint(3, "%s:\n", __func__);
+    dprint(3, "%s/%d:\n", __func__, ssd->qxl.id);
     graphic_hw_update(ssd->dcl.con);
 
     qemu_mutex_lock(&ssd->lock);
@@ -427,7 +428,7 @@ void qemu_spice_display_refresh(SimpleSpiceDisplay *ssd)
     if (ssd->notify) {
         ssd->notify = 0;
         qemu_spice_wakeup(ssd);
-        dprint(2, "%s: notify\n", __FUNCTION__);
+        dprint(2, "%s/%d: notify\n", __func__, ssd->qxl.id);
     }
 }
 
@@ -437,19 +438,19 @@ static void interface_attach_worker(QXLInstance *sin, QXLWorker *qxl_worker)
 {
     SimpleSpiceDisplay *ssd = container_of(sin, SimpleSpiceDisplay, qxl);
 
-    dprint(1, "%s:\n", __FUNCTION__);
+    dprint(1, "%s/%d:\n", __func__, ssd->qxl.id);
     ssd->worker = qxl_worker;
 }
 
 static void interface_set_compression_level(QXLInstance *sin, int level)
 {
-    dprint(1, "%s:\n", __FUNCTION__);
+    dprint(1, "%s/%d:\n", __func__, sin->id);
     /* nothing to do */
 }
 
 static void interface_set_mm_time(QXLInstance *sin, uint32_t mm_time)
 {
-    dprint(3, "%s:\n", __FUNCTION__);
+    dprint(3, "%s/%d:\n", __func__, sin->id);
     /* nothing to do */
 }
 
@@ -472,7 +473,7 @@ static int interface_get_command(QXLInstance *sin, struct QXLCommandExt *ext)
     SimpleSpiceUpdate *update;
     int ret = false;
 
-    dprint(3, "%s:\n", __FUNCTION__);
+    dprint(3, "%s/%d:\n", __func__, ssd->qxl.id);
 
     qemu_mutex_lock(&ssd->lock);
     update = QTAILQ_FIRST(&ssd->updates);
@@ -488,7 +489,7 @@ static int interface_get_command(QXLInstance *sin, struct QXLCommandExt *ext)
 
 static int interface_req_cmd_notification(QXLInstance *sin)
 {
-    dprint(1, "%s:\n", __FUNCTION__);
+    dprint(1, "%s/%d:\n", __func__, sin->id);
     return 1;
 }
 
@@ -498,7 +499,7 @@ static void interface_release_resource(QXLInstance *sin,
     SimpleSpiceDisplay *ssd = container_of(sin, SimpleSpiceDisplay, qxl);
     uintptr_t id;
 
-    dprint(2, "%s:\n", __FUNCTION__);
+    dprint(2, "%s/%d:\n", __func__, ssd->qxl.id);
     id = ext.info->id;
     qemu_spice_destroy_update(ssd, (void*)id);
 }
@@ -611,21 +612,38 @@ static const DisplayChangeListenerOps display_listener_ops = {
     .dpy_refresh     = display_refresh,
 };
 
-void qemu_spice_display_init(DisplayState *ds)
+static void qemu_spice_display_init_one(QemuConsole *con)
 {
     SimpleSpiceDisplay *ssd = g_new0(SimpleSpiceDisplay, 1);
 
     qemu_spice_display_init_common(ssd);
 
     ssd->qxl.base.sif = &dpy_interface.base;
-    qemu_spice_add_interface(&ssd->qxl.base);
+    qemu_spice_add_display_interface(&ssd->qxl, con);
     assert(ssd->worker);
 
     qemu_spice_create_host_memslot(ssd);
 
     ssd->dcl.ops = &display_listener_ops;
-    ssd->dcl.con = qemu_console_lookup_by_index(0);
+    ssd->dcl.con = con;
     register_displaychangelistener(&ssd->dcl);
 
     qemu_spice_create_host_primary(ssd);
 }
+
+void qemu_spice_display_init(void)
+{
+    QemuConsole *con;
+    int i;
+
+    for (i = 0;; i++) {
+        con = qemu_console_lookup_by_index(i);
+        if (!con || !qemu_console_is_graphic(con)) {
+            break;
+        }
+        if (qemu_spice_have_display_interface(con)) {
+            continue;
+        }
+        qemu_spice_display_init_one(con);
+    }
+}
diff --git a/util/compatfd.c b/util/compatfd.c
index 9cf3f2834d..430a41c855 100644
--- a/util/compatfd.c
+++ b/util/compatfd.c
@@ -15,9 +15,9 @@
 
 #include "qemu-common.h"
 #include "qemu/compatfd.h"
+#include "qemu/thread.h"
 
 #include <sys/syscall.h>
-#include <pthread.h>
 
 struct sigfd_compat_info
 {
@@ -28,10 +28,6 @@ struct sigfd_compat_info
 static void *sigwait_compat(void *opaque)
 {
     struct sigfd_compat_info *info = opaque;
-    sigset_t all;
-
-    sigfillset(&all);
-    pthread_sigmask(SIG_BLOCK, &all, NULL);
 
     while (1) {
         int sig;
@@ -71,9 +67,8 @@ static void *sigwait_compat(void *opaque)
 
 static int qemu_signalfd_compat(const sigset_t *mask)
 {
-    pthread_attr_t attr;
-    pthread_t tid;
     struct sigfd_compat_info *info;
+    QemuThread thread;
     int fds[2];
 
     info = malloc(sizeof(*info));
@@ -93,12 +88,7 @@ static int qemu_signalfd_compat(const sigset_t *mask)
     memcpy(&info->mask, mask, sizeof(*mask));
     info->fd = fds[1];
 
-    pthread_attr_init(&attr);
-    pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
-
-    pthread_create(&tid, &attr, sigwait_compat, info);
-
-    pthread_attr_destroy(&attr);
+    qemu_thread_create(&thread, sigwait_compat, info, QEMU_THREAD_DETACHED);
 
     return fds[0];
 }
diff --git a/util/iov.c b/util/iov.c
index f705586808..bb46c04e4d 100644
--- a/util/iov.c
+++ b/util/iov.c
@@ -181,13 +181,11 @@ ssize_t iov_send_recv(int sockfd, struct iovec *iov, unsigned iov_cnt,
             assert(iov[niov].iov_len > tail);
             orig_len = iov[niov].iov_len;
             iov[niov++].iov_len = tail;
-        }
-
-        ret = do_send_recv(sockfd, iov, niov, do_send);
-
-        /* Undo the changes above before checking for errors */
-        if (tail) {
+            ret = do_send_recv(sockfd, iov, niov, do_send);
+            /* Undo the changes above before checking for errors */
             iov[niov-1].iov_len = orig_len;
+        } else {
+            ret = do_send_recv(sockfd, iov, niov, do_send);
         }
         if (offset) {
             iov[0].iov_base -= offset;
diff --git a/util/osdep.c b/util/osdep.c
index 685c8ae889..62072b4be3 100644
--- a/util/osdep.c
+++ b/util/osdep.c
@@ -207,6 +207,13 @@ int qemu_open(const char *name, int flags, ...)
     }
 #endif
 
+#ifdef O_DIRECT
+    if (ret == -1 && errno == EINVAL && (flags & O_DIRECT)) {
+        error_report("file system may not support O_DIRECT");
+        errno = EINVAL; /* in case it was clobbered */
+    }
+#endif /* O_DIRECT */
+
     return ret;
 }
 
diff --git a/util/oslib-posix.c b/util/oslib-posix.c
index 3dc8b1b074..e00a44c86f 100644
--- a/util/oslib-posix.c
+++ b/util/oslib-posix.c
@@ -112,9 +112,7 @@ void *qemu_anon_ram_alloc(size_t size)
     size_t offset = QEMU_ALIGN_UP((uintptr_t)ptr, align) - (uintptr_t)ptr;
 
     if (ptr == MAP_FAILED) {
-        fprintf(stderr, "Failed to allocate %zu B: %s\n",
-                size, strerror(errno));
-        abort();
+        return NULL;
     }
 
     ptr += offset;
@@ -159,6 +157,18 @@ void qemu_set_nonblock(int fd)
     fcntl(fd, F_SETFL, f | O_NONBLOCK);
 }
 
+int socket_set_fast_reuse(int fd)
+{
+    int val = 1, ret;
+
+    ret = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR,
+                     (const char *)&val, sizeof(val));
+
+    assert(ret == 0);
+
+    return ret;
+}
+
 void qemu_set_cloexec(int fd)
 {
     int f;
diff --git a/util/oslib-win32.c b/util/oslib-win32.c
index 961fbf5e3d..776ccfaaf0 100644
--- a/util/oslib-win32.c
+++ b/util/oslib-win32.c
@@ -65,10 +65,7 @@ void *qemu_anon_ram_alloc(size_t size)
     /* FIXME: this is not exactly optimal solution since VirtualAlloc
        has 64Kb granularity, but at least it guarantees us that the
        memory is page aligned. */
-    if (!size) {
-        abort();
-    }
-    ptr = qemu_oom_check(VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE));
+    ptr = VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE);
     trace_qemu_anon_ram_alloc(size, ptr);
     return ptr;
 }
@@ -127,6 +124,16 @@ void qemu_set_nonblock(int fd)
     qemu_fd_register(fd);
 }
 
+int socket_set_fast_reuse(int fd)
+{
+    /* Enabling the reuse of an endpoint that was used by a socket still in
+     * TIME_WAIT state is usually performed by setting SO_REUSEADDR. On Windows
+     * fast reuse is the default and SO_REUSEADDR does strange things. So we
+     * don't have to do anything here. More info can be found at:
+     * http://msdn.microsoft.com/en-us/library/windows/desktop/ms740621.aspx */
+    return 0;
+}
+
 int inet_aton(const char *cp, struct in_addr *ia)
 {
     uint32_t addr = inet_addr(cp);
diff --git a/util/path.c b/util/path.c
index f0c69627c7..623219e4c5 100644
--- a/util/path.c
+++ b/util/path.c
@@ -39,7 +39,7 @@ static int strneq(const char *s1, unsigned int n, const char *s2)
 }
 
 static struct pathelem *add_entry(struct pathelem *root, const char *name,
-                                  unsigned char type);
+                                  unsigned type);
 
 static struct pathelem *new_entry(const char *root,
                                   struct pathelem *parent,
@@ -82,7 +82,7 @@ static struct pathelem *add_dir_maybe(struct pathelem *path)
 }
 
 static struct pathelem *add_entry(struct pathelem *root, const char *name,
-                                  unsigned char type)
+                                  unsigned type)
 {
     struct pathelem **e;
 
diff --git a/util/qemu-option.c b/util/qemu-option.c
index e0844a966c..efcb5dcfcb 100644
--- a/util/qemu-option.c
+++ b/util/qemu-option.c
@@ -834,6 +834,12 @@ const char *qemu_opts_id(QemuOpts *opts)
     return opts->id;
 }
 
+/* The id string will be g_free()d by qemu_opts_del */
+void qemu_opts_set_id(QemuOpts *opts, char *id)
+{
+    opts->id = id;
+}
+
 void qemu_opts_del(QemuOpts *opts)
 {
     QemuOpt *opt;
diff --git a/util/qemu-sockets.c b/util/qemu-sockets.c
index 095716ecdb..6b97dc11f9 100644
--- a/util/qemu-sockets.c
+++ b/util/qemu-sockets.c
@@ -155,7 +155,7 @@ int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp)
             continue;
         }
 
-        qemu_setsockopt(slisten, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+        socket_set_fast_reuse(slisten);
 #ifdef IPV6_V6ONLY
         if (e->ai_family == PF_INET6) {
             /* listen on both ipv4 and ipv6 */
@@ -274,7 +274,7 @@ static int inet_connect_addr(struct addrinfo *addr, bool *in_progress,
         error_set_errno(errp, errno, QERR_SOCKET_CREATE_FAILED);
         return -1;
     }
-    qemu_setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+    socket_set_fast_reuse(sock);
     if (connect_state != NULL) {
         qemu_set_nonblock(sock);
     }
@@ -455,7 +455,7 @@ int inet_dgram_opts(QemuOpts *opts, Error **errp)
         error_set_errno(errp, errno, QERR_SOCKET_CREATE_FAILED);
         goto err;
     }
-    qemu_setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+    socket_set_fast_reuse(sock);
 
     /* bind socket */
     if (bind(sock, local->ai_addr, local->ai_addrlen) < 0) {
diff --git a/util/qemu-thread-posix.c b/util/qemu-thread-posix.c
index 4de133e7b2..37dd298631 100644
--- a/util/qemu-thread-posix.c
+++ b/util/qemu-thread-posix.c
@@ -20,7 +20,12 @@
 #include <limits.h>
 #include <unistd.h>
 #include <sys/time.h>
+#ifdef __linux__
+#include <sys/syscall.h>
+#include <linux/futex.h>
+#endif
 #include "qemu/thread.h"
+#include "qemu/atomic.h"
 
 static void error_exit(int err, const char *msg)
 {
@@ -272,6 +277,117 @@ void qemu_sem_wait(QemuSemaphore *sem)
 #endif
 }
 
+#ifdef __linux__
+#define futex(...)              syscall(__NR_futex, __VA_ARGS__)
+
+static inline void futex_wake(QemuEvent *ev, int n)
+{
+    futex(ev, FUTEX_WAKE, n, NULL, NULL, 0);
+}
+
+static inline void futex_wait(QemuEvent *ev, unsigned val)
+{
+    futex(ev, FUTEX_WAIT, (int) val, NULL, NULL, 0);
+}
+#else
+static inline void futex_wake(QemuEvent *ev, int n)
+{
+    if (n == 1) {
+        pthread_cond_signal(&ev->cond);
+    } else {
+        pthread_cond_broadcast(&ev->cond);
+    }
+}
+
+static inline void futex_wait(QemuEvent *ev, unsigned val)
+{
+    pthread_mutex_lock(&ev->lock);
+    if (ev->value == val) {
+        pthread_cond_wait(&ev->cond, &ev->lock);
+    }
+    pthread_mutex_unlock(&ev->lock);
+}
+#endif
+
+/* Valid transitions:
+ * - free->set, when setting the event
+ * - busy->set, when setting the event, followed by futex_wake
+ * - set->free, when resetting the event
+ * - free->busy, when waiting
+ *
+ * set->busy does not happen (it can be observed from the outside but
+ * it really is set->free->busy).
+ *
+ * busy->free provably cannot happen; to enforce it, the set->free transition
+ * is done with an OR, which becomes a no-op if the event has concurrently
+ * transitioned to free or busy.
+ */
+
+#define EV_SET         0
+#define EV_FREE        1
+#define EV_BUSY       -1
+
+void qemu_event_init(QemuEvent *ev, bool init)
+{
+#ifndef __linux__
+    pthread_mutex_init(&ev->lock, NULL);
+    pthread_cond_init(&ev->cond, NULL);
+#endif
+
+    ev->value = (init ? EV_SET : EV_FREE);
+}
+
+void qemu_event_destroy(QemuEvent *ev)
+{
+#ifndef __linux__
+    pthread_mutex_destroy(&ev->lock);
+    pthread_cond_destroy(&ev->cond);
+#endif
+}
+
+void qemu_event_set(QemuEvent *ev)
+{
+    if (atomic_mb_read(&ev->value) != EV_SET) {
+        if (atomic_xchg(&ev->value, EV_SET) == EV_BUSY) {
+            /* There were waiters, wake them up.  */
+            futex_wake(ev, INT_MAX);
+        }
+    }
+}
+
+void qemu_event_reset(QemuEvent *ev)
+{
+    if (atomic_mb_read(&ev->value) == EV_SET) {
+        /*
+         * If there was a concurrent reset (or even reset+wait),
+         * do nothing.  Otherwise change EV_SET->EV_FREE.
+         */
+        atomic_or(&ev->value, EV_FREE);
+    }
+}
+
+void qemu_event_wait(QemuEvent *ev)
+{
+    unsigned value;
+
+    value = atomic_mb_read(&ev->value);
+    if (value != EV_SET) {
+        if (value == EV_FREE) {
+            /*
+             * Leave the event reset and tell qemu_event_set that there
+             * are waiters.  No need to retry, because there cannot be
+             * a concurent busy->free transition.  After the CAS, the
+             * event will be either set or busy.
+             */
+            if (atomic_cmpxchg(&ev->value, EV_FREE, EV_BUSY) == EV_SET) {
+                return;
+            }
+        }
+        futex_wait(ev, EV_BUSY);
+    }
+}
+
+
 void qemu_thread_create(QemuThread *thread,
                        void *(*start_routine)(void*),
                        void *arg, int mode)
diff --git a/util/qemu-thread-win32.c b/util/qemu-thread-win32.c
index 517878dcc1..27a5217769 100644
--- a/util/qemu-thread-win32.c
+++ b/util/qemu-thread-win32.c
@@ -227,6 +227,32 @@ void qemu_sem_wait(QemuSemaphore *sem)
     }
 }
 
+void qemu_event_init(QemuEvent *ev, bool init)
+{
+    /* Manual reset.  */
+    ev->event = CreateEvent(NULL, TRUE, init, NULL);
+}
+
+void qemu_event_destroy(QemuEvent *ev)
+{
+    CloseHandle(ev->event);
+}
+
+void qemu_event_set(QemuEvent *ev)
+{
+    SetEvent(ev->event);
+}
+
+void qemu_event_reset(QemuEvent *ev)
+{
+    ResetEvent(ev->event);
+}
+
+void qemu_event_wait(QemuEvent *ev)
+{
+    WaitForSingleObject(ev->event, INFINITE);
+}
+
 struct QemuThreadData {
     /* Passed to win32_start_routine.  */
     void             *(*start_routine)(void *);
diff --git a/version.rc b/version.rc
index a50d62fa0c..d42ef62962 100644
--- a/version.rc
+++ b/version.rc
@@ -13,7 +13,7 @@ FILESUBTYPE VFT2_UNKNOWN
   {
     BLOCK "040904E4"
     {
-      VALUE "CompanyName", "http://www.qemu.org"
+      VALUE "CompanyName", "http://www.qemu-project.org"
       VALUE "FileDescription", "QEMU machine emulators and tools"
       VALUE "FileVersion", QEMU_VERSION
       VALUE "LegalCopyright", "Copyright various authors. Released under the GNU General Public License."
diff --git a/vl.c b/vl.c
index 46c29c42d8..efbff6512a 100644
--- a/vl.c
+++ b/vl.c
@@ -1719,14 +1719,14 @@ static pid_t shutdown_pid;
 static int powerdown_requested;
 static int debug_requested;
 static int suspend_requested;
-static int wakeup_requested;
+static WakeupReason wakeup_reason;
 static NotifierList powerdown_notifiers =
     NOTIFIER_LIST_INITIALIZER(powerdown_notifiers);
 static NotifierList suspend_notifiers =
     NOTIFIER_LIST_INITIALIZER(suspend_notifiers);
 static NotifierList wakeup_notifiers =
     NOTIFIER_LIST_INITIALIZER(wakeup_notifiers);
-static uint32_t wakeup_reason_mask = ~0;
+static uint32_t wakeup_reason_mask = ~(1 << QEMU_WAKEUP_REASON_NONE);
 static RunState vmstop_requested = RUN_STATE_MAX;
 
 int qemu_shutdown_requested_get(void)
@@ -1776,11 +1776,9 @@ static int qemu_suspend_requested(void)
     return r;
 }
 
-static int qemu_wakeup_requested(void)
+static WakeupReason qemu_wakeup_requested(void)
 {
-    int r = wakeup_requested;
-    wakeup_requested = 0;
-    return r;
+    return wakeup_reason;
 }
 
 static int qemu_powerdown_requested(void)
@@ -1897,8 +1895,7 @@ void qemu_system_wakeup_request(WakeupReason reason)
         return;
     }
     runstate_set(RUN_STATE_RUNNING);
-    notifier_list_notify(&wakeup_notifiers, &reason);
-    wakeup_requested = 1;
+    wakeup_reason = reason;
     qemu_notify_event();
 }
 
@@ -1990,6 +1987,8 @@ static bool main_loop_should_exit(void)
         pause_all_vcpus();
         cpu_synchronize_all_states();
         qemu_system_reset(VMRESET_SILENT);
+        notifier_list_notify(&wakeup_notifiers, &wakeup_reason);
+        wakeup_reason = QEMU_WAKEUP_REASON_NONE;
         resume_all_vcpus();
         monitor_protocol_event(QEVENT_WAKEUP, NULL);
     }
@@ -2826,7 +2825,7 @@ int main(int argc, char **argv, char **envp)
     const char *icount_option = NULL;
     const char *initrd_filename;
     const char *kernel_filename, *kernel_cmdline;
-    const char *boot_order = NULL;
+    const char *boot_order;
     DisplayState *ds;
     int cyls, heads, secs, translation;
     QemuOpts *hda_opts = NULL, *opts, *machine_opts;
@@ -4051,9 +4050,7 @@ int main(int argc, char **argv, char **envp)
     initrd_filename = qemu_opt_get(machine_opts, "initrd");
     kernel_cmdline = qemu_opt_get(machine_opts, "append");
 
-    if (!boot_order) {
-        boot_order = machine->default_boot_order;
-    }
+    boot_order = machine->default_boot_order;
     opts = qemu_opts_find(qemu_find_opts("boot-opts"), NULL);
     if (opts) {
         char *normal_boot_order;
@@ -4318,8 +4315,8 @@ int main(int argc, char **argv, char **envp)
     }
 #endif
 #ifdef CONFIG_SPICE
-    if (using_spice && !spice_displays) {
-        qemu_spice_display_init(ds);
+    if (using_spice) {
+        qemu_spice_display_init();
     }
 #endif
 
diff --git a/xen-all.c b/xen-all.c
index 839f14f53c..9a27899ca1 100644
--- a/xen-all.c
+++ b/xen-all.c
@@ -98,6 +98,7 @@ typedef struct XenIOState {
 
     Notifier exit;
     Notifier suspend;
+    Notifier wakeup;
 } XenIOState;
 
 /* Xen specific function for piix pci */
@@ -613,13 +614,13 @@ static ioreq_t *cpu_get_ioreq(XenIOState *state)
     }
 
     if (port != -1) {
-        for (i = 0; i < smp_cpus; i++) {
+        for (i = 0; i < max_cpus; i++) {
             if (state->ioreq_local_port[i] == port) {
                 break;
             }
         }
 
-        if (i == smp_cpus) {
+        if (i == max_cpus) {
             hw_error("Fatal error while trying to get io event!\n");
         }
 
@@ -948,7 +949,7 @@ static void xenstore_record_dm_state(struct xs_handle *xs, const char *state)
         exit(1);
     }
 
-    snprintf(path, sizeof (path), "/local/domain/0/device-model/%u/state", xen_domid);
+    snprintf(path, sizeof (path), "device-model/%u/state", xen_domid);
     if (!xs_write(xs, XBT_NULL, path, state, strlen(state))) {
         fprintf(stderr, "error recording dm state\n");
         exit(1);
@@ -1060,6 +1061,11 @@ static void xen_read_physmap(XenIOState *state)
     free(entries);
 }
 
+static void xen_wakeup_notifier(Notifier *notifier, void *data)
+{
+    xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 0);
+}
+
 int xen_hvm_init(MemoryRegion **ram_memory)
 {
     int i, rc;
@@ -1089,6 +1095,9 @@ int xen_hvm_init(MemoryRegion **ram_memory)
     state->suspend.notify = xen_suspend_notifier;
     qemu_register_suspend_notifier(&state->suspend);
 
+    state->wakeup.notify = xen_wakeup_notifier;
+    qemu_register_wakeup_notifier(&state->wakeup);
+
     xc_get_hvm_param(xen_xc, xen_domid, HVM_PARAM_IOREQ_PFN, &ioreq_pfn);
     DPRINTF("shared page at pfn %lx\n", ioreq_pfn);
     state->shared_page = xc_map_foreign_range(xen_xc, xen_domid, XC_PAGE_SIZE,
@@ -1106,10 +1115,10 @@ int xen_hvm_init(MemoryRegion **ram_memory)
         hw_error("map buffered IO page returned error %d", errno);
     }
 
-    state->ioreq_local_port = g_malloc0(smp_cpus * sizeof (evtchn_port_t));
+    state->ioreq_local_port = g_malloc0(max_cpus * sizeof (evtchn_port_t));
 
     /* FIXME: how about if we overflow the page here? */
-    for (i = 0; i < smp_cpus; i++) {
+    for (i = 0; i < max_cpus; i++) {
         rc = xc_evtchn_bind_interdomain(state->xce_handle, xen_domid,
                                         xen_vcpu_eport(state->shared_page, i));
         if (rc == -1) {