27 files changed, 628 insertions, 394 deletions

diff --git a/tcg/aarch64/tcg-target.c b/tcg/aarch64/tcg-target.c
index 41a17f8a62..6379df1f68 100644
--- a/tcg/aarch64/tcg-target.c
+++ b/tcg/aarch64/tcg-target.c
@@ -88,7 +88,7 @@ static inline void reloc_pc19(void *code_ptr, tcg_target_long target)
 }
 
 static inline void patch_reloc(uint8_t *code_ptr, int type,
-                               tcg_target_long value, tcg_target_long addend)
+                               intptr_t value, intptr_t addend)
 {
     value += addend;
 
@@ -423,14 +423,14 @@ static inline void tcg_out_mov(TCGContext *s,
 }
 
 static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg,
-                              TCGReg arg1, tcg_target_long arg2)
+                              TCGReg arg1, intptr_t arg2)
 {
     tcg_out_ldst(s, (type == TCG_TYPE_I64) ? LDST_64 : LDST_32, LDST_LD,
                  arg, arg1, arg2);
 }
 
 static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
-                              TCGReg arg1, tcg_target_long arg2)
+                              TCGReg arg1, intptr_t arg2)
 {
     tcg_out_ldst(s, (type == TCG_TYPE_I64) ? LDST_64 : LDST_32, LDST_ST,
                  arg, arg1, arg2);
@@ -778,8 +778,6 @@ static inline void tcg_out_nop(TCGContext *s)
 }
 
 #ifdef CONFIG_SOFTMMU
-#include "exec/softmmu_defs.h"
-
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h
index 51e50920b2..d3a1bc2437 100644
--- a/tcg/aarch64/tcg-target.h
+++ b/tcg/aarch64/tcg-target.h
@@ -61,6 +61,8 @@ typedef enum {
 #define TCG_TARGET_HAS_sub2_i32         0
 #define TCG_TARGET_HAS_mulu2_i32        0
 #define TCG_TARGET_HAS_muls2_i32        0
+#define TCG_TARGET_HAS_muluh_i32        0
+#define TCG_TARGET_HAS_mulsh_i32        0
 
 #define TCG_TARGET_HAS_div_i64          0
 #define TCG_TARGET_HAS_rem_i64          0
@@ -87,13 +89,14 @@ typedef enum {
 #define TCG_TARGET_HAS_sub2_i64         0
 #define TCG_TARGET_HAS_mulu2_i64        0
 #define TCG_TARGET_HAS_muls2_i64        0
+#define TCG_TARGET_HAS_muluh_i64        0
+#define TCG_TARGET_HAS_mulsh_i64        0
 
 enum {
     TCG_AREG0 = TCG_REG_X19,
 };
 
-static inline void flush_icache_range(tcg_target_ulong start,
-                                      tcg_target_ulong stop)
+static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
 {
     __builtin___clear_cache((char *)start, (char *)stop);
 }
diff --git a/tcg/arm/tcg-target.c b/tcg/arm/tcg-target.c
index 6c4854dbb0..eb0e84ce44 100644
--- a/tcg/arm/tcg-target.c
+++ b/tcg/arm/tcg-target.c
@@ -108,21 +108,21 @@ static const int tcg_target_call_oarg_regs[2] = {
 
 #define TCG_REG_TMP  TCG_REG_R12
 
-static inline void reloc_abs32(void *code_ptr, tcg_target_long target)
+static inline void reloc_abs32(void *code_ptr, intptr_t target)
 {
     *(uint32_t *) code_ptr = target;
 }
 
-static inline void reloc_pc24(void *code_ptr, tcg_target_long target)
+static inline void reloc_pc24(void *code_ptr, intptr_t target)
 {
-    uint32_t offset = ((target - ((tcg_target_long) code_ptr + 8)) >> 2);
+    uint32_t offset = ((target - ((intptr_t)code_ptr + 8)) >> 2);
 
     *(uint32_t *) code_ptr = ((*(uint32_t *) code_ptr) & ~0xffffff)
                              | (offset & 0xffffff);
 }
 
 static void patch_reloc(uint8_t *code_ptr, int type,
-                tcg_target_long value, tcg_target_long addend)
+                        intptr_t value, intptr_t addend)
 {
     switch (type) {
     case R_ARM_ABS32:
@@ -1058,8 +1058,6 @@ static inline void tcg_out_goto_label(TCGContext *s, int cond, int label_index)
 
 #ifdef CONFIG_SOFTMMU
 
-#include "exec/softmmu_defs.h"
-
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
@@ -2065,13 +2063,13 @@ static void tcg_target_init(TCGContext *s)
 }
 
 static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg,
-                              TCGReg arg1, tcg_target_long arg2)
+                              TCGReg arg1, intptr_t arg2)
 {
     tcg_out_ld32u(s, COND_AL, arg, arg1, arg2);
 }
 
 static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
-                              TCGReg arg1, tcg_target_long arg2)
+                              TCGReg arg1, intptr_t arg2)
 {
     tcg_out_st32(s, COND_AL, arg, arg1, arg2);
 }
diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h
index 5cd9d6a679..9482bfa993 100644
--- a/tcg/arm/tcg-target.h
+++ b/tcg/arm/tcg-target.h
@@ -80,6 +80,8 @@ extern bool use_idiv_instructions;
 #define TCG_TARGET_HAS_deposit_i32      1
 #define TCG_TARGET_HAS_movcond_i32      1
 #define TCG_TARGET_HAS_muls2_i32        1
+#define TCG_TARGET_HAS_muluh_i32        0
+#define TCG_TARGET_HAS_mulsh_i32        0
 #define TCG_TARGET_HAS_div_i32          use_idiv_instructions
 #define TCG_TARGET_HAS_rem_i32          0
 
@@ -90,15 +92,14 @@ enum {
     TCG_AREG0 = TCG_REG_R6,
 };
 
-static inline void flush_icache_range(tcg_target_ulong start,
-                                      tcg_target_ulong stop)
+static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
 {
 #if QEMU_GNUC_PREREQ(4, 1)
     __builtin___clear_cache((char *) start, (char *) stop);
 #else
-    register unsigned long _beg __asm ("a1") = start;
-    register unsigned long _end __asm ("a2") = stop;
-    register unsigned long _flg __asm ("a3") = 0;
+    register uintptr_t _beg __asm("a1") = start;
+    register uintptr_t _end __asm("a2") = stop;
+    register uintptr_t _flg __asm("a3") = 0;
     __asm __volatile__ ("swi 0x9f0002" : : "r" (_beg), "r" (_end), "r" (_flg));
 #endif
 }
diff --git a/tcg/hppa/tcg-target.c b/tcg/hppa/tcg-target.c
index 68f77ba4dd..236b39c31f 100644
--- a/tcg/hppa/tcg-target.c
+++ b/tcg/hppa/tcg-target.c
@@ -22,6 +22,10 @@
  * THE SOFTWARE.
  */
 
+#if TCG_TARGET_REG_BITS != 32
+#error unsupported
+#endif
+
 #ifndef NDEBUG
 static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
     "%r0", "%r1", "%rp", "%r3", "%r4", "%r5", "%r6", "%r7",
@@ -145,14 +149,14 @@ static int reassemble_21(int as21)
 #define R_PARISC_PCREL12F  R_PARISC_NONE
 
 static void patch_reloc(uint8_t *code_ptr, int type,
-                        tcg_target_long value, tcg_target_long addend)
+                        intptr_t value, intptr_t addend)
 {
     uint32_t *insn_ptr = (uint32_t *)code_ptr;
     uint32_t insn = *insn_ptr;
-    tcg_target_long pcrel;
+    intptr_t pcrel;
 
     value += addend;
-    pcrel = (value - ((tcg_target_long)code_ptr + 8)) >> 2;
+    pcrel = (value - ((intptr_t)code_ptr + 8)) >> 2;
 
     switch (type) {
     case R_PARISC_PCREL12F:
@@ -388,14 +392,14 @@ static void tcg_out_ldst(TCGContext *s, int ret, int addr,
 
 /* This function is required by tcg.c.  */
 static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
-                              TCGReg arg1, tcg_target_long arg2)
+                              TCGReg arg1, intptr_t arg2)
 {
     tcg_out_ldst(s, ret, arg1, arg2, INSN_LDW);
 }
 
 /* This function is required by tcg.c.  */
 static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg ret,
-                              TCGReg arg1, tcg_target_long arg2)
+                              TCGReg arg1, intptr_t arg2)
 {
     tcg_out_ldst(s, ret, arg1, arg2, INSN_STW);
 }
@@ -906,8 +910,6 @@ static void tcg_out_movcond(TCGContext *s, int cond, TCGArg ret,
 }
 
 #if defined(CONFIG_SOFTMMU)
-#include "exec/softmmu_defs.h"
-
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
diff --git a/tcg/hppa/tcg-target.h b/tcg/hppa/tcg-target.h
index 25467bdd43..122edce7a7 100644
--- a/tcg/hppa/tcg-target.h
+++ b/tcg/hppa/tcg-target.h
@@ -25,10 +25,6 @@
 #ifndef TCG_TARGET_HPPA
 #define TCG_TARGET_HPPA 1
 
-#if TCG_TARGET_REG_BITS != 32
-#error unsupported
-#endif
-
 #define TCG_TARGET_WORDS_BIGENDIAN
 
 #define TCG_TARGET_NB_REGS 32
@@ -100,6 +96,8 @@ typedef enum {
 #define TCG_TARGET_HAS_deposit_i32      1
 #define TCG_TARGET_HAS_movcond_i32      1
 #define TCG_TARGET_HAS_muls2_i32        0
+#define TCG_TARGET_HAS_muluh_i32        0
+#define TCG_TARGET_HAS_mulsh_i32        0
 
 /* optional instructions automatically implemented */
 #define TCG_TARGET_HAS_neg_i32          0 /* sub rd, 0, rs */
@@ -109,8 +107,7 @@ typedef enum {
 #define TCG_AREG0 TCG_REG_R17
 
 
-static inline void flush_icache_range(tcg_target_ulong start,
-                                      tcg_target_ulong stop)
+static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
 {
     start &= ~31;
     while (start <= stop) {
diff --git a/tcg/i386/tcg-target.c b/tcg/i386/tcg-target.c
index 12a7ca3440..c1f07415ab 100644
--- a/tcg/i386/tcg-target.c
+++ b/tcg/i386/tcg-target.c
@@ -112,7 +112,7 @@ static bool have_cmov;
 static uint8_t *tb_ret_addr;
 
 static void patch_reloc(uint8_t *code_ptr, int type,
-                        tcg_target_long value, tcg_target_long addend)
+                        intptr_t value, intptr_t addend)
 {
     value += addend;
     switch(type) {
@@ -430,8 +430,7 @@ static void tcg_out_modrm(TCGContext *s, int opc, int r, int rm)
    that will follow the instruction.  */
 
 static void tcg_out_modrm_sib_offset(TCGContext *s, int opc, int r, int rm,
-                                     int index, int shift,
-                                     tcg_target_long offset)
+                                     int index, int shift, intptr_t offset)
 {
     int mod, len;
 
@@ -439,8 +438,8 @@ static void tcg_out_modrm_sib_offset(TCGContext *s, int opc, int r, int rm,
         if (TCG_TARGET_REG_BITS == 64) {
             /* Try for a rip-relative addressing mode.  This has replaced
                the 32-bit-mode absolute addressing encoding.  */
-            tcg_target_long pc = (tcg_target_long)s->code_ptr + 5 + ~rm;
-            tcg_target_long disp = offset - pc;
+            intptr_t pc = (intptr_t)s->code_ptr + 5 + ~rm;
+            intptr_t disp = offset - pc;
             if (disp == (int32_t)disp) {
                 tcg_out_opc(s, opc, r, 0, 0);
                 tcg_out8(s, (LOWREGMASK(r) << 3) | 5);
@@ -514,7 +513,7 @@ static void tcg_out_modrm_sib_offset(TCGContext *s, int opc, int r, int rm,
 
 /* A simplification of the above with no index or shift.  */
 static inline void tcg_out_modrm_offset(TCGContext *s, int opc, int r,
-                                        int rm, tcg_target_long offset)
+                                        int rm, intptr_t offset)
 {
     tcg_out_modrm_sib_offset(s, opc, r, rm, -1, 0, offset);
 }
@@ -559,7 +558,7 @@ static void tcg_out_movi(TCGContext *s, TCGType type,
     }
 
     /* Try a 7 byte pc-relative lea before the 10 byte movq.  */
-    diff = arg - ((tcg_target_long)s->code_ptr + 7);
+    diff = arg - ((uintptr_t)s->code_ptr + 7);
     if (diff == (int32_t)diff) {
         tcg_out_opc(s, OPC_LEA | P_REXW, ret, 0, 0);
         tcg_out8(s, (LOWREGMASK(ret) << 3) | 5);
@@ -595,14 +594,14 @@ static inline void tcg_out_pop(TCGContext *s, int reg)
 }
 
 static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
-                              TCGReg arg1, tcg_target_long arg2)
+                              TCGReg arg1, intptr_t arg2)
 {
     int opc = OPC_MOVL_GvEv + (type == TCG_TYPE_I64 ? P_REXW : 0);
     tcg_out_modrm_offset(s, opc, ret, arg1, arg2);
 }
 
 static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
-                              TCGReg arg1, tcg_target_long arg2)
+                              TCGReg arg1, intptr_t arg2)
 {
     int opc = OPC_MOVL_EvGv + (type == TCG_TYPE_I64 ? P_REXW : 0);
     tcg_out_modrm_offset(s, opc, arg, arg1, arg2);
@@ -757,7 +756,7 @@ static void tcg_out_jxx(TCGContext *s, int opc, int label_index, int small)
     TCGLabel *l = &s->labels[label_index];
 
     if (l->has_value) {
-        val = l->u.value - (tcg_target_long)s->code_ptr;
+        val = l->u.value - (intptr_t)s->code_ptr;
         val1 = val - 2;
         if ((int8_t)val1 == val1) {
             if (opc == -1) {
@@ -997,9 +996,9 @@ static void tcg_out_movcond64(TCGContext *s, TCGCond cond, TCGArg dest,
 }
 #endif
 
-static void tcg_out_branch(TCGContext *s, int call, tcg_target_long dest)
+static void tcg_out_branch(TCGContext *s, int call, uintptr_t dest)
 {
-    tcg_target_long disp = dest - (tcg_target_long)s->code_ptr - 5;
+    intptr_t disp = dest - (intptr_t)s->code_ptr - 5;
 
     if (disp == (int32_t)disp) {
         tcg_out_opc(s, call ? OPC_CALL_Jz : OPC_JMP_long, 0, 0, 0);
@@ -1011,27 +1010,24 @@ static void tcg_out_branch(TCGContext *s, int call, tcg_target_long dest)
     }
 }
 
-static inline void tcg_out_calli(TCGContext *s, tcg_target_long dest)
+static inline void tcg_out_calli(TCGContext *s, uintptr_t dest)
 {
     tcg_out_branch(s, 1, dest);
 }
 
-static void tcg_out_jmp(TCGContext *s, tcg_target_long dest)
+static void tcg_out_jmp(TCGContext *s, uintptr_t dest)
 {
     tcg_out_branch(s, 0, dest);
 }
 
 #if defined(CONFIG_SOFTMMU)
-
-#include "exec/softmmu_defs.h"
-
 /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
  *                                     int mmu_idx, uintptr_t ra)
  */
 static const void * const qemu_ld_helpers[4] = {
-    helper_ret_ldb_mmu,
-    helper_ret_ldw_mmu,
-    helper_ret_ldl_mmu,
+    helper_ret_ldub_mmu,
+    helper_ret_lduw_mmu,
+    helper_ret_ldul_mmu,
     helper_ret_ldq_mmu,
 };
 
@@ -1086,33 +1082,46 @@ static inline void tcg_out_tlb_load(TCGContext *s, int addrlo_idx,
     const int addrlo = args[addrlo_idx];
     const int r0 = TCG_REG_L0;
     const int r1 = TCG_REG_L1;
-    TCGType type = TCG_TYPE_I32;
-    int rexw = 0;
+    TCGType ttype = TCG_TYPE_I32;
+    TCGType htype = TCG_TYPE_I32;
+    int trexw = 0, hrexw = 0;
 
-    if (TCG_TARGET_REG_BITS == 64 && TARGET_LONG_BITS == 64) {
-        type = TCG_TYPE_I64;
-        rexw = P_REXW;
+    if (TCG_TARGET_REG_BITS == 64) {
+        if (TARGET_LONG_BITS == 64) {
+            ttype = TCG_TYPE_I64;
+            trexw = P_REXW;
+        }
+        if (TCG_TYPE_PTR == TCG_TYPE_I64) {
+            htype = TCG_TYPE_I64;
+            hrexw = P_REXW;
+        }
     }
 
-    tcg_out_mov(s, type, r0, addrlo);
-    tcg_out_mov(s, type, r1, addrlo);
+    tcg_out_mov(s, htype, r0, addrlo);
+    tcg_out_mov(s, ttype, r1, addrlo);
 
-    tcg_out_shifti(s, SHIFT_SHR + rexw, r0,
+    tcg_out_shifti(s, SHIFT_SHR + hrexw, r0,
                    TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
 
-    tgen_arithi(s, ARITH_AND + rexw, r1,
+    tgen_arithi(s, ARITH_AND + trexw, r1,
                 TARGET_PAGE_MASK | ((1 << s_bits) - 1), 0);
-    tgen_arithi(s, ARITH_AND + rexw, r0,
+    tgen_arithi(s, ARITH_AND + hrexw, r0,
                 (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS, 0);
 
-    tcg_out_modrm_sib_offset(s, OPC_LEA + P_REXW, r0, TCG_AREG0, r0, 0,
+    tcg_out_modrm_sib_offset(s, OPC_LEA + hrexw, r0, TCG_AREG0, r0, 0,
                              offsetof(CPUArchState, tlb_table[mem_index][0])
                              + which);
 
     /* cmp 0(r0), r1 */
-    tcg_out_modrm_offset(s, OPC_CMP_GvEv + rexw, r1, r0, 0);
+    tcg_out_modrm_offset(s, OPC_CMP_GvEv + trexw, r1, r0, 0);
 
-    tcg_out_mov(s, type, r1, addrlo);
+    /* Prepare for both the fast path add of the tlb addend, and the slow
+       path function argument setup.  There are two cases worth note:
+       For 32-bit guest and x86_64 host, MOVL zero-extends the guest address
+       before the fastpath ADDQ below.  For 64-bit guest and x32 host, MOVQ
+       copies the entire guest address for the slow path, while truncation
+       for the 32-bit host happens with the fastpath ADDL below.  */
+    tcg_out_mov(s, ttype, r1, addrlo);
 
     /* jne slow_path */
     tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0);
@@ -1132,7 +1141,7 @@ static inline void tcg_out_tlb_load(TCGContext *s, int addrlo_idx,
     /* TLB Hit.  */
 
     /* add addend(r0), r1 */
-    tcg_out_modrm_offset(s, OPC_ADD_GvEv + P_REXW, r1, r0,
+    tcg_out_modrm_offset(s, OPC_ADD_GvEv + hrexw, r1, r0,
                          offsetof(CPUTLBEntry, addend) - which);
 }
 #elif defined(__x86_64__) && defined(__linux__)
@@ -1154,8 +1163,7 @@ static inline void setup_guest_base_seg(void) { }
 #endif /* SOFTMMU */
 
 static void tcg_out_qemu_ld_direct(TCGContext *s, int datalo, int datahi,
-                                   int base, tcg_target_long ofs, int seg,
-                                   int sizeop)
+                                   int base, intptr_t ofs, int seg, int sizeop)
 {
 #ifdef TARGET_WORDS_BIGENDIAN
     const int bswap = 1;
@@ -1305,7 +1313,7 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args,
 }
 
 static void tcg_out_qemu_st_direct(TCGContext *s, int datalo, int datahi,
-                                   int base, tcg_target_long ofs, int seg,
+                                   int base, intptr_t ofs, int seg,
                                    int sizeop)
 {
 #ifdef TARGET_WORDS_BIGENDIAN
@@ -1470,12 +1478,6 @@ static void add_qemu_ldst_label(TCGContext *s,
     }
 }
 
-/* See the GETPC definition in include/exec/exec-all.h.  */
-static inline uintptr_t do_getpc(uint8_t *raddr)
-{
-    return (uintptr_t)raddr - 1;
-}
-
 /*
  * Generate code for the slow path for a load at the end of block
  */
@@ -1509,17 +1511,17 @@ static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
         tcg_out_sti(s, TCG_TYPE_I32, TCG_REG_ESP, ofs, l->mem_index);
         ofs += 4;
 
-        tcg_out_sti(s, TCG_TYPE_I32, TCG_REG_ESP, ofs, do_getpc(l->raddr));
+        tcg_out_sti(s, TCG_TYPE_I32, TCG_REG_ESP, ofs, (uintptr_t)l->raddr);
     } else {
         tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
         /* The second argument is already loaded with addrlo.  */
         tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[2],
                      l->mem_index);
         tcg_out_movi(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[3],
-                     do_getpc(l->raddr));
+                     (uintptr_t)l->raddr);
     }
 
-    tcg_out_calli(s, (tcg_target_long)qemu_ld_helpers[s_bits]);
+    tcg_out_calli(s, (uintptr_t)qemu_ld_helpers[s_bits]);
 
     data_reg = l->datalo_reg;
     switch(opc) {
@@ -1529,20 +1531,17 @@ static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
     case 1 | 4:
         tcg_out_ext16s(s, data_reg, TCG_REG_EAX, P_REXW);
         break;
-    case 0:
-        tcg_out_ext8u(s, data_reg, TCG_REG_EAX);
-        break;
-    case 1:
-        tcg_out_ext16u(s, data_reg, TCG_REG_EAX);
-        break;
-    case 2:
-        tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
-        break;
 #if TCG_TARGET_REG_BITS == 64
     case 2 | 4:
         tcg_out_ext32s(s, data_reg, TCG_REG_EAX);
         break;
 #endif
+    case 0:
+    case 1:
+        /* Note that the helpers have zero-extended to tcg_target_long.  */
+    case 2:
+        tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
+        break;
     case 3:
         if (TCG_TARGET_REG_BITS == 64) {
             tcg_out_mov(s, TCG_TYPE_I64, data_reg, TCG_REG_RAX);
@@ -1560,7 +1559,7 @@ static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
     }
 
     /* Jump to the code corresponding to next IR of qemu_st */
-    tcg_out_jmp(s, (tcg_target_long)l->raddr);
+    tcg_out_jmp(s, (uintptr_t)l->raddr);
 }
 
 /*
@@ -1571,6 +1570,7 @@ static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
     int opc = l->opc;
     int s_bits = opc & 3;
     uint8_t **label_ptr = &l->label_ptr[0];
+    TCGReg retaddr;
 
     /* resolve label address */
     *(uint32_t *)label_ptr[0] = (uint32_t)(s->code_ptr - label_ptr[0] - 4);
@@ -1603,10 +1603,10 @@ static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
         tcg_out_sti(s, TCG_TYPE_I32, TCG_REG_ESP, ofs, l->mem_index);
         ofs += 4;
 
-        tcg_out_sti(s, TCG_TYPE_I32, TCG_REG_ESP, ofs, do_getpc(l->raddr));
+        retaddr = TCG_REG_EAX;
+        tcg_out_movi(s, TCG_TYPE_I32, retaddr, (uintptr_t)l->raddr);
+        tcg_out_st(s, TCG_TYPE_I32, retaddr, TCG_REG_ESP, ofs);
     } else {
-        uintptr_t pc;
-
         tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
         /* The second argument is already loaded with addrlo.  */
         tcg_out_mov(s, (opc == 3 ? TCG_TYPE_I64 : TCG_TYPE_I32),
@@ -1614,20 +1614,19 @@ static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
         tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[3],
                      l->mem_index);
 
-        pc = do_getpc(l->raddr);
         if (ARRAY_SIZE(tcg_target_call_iarg_regs) > 4) {
-            tcg_out_movi(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[4], pc);
-        } else if (pc == (int32_t)pc) {
-            tcg_out_sti(s, TCG_TYPE_PTR, TCG_REG_ESP, 0, pc);
+            retaddr = tcg_target_call_iarg_regs[4];
+            tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr);
         } else {
-            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_RAX, pc);
-            tcg_out_st(s, TCG_TYPE_PTR, TCG_REG_RAX, TCG_REG_ESP, 0);
+            retaddr = TCG_REG_RAX;
+            tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr);
+            tcg_out_st(s, TCG_TYPE_PTR, retaddr, TCG_REG_ESP, 0);
         }
     }
 
-    tcg_out_calli(s, (tcg_target_long)qemu_st_helpers[s_bits]);
-
-    tcg_out_jmp(s, (tcg_target_long)l->raddr);
+    /* "Tail call" to the helper, with the return address back inline.  */
+    tcg_out_push(s, retaddr);
+    tcg_out_jmp(s, (uintptr_t)qemu_st_helpers[s_bits]);
 }
 
 /*
@@ -1668,7 +1667,7 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     switch(opc) {
     case INDEX_op_exit_tb:
         tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_EAX, args[0]);
-        tcg_out_jmp(s, (tcg_target_long) tb_ret_addr);
+        tcg_out_jmp(s, (uintptr_t)tb_ret_addr);
         break;
     case INDEX_op_goto_tb:
         if (s->tb_jmp_offset) {
@@ -1679,7 +1678,7 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         } else {
             /* indirect jump method */
             tcg_out_modrm_offset(s, OPC_GRP5, EXT5_JMPN_Ev, -1,
-                                 (tcg_target_long)(s->tb_next + args[0]));
+                                 (intptr_t)(s->tb_next + args[0]));
         }
         s->tb_next_offset[args[0]] = s->code_ptr - s->code_buf;
         break;
@@ -2372,7 +2371,7 @@ static DebugFrame debug_frame = {
 #if defined(ELF_HOST_MACHINE)
 void tcg_register_jit(void *buf, size_t buf_size)
 {
-    debug_frame.fde.func_start = (tcg_target_long) buf;
+    debug_frame.fde.func_start = (uintptr_t)buf;
     debug_frame.fde.func_len = buf_size;
 
     tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h
index e3f6bb965f..d32d7ef6f0 100644
--- a/tcg/i386/tcg-target.h
+++ b/tcg/i386/tcg-target.h
@@ -24,12 +24,14 @@
 #ifndef TCG_TARGET_I386 
 #define TCG_TARGET_I386 1
 
-//#define TCG_TARGET_WORDS_BIGENDIAN
+#undef TCG_TARGET_WORDS_BIGENDIAN
 
-#if TCG_TARGET_REG_BITS == 64
-# define TCG_TARGET_NB_REGS 16
+#ifdef __x86_64__
+# define TCG_TARGET_REG_BITS  64
+# define TCG_TARGET_NB_REGS   16
 #else
-# define TCG_TARGET_NB_REGS 8
+# define TCG_TARGET_REG_BITS  32
+# define TCG_TARGET_NB_REGS    8
 #endif
 
 typedef enum {
@@ -96,6 +98,8 @@ typedef enum {
 #define TCG_TARGET_HAS_sub2_i32         1
 #define TCG_TARGET_HAS_mulu2_i32        1
 #define TCG_TARGET_HAS_muls2_i32        1
+#define TCG_TARGET_HAS_muluh_i32        0
+#define TCG_TARGET_HAS_mulsh_i32        0
 
 #if TCG_TARGET_REG_BITS == 64
 #define TCG_TARGET_HAS_div2_i64         1
@@ -122,6 +126,8 @@ typedef enum {
 #define TCG_TARGET_HAS_sub2_i64         1
 #define TCG_TARGET_HAS_mulu2_i64        1
 #define TCG_TARGET_HAS_muls2_i64        1
+#define TCG_TARGET_HAS_muluh_i64        0
+#define TCG_TARGET_HAS_mulsh_i64        0
 #endif
 
 #define TCG_TARGET_deposit_i32_valid(ofs, len) \
@@ -135,8 +141,7 @@ typedef enum {
 # define TCG_AREG0 TCG_REG_EBP
 #endif
 
-static inline void flush_icache_range(tcg_target_ulong start,
-                                      tcg_target_ulong stop)
+static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
 {
 }
 
diff --git a/tcg/ia64/tcg-target.c b/tcg/ia64/tcg-target.c
index 2373d9ef79..cd4f1ae1db 100644
--- a/tcg/ia64/tcg-target.c
+++ b/tcg/ia64/tcg-target.c
@@ -668,16 +668,16 @@ static inline uint64_t tcg_opc_x3(int qp, uint64_t opc, uint64_t imm)
  * Relocations
  */
 
-static inline void reloc_pcrel21b (void *pc, tcg_target_long target)
+static inline void reloc_pcrel21b(void *pc, intptr_t target)
 {
     uint64_t imm;
     int64_t disp;
     int slot;
 
-    slot = (tcg_target_long) pc & 3;
-    pc = (void *)((tcg_target_long) pc & ~3);
+    slot = (intptr_t)pc & 3;
+    pc = (void *)((intptr_t)pc & ~3);
 
-    disp = target - (tcg_target_long) pc;
+    disp = target - (intptr_t)pc;
     imm = (uint64_t) disp >> 4;
 
     switch(slot) {
@@ -728,12 +728,12 @@ static inline uint64_t get_reloc_pcrel21b (void *pc)
     }
 }
 
-static inline void reloc_pcrel60b (void *pc, tcg_target_long target)
+static inline void reloc_pcrel60b(void *pc, intptr_t target)
 {
     int64_t disp;
     uint64_t imm;
 
-    disp = target - (tcg_target_long) pc;
+    disp = target - (intptr_t)pc;
     imm = (uint64_t) disp >> 4;
 
     *(uint64_t *)(pc + 8) = (*(uint64_t *)(pc + 8) & 0xf700000fff800000ull)
@@ -759,7 +759,7 @@ static inline uint64_t get_reloc_pcrel60b (void *pc)
 
 
 static void patch_reloc(uint8_t *code_ptr, int type,
-                        tcg_target_long value, tcg_target_long addend)
+                        intptr_t value, intptr_t addend)
 {
     value += addend;
     switch (type) {
@@ -993,7 +993,7 @@ static inline void tcg_out_st_rel(TCGContext *s, uint64_t opc_m4, TCGArg arg,
 }
 
 static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg,
-                              TCGReg arg1, tcg_target_long arg2)
+                              TCGReg arg1, intptr_t arg2)
 {
     if (type == TCG_TYPE_I32) {
         tcg_out_ld_rel(s, OPC_LD4_M1, arg, arg1, arg2);
@@ -1003,7 +1003,7 @@ static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg,
 }
 
 static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
-                              TCGReg arg1, tcg_target_long arg2)
+                              TCGReg arg1, intptr_t arg2)
 {
     if (type == TCG_TYPE_I32) {
         tcg_out_st_rel(s, OPC_ST4_M4, arg, arg1, arg2);
@@ -1490,9 +1490,6 @@ static inline void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGArg ret,
 }
 
 #if defined(CONFIG_SOFTMMU)
-
-#include "exec/softmmu_defs.h"
-
 /* Load and compare a TLB entry, and return the result in (p6, p7).
    R2 is loaded with the address of the addend TLB entry.
    R57 is loaded with the address, zero extented on 32-bit targets. */
diff --git a/tcg/ia64/tcg-target.h b/tcg/ia64/tcg-target.h
index f32d5199cb..4330c9cdd3 100644
--- a/tcg/ia64/tcg-target.h
+++ b/tcg/ia64/tcg-target.h
@@ -146,6 +146,10 @@ typedef enum {
 #define TCG_TARGET_HAS_mulu2_i64        0
 #define TCG_TARGET_HAS_muls2_i32        0
 #define TCG_TARGET_HAS_muls2_i64        0
+#define TCG_TARGET_HAS_muluh_i32        0
+#define TCG_TARGET_HAS_muluh_i64        0
+#define TCG_TARGET_HAS_mulsh_i32        0
+#define TCG_TARGET_HAS_mulsh_i64        0
 
 #define TCG_TARGET_deposit_i32_valid(ofs, len) ((len) <= 16)
 #define TCG_TARGET_deposit_i64_valid(ofs, len) ((len) <= 16)
@@ -158,8 +162,7 @@ typedef enum {
 
 #define TCG_AREG0 TCG_REG_R7
 
-static inline void flush_icache_range(tcg_target_ulong start,
-                                      tcg_target_ulong stop)
+static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
 {
     start = start & ~(32UL - 1UL);
     stop = (stop + (32UL - 1UL)) & ~(32UL - 1UL);
diff --git a/tcg/mips/tcg-target.c b/tcg/mips/tcg-target.c
index 793532ec95..5f0a65b4ea 100644
--- a/tcg/mips/tcg-target.c
+++ b/tcg/mips/tcg-target.c
@@ -108,33 +108,33 @@ static const TCGReg tcg_target_call_oarg_regs[2] = {
 
 static uint8_t *tb_ret_addr;
 
-static inline uint32_t reloc_lo16_val (void *pc, tcg_target_long target)
+static inline uint32_t reloc_lo16_val(void *pc, intptr_t target)
 {
     return target & 0xffff;
 }
 
-static inline void reloc_lo16 (void *pc, tcg_target_long target)
+static inline void reloc_lo16(void *pc, intptr_t target)
 {
     *(uint32_t *) pc = (*(uint32_t *) pc & ~0xffff)
                        | reloc_lo16_val(pc, target);
 }
 
-static inline uint32_t reloc_hi16_val (void *pc, tcg_target_long target)
+static inline uint32_t reloc_hi16_val(void *pc, intptr_t target)
 {
     return (target >> 16) & 0xffff;
 }
 
-static inline void reloc_hi16 (void *pc, tcg_target_long target)
+static inline void reloc_hi16(void *pc, intptr_t target)
 {
     *(uint32_t *) pc = (*(uint32_t *) pc & ~0xffff)
                        | reloc_hi16_val(pc, target);
 }
 
-static inline uint32_t reloc_pc16_val (void *pc, tcg_target_long target)
+static inline uint32_t reloc_pc16_val(void *pc, intptr_t target)
 {
     int32_t disp;
 
-    disp = target - (tcg_target_long) pc - 4;
+    disp = target - (intptr_t)pc - 4;
     if (disp != (disp << 14) >> 14) {
         tcg_abort ();
     }
@@ -157,14 +157,14 @@ static inline uint32_t reloc_26_val (void *pc, tcg_target_long target)
     return (target >> 2) & 0x3ffffff;
 }
 
-static inline void reloc_pc26 (void *pc, tcg_target_long target)
+static inline void reloc_pc26(void *pc, intptr_t target)
 {
     *(uint32_t *) pc = (*(uint32_t *) pc & ~0x3ffffff)
                        | reloc_26_val(pc, target);
 }
 
 static void patch_reloc(uint8_t *code_ptr, int type,
-                        tcg_target_long value, tcg_target_long addend)
+                        intptr_t value, intptr_t addend)
 {
     value += addend;
     switch(type) {
@@ -422,83 +422,83 @@ static inline void tcg_out_movi(TCGContext *s, TCGType type,
 
 static inline void tcg_out_bswap16(TCGContext *s, TCGReg ret, TCGReg arg)
 {
-#if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
-    tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg);
-#else
-    /* ret and arg can't be register at */
-    if (ret == TCG_REG_AT || arg == TCG_REG_AT) {
-        tcg_abort();
-    }
+    if (use_mips32r2_instructions) {
+        tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg);
+    } else {
+        /* ret and arg can't be register at */
+        if (ret == TCG_REG_AT || arg == TCG_REG_AT) {
+            tcg_abort();
+        }
 
-    tcg_out_opc_sa(s, OPC_SRL, TCG_REG_AT, arg, 8);
-    tcg_out_opc_sa(s, OPC_SLL, ret, arg, 8);
-    tcg_out_opc_imm(s, OPC_ANDI, ret, ret, 0xff00);
-    tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_REG_AT);
-#endif
+        tcg_out_opc_sa(s, OPC_SRL, TCG_REG_AT, arg, 8);
+        tcg_out_opc_sa(s, OPC_SLL, ret, arg, 8);
+        tcg_out_opc_imm(s, OPC_ANDI, ret, ret, 0xff00);
+        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_REG_AT);
+    }
 }
 
 static inline void tcg_out_bswap16s(TCGContext *s, TCGReg ret, TCGReg arg)
 {
-#if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
-    tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg);
-    tcg_out_opc_reg(s, OPC_SEH, ret, 0, ret);
-#else
-    /* ret and arg can't be register at */
-    if (ret == TCG_REG_AT || arg == TCG_REG_AT) {
-        tcg_abort();
-    }
+    if (use_mips32r2_instructions) {
+        tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg);
+        tcg_out_opc_reg(s, OPC_SEH, ret, 0, ret);
+    } else {
+        /* ret and arg can't be register at */
+        if (ret == TCG_REG_AT || arg == TCG_REG_AT) {
+            tcg_abort();
+        }
 
-    tcg_out_opc_sa(s, OPC_SRL, TCG_REG_AT, arg, 8);
-    tcg_out_opc_sa(s, OPC_SLL, ret, arg, 24);
-    tcg_out_opc_sa(s, OPC_SRA, ret, ret, 16);
-    tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_REG_AT);
-#endif
+        tcg_out_opc_sa(s, OPC_SRL, TCG_REG_AT, arg, 8);
+        tcg_out_opc_sa(s, OPC_SLL, ret, arg, 24);
+        tcg_out_opc_sa(s, OPC_SRA, ret, ret, 16);
+        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_REG_AT);
+    }
 }
 
 static inline void tcg_out_bswap32(TCGContext *s, TCGReg ret, TCGReg arg)
 {
-#if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
-    tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg);
-    tcg_out_opc_sa(s, OPC_ROTR, ret, ret, 16);
-#else
-    /* ret and arg must be different and can't be register at */
-    if (ret == arg || ret == TCG_REG_AT || arg == TCG_REG_AT) {
-        tcg_abort();
-    }
+    if (use_mips32r2_instructions) {
+        tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg);
+        tcg_out_opc_sa(s, OPC_ROTR, ret, ret, 16);
+    } else {
+        /* ret and arg must be different and can't be register at */
+        if (ret == arg || ret == TCG_REG_AT || arg == TCG_REG_AT) {
+            tcg_abort();
+        }
 
-    tcg_out_opc_sa(s, OPC_SLL, ret, arg, 24);
+        tcg_out_opc_sa(s, OPC_SLL, ret, arg, 24);
 
-    tcg_out_opc_sa(s, OPC_SRL, TCG_REG_AT, arg, 24);
-    tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_REG_AT);
+        tcg_out_opc_sa(s, OPC_SRL, TCG_REG_AT, arg, 24);
+        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_REG_AT);
 
-    tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_AT, arg, 0xff00);
-    tcg_out_opc_sa(s, OPC_SLL, TCG_REG_AT, TCG_REG_AT, 8);
-    tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_REG_AT);
+        tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_AT, arg, 0xff00);
+        tcg_out_opc_sa(s, OPC_SLL, TCG_REG_AT, TCG_REG_AT, 8);
+        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_REG_AT);
 
-    tcg_out_opc_sa(s, OPC_SRL, TCG_REG_AT, arg, 8);
-    tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_AT, TCG_REG_AT, 0xff00);
-    tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_REG_AT);
-#endif
+        tcg_out_opc_sa(s, OPC_SRL, TCG_REG_AT, arg, 8);
+        tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_AT, TCG_REG_AT, 0xff00);
+        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_REG_AT);
+    }
 }
 
 static inline void tcg_out_ext8s(TCGContext *s, TCGReg ret, TCGReg arg)
 {
-#if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
-    tcg_out_opc_reg(s, OPC_SEB, ret, 0, arg);
-#else
-    tcg_out_opc_sa(s, OPC_SLL, ret, arg, 24);
-    tcg_out_opc_sa(s, OPC_SRA, ret, ret, 24);
-#endif
+    if (use_mips32r2_instructions) {
+        tcg_out_opc_reg(s, OPC_SEB, ret, 0, arg);
+    } else {
+        tcg_out_opc_sa(s, OPC_SLL, ret, arg, 24);
+        tcg_out_opc_sa(s, OPC_SRA, ret, ret, 24);
+    }
 }
 
 static inline void tcg_out_ext16s(TCGContext *s, TCGReg ret, TCGReg arg)
 {
-#if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
-    tcg_out_opc_reg(s, OPC_SEH, ret, 0, arg);
-#else
-    tcg_out_opc_sa(s, OPC_SLL, ret, arg, 16);
-    tcg_out_opc_sa(s, OPC_SRA, ret, ret, 16);
-#endif
+    if (use_mips32r2_instructions) {
+        tcg_out_opc_reg(s, OPC_SEH, ret, 0, arg);
+    } else {
+        tcg_out_opc_sa(s, OPC_SLL, ret, arg, 16);
+        tcg_out_opc_sa(s, OPC_SRA, ret, ret, 16);
+    }
 }
 
 static inline void tcg_out_ldst(TCGContext *s, int opc, TCGArg arg,
@@ -514,13 +514,13 @@ static inline void tcg_out_ldst(TCGContext *s, int opc, TCGArg arg,
 }
 
 static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg,
-                              TCGReg arg1, tcg_target_long arg2)
+                              TCGReg arg1, intptr_t arg2)
 {
     tcg_out_ldst(s, OPC_LW, arg, arg1, arg2);
 }
 
 static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
-                              TCGReg arg1, tcg_target_long arg2)
+                              TCGReg arg1, intptr_t arg2)
 {
     tcg_out_ldst(s, OPC_SW, arg, arg1, arg2);
 }
@@ -919,9 +919,6 @@ static void tcg_out_setcond2(TCGContext *s, TCGCond cond, TCGReg ret,
 }
 
 #if defined(CONFIG_SOFTMMU)
-
-#include "exec/softmmu_defs.h"
-
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
@@ -1406,12 +1403,12 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_mov(s, TCG_TYPE_I32, args[0], TCG_REG_AT);
         break;
     case INDEX_op_mul_i32:
-#if defined(__mips_isa_rev) && (__mips_isa_rev >= 1)
-        tcg_out_opc_reg(s, OPC_MUL, args[0], args[1], args[2]);
-#else
-        tcg_out_opc_reg(s, OPC_MULT, 0, args[1], args[2]);
-        tcg_out_opc_reg(s, OPC_MFLO, args[0], 0, 0);
-#endif
+        if (use_mips32_instructions) {
+            tcg_out_opc_reg(s, OPC_MUL, args[0], args[1], args[2]);
+        } else {
+            tcg_out_opc_reg(s, OPC_MULT, 0, args[1], args[2]);
+            tcg_out_opc_reg(s, OPC_MFLO, args[0], 0, 0);
+        }
         break;
     case INDEX_op_muls2_i32:
         tcg_out_opc_reg(s, OPC_MULT, 0, args[2], args[3]);
@@ -1423,6 +1420,14 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_opc_reg(s, OPC_MFLO, args[0], 0, 0);
         tcg_out_opc_reg(s, OPC_MFHI, args[1], 0, 0);
         break;
+    case INDEX_op_mulsh_i32:
+        tcg_out_opc_reg(s, OPC_MULT, 0, args[1], args[2]);
+        tcg_out_opc_reg(s, OPC_MFHI, args[0], 0, 0);
+        break;
+    case INDEX_op_muluh_i32:
+        tcg_out_opc_reg(s, OPC_MULTU, 0, args[1], args[2]);
+        tcg_out_opc_reg(s, OPC_MFHI, args[0], 0, 0);
+        break;
     case INDEX_op_div_i32:
         tcg_out_opc_reg(s, OPC_DIV, 0, args[1], args[2]);
         tcg_out_opc_reg(s, OPC_MFLO, args[0], 0, 0);
@@ -1506,20 +1511,19 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         }
         break;
 
-    /* The bswap routines do not work on non-R2 CPU. In that case
-       we let TCG generating the corresponding code. */
     case INDEX_op_bswap16_i32:
-        tcg_out_bswap16(s, args[0], args[1]);
+        tcg_out_opc_reg(s, OPC_WSBH, args[0], 0, args[1]);
         break;
     case INDEX_op_bswap32_i32:
-        tcg_out_bswap32(s, args[0], args[1]);
+        tcg_out_opc_reg(s, OPC_WSBH, args[0], 0, args[1]);
+        tcg_out_opc_sa(s, OPC_ROTR, args[0], args[0], 16);
         break;
 
     case INDEX_op_ext8s_i32:
-        tcg_out_ext8s(s, args[0], args[1]);
+        tcg_out_opc_reg(s, OPC_SEB, args[0], 0, args[1]);
         break;
     case INDEX_op_ext16s_i32:
-        tcg_out_ext16s(s, args[0], args[1]);
+        tcg_out_opc_reg(s, OPC_SEH, args[0], 0, args[1]);
         break;
 
     case INDEX_op_deposit_i32:
@@ -1602,6 +1606,8 @@ static const TCGTargetOpDef mips_op_defs[] = {
     { INDEX_op_mul_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_muls2_i32, { "r", "r", "rZ", "rZ" } },
     { INDEX_op_mulu2_i32, { "r", "r", "rZ", "rZ" } },
+    { INDEX_op_mulsh_i32, { "r", "rZ", "rZ" } },
+    { INDEX_op_muluh_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_div_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_divu_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_rem_i32, { "r", "rZ", "rZ" } },
@@ -1617,29 +1623,19 @@ static const TCGTargetOpDef mips_op_defs[] = {
     { INDEX_op_shl_i32, { "r", "rZ", "ri" } },
     { INDEX_op_shr_i32, { "r", "rZ", "ri" } },
     { INDEX_op_sar_i32, { "r", "rZ", "ri" } },
-#if TCG_TARGET_HAS_rot_i32
     { INDEX_op_rotr_i32, { "r", "rZ", "ri" } },
     { INDEX_op_rotl_i32, { "r", "rZ", "ri" } },
-#endif
 
-#if TCG_TARGET_HAS_bswap16_i32
     { INDEX_op_bswap16_i32, { "r", "r" } },
-#endif
-#if TCG_TARGET_HAS_bswap32_i32
     { INDEX_op_bswap32_i32, { "r", "r" } },
-#endif
 
     { INDEX_op_ext8s_i32, { "r", "rZ" } },
     { INDEX_op_ext16s_i32, { "r", "rZ" } },
 
-#if TCG_TARGET_HAS_deposit_i32
     { INDEX_op_deposit_i32, { "r", "0", "rZ" } },
-#endif
 
     { INDEX_op_brcond_i32, { "rZ", "rZ" } },
-#if TCG_TARGET_HAS_movcond_i32
     { INDEX_op_movcond_i32, { "r", "rZ", "rZ", "rZ", "0" } },
-#endif
     { INDEX_op_setcond_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_setcond2_i32, { "r", "rZ", "rZ", "rZ", "rZ" } },
 
@@ -1688,6 +1684,86 @@ static int tcg_target_callee_save_regs[] = {
     TCG_REG_RA,       /* should be last for ABI compliance */
 };
 
+/* The Linux kernel doesn't provide any information about the available
+   instruction set. Probe it using a signal handler. */
+
+#include <signal.h>
+
+#ifndef use_movnz_instructions
+bool use_movnz_instructions = false;
+#endif
+
+#ifndef use_mips32_instructions
+bool use_mips32_instructions = false;
+#endif
+
+#ifndef use_mips32r2_instructions
+bool use_mips32r2_instructions = false;
+#endif
+
+static volatile sig_atomic_t got_sigill;
+
+static void sigill_handler(int signo, siginfo_t *si, void *data)
+{
+    /* Skip the faulty instruction */
+    ucontext_t *uc = (ucontext_t *)data;
+    uc->uc_mcontext.pc += 4;
+
+    got_sigill = 1;
+}
+
+static void tcg_target_detect_isa(void)
+{
+    struct sigaction sa_old, sa_new;
+
+    memset(&sa_new, 0, sizeof(sa_new));
+    sa_new.sa_flags = SA_SIGINFO;
+    sa_new.sa_sigaction = sigill_handler;
+    sigaction(SIGILL, &sa_new, &sa_old);
+
+    /* Probe for movn/movz, necessary to implement movcond. */
+#ifndef use_movnz_instructions
+    got_sigill = 0;
+    asm volatile(".set push\n"
+                 ".set mips32\n"
+                 "movn $zero, $zero, $zero\n"
+                 "movz $zero, $zero, $zero\n"
+                 ".set pop\n"
+                 : : : );
+    use_movnz_instructions = !got_sigill;
+#endif
+
+    /* Probe for MIPS32 instructions. As no subsetting is allowed
+       by the specification, it is only necessary to probe for one
+       of the instructions. */
+#ifndef use_mips32_instructions
+    got_sigill = 0;
+    asm volatile(".set push\n"
+                 ".set mips32\n"
+                 "mul $zero, $zero\n"
+                 ".set pop\n"
+                 : : : );
+    use_mips32_instructions = !got_sigill;
+#endif
+
+    /* Probe for MIPS32r2 instructions if MIPS32 instructions are
+       available. As no subsetting is allowed by the specification,
+       it is only necessary to probe for one of the instructions. */
+#ifndef use_mips32r2_instructions
+    if (use_mips32_instructions) {
+        got_sigill = 0;
+        asm volatile(".set push\n"
+                     ".set mips32r2\n"
+                     "seb $zero, $zero\n"
+                     ".set pop\n"
+                     : : : );
+        use_mips32r2_instructions = !got_sigill;
+    }
+#endif
+
+    sigaction(SIGILL, &sa_old, NULL);
+}
+
 /* Generate global QEMU prologue and epilogue code */
 static void tcg_target_qemu_prologue(TCGContext *s)
 {
@@ -1727,6 +1803,7 @@ static void tcg_target_qemu_prologue(TCGContext *s)
 
 static void tcg_target_init(TCGContext *s)
 {
+    tcg_target_detect_isa();
     tcg_regset_set(tcg_target_available_regs[TCG_TYPE_I32], 0xffffffff);
     tcg_regset_set(tcg_target_call_clobber_regs,
                    (1 << TCG_REG_V0) |
diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h
index a438950bc1..c37252269f 100644
--- a/tcg/mips/tcg-target.h
+++ b/tcg/mips/tcg-target.h
@@ -77,40 +77,50 @@ typedef enum {
 #define TCG_TARGET_CALL_STACK_OFFSET 16
 #define TCG_TARGET_CALL_ALIGN_ARGS 1
 
+/* MOVN/MOVZ instructions detection */
+#if (defined(__mips_isa_rev) && (__mips_isa_rev >= 1)) || \
+    defined(_MIPS_ARCH_LOONGSON2E) || defined(_MIPS_ARCH_LOONGSON2F) || \
+    defined(_MIPS_ARCH_MIPS4)
+#define use_movnz_instructions  1
+#else
+extern bool use_movnz_instructions;
+#endif
+
+/* MIPS32 instruction set detection */
+#if defined(__mips_isa_rev) && (__mips_isa_rev >= 1)
+#define use_mips32_instructions  1
+#else
+extern bool use_mips32_instructions;
+#endif
+
+/* MIPS32R2 instruction set detection */
+#if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
+#define use_mips32r2_instructions  1
+#else
+extern bool use_mips32r2_instructions;
+#endif
+
 /* optional instructions */
 #define TCG_TARGET_HAS_div_i32          1
 #define TCG_TARGET_HAS_rem_i32          1
 #define TCG_TARGET_HAS_not_i32          1
 #define TCG_TARGET_HAS_nor_i32          1
-#define TCG_TARGET_HAS_ext8s_i32        1
-#define TCG_TARGET_HAS_ext16s_i32       1
 #define TCG_TARGET_HAS_andc_i32         0
 #define TCG_TARGET_HAS_orc_i32          0
 #define TCG_TARGET_HAS_eqv_i32          0
 #define TCG_TARGET_HAS_nand_i32         0
 #define TCG_TARGET_HAS_muls2_i32        1
+#define TCG_TARGET_HAS_muluh_i32        1
+#define TCG_TARGET_HAS_mulsh_i32        1
 
-/* optional instructions only implemented on MIPS4, MIPS32 and Loongson 2 */
-#if (defined(__mips_isa_rev) && (__mips_isa_rev >= 1)) || \
-    defined(_MIPS_ARCH_LOONGSON2E) || defined(_MIPS_ARCH_LOONGSON2F) || \
-    defined(_MIPS_ARCH_MIPS4)
-#define TCG_TARGET_HAS_movcond_i32      1
-#else
-#define TCG_TARGET_HAS_movcond_i32      0
-#endif
-
-/* optional instructions only implemented on MIPS32R2 */
-#if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
-#define TCG_TARGET_HAS_bswap16_i32      1
-#define TCG_TARGET_HAS_bswap32_i32      1
-#define TCG_TARGET_HAS_rot_i32          1
-#define TCG_TARGET_HAS_deposit_i32      1
-#else
-#define TCG_TARGET_HAS_bswap16_i32      0
-#define TCG_TARGET_HAS_bswap32_i32      0
-#define TCG_TARGET_HAS_rot_i32          0
-#define TCG_TARGET_HAS_deposit_i32      0
-#endif
+/* optional instructions detected at runtime */
+#define TCG_TARGET_HAS_movcond_i32      use_movnz_instructions
+#define TCG_TARGET_HAS_bswap16_i32      use_mips32r2_instructions
+#define TCG_TARGET_HAS_bswap32_i32      use_mips32r2_instructions
+#define TCG_TARGET_HAS_deposit_i32      use_mips32r2_instructions
+#define TCG_TARGET_HAS_ext8s_i32        use_mips32r2_instructions
+#define TCG_TARGET_HAS_ext16s_i32       use_mips32r2_instructions
+#define TCG_TARGET_HAS_rot_i32          use_mips32r2_instructions
 
 /* optional instructions automatically implemented */
 #define TCG_TARGET_HAS_neg_i32          0 /* sub  rd, zero, rt   */
@@ -125,8 +135,7 @@ typedef enum {
 #include <sys/cachectl.h>
 #endif
 
-static inline void flush_icache_range(tcg_target_ulong start,
-                                      tcg_target_ulong stop)
+static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
 {
     cacheflush ((void *)start, stop-start, ICACHE);
 }
diff --git a/tcg/optimize.c b/tcg/optimize.c
index b35868afbc..b29bf25b67 100644
--- a/tcg/optimize.c
+++ b/tcg/optimize.c
@@ -198,6 +198,8 @@ static TCGOpcode op_to_mov(TCGOpcode op)
 
 static TCGArg do_constant_folding_2(TCGOpcode op, TCGArg x, TCGArg y)
 {
+    uint64_t l64, h64;
+
     switch (op) {
     CASE_OP_32_64(add):
         return x + y;
@@ -290,6 +292,37 @@ static TCGArg do_constant_folding_2(TCGOpcode op, TCGArg x, TCGArg y)
     case INDEX_op_ext32u_i64:
         return (uint32_t)x;
 
+    case INDEX_op_muluh_i32:
+        return ((uint64_t)(uint32_t)x * (uint32_t)y) >> 32;
+    case INDEX_op_mulsh_i32:
+        return ((int64_t)(int32_t)x * (int32_t)y) >> 32;
+
+    case INDEX_op_muluh_i64:
+        mulu64(&l64, &h64, x, y);
+        return h64;
+    case INDEX_op_mulsh_i64:
+        muls64(&l64, &h64, x, y);
+        return h64;
+
+    case INDEX_op_div_i32:
+        /* Avoid crashing on divide by zero, otherwise undefined.  */
+        return (int32_t)x / ((int32_t)y ? : 1);
+    case INDEX_op_divu_i32:
+        return (uint32_t)x / ((uint32_t)y ? : 1);
+    case INDEX_op_div_i64:
+        return (int64_t)x / ((int64_t)y ? : 1);
+    case INDEX_op_divu_i64:
+        return (uint64_t)x / ((uint64_t)y ? : 1);
+
+    case INDEX_op_rem_i32:
+        return (int32_t)x % ((int32_t)y ? : 1);
+    case INDEX_op_remu_i32:
+        return (uint32_t)x % ((uint32_t)y ? : 1);
+    case INDEX_op_rem_i64:
+        return (int64_t)x % ((int64_t)y ? : 1);
+    case INDEX_op_remu_i64:
+        return (uint64_t)x % ((uint64_t)y ? : 1);
+
     default:
         fprintf(stderr,
                 "Unrecognized operation %d in do_constant_folding.\n", op);
@@ -531,6 +564,8 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
         CASE_OP_32_64(eqv):
         CASE_OP_32_64(nand):
         CASE_OP_32_64(nor):
+        CASE_OP_32_64(muluh):
+        CASE_OP_32_64(mulsh):
             swap_commutative(args[0], &args[1], &args[2]);
             break;
         CASE_OP_32_64(brcond):
@@ -771,6 +806,8 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
         switch (op) {
         CASE_OP_32_64(and):
         CASE_OP_32_64(mul):
+        CASE_OP_32_64(muluh):
+        CASE_OP_32_64(mulsh):
             if ((temps[args[2]].state == TCG_TEMP_CONST
                 && temps[args[2]].val == 0)) {
                 s->gen_opc_buf[op_index] = op_to_movi(op);
@@ -882,6 +919,12 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
         CASE_OP_32_64(eqv):
         CASE_OP_32_64(nand):
         CASE_OP_32_64(nor):
+        CASE_OP_32_64(muluh):
+        CASE_OP_32_64(mulsh):
+        CASE_OP_32_64(div):
+        CASE_OP_32_64(divu):
+        CASE_OP_32_64(rem):
+        CASE_OP_32_64(remu):
             if (temps[args[1]].state == TCG_TEMP_CONST
                 && temps[args[2]].state == TCG_TEMP_CONST) {
                 s->gen_opc_buf[op_index] = op_to_movi(op);
diff --git a/tcg/ppc/tcg-target.c b/tcg/ppc/tcg-target.c
index 453ab6b580..25955563b8 100644
--- a/tcg/ppc/tcg-target.c
+++ b/tcg/ppc/tcg-target.c
@@ -204,7 +204,7 @@ static void reloc_pc14 (void *pc, tcg_target_long target)
 }
 
 static void patch_reloc(uint8_t *code_ptr, int type,
-                        tcg_target_long value, tcg_target_long addend)
+                        intptr_t value, intptr_t addend)
 {
     value += addend;
     switch (type) {
@@ -549,8 +549,6 @@ static void add_qemu_ldst_label (TCGContext *s,
     label->label_ptr[0] = label_ptr;
 }
 
-#include "exec/softmmu_defs.h"
-
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
@@ -1062,14 +1060,14 @@ static void tcg_target_qemu_prologue (TCGContext *s)
 #endif
 }
 
-static void tcg_out_ld (TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
-                        tcg_target_long arg2)
+static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
+                       intptr_t arg2)
 {
     tcg_out_ldst (s, ret, arg1, arg2, LWZ, LWZX);
 }
 
-static void tcg_out_st (TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
-                        tcg_target_long arg2)
+static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
+                       intptr_t arg2)
 {
     tcg_out_ldst (s, arg, arg1, arg2, STW, STWX);
 }
diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h
index b42d97cc24..c9f8ff5206 100644
--- a/tcg/ppc/tcg-target.h
+++ b/tcg/ppc/tcg-target.h
@@ -96,11 +96,13 @@ typedef enum {
 #define TCG_TARGET_HAS_deposit_i32      1
 #define TCG_TARGET_HAS_movcond_i32      1
 #define TCG_TARGET_HAS_muls2_i32        0
+#define TCG_TARGET_HAS_muluh_i32        0
+#define TCG_TARGET_HAS_mulsh_i32        0
 
 #define TCG_AREG0 TCG_REG_R27
 
 #define tcg_qemu_tb_exec(env, tb_ptr) \
-    ((long __attribute__ ((longcall)) \
+    ((uintptr_t __attribute__ ((longcall)) \
       (*)(void *, void *))tcg_ctx.code_gen_prologue)(env, tb_ptr)
 
 #endif
diff --git a/tcg/ppc64/tcg-target.c b/tcg/ppc64/tcg-target.c
index 0678de2045..0bd1e0ce8c 100644
--- a/tcg/ppc64/tcg-target.c
+++ b/tcg/ppc64/tcg-target.c
@@ -208,7 +208,7 @@ static void reloc_pc14 (void *pc, tcg_target_long target)
 }
 
 static void patch_reloc (uint8_t *code_ptr, int type,
-                         tcg_target_long value, tcg_target_long addend)
+                         intptr_t value, intptr_t addend)
 {
     value += addend;
     switch (type) {
@@ -750,9 +750,6 @@ static void tcg_out_ldsta(TCGContext *s, TCGReg ret, TCGReg addr,
 }
 
 #if defined (CONFIG_SOFTMMU)
-
-#include "exec/softmmu_defs.h"
-
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
@@ -1072,8 +1069,8 @@ static void tcg_target_qemu_prologue (TCGContext *s)
     tcg_out32(s, BCLR | BO_ALWAYS);
 }
 
-static void tcg_out_ld (TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
-                        tcg_target_long arg2)
+static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
+                       intptr_t arg2)
 {
     if (type == TCG_TYPE_I32)
         tcg_out_ldst (s, ret, arg1, arg2, LWZ, LWZX);
@@ -1081,8 +1078,8 @@ static void tcg_out_ld (TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
         tcg_out_ldsta (s, ret, arg1, arg2, LD, LDX);
 }
 
-static void tcg_out_st (TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
-                        tcg_target_long arg2)
+static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
+                       intptr_t arg2)
 {
     if (type == TCG_TYPE_I32)
         tcg_out_ldst (s, arg, arg1, arg2, STW, STWX);
@@ -1975,29 +1972,11 @@ static void tcg_out_op (TCGContext *s, TCGOpcode opc, const TCGArg *args,
         }
         break;
 
-    case INDEX_op_mulu2_i64:
-    case INDEX_op_muls2_i64:
-        {
-            int oph = (opc == INDEX_op_mulu2_i64 ? MULHDU : MULHD);
-            TCGReg outl = args[0], outh = args[1];
-            a0 = args[2], a1 = args[3];
-
-            if (outl == a0 || outl == a1) {
-                if (outh == a0 || outh == a1) {
-                    outl = TCG_REG_R0;
-                } else {
-                    tcg_out32(s, oph | TAB(outh, a0, a1));
-                    oph = 0;
-                }
-            }
-            tcg_out32(s, MULLD | TAB(outl, a0, a1));
-            if (oph != 0) {
-                tcg_out32(s, oph | TAB(outh, a0, a1));
-            }
-            if (outl != args[0]) {
-                tcg_out_mov(s, TCG_TYPE_I64, args[0], outl);
-            }
-        }
+    case INDEX_op_muluh_i64:
+        tcg_out32(s, MULHDU | TAB(args[0], args[1], args[2]));
+        break;
+    case INDEX_op_mulsh_i64:
+        tcg_out32(s, MULHD | TAB(args[0], args[1], args[2]));
         break;
 
     default:
@@ -2124,8 +2103,8 @@ static const TCGTargetOpDef ppc_op_defs[] = {
 
     { INDEX_op_add2_i64, { "r", "r", "r", "r", "rI", "rZM" } },
     { INDEX_op_sub2_i64, { "r", "r", "rI", "r", "rZM", "r" } },
-    { INDEX_op_muls2_i64, { "r", "r", "r", "r" } },
-    { INDEX_op_mulu2_i64, { "r", "r", "r", "r" } },
+    { INDEX_op_mulsh_i64, { "r", "r", "r" } },
+    { INDEX_op_muluh_i64, { "r", "r", "r" } },
 
     { -1 },
 };
diff --git a/tcg/ppc64/tcg-target.h b/tcg/ppc64/tcg-target.h
index 48fc6e2e54..fa4b9da093 100644
--- a/tcg/ppc64/tcg-target.h
+++ b/tcg/ppc64/tcg-target.h
@@ -95,6 +95,8 @@ typedef enum {
 #define TCG_TARGET_HAS_sub2_i32         0
 #define TCG_TARGET_HAS_mulu2_i32        0
 #define TCG_TARGET_HAS_muls2_i32        0
+#define TCG_TARGET_HAS_muluh_i32        0
+#define TCG_TARGET_HAS_mulsh_i32        0
 
 #define TCG_TARGET_HAS_div_i64          1
 #define TCG_TARGET_HAS_rem_i64          0
@@ -116,8 +118,10 @@ typedef enum {
 #define TCG_TARGET_HAS_movcond_i64      1
 #define TCG_TARGET_HAS_add2_i64         1
 #define TCG_TARGET_HAS_sub2_i64         1
-#define TCG_TARGET_HAS_mulu2_i64        1
-#define TCG_TARGET_HAS_muls2_i64        1
+#define TCG_TARGET_HAS_mulu2_i64        0
+#define TCG_TARGET_HAS_muls2_i64        0
+#define TCG_TARGET_HAS_muluh_i64        1
+#define TCG_TARGET_HAS_mulsh_i64        1
 
 #define TCG_AREG0 TCG_REG_R27
 
diff --git a/tcg/s390/tcg-target.c b/tcg/s390/tcg-target.c
index f229f1c346..1b44aeee96 100644
--- a/tcg/s390/tcg-target.c
+++ b/tcg/s390/tcg-target.c
@@ -315,9 +315,6 @@ static const uint8_t tcg_cond_to_ltr_cond[] = {
 };
 
 #ifdef CONFIG_SOFTMMU
-
-#include "exec/softmmu_defs.h"
-
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
@@ -351,10 +348,10 @@ static uint8_t *tb_ret_addr;
 static uint64_t facilities;
 
 static void patch_reloc(uint8_t *code_ptr, int type,
-                        tcg_target_long value, tcg_target_long addend)
+                        intptr_t value, intptr_t addend)
 {
-    tcg_target_long code_ptr_tl = (tcg_target_long)code_ptr;
-    tcg_target_long pcrel2;
+    intptr_t code_ptr_tl = (intptr_t)code_ptr;
+    intptr_t pcrel2;
 
     /* ??? Not the usual definition of "addend".  */
     pcrel2 = (value - (code_ptr_tl + addend)) >> 1;
@@ -771,7 +768,7 @@ static void tcg_out_mem(TCGContext *s, S390Opcode opc_rx, S390Opcode opc_rxy,
 
 /* load data without address translation or endianness conversion */
 static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg data,
-                              TCGReg base, tcg_target_long ofs)
+                              TCGReg base, intptr_t ofs)
 {
     if (type == TCG_TYPE_I32) {
         tcg_out_mem(s, RX_L, RXY_LY, data, base, TCG_REG_NONE, ofs);
@@ -781,7 +778,7 @@ static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg data,
 }
 
 static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg data,
-                              TCGReg base, tcg_target_long ofs)
+                              TCGReg base, intptr_t ofs)
 {
     if (type == TCG_TYPE_I32) {
         tcg_out_mem(s, RX_ST, RXY_STY, data, base, TCG_REG_NONE, ofs);
diff --git a/tcg/s390/tcg-target.h b/tcg/s390/tcg-target.h
index 42ca36c0e9..6142fb26a2 100644
--- a/tcg/s390/tcg-target.h
+++ b/tcg/s390/tcg-target.h
@@ -69,6 +69,8 @@ typedef enum TCGReg {
 #define TCG_TARGET_HAS_sub2_i32         1
 #define TCG_TARGET_HAS_mulu2_i32        0
 #define TCG_TARGET_HAS_muls2_i32        0
+#define TCG_TARGET_HAS_muluh_i32        0
+#define TCG_TARGET_HAS_mulsh_i32        0
 
 #define TCG_TARGET_HAS_div2_i64         1
 #define TCG_TARGET_HAS_rot_i64          1
@@ -94,6 +96,8 @@ typedef enum TCGReg {
 #define TCG_TARGET_HAS_sub2_i64         1
 #define TCG_TARGET_HAS_mulu2_i64        1
 #define TCG_TARGET_HAS_muls2_i64        0
+#define TCG_TARGET_HAS_muluh_i64        0
+#define TCG_TARGET_HAS_mulsh_i64        0
 
 extern bool tcg_target_deposit_valid(int ofs, int len);
 #define TCG_TARGET_deposit_i32_valid  tcg_target_deposit_valid
@@ -110,8 +114,7 @@ enum {
     TCG_AREG0 = TCG_REG_R10,
 };
 
-static inline void flush_icache_range(tcg_target_ulong start,
-                                      tcg_target_ulong stop)
+static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
 {
 }
 
diff --git a/tcg/sparc/tcg-target.c b/tcg/sparc/tcg-target.c
index 5bfd29c3b4..9574954ac4 100644
--- a/tcg/sparc/tcg-target.c
+++ b/tcg/sparc/tcg-target.c
@@ -252,7 +252,7 @@ static inline int check_fit_i32(uint32_t val, unsigned int bits)
 }
 
 static void patch_reloc(uint8_t *code_ptr, int type,
-                        tcg_target_long value, tcg_target_long addend)
+                        intptr_t value, intptr_t addend)
 {
     uint32_t insn;
     value += addend;
@@ -264,7 +264,7 @@ static void patch_reloc(uint8_t *code_ptr, int type,
         *(uint32_t *)code_ptr = value;
         break;
     case R_SPARC_WDISP16:
-        value -= (long)code_ptr;
+        value -= (intptr_t)code_ptr;
         if (!check_fit_tl(value >> 2, 16)) {
             tcg_abort();
         }
@@ -274,7 +274,7 @@ static void patch_reloc(uint8_t *code_ptr, int type,
         *(uint32_t *)code_ptr = insn;
         break;
     case R_SPARC_WDISP19:
-        value -= (long)code_ptr;
+        value -= (intptr_t)code_ptr;
         if (!check_fit_tl(value >> 2, 19)) {
             tcg_abort();
         }
@@ -436,13 +436,13 @@ static inline void tcg_out_ldst(TCGContext *s, int ret, int addr,
 }
 
 static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
-                              TCGReg arg1, tcg_target_long arg2)
+                              TCGReg arg1, intptr_t arg2)
 {
     tcg_out_ldst(s, ret, arg1, arg2, (type == TCG_TYPE_I32 ? LDUW : LDX));
 }
 
 static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
-                              TCGReg arg1, tcg_target_long arg2)
+                              TCGReg arg1, intptr_t arg2)
 {
     tcg_out_ldst(s, arg, arg1, arg2, (type == TCG_TYPE_I32 ? STW : STX));
 }
@@ -831,8 +831,6 @@ static void tcg_target_qemu_prologue(TCGContext *s)
 
 #if defined(CONFIG_SOFTMMU)
 
-#include "exec/softmmu_defs.h"
-
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
diff --git a/tcg/sparc/tcg-target.h b/tcg/sparc/tcg-target.h
index dab52d7176..2edf858733 100644
--- a/tcg/sparc/tcg-target.h
+++ b/tcg/sparc/tcg-target.h
@@ -24,6 +24,14 @@
 #ifndef TCG_TARGET_SPARC 
 #define TCG_TARGET_SPARC 1
 
+#if UINTPTR_MAX == UINT32_MAX
+# define TCG_TARGET_REG_BITS 32
+#elif UINTPTR_MAX == UINT64_MAX
+# define TCG_TARGET_REG_BITS 64
+#else
+# error Unknown pointer size for tcg target
+#endif
+
 #define TCG_TARGET_WORDS_BIGENDIAN
 
 #define TCG_TARGET_NB_REGS 32
@@ -107,6 +115,8 @@ typedef enum {
 #define TCG_TARGET_HAS_sub2_i32         1
 #define TCG_TARGET_HAS_mulu2_i32        1
 #define TCG_TARGET_HAS_muls2_i32        0
+#define TCG_TARGET_HAS_muluh_i32        0
+#define TCG_TARGET_HAS_mulsh_i32        0
 
 #if TCG_TARGET_REG_BITS == 64
 #define TCG_TARGET_HAS_div_i64          1
@@ -134,20 +144,18 @@ typedef enum {
 #define TCG_TARGET_HAS_sub2_i64         0
 #define TCG_TARGET_HAS_mulu2_i64        0
 #define TCG_TARGET_HAS_muls2_i64        0
+#define TCG_TARGET_HAS_muluh_i64        0
+#define TCG_TARGET_HAS_mulsh_i64        0
 #endif
 
 #define TCG_AREG0 TCG_REG_I0
 
-static inline void flush_icache_range(tcg_target_ulong start,
-                                      tcg_target_ulong stop)
+static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
 {
-    unsigned long p;
-
-    p = start & ~(8UL - 1UL);
-    stop = (stop + (8UL - 1UL)) & ~(8UL - 1UL);
-
-    for (; p < stop; p += 8)
+    uintptr_t p;
+    for (p = start & -8; p < (stop + 7) & -8; p += 8) {
         __asm__ __volatile__("flush\t%0" : : "r" (p));
+    }
 }
 
 #endif
diff --git a/tcg/tcg-op.h b/tcg/tcg-op.h
index 364964d8d4..bb30a7cf39 100644
--- a/tcg/tcg-op.h
+++ b/tcg/tcg-op.h
@@ -1039,10 +1039,18 @@ static inline void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
     t0 = tcg_temp_new_i64();
     t1 = tcg_temp_new_i32();
 
-    tcg_gen_op4_i32(INDEX_op_mulu2_i32, TCGV_LOW(t0), TCGV_HIGH(t0),
-                    TCGV_LOW(arg1), TCGV_LOW(arg2));
-    /* Allow the optimizer room to replace mulu2 with two moves.  */
-    tcg_gen_op0(INDEX_op_nop);
+    if (TCG_TARGET_HAS_mulu2_i32) {
+        tcg_gen_op4_i32(INDEX_op_mulu2_i32, TCGV_LOW(t0), TCGV_HIGH(t0),
+                        TCGV_LOW(arg1), TCGV_LOW(arg2));
+        /* Allow the optimizer room to replace mulu2 with two moves.  */
+        tcg_gen_op0(INDEX_op_nop);
+    } else {
+        tcg_debug_assert(TCG_TARGET_HAS_muluh_i32);
+        tcg_gen_op3_i32(INDEX_op_mul_i32, TCGV_LOW(t0),
+                        TCGV_LOW(arg1), TCGV_LOW(arg2));
+        tcg_gen_op3_i32(INDEX_op_muluh_i32, TCGV_HIGH(t0),
+                        TCGV_LOW(arg1), TCGV_LOW(arg2));
+    }
 
     tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
     tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
@@ -2401,6 +2409,12 @@ static inline void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh,
         tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2);
         /* Allow the optimizer room to replace mulu2 with two moves.  */
         tcg_gen_op0(INDEX_op_nop);
+    } else if (TCG_TARGET_HAS_muluh_i32) {
+        TCGv_i32 t = tcg_temp_new_i32();
+        tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
+        tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2);
+        tcg_gen_mov_i32(rl, t);
+        tcg_temp_free_i32(t);
     } else {
         TCGv_i64 t0 = tcg_temp_new_i64();
         TCGv_i64 t1 = tcg_temp_new_i64();
@@ -2420,6 +2434,12 @@ static inline void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh,
         tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2);
         /* Allow the optimizer room to replace muls2 with two moves.  */
         tcg_gen_op0(INDEX_op_nop);
+    } else if (TCG_TARGET_HAS_mulsh_i32) {
+        TCGv_i32 t = tcg_temp_new_i32();
+        tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
+        tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2);
+        tcg_gen_mov_i32(rl, t);
+        tcg_temp_free_i32(t);
     } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_mulu2_i32) {
         TCGv_i32 t0 = tcg_temp_new_i32();
         TCGv_i32 t1 = tcg_temp_new_i32();
@@ -2499,6 +2519,12 @@ static inline void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh,
         tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2);
         /* Allow the optimizer room to replace mulu2 with two moves.  */
         tcg_gen_op0(INDEX_op_nop);
+    } else if (TCG_TARGET_HAS_muluh_i64) {
+        TCGv_i64 t = tcg_temp_new_i64();
+        tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
+        tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2);
+        tcg_gen_mov_i64(rl, t);
+        tcg_temp_free_i64(t);
     } else if (TCG_TARGET_HAS_mulu2_i64) {
         TCGv_i64 t0 = tcg_temp_new_i64();
         TCGv_i64 t1 = tcg_temp_new_i64();
@@ -2540,6 +2566,12 @@ static inline void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh,
         tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2);
         /* Allow the optimizer room to replace muls2 with two moves.  */
         tcg_gen_op0(INDEX_op_nop);
+    } else if (TCG_TARGET_HAS_mulsh_i64) {
+        TCGv_i64 t = tcg_temp_new_i64();
+        tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
+        tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2);
+        tcg_gen_mov_i64(rl, t);
+        tcg_temp_free_i64(t);
     } else {
         TCGv_i64 t0 = tcg_temp_new_i64();
         int sizemask = 0;
@@ -2599,7 +2631,7 @@ static inline void tcg_gen_debug_insn_start(uint64_t pc)
 #endif
 }
 
-static inline void tcg_gen_exit_tb(tcg_target_long val)
+static inline void tcg_gen_exit_tb(uintptr_t val)
 {
     tcg_gen_op1i(INDEX_op_exit_tb, val);
 }
diff --git a/tcg/tcg-opc.h b/tcg/tcg-opc.h
index a8af5b96a4..a75c29d518 100644
--- a/tcg/tcg-opc.h
+++ b/tcg/tcg-opc.h
@@ -91,6 +91,8 @@ DEF(add2_i32, 2, 4, 0, IMPL(TCG_TARGET_HAS_add2_i32))
 DEF(sub2_i32, 2, 4, 0, IMPL(TCG_TARGET_HAS_sub2_i32))
 DEF(mulu2_i32, 2, 2, 0, IMPL(TCG_TARGET_HAS_mulu2_i32))
 DEF(muls2_i32, 2, 2, 0, IMPL(TCG_TARGET_HAS_muls2_i32))
+DEF(muluh_i32, 1, 2, 0, IMPL(TCG_TARGET_HAS_muluh_i32))
+DEF(mulsh_i32, 1, 2, 0, IMPL(TCG_TARGET_HAS_mulsh_i32))
 DEF(brcond2_i32, 0, 4, 2, TCG_OPF_BB_END | IMPL(TCG_TARGET_REG_BITS == 32))
 DEF(setcond2_i32, 1, 4, 1, IMPL(TCG_TARGET_REG_BITS == 32))
 
@@ -167,6 +169,8 @@ DEF(add2_i64, 2, 4, 0, IMPL64 | IMPL(TCG_TARGET_HAS_add2_i64))
 DEF(sub2_i64, 2, 4, 0, IMPL64 | IMPL(TCG_TARGET_HAS_sub2_i64))
 DEF(mulu2_i64, 2, 2, 0, IMPL64 | IMPL(TCG_TARGET_HAS_mulu2_i64))
 DEF(muls2_i64, 2, 2, 0, IMPL64 | IMPL(TCG_TARGET_HAS_muls2_i64))
+DEF(muluh_i64, 1, 2, 0, IMPL(TCG_TARGET_HAS_muluh_i64))
+DEF(mulsh_i64, 1, 2, 0, IMPL(TCG_TARGET_HAS_mulsh_i64))
 
 /* QEMU specific */
 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
diff --git a/tcg/tcg.c b/tcg/tcg.c
index 19bd5a39bf..fd7fb6b85e 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -49,10 +49,10 @@
 
 #include "tcg-op.h"
 
-#if TCG_TARGET_REG_BITS == 64
-# define ELF_CLASS  ELFCLASS64
-#else
+#if UINTPTR_MAX == UINT32_MAX
 # define ELF_CLASS  ELFCLASS32
+#else
+# define ELF_CLASS  ELFCLASS64
 #endif
 #ifdef HOST_WORDS_BIGENDIAN
 # define ELF_DATA   ELFDATA2MSB
@@ -66,7 +66,7 @@
 static void tcg_target_init(TCGContext *s);
 static void tcg_target_qemu_prologue(TCGContext *s);
 static void patch_reloc(uint8_t *code_ptr, int type, 
-                        tcg_target_long value, tcg_target_long addend);
+                        intptr_t value, intptr_t addend);
 
 /* The CIE and FDE header definitions will be common to all hosts.  */
 typedef struct {
@@ -82,8 +82,8 @@ typedef struct {
 typedef struct QEMU_PACKED {
     uint32_t len __attribute__((aligned((sizeof(void *)))));
     uint32_t cie_offset;
-    tcg_target_long func_start;
-    tcg_target_long func_len;
+    uintptr_t func_start;
+    uintptr_t func_len;
 } DebugFrameFDEHeader;
 
 static void tcg_register_jit_int(void *buf, size_t size,
@@ -93,14 +93,14 @@ static void tcg_register_jit_int(void *buf, size_t size,
 /* Forward declarations for functions declared and used in tcg-target.c. */
 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str);
 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
-                       tcg_target_long arg2);
+                       intptr_t arg2);
 static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
 static void tcg_out_movi(TCGContext *s, TCGType type,
                          TCGReg ret, tcg_target_long arg);
 static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
                        const int *const_args);
 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
-                       tcg_target_long arg2);
+                       intptr_t arg2);
 static int tcg_target_const_match(tcg_target_long val,
                                   const TCGArgConstraint *arg_ct);
 
@@ -143,7 +143,7 @@ static inline void tcg_out64(TCGContext *s, uint64_t v)
 /* label relocation processing */
 
 static void tcg_out_reloc(TCGContext *s, uint8_t *code_ptr, int type,
-                          int label_index, long addend)
+                          int label_index, intptr_t addend)
 {
     TCGLabel *l;
     TCGRelocation *r;
@@ -169,11 +169,12 @@ static void tcg_out_label(TCGContext *s, int label_index, void *ptr)
 {
     TCGLabel *l;
     TCGRelocation *r;
-    tcg_target_long value = (tcg_target_long)ptr;
+    intptr_t value = (intptr_t)ptr;
 
     l = &s->labels[label_index];
-    if (l->has_value)
+    if (l->has_value) {
         tcg_abort();
+    }
     r = l->u.first_reloc;
     while (r != NULL) {
         patch_reloc(r->ptr, r->type, value, r->addend);
@@ -293,8 +294,7 @@ void tcg_prologue_init(TCGContext *s)
     s->code_buf = s->code_gen_prologue;
     s->code_ptr = s->code_buf;
     tcg_target_qemu_prologue(s);
-    flush_icache_range((tcg_target_ulong)s->code_buf,
-                       (tcg_target_ulong)s->code_ptr);
+    flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr);
 
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
@@ -307,8 +307,7 @@ void tcg_prologue_init(TCGContext *s)
 #endif
 }
 
-void tcg_set_frame(TCGContext *s, int reg,
-                   tcg_target_long start, tcg_target_long size)
+void tcg_set_frame(TCGContext *s, int reg, intptr_t start, intptr_t size)
 {
     s->frame_start = start;
     s->frame_end = start + size;
@@ -391,7 +390,7 @@ TCGv_i64 tcg_global_reg_new_i64(int reg, const char *name)
 }
 
 static inline int tcg_global_mem_new_internal(TCGType type, int reg,
-                                              tcg_target_long offset,
+                                              intptr_t offset,
                                               const char *name)
 {
     TCGContext *s = &tcg_ctx;
@@ -451,21 +450,15 @@ static inline int tcg_global_mem_new_internal(TCGType type, int reg,
     return idx;
 }
 
-TCGv_i32 tcg_global_mem_new_i32(int reg, tcg_target_long offset,
-                                const char *name)
+TCGv_i32 tcg_global_mem_new_i32(int reg, intptr_t offset, const char *name)
 {
-    int idx;
-
-    idx = tcg_global_mem_new_internal(TCG_TYPE_I32, reg, offset, name);
+    int idx = tcg_global_mem_new_internal(TCG_TYPE_I32, reg, offset, name);
     return MAKE_TCGV_I32(idx);
 }
 
-TCGv_i64 tcg_global_mem_new_i64(int reg, tcg_target_long offset,
-                                const char *name)
+TCGv_i64 tcg_global_mem_new_i64(int reg, intptr_t offset, const char *name)
 {
-    int idx;
-
-    idx = tcg_global_mem_new_internal(TCG_TYPE_I64, reg, offset, name);
+    int idx = tcg_global_mem_new_internal(TCG_TYPE_I64, reg, offset, name);
     return MAKE_TCGV_I64(idx);
 }
 
@@ -641,7 +634,7 @@ void tcg_register_helper(void *func, const char *name)
         s->helpers = realloc(s->helpers, n * sizeof(TCGHelperInfo));
         s->allocated_helpers = n;
     }
-    s->helpers[s->nb_helpers].func = (tcg_target_ulong)func;
+    s->helpers[s->nb_helpers].func = (uintptr_t)func;
     s->helpers[s->nb_helpers].name = name;
     s->nb_helpers++;
 }
@@ -871,11 +864,11 @@ static int helper_cmp(const void *p1, const void *p2)
 }
 
 /* find helper definition (Note: A hash table would be better) */
-static TCGHelperInfo *tcg_find_helper(TCGContext *s, tcg_target_ulong val)
+static TCGHelperInfo *tcg_find_helper(TCGContext *s, uintptr_t val)
 {
     int m, m_min, m_max;
     TCGHelperInfo *th;
-    tcg_target_ulong v;
+    uintptr_t v;
 
     if (unlikely(!s->helpers_sorted)) {
         qsort(s->helpers, s->nb_helpers, sizeof(TCGHelperInfo), 
@@ -1252,12 +1245,13 @@ static inline void tcg_la_bb_end(TCGContext *s, uint8_t *dead_temps,
 static void tcg_liveness_analysis(TCGContext *s)
 {
     int i, op_index, nb_args, nb_iargs, nb_oargs, arg, nb_ops;
-    TCGOpcode op, op_new;
+    TCGOpcode op, op_new, op_new2;
     TCGArg *args;
     const TCGOpDef *def;
     uint8_t *dead_temps, *mem_temps;
     uint16_t dead_args;
     uint8_t sync_args;
+    bool have_op_new2;
     
     s->gen_opc_ptr++; /* skip end */
 
@@ -1394,29 +1388,52 @@ static void tcg_liveness_analysis(TCGContext *s)
             goto do_not_remove;
 
         case INDEX_op_mulu2_i32:
+            op_new = INDEX_op_mul_i32;
+            op_new2 = INDEX_op_muluh_i32;
+            have_op_new2 = TCG_TARGET_HAS_muluh_i32;
+            goto do_mul2;
         case INDEX_op_muls2_i32:
             op_new = INDEX_op_mul_i32;
+            op_new2 = INDEX_op_mulsh_i32;
+            have_op_new2 = TCG_TARGET_HAS_mulsh_i32;
             goto do_mul2;
         case INDEX_op_mulu2_i64:
+            op_new = INDEX_op_mul_i64;
+            op_new2 = INDEX_op_muluh_i64;
+            have_op_new2 = TCG_TARGET_HAS_muluh_i64;
+            goto do_mul2;
         case INDEX_op_muls2_i64:
             op_new = INDEX_op_mul_i64;
+            op_new2 = INDEX_op_mulsh_i64;
+            have_op_new2 = TCG_TARGET_HAS_mulsh_i64;
+            goto do_mul2;
         do_mul2:
             args -= 4;
             nb_iargs = 2;
             nb_oargs = 2;
-            /* Likewise, test for the high part of the operation dead.  */
             if (dead_temps[args[1]] && !mem_temps[args[1]]) {
                 if (dead_temps[args[0]] && !mem_temps[args[0]]) {
+                    /* Both parts of the operation are dead.  */
                     goto do_remove;
                 }
+                /* The high part of the operation is dead; generate the low. */
                 s->gen_opc_buf[op_index] = op = op_new;
                 args[1] = args[2];
                 args[2] = args[3];
-                assert(s->gen_opc_buf[op_index + 1] == INDEX_op_nop);
-                tcg_set_nop(s, s->gen_opc_buf + op_index + 1, args + 3, 1);
-                /* Fall through and mark the single-word operation live.  */
-                nb_oargs = 1;
+            } else if (have_op_new2 && dead_temps[args[0]]
+                       && !mem_temps[args[0]]) {
+                /* The low part of the operation is dead; generate the high.  */
+                s->gen_opc_buf[op_index] = op = op_new2;
+                args[0] = args[1];
+                args[1] = args[2];
+                args[2] = args[3];
+            } else {
+                goto do_not_remove;
             }
+            assert(s->gen_opc_buf[op_index + 1] == INDEX_op_nop);
+            tcg_set_nop(s, s->gen_opc_buf + op_index + 1, args + 3, 1);
+            /* Mark the single-word operation live.  */
+            nb_oargs = 1;
             goto do_not_remove;
 
         default:
@@ -1590,7 +1607,7 @@ static void temp_allocate_frame(TCGContext *s, int temp)
     ts->mem_offset = s->current_frame_offset;
     ts->mem_reg = s->frame_reg;
     ts->mem_allocated = 1;
-    s->current_frame_offset += (tcg_target_long)sizeof(tcg_target_long);
+    s->current_frame_offset += sizeof(tcg_target_long);
 }
 
 /* sync register 'reg' by saving it to the corresponding temporary */
@@ -2053,7 +2070,9 @@ static int tcg_reg_alloc_call(TCGContext *s, const TCGOpDef *def,
     int nb_iargs, nb_oargs, flags, nb_regs, i, reg, nb_params;
     TCGArg arg, func_arg;
     TCGTemp *ts;
-    tcg_target_long stack_offset, call_stack_size, func_addr;
+    intptr_t stack_offset;
+    size_t call_stack_size;
+    uintptr_t func_addr;
     int const_func_arg, allocate_args;
     TCGRegSet allocated_regs;
     const TCGArgConstraint *arg_ct;
@@ -2391,8 +2410,7 @@ int tcg_gen_code(TCGContext *s, uint8_t *gen_code_buf)
     tcg_gen_code_common(s, gen_code_buf, -1);
 
     /* flush instruction cache */
-    flush_icache_range((tcg_target_ulong)gen_code_buf,
-                       (tcg_target_ulong)s->code_ptr);
+    flush_icache_range((uintptr_t)gen_code_buf, (uintptr_t)s->code_ptr);
 
     return s->code_ptr -  gen_code_buf;
 }
diff --git a/tcg/tcg.h b/tcg/tcg.h
index f3f9889694..902c751d26 100644
--- a/tcg/tcg.h
+++ b/tcg/tcg.h
@@ -21,15 +21,23 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+
+#ifndef TCG_H
+#define TCG_H
+
 #include "qemu-common.h"
 
-/* Target word size (must be identical to pointer size). */
-#if UINTPTR_MAX == UINT32_MAX
-# define TCG_TARGET_REG_BITS 32
-#elif UINTPTR_MAX == UINT64_MAX
-# define TCG_TARGET_REG_BITS 64
-#else
-# error Unknown pointer size for tcg target
+#include "tcg-target.h"
+
+/* Default target word size to pointer size.  */
+#ifndef TCG_TARGET_REG_BITS
+# if UINTPTR_MAX == UINT32_MAX
+#  define TCG_TARGET_REG_BITS 32
+# elif UINTPTR_MAX == UINT64_MAX
+#  define TCG_TARGET_REG_BITS 64
+# else
+#  error Unknown pointer size for tcg target
+# endif
 #endif
 
 #if TCG_TARGET_REG_BITS == 32
@@ -46,7 +54,6 @@ typedef uint64_t tcg_target_ulong;
 #error unsupported
 #endif
 
-#include "tcg-target.h"
 #include "tcg-runtime.h"
 
 #if TCG_TARGET_NB_REGS <= 32
@@ -85,6 +92,8 @@ typedef uint64_t TCGRegSet;
 #define TCG_TARGET_HAS_sub2_i64         0
 #define TCG_TARGET_HAS_mulu2_i64        0
 #define TCG_TARGET_HAS_muls2_i64        0
+#define TCG_TARGET_HAS_muluh_i64        0
+#define TCG_TARGET_HAS_mulsh_i64        0
 /* Turn some undef macros into true macros.  */
 #define TCG_TARGET_HAS_add2_i32         1
 #define TCG_TARGET_HAS_sub2_i32         1
@@ -134,13 +143,13 @@ typedef struct TCGRelocation {
     struct TCGRelocation *next;
     int type;
     uint8_t *ptr;
-    tcg_target_long addend;
+    intptr_t addend;
 } TCGRelocation; 
 
 typedef struct TCGLabel {
     int has_value;
     union {
-        tcg_target_ulong value;
+        uintptr_t value;
         TCGRelocation *first_reloc;
     } u;
 } TCGLabel;
@@ -173,9 +182,12 @@ typedef enum TCGType {
     TCG_TYPE_REG = TCG_TYPE_I64,
 #endif
 
-    /* An alias for the size of the native pointer.  We don't currently
-       support any hosts with 64-bit registers and 32-bit pointers.  */
-    TCG_TYPE_PTR = TCG_TYPE_REG,
+    /* An alias for the size of the native pointer.  */
+#if UINTPTR_MAX == UINT32_MAX
+    TCG_TYPE_PTR = TCG_TYPE_I32,
+#else
+    TCG_TYPE_PTR = TCG_TYPE_I64,
+#endif
 
     /* An alias for the size of the target "long", aka register.  */
 #if TARGET_LONG_BITS == 64
@@ -380,7 +392,7 @@ typedef struct TCGTemp {
     int reg;
     tcg_target_long val;
     int mem_reg;
-    tcg_target_long mem_offset;
+    intptr_t mem_offset;
     unsigned int fixed_reg:1;
     unsigned int mem_coherent:1;
     unsigned int mem_allocated:1;
@@ -394,7 +406,7 @@ typedef struct TCGTemp {
 } TCGTemp;
 
 typedef struct TCGHelperInfo {
-    tcg_target_ulong func;
+    uintptr_t func;
     const char *name;
 } TCGHelperInfo;
 
@@ -427,9 +439,9 @@ struct TCGContext {
        into account fixed registers */
     int reg_to_temp[TCG_TARGET_NB_REGS];
     TCGRegSet reserved_regs;
-    tcg_target_long current_frame_offset;
-    tcg_target_long frame_start;
-    tcg_target_long frame_end;
+    intptr_t current_frame_offset;
+    intptr_t frame_start;
+    intptr_t frame_end;
     int frame_reg;
 
     uint8_t *code_ptr;
@@ -522,12 +534,10 @@ void tcg_func_start(TCGContext *s);
 int tcg_gen_code(TCGContext *s, uint8_t *gen_code_buf);
 int tcg_gen_code_search_pc(TCGContext *s, uint8_t *gen_code_buf, long offset);
 
-void tcg_set_frame(TCGContext *s, int reg,
-                   tcg_target_long start, tcg_target_long size);
+void tcg_set_frame(TCGContext *s, int reg, intptr_t start, intptr_t size);
 
 TCGv_i32 tcg_global_reg_new_i32(int reg, const char *name);
-TCGv_i32 tcg_global_mem_new_i32(int reg, tcg_target_long offset,
-                                const char *name);
+TCGv_i32 tcg_global_mem_new_i32(int reg, intptr_t offset, const char *name);
 TCGv_i32 tcg_temp_new_internal_i32(int temp_local);
 static inline TCGv_i32 tcg_temp_new_i32(void)
 {
@@ -541,8 +551,7 @@ void tcg_temp_free_i32(TCGv_i32 arg);
 char *tcg_get_arg_str_i32(TCGContext *s, char *buf, int buf_size, TCGv_i32 arg);
 
 TCGv_i64 tcg_global_reg_new_i64(int reg, const char *name);
-TCGv_i64 tcg_global_mem_new_i64(int reg, tcg_target_long offset,
-                                const char *name);
+TCGv_i64 tcg_global_mem_new_i64(int reg, intptr_t offset, const char *name);
 TCGv_i64 tcg_temp_new_internal_i64(int temp_local);
 static inline TCGv_i64 tcg_temp_new_i64(void)
 {
@@ -637,11 +646,11 @@ do {\
 
 void tcg_add_target_add_op_defs(const TCGTargetOpDef *tdefs);
 
-#if TCG_TARGET_REG_BITS == 32
+#if UINTPTR_MAX == UINT32_MAX
 #define TCGV_NAT_TO_PTR(n) MAKE_TCGV_PTR(GET_TCGV_I32(n))
 #define TCGV_PTR_TO_NAT(n) MAKE_TCGV_I32(GET_TCGV_PTR(n))
 
-#define tcg_const_ptr(V) TCGV_NAT_TO_PTR(tcg_const_i32((tcg_target_long)(V)))
+#define tcg_const_ptr(V) TCGV_NAT_TO_PTR(tcg_const_i32((intptr_t)(V)))
 #define tcg_global_reg_new_ptr(R, N) \
     TCGV_NAT_TO_PTR(tcg_global_reg_new_i32((R), (N)))
 #define tcg_global_mem_new_ptr(R, O, N) \
@@ -652,7 +661,7 @@ void tcg_add_target_add_op_defs(const TCGTargetOpDef *tdefs);
 #define TCGV_NAT_TO_PTR(n) MAKE_TCGV_PTR(GET_TCGV_I64(n))
 #define TCGV_PTR_TO_NAT(n) MAKE_TCGV_I64(GET_TCGV_PTR(n))
 
-#define tcg_const_ptr(V) TCGV_NAT_TO_PTR(tcg_const_i64((tcg_target_long)(V)))
+#define tcg_const_ptr(V) TCGV_NAT_TO_PTR(tcg_const_i64((intptr_t)(V)))
 #define tcg_global_reg_new_ptr(R, N) \
     TCGV_NAT_TO_PTR(tcg_global_reg_new_i64((R), (N)))
 #define tcg_global_mem_new_ptr(R, O, N) \
@@ -731,8 +740,7 @@ TCGv_i64 tcg_const_local_i64(int64_t val);
 
 #if !defined(tcg_qemu_tb_exec)
 # define tcg_qemu_tb_exec(env, tb_ptr) \
-    ((tcg_target_ulong (*)(void *, void *))tcg_ctx.code_gen_prologue)(env, \
-                                                                      tb_ptr)
+    ((uintptr_t (*)(void *, void *))tcg_ctx.code_gen_prologue)(env, tb_ptr)
 #endif
 
 void tcg_register_jit(void *buf, size_t buf_size);
@@ -741,3 +749,51 @@ void tcg_register_jit(void *buf, size_t buf_size);
 /* Generate TB finalization at the end of block */
 void tcg_out_tb_finalize(TCGContext *s);
 #endif
+
+/*
+ * Memory helpers that will be used by TCG generated code.
+ */
+#ifdef CONFIG_SOFTMMU
+/* Value zero-extended to tcg register size.  */
+tcg_target_ulong helper_ret_ldub_mmu(CPUArchState *env, target_ulong addr,
+                                     int mmu_idx, uintptr_t retaddr);
+tcg_target_ulong helper_ret_lduw_mmu(CPUArchState *env, target_ulong addr,
+                                     int mmu_idx, uintptr_t retaddr);
+tcg_target_ulong helper_ret_ldul_mmu(CPUArchState *env, target_ulong addr,
+                                     int mmu_idx, uintptr_t retaddr);
+uint64_t helper_ret_ldq_mmu(CPUArchState *env, target_ulong addr,
+                            int mmu_idx, uintptr_t retaddr);
+
+/* Value sign-extended to tcg register size.  */
+tcg_target_ulong helper_ret_ldsb_mmu(CPUArchState *env, target_ulong addr,
+                                     int mmu_idx, uintptr_t retaddr);
+tcg_target_ulong helper_ret_ldsw_mmu(CPUArchState *env, target_ulong addr,
+                                     int mmu_idx, uintptr_t retaddr);
+tcg_target_ulong helper_ret_ldsl_mmu(CPUArchState *env, target_ulong addr,
+                                     int mmu_idx, uintptr_t retaddr);
+
+void helper_ret_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val,
+                        int mmu_idx, uintptr_t retaddr);
+void helper_ret_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
+                        int mmu_idx, uintptr_t retaddr);
+void helper_ret_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
+                        int mmu_idx, uintptr_t retaddr);
+void helper_ret_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
+                        int mmu_idx, uintptr_t retaddr);
+
+uint8_t helper_ldb_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
+uint16_t helper_ldw_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
+uint32_t helper_ldl_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
+uint64_t helper_ldq_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
+
+void helper_stb_mmu(CPUArchState *env, target_ulong addr,
+                    uint8_t val, int mmu_idx);
+void helper_stw_mmu(CPUArchState *env, target_ulong addr,
+                    uint16_t val, int mmu_idx);
+void helper_stl_mmu(CPUArchState *env, target_ulong addr,
+                    uint32_t val, int mmu_idx);
+void helper_stq_mmu(CPUArchState *env, target_ulong addr,
+                    uint64_t val, int mmu_idx);
+#endif /* CONFIG_SOFTMMU */
+
+#endif /* TCG_H */
diff --git a/tcg/tci/tcg-target.c b/tcg/tci/tcg-target.c
index e118bc7179..233ab3bf35 100644
--- a/tcg/tci/tcg-target.c
+++ b/tcg/tci/tcg-target.c
@@ -370,7 +370,7 @@ static const char *const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
 #endif
 
 static void patch_reloc(uint8_t *code_ptr, int type,
-                        tcg_target_long value, tcg_target_long addend)
+                        intptr_t value, intptr_t addend)
 {
     /* tcg_out_reloc always uses the same type, addend. */
     assert(type == sizeof(tcg_target_long));
@@ -415,13 +415,6 @@ static void tcg_out_i(TCGContext *s, tcg_target_ulong v)
     s->code_ptr += sizeof(tcg_target_ulong);
 }
 
-/* Write 64 bit value. */
-static void tcg_out64(TCGContext *s, uint64_t v)
-{
-    *(uint64_t *)s->code_ptr = v;
-    s->code_ptr += sizeof(v);
-}
-
 /* Write opcode. */
 static void tcg_out_op_t(TCGContext *s, TCGOpcode op)
 {
@@ -488,7 +481,7 @@ static void tci_out_label(TCGContext *s, TCGArg arg)
 }
 
 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
-                       tcg_target_long arg2)
+                       intptr_t arg2)
 {
     uint8_t *old_code_ptr = s->code_ptr;
     if (type == TCG_TYPE_I32) {
@@ -842,7 +835,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
 }
 
 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
-                       tcg_target_long arg2)
+                       intptr_t arg2)
 {
     uint8_t *old_code_ptr = s->code_ptr;
     if (type == TCG_TYPE_I32) {
diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h
index d7fc14eb17..c2ecfbe047 100644
--- a/tcg/tci/tcg-target.h
+++ b/tcg/tci/tcg-target.h
@@ -44,6 +44,14 @@
 
 #define TCG_TARGET_INTERPRETER 1
 
+#if UINTPTR_MAX == UINT32_MAX
+# define TCG_TARGET_REG_BITS 32
+#elif UINTPTR_MAX == UINT64_MAX
+# define TCG_TARGET_REG_BITS 64
+#else
+# error Unknown pointer size for tci target
+#endif
+
 #ifdef CONFIG_DEBUG_TCG
 /* Enable debug output. */
 #define CONFIG_DEBUG_TCG_INTERPRETER
@@ -76,6 +84,8 @@
 #define TCG_TARGET_HAS_rot_i32          1
 #define TCG_TARGET_HAS_movcond_i32      0
 #define TCG_TARGET_HAS_muls2_i32        0
+#define TCG_TARGET_HAS_muluh_i32        0
+#define TCG_TARGET_HAS_mulsh_i32        0
 
 #if TCG_TARGET_REG_BITS == 64
 #define TCG_TARGET_HAS_bswap16_i64      1
@@ -100,13 +110,14 @@
 #define TCG_TARGET_HAS_rot_i64          1
 #define TCG_TARGET_HAS_movcond_i64      0
 #define TCG_TARGET_HAS_muls2_i64        0
-
 #define TCG_TARGET_HAS_add2_i32         0
 #define TCG_TARGET_HAS_sub2_i32         0
 #define TCG_TARGET_HAS_mulu2_i32        0
 #define TCG_TARGET_HAS_add2_i64         0
 #define TCG_TARGET_HAS_sub2_i64         0
 #define TCG_TARGET_HAS_mulu2_i64        0
+#define TCG_TARGET_HAS_muluh_i64        0
+#define TCG_TARGET_HAS_mulsh_i64        0
 #endif /* TCG_TARGET_REG_BITS == 64 */
 
 /* Number of registers available.
@@ -166,11 +177,10 @@ typedef enum {
 
 void tci_disas(uint8_t opc);
 
-tcg_target_ulong tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr);
+uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr);
 #define tcg_qemu_tb_exec tcg_qemu_tb_exec
 
-static inline void flush_icache_range(tcg_target_ulong start,
-                                      tcg_target_ulong stop)
+static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
 {
 }