Commit 3972ef6f authored by Richard Henderson's avatar Richard Henderson
Browse files

tcg: Push merged memop+mmu_idx parameter to softmmu routines



The extra information is not yet used but it is now available.
This requires minor changes through all of the tcg backends.

Reviewed-by: default avatarPeter Maydell <peter.maydell@linaro.org>
Signed-off-by: default avatarRichard Henderson <rth@twiddle.net>
parent 59227d5d
Loading
Loading
Loading
Loading
+24 −18
Original line number Diff line number Diff line
@@ -168,9 +168,10 @@ static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,
#ifdef SOFTMMU_CODE_ACCESS
static __attribute__((unused))
#endif
WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr, int mmu_idx,
                            uintptr_t retaddr)
WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,
                            TCGMemOpIdx oi, uintptr_t retaddr)
{
    unsigned mmu_idx = get_mmuidx(oi);
    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
    uintptr_t haddr;
@@ -226,8 +227,8 @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr, int mmu_idx,
        addr2 = addr1 + DATA_SIZE;
        /* Note the adjustment at the beginning of the function.
           Undo that for the recursion.  */
        res1 = helper_le_ld_name(env, addr1, mmu_idx, retaddr + GETPC_ADJ);
        res2 = helper_le_ld_name(env, addr2, mmu_idx, retaddr + GETPC_ADJ);
        res1 = helper_le_ld_name(env, addr1, oi, retaddr + GETPC_ADJ);
        res2 = helper_le_ld_name(env, addr2, oi, retaddr + GETPC_ADJ);
        shift = (addr & (DATA_SIZE - 1)) * 8;

        /* Little-endian combine.  */
@@ -256,9 +257,10 @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr, int mmu_idx,
#ifdef SOFTMMU_CODE_ACCESS
static __attribute__((unused))
#endif
WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr, int mmu_idx,
                            uintptr_t retaddr)
WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr,
                            TCGMemOpIdx oi, uintptr_t retaddr)
{
    unsigned mmu_idx = get_mmuidx(oi);
    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
    uintptr_t haddr;
@@ -314,8 +316,8 @@ WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr, int mmu_idx,
        addr2 = addr1 + DATA_SIZE;
        /* Note the adjustment at the beginning of the function.
           Undo that for the recursion.  */
        res1 = helper_be_ld_name(env, addr1, mmu_idx, retaddr + GETPC_ADJ);
        res2 = helper_be_ld_name(env, addr2, mmu_idx, retaddr + GETPC_ADJ);
        res1 = helper_be_ld_name(env, addr1, oi, retaddr + GETPC_ADJ);
        res2 = helper_be_ld_name(env, addr2, oi, retaddr + GETPC_ADJ);
        shift = (addr & (DATA_SIZE - 1)) * 8;

        /* Big-endian combine.  */
@@ -341,7 +343,8 @@ DATA_TYPE
glue(glue(helper_ld, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,
                                         int mmu_idx)
{
    return helper_te_ld_name (env, addr, mmu_idx, GETRA());
    TCGMemOpIdx oi = make_memop_idx(SHIFT, mmu_idx);
    return helper_te_ld_name (env, addr, oi, GETRA());
}

#ifndef SOFTMMU_CODE_ACCESS
@@ -350,16 +353,16 @@ glue(glue(helper_ld, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,
   avoid this for 64-bit data, or for 32-bit data on 32-bit host.  */
#if DATA_SIZE * 8 < TCG_TARGET_REG_BITS
WORD_TYPE helper_le_lds_name(CPUArchState *env, target_ulong addr,
                             int mmu_idx, uintptr_t retaddr)
                             TCGMemOpIdx oi, uintptr_t retaddr)
{
    return (SDATA_TYPE)helper_le_ld_name(env, addr, mmu_idx, retaddr);
    return (SDATA_TYPE)helper_le_ld_name(env, addr, oi, retaddr);
}

# if DATA_SIZE > 1
WORD_TYPE helper_be_lds_name(CPUArchState *env, target_ulong addr,
                             int mmu_idx, uintptr_t retaddr)
                             TCGMemOpIdx oi, uintptr_t retaddr)
{
    return (SDATA_TYPE)helper_be_ld_name(env, addr, mmu_idx, retaddr);
    return (SDATA_TYPE)helper_be_ld_name(env, addr, oi, retaddr);
}
# endif
#endif
@@ -386,8 +389,9 @@ static inline void glue(io_write, SUFFIX)(CPUArchState *env,
}

void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
                       int mmu_idx, uintptr_t retaddr)
                       TCGMemOpIdx oi, uintptr_t retaddr)
{
    unsigned mmu_idx = get_mmuidx(oi);
    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
    uintptr_t haddr;
@@ -444,7 +448,7 @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
            /* Note the adjustment at the beginning of the function.
               Undo that for the recursion.  */
            glue(helper_ret_stb, MMUSUFFIX)(env, addr + i, val8,
                                            mmu_idx, retaddr + GETPC_ADJ);
                                            oi, retaddr + GETPC_ADJ);
        }
        return;
    }
@@ -467,8 +471,9 @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,

#if DATA_SIZE > 1
void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
                       int mmu_idx, uintptr_t retaddr)
                       TCGMemOpIdx oi, uintptr_t retaddr)
{
    unsigned mmu_idx = get_mmuidx(oi);
    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
    uintptr_t haddr;
@@ -525,7 +530,7 @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
            /* Note the adjustment at the beginning of the function.
               Undo that for the recursion.  */
            glue(helper_ret_stb, MMUSUFFIX)(env, addr + i, val8,
                                            mmu_idx, retaddr + GETPC_ADJ);
                                            oi, retaddr + GETPC_ADJ);
        }
        return;
    }
@@ -547,7 +552,8 @@ void
glue(glue(helper_st, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,
                                         DATA_TYPE val, int mmu_idx)
{
    helper_te_st_name(env, addr, val, mmu_idx, GETRA());
    TCGMemOpIdx oi = make_memop_idx(SHIFT, mmu_idx);
    helper_te_st_name(env, addr, val, oi, GETRA());
}

#endif /* !defined(SOFTMMU_CODE_ACCESS) */
+6 −4
Original line number Diff line number Diff line
@@ -6013,13 +6013,15 @@ void HELPER(dc_zva)(CPUARMState *env, uint64_t vaddr_in)
        int maxidx = DIV_ROUND_UP(blocklen, TARGET_PAGE_SIZE);
        void *hostaddr[maxidx];
        int try, i;
        unsigned mmu_idx = cpu_mmu_index(env);
        TCGMemOpIdx oi = make_memop_idx(MO_UB, mmu_idx);

        for (try = 0; try < 2; try++) {

            for (i = 0; i < maxidx; i++) {
                hostaddr[i] = tlb_vaddr_to_host(env,
                                                vaddr + TARGET_PAGE_SIZE * i,
                                                1, cpu_mmu_index(env));
                                                1, mmu_idx);
                if (!hostaddr[i]) {
                    break;
                }
@@ -6040,12 +6042,12 @@ void HELPER(dc_zva)(CPUARMState *env, uint64_t vaddr_in)
             * this purpose use the actual register value passed to us
             * so that we get the fault address right.
             */
            helper_ret_stb_mmu(env, vaddr_in, 0, cpu_mmu_index(env), GETRA());
            helper_ret_stb_mmu(env, vaddr_in, 0, oi, GETRA());
            /* Now we can populate the other TLB entries, if any */
            for (i = 0; i < maxidx; i++) {
                uint64_t va = vaddr + TARGET_PAGE_SIZE * i;
                if (va != (vaddr_in & TARGET_PAGE_MASK)) {
                    helper_ret_stb_mmu(env, va, 0, cpu_mmu_index(env), GETRA());
                    helper_ret_stb_mmu(env, va, 0, oi, GETRA());
                }
            }
        }
@@ -6062,7 +6064,7 @@ void HELPER(dc_zva)(CPUARMState *env, uint64_t vaddr_in)
         *    bounce buffer was in use
         */
        for (i = 0; i < blocklen; i++) {
            helper_ret_stb_mmu(env, vaddr + i, 0, cpu_mmu_index(env), GETRA());
            helper_ret_stb_mmu(env, vaddr + i, 0, oi, GETRA());
        }
    }
#else
+18 −17
Original line number Diff line number Diff line
@@ -959,7 +959,7 @@ static inline void tcg_out_addsub2(TCGContext *s, int ext, TCGReg rl,

#ifdef CONFIG_SOFTMMU
/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
 *                                     int mmu_idx, uintptr_t ra)
 *                                     TCGMemOpIdx oi, uintptr_t ra)
 */
static void * const qemu_ld_helpers[16] = {
    [MO_UB]   = helper_ret_ldub_mmu,
@@ -972,7 +972,8 @@ static void * const qemu_ld_helpers[16] = {
};

/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,
 *                                     uintxx_t val, int mmu_idx, uintptr_t ra)
 *                                     uintxx_t val, TCGMemOpIdx oi,
 *                                     uintptr_t ra)
 */
static void * const qemu_st_helpers[16] = {
    [MO_UB]   = helper_ret_stb_mmu,
@@ -993,14 +994,15 @@ static inline void tcg_out_adr(TCGContext *s, TCGReg rd, void *target)

static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
{
    TCGMemOp opc = lb->opc;
    TCGMemOpIdx oi = lb->oi;
    TCGMemOp opc = get_memop(oi);
    TCGMemOp size = opc & MO_SIZE;

    reloc_pc19(lb->label_ptr[0], s->code_ptr);

    tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_X0, TCG_AREG0);
    tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0);
    tcg_out_mov(s, TARGET_LONG_BITS == 64, TCG_REG_X1, lb->addrlo_reg);
    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X2, lb->mem_index);
    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X2, oi);
    tcg_out_adr(s, TCG_REG_X3, lb->raddr);
    tcg_out_call(s, qemu_ld_helpers[opc & ~MO_SIGN]);
    if (opc & MO_SIGN) {
@@ -1014,33 +1016,32 @@ static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)

static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
{
    TCGMemOp opc = lb->opc;
    TCGMemOpIdx oi = lb->oi;
    TCGMemOp opc = get_memop(oi);
    TCGMemOp size = opc & MO_SIZE;

    reloc_pc19(lb->label_ptr[0], s->code_ptr);

    tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_X0, TCG_AREG0);
    tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0);
    tcg_out_mov(s, TARGET_LONG_BITS == 64, TCG_REG_X1, lb->addrlo_reg);
    tcg_out_mov(s, size == MO_64, TCG_REG_X2, lb->datalo_reg);
    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X3, lb->mem_index);
    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X3, oi);
    tcg_out_adr(s, TCG_REG_X4, lb->raddr);
    tcg_out_call(s, qemu_st_helpers[opc]);
    tcg_out_goto(s, lb->raddr);
}

static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOp opc,
static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
                                TCGType ext, TCGReg data_reg, TCGReg addr_reg,
                                int mem_index, tcg_insn_unit *raddr,
                                tcg_insn_unit *label_ptr)
                                tcg_insn_unit *raddr, tcg_insn_unit *label_ptr)
{
    TCGLabelQemuLdst *label = new_ldst_label(s);

    label->is_ld = is_ld;
    label->opc = opc;
    label->oi = oi;
    label->type = ext;
    label->datalo_reg = data_reg;
    label->addrlo_reg = addr_reg;
    label->mem_index = mem_index;
    label->raddr = raddr;
    label->label_ptr[0] = label_ptr;
}
@@ -1207,8 +1208,8 @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,

    tcg_out_tlb_read(s, addr_reg, s_bits, &label_ptr, mem_index, 1);
    tcg_out_qemu_ld_direct(s, memop, ext, data_reg, addr_reg, TCG_REG_X1);
    add_qemu_ldst_label(s, true, memop, ext, data_reg, addr_reg,
                        mem_index, s->code_ptr, label_ptr);
    add_qemu_ldst_label(s, true, oi, ext, data_reg, addr_reg,
                        s->code_ptr, label_ptr);
#else /* !CONFIG_SOFTMMU */
    tcg_out_qemu_ld_direct(s, memop, ext, data_reg, addr_reg,
                           GUEST_BASE ? TCG_REG_GUEST_BASE : TCG_REG_XZR);
@@ -1226,8 +1227,8 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,

    tcg_out_tlb_read(s, addr_reg, s_bits, &label_ptr, mem_index, 0);
    tcg_out_qemu_st_direct(s, memop, data_reg, addr_reg, TCG_REG_X1);
    add_qemu_ldst_label(s, false, memop, s_bits == MO_64, data_reg, addr_reg,
                        mem_index, s->code_ptr, label_ptr);
    add_qemu_ldst_label(s, false, oi, s_bits == MO_64, data_reg, addr_reg,
                        s->code_ptr, label_ptr);
#else /* !CONFIG_SOFTMMU */
    tcg_out_qemu_st_direct(s, memop, data_reg, addr_reg,
                           GUEST_BASE ? TCG_REG_GUEST_BASE : TCG_REG_XZR);
+14 −13
Original line number Diff line number Diff line
@@ -1221,20 +1221,19 @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addrlo, TCGReg addrhi,
/* Record the context of a call to the out of line helper code for the slow
   path for a load or store, so that we can later generate the correct
   helper code.  */
static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOp opc,
static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
                                TCGReg datalo, TCGReg datahi, TCGReg addrlo,
                                TCGReg addrhi, int mem_index,
                                tcg_insn_unit *raddr, tcg_insn_unit *label_ptr)
                                TCGReg addrhi, tcg_insn_unit *raddr,
                                tcg_insn_unit *label_ptr)
{
    TCGLabelQemuLdst *label = new_ldst_label(s);

    label->is_ld = is_ld;
    label->opc = opc;
    label->oi = oi;
    label->datalo_reg = datalo;
    label->datahi_reg = datahi;
    label->addrlo_reg = addrlo;
    label->addrhi_reg = addrhi;
    label->mem_index = mem_index;
    label->raddr = raddr;
    label->label_ptr[0] = label_ptr;
}
@@ -1242,7 +1241,8 @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOp opc,
static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
{
    TCGReg argreg, datalo, datahi;
    TCGMemOp opc = lb->opc;
    TCGMemOpIdx oi = lb->oi;
    TCGMemOp opc = get_memop(oi);
    void *func;

    reloc_pc24(lb->label_ptr[0], s->code_ptr);
@@ -1253,7 +1253,7 @@ static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
    } else {
        argreg = tcg_out_arg_reg32(s, argreg, lb->addrlo_reg);
    }
    argreg = tcg_out_arg_imm32(s, argreg, lb->mem_index);
    argreg = tcg_out_arg_imm32(s, argreg, oi);
    argreg = tcg_out_arg_reg32(s, argreg, TCG_REG_R14);

    /* For armv6 we can use the canonical unsigned helpers and minimize
@@ -1302,7 +1302,8 @@ static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
{
    TCGReg argreg, datalo, datahi;
    TCGMemOp opc = lb->opc;
    TCGMemOpIdx oi = lb->oi;
    TCGMemOp opc = get_memop(oi);

    reloc_pc24(lb->label_ptr[0], s->code_ptr);

@@ -1332,7 +1333,7 @@ static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
        break;
    }

    argreg = tcg_out_arg_imm32(s, argreg, lb->mem_index);
    argreg = tcg_out_arg_imm32(s, argreg, oi);
    argreg = tcg_out_arg_reg32(s, argreg, TCG_REG_R14);

    /* Tail-call to the helper, which will return to the fast path.  */
@@ -1489,8 +1490,8 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)

    tcg_out_qemu_ld_index(s, opc, datalo, datahi, addrlo, addend);

    add_qemu_ldst_label(s, true, opc, datalo, datahi, addrlo, addrhi,
                        mem_index, s->code_ptr, label_ptr);
    add_qemu_ldst_label(s, true, oi, datalo, datahi, addrlo, addrhi,
                        s->code_ptr, label_ptr);
#else /* !CONFIG_SOFTMMU */
    if (GUEST_BASE) {
        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP, GUEST_BASE);
@@ -1619,8 +1620,8 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
    label_ptr = s->code_ptr;
    tcg_out_bl_noaddr(s, COND_NE);

    add_qemu_ldst_label(s, false, opc, datalo, datahi, addrlo, addrhi,
                        mem_index, s->code_ptr, label_ptr);
    add_qemu_ldst_label(s, false, oi, datalo, datahi, addrlo, addrhi,
                        s->code_ptr, label_ptr);
#else /* !CONFIG_SOFTMMU */
    if (GUEST_BASE) {
        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP, GUEST_BASE);
+18 −19
Original line number Diff line number Diff line
@@ -1244,21 +1244,20 @@ static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi,
 * Record the context of a call to the out of line helper code for the slow path
 * for a load or store, so that we can later generate the correct helper code
 */
static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOp opc,
static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
                                TCGReg datalo, TCGReg datahi,
                                TCGReg addrlo, TCGReg addrhi,
                                int mem_index, tcg_insn_unit *raddr,
                                tcg_insn_unit *raddr,
                                tcg_insn_unit **label_ptr)
{
    TCGLabelQemuLdst *label = new_ldst_label(s);

    label->is_ld = is_ld;
    label->opc = opc;
    label->oi = oi;
    label->datalo_reg = datalo;
    label->datahi_reg = datahi;
    label->addrlo_reg = addrlo;
    label->addrhi_reg = addrhi;
    label->mem_index = mem_index;
    label->raddr = raddr;
    label->label_ptr[0] = label_ptr[0];
    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
@@ -1271,7 +1270,8 @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOp opc,
 */
static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
{
    TCGMemOp opc = l->opc;
    TCGMemOpIdx oi = l->oi;
    TCGMemOp opc = get_memop(oi);
    TCGReg data_reg;
    tcg_insn_unit **label_ptr = &l->label_ptr[0];

@@ -1295,15 +1295,14 @@ static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
            ofs += 4;
        }

        tcg_out_sti(s, TCG_TYPE_I32, TCG_REG_ESP, ofs, l->mem_index);
        tcg_out_sti(s, TCG_TYPE_I32, TCG_REG_ESP, ofs, oi);
        ofs += 4;

        tcg_out_sti(s, TCG_TYPE_I32, TCG_REG_ESP, ofs, (uintptr_t)l->raddr);
        tcg_out_sti(s, TCG_TYPE_PTR, TCG_REG_ESP, ofs, (uintptr_t)l->raddr);
    } else {
        tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
        /* The second argument is already loaded with addrlo.  */
        tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[2],
                     l->mem_index);
        tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[2], oi);
        tcg_out_movi(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[3],
                     (uintptr_t)l->raddr);
    }
@@ -1354,7 +1353,8 @@ static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
 */
static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
{
    TCGMemOp opc = l->opc;
    TCGMemOpIdx oi = l->oi;
    TCGMemOp opc = get_memop(oi);
    TCGMemOp s_bits = opc & MO_SIZE;
    tcg_insn_unit **label_ptr = &l->label_ptr[0];
    TCGReg retaddr;
@@ -1387,19 +1387,18 @@ static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
            ofs += 4;
        }

        tcg_out_sti(s, TCG_TYPE_I32, TCG_REG_ESP, ofs, l->mem_index);
        tcg_out_sti(s, TCG_TYPE_I32, TCG_REG_ESP, ofs, oi);
        ofs += 4;

        retaddr = TCG_REG_EAX;
        tcg_out_movi(s, TCG_TYPE_I32, retaddr, (uintptr_t)l->raddr);
        tcg_out_st(s, TCG_TYPE_I32, retaddr, TCG_REG_ESP, ofs);
        tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr);
        tcg_out_st(s, TCG_TYPE_PTR, retaddr, TCG_REG_ESP, ofs);
    } else {
        tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
        /* The second argument is already loaded with addrlo.  */
        tcg_out_mov(s, (s_bits == MO_64 ? TCG_TYPE_I64 : TCG_TYPE_I32),
                    tcg_target_call_iarg_regs[2], l->datalo_reg);
        tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[3],
                     l->mem_index);
        tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[3], oi);

        if (ARRAY_SIZE(tcg_target_call_iarg_regs) > 4) {
            retaddr = tcg_target_call_iarg_regs[4];
@@ -1557,8 +1556,8 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
    tcg_out_qemu_ld_direct(s, datalo, datahi, TCG_REG_L1, 0, 0, opc);

    /* Record the current context of a load into ldst label */
    add_qemu_ldst_label(s, true, opc, datalo, datahi, addrlo, addrhi,
                        mem_index, s->code_ptr, label_ptr);
    add_qemu_ldst_label(s, true, oi, datalo, datahi, addrlo, addrhi,
                        s->code_ptr, label_ptr);
#else
    {
        int32_t offset = GUEST_BASE;
@@ -1690,8 +1689,8 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
    tcg_out_qemu_st_direct(s, datalo, datahi, TCG_REG_L1, 0, 0, opc);

    /* Record the current context of a store into ldst label */
    add_qemu_ldst_label(s, false, opc, datalo, datahi, addrlo, addrhi,
                        mem_index, s->code_ptr, label_ptr);
    add_qemu_ldst_label(s, false, oi, datalo, datahi, addrlo, addrhi,
                        s->code_ptr, label_ptr);
#else
    {
        int32_t offset = GUEST_BASE;
Loading