Commit 1f00b27f authored by Sergey Sorokin's avatar Sergey Sorokin Committed by Richard Henderson
Browse files

tcg: Improve the alignment check infrastructure 



Some architectures (e.g. ARMv8) need the address which is aligned
to a size more than the size of the memory access.
To support such check it's enough the current costless alignment
check implementation in QEMU, but we need to support
an alignment size specifying.

Signed-off-by: default avatarSergey Sorokin <afarallax@yandex.ru>
Message-Id: <1466705806-679898-1-git-send-email-afarallax@yandex.ru>
Signed-off-by: default avatarRichard Henderson <rth@twiddle.net>
[rth: Assert in tcg_canonicalize_memop.  Leave get_alignment_bits
available for, though unused by, user-mode.  Retain logging difference
based on ALIGNED_ONLY.]
parent 59d7c14e

include/exec/cpu-all.h

+12 −4
Original line number Diff line number Diff line
@@ -288,14 +288,22 @@ CPUArchState *cpu_copy(CPUArchState *env); 
#if !defined(CONFIG_USER_ONLY)



/* Flags stored in the low bits of the TLB virtual address.  These are

   defined so that fast path ram access is all zeros.  */

 * defined so that fast path ram access is all zeros.

 * The flags all must be between TARGET_PAGE_BITS and

 * maximum address alignment bit.

 */

/* Zero if TLB entry is valid.  */

#define TLB_INVALID_MASK   (1 << 3)

#define TLB_INVALID_MASK    (1 << (TARGET_PAGE_BITS - 1))

/* Set if TLB entry references a clean RAM page.  The iotlb entry will

   contain the page physical address.  */

#define TLB_NOTDIRTY    (1 << 4)

#define TLB_NOTDIRTY        (1 << (TARGET_PAGE_BITS - 2))

/* Set if TLB entry is an IO callback.  */

#define TLB_MMIO        (1 << 5)

#define TLB_MMIO            (1 << (TARGET_PAGE_BITS - 3))



/* Use this mask to check interception with an alignment mask

 * in a TCG backend.

 */

#define TLB_FLAGS_MASK  (TLB_INVALID_MASK | TLB_NOTDIRTY | TLB_MMIO)



void dump_exec_info(FILE *f, fprintf_function cpu_fprintf);

void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf);

softmmu_template.h

+24 −64
Original line number Diff line number Diff line
@@ -171,20 +171,21 @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr, 
    unsigned mmu_idx = get_mmuidx(oi);

    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);

    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;

    int a_bits = get_alignment_bits(get_memop(oi));

    uintptr_t haddr;

    DATA_TYPE res;



    /* Adjust the given return address.  */

    retaddr -= GETPC_ADJ;



    /* If the TLB entry is for a different page, reload and try again.  */

    if ((addr & TARGET_PAGE_MASK)

         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {

        if ((addr & (DATA_SIZE - 1)) != 0

            && (get_memop(oi) & MO_AMASK) == MO_ALIGN) {

    if (a_bits > 0 && (addr & ((1 << a_bits) - 1)) != 0) {

        cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,

                             mmu_idx, retaddr);

    }



    /* If the TLB entry is for a different page, reload and try again.  */

    if ((addr & TARGET_PAGE_MASK)

         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {

        if (!VICTIM_TLB_HIT(ADDR_READ)) {

            tlb_fill(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,

                     mmu_idx, retaddr);
@@ -215,10 +216,6 @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr, 
        DATA_TYPE res1, res2;

        unsigned shift;

    do_unaligned_access:

        if ((get_memop(oi) & MO_AMASK) == MO_ALIGN) {

            cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,

                                 mmu_idx, retaddr);

        }

        addr1 = addr & ~(DATA_SIZE - 1);

        addr2 = addr1 + DATA_SIZE;

        /* Note the adjustment at the beginning of the function.
@@ -232,13 +229,6 @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr, 
        return res;

    }



    /* Handle aligned access or unaligned access in the same page.  */

    if ((addr & (DATA_SIZE - 1)) != 0

        && (get_memop(oi) & MO_AMASK) == MO_ALIGN) {

        cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,

                             mmu_idx, retaddr);

    }



    haddr = addr + env->tlb_table[mmu_idx][index].addend;

#if DATA_SIZE == 1

    res = glue(glue(ld, LSUFFIX), _p)((uint8_t *)haddr);
@@ -255,20 +245,21 @@ WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr, 
    unsigned mmu_idx = get_mmuidx(oi);

    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);

    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;

    int a_bits = get_alignment_bits(get_memop(oi));

    uintptr_t haddr;

    DATA_TYPE res;



    /* Adjust the given return address.  */

    retaddr -= GETPC_ADJ;



    /* If the TLB entry is for a different page, reload and try again.  */

    if ((addr & TARGET_PAGE_MASK)

         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {

        if ((addr & (DATA_SIZE - 1)) != 0

            && (get_memop(oi) & MO_AMASK) == MO_ALIGN) {

    if (a_bits > 0 && (addr & ((1 << a_bits) - 1)) != 0) {

        cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,

                             mmu_idx, retaddr);

    }



    /* If the TLB entry is for a different page, reload and try again.  */

    if ((addr & TARGET_PAGE_MASK)

         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {

        if (!VICTIM_TLB_HIT(ADDR_READ)) {

            tlb_fill(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,

                     mmu_idx, retaddr);
@@ -299,10 +290,6 @@ WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr, 
        DATA_TYPE res1, res2;

        unsigned shift;

    do_unaligned_access:

        if ((get_memop(oi) & MO_AMASK) == MO_ALIGN) {

            cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,

                                 mmu_idx, retaddr);

        }

        addr1 = addr & ~(DATA_SIZE - 1);

        addr2 = addr1 + DATA_SIZE;

        /* Note the adjustment at the beginning of the function.
@@ -316,13 +303,6 @@ WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr, 
        return res;

    }



    /* Handle aligned access or unaligned access in the same page.  */

    if ((addr & (DATA_SIZE - 1)) != 0

        && (get_memop(oi) & MO_AMASK) == MO_ALIGN) {

        cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,

                             mmu_idx, retaddr);

    }



    haddr = addr + env->tlb_table[mmu_idx][index].addend;

    res = glue(glue(ld, LSUFFIX), _be_p)((uint8_t *)haddr);

    return res;
@@ -376,19 +356,20 @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, 
    unsigned mmu_idx = get_mmuidx(oi);

    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);

    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;

    int a_bits = get_alignment_bits(get_memop(oi));

    uintptr_t haddr;



    /* Adjust the given return address.  */

    retaddr -= GETPC_ADJ;



    /* If the TLB entry is for a different page, reload and try again.  */

    if ((addr & TARGET_PAGE_MASK)

        != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {

        if ((addr & (DATA_SIZE - 1)) != 0

            && (get_memop(oi) & MO_AMASK) == MO_ALIGN) {

    if (a_bits > 0 && (addr & ((1 << a_bits) - 1)) != 0) {

        cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,

                             mmu_idx, retaddr);

    }



    /* If the TLB entry is for a different page, reload and try again.  */

    if ((addr & TARGET_PAGE_MASK)

        != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {

        if (!VICTIM_TLB_HIT(addr_write)) {

            tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_STORE, mmu_idx, retaddr);

        }
@@ -416,10 +397,6 @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, 
                     >= TARGET_PAGE_SIZE)) {

        int i;

    do_unaligned_access:

        if ((get_memop(oi) & MO_AMASK) == MO_ALIGN) {

            cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,

                                 mmu_idx, retaddr);

        }

        /* XXX: not efficient, but simple */

        /* Note: relies on the fact that tlb_fill() does not remove the

         * previous page from the TLB cache.  */
@@ -434,13 +411,6 @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, 
        return;

    }



    /* Handle aligned access or unaligned access in the same page.  */

    if ((addr & (DATA_SIZE - 1)) != 0

        && (get_memop(oi) & MO_AMASK) == MO_ALIGN) {

        cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,

                             mmu_idx, retaddr);

    }



    haddr = addr + env->tlb_table[mmu_idx][index].addend;

#if DATA_SIZE == 1

    glue(glue(st, SUFFIX), _p)((uint8_t *)haddr, val);
@@ -456,19 +426,20 @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, 
    unsigned mmu_idx = get_mmuidx(oi);

    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);

    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;

    int a_bits = get_alignment_bits(get_memop(oi));

    uintptr_t haddr;



    /* Adjust the given return address.  */

    retaddr -= GETPC_ADJ;



    /* If the TLB entry is for a different page, reload and try again.  */

    if ((addr & TARGET_PAGE_MASK)

        != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {

        if ((addr & (DATA_SIZE - 1)) != 0

            && (get_memop(oi) & MO_AMASK) == MO_ALIGN) {

    if (a_bits > 0 && (addr & ((1 << a_bits) - 1)) != 0) {

        cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,

                             mmu_idx, retaddr);

    }



    /* If the TLB entry is for a different page, reload and try again.  */

    if ((addr & TARGET_PAGE_MASK)

        != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {

        if (!VICTIM_TLB_HIT(addr_write)) {

            tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_STORE, mmu_idx, retaddr);

        }
@@ -496,10 +467,6 @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, 
                     >= TARGET_PAGE_SIZE)) {

        int i;

    do_unaligned_access:

        if ((get_memop(oi) & MO_AMASK) == MO_ALIGN) {

            cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,

                                 mmu_idx, retaddr);

        }

        /* XXX: not efficient, but simple */

        /* Note: relies on the fact that tlb_fill() does not remove the

         * previous page from the TLB cache.  */
@@ -514,13 +481,6 @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, 
        return;

    }



    /* Handle aligned access or unaligned access in the same page.  */

    if ((addr & (DATA_SIZE - 1)) != 0

        && (get_memop(oi) & MO_AMASK) == MO_ALIGN) {

        cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,

                             mmu_idx, retaddr);

    }



    haddr = addr + env->tlb_table[mmu_idx][index].addend;

    glue(glue(st, SUFFIX), _be_p)((uint8_t *)haddr, val);

}

tcg/aarch64/tcg-target.inc.c

+5 −4
Original line number Diff line number Diff line
@@ -1081,19 +1081,20 @@ static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, TCGMemOp opc, 
    int tlb_offset = is_read ?

        offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)

        : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write);

    int s_mask = (1 << (opc & MO_SIZE)) - 1;

    int a_bits = get_alignment_bits(opc);

    TCGReg base = TCG_AREG0, x3;

    uint64_t tlb_mask;



    /* For aligned accesses, we check the first byte and include the alignment

       bits within the address.  For unaligned access, we check that we don't

       cross pages using the address of the last byte of the access.  */

    if ((opc & MO_AMASK) == MO_ALIGN || s_mask == 0) {

        tlb_mask = TARGET_PAGE_MASK | s_mask;

    if (a_bits >= 0) {

        /* A byte access or an alignment check required */

        tlb_mask = TARGET_PAGE_MASK | ((1 << a_bits) - 1);

        x3 = addr_reg;

    } else {

        tcg_out_insn(s, 3401, ADDI, TARGET_LONG_BITS == 64,

                     TCG_REG_X3, addr_reg, s_mask);

                     TCG_REG_X3, addr_reg, (1 << (opc & MO_SIZE)) - 1);

        tlb_mask = TARGET_PAGE_MASK;

        x3 = TCG_REG_X3;

    }

tcg/i386/tcg-target.inc.c

+9 −6
Original line number Diff line number Diff line
@@ -1202,8 +1202,8 @@ static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi, 
    TCGType ttype = TCG_TYPE_I32;

    TCGType tlbtype = TCG_TYPE_I32;

    int trexw = 0, hrexw = 0, tlbrexw = 0;

    int s_mask = (1 << (opc & MO_SIZE)) - 1;

    bool aligned = (opc & MO_AMASK) == MO_ALIGN || s_mask == 0;

    int a_bits = get_alignment_bits(opc);

    target_ulong tlb_mask;



    if (TCG_TARGET_REG_BITS == 64) {

        if (TARGET_LONG_BITS == 64) {
@@ -1220,19 +1220,22 @@ static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi, 
    }



    tcg_out_mov(s, tlbtype, r0, addrlo);

    if (aligned) {

    if (a_bits >= 0) {

        /* A byte access or an alignment check required */

        tcg_out_mov(s, ttype, r1, addrlo);

        tlb_mask = TARGET_PAGE_MASK | ((1 << a_bits) - 1);

    } else {

        /* For unaligned access check that we don't cross pages using

           the page address of the last byte.  */

        tcg_out_modrm_offset(s, OPC_LEA + trexw, r1, addrlo, s_mask);

        tcg_out_modrm_offset(s, OPC_LEA + trexw, r1, addrlo,

                             (1 << (opc & MO_SIZE)) - 1);

        tlb_mask = TARGET_PAGE_MASK;

    }



    tcg_out_shifti(s, SHIFT_SHR + tlbrexw, r0,

                   TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);



    tgen_arithi(s, ARITH_AND + trexw, r1,

                TARGET_PAGE_MASK | (aligned ? s_mask : 0), 0);

    tgen_arithi(s, ARITH_AND + trexw, r1, tlb_mask, 0);

    tgen_arithi(s, ARITH_AND + tlbrexw, r0,

                (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS, 0);

tcg/ppc/tcg-target.inc.c

+9 −5
Original line number Diff line number Diff line
@@ -1405,6 +1405,7 @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGMemOp opc, 
    int add_off = offsetof(CPUArchState, tlb_table[mem_index][0].addend);

    TCGReg base = TCG_AREG0;

    TCGMemOp s_bits = opc & MO_SIZE;

    int a_bits = get_alignment_bits(opc);



    /* Extract the page index, shifted into place for tlb index.  */

    if (TCG_TARGET_REG_BITS == 64) {
@@ -1462,14 +1463,17 @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGMemOp opc, 
         * the bottom bits and thus trigger a comparison failure on

         * unaligned accesses

         */

        if (a_bits < 0) {

            a_bits = s_bits;

        }

        tcg_out_rlw(s, RLWINM, TCG_REG_R0, addrlo, 0,

                    (32 - s_bits) & 31, 31 - TARGET_PAGE_BITS);

    } else if (s_bits) {

        /* > byte access, we need to handle alignment */

        if ((opc & MO_AMASK) == MO_ALIGN) {

                    (32 - a_bits) & 31, 31 - TARGET_PAGE_BITS);

    } else if (a_bits) {

        /* More than byte access, we need to handle alignment */

        if (a_bits > 0) {

            /* Alignment required by the front-end, same as 32-bits */

            tcg_out_rld(s, RLDICL, TCG_REG_R0, addrlo,

                        64 - TARGET_PAGE_BITS, TARGET_PAGE_BITS - s_bits);

                        64 - TARGET_PAGE_BITS, TARGET_PAGE_BITS - a_bits);

            tcg_out_rld(s, RLDICL, TCG_REG_R0, TCG_REG_R0, TARGET_PAGE_BITS, 0);

       } else {

           /* We support unaligned accesses, we need to make sure we fail