Commit aac1fb05 authored by Richard Henderson's avatar Richard Henderson
Browse files

tcg: Tidy softmmu_template.h 



Avoid a loop in the tlb_fill path; the fill will either succeed or
generate an exception.

Inline the slow_ld/st function; it was a complete copy of the main
helper except for the actual cross-page unaligned code, and the
compiler was inlining it anyway.

Add unlikely markers optimizing for the most common case of simple
tlb miss.

Make sure the compiler can optimize away the unaligned paths for a
1 byte access.

Reviewed-by: default avatarAurelien Jarno <aurelien@aurel32.net>
Signed-off-by: default avatarRichard Henderson <rth@twiddle.net>
parent e25c3887

include/exec/softmmu_template.h

+104 −183
Original line number Diff line number Diff line
@@ -54,10 +54,6 @@ 
#define ADDR_READ addr_read

#endif



static DATA_TYPE glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(CPUArchState *env,

                                                        target_ulong addr,

                                                        int mmu_idx,

                                                        uintptr_t retaddr);

static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,

                                              hwaddr physaddr,

                                              target_ulong addr,
@@ -86,92 +82,48 @@ glue(glue(helper_ret_ld, SUFFIX), MMUSUFFIX)(CPUArchState *env, 
                                             target_ulong addr, int mmu_idx,

                                             uintptr_t retaddr)

{

    DATA_TYPE res;

    int index;

    target_ulong tlb_addr;

    hwaddr ioaddr;

    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);

    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;

    uintptr_t haddr;



    /* test if there is match for unaligned or IO access */

    /* XXX: could done more in memory macro in a non portable way */

    index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);

 redo:

    tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;

    if ((addr & TARGET_PAGE_MASK) == (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {

        if (tlb_addr & ~TARGET_PAGE_MASK) {

            /* IO access */

            if ((addr & (DATA_SIZE - 1)) != 0)

                goto do_unaligned_access;

            ioaddr = env->iotlb[mmu_idx][index];

            res = glue(io_read, SUFFIX)(env, ioaddr, addr, retaddr);

        } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {

            /* slow unaligned access (it spans two pages or IO) */

        do_unaligned_access:

#ifdef ALIGNED_ONLY

            do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);

#endif

            res = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(env, addr,

                                                         mmu_idx, retaddr);

        } else {

            /* unaligned/aligned access in the same page */

            uintptr_t addend;

    /* If the TLB entry is for a different page, reload and try again.  */

    if ((addr & TARGET_PAGE_MASK)

         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {

#ifdef ALIGNED_ONLY

        if ((addr & (DATA_SIZE - 1)) != 0) {

            do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);

        }

#endif

            addend = env->tlb_table[mmu_idx][index].addend;

            res = glue(glue(ld, USUFFIX), _raw)((uint8_t *)(intptr_t)

                                                (addr + addend));

        }

    } else {

#ifdef ALIGNED_ONLY

        if ((addr & (DATA_SIZE - 1)) != 0)

            do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);

#endif

        tlb_fill(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);

        goto redo;

    }

    return res;

}



DATA_TYPE

glue(glue(helper_ld, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,

                                         int mmu_idx)

{

    return glue(glue(helper_ret_ld, SUFFIX), MMUSUFFIX)(env, addr, mmu_idx,

                                                        GETPC_EXT());

        tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;

    }



/* handle all unaligned cases */

static DATA_TYPE

glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(CPUArchState *env,

                                       target_ulong addr,

                                       int mmu_idx,

                                       uintptr_t retaddr)

{

    DATA_TYPE res, res1, res2;

    int index, shift;

    /* Handle an IO access.  */

    if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {

        hwaddr ioaddr;

    target_ulong tlb_addr, addr1, addr2;



    index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);

 redo:

    tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;

    if ((addr & TARGET_PAGE_MASK) == (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {

        if (tlb_addr & ~TARGET_PAGE_MASK) {

            /* IO access */

            if ((addr & (DATA_SIZE - 1)) != 0)

        if ((addr & (DATA_SIZE - 1)) != 0) {

            goto do_unaligned_access;

        }

        ioaddr = env->iotlb[mmu_idx][index];

            res = glue(io_read, SUFFIX)(env, ioaddr, addr, retaddr);

        } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {

        return glue(io_read, SUFFIX)(env, ioaddr, addr, retaddr);

    }



    /* Handle slow unaligned access (it spans two pages or IO).  */

    if (DATA_SIZE > 1

        && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1

                    >= TARGET_PAGE_SIZE)) {

        target_ulong addr1, addr2;

        DATA_TYPE res1, res2, res;

        unsigned shift;

    do_unaligned_access:

            /* slow unaligned access (it spans two pages) */

#ifdef ALIGNED_ONLY

        do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);

#endif

        addr1 = addr & ~(DATA_SIZE - 1);

        addr2 = addr1 + DATA_SIZE;

            res1 = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(env, addr1,

        res1 = glue(glue(helper_ret_ld, SUFFIX), MMUSUFFIX)(env, addr1,

                                                            mmu_idx, retaddr);

            res2 = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(env, addr2,

        res2 = glue(glue(helper_ret_ld, SUFFIX), MMUSUFFIX)(env, addr2,

                                                            mmu_idx, retaddr);

        shift = (addr & (DATA_SIZE - 1)) * 8;

#ifdef TARGET_WORDS_BIGENDIAN
@@ -179,28 +131,29 @@ glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(CPUArchState *env, 
#else

        res = (res1 >> shift) | (res2 << ((DATA_SIZE * 8) - shift));

#endif

            res = (DATA_TYPE)res;

        } else {

            /* unaligned/aligned access in the same page */

            uintptr_t addend = env->tlb_table[mmu_idx][index].addend;

            res = glue(glue(ld, USUFFIX), _raw)((uint8_t *)(intptr_t)

                                                (addr + addend));

        return res;

    }

    } else {

        /* the page is not in the TLB : fill it */

        tlb_fill(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);

        goto redo;



    /* Handle aligned access or unaligned access in the same page.  */

#ifdef ALIGNED_ONLY

    if ((addr & (DATA_SIZE - 1)) != 0) {

        do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);

    }

    return res;

#endif



    haddr = addr + env->tlb_table[mmu_idx][index].addend;

    return glue(glue(ld, USUFFIX), _raw)((uint8_t *)haddr);

}



#ifndef SOFTMMU_CODE_ACCESS

DATA_TYPE

glue(glue(helper_ld, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,

                                         int mmu_idx)

{

    return glue(glue(helper_ret_ld, SUFFIX), MMUSUFFIX)(env, addr, mmu_idx,

                                                        GETPC_EXT());

}



static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(CPUArchState *env,

                                                   target_ulong addr,

                                                   DATA_TYPE val,

                                                   int mmu_idx,

                                                   uintptr_t retaddr);

#ifndef SOFTMMU_CODE_ACCESS



static inline void glue(io_write, SUFFIX)(CPUArchState *env,

                                          hwaddr physaddr,
@@ -225,106 +178,74 @@ glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)(CPUArchState *env, 
                                             target_ulong addr, DATA_TYPE val,

                                             int mmu_idx, uintptr_t retaddr)

{

    hwaddr ioaddr;

    target_ulong tlb_addr;

    int index;

    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);

    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;

    uintptr_t haddr;



    index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);

 redo:

    tlb_addr = env->tlb_table[mmu_idx][index].addr_write;

    if ((addr & TARGET_PAGE_MASK) == (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {

        if (tlb_addr & ~TARGET_PAGE_MASK) {

            /* IO access */

            if ((addr & (DATA_SIZE - 1)) != 0)

                goto do_unaligned_access;

            ioaddr = env->iotlb[mmu_idx][index];

            glue(io_write, SUFFIX)(env, ioaddr, val, addr, retaddr);

        } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {

        do_unaligned_access:

#ifdef ALIGNED_ONLY

            do_unaligned_access(env, addr, 1, mmu_idx, retaddr);

#endif

            glue(glue(slow_st, SUFFIX), MMUSUFFIX)(env, addr, val,

                                                   mmu_idx, retaddr);

        } else {

            /* aligned/unaligned access in the same page */

            uintptr_t addend;

    /* If the TLB entry is for a different page, reload and try again.  */

    if ((addr & TARGET_PAGE_MASK)

        != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {

#ifdef ALIGNED_ONLY

        if ((addr & (DATA_SIZE - 1)) != 0) {

            do_unaligned_access(env, addr, 1, mmu_idx, retaddr);

        }

#endif

            addend = env->tlb_table[mmu_idx][index].addend;

            glue(glue(st, SUFFIX), _raw)((uint8_t *)(intptr_t)

                                         (addr + addend), val);

        }

    } else {

        /* the page is not in the TLB : fill it */

#ifdef ALIGNED_ONLY

        if ((addr & (DATA_SIZE - 1)) != 0)

            do_unaligned_access(env, addr, 1, mmu_idx, retaddr);

#endif

        tlb_fill(env, addr, 1, mmu_idx, retaddr);

        goto redo;

    }

}



void

glue(glue(helper_st, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,

                                         DATA_TYPE val, int mmu_idx)

{

    glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)(env, addr, val, mmu_idx,

                                                 GETPC_EXT());

        tlb_addr = env->tlb_table[mmu_idx][index].addr_write;

    }



/* handles all unaligned cases */

static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(CPUArchState *env,

                                                   target_ulong addr,

                                                   DATA_TYPE val,

                                                   int mmu_idx,

                                                   uintptr_t retaddr)

{

    /* Handle an IO access.  */

    if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {

        hwaddr ioaddr;

    target_ulong tlb_addr;

    int index, i;



    index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);

 redo:

    tlb_addr = env->tlb_table[mmu_idx][index].addr_write;

    if ((addr & TARGET_PAGE_MASK) == (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {

        if (tlb_addr & ~TARGET_PAGE_MASK) {

            /* IO access */

            if ((addr & (DATA_SIZE - 1)) != 0)

        if ((addr & (DATA_SIZE - 1)) != 0) {

            goto do_unaligned_access;

        }

        ioaddr = env->iotlb[mmu_idx][index];

        glue(io_write, SUFFIX)(env, ioaddr, val, addr, retaddr);

        } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {

        return;

    }



    /* Handle slow unaligned access (it spans two pages or IO).  */

    if (DATA_SIZE > 1

        && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1

                     >= TARGET_PAGE_SIZE)) {

        int i;

    do_unaligned_access:

#ifdef ALIGNED_ONLY

        do_unaligned_access(env, addr, 1, mmu_idx, retaddr);

#endif

        /* XXX: not efficient, but simple */

        /* Note: relies on the fact that tlb_fill() does not remove the

         * previous page from the TLB cache.  */

        for (i = DATA_SIZE - 1; i >= 0; i--) {

#ifdef TARGET_WORDS_BIGENDIAN

                glue(slow_stb, MMUSUFFIX)(env, addr + i,

                                          val >> (((DATA_SIZE - 1) * 8) - (i * 8)),

                                          mmu_idx, retaddr);

            uint8_t val8 = val >> (((DATA_SIZE - 1) * 8) - (i * 8));

#else

                glue(slow_stb, MMUSUFFIX)(env, addr + i,

                                          val >> (i * 8),

                                          mmu_idx, retaddr);

            uint8_t val8 = val >> (i * 8);

#endif

            glue(helper_ret_stb, MMUSUFFIX)(env, addr + i, val8,

                                            mmu_idx, retaddr);

        }

        } else {

            /* aligned/unaligned access in the same page */

            uintptr_t addend = env->tlb_table[mmu_idx][index].addend;

            glue(glue(st, SUFFIX), _raw)((uint8_t *)(intptr_t)

                                         (addr + addend), val);

        return;

    }

    } else {

        /* the page is not in the TLB : fill it */

        tlb_fill(env, addr, 1, mmu_idx, retaddr);

        goto redo;



    /* Handle aligned access or unaligned access in the same page.  */

#ifdef ALIGNED_ONLY

    if ((addr & (DATA_SIZE - 1)) != 0) {

        do_unaligned_access(env, addr, 1, mmu_idx, retaddr);

    }

#endif



    haddr = addr + env->tlb_table[mmu_idx][index].addend;

    glue(glue(st, SUFFIX), _raw)((uint8_t *)haddr, val);

}



void

glue(glue(helper_st, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,

                                         DATA_TYPE val, int mmu_idx)

{

    glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)(env, addr, val, mmu_idx,

                                                 GETPC_EXT());

}



#endif /* !defined(SOFTMMU_CODE_ACCESS) */