Commit 37ec01d4 authored by Avi Kivity's avatar Avi Kivity
Browse files

memory: dispatch directly via MemoryRegion 



Instead of indirecting via io_mem_region, dispatch directly
through the MemoryRegion obtained from the iotlb or phys_page_find().

Signed-off-by: default avatarAvi Kivity <avi@redhat.com>
parent ce5d64c2

exec-all.h

+5 −4
Original line number Diff line number Diff line
@@ -299,10 +299,11 @@ extern void *tci_tb_ptr; 


#if !defined(CONFIG_USER_ONLY)



target_phys_addr_t section_to_ioaddr(target_phys_addr_t section_io_addr);

uint64_t io_mem_read(int index, target_phys_addr_t addr, unsigned size);

void io_mem_write(int index, target_phys_addr_t addr, uint64_t value,

struct MemoryRegion *iotlb_to_region(target_phys_addr_t index);

uint64_t io_mem_read(struct MemoryRegion *mr, target_phys_addr_t addr,

                     unsigned size);

void io_mem_write(struct MemoryRegion *mr, target_phys_addr_t addr,

                  uint64_t value, unsigned size);

extern struct MemoryRegion *io_mem_region[IO_MEM_NB_ENTRIES];



void tlb_fill(CPUState *env1, target_ulong addr, int is_write, int mmu_idx,

exec.c

+36 −66
Original line number Diff line number Diff line
@@ -3399,7 +3399,7 @@ static uint64_t subpage_read(void *opaque, target_phys_addr_t addr, 
    addr += mmio->base;

    addr -= section->offset_within_address_space;

    addr += section->offset_within_region;

    return io_mem_read(section->mr->ram_addr, addr, len);

    return io_mem_read(section->mr, addr, len);

}



static void subpage_write(void *opaque, target_phys_addr_t addr,
@@ -3418,7 +3418,7 @@ static void subpage_write(void *opaque, target_phys_addr_t addr, 
    addr += mmio->base;

    addr -= section->offset_within_address_space;

    addr += section->offset_within_region;

    io_mem_write(section->mr->ram_addr, addr, value, len);

    io_mem_write(section->mr, addr, value, len);

}



static const MemoryRegionOps subpage_ops = {
@@ -3562,13 +3562,9 @@ static uint16_t dummy_section(MemoryRegion *mr) 
    return phys_section_add(&section);

}



target_phys_addr_t section_to_ioaddr(target_phys_addr_t section_io_addr)

MemoryRegion *iotlb_to_region(target_phys_addr_t index)

{

    MemoryRegionSection *section;



    section = &phys_sections[section_io_addr & ~TARGET_PAGE_MASK];

    return (section_io_addr & TARGET_PAGE_MASK)

        | (section->mr->ram_addr & ~TARGET_PAGE_MASK);

    return phys_sections[index & ~TARGET_PAGE_MASK].mr;

}



static void io_mem_init(void)
@@ -3838,7 +3834,7 @@ int cpu_memory_rw_debug(CPUState *env, target_ulong addr, 
void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,

                            int len, int is_write)

{

    int l, io_index;

    int l;

    uint8_t *ptr;

    uint32_t val;

    target_phys_addr_t page;
@@ -3854,25 +3850,23 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf, 
        if (is_write) {

            if (!memory_region_is_ram(section->mr)) {

                target_phys_addr_t addr1;

                io_index = memory_region_get_ram_addr(section->mr)

                    & (IO_MEM_NB_ENTRIES - 1);

                addr1 = section_addr(section, addr);

                /* XXX: could force cpu_single_env to NULL to avoid

                   potential bugs */

                if (l >= 4 && ((addr1 & 3) == 0)) {

                    /* 32 bit write access */

                    val = ldl_p(buf);

                    io_mem_write(io_index, addr1, val, 4);

                    io_mem_write(section->mr, addr1, val, 4);

                    l = 4;

                } else if (l >= 2 && ((addr1 & 1) == 0)) {

                    /* 16 bit write access */

                    val = lduw_p(buf);

                    io_mem_write(io_index, addr1, val, 2);

                    io_mem_write(section->mr, addr1, val, 2);

                    l = 2;

                } else {

                    /* 8 bit write access */

                    val = ldub_p(buf);

                    io_mem_write(io_index, addr1, val, 1);

                    io_mem_write(section->mr, addr1, val, 1);

                    l = 1;

                }

            } else if (!section->readonly) {
@@ -3895,22 +3889,20 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf, 
            if (!is_ram_rom_romd(section)) {

                target_phys_addr_t addr1;

                /* I/O case */

                io_index = memory_region_get_ram_addr(section->mr)

                    & (IO_MEM_NB_ENTRIES - 1);

                addr1 = section_addr(section, addr);

                if (l >= 4 && ((addr1 & 3) == 0)) {

                    /* 32 bit read access */

                    val = io_mem_read(io_index, addr1, 4);

                    val = io_mem_read(section->mr, addr1, 4);

                    stl_p(buf, val);

                    l = 4;

                } else if (l >= 2 && ((addr1 & 1) == 0)) {

                    /* 16 bit read access */

                    val = io_mem_read(io_index, addr1, 2);

                    val = io_mem_read(section->mr, addr1, 2);

                    stw_p(buf, val);

                    l = 2;

                } else {

                    /* 8 bit read access */

                    val = io_mem_read(io_index, addr1, 1);

                    val = io_mem_read(section->mr, addr1, 1);

                    stb_p(buf, val);

                    l = 1;

                }
@@ -4106,7 +4098,6 @@ void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len, 
static inline uint32_t ldl_phys_internal(target_phys_addr_t addr,

                                         enum device_endian endian)

{

    int io_index;

    uint8_t *ptr;

    uint32_t val;

    MemoryRegionSection *section;
@@ -4115,10 +4106,8 @@ static inline uint32_t ldl_phys_internal(target_phys_addr_t addr, 


    if (!is_ram_rom_romd(section)) {

        /* I/O case */

        io_index = memory_region_get_ram_addr(section->mr)

            & (IO_MEM_NB_ENTRIES - 1);

        addr = section_addr(section, addr);

        val = io_mem_read(io_index, addr, 4);

        val = io_mem_read(section->mr, addr, 4);

#if defined(TARGET_WORDS_BIGENDIAN)

        if (endian == DEVICE_LITTLE_ENDIAN) {

            val = bswap32(val);
@@ -4167,7 +4156,6 @@ uint32_t ldl_be_phys(target_phys_addr_t addr) 
static inline uint64_t ldq_phys_internal(target_phys_addr_t addr,

                                         enum device_endian endian)

{

    int io_index;

    uint8_t *ptr;

    uint64_t val;

    MemoryRegionSection *section;
@@ -4176,18 +4164,16 @@ static inline uint64_t ldq_phys_internal(target_phys_addr_t addr, 


    if (!is_ram_rom_romd(section)) {

        /* I/O case */

        io_index = memory_region_get_ram_addr(section->mr)

            & (IO_MEM_NB_ENTRIES - 1);

        addr = section_addr(section, addr);



        /* XXX This is broken when device endian != cpu endian.

               Fix and add "endian" variable check */

#ifdef TARGET_WORDS_BIGENDIAN

        val = io_mem_read(io_index, addr, 4) << 32;

        val |= io_mem_read(io_index, addr + 4, 4);

        val = io_mem_read(section->mr, addr, 4) << 32;

        val |= io_mem_read(section->mr, addr + 4, 4);

#else

        val = io_mem_read(io_index, addr, 4);

        val |= io_mem_read(io_index, addr + 4, 4) << 32;

        val = io_mem_read(section->mr, addr, 4);

        val |= io_mem_read(section->mr, addr + 4, 4) << 32;

#endif

    } else {

        /* RAM case */
@@ -4236,7 +4222,6 @@ uint32_t ldub_phys(target_phys_addr_t addr) 
static inline uint32_t lduw_phys_internal(target_phys_addr_t addr,

                                          enum device_endian endian)

{

    int io_index;

    uint8_t *ptr;

    uint64_t val;

    MemoryRegionSection *section;
@@ -4245,10 +4230,8 @@ static inline uint32_t lduw_phys_internal(target_phys_addr_t addr, 


    if (!is_ram_rom_romd(section)) {

        /* I/O case */

        io_index = memory_region_get_ram_addr(section->mr)

            & (IO_MEM_NB_ENTRIES - 1);

        addr = section_addr(section, addr);

        val = io_mem_read(io_index, addr, 2);

        val = io_mem_read(section->mr, addr, 2);

#if defined(TARGET_WORDS_BIGENDIAN)

        if (endian == DEVICE_LITTLE_ENDIAN) {

            val = bswap16(val);
@@ -4298,20 +4281,17 @@ uint32_t lduw_be_phys(target_phys_addr_t addr) 
   bits are used to track modified PTEs */

void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)

{

    int io_index;

    uint8_t *ptr;

    MemoryRegionSection *section;



    section = phys_page_find(addr >> TARGET_PAGE_BITS);



    if (!memory_region_is_ram(section->mr) || section->readonly) {

        addr = section_addr(section, addr);

        if (memory_region_is_ram(section->mr)) {

            io_index = io_mem_rom.ram_addr;

        } else {

            io_index = memory_region_get_ram_addr(section->mr);

            section = &phys_sections[phys_section_rom];

        }

        addr = section_addr(section, addr);

        io_mem_write(io_index, addr, val, 4);

        io_mem_write(section->mr, addr, val, 4);

    } else {

        unsigned long addr1 = (memory_region_get_ram_addr(section->mr)

                               & TARGET_PAGE_MASK)
@@ -4333,26 +4313,22 @@ void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val) 


void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)

{

    int io_index;

    uint8_t *ptr;

    MemoryRegionSection *section;



    section = phys_page_find(addr >> TARGET_PAGE_BITS);



    if (!memory_region_is_ram(section->mr) || section->readonly) {

        addr = section_addr(section, addr);

        if (memory_region_is_ram(section->mr)) {

                io_index = io_mem_rom.ram_addr;

        } else {

            io_index = memory_region_get_ram_addr(section->mr)

                & (IO_MEM_NB_ENTRIES - 1);

            section = &phys_sections[phys_section_rom];

        }

        addr = section_addr(section, addr);

#ifdef TARGET_WORDS_BIGENDIAN

        io_mem_write(io_index, addr, val >> 32, 4);

        io_mem_write(io_index, addr + 4, (uint32_t)val, 4);

        io_mem_write(section->mr, addr, val >> 32, 4);

        io_mem_write(section->mr, addr + 4, (uint32_t)val, 4);

#else

        io_mem_write(io_index, addr, (uint32_t)val, 4);

        io_mem_write(io_index, addr + 4, val >> 32, 4);

        io_mem_write(section->mr, addr, (uint32_t)val, 4);

        io_mem_write(section->mr, addr + 4, val >> 32, 4);

#endif

    } else {

        ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
@@ -4366,20 +4342,16 @@ void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val) 
static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val,

                                     enum device_endian endian)

{

    int io_index;

    uint8_t *ptr;

    MemoryRegionSection *section;



    section = phys_page_find(addr >> TARGET_PAGE_BITS);



    if (!memory_region_is_ram(section->mr) || section->readonly) {

        addr = section_addr(section, addr);

        if (memory_region_is_ram(section->mr)) {

            io_index = io_mem_rom.ram_addr;

        } else {

            io_index = memory_region_get_ram_addr(section->mr)

                & (IO_MEM_NB_ENTRIES - 1);

            section = &phys_sections[phys_section_rom];

        }

        addr = section_addr(section, addr);

#if defined(TARGET_WORDS_BIGENDIAN)

        if (endian == DEVICE_LITTLE_ENDIAN) {

            val = bswap32(val);
@@ -4389,7 +4361,7 @@ static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val, 
            val = bswap32(val);

        }

#endif

        io_mem_write(io_index, addr, val, 4);

        io_mem_write(section->mr, addr, val, 4);

    } else {

        unsigned long addr1;

        addr1 = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
@@ -4443,20 +4415,16 @@ void stb_phys(target_phys_addr_t addr, uint32_t val) 
static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val,

                                     enum device_endian endian)

{

    int io_index;

    uint8_t *ptr;

    MemoryRegionSection *section;



    section = phys_page_find(addr >> TARGET_PAGE_BITS);



    if (!memory_region_is_ram(section->mr) || section->readonly) {

        addr = section_addr(section, addr);

        if (memory_region_is_ram(section->mr)) {

            io_index = io_mem_rom.ram_addr;

        } else {

            io_index = memory_region_get_ram_addr(section->mr)

                & (IO_MEM_NB_ENTRIES - 1);

            section = &phys_sections[phys_section_rom];

        }

        addr = section_addr(section, addr);

#if defined(TARGET_WORDS_BIGENDIAN)

        if (endian == DEVICE_LITTLE_ENDIAN) {

            val = bswap16(val);
@@ -4466,7 +4434,7 @@ static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val, 
            val = bswap16(val);

        }

#endif

        io_mem_write(io_index, addr, val, 2);

        io_mem_write(section->mr, addr, val, 2);

    } else {

        unsigned long addr1;

        addr1 = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
@@ -4678,6 +4646,7 @@ tb_page_addr_t get_page_addr_code(CPUState *env1, target_ulong addr) 
{

    int mmu_idx, page_index, pd;

    void *p;

    MemoryRegion *mr;



    page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);

    mmu_idx = cpu_mmu_index(env1);
@@ -4686,8 +4655,9 @@ tb_page_addr_t get_page_addr_code(CPUState *env1, target_ulong addr) 
        ldub_code(addr);

    }

    pd = env1->iotlb[mmu_idx][page_index] & ~TARGET_PAGE_MASK;

    if (pd != io_mem_ram.ram_addr && pd != io_mem_rom.ram_addr

        && !io_mem_region[pd]->rom_device) {

    mr = iotlb_to_region(pd);

    if (mr != &io_mem_ram && mr != &io_mem_rom

        && mr != &io_mem_notdirty && !mr->rom_device) {

#if defined(TARGET_ALPHA) || defined(TARGET_MIPS) || defined(TARGET_SPARC)

        cpu_unassigned_access(env1, addr, 0, 1, 0, 4);

#else

memory.c

+4 −4
Original line number Diff line number Diff line
@@ -1501,15 +1501,15 @@ void set_system_io_map(MemoryRegion *mr) 
    memory_region_update_topology(NULL);

}



uint64_t io_mem_read(int io_index, target_phys_addr_t addr, unsigned size)

uint64_t io_mem_read(MemoryRegion *mr, target_phys_addr_t addr, unsigned size)

{

    return memory_region_dispatch_read(io_mem_region[io_index], addr, size);

    return memory_region_dispatch_read(mr, addr, size);

}



void io_mem_write(int io_index, target_phys_addr_t addr,

void io_mem_write(MemoryRegion *mr, target_phys_addr_t addr,

                  uint64_t val, unsigned size)

{

    memory_region_dispatch_write(io_mem_region[io_index], addr, val, size);

    memory_region_dispatch_write(mr, addr, val, size);

}



typedef struct MemoryRegionList MemoryRegionList;

softmmu_template.h

+24 −24
Original line number Diff line number Diff line
@@ -62,27 +62,27 @@ static inline DATA_TYPE glue(io_read, SUFFIX)(target_phys_addr_t physaddr, 
                                              void *retaddr)

{

    DATA_TYPE res;

    int index;

    index = physaddr & (IO_MEM_NB_ENTRIES - 1);

    MemoryRegion *mr = iotlb_to_region(physaddr);



    physaddr = (physaddr & TARGET_PAGE_MASK) + addr;

    env->mem_io_pc = (unsigned long)retaddr;

    if (index != io_mem_ram.ram_addr && index != io_mem_rom.ram_addr

        && index != io_mem_unassigned.ram_addr

        && index != io_mem_notdirty.ram_addr

    if (mr != &io_mem_ram && mr != &io_mem_rom

        && mr != &io_mem_unassigned

        && mr != &io_mem_notdirty

            && !can_do_io(env)) {

        cpu_io_recompile(env, retaddr);

    }



    env->mem_io_vaddr = addr;

#if SHIFT <= 2

    res = io_mem_read(index, physaddr, 1 << SHIFT);

    res = io_mem_read(mr, physaddr, 1 << SHIFT);

#else

#ifdef TARGET_WORDS_BIGENDIAN

    res = io_mem_read(index, physaddr, 4) << 32;

    res |= io_mem_read(index, physaddr + 4, 4);

    res = io_mem_read(mr, physaddr, 4) << 32;

    res |= io_mem_read(mr, physaddr + 4, 4);

#else

    res = io_mem_read(index, physaddr, 4);

    res |= io_mem_read(index, physaddr + 4, 4) << 32;

    res = io_mem_read(mr, physaddr, 4);

    res |= io_mem_read(mr, physaddr + 4, 4) << 32;

#endif

#endif /* SHIFT > 2 */

    return res;
@@ -110,7 +110,7 @@ DATA_TYPE REGPARM glue(glue(__ld, SUFFIX), MMUSUFFIX)(target_ulong addr, 
            if ((addr & (DATA_SIZE - 1)) != 0)

                goto do_unaligned_access;

            retaddr = GETPC();

            ioaddr = section_to_ioaddr(env->iotlb[mmu_idx][index]);

            ioaddr = env->iotlb[mmu_idx][index];

            res = glue(io_read, SUFFIX)(ioaddr, addr, retaddr);

        } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {

            /* slow unaligned access (it spans two pages or IO) */
@@ -164,7 +164,7 @@ static DATA_TYPE glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(target_ulong addr, 
            /* IO access */

            if ((addr & (DATA_SIZE - 1)) != 0)

                goto do_unaligned_access;

            ioaddr = section_to_ioaddr(env->iotlb[mmu_idx][index]);

            ioaddr = env->iotlb[mmu_idx][index];

            res = glue(io_read, SUFFIX)(ioaddr, addr, retaddr);

        } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {

        do_unaligned_access:
@@ -207,12 +207,12 @@ static inline void glue(io_write, SUFFIX)(target_phys_addr_t physaddr, 
                                          target_ulong addr,

                                          void *retaddr)

{

    int index;

    index = physaddr & (IO_MEM_NB_ENTRIES - 1);

    MemoryRegion *mr = iotlb_to_region(physaddr);



    physaddr = (physaddr & TARGET_PAGE_MASK) + addr;

    if (index != io_mem_ram.ram_addr && index != io_mem_rom.ram_addr

        && index != io_mem_unassigned.ram_addr

        && index != io_mem_notdirty.ram_addr

    if (mr != &io_mem_ram && mr != &io_mem_rom

        && mr != &io_mem_unassigned

        && mr != &io_mem_notdirty

            && !can_do_io(env)) {

        cpu_io_recompile(env, retaddr);

    }
@@ -220,14 +220,14 @@ static inline void glue(io_write, SUFFIX)(target_phys_addr_t physaddr, 
    env->mem_io_vaddr = addr;

    env->mem_io_pc = (unsigned long)retaddr;

#if SHIFT <= 2

    io_mem_write(index, physaddr, val, 1 << SHIFT);

    io_mem_write(mr, physaddr, val, 1 << SHIFT);

#else

#ifdef TARGET_WORDS_BIGENDIAN

    io_mem_write(index, physaddr, (val >> 32), 4);

    io_mem_write(index, physaddr + 4, (uint32_t)val, 4);

    io_mem_write(mr, physaddr, (val >> 32), 4);

    io_mem_write(mr, physaddr + 4, (uint32_t)val, 4);

#else

    io_mem_write(index, physaddr, (uint32_t)val, 4);

    io_mem_write(index, physaddr + 4, val >> 32, 4);

    io_mem_write(mr, physaddr, (uint32_t)val, 4);

    io_mem_write(mr, physaddr + 4, val >> 32, 4);

#endif

#endif /* SHIFT > 2 */

}
@@ -251,7 +251,7 @@ void REGPARM glue(glue(__st, SUFFIX), MMUSUFFIX)(target_ulong addr, 
            if ((addr & (DATA_SIZE - 1)) != 0)

                goto do_unaligned_access;

            retaddr = GETPC();

            ioaddr = section_to_ioaddr(env->iotlb[mmu_idx][index]);

            ioaddr = env->iotlb[mmu_idx][index];

            glue(io_write, SUFFIX)(ioaddr, val, addr, retaddr);

        } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {

        do_unaligned_access:
@@ -303,7 +303,7 @@ static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(target_ulong addr, 
            /* IO access */

            if ((addr & (DATA_SIZE - 1)) != 0)

                goto do_unaligned_access;

            ioaddr = section_to_ioaddr(env->iotlb[mmu_idx][index]);

            ioaddr = env->iotlb[mmu_idx][index];

            glue(io_write, SUFFIX)(ioaddr, val, addr, retaddr);

        } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {

        do_unaligned_access: