Commit ac1970fb authored by Avi Kivity's avatar Avi Kivity
Browse files

memory: per-AddressSpace dispatch 



Currently we use a global radix tree to dispatch memory access.  This only
works with a single address space; to support multiple address spaces we
make the radix tree a member of AddressSpace (via an intermediate structure
AddressSpaceDispatch to avoid exposing too many internals).

A side effect is that address_space_io also gains a dispatch table.  When
we remove all the pre-memory-API I/O registrations, we can use that for
dispatching I/O and get rid of the original I/O dispatch.

Signed-off-by: default avatarAvi Kivity <avi@redhat.com>
parent 0e8a6d47

cputlb.c

+2 −1
Original line number Diff line number Diff line
@@ -21,6 +21,7 @@ 
#include "cpu.h"

#include "exec-all.h"

#include "memory.h"

#include "exec-memory.h"



#include "cputlb.h"
@@ -251,7 +252,7 @@ void tlb_set_page(CPUArchState *env, target_ulong vaddr, 
    if (size != TARGET_PAGE_SIZE) {

        tlb_add_large_page(env, vaddr, size);

    }

    section = phys_page_find(paddr >> TARGET_PAGE_BITS);

    section = phys_page_find(address_space_memory.dispatch, paddr >> TARGET_PAGE_BITS);

#if defined(DEBUG_TLB)

    printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx

           " prot=%x idx=%d pd=0x%08lx\n",

cputlb.h

+2 −1
Original line number Diff line number Diff line
@@ -26,7 +26,8 @@ void tlb_unprotect_code_phys(CPUArchState *env, ram_addr_t ram_addr, 
                             target_ulong vaddr);

void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start,

                           uintptr_t length);

MemoryRegionSection *phys_page_find(target_phys_addr_t index);

MemoryRegionSection *phys_page_find(struct AddressSpaceDispatch *d,

                                    target_phys_addr_t index);

void cpu_tlb_reset_dirty_all(ram_addr_t start1, ram_addr_t length);

void tlb_set_dirty(CPUArchState *env, target_ulong vaddr);

extern int tlb_flush_count;

exec.c

+107 −67
Original line number Diff line number Diff line
@@ -187,7 +187,6 @@ uintptr_t qemu_host_page_mask; 
static void *l1_map[V_L1_SIZE];



#if !defined(CONFIG_USER_ONLY)

typedef struct PhysPageEntry PhysPageEntry;



static MemoryRegionSection *phys_sections;

static unsigned phys_sections_nb, phys_sections_nb_alloc;
@@ -196,22 +195,12 @@ static uint16_t phys_section_notdirty; 
static uint16_t phys_section_rom;

static uint16_t phys_section_watch;



struct PhysPageEntry {

    uint16_t is_leaf : 1;

     /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */

    uint16_t ptr : 15;

};



/* Simple allocator for PhysPageEntry nodes */

static PhysPageEntry (*phys_map_nodes)[L2_SIZE];

static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc;



#define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)



/* This is a multi-level map on the physical address space.

   The bottom level has pointers to MemoryRegionSections.  */

static PhysPageEntry phys_map = { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };



static void io_mem_init(void);

static void memory_map_init(void);
@@ -459,18 +448,19 @@ static void phys_page_set_level(PhysPageEntry *lp, target_phys_addr_t *index, 
    }

}



static void phys_page_set(target_phys_addr_t index, target_phys_addr_t nb,

static void phys_page_set(AddressSpaceDispatch *d,

                          target_phys_addr_t index, target_phys_addr_t nb,

                          uint16_t leaf)

{

    /* Wildly overreserve - it doesn't matter much. */

    phys_map_node_reserve(3 * P_L2_LEVELS);



    phys_page_set_level(&phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);

    phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);

}



MemoryRegionSection *phys_page_find(target_phys_addr_t index)

MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, target_phys_addr_t index)

{

    PhysPageEntry lp = phys_map;

    PhysPageEntry lp = d->phys_map;

    PhysPageEntry *p;

    int i;

    uint16_t s_index = phys_section_unassigned;
@@ -1472,7 +1462,7 @@ void tb_invalidate_phys_addr(target_phys_addr_t addr) 
    ram_addr_t ram_addr;

    MemoryRegionSection *section;



    section = phys_page_find(addr >> TARGET_PAGE_BITS);

    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);

    if (!(memory_region_is_ram(section->mr)

          || (section->mr->rom_device && section->mr->readable))) {

        return;
@@ -2210,9 +2200,9 @@ static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level) 
    lp->ptr = PHYS_MAP_NODE_NIL;

}



static void destroy_all_mappings(void)

static void destroy_all_mappings(AddressSpaceDispatch *d)

{

    destroy_l2_mapping(&phys_map, P_L2_LEVELS - 1);

    destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);

    phys_map_nodes_reset();

}
@@ -2232,12 +2222,12 @@ static void phys_sections_clear(void) 
    phys_sections_nb = 0;

}



static void register_subpage(MemoryRegionSection *section)

static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)

{

    subpage_t *subpage;

    target_phys_addr_t base = section->offset_within_address_space

        & TARGET_PAGE_MASK;

    MemoryRegionSection *existing = phys_page_find(base >> TARGET_PAGE_BITS);

    MemoryRegionSection *existing = phys_page_find(d, base >> TARGET_PAGE_BITS);

    MemoryRegionSection subsection = {

        .offset_within_address_space = base,

        .size = TARGET_PAGE_SIZE,
@@ -2249,7 +2239,7 @@ static void register_subpage(MemoryRegionSection *section) 
    if (!(existing->mr->subpage)) {

        subpage = subpage_init(base);

        subsection.mr = &subpage->iomem;

        phys_page_set(base >> TARGET_PAGE_BITS, 1,

        phys_page_set(d, base >> TARGET_PAGE_BITS, 1,

                      phys_section_add(&subsection));

    } else {

        subpage = container_of(existing->mr, subpage_t, iomem);
@@ -2260,7 +2250,7 @@ static void register_subpage(MemoryRegionSection *section) 
}





static void register_multipage(MemoryRegionSection *section)

static void register_multipage(AddressSpaceDispatch *d, MemoryRegionSection *section)

{

    target_phys_addr_t start_addr = section->offset_within_address_space;

    ram_addr_t size = section->size;
@@ -2270,13 +2260,13 @@ static void register_multipage(MemoryRegionSection *section) 
    assert(size);



    addr = start_addr;

    phys_page_set(addr >> TARGET_PAGE_BITS, size >> TARGET_PAGE_BITS,

    phys_page_set(d, addr >> TARGET_PAGE_BITS, size >> TARGET_PAGE_BITS,

                  section_index);

}



void cpu_register_physical_memory_log(MemoryRegionSection *section,

                                      bool readonly)

static void mem_add(MemoryListener *listener, MemoryRegionSection *section)

{

    AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);

    MemoryRegionSection now = *section, remain = *section;



    if ((now.offset_within_address_space & ~TARGET_PAGE_MASK)
@@ -2284,7 +2274,7 @@ void cpu_register_physical_memory_log(MemoryRegionSection *section, 
        now.size = MIN(TARGET_PAGE_ALIGN(now.offset_within_address_space)

                       - now.offset_within_address_space,

                       now.size);

        register_subpage(&now);

        register_subpage(d, &now);

        remain.size -= now.size;

        remain.offset_within_address_space += now.size;

        remain.offset_within_region += now.size;
@@ -2293,10 +2283,10 @@ void cpu_register_physical_memory_log(MemoryRegionSection *section, 
        now = remain;

        if (remain.offset_within_region & ~TARGET_PAGE_MASK) {

            now.size = TARGET_PAGE_SIZE;

            register_subpage(&now);

            register_subpage(d, &now);

        } else {

            now.size &= TARGET_PAGE_MASK;

            register_multipage(&now);

            register_multipage(d, &now);

        }

        remain.size -= now.size;

        remain.offset_within_address_space += now.size;
@@ -2304,7 +2294,7 @@ void cpu_register_physical_memory_log(MemoryRegionSection *section, 
    }

    now = remain;

    if (now.size) {

        register_subpage(&now);

        register_subpage(d, &now);

    }

}
@@ -3155,11 +3145,17 @@ static void io_mem_init(void) 
                          "watch", UINT64_MAX);

}



static void mem_begin(MemoryListener *listener)

{

    AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);



    destroy_all_mappings(d);

    d->phys_map.ptr = PHYS_MAP_NODE_NIL;

}



static void core_begin(MemoryListener *listener)

{

    destroy_all_mappings();

    phys_sections_clear();

    phys_map.ptr = PHYS_MAP_NODE_NIL;

    phys_section_unassigned = dummy_section(&io_mem_unassigned);

    phys_section_notdirty = dummy_section(&io_mem_notdirty);

    phys_section_rom = dummy_section(&io_mem_rom);
@@ -3178,18 +3174,6 @@ static void tcg_commit(MemoryListener *listener) 
    }

}



static void core_region_add(MemoryListener *listener,

                            MemoryRegionSection *section)

{

    cpu_register_physical_memory_log(section, section->readonly);

}



static void core_region_nop(MemoryListener *listener,

                            MemoryRegionSection *section)

{

    cpu_register_physical_memory_log(section, section->readonly);

}



static void core_log_global_start(MemoryListener *listener)

{

    cpu_physical_memory_set_dirty_tracking(1);
@@ -3220,11 +3204,9 @@ static void io_region_del(MemoryListener *listener, 


static MemoryListener core_memory_listener = {

    .begin = core_begin,

    .region_add = core_region_add,

    .region_nop = core_region_nop,

    .log_global_start = core_log_global_start,

    .log_global_stop = core_log_global_stop,

    .priority = 0,

    .priority = 1,

};



static MemoryListener io_memory_listener = {
@@ -3237,6 +3219,21 @@ static MemoryListener tcg_memory_listener = { 
    .commit = tcg_commit,

};



void address_space_init_dispatch(AddressSpace *as)

{

    AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);



    d->phys_map  = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };

    d->listener = (MemoryListener) {

        .begin = mem_begin,

        .region_add = mem_add,

        .region_nop = mem_add,

        .priority = 0,

    };

    as->dispatch = d;

    memory_listener_register(&d->listener, as);

}



static void memory_map_init(void)

{

    system_memory = g_malloc(sizeof(*system_memory));
@@ -3321,9 +3318,10 @@ static void invalidate_and_set_dirty(target_phys_addr_t addr, 
    xen_modified_memory(addr, length);

}



void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,

                            int len, int is_write)

void address_space_rw(AddressSpace *as, target_phys_addr_t addr, uint8_t *buf,

                      int len, bool is_write)

{

    AddressSpaceDispatch *d = as->dispatch;

    int l;

    uint8_t *ptr;

    uint32_t val;
@@ -3335,7 +3333,7 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf, 
        l = (page + TARGET_PAGE_SIZE) - addr;

        if (l > len)

            l = len;

        section = phys_page_find(page >> TARGET_PAGE_BITS);

        section = phys_page_find(d, page >> TARGET_PAGE_BITS);



        if (is_write) {

            if (!memory_region_is_ram(section->mr)) {
@@ -3406,10 +3404,36 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf, 
    }

}



void address_space_write(AddressSpace *as, target_phys_addr_t addr,

                         const uint8_t *buf, int len)

{

    address_space_rw(as, addr, (uint8_t *)buf, len, true);

}



/**

 * address_space_read: read from an address space.

 *

 * @as: #AddressSpace to be accessed

 * @addr: address within that address space

 * @buf: buffer with the data transferred

 */

void address_space_read(AddressSpace *as, target_phys_addr_t addr, uint8_t *buf, int len)

{

    address_space_rw(as, addr, buf, len, false);

}





void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,

                            int len, int is_write)

{

    return address_space_rw(&address_space_memory, addr, buf, len, is_write);

}



/* used for ROM loading : can write in RAM and ROM */

void cpu_physical_memory_write_rom(target_phys_addr_t addr,

                                   const uint8_t *buf, int len)

{

    AddressSpaceDispatch *d = address_space_memory.dispatch;

    int l;

    uint8_t *ptr;

    target_phys_addr_t page;
@@ -3420,7 +3444,7 @@ void cpu_physical_memory_write_rom(target_phys_addr_t addr, 
        l = (page + TARGET_PAGE_SIZE) - addr;

        if (l > len)

            l = len;

        section = phys_page_find(page >> TARGET_PAGE_BITS);

        section = phys_page_find(d, page >> TARGET_PAGE_BITS);



        if (!(memory_region_is_ram(section->mr) ||

              memory_region_is_romd(section->mr))) {
@@ -3494,10 +3518,12 @@ static void cpu_notify_map_clients(void) 
 * Use cpu_register_map_client() to know when retrying the map operation is

 * likely to succeed.

 */

void *cpu_physical_memory_map(target_phys_addr_t addr,

void *address_space_map(AddressSpace *as,

                        target_phys_addr_t addr,

                        target_phys_addr_t *plen,

                              int is_write)

                        bool is_write)

{

    AddressSpaceDispatch *d = as->dispatch;

    target_phys_addr_t len = *plen;

    target_phys_addr_t todo = 0;

    int l;
@@ -3512,7 +3538,7 @@ void *cpu_physical_memory_map(target_phys_addr_t addr, 
        l = (page + TARGET_PAGE_SIZE) - addr;

        if (l > len)

            l = len;

        section = phys_page_find(page >> TARGET_PAGE_BITS);

        section = phys_page_find(d, page >> TARGET_PAGE_BITS);



        if (!(memory_region_is_ram(section->mr) && !section->readonly)) {

            if (todo || bounce.buffer) {
@@ -3522,7 +3548,7 @@ void *cpu_physical_memory_map(target_phys_addr_t addr, 
            bounce.addr = addr;

            bounce.len = l;

            if (!is_write) {

                cpu_physical_memory_read(addr, bounce.buffer, l);

                address_space_read(as, addr, bounce.buffer, l);

            }



            *plen = l;
@@ -3543,11 +3569,11 @@ void *cpu_physical_memory_map(target_phys_addr_t addr, 
    return ret;

}



/* Unmaps a memory region previously mapped by cpu_physical_memory_map().

/* Unmaps a memory region previously mapped by address_space_map().

 * Will also mark the memory as dirty if is_write == 1.  access_len gives

 * the amount of memory that was actually read or written by the caller.

 */

void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,

void address_space_unmap(AddressSpace *as, void *buffer, target_phys_addr_t len,

                         int is_write, target_phys_addr_t access_len)

{

    if (buffer != bounce.buffer) {
@@ -3569,13 +3595,26 @@ void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len, 
        return;

    }

    if (is_write) {

        cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);

        address_space_write(as, bounce.addr, bounce.buffer, access_len);

    }

    qemu_vfree(bounce.buffer);

    bounce.buffer = NULL;

    cpu_notify_map_clients();

}



void *cpu_physical_memory_map(target_phys_addr_t addr,

                              target_phys_addr_t *plen,

                              int is_write)

{

    return address_space_map(&address_space_memory, addr, plen, is_write);

}



void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,

                               int is_write, target_phys_addr_t access_len)

{

    return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);

}



/* warning: addr must be aligned */

static inline uint32_t ldl_phys_internal(target_phys_addr_t addr,

                                         enum device_endian endian)
@@ -3584,7 +3623,7 @@ static inline uint32_t ldl_phys_internal(target_phys_addr_t addr, 
    uint32_t val;

    MemoryRegionSection *section;



    section = phys_page_find(addr >> TARGET_PAGE_BITS);

    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);



    if (!(memory_region_is_ram(section->mr) ||

          memory_region_is_romd(section->mr))) {
@@ -3643,7 +3682,7 @@ static inline uint64_t ldq_phys_internal(target_phys_addr_t addr, 
    uint64_t val;

    MemoryRegionSection *section;



    section = phys_page_find(addr >> TARGET_PAGE_BITS);

    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);



    if (!(memory_region_is_ram(section->mr) ||

          memory_region_is_romd(section->mr))) {
@@ -3710,7 +3749,7 @@ static inline uint32_t lduw_phys_internal(target_phys_addr_t addr, 
    uint64_t val;

    MemoryRegionSection *section;



    section = phys_page_find(addr >> TARGET_PAGE_BITS);

    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);



    if (!(memory_region_is_ram(section->mr) ||

          memory_region_is_romd(section->mr))) {
@@ -3769,7 +3808,7 @@ void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val) 
    uint8_t *ptr;

    MemoryRegionSection *section;



    section = phys_page_find(addr >> TARGET_PAGE_BITS);

    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);



    if (!memory_region_is_ram(section->mr) || section->readonly) {

        addr = memory_region_section_addr(section, addr);
@@ -3801,7 +3840,7 @@ void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val) 
    uint8_t *ptr;

    MemoryRegionSection *section;



    section = phys_page_find(addr >> TARGET_PAGE_BITS);

    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);



    if (!memory_region_is_ram(section->mr) || section->readonly) {

        addr = memory_region_section_addr(section, addr);
@@ -3830,7 +3869,7 @@ static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val, 
    uint8_t *ptr;

    MemoryRegionSection *section;



    section = phys_page_find(addr >> TARGET_PAGE_BITS);

    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);



    if (!memory_region_is_ram(section->mr) || section->readonly) {

        addr = memory_region_section_addr(section, addr);
@@ -3897,7 +3936,7 @@ static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val, 
    uint8_t *ptr;

    MemoryRegionSection *section;



    section = phys_page_find(addr >> TARGET_PAGE_BITS);

    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);



    if (!memory_region_is_ram(section->mr) || section->readonly) {

        addr = memory_region_section_addr(section, addr);
@@ -4133,7 +4172,8 @@ bool cpu_physical_memory_is_io(target_phys_addr_t phys_addr) 
{

    MemoryRegionSection *section;



    section = phys_page_find(phys_addr >> TARGET_PAGE_BITS);

    section = phys_page_find(address_space_memory.dispatch,

                             phys_addr >> TARGET_PAGE_BITS);



    return !(memory_region_is_ram(section->mr) ||

             memory_region_is_romd(section->mr));

memory-internal.h

+20 −2
Original line number Diff line number Diff line
@@ -22,6 +22,26 @@ 
#ifndef CONFIG_USER_ONLY

#include "hw/xen.h"



typedef struct PhysPageEntry PhysPageEntry;



struct PhysPageEntry {

    uint16_t is_leaf : 1;

     /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */

    uint16_t ptr : 15;

};



typedef struct AddressSpaceDispatch AddressSpaceDispatch;



struct AddressSpaceDispatch {

    /* This is a multi-level map on the physical address space.

     * The bottom level has pointers to MemoryRegionSections.

     */

    PhysPageEntry phys_map;

    MemoryListener listener;

};



void address_space_init_dispatch(AddressSpace *as);



ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,

                                   MemoryRegion *mr);

ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr);
@@ -30,8 +50,6 @@ void qemu_ram_free_from_ptr(ram_addr_t addr); 


struct MemoryRegion;

struct MemoryRegionSection;

void cpu_register_physical_memory_log(struct MemoryRegionSection *section,

                                      bool readonly);



void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size);

void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size);

memory.c

+1 −0
Original line number Diff line number Diff line
@@ -1539,6 +1539,7 @@ void address_space_init(AddressSpace *as, MemoryRegion *root) 
    QTAILQ_INSERT_TAIL(&address_spaces, as, address_spaces_link);

    as->name = NULL;

    memory_region_transaction_commit();

    address_space_init_dispatch(as);

}



uint64_t io_mem_read(MemoryRegion *mr, target_phys_addr_t addr, unsigned size)