Commit 3cc8f884 authored by Paolo Bonzini's avatar Paolo Bonzini
Browse files

memory: try to inline constant-length reads 



memcpy can take a large amount of time for small reads and writes.
Handle the common case of reading s/g descriptors from memory (there
is no corresponding "write" case that is as common, because writes
often use address_space_st* functions) by inlining the relevant
parts of address_space_read into the caller.

Signed-off-by: default avatarPaolo Bonzini <pbonzini@redhat.com>
parent 1619d1fe

exec.c

+2 −13
Original line number Diff line number Diff line
@@ -390,17 +390,6 @@ address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *x 
    return section;

}



static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)

{

    if (is_write) {

        return memory_region_is_ram(mr) && !mr->readonly;

    } else {

        return memory_region_is_ram(mr) || memory_region_is_romd(mr);

    }



    return false;

}



/* Called from RCU critical section */

MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,

                                      hwaddr *xlat, hwaddr *plen,
@@ -2632,8 +2621,8 @@ MemTxResult address_space_read_continue(AddressSpace *as, hwaddr addr, 
    return result;

}



MemTxResult address_space_read(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,

                               uint8_t *buf, int len)

MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,

                                    MemTxAttrs attrs, uint8_t *buf, int len)

{

    hwaddr l;

    hwaddr addr1;

include/exec/memory.h

+57 −17
Original line number Diff line number Diff line
@@ -1234,23 +1234,7 @@ MemTxResult address_space_write(AddressSpace *as, hwaddr addr, 
                                MemTxAttrs attrs,

                                const uint8_t *buf, int len);



/**

 * address_space_read: read from an address space.

 *

 * Return a MemTxResult indicating whether the operation succeeded

 * or failed (eg unassigned memory, device rejected the transaction,

 * IOMMU fault).

 *

 * @as: #AddressSpace to be accessed

 * @addr: address within that address space

 * @attrs: memory transaction attributes

 * @buf: buffer with the data transferred

 */

MemTxResult address_space_read(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,

                               uint8_t *buf, int len);



/**

 * address_space_ld*: load from an address space

/* address_space_ld*: load from an address space

 * address_space_st*: store to an address space

 *

 * These functions perform a load or store of the byte, word,
@@ -1385,6 +1369,62 @@ MemTxResult address_space_read_continue(AddressSpace *as, hwaddr addr, 
                                        MemTxAttrs attrs, uint8_t *buf,

                                        int len, hwaddr addr1, hwaddr l,

					MemoryRegion *mr);

MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,

                                    MemTxAttrs attrs, uint8_t *buf, int len);

void *qemu_get_ram_ptr(ram_addr_t addr);



static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)

{

    if (is_write) {

        return memory_region_is_ram(mr) && !mr->readonly;

    } else {

        return memory_region_is_ram(mr) || memory_region_is_romd(mr);

    }



    return false;

}



/**

 * address_space_read: read from an address space.

 *

 * Return a MemTxResult indicating whether the operation succeeded

 * or failed (eg unassigned memory, device rejected the transaction,

 * IOMMU fault).

 *

 * @as: #AddressSpace to be accessed

 * @addr: address within that address space

 * @attrs: memory transaction attributes

 * @buf: buffer with the data transferred

 */

static inline __attribute__((__always_inline__))

MemTxResult address_space_read(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,

                               uint8_t *buf, int len)

{

    MemTxResult result = MEMTX_OK;

    hwaddr l, addr1;

    void *ptr;

    MemoryRegion *mr;



    if (__builtin_constant_p(len)) {

        if (len) {

            rcu_read_lock();

            l = len;

            mr = address_space_translate(as, addr, &addr1, &l, false);

            if (len == l && memory_access_is_direct(mr, false)) {

                addr1 += memory_region_get_ram_addr(mr);

                ptr = qemu_get_ram_ptr(addr1);

                memcpy(buf, ptr, len);

            } else {

                result = address_space_read_continue(as, addr, attrs, buf, len,

                                                     addr1, l, mr);

            }

            rcu_read_unlock();

        }

    } else {

        result = address_space_read_full(as, addr, attrs, buf, len);

    }

    return result;

}



#endif

include/exec/ram_addr.h

+0 −1
Original line number Diff line number Diff line
@@ -73,7 +73,6 @@ ram_addr_t qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size, 
                                     MemoryRegion *mr, Error **errp);

int qemu_get_ram_fd(ram_addr_t addr);

void *qemu_get_ram_block_host_ptr(ram_addr_t addr);

void *qemu_get_ram_ptr(ram_addr_t addr);

void qemu_ram_free(ram_addr_t addr);



int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp);