Commit d92fa2dc authored by Le Tan's avatar Le Tan Committed by Michael S. Tsirkin
Browse files

intel-iommu: add context-cache to cache context-entry 



Add context-cache to cache context-entry encountered on a page-walk. Each
VTDAddressSpace has a member of VTDContextCacheEntry which represents an entry
in the context-cache. Since devices with different bus_num and devfn have their
respective VTDAddressSpace, this will be a good way to reference the cached
entries.
Each VTDContextCacheEntry will have a context_cache_gen and the cached entry
is valid only when context_cache_gen equals IntelIOMMUState.context_cache_gen.

Signed-off-by: default avatarLe Tan <tamlokveer@gmail.com>
Reviewed-by: default avatarMichael S. Tsirkin <mst@redhat.com>
Signed-off-by: default avatarMichael S. Tsirkin <mst@redhat.com>
parent ed7b8fbc

hw/i386/intel_iommu.c

+160 −28
Original line number Diff line number Diff line
@@ -27,6 +27,7 @@ 
#ifdef DEBUG_INTEL_IOMMU

enum {

    DEBUG_GENERAL, DEBUG_CSR, DEBUG_INV, DEBUG_MMU, DEBUG_FLOG,

    DEBUG_CACHE,

};

#define VTD_DBGBIT(x)   (1 << DEBUG_##x)

static int vtd_dbgflags = VTD_DBGBIT(GENERAL) | VTD_DBGBIT(CSR);
@@ -131,6 +132,33 @@ static uint64_t vtd_set_clear_mask_quad(IntelIOMMUState *s, hwaddr addr, 
    return new_val;

}



/* Reset all the gen of VTDAddressSpace to zero and set the gen of

 * IntelIOMMUState to 1.

 */

static void vtd_reset_context_cache(IntelIOMMUState *s)

{

    VTDAddressSpace **pvtd_as;

    VTDAddressSpace *vtd_as;

    uint32_t bus_it;

    uint32_t devfn_it;



    VTD_DPRINTF(CACHE, "global context_cache_gen=1");

    for (bus_it = 0; bus_it < VTD_PCI_BUS_MAX; ++bus_it) {

        pvtd_as = s->address_spaces[bus_it];

        if (!pvtd_as) {

            continue;

        }

        for (devfn_it = 0; devfn_it < VTD_PCI_DEVFN_MAX; ++devfn_it) {

            vtd_as = pvtd_as[devfn_it];

            if (!vtd_as) {

                continue;

            }

            vtd_as->context_cache_entry.context_cache_gen = 0;

        }

    }

    s->context_cache_gen = 1;

}



/* Given the reg addr of both the message data and address, generate an

 * interrupt via MSI.

 */
@@ -651,11 +679,13 @@ static inline bool vtd_is_interrupt_addr(hwaddr addr) 
 * @is_write: The access is a write operation

 * @entry: IOMMUTLBEntry that contain the addr to be translated and result

 */

static void vtd_do_iommu_translate(IntelIOMMUState *s, uint8_t bus_num,

static void vtd_do_iommu_translate(VTDAddressSpace *vtd_as, uint8_t bus_num,

                                   uint8_t devfn, hwaddr addr, bool is_write,

                                   IOMMUTLBEntry *entry)

{

    IntelIOMMUState *s = vtd_as->iommu_state;

    VTDContextEntry ce;

    VTDContextCacheEntry *cc_entry = &vtd_as->context_cache_entry;

    uint64_t slpte;

    uint32_t level;

    uint16_t source_id = vtd_make_source_id(bus_num, devfn);
@@ -686,19 +716,36 @@ static void vtd_do_iommu_translate(IntelIOMMUState *s, uint8_t bus_num, 
            return;

        }

    }



    /* Try to fetch context-entry from cache first */

    if (cc_entry->context_cache_gen == s->context_cache_gen) {

        VTD_DPRINTF(CACHE, "hit context-cache bus %d devfn %d "

                    "(hi %"PRIx64 " lo %"PRIx64 " gen %"PRIu32 ")",

                    bus_num, devfn, cc_entry->context_entry.hi,

                    cc_entry->context_entry.lo, cc_entry->context_cache_gen);

        ce = cc_entry->context_entry;

        is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;

    } else {

        ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce);

        is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;

        if (ret_fr) {

            ret_fr = -ret_fr;

            if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) {

            VTD_DPRINTF(FLOG, "fault processing is disabled for DMA requests "

                        "through this context-entry (with FPD Set)");

                VTD_DPRINTF(FLOG, "fault processing is disabled for DMA "

                            "requests through this context-entry "

                            "(with FPD Set)");

            } else {

                vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write);

            }

            return;

        }

        /* Update context-cache */

        VTD_DPRINTF(CACHE, "update context-cache bus %d devfn %d "

                    "(hi %"PRIx64 " lo %"PRIx64 " gen %"PRIu32 "->%"PRIu32 ")",

                    bus_num, devfn, ce.hi, ce.lo,

                    cc_entry->context_cache_gen, s->context_cache_gen);

        cc_entry->context_entry = ce;

        cc_entry->context_cache_gen = s->context_cache_gen;

    }



    ret_fr = vtd_gpa_to_slpte(&ce, addr, is_write, &slpte, &level,

                              &reads, &writes);
@@ -729,6 +776,57 @@ static void vtd_root_table_setup(IntelIOMMUState *s) 
                (s->root_extended ? "(extended)" : ""));

}



static void vtd_context_global_invalidate(IntelIOMMUState *s)

{

    s->context_cache_gen++;

    if (s->context_cache_gen == VTD_CONTEXT_CACHE_GEN_MAX) {

        vtd_reset_context_cache(s);

    }

}



/* Do a context-cache device-selective invalidation.

 * @func_mask: FM field after shifting

 */

static void vtd_context_device_invalidate(IntelIOMMUState *s,

                                          uint16_t source_id,

                                          uint16_t func_mask)

{

    uint16_t mask;

    VTDAddressSpace **pvtd_as;

    VTDAddressSpace *vtd_as;

    uint16_t devfn;

    uint16_t devfn_it;



    switch (func_mask & 3) {

    case 0:

        mask = 0;   /* No bits in the SID field masked */

        break;

    case 1:

        mask = 4;   /* Mask bit 2 in the SID field */

        break;

    case 2:

        mask = 6;   /* Mask bit 2:1 in the SID field */

        break;

    case 3:

        mask = 7;   /* Mask bit 2:0 in the SID field */

        break;

    }

    VTD_DPRINTF(INV, "device-selective invalidation source 0x%"PRIx16

                    " mask %"PRIu16, source_id, mask);

    pvtd_as = s->address_spaces[VTD_SID_TO_BUS(source_id)];

    if (pvtd_as) {

        devfn = VTD_SID_TO_DEVFN(source_id);

        for (devfn_it = 0; devfn_it < VTD_PCI_DEVFN_MAX; ++devfn_it) {

            vtd_as = pvtd_as[devfn_it];

            if (vtd_as && ((devfn_it & mask) == (devfn & mask))) {

                VTD_DPRINTF(INV, "invalidate context-cahce of devfn 0x%"PRIx16,

                            devfn_it);

                vtd_as->context_cache_entry.context_cache_gen = 0;

            }

        }

    }

}



/* Context-cache invalidation

 * Returns the Context Actual Invalidation Granularity.

 * @val: the content of the CCMD_REG
@@ -739,24 +837,23 @@ static uint64_t vtd_context_cache_invalidate(IntelIOMMUState *s, uint64_t val) 
    uint64_t type = val & VTD_CCMD_CIRG_MASK;



    switch (type) {

    case VTD_CCMD_DOMAIN_INVL:

        VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16,

                    (uint16_t)VTD_CCMD_DID(val));

        /* Fall through */

    case VTD_CCMD_GLOBAL_INVL:

        VTD_DPRINTF(INV, "Global invalidation request");

        VTD_DPRINTF(INV, "global invalidation");

        caig = VTD_CCMD_GLOBAL_INVL_A;

        break;



    case VTD_CCMD_DOMAIN_INVL:

        VTD_DPRINTF(INV, "Domain-selective invalidation request");

        caig = VTD_CCMD_DOMAIN_INVL_A;

        vtd_context_global_invalidate(s);

        break;



    case VTD_CCMD_DEVICE_INVL:

        VTD_DPRINTF(INV, "Domain-selective invalidation request");

        caig = VTD_CCMD_DEVICE_INVL_A;

        vtd_context_device_invalidate(s, VTD_CCMD_SID(val), VTD_CCMD_FM(val));

        break;



    default:

        VTD_DPRINTF(GENERAL,

                    "error: wrong context-cache invalidation granularity");

        VTD_DPRINTF(GENERAL, "error: invalid granularity");

        caig = 0;

    }

    return caig;
@@ -994,6 +1091,38 @@ static bool vtd_process_wait_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc) 
    return true;

}



static bool vtd_process_context_cache_desc(IntelIOMMUState *s,

                                           VTDInvDesc *inv_desc)

{

    if ((inv_desc->lo & VTD_INV_DESC_CC_RSVD) || inv_desc->hi) {

        VTD_DPRINTF(GENERAL, "error: non-zero reserved field in Context-cache "

                    "Invalidate Descriptor");

        return false;

    }

    switch (inv_desc->lo & VTD_INV_DESC_CC_G) {

    case VTD_INV_DESC_CC_DOMAIN:

        VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16,

                    (uint16_t)VTD_INV_DESC_CC_DID(inv_desc->lo));

        /* Fall through */

    case VTD_INV_DESC_CC_GLOBAL:

        VTD_DPRINTF(INV, "global invalidation");

        vtd_context_global_invalidate(s);

        break;



    case VTD_INV_DESC_CC_DEVICE:

        vtd_context_device_invalidate(s, VTD_INV_DESC_CC_SID(inv_desc->lo),

                                      VTD_INV_DESC_CC_FM(inv_desc->lo));

        break;



    default:

        VTD_DPRINTF(GENERAL, "error: invalid granularity in Context-cache "

                    "Invalidate Descriptor hi 0x%"PRIx64  " lo 0x%"PRIx64,

                    inv_desc->hi, inv_desc->lo);

        return false;

    }

    return true;

}



static bool vtd_process_inv_desc(IntelIOMMUState *s)

{

    VTDInvDesc inv_desc;
@@ -1012,6 +1141,9 @@ static bool vtd_process_inv_desc(IntelIOMMUState *s) 
    case VTD_INV_DESC_CC:

        VTD_DPRINTF(INV, "Context-cache Invalidate Descriptor hi 0x%"PRIx64

                    " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo);

        if (!vtd_process_context_cache_desc(s, &inv_desc)) {

            return false;

        }

        break;



    case VTD_INV_DESC_IOTLB:
@@ -1453,8 +1585,6 @@ static IOMMUTLBEntry vtd_iommu_translate(MemoryRegion *iommu, hwaddr addr, 
{

    VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu);

    IntelIOMMUState *s = vtd_as->iommu_state;

    uint8_t bus_num = vtd_as->bus_num;

    uint8_t devfn = vtd_as->devfn;

    IOMMUTLBEntry ret = {

        .target_as = &address_space_memory,

        .iova = addr,
@@ -1472,13 +1602,13 @@ static IOMMUTLBEntry vtd_iommu_translate(MemoryRegion *iommu, hwaddr addr, 
        return ret;

    }



    vtd_do_iommu_translate(s, bus_num, devfn, addr, is_write, &ret);



    vtd_do_iommu_translate(vtd_as, vtd_as->bus_num, vtd_as->devfn, addr,

                           is_write, &ret);

    VTD_DPRINTF(MMU,

                "bus %"PRIu8 " slot %"PRIu8 " func %"PRIu8 " devfn %"PRIu8

                " gpa 0x%"PRIx64 " hpa 0x%"PRIx64, bus_num,

                VTD_PCI_SLOT(devfn), VTD_PCI_FUNC(devfn), devfn, addr,

                ret.translated_addr);

                " gpa 0x%"PRIx64 " hpa 0x%"PRIx64, vtd_as->bus_num,

                VTD_PCI_SLOT(vtd_as->devfn), VTD_PCI_FUNC(vtd_as->devfn),

                vtd_as->devfn, addr, ret.translated_addr);

    return ret;

}
@@ -1531,6 +1661,8 @@ static void vtd_init(IntelIOMMUState *s) 
             VTD_CAP_SAGAW;

    s->ecap = VTD_ECAP_QI | VTD_ECAP_IRO;



    vtd_reset_context_cache(s);



    /* Define registers with default values and bit semantics */

    vtd_define_long(s, DMAR_VER_REG, 0x10UL, 0, 0);

    vtd_define_quad(s, DMAR_CAP_REG, s->cap, 0, 0);

hw/i386/intel_iommu_internal.h

+16 −7
Original line number Diff line number Diff line
@@ -154,6 +154,9 @@ 
#define VTD_CCMD_DOMAIN_INVL_A      (2ULL << 59)

#define VTD_CCMD_DEVICE_INVL_A      (3ULL << 59)

#define VTD_CCMD_CAIG_MASK          (3ULL << 59)

#define VTD_CCMD_DID(val)           ((val) & VTD_DOMAIN_ID_MASK)

#define VTD_CCMD_SID(val)           (((val) >> 16) & 0xffffULL)

#define VTD_CCMD_FM(val)            (((val) >> 32) & 3ULL)



/* RTADDR_REG */

#define VTD_RTADDR_RTT              (1ULL << 11)
@@ -169,6 +172,7 @@ 
#define VTD_CAP_FRO                 (DMAR_FRCD_REG_OFFSET << 20)

#define VTD_CAP_NFR                 ((DMAR_FRCD_REG_NR - 1) << 40)

#define VTD_DOMAIN_ID_SHIFT         16  /* 16-bit domain id for 64K domains */

#define VTD_DOMAIN_ID_MASK          ((1UL << VTD_DOMAIN_ID_SHIFT) - 1)

#define VTD_CAP_ND                  (((VTD_DOMAIN_ID_SHIFT - 4) / 2) & 7ULL)

#define VTD_MGAW                    39  /* Maximum Guest Address Width */

#define VTD_CAP_MGAW                (((VTD_MGAW - 1) & 0x3fULL) << 16)
@@ -255,6 +259,8 @@ typedef enum VTDFaultReason { 
    VTD_FR_MAX,                 /* Guard */

} VTDFaultReason;



#define VTD_CONTEXT_CACHE_GEN_MAX       0xffffffffUL



/* Queued Invalidation Descriptor */

struct VTDInvDesc {

    uint64_t lo;
@@ -277,6 +283,16 @@ typedef struct VTDInvDesc VTDInvDesc; 
#define VTD_INV_DESC_WAIT_RSVD_LO       0Xffffff80ULL

#define VTD_INV_DESC_WAIT_RSVD_HI       3ULL



/* Masks for Context-cache Invalidation Descriptor */

#define VTD_INV_DESC_CC_G               (3ULL << 4)

#define VTD_INV_DESC_CC_GLOBAL          (1ULL << 4)

#define VTD_INV_DESC_CC_DOMAIN          (2ULL << 4)

#define VTD_INV_DESC_CC_DEVICE          (3ULL << 4)

#define VTD_INV_DESC_CC_DID(val)        (((val) >> 16) & VTD_DOMAIN_ID_MASK)

#define VTD_INV_DESC_CC_SID(val)        (((val) >> 32) & 0xffffUL)

#define VTD_INV_DESC_CC_FM(val)         (((val) >> 48) & 3UL)

#define VTD_INV_DESC_CC_RSVD            0xfffc00000000ffc0ULL



/* Pagesize of VTD paging structures, including root and context tables */

#define VTD_PAGE_SHIFT              12

#define VTD_PAGE_SIZE               (1ULL << VTD_PAGE_SHIFT)
@@ -301,13 +317,6 @@ typedef struct VTDRootEntry VTDRootEntry; 
#define VTD_ROOT_ENTRY_NR           (VTD_PAGE_SIZE / sizeof(VTDRootEntry))

#define VTD_ROOT_ENTRY_RSVD         (0xffeULL | ~VTD_HAW_MASK)



/* Context-Entry */

struct VTDContextEntry {

    uint64_t lo;

    uint64_t hi;

};

typedef struct VTDContextEntry VTDContextEntry;



/* Masks for struct VTDContextEntry */

/* lo */

#define VTD_CONTEXT_ENTRY_P         (1ULL << 0)

hw/pci-host/q35.c

+1 −0
Original line number Diff line number Diff line
@@ -368,6 +368,7 @@ static AddressSpace *q35_host_dma_iommu(PCIBus *bus, void *opaque, int devfn) 
        pvtd_as[devfn]->bus_num = (uint8_t)bus_num;

        pvtd_as[devfn]->devfn = (uint8_t)devfn;

        pvtd_as[devfn]->iommu_state = s;

        pvtd_as[devfn]->context_cache_entry.context_cache_gen = 0;

        memory_region_init_iommu(&pvtd_as[devfn]->iommu, OBJECT(s),

                                 &s->iommu_ops, "intel_iommu", UINT64_MAX);

        address_space_init(&pvtd_as[devfn]->as,

include/hw/i386/intel_iommu.h

+22 −0
Original line number Diff line number Diff line
@@ -37,20 +37,40 @@ 
#define VTD_PCI_DEVFN_MAX           256

#define VTD_PCI_SLOT(devfn)         (((devfn) >> 3) & 0x1f)

#define VTD_PCI_FUNC(devfn)         ((devfn) & 0x07)

#define VTD_SID_TO_BUS(sid)         (((sid) >> 8) && 0xff)

#define VTD_SID_TO_DEVFN(sid)       ((sid) & 0xff)



#define DMAR_REG_SIZE               0x230

#define VTD_HOST_ADDRESS_WIDTH      39

#define VTD_HAW_MASK                ((1ULL << VTD_HOST_ADDRESS_WIDTH) - 1)



typedef struct VTDContextEntry VTDContextEntry;

typedef struct VTDContextCacheEntry VTDContextCacheEntry;

typedef struct IntelIOMMUState IntelIOMMUState;

typedef struct VTDAddressSpace VTDAddressSpace;





/* Context-Entry */

struct VTDContextEntry {

    uint64_t lo;

    uint64_t hi;

};



struct VTDContextCacheEntry {

    /* The cache entry is obsolete if

     * context_cache_gen!=IntelIOMMUState.context_cache_gen

     */

    uint32_t context_cache_gen;

    struct VTDContextEntry context_entry;

};



struct VTDAddressSpace {

    uint8_t bus_num;

    uint8_t devfn;

    AddressSpace as;

    MemoryRegion iommu;

    IntelIOMMUState *iommu_state;

    VTDContextCacheEntry context_cache_entry;

};



/* The iommu (DMAR) device state struct */
@@ -82,6 +102,8 @@ struct IntelIOMMUState { 
    uint64_t cap;                   /* The value of capability reg */

    uint64_t ecap;                  /* The value of extended capability reg */



    uint32_t context_cache_gen;     /* Should be in [1,MAX] */



    MemoryRegionIOMMUOps iommu_ops;

    VTDAddressSpace **address_spaces[VTD_PCI_BUS_MAX];

};