Commit 71b9f394 authored by Richard Henderson's avatar Richard Henderson Committed by Peter Maydell
Browse files

target/arm: Add mte helpers for sve scalar + int stores 



Because the elements are sequential, we can eliminate many tests all
at once when the tag hits TCMA, or if the page(s) are not Tagged.

Reviewed-by: default avatarPeter Maydell <peter.maydell@linaro.org>
Signed-off-by: default avatarRichard Henderson <richard.henderson@linaro.org>
Message-id: 20200626033144.790098-35-richard.henderson@linaro.org
Signed-off-by: default avatarPeter Maydell <peter.maydell@linaro.org>
parent 206adacf

target/arm/helper-sve.h

+47 −0
Original line number Diff line number Diff line
@@ -1363,6 +1363,53 @@ DEF_HELPER_FLAGS_4(sve_st1hd_be_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32) 
DEF_HELPER_FLAGS_4(sve_st1sd_le_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st1sd_be_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_st1bb_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st2bb_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st3bb_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st4bb_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_st1hh_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st2hh_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st3hh_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st4hh_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_st1hh_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st2hh_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st3hh_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st4hh_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_st1ss_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st2ss_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st3ss_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st4ss_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_st1ss_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st2ss_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st3ss_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st4ss_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_st1dd_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st2dd_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st3dd_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st4dd_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_st1dd_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st2dd_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st3dd_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st4dd_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_st1bh_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st1bs_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st1bd_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_st1hs_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st1hd_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st1hs_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st1hd_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_st1sd_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st1sd_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_6(sve_ldbsu_zsu, TCG_CALL_NO_WG,

                   void, env, ptr, ptr, ptr, tl, i32)

DEF_HELPER_FLAGS_6(sve_ldhsu_le_zsu, TCG_CALL_NO_WG,

target/arm/sve_helper.c

+72 −23
Original line number Diff line number Diff line
@@ -5022,11 +5022,12 @@ DO_LDFF1_LDNF1_2(dd, MO_64, MO_64) 
 */



static inline QEMU_ALWAYS_INLINE

void sve_stN_r(CPUARMState *env, uint64_t *vg, target_ulong addr, uint32_t desc,

               const uintptr_t retaddr, const int esz,

               const int msz, const int N,

void sve_stN_r(CPUARMState *env, uint64_t *vg, target_ulong addr,

               uint32_t desc, const uintptr_t retaddr,

               const int esz, const int msz, const int N, uint32_t mtedesc,

               sve_ldst1_host_fn *host_fn,

               sve_ldst1_tlb_fn *tlb_fn)

               sve_ldst1_tlb_fn *tlb_fn,

               sve_cont_ldst_mte_check_fn *mte_check_fn)

{

    const unsigned rd = simd_data(desc);

    const intptr_t reg_max = simd_oprsz(desc);
@@ -5048,7 +5049,14 @@ void sve_stN_r(CPUARMState *env, uint64_t *vg, target_ulong addr, uint32_t desc, 
    sve_cont_ldst_watchpoints(&info, env, vg, addr, 1 << esz, N << msz,

                              BP_MEM_WRITE, retaddr);



    /* TODO: MTE check. */

    /*

     * Handle mte checks for all active elements.

     * Since TBI must be set for MTE, !mtedesc => !mte_active.

     */

    if (mte_check_fn && mtedesc) {

        mte_check_fn(&info, env, vg, addr, 1 << esz, N << msz,

                     mtedesc, retaddr);

    }



    flags = info.page[0].flags | info.page[1].flags;

    if (unlikely(flags != 0)) {
@@ -5142,11 +5150,40 @@ void sve_stN_r(CPUARMState *env, uint64_t *vg, target_ulong addr, uint32_t desc, 
    }

}



static inline QEMU_ALWAYS_INLINE

void sve_stN_r_mte(CPUARMState *env, uint64_t *vg, target_ulong addr,

                   uint32_t desc, const uintptr_t ra,

                   const int esz, const int msz, const int N,

                   sve_ldst1_host_fn *host_fn,

                   sve_ldst1_tlb_fn *tlb_fn)

{

    uint32_t mtedesc = desc >> (SIMD_DATA_SHIFT + SVE_MTEDESC_SHIFT);

    int bit55 = extract64(addr, 55, 1);



    /* Remove mtedesc from the normal sve descriptor. */

    desc = extract32(desc, 0, SIMD_DATA_SHIFT + SVE_MTEDESC_SHIFT);



    /* Perform gross MTE suppression early. */

    if (!tbi_check(desc, bit55) ||

        tcma_check(desc, bit55, allocation_tag_from_addr(addr))) {

        mtedesc = 0;

    }



    sve_stN_r(env, vg, addr, desc, ra, esz, msz, N, mtedesc, host_fn, tlb_fn,

              N == 1 ? sve_cont_ldst_mte_check1 : sve_cont_ldst_mte_checkN);

}



#define DO_STN_1(N, NAME, ESZ)                                          \

void HELPER(sve_st##N##NAME##_r)(CPUARMState *env, void *vg,            \

                                 target_ulong addr, uint32_t desc)      \

{                                                                       \

    sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MO_8, N,           \

    sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MO_8, N, 0,            \

              sve_st1##NAME##_host, sve_st1##NAME##_tlb, NULL);         \

}                                                                       \

void HELPER(sve_st##N##NAME##_r_mte)(CPUARMState *env, void *vg,        \

                                     target_ulong addr, uint32_t desc)  \

{                                                                       \

    sve_stN_r_mte(env, vg, addr, desc, GETPC(), ESZ, MO_8, N,           \

                  sve_st1##NAME##_host, sve_st1##NAME##_tlb);           \

}
@@ -5154,13 +5191,25 @@ void HELPER(sve_st##N##NAME##_r)(CPUARMState *env, void *vg, \ 
void HELPER(sve_st##N##NAME##_le_r)(CPUARMState *env, void *vg,         \

                                    target_ulong addr, uint32_t desc)   \

{                                                                       \

    sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, N,              \

              sve_st1##NAME##_le_host, sve_st1##NAME##_le_tlb);       \

    sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, N, 0,             \

              sve_st1##NAME##_le_host, sve_st1##NAME##_le_tlb, NULL);   \

}                                                                       \

void HELPER(sve_st##N##NAME##_be_r)(CPUARMState *env, void *vg,         \

                                    target_ulong addr, uint32_t desc)   \

{                                                                       \

    sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, N,              \

    sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, N, 0,             \

              sve_st1##NAME##_be_host, sve_st1##NAME##_be_tlb, NULL);   \

}                                                                       \

void HELPER(sve_st##N##NAME##_le_r_mte)(CPUARMState *env, void *vg,     \

                                        target_ulong addr, uint32_t desc) \

{                                                                       \

    sve_stN_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ, N,            \

                  sve_st1##NAME##_le_host, sve_st1##NAME##_le_tlb);     \

}                                                                       \

void HELPER(sve_st##N##NAME##_be_r_mte)(CPUARMState *env, void *vg,     \

                                        target_ulong addr, uint32_t desc) \

{                                                                       \

    sve_stN_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ, N,            \

                  sve_st1##NAME##_be_host, sve_st1##NAME##_be_tlb);     \

}

target/arm/translate-sve.c

+107 −55
Original line number Diff line number Diff line
@@ -5018,8 +5018,8 @@ static bool trans_LD1R_zpri(DisasContext *s, arg_rpri_load *a) 
static void do_st_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr,

                      int msz, int esz, int nreg)

{

    static gen_helper_gvec_mem * const fn_single[2][4][4] = {

        { { gen_helper_sve_st1bb_r,

    static gen_helper_gvec_mem * const fn_single[2][2][4][4] = {

        { { { gen_helper_sve_st1bb_r,

              gen_helper_sve_st1bh_r,

              gen_helper_sve_st1bs_r,

              gen_helper_sve_st1bd_r },
@@ -5044,10 +5044,37 @@ static void do_st_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr, 
              gen_helper_sve_st1ss_be_r,

              gen_helper_sve_st1sd_be_r },

            { NULL, NULL, NULL,

            gen_helper_sve_st1dd_be_r } },

              gen_helper_sve_st1dd_be_r } } },



        { { { gen_helper_sve_st1bb_r_mte,

              gen_helper_sve_st1bh_r_mte,

              gen_helper_sve_st1bs_r_mte,

              gen_helper_sve_st1bd_r_mte },

            { NULL,

              gen_helper_sve_st1hh_le_r_mte,

              gen_helper_sve_st1hs_le_r_mte,

              gen_helper_sve_st1hd_le_r_mte },

            { NULL, NULL,

              gen_helper_sve_st1ss_le_r_mte,

              gen_helper_sve_st1sd_le_r_mte },

            { NULL, NULL, NULL,

              gen_helper_sve_st1dd_le_r_mte } },

          { { gen_helper_sve_st1bb_r_mte,

              gen_helper_sve_st1bh_r_mte,

              gen_helper_sve_st1bs_r_mte,

              gen_helper_sve_st1bd_r_mte },

            { NULL,

              gen_helper_sve_st1hh_be_r_mte,

              gen_helper_sve_st1hs_be_r_mte,

              gen_helper_sve_st1hd_be_r_mte },

            { NULL, NULL,

              gen_helper_sve_st1ss_be_r_mte,

              gen_helper_sve_st1sd_be_r_mte },

            { NULL, NULL, NULL,

              gen_helper_sve_st1dd_be_r_mte } } },

    };

    static gen_helper_gvec_mem * const fn_multiple[2][3][4] = {

        { { gen_helper_sve_st2bb_r,

    static gen_helper_gvec_mem * const fn_multiple[2][2][3][4] = {

        { { { gen_helper_sve_st2bb_r,

              gen_helper_sve_st2hh_le_r,

              gen_helper_sve_st2ss_le_r,

              gen_helper_sve_st2dd_le_r },
@@ -5070,21 +5097,46 @@ static void do_st_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr, 
            { gen_helper_sve_st4bb_r,

              gen_helper_sve_st4hh_be_r,

              gen_helper_sve_st4ss_be_r,

            gen_helper_sve_st4dd_be_r } },

              gen_helper_sve_st4dd_be_r } } },

        { { { gen_helper_sve_st2bb_r_mte,

              gen_helper_sve_st2hh_le_r_mte,

              gen_helper_sve_st2ss_le_r_mte,

              gen_helper_sve_st2dd_le_r_mte },

            { gen_helper_sve_st3bb_r_mte,

              gen_helper_sve_st3hh_le_r_mte,

              gen_helper_sve_st3ss_le_r_mte,

              gen_helper_sve_st3dd_le_r_mte },

            { gen_helper_sve_st4bb_r_mte,

              gen_helper_sve_st4hh_le_r_mte,

              gen_helper_sve_st4ss_le_r_mte,

              gen_helper_sve_st4dd_le_r_mte } },

          { { gen_helper_sve_st2bb_r_mte,

              gen_helper_sve_st2hh_be_r_mte,

              gen_helper_sve_st2ss_be_r_mte,

              gen_helper_sve_st2dd_be_r_mte },

            { gen_helper_sve_st3bb_r_mte,

              gen_helper_sve_st3hh_be_r_mte,

              gen_helper_sve_st3ss_be_r_mte,

              gen_helper_sve_st3dd_be_r_mte },

            { gen_helper_sve_st4bb_r_mte,

              gen_helper_sve_st4hh_be_r_mte,

              gen_helper_sve_st4ss_be_r_mte,

              gen_helper_sve_st4dd_be_r_mte } } },

    };

    gen_helper_gvec_mem *fn;

    int be = s->be_data == MO_BE;



    if (nreg == 0) {

        /* ST1 */

        fn = fn_single[be][msz][esz];

        fn = fn_single[s->mte_active[0]][be][msz][esz];

        nreg = 1;

    } else {

        /* ST2, ST3, ST4 -- msz == esz, enforced by encoding */

        assert(msz == esz);

        fn = fn_multiple[be][nreg - 1][msz];

        fn = fn_multiple[s->mte_active[0]][be][nreg - 1][msz];

    }

    assert(fn != NULL);

    do_mem_zpa(s, zt, pg, addr, msz_dtype(s, msz), 0, true, fn);

    do_mem_zpa(s, zt, pg, addr, msz_dtype(s, msz), nreg, true, fn);

}



static bool trans_ST_zprr(DisasContext *s, arg_rprr_store *a)