Commit aa13f7c3 authored by Richard Henderson's avatar Richard Henderson Committed by Peter Maydell
Browse files

target/arm: Add mte helpers for sve scalar + int ff/nf loads 



Because the elements are sequential, we can eliminate many tests all
at once when the tag hits TCMA, or if the page(s) are not Tagged.

Reviewed-by: default avatarPeter Maydell <peter.maydell@linaro.org>
Signed-off-by: default avatarRichard Henderson <richard.henderson@linaro.org>
Message-id: 20200626033144.790098-36-richard.henderson@linaro.org
Signed-off-by: default avatarPeter Maydell <peter.maydell@linaro.org>
parent 71b9f394

target/arm/helper-sve.h

+98 −0
Original line number Diff line number Diff line
@@ -1285,6 +1285,55 @@ DEF_HELPER_FLAGS_4(sve_ldff1sds_be_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32) 
DEF_HELPER_FLAGS_4(sve_ldff1dd_le_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldff1dd_be_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_ldff1bb_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldff1bhu_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldff1bsu_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldff1bdu_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldff1bhs_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldff1bss_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldff1bds_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_ldff1hh_le_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldff1hsu_le_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldff1hdu_le_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldff1hss_le_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldff1hds_le_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_ldff1hh_be_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldff1hsu_be_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldff1hdu_be_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldff1hss_be_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldff1hds_be_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_ldff1ss_le_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldff1sdu_le_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldff1sds_le_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_ldff1ss_be_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldff1sdu_be_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldff1sds_be_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_ldff1dd_le_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldff1dd_be_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_ldnf1bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldnf1bhu_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldnf1bsu_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
@@ -1316,6 +1365,55 @@ DEF_HELPER_FLAGS_4(sve_ldnf1sds_be_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32) 
DEF_HELPER_FLAGS_4(sve_ldnf1dd_le_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldnf1dd_be_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_ldnf1bb_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldnf1bhu_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldnf1bsu_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldnf1bdu_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldnf1bhs_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldnf1bss_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldnf1bds_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_ldnf1hh_le_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldnf1hsu_le_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldnf1hdu_le_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldnf1hss_le_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldnf1hds_le_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_ldnf1hh_be_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldnf1hsu_be_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldnf1hdu_be_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldnf1hss_be_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldnf1hds_be_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_ldnf1ss_le_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldnf1sdu_le_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldnf1sds_le_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_ldnf1ss_be_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldnf1sdu_be_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldnf1sds_be_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_ldnf1dd_le_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_ldnf1dd_be_r_mte, TCG_CALL_NO_WG,

                   void, env, ptr, tl, i32)



DEF_HELPER_FLAGS_4(sve_st1bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st2bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

DEF_HELPER_FLAGS_4(sve_st3bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

target/arm/sve_helper.c

+87 −12
Original line number Diff line number Diff line
@@ -4794,7 +4794,7 @@ static void record_fault(CPUARMState *env, uintptr_t i, uintptr_t oprsz) 
 */

static inline QEMU_ALWAYS_INLINE

void sve_ldnfff1_r(CPUARMState *env, void *vg, const target_ulong addr,

                   uint32_t desc, const uintptr_t retaddr,

                   uint32_t desc, const uintptr_t retaddr, uint32_t mtedesc,

                   const int esz, const int msz, const SVEContFault fault,

                   sve_ldst1_host_fn *host_fn,

                   sve_ldst1_tlb_fn *tlb_fn)
@@ -4826,13 +4826,25 @@ void sve_ldnfff1_r(CPUARMState *env, void *vg, const target_ulong addr, 
    mem_off = info.mem_off_first[0];

    flags = info.page[0].flags;



    /*

     * Disable MTE checking if the Tagged bit is not set.  Since TBI must

     * be set within MTEDESC for MTE, !mtedesc => !mte_active.

     */

    if (arm_tlb_mte_tagged(&info.page[0].attrs)) {

        mtedesc = 0;

    }



    if (fault == FAULT_FIRST) {

        /* Trapping mte check for the first-fault element.  */

        if (mtedesc) {

            mte_check1(env, mtedesc, addr + mem_off, retaddr);

        }



        /*

         * Special handling of the first active element,

         * if it crosses a page boundary or is MMIO.

         */

        bool is_split = mem_off == info.mem_off_split;

        /* TODO: MTE check. */

        if (unlikely(flags != 0) || unlikely(is_split)) {

            /*

             * Use the slow path for cross-page handling.
@@ -4868,7 +4880,9 @@ void sve_ldnfff1_r(CPUARMState *env, void *vg, const target_ulong addr, 
                /* Watchpoint hit, see below. */

                goto do_fault;

            }

            /* TODO: MTE check. */

            if (mtedesc && !mte_probe1(env, mtedesc, addr + mem_off)) {

                goto do_fault;

            }

            /*

             * Use the slow path for cross-page handling.

             * This is RAM, without a watchpoint, and will not trap.
@@ -4916,7 +4930,9 @@ void sve_ldnfff1_r(CPUARMState *env, void *vg, const target_ulong addr, 
                     & BP_MEM_READ)) {

                    goto do_fault;

                }

                /* TODO: MTE check. */

                if (mtedesc && !mte_probe1(env, mtedesc, addr + mem_off)) {

                    goto do_fault;

                }

                host_fn(vd, reg_off, host + mem_off);

            }

            reg_off += 1 << esz;
@@ -4954,17 +4970,52 @@ void sve_ldnfff1_r(CPUARMState *env, void *vg, const target_ulong addr, 
    record_fault(env, reg_off, reg_max);

}



static inline QEMU_ALWAYS_INLINE

void sve_ldnfff1_r_mte(CPUARMState *env, void *vg, target_ulong addr,

                       uint32_t desc, const uintptr_t retaddr,

                       const int esz, const int msz, const SVEContFault fault,

                       sve_ldst1_host_fn *host_fn,

                       sve_ldst1_tlb_fn *tlb_fn)

{

    uint32_t mtedesc = desc >> (SIMD_DATA_SHIFT + SVE_MTEDESC_SHIFT);

    int bit55 = extract64(addr, 55, 1);



    /* Remove mtedesc from the normal sve descriptor. */

    desc = extract32(desc, 0, SIMD_DATA_SHIFT + SVE_MTEDESC_SHIFT);



    /* Perform gross MTE suppression early. */

    if (!tbi_check(desc, bit55) ||

        tcma_check(desc, bit55, allocation_tag_from_addr(addr))) {

        mtedesc = 0;

    }



    sve_ldnfff1_r(env, vg, addr, desc, retaddr, mtedesc,

                  esz, msz, fault, host_fn, tlb_fn);

}



#define DO_LDFF1_LDNF1_1(PART, ESZ)                                     \

void HELPER(sve_ldff1##PART##_r)(CPUARMState *env, void *vg,            \

                                 target_ulong addr, uint32_t desc)      \

{                                                                       \

    sve_ldnfff1_r(env, vg, addr, desc, GETPC(), ESZ, MO_8, FAULT_FIRST, \

    sve_ldnfff1_r(env, vg, addr, desc, GETPC(), 0, ESZ, MO_8, FAULT_FIRST, \

                  sve_ld1##PART##_host, sve_ld1##PART##_tlb);           \

}                                                                       \

void HELPER(sve_ldnf1##PART##_r)(CPUARMState *env, void *vg,            \

                                 target_ulong addr, uint32_t desc)      \

{                                                                       \

    sve_ldnfff1_r(env, vg, addr, desc, GETPC(), ESZ, MO_8, FAULT_NO,    \

    sve_ldnfff1_r(env, vg, addr, desc, GETPC(), 0, ESZ, MO_8, FAULT_NO, \

                  sve_ld1##PART##_host, sve_ld1##PART##_tlb);           \

}                                                                       \

void HELPER(sve_ldff1##PART##_r_mte)(CPUARMState *env, void *vg,        \

                                     target_ulong addr, uint32_t desc)  \

{                                                                       \

    sve_ldnfff1_r_mte(env, vg, addr, desc, GETPC(), ESZ, MO_8, FAULT_FIRST, \

                      sve_ld1##PART##_host, sve_ld1##PART##_tlb);       \

}                                                                       \

void HELPER(sve_ldnf1##PART##_r_mte)(CPUARMState *env, void *vg,        \

                                     target_ulong addr, uint32_t desc)  \

{                                                                       \

    sve_ldnfff1_r_mte(env, vg, addr, desc, GETPC(), ESZ, MO_8, FAULT_NO, \

                  sve_ld1##PART##_host, sve_ld1##PART##_tlb);           \

}
@@ -4972,25 +5023,49 @@ void HELPER(sve_ldnf1##PART##_r)(CPUARMState *env, void *vg, \ 
void HELPER(sve_ldff1##PART##_le_r)(CPUARMState *env, void *vg,         \

                                    target_ulong addr, uint32_t desc)   \

{                                                                       \

    sve_ldnfff1_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, FAULT_FIRST,  \

    sve_ldnfff1_r(env, vg, addr, desc, GETPC(), 0, ESZ, MSZ, FAULT_FIRST, \

                  sve_ld1##PART##_le_host, sve_ld1##PART##_le_tlb);     \

}                                                                       \

void HELPER(sve_ldnf1##PART##_le_r)(CPUARMState *env, void *vg,         \

                                    target_ulong addr, uint32_t desc)   \

{                                                                       \

    sve_ldnfff1_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, FAULT_NO,     \

    sve_ldnfff1_r(env, vg, addr, desc, GETPC(), 0, ESZ, MSZ, FAULT_NO,  \

                  sve_ld1##PART##_le_host, sve_ld1##PART##_le_tlb);     \

}                                                                       \

void HELPER(sve_ldff1##PART##_be_r)(CPUARMState *env, void *vg,         \

                                    target_ulong addr, uint32_t desc)   \

{                                                                       \

    sve_ldnfff1_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, FAULT_FIRST,  \

    sve_ldnfff1_r(env, vg, addr, desc, GETPC(), 0, ESZ, MSZ, FAULT_FIRST, \

                  sve_ld1##PART##_be_host, sve_ld1##PART##_be_tlb);     \

}                                                                       \

void HELPER(sve_ldnf1##PART##_be_r)(CPUARMState *env, void *vg,         \

                                    target_ulong addr, uint32_t desc)   \

{                                                                       \

    sve_ldnfff1_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, FAULT_NO,     \

    sve_ldnfff1_r(env, vg, addr, desc, GETPC(), 0, ESZ, MSZ, FAULT_NO,  \

                  sve_ld1##PART##_be_host, sve_ld1##PART##_be_tlb);     \

}                                                                       \

void HELPER(sve_ldff1##PART##_le_r_mte)(CPUARMState *env, void *vg,     \

                                        target_ulong addr, uint32_t desc) \

{                                                                       \

    sve_ldnfff1_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ, FAULT_FIRST, \

                      sve_ld1##PART##_le_host, sve_ld1##PART##_le_tlb); \

}                                                                       \

void HELPER(sve_ldnf1##PART##_le_r_mte)(CPUARMState *env, void *vg,     \

                                        target_ulong addr, uint32_t desc) \

{                                                                       \

    sve_ldnfff1_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ, FAULT_NO, \

                      sve_ld1##PART##_le_host, sve_ld1##PART##_le_tlb); \

}                                                                       \

void HELPER(sve_ldff1##PART##_be_r_mte)(CPUARMState *env, void *vg,     \

                                        target_ulong addr, uint32_t desc) \

{                                                                       \

    sve_ldnfff1_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ, FAULT_FIRST, \

                      sve_ld1##PART##_be_host, sve_ld1##PART##_be_tlb); \

}                                                                       \

void HELPER(sve_ldnf1##PART##_be_r_mte)(CPUARMState *env, void *vg,     \

                                        target_ulong addr, uint32_t desc) \

{                                                                       \

    sve_ldnfff1_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ, FAULT_NO, \

                      sve_ld1##PART##_be_host, sve_ld1##PART##_be_tlb); \

}

target/arm/translate-sve.c

+172 −88
Original line number Diff line number Diff line
@@ -4773,8 +4773,8 @@ static bool trans_LD_zpri(DisasContext *s, arg_rpri_load *a) 


static bool trans_LDFF1_zprr(DisasContext *s, arg_rprr_load *a)

{

    static gen_helper_gvec_mem * const fns[2][16] = {

        /* Little-endian */

    static gen_helper_gvec_mem * const fns[2][2][16] = {

        { /* mte inactive, little-endian */

          { gen_helper_sve_ldff1bb_r,

            gen_helper_sve_ldff1bhu_r,

            gen_helper_sve_ldff1bsu_r,
@@ -4795,7 +4795,7 @@ static bool trans_LDFF1_zprr(DisasContext *s, arg_rprr_load *a) 
            gen_helper_sve_ldff1bhs_r,

            gen_helper_sve_ldff1dd_le_r },



        /* Big-endian */

          /* mte inactive, big-endian */

          { gen_helper_sve_ldff1bb_r,

            gen_helper_sve_ldff1bhu_r,

            gen_helper_sve_ldff1bsu_r,
@@ -4814,23 +4814,65 @@ static bool trans_LDFF1_zprr(DisasContext *s, arg_rprr_load *a) 
            gen_helper_sve_ldff1bds_r,

            gen_helper_sve_ldff1bss_r,

            gen_helper_sve_ldff1bhs_r,

          gen_helper_sve_ldff1dd_be_r },

            gen_helper_sve_ldff1dd_be_r } },



        { /* mte active, little-endian */

          { gen_helper_sve_ldff1bb_r_mte,

            gen_helper_sve_ldff1bhu_r_mte,

            gen_helper_sve_ldff1bsu_r_mte,

            gen_helper_sve_ldff1bdu_r_mte,



            gen_helper_sve_ldff1sds_le_r_mte,

            gen_helper_sve_ldff1hh_le_r_mte,

            gen_helper_sve_ldff1hsu_le_r_mte,

            gen_helper_sve_ldff1hdu_le_r_mte,



            gen_helper_sve_ldff1hds_le_r_mte,

            gen_helper_sve_ldff1hss_le_r_mte,

            gen_helper_sve_ldff1ss_le_r_mte,

            gen_helper_sve_ldff1sdu_le_r_mte,



            gen_helper_sve_ldff1bds_r_mte,

            gen_helper_sve_ldff1bss_r_mte,

            gen_helper_sve_ldff1bhs_r_mte,

            gen_helper_sve_ldff1dd_le_r_mte },



          /* mte active, big-endian */

          { gen_helper_sve_ldff1bb_r_mte,

            gen_helper_sve_ldff1bhu_r_mte,

            gen_helper_sve_ldff1bsu_r_mte,

            gen_helper_sve_ldff1bdu_r_mte,



            gen_helper_sve_ldff1sds_be_r_mte,

            gen_helper_sve_ldff1hh_be_r_mte,

            gen_helper_sve_ldff1hsu_be_r_mte,

            gen_helper_sve_ldff1hdu_be_r_mte,



            gen_helper_sve_ldff1hds_be_r_mte,

            gen_helper_sve_ldff1hss_be_r_mte,

            gen_helper_sve_ldff1ss_be_r_mte,

            gen_helper_sve_ldff1sdu_be_r_mte,



            gen_helper_sve_ldff1bds_r_mte,

            gen_helper_sve_ldff1bss_r_mte,

            gen_helper_sve_ldff1bhs_r_mte,

            gen_helper_sve_ldff1dd_be_r_mte } },

    };



    if (sve_access_check(s)) {

        TCGv_i64 addr = new_tmp_a64(s);

        tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype));

        tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));

        do_mem_zpa(s, a->rd, a->pg, addr, a->dtype, 0, false,

                   fns[s->be_data == MO_BE][a->dtype]);

        do_mem_zpa(s, a->rd, a->pg, addr, a->dtype, 1, false,

                   fns[s->mte_active[0]][s->be_data == MO_BE][a->dtype]);

    }

    return true;

}



static bool trans_LDNF1_zpri(DisasContext *s, arg_rpri_load *a)

{

    static gen_helper_gvec_mem * const fns[2][16] = {

        /* Little-endian */

    static gen_helper_gvec_mem * const fns[2][2][16] = {

        { /* mte inactive, little-endian */

          { gen_helper_sve_ldnf1bb_r,

            gen_helper_sve_ldnf1bhu_r,

            gen_helper_sve_ldnf1bsu_r,
@@ -4851,7 +4893,7 @@ static bool trans_LDNF1_zpri(DisasContext *s, arg_rpri_load *a) 
            gen_helper_sve_ldnf1bhs_r,

            gen_helper_sve_ldnf1dd_le_r },



        /* Big-endian */

          /* mte inactive, big-endian */

          { gen_helper_sve_ldnf1bb_r,

            gen_helper_sve_ldnf1bhu_r,

            gen_helper_sve_ldnf1bsu_r,
@@ -4870,7 +4912,49 @@ static bool trans_LDNF1_zpri(DisasContext *s, arg_rpri_load *a) 
            gen_helper_sve_ldnf1bds_r,

            gen_helper_sve_ldnf1bss_r,

            gen_helper_sve_ldnf1bhs_r,

          gen_helper_sve_ldnf1dd_be_r },

            gen_helper_sve_ldnf1dd_be_r } },



        { /* mte inactive, little-endian */

          { gen_helper_sve_ldnf1bb_r_mte,

            gen_helper_sve_ldnf1bhu_r_mte,

            gen_helper_sve_ldnf1bsu_r_mte,

            gen_helper_sve_ldnf1bdu_r_mte,



            gen_helper_sve_ldnf1sds_le_r_mte,

            gen_helper_sve_ldnf1hh_le_r_mte,

            gen_helper_sve_ldnf1hsu_le_r_mte,

            gen_helper_sve_ldnf1hdu_le_r_mte,



            gen_helper_sve_ldnf1hds_le_r_mte,

            gen_helper_sve_ldnf1hss_le_r_mte,

            gen_helper_sve_ldnf1ss_le_r_mte,

            gen_helper_sve_ldnf1sdu_le_r_mte,



            gen_helper_sve_ldnf1bds_r_mte,

            gen_helper_sve_ldnf1bss_r_mte,

            gen_helper_sve_ldnf1bhs_r_mte,

            gen_helper_sve_ldnf1dd_le_r_mte },



          /* mte inactive, big-endian */

          { gen_helper_sve_ldnf1bb_r_mte,

            gen_helper_sve_ldnf1bhu_r_mte,

            gen_helper_sve_ldnf1bsu_r_mte,

            gen_helper_sve_ldnf1bdu_r_mte,



            gen_helper_sve_ldnf1sds_be_r_mte,

            gen_helper_sve_ldnf1hh_be_r_mte,

            gen_helper_sve_ldnf1hsu_be_r_mte,

            gen_helper_sve_ldnf1hdu_be_r_mte,



            gen_helper_sve_ldnf1hds_be_r_mte,

            gen_helper_sve_ldnf1hss_be_r_mte,

            gen_helper_sve_ldnf1ss_be_r_mte,

            gen_helper_sve_ldnf1sdu_be_r_mte,



            gen_helper_sve_ldnf1bds_r_mte,

            gen_helper_sve_ldnf1bss_r_mte,

            gen_helper_sve_ldnf1bhs_r_mte,

            gen_helper_sve_ldnf1dd_be_r_mte } },

    };



    if (sve_access_check(s)) {
@@ -4880,8 +4964,8 @@ static bool trans_LDNF1_zpri(DisasContext *s, arg_rpri_load *a) 
        TCGv_i64 addr = new_tmp_a64(s);



        tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), off);

        do_mem_zpa(s, a->rd, a->pg, addr, a->dtype, 0, false,

                   fns[s->be_data == MO_BE][a->dtype]);

        do_mem_zpa(s, a->rd, a->pg, addr, a->dtype, 1, false,

                   fns[s->mte_active[0]][s->be_data == MO_BE][a->dtype]);

    }

    return true;

}