Commit c482cb11 authored by Richard Henderson's avatar Richard Henderson
Browse files

tcg: Add atomic helpers 



Add all of cmpxchg, op_fetch, fetch_op, and xchg.
Handle both endian-ness, and sizes up to 8.
Handle expanding non-atomically, when emulating in serial.

Reviewed-by: default avatarEmilio G. Cota <cota@braap.org>
Reviewed-by: default avatarAlex Bennée <alex.bennee@linaro.org>
Signed-off-by: default avatarRichard Henderson <rth@twiddle.net>
parent c86c6e4c

Makefile.objs

+1 −1
Original line number Diff line number Diff line
@@ -89,7 +89,7 @@ endif 


#######################################################################

# Target-independent parts used in system and user emulation

common-obj-y += tcg-runtime.o cpus-common.o

common-obj-y += cpus-common.o

common-obj-y += hw/

common-obj-y += qom/

common-obj-y += disas/

Makefile.target

+1 −0
Original line number Diff line number Diff line
@@ -94,6 +94,7 @@ obj-$(CONFIG_TCG_INTERPRETER) += disas/tci.o 
obj-y += fpu/softfloat.o

obj-y += target-$(TARGET_BASE_ARCH)/

obj-y += disas.o

obj-y += tcg-runtime.o

obj-$(call notempty,$(TARGET_XML_FILES)) += gdbstub-xml.o

obj-$(call lnot,$(CONFIG_KVM)) += kvm-stub.o

atomic_template.h

0 → 100644
+177 −0
Original line number Diff line number Diff line 
/*

 * Atomic helper templates

 * Included from tcg-runtime.c and cputlb.c.

 *

 * Copyright (c) 2016 Red Hat, Inc

 *

 * This library is free software; you can redistribute it and/or

 * modify it under the terms of the GNU Lesser General Public

 * License as published by the Free Software Foundation; either

 * version 2 of the License, or (at your option) any later version.

 *

 * This library is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

 * Lesser General Public License for more details.

 *

 * You should have received a copy of the GNU Lesser General Public

 * License along with this library; if not, see <http://www.gnu.org/licenses/>.

 */



#if DATA_SIZE == 8

# define SUFFIX     q

# define DATA_TYPE  uint64_t

# define BSWAP      bswap64

#elif DATA_SIZE == 4

# define SUFFIX     l

# define DATA_TYPE  uint32_t

# define BSWAP      bswap32

#elif DATA_SIZE == 2

# define SUFFIX     w

# define DATA_TYPE  uint16_t

# define BSWAP      bswap16

#elif DATA_SIZE == 1

# define SUFFIX     b

# define DATA_TYPE  uint8_t

# define BSWAP

#else

# error unsupported data size

#endif



#if DATA_SIZE >= 4

# define ABI_TYPE  DATA_TYPE

#else

# define ABI_TYPE  uint32_t

#endif



/* Define host-endian atomic operations.  Note that END is used within

   the ATOMIC_NAME macro, and redefined below.  */

#if DATA_SIZE == 1

# define END

#elif defined(HOST_WORDS_BIGENDIAN)

# define END  _be

#else

# define END  _le

#endif



ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,

                              ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)

{

    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;

    return atomic_cmpxchg__nocheck(haddr, cmpv, newv);

}



ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,

                           ABI_TYPE val EXTRA_ARGS)

{

    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;

    return atomic_xchg__nocheck(haddr, val);

}



#define GEN_ATOMIC_HELPER(X)                                        \

ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \

                 ABI_TYPE val EXTRA_ARGS)                           \

{                                                                   \

    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \

    return atomic_##X(haddr, val);                                  \

}                                                                   \



GEN_ATOMIC_HELPER(fetch_add)

GEN_ATOMIC_HELPER(fetch_and)

GEN_ATOMIC_HELPER(fetch_or)

GEN_ATOMIC_HELPER(fetch_xor)

GEN_ATOMIC_HELPER(add_fetch)

GEN_ATOMIC_HELPER(and_fetch)

GEN_ATOMIC_HELPER(or_fetch)

GEN_ATOMIC_HELPER(xor_fetch)



#undef GEN_ATOMIC_HELPER

#undef END



#if DATA_SIZE > 1



/* Define reverse-host-endian atomic operations.  Note that END is used

   within the ATOMIC_NAME macro.  */

#ifdef HOST_WORDS_BIGENDIAN

# define END  _le

#else

# define END  _be

#endif



ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,

                              ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)

{

    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;

    return BSWAP(atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)));

}



ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,

                           ABI_TYPE val EXTRA_ARGS)

{

    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;

    return BSWAP(atomic_xchg__nocheck(haddr, BSWAP(val)));

}



#define GEN_ATOMIC_HELPER(X)                                        \

ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \

                 ABI_TYPE val EXTRA_ARGS)                           \

{                                                                   \

    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \

    return BSWAP(atomic_##X(haddr, BSWAP(val)));                    \

}



GEN_ATOMIC_HELPER(fetch_and)

GEN_ATOMIC_HELPER(fetch_or)

GEN_ATOMIC_HELPER(fetch_xor)

GEN_ATOMIC_HELPER(and_fetch)

GEN_ATOMIC_HELPER(or_fetch)

GEN_ATOMIC_HELPER(xor_fetch)



#undef GEN_ATOMIC_HELPER



/* Note that for addition, we need to use a separate cmpxchg loop instead

   of bswaps for the reverse-host-endian helpers.  */

ABI_TYPE ATOMIC_NAME(fetch_add)(CPUArchState *env, target_ulong addr,

                         ABI_TYPE val EXTRA_ARGS)

{

    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;

    DATA_TYPE ldo, ldn, ret, sto;



    ldo = atomic_read__nocheck(haddr);

    while (1) {

        ret = BSWAP(ldo);

        sto = BSWAP(ret + val);

        ldn = atomic_cmpxchg__nocheck(haddr, ldo, sto);

        if (ldn == ldo) {

            return ret;

        }

        ldo = ldn;

    }

}



ABI_TYPE ATOMIC_NAME(add_fetch)(CPUArchState *env, target_ulong addr,

                         ABI_TYPE val EXTRA_ARGS)

{

    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;

    DATA_TYPE ldo, ldn, ret, sto;



    ldo = atomic_read__nocheck(haddr);

    while (1) {

        ret = BSWAP(ldo) + val;

        sto = BSWAP(ret);

        ldn = atomic_cmpxchg__nocheck(haddr, ldo, sto);

        if (ldn == ldo) {

            return ret;

        }

        ldo = ldn;

    }

}



#undef END

#endif /* DATA_SIZE > 1 */



#undef BSWAP

#undef ABI_TYPE

#undef DATA_TYPE

#undef SUFFIX

#undef DATA_SIZE

cputlb.c

+109 −3
Original line number Diff line number Diff line
@@ -23,15 +23,15 @@ 
#include "exec/memory.h"

#include "exec/address-spaces.h"

#include "exec/cpu_ldst.h"



#include "exec/cputlb.h"



#include "exec/memory-internal.h"

#include "exec/ram_addr.h"

#include "exec/exec-all.h"

#include "tcg/tcg.h"

#include "qemu/error-report.h"

#include "exec/log.h"

#include "exec/helper-proto.h"

#include "qemu/atomic.h"



/* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */

/* #define DEBUG_TLB */
@@ -585,6 +585,69 @@ void probe_write(CPUArchState *env, target_ulong addr, int mmu_idx, 
    }

}



/* Probe for a read-modify-write atomic operation.  Do not allow unaligned

 * operations, or io operations to proceed.  Return the host address.  */

static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,

                               TCGMemOpIdx oi, uintptr_t retaddr)

{

    size_t mmu_idx = get_mmuidx(oi);

    size_t index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);

    CPUTLBEntry *tlbe = &env->tlb_table[mmu_idx][index];

    target_ulong tlb_addr = tlbe->addr_write;

    TCGMemOp mop = get_memop(oi);

    int a_bits = get_alignment_bits(mop);

    int s_bits = mop & MO_SIZE;



    /* Adjust the given return address.  */

    retaddr -= GETPC_ADJ;



    /* Enforce guest required alignment.  */

    if (unlikely(a_bits > 0 && (addr & ((1 << a_bits) - 1)))) {

        /* ??? Maybe indicate atomic op to cpu_unaligned_access */

        cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,

                             mmu_idx, retaddr);

    }



    /* Enforce qemu required alignment.  */

    if (unlikely(addr & ((1 << s_bits) - 1))) {

        /* We get here if guest alignment was not requested,

           or was not enforced by cpu_unaligned_access above.

           We might widen the access and emulate, but for now

           mark an exception and exit the cpu loop.  */

        goto stop_the_world;

    }



    /* Check TLB entry and enforce page permissions.  */

    if ((addr & TARGET_PAGE_MASK)

        != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {

        if (!VICTIM_TLB_HIT(addr_write, addr)) {

            tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_STORE, mmu_idx, retaddr);

        }

        tlb_addr = tlbe->addr_write;

    }



    /* Notice an IO access, or a notdirty page.  */

    if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {

        /* There's really nothing that can be done to

           support this apart from stop-the-world.  */

        goto stop_the_world;

    }



    /* Let the guest notice RMW on a write-only page.  */

    if (unlikely(tlbe->addr_read != tlb_addr)) {

        tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_LOAD, mmu_idx, retaddr);

        /* Since we don't support reads and writes to different addresses,

           and we do have the proper page loaded for write, this shouldn't

           ever return.  But just in case, handle via stop-the-world.  */

        goto stop_the_world;

    }



    return (void *)((uintptr_t)addr + tlbe->addend);



 stop_the_world:

    cpu_loop_exit_atomic(ENV_GET_CPU(env), retaddr);

}



#ifdef TARGET_WORDS_BIGENDIAN

# define TGT_BE(X)  (X)

# define TGT_LE(X)  BSWAP(X)
@@ -606,8 +669,51 @@ void probe_write(CPUArchState *env, target_ulong addr, int mmu_idx, 


#define DATA_SIZE 8

#include "softmmu_template.h"

#undef MMUSUFFIX



/* First set of helpers allows passing in of OI and RETADDR.  This makes

   them callable from other helpers.  */



#define EXTRA_ARGS     , TCGMemOpIdx oi, uintptr_t retaddr

#define ATOMIC_NAME(X) \

    HELPER(glue(glue(glue(atomic_ ## X, SUFFIX), END), _mmu))

#define ATOMIC_MMU_LOOKUP  atomic_mmu_lookup(env, addr, oi, retaddr)



#define DATA_SIZE 1

#include "atomic_template.h"



#define DATA_SIZE 2

#include "atomic_template.h"



#define DATA_SIZE 4

#include "atomic_template.h"



#define DATA_SIZE 8

#include "atomic_template.h"



/* Second set of helpers are directly callable from TCG as helpers.  */



#undef EXTRA_ARGS

#undef ATOMIC_NAME

#undef ATOMIC_MMU_LOOKUP

#define EXTRA_ARGS         , TCGMemOpIdx oi

#define ATOMIC_NAME(X)     HELPER(glue(glue(atomic_ ## X, SUFFIX), END))

#define ATOMIC_MMU_LOOKUP  atomic_mmu_lookup(env, addr, oi, GETPC())



#define DATA_SIZE 1

#include "atomic_template.h"



#define DATA_SIZE 2

#include "atomic_template.h"



#define DATA_SIZE 4

#include "atomic_template.h"



#define DATA_SIZE 8

#include "atomic_template.h"



/* Code access functions.  */



#undef MMUSUFFIX

#define MMUSUFFIX _cmmu

#undef GETPC

#define GETPC() ((uintptr_t)0)

tcg-runtime.c

+38 −11
Original line number Diff line number Diff line
@@ -23,17 +23,10 @@ 
 */

#include "qemu/osdep.h"

#include "qemu/host-utils.h"



/* This file is compiled once, and thus we can't include the standard

   "exec/helper-proto.h", which has includes that are target specific.  */



#include "exec/helper-head.h"



#define DEF_HELPER_FLAGS_2(name, flags, ret, t1, t2) \

  dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2));



#include "tcg-runtime.h"



#include "cpu.h"

#include "exec/helper-proto.h"

#include "exec/cpu_ldst.h"

#include "exec/exec-all.h"



/* 32-bit helpers */
@@ -107,3 +100,37 @@ int64_t HELPER(mulsh_i64)(int64_t arg1, int64_t arg2) 
    muls64(&l, &h, arg1, arg2);

    return h;

}



#ifndef CONFIG_SOFTMMU

/* The softmmu versions of these helpers are in cputlb.c.  */



/* Do not allow unaligned operations to proceed.  Return the host address.  */

static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,

                               int size, uintptr_t retaddr)

{

    /* Enforce qemu required alignment.  */

    if (unlikely(addr & (size - 1))) {

        cpu_loop_exit_atomic(ENV_GET_CPU(env), retaddr);

    }

    return g2h(addr);

}



/* Macro to call the above, with local variables from the use context.  */

#define ATOMIC_MMU_LOOKUP  atomic_mmu_lookup(env, addr, DATA_SIZE, GETPC())



#define ATOMIC_NAME(X)   HELPER(glue(glue(atomic_ ## X, SUFFIX), END))

#define EXTRA_ARGS



#define DATA_SIZE 1

#include "atomic_template.h"



#define DATA_SIZE 2

#include "atomic_template.h"



#define DATA_SIZE 4

#include "atomic_template.h"



#define DATA_SIZE 8

#include "atomic_template.h"



#endif /* !CONFIG_SOFTMMU */