Commit 44656d3d authored by Marco Elver's avatar Marco Elver Committed by Paul E. McKenney
Browse files

kcsan: Add current->state to implicitly atomic accesses 



Add volatile current->state to list of implicitly atomic accesses. This
is in preparation to eventually enable KCSAN on kernel/sched (which
currently still has KCSAN_SANITIZE := n).

Since accesses that match the special check in atomic.h are rare, it
makes more sense to move this check to the slow-path, avoiding the
additional compare in the fast-path. With the microbenchmark, a speedup
of ~6% is measured.

Signed-off-by: default avatarMarco Elver <elver@google.com>
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent 2402d0ea

kernel/kcsan/atomic.h

+7 −14
Original line number Diff line number Diff line
@@ -4,24 +4,17 @@ 
#define _KERNEL_KCSAN_ATOMIC_H



#include <linux/jiffies.h>

#include <linux/sched.h>



/*

 * Helper that returns true if access to @ptr should be considered an atomic

 * access, even though it is not explicitly atomic.

 *

 * List all volatile globals that have been observed in races, to suppress

 * data race reports between accesses to these variables.

 *

 * For now, we assume that volatile accesses of globals are as strong as atomic

 * accesses (READ_ONCE, WRITE_ONCE cast to volatile). The situation is still not

 * entirely clear, as on some architectures (Alpha) READ_ONCE/WRITE_ONCE do more

 * than cast to volatile. Eventually, we hope to be able to remove this

 * function.

 * Special rules for certain memory where concurrent conflicting accesses are

 * common, however, the current convention is to not mark them; returns true if

 * access to @ptr should be considered atomic. Called from slow-path.

 */

static __always_inline bool kcsan_is_atomic(const volatile void *ptr)

static bool kcsan_is_atomic_special(const volatile void *ptr)

{

	/* only jiffies for now */

	return ptr == &jiffies;

	/* volatile globals that have been observed in data races. */

	return ptr == &jiffies || ptr == &current->state;

}



#endif /* _KERNEL_KCSAN_ATOMIC_H */

kernel/kcsan/core.c

+15 −7
Original line number Diff line number Diff line
@@ -188,12 +188,13 @@ static __always_inline struct kcsan_ctx *get_ctx(void) 
	return in_task() ? &current->kcsan_ctx : raw_cpu_ptr(&kcsan_cpu_ctx);

}



/* Rules for generic atomic accesses. Called from fast-path. */

static __always_inline bool

is_atomic(const volatile void *ptr, size_t size, int type)

{

	struct kcsan_ctx *ctx;



	if ((type & KCSAN_ACCESS_ATOMIC) != 0)

	if (type & KCSAN_ACCESS_ATOMIC)

		return true;



	/*
@@ -201,16 +202,16 @@ is_atomic(const volatile void *ptr, size_t size, int type) 
	 * as atomic. This allows using them also in atomic regions, such as

	 * seqlocks, without implicitly changing their semantics.

	 */

	if ((type & KCSAN_ACCESS_ASSERT) != 0)

	if (type & KCSAN_ACCESS_ASSERT)

		return false;



	if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC) &&

	    (type & KCSAN_ACCESS_WRITE) != 0 && size <= sizeof(long) &&

	    (type & KCSAN_ACCESS_WRITE) && size <= sizeof(long) &&

	    IS_ALIGNED((unsigned long)ptr, size))

		return true; /* Assume aligned writes up to word size are atomic. */



	ctx = get_ctx();

	if (unlikely(ctx->atomic_next > 0)) {

	if (ctx->atomic_next > 0) {

		/*

		 * Because we do not have separate contexts for nested

		 * interrupts, in case atomic_next is set, we simply assume that
@@ -224,10 +225,8 @@ is_atomic(const volatile void *ptr, size_t size, int type) 
			--ctx->atomic_next; /* in task, or outer interrupt */

		return true;

	}

	if (unlikely(ctx->atomic_nest_count > 0 || ctx->in_flat_atomic))

		return true;



	return kcsan_is_atomic(ptr);

	return ctx->atomic_nest_count > 0 || ctx->in_flat_atomic;

}



static __always_inline bool
@@ -367,6 +366,15 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type) 
	if (!kcsan_is_enabled())

		goto out;



	/*

	 * Special atomic rules: unlikely to be true, so we check them here in

	 * the slow-path, and not in the fast-path in is_atomic(). Call after

	 * kcsan_is_enabled(), as we may access memory that is not yet

	 * initialized during early boot.

	 */

	if (!is_assert && kcsan_is_atomic_special(ptr))

		goto out;



	if (!check_encodable((unsigned long)ptr, size)) {

		kcsan_counter_inc(KCSAN_COUNTER_UNENCODABLE_ACCESSES);

		goto out;

kernel/kcsan/debugfs.c

+18 −9
Original line number Diff line number Diff line
@@ -74,25 +74,34 @@ void kcsan_counter_dec(enum kcsan_counter_id id) 
 */

static noinline void microbenchmark(unsigned long iters)

{

	const struct kcsan_ctx ctx_save = current->kcsan_ctx;

	const bool was_enabled = READ_ONCE(kcsan_enabled);

	cycles_t cycles;



	/* We may have been called from an atomic region; reset context. */

	memset(&current->kcsan_ctx, 0, sizeof(current->kcsan_ctx));

	/*

	 * Disable to benchmark fast-path for all accesses, and (expected

	 * negligible) call into slow-path, but never set up watchpoints.

	 */

	WRITE_ONCE(kcsan_enabled, false);



	pr_info("KCSAN: %s begin | iters: %lu\n", __func__, iters);



	cycles = get_cycles();

	while (iters--) {

		/*

		 * We can run this benchmark from multiple tasks; this address

		 * calculation increases likelyhood of some accesses

		 * overlapping. Make the access type an atomic read, to never

		 * set up watchpoints and test the fast-path only.

		 */

		unsigned long addr =

			iters % (CONFIG_KCSAN_NUM_WATCHPOINTS * PAGE_SIZE);

		__kcsan_check_access((void *)addr, sizeof(long), KCSAN_ACCESS_ATOMIC);

		unsigned long addr = iters & ((PAGE_SIZE << 8) - 1);

		int type = !(iters & 0x7f) ? KCSAN_ACCESS_ATOMIC :

				(!(iters & 0xf) ? KCSAN_ACCESS_WRITE : 0);

		__kcsan_check_access((void *)addr, sizeof(long), type);

	}

	cycles = get_cycles() - cycles;



	pr_info("KCSAN: %s end   | cycles: %llu\n", __func__, cycles);



	WRITE_ONCE(kcsan_enabled, was_enabled);

	/* restore context */

	current->kcsan_ctx = ctx_save;

}



/*