util: add stats64 module (ae2d489c) · Commits · SUMMER2020 / students / proj-2021291

include/qemu/stats64.h

0 → 100644

+193 −0

Original line number	Diff line number	Diff line
		/*
		* Atomic operations on 64-bit quantities.
		*
		* Copyright (C) 2017 Red Hat, Inc.
		*
		* Author: Paolo Bonzini <pbonzini@redhat.com>
		*
		* This work is licensed under the terms of the GNU GPL, version 2 or later.
		* See the COPYING file in the top-level directory.
		*/

		#ifndef QEMU_STATS64_H
		#define QEMU_STATS64_H 1

		#include "qemu/atomic.h"

		/* This provides atomic operations on 64-bit type, using a reader-writer
		* spinlock on architectures that do not have 64-bit accesses. Even on
		* those architectures, it tries hard not to take the lock.
		*/

		typedef struct Stat64 {
		#ifdef CONFIG_ATOMIC64
		uint64_t value;
		#else
		uint32_t low, high;
		uint32_t lock;
		#endif
		} Stat64;

		#ifdef CONFIG_ATOMIC64
		static inline void stat64_init(Stat64 *s, uint64_t value)
		{
		/* This is not guaranteed to be atomic! */
		*s = (Stat64) { value };
		}

		static inline uint64_t stat64_get(const Stat64 *s)
		{
		return atomic_read__nocheck(&s->value);
		}

		static inline void stat64_add(Stat64 *s, uint64_t value)
		{
		atomic_add(&s->value, value);
		}

		static inline void stat64_min(Stat64 *s, uint64_t value)
		{
		uint64_t orig = atomic_read__nocheck(&s->value);
		while (orig > value) {
		orig = atomic_cmpxchg__nocheck(&s->value, orig, value);
		}
		}

		static inline void stat64_max(Stat64 *s, uint64_t value)
		{
		uint64_t orig = atomic_read__nocheck(&s->value);
		while (orig < value) {
		orig = atomic_cmpxchg__nocheck(&s->value, orig, value);
		}
		}
		#else
		uint64_t stat64_get(const Stat64 *s);
		bool stat64_min_slow(Stat64 *s, uint64_t value);
		bool stat64_max_slow(Stat64 *s, uint64_t value);
		bool stat64_add32_carry(Stat64 *s, uint32_t low, uint32_t high);

		static inline void stat64_init(Stat64 *s, uint64_t value)
		{
		/* This is not guaranteed to be atomic! */
		*s = (Stat64) { .low = value, .high = value >> 32, .lock = 0 };
		}

		static inline void stat64_add(Stat64 *s, uint64_t value)
		{
		uint32_t low, high;
		high = value >> 32;
		low = (uint32_t) value;
		if (!low) {
		if (high) {
		atomic_add(&s->high, high);
		}
		return;
		}

		for (;;) {
		uint32_t orig = s->low;
		uint32_t result = orig + low;
		uint32_t old;

		if (result < low \|\| high) {
		/* If the high part is affected, take the lock. */
		if (stat64_add32_carry(s, low, high)) {
		return;
		}
		continue;
		}

		/* No carry, try with a 32-bit cmpxchg. The result is independent of
		* the high 32 bits, so it can race just fine with stat64_add32_carry
		* and even stat64_get!
		*/
		old = atomic_cmpxchg(&s->low, orig, result);
		if (orig == old) {
		return;
		}
		}
		}

		static inline void stat64_min(Stat64 *s, uint64_t value)
		{
		uint32_t low, high;
		uint32_t orig_low, orig_high;

		high = value >> 32;
		low = (uint32_t) value;
		do {
		orig_high = atomic_read(&s->high);
		if (orig_high < high) {
		return;
		}

		if (orig_high == high) {
		/* High 32 bits are equal. Read low after high, otherwise we
		* can get a false positive (e.g. 0x1235,0x0000 changes to
		* 0x1234,0x8000 and we read it as 0x1234,0x0000). Pairs with
		* the write barrier in stat64_min_slow.
		*/
		smp_rmb();
		orig_low = atomic_read(&s->low);
		if (orig_low <= low) {
		return;
		}

		/* See if we were lucky and a writer raced against us. The
		* barrier is theoretically unnecessary, but if we remove it
		* we may miss being lucky.
		*/
		smp_rmb();
		orig_high = atomic_read(&s->high);
		if (orig_high < high) {
		return;
		}
		}

		/* If the value changes in any way, we have to take the lock. */
		} while (!stat64_min_slow(s, value));
		}

		static inline void stat64_max(Stat64 *s, uint64_t value)
		{
		uint32_t low, high;
		uint32_t orig_low, orig_high;

		high = value >> 32;
		low = (uint32_t) value;
		do {
		orig_high = atomic_read(&s->high);
		if (orig_high > high) {
		return;
		}

		if (orig_high == high) {
		/* High 32 bits are equal. Read low after high, otherwise we
		* can get a false positive (e.g. 0x1234,0x8000 changes to
		* 0x1235,0x0000 and we read it as 0x1235,0x8000). Pairs with
		* the write barrier in stat64_max_slow.
		*/
		smp_rmb();
		orig_low = atomic_read(&s->low);
		if (orig_low >= low) {
		return;
		}

		/* See if we were lucky and a writer raced against us. The
		* barrier is theoretically unnecessary, but if we remove it
		* we may miss being lucky.
		*/
		smp_rmb();
		orig_high = atomic_read(&s->high);
		if (orig_high > high) {
		return;
		}
		}

		/* If the value changes in any way, we have to take the lock. */
		} while (!stat64_max_slow(s, value));
		}

		#endif

		#endif

util/Makefile.objs

+1 −0

Original line number	Diff line number	Diff line
		@@ -42,4 +42,5 @@ util-obj-y += log.o
		util-obj-y += qdist.o
		util-obj-y += qht.o
		util-obj-y += range.o
		util-obj-y += stats64.o
		util-obj-y += systemd.o

util/stats64.c

0 → 100644

+137 −0

Original line number	Diff line number	Diff line
		/*
		* Atomic operations on 64-bit quantities.
		*
		* Copyright (C) 2017 Red Hat, Inc.
		*
		* Author: Paolo Bonzini <pbonzini@redhat.com>
		*
		* This work is licensed under the terms of the GNU GPL, version 2 or later.
		* See the COPYING file in the top-level directory.
		*/

		#include "qemu/osdep.h"
		#include "qemu/atomic.h"
		#include "qemu/stats64.h"
		#include "qemu/processor.h"

		#ifndef CONFIG_ATOMIC64
		static inline void stat64_rdlock(Stat64 *s)
		{
		/* Keep out incoming writers to avoid them starving us. */
		atomic_add(&s->lock, 2);

		/* If there is a concurrent writer, wait for it. */
		while (atomic_read(&s->lock) & 1) {
		cpu_relax();
		}
		}

		static inline void stat64_rdunlock(Stat64 *s)
		{
		atomic_sub(&s->lock, 2);
		}

		static inline bool stat64_wrtrylock(Stat64 *s)
		{
		return atomic_cmpxchg(&s->lock, 0, 1) == 0;
		}

		static inline void stat64_wrunlock(Stat64 *s)
		{
		atomic_dec(&s->lock);
		}

		uint64_t stat64_get(const Stat64 *s)
		{
		uint32_t high, low;

		stat64_rdlock((Stat64 *)s);

		/* 64-bit writes always take the lock, so we can read in
		* any order.
		*/
		high = atomic_read(&s->high);
		low = atomic_read(&s->low);
		stat64_rdunlock((Stat64 *)s);

		return ((uint64_t)high << 32) \| low;
		}

		bool stat64_add32_carry(Stat64 *s, uint32_t low, uint32_t high)
		{
		uint32_t old;

		if (!stat64_wrtrylock(s)) {
		cpu_relax();
		return false;
		}

		/* 64-bit reads always take the lock, so they don't care about the
		* order of our update. By updating s->low first, we can check
		* whether we have to carry into s->high.
		*/
		old = atomic_fetch_add(&s->low, low);
		high += (old + low) < old;
		atomic_add(&s->high, high);
		stat64_wrunlock(s);
		return true;
		}

		bool stat64_min_slow(Stat64 *s, uint64_t value)
		{
		uint32_t high, low;
		uint64_t orig;

		if (!stat64_wrtrylock(s)) {
		cpu_relax();
		return false;
		}

		high = atomic_read(&s->high);
		low = atomic_read(&s->low);

		orig = ((uint64_t)high << 32) \| low;
		if (orig < value) {
		/* We have to set low before high, just like stat64_min reads
		* high before low. The value may become higher temporarily, but
		* stat64_get does not notice (it takes the lock) and the only ill
		* effect on stat64_min is that the slow path may be triggered
		* unnecessarily.
		*/
		atomic_set(&s->low, (uint32_t)value);
		smp_wmb();
		atomic_set(&s->high, value >> 32);
		}
		stat64_wrunlock(s);
		return true;
		}

		bool stat64_max_slow(Stat64 *s, uint64_t value)
		{
		uint32_t high, low;
		uint64_t orig;

		if (!stat64_wrtrylock(s)) {
		cpu_relax();
		return false;
		}

		high = atomic_read(&s->high);
		low = atomic_read(&s->low);

		orig = ((uint64_t)high << 32) \| low;
		if (orig > value) {
		/* We have to set low before high, just like stat64_max reads
		* high before low. The value may become lower temporarily, but
		* stat64_get does not notice (it takes the lock) and the only ill
		* effect on stat64_max is that the slow path may be triggered
		* unnecessarily.
		*/
		atomic_set(&s->low, (uint32_t)value);
		smp_wmb();
		atomic_set(&s->high, value >> 32);
		}
		stat64_wrunlock(s);
		return true;
		}
		#endif