Unverified Commit 26fb751c authored by Atish Patra's avatar Atish Patra Committed by Palmer Dabbelt
Browse files

RISC-V: Do not use cpumask data structure for hartid bitmap



Currently, SBI APIs accept a hartmask that is generated from struct
cpumask. Cpumask data structure can hold upto NR_CPUs value. Thus, it
is not the correct data structure for hartids as it can be higher
than NR_CPUs for platforms with sparse or discontguous hartids.

Remove all association between hartid mask and struct cpumask.

Reviewed-by: Anup Patel <anup@brainfault.org> (For Linux RISC-V changes)
Acked-by: Anup Patel <anup@brainfault.org> (For KVM RISC-V changes)
Signed-off-by: default avatarAtish Patra <atishp@rivosinc.com>
Signed-off-by: default avatarPalmer Dabbelt <palmer@rivosinc.com>
parent 2ffc48fc
Loading
Loading
Loading
Loading
+10 −9
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@
#define _ASM_RISCV_SBI_H

#include <linux/types.h>
#include <linux/cpumask.h>

#ifdef CONFIG_RISCV_SBI
enum sbi_ext_id {
@@ -128,27 +129,27 @@ long sbi_get_mimpid(void);
void sbi_set_timer(uint64_t stime_value);
void sbi_shutdown(void);
void sbi_clear_ipi(void);
int sbi_send_ipi(const unsigned long *hart_mask);
int sbi_remote_fence_i(const unsigned long *hart_mask);
int sbi_remote_sfence_vma(const unsigned long *hart_mask,
int sbi_send_ipi(const struct cpumask *cpu_mask);
int sbi_remote_fence_i(const struct cpumask *cpu_mask);
int sbi_remote_sfence_vma(const struct cpumask *cpu_mask,
			   unsigned long start,
			   unsigned long size);

int sbi_remote_sfence_vma_asid(const unsigned long *hart_mask,
int sbi_remote_sfence_vma_asid(const struct cpumask *cpu_mask,
				unsigned long start,
				unsigned long size,
				unsigned long asid);
int sbi_remote_hfence_gvma(const unsigned long *hart_mask,
int sbi_remote_hfence_gvma(const struct cpumask *cpu_mask,
			   unsigned long start,
			   unsigned long size);
int sbi_remote_hfence_gvma_vmid(const unsigned long *hart_mask,
int sbi_remote_hfence_gvma_vmid(const struct cpumask *cpu_mask,
				unsigned long start,
				unsigned long size,
				unsigned long vmid);
int sbi_remote_hfence_vvma(const unsigned long *hart_mask,
int sbi_remote_hfence_vvma(const struct cpumask *cpu_mask,
			   unsigned long start,
			   unsigned long size);
int sbi_remote_hfence_vvma_asid(const unsigned long *hart_mask,
int sbi_remote_hfence_vvma_asid(const struct cpumask *cpu_mask,
				unsigned long start,
				unsigned long size,
				unsigned long asid);
@@ -183,7 +184,7 @@ static inline unsigned long sbi_mk_version(unsigned long major,

int sbi_err_map_linux_errno(int err);
#else /* CONFIG_RISCV_SBI */
static inline int sbi_remote_fence_i(const unsigned long *hart_mask) { return -1; }
static inline int sbi_remote_fence_i(const struct cpumask *cpu_mask) { return -1; }
static inline void sbi_init(void) {}
#endif /* CONFIG_RISCV_SBI */
#endif /* _ASM_RISCV_SBI_H */
+0 −2
Original line number Diff line number Diff line
@@ -92,8 +92,6 @@ static inline void riscv_clear_ipi(void)

#endif /* CONFIG_SMP */

void riscv_cpuid_to_hartid_mask(const struct cpumask *in, struct cpumask *out);

#if defined(CONFIG_HOTPLUG_CPU) && (CONFIG_SMP)
bool cpu_has_hotplug(unsigned int cpu);
#else
+106 −83
Original line number Diff line number Diff line
@@ -16,8 +16,8 @@ unsigned long sbi_spec_version __ro_after_init = SBI_SPEC_VERSION_DEFAULT;
EXPORT_SYMBOL(sbi_spec_version);

static void (*__sbi_set_timer)(uint64_t stime) __ro_after_init;
static int (*__sbi_send_ipi)(const unsigned long *hart_mask) __ro_after_init;
static int (*__sbi_rfence)(int fid, const unsigned long *hart_mask,
static int (*__sbi_send_ipi)(const struct cpumask *cpu_mask) __ro_after_init;
static int (*__sbi_rfence)(int fid, const struct cpumask *cpu_mask,
			   unsigned long start, unsigned long size,
			   unsigned long arg4, unsigned long arg5) __ro_after_init;

@@ -67,6 +67,30 @@ int sbi_err_map_linux_errno(int err)
EXPORT_SYMBOL(sbi_err_map_linux_errno);

#ifdef CONFIG_RISCV_SBI_V01
static unsigned long __sbi_v01_cpumask_to_hartmask(const struct cpumask *cpu_mask)
{
	unsigned long cpuid, hartid;
	unsigned long hmask = 0;

	/*
	 * There is no maximum hartid concept in RISC-V and NR_CPUS must not be
	 * associated with hartid. As SBI v0.1 is only kept for backward compatibility
	 * and will be removed in the future, there is no point in supporting hartid
	 * greater than BITS_PER_LONG (32 for RV32 and 64 for RV64). Ideally, SBI v0.2
	 * should be used for platforms with hartid greater than BITS_PER_LONG.
	 */
	for_each_cpu(cpuid, cpu_mask) {
		hartid = cpuid_to_hartid_map(cpuid);
		if (hartid >= BITS_PER_LONG) {
			pr_warn("Unable to send any request to hartid > BITS_PER_LONG for SBI v0.1\n");
			break;
		}
		hmask |= 1 << hartid;
	}

	return hmask;
}

/**
 * sbi_console_putchar() - Writes given character to the console device.
 * @ch: The data to be written to the console.
@@ -132,33 +156,44 @@ static void __sbi_set_timer_v01(uint64_t stime_value)
#endif
}

static int __sbi_send_ipi_v01(const unsigned long *hart_mask)
static int __sbi_send_ipi_v01(const struct cpumask *cpu_mask)
{
	sbi_ecall(SBI_EXT_0_1_SEND_IPI, 0, (unsigned long)hart_mask,
	unsigned long hart_mask;

	if (!cpu_mask)
		cpu_mask = cpu_online_mask;
	hart_mask = __sbi_v01_cpumask_to_hartmask(cpu_mask);

	sbi_ecall(SBI_EXT_0_1_SEND_IPI, 0, (unsigned long)(&hart_mask),
		  0, 0, 0, 0, 0);
	return 0;
}

static int __sbi_rfence_v01(int fid, const unsigned long *hart_mask,
static int __sbi_rfence_v01(int fid, const struct cpumask *cpu_mask,
			    unsigned long start, unsigned long size,
			    unsigned long arg4, unsigned long arg5)
{
	int result = 0;
	unsigned long hart_mask;

	if (!cpu_mask)
		cpu_mask = cpu_online_mask;
	hart_mask = __sbi_v01_cpumask_to_hartmask(cpu_mask);

	/* v0.2 function IDs are equivalent to v0.1 extension IDs */
	switch (fid) {
	case SBI_EXT_RFENCE_REMOTE_FENCE_I:
		sbi_ecall(SBI_EXT_0_1_REMOTE_FENCE_I, 0,
			  (unsigned long)hart_mask, 0, 0, 0, 0, 0);
			  (unsigned long)&hart_mask, 0, 0, 0, 0, 0);
		break;
	case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA:
		sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA, 0,
			  (unsigned long)hart_mask, start, size,
			  (unsigned long)&hart_mask, start, size,
			  0, 0, 0);
		break;
	case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID:
		sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID, 0,
			  (unsigned long)hart_mask, start, size,
			  (unsigned long)&hart_mask, start, size,
			  arg4, 0, 0);
		break;
	default:
@@ -180,7 +215,7 @@ static void __sbi_set_timer_v01(uint64_t stime_value)
		sbi_major_version(), sbi_minor_version());
}

static int __sbi_send_ipi_v01(const unsigned long *hart_mask)
static int __sbi_send_ipi_v01(const struct cpumask *cpu_mask)
{
	pr_warn("IPI extension is not available in SBI v%lu.%lu\n",
		sbi_major_version(), sbi_minor_version());
@@ -188,7 +223,7 @@ static int __sbi_send_ipi_v01(const unsigned long *hart_mask)
	return 0;
}

static int __sbi_rfence_v01(int fid, const unsigned long *hart_mask,
static int __sbi_rfence_v01(int fid, const struct cpumask *cpu_mask,
			    unsigned long start, unsigned long size,
			    unsigned long arg4, unsigned long arg5)
{
@@ -212,37 +247,33 @@ static void __sbi_set_timer_v02(uint64_t stime_value)
#endif
}

static int __sbi_send_ipi_v02(const unsigned long *hart_mask)
static int __sbi_send_ipi_v02(const struct cpumask *cpu_mask)
{
	unsigned long hartid, hmask_val, hbase;
	struct cpumask tmask;
	unsigned long hartid, cpuid, hmask = 0, hbase = 0;
	struct sbiret ret = {0};
	int result;

	if (!hart_mask || !(*hart_mask)) {
		riscv_cpuid_to_hartid_mask(cpu_online_mask, &tmask);
		hart_mask = cpumask_bits(&tmask);
	}
	if (!cpu_mask)
		cpu_mask = cpu_online_mask;

	hmask_val = 0;
	hbase = 0;
	for_each_set_bit(hartid, hart_mask, NR_CPUS) {
		if (hmask_val && ((hbase + BITS_PER_LONG) <= hartid)) {
	for_each_cpu(cpuid, cpu_mask) {
		hartid = cpuid_to_hartid_map(cpuid);
		if (hmask && ((hbase + BITS_PER_LONG) <= hartid)) {
			ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI,
					hmask_val, hbase, 0, 0, 0, 0);
					hmask, hbase, 0, 0, 0, 0);
			if (ret.error)
				goto ecall_failed;
			hmask_val = 0;
			hmask = 0;
			hbase = 0;
		}
		if (!hmask_val)
		if (!hmask)
			hbase = hartid;
		hmask_val |= 1UL << (hartid - hbase);
		hmask |= 1UL << (hartid - hbase);
	}

	if (hmask_val) {
	if (hmask) {
		ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI,
				hmask_val, hbase, 0, 0, 0, 0);
				hmask, hbase, 0, 0, 0, 0);
		if (ret.error)
			goto ecall_failed;
	}
@@ -252,11 +283,11 @@ static int __sbi_send_ipi_v02(const unsigned long *hart_mask)
ecall_failed:
	result = sbi_err_map_linux_errno(ret.error);
	pr_err("%s: hbase = [%lu] hmask = [0x%lx] failed (error [%d])\n",
	       __func__, hbase, hmask_val, result);
	       __func__, hbase, hmask, result);
	return result;
}

static int __sbi_rfence_v02_call(unsigned long fid, unsigned long hmask_val,
static int __sbi_rfence_v02_call(unsigned long fid, unsigned long hmask,
				 unsigned long hbase, unsigned long start,
				 unsigned long size, unsigned long arg4,
				 unsigned long arg5)
@@ -267,31 +298,31 @@ static int __sbi_rfence_v02_call(unsigned long fid, unsigned long hmask_val,

	switch (fid) {
	case SBI_EXT_RFENCE_REMOTE_FENCE_I:
		ret = sbi_ecall(ext, fid, hmask_val, hbase, 0, 0, 0, 0);
		ret = sbi_ecall(ext, fid, hmask, hbase, 0, 0, 0, 0);
		break;
	case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA:
		ret = sbi_ecall(ext, fid, hmask_val, hbase, start,
		ret = sbi_ecall(ext, fid, hmask, hbase, start,
				size, 0, 0);
		break;
	case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID:
		ret = sbi_ecall(ext, fid, hmask_val, hbase, start,
		ret = sbi_ecall(ext, fid, hmask, hbase, start,
				size, arg4, 0);
		break;

	case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA:
		ret = sbi_ecall(ext, fid, hmask_val, hbase, start,
		ret = sbi_ecall(ext, fid, hmask, hbase, start,
				size, 0, 0);
		break;
	case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID:
		ret = sbi_ecall(ext, fid, hmask_val, hbase, start,
		ret = sbi_ecall(ext, fid, hmask, hbase, start,
				size, arg4, 0);
		break;
	case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA:
		ret = sbi_ecall(ext, fid, hmask_val, hbase, start,
		ret = sbi_ecall(ext, fid, hmask, hbase, start,
				size, 0, 0);
		break;
	case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID:
		ret = sbi_ecall(ext, fid, hmask_val, hbase, start,
		ret = sbi_ecall(ext, fid, hmask, hbase, start,
				size, arg4, 0);
		break;
	default:
@@ -303,43 +334,39 @@ static int __sbi_rfence_v02_call(unsigned long fid, unsigned long hmask_val,
	if (ret.error) {
		result = sbi_err_map_linux_errno(ret.error);
		pr_err("%s: hbase = [%lu] hmask = [0x%lx] failed (error [%d])\n",
		       __func__, hbase, hmask_val, result);
		       __func__, hbase, hmask, result);
	}

	return result;
}

static int __sbi_rfence_v02(int fid, const unsigned long *hart_mask,
static int __sbi_rfence_v02(int fid, const struct cpumask *cpu_mask,
			    unsigned long start, unsigned long size,
			    unsigned long arg4, unsigned long arg5)
{
	unsigned long hmask_val, hartid, hbase;
	struct cpumask tmask;
	unsigned long hartid, cpuid, hmask = 0, hbase = 0;
	int result;

	if (!hart_mask || !(*hart_mask)) {
		riscv_cpuid_to_hartid_mask(cpu_online_mask, &tmask);
		hart_mask = cpumask_bits(&tmask);
	}
	if (!cpu_mask)
		cpu_mask = cpu_online_mask;

	hmask_val = 0;
	hbase = 0;
	for_each_set_bit(hartid, hart_mask, NR_CPUS) {
		if (hmask_val && ((hbase + BITS_PER_LONG) <= hartid)) {
			result = __sbi_rfence_v02_call(fid, hmask_val, hbase,
	for_each_cpu(cpuid, cpu_mask) {
		hartid = cpuid_to_hartid_map(cpuid);
		if (hmask && ((hbase + BITS_PER_LONG) <= hartid)) {
			result = __sbi_rfence_v02_call(fid, hmask, hbase,
						       start, size, arg4, arg5);
			if (result)
				return result;
			hmask_val = 0;
			hmask = 0;
			hbase = 0;
		}
		if (!hmask_val)
		if (!hmask)
			hbase = hartid;
		hmask_val |= 1UL << (hartid - hbase);
		hmask |= 1UL << (hartid - hbase);
	}

	if (hmask_val) {
		result = __sbi_rfence_v02_call(fid, hmask_val, hbase,
	if (hmask) {
		result = __sbi_rfence_v02_call(fid, hmask, hbase,
					       start, size, arg4, arg5);
		if (result)
			return result;
@@ -361,44 +388,44 @@ void sbi_set_timer(uint64_t stime_value)

/**
 * sbi_send_ipi() - Send an IPI to any hart.
 * @hart_mask: A cpu mask containing all the target harts.
 * @cpu_mask: A cpu mask containing all the target harts.
 *
 * Return: 0 on success, appropriate linux error code otherwise.
 */
int sbi_send_ipi(const unsigned long *hart_mask)
int sbi_send_ipi(const struct cpumask *cpu_mask)
{
	return __sbi_send_ipi(hart_mask);
	return __sbi_send_ipi(cpu_mask);
}
EXPORT_SYMBOL(sbi_send_ipi);

/**
 * sbi_remote_fence_i() - Execute FENCE.I instruction on given remote harts.
 * @hart_mask: A cpu mask containing all the target harts.
 * @cpu_mask: A cpu mask containing all the target harts.
 *
 * Return: 0 on success, appropriate linux error code otherwise.
 */
int sbi_remote_fence_i(const unsigned long *hart_mask)
int sbi_remote_fence_i(const struct cpumask *cpu_mask)
{
	return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_FENCE_I,
			    hart_mask, 0, 0, 0, 0);
			    cpu_mask, 0, 0, 0, 0);
}
EXPORT_SYMBOL(sbi_remote_fence_i);

/**
 * sbi_remote_sfence_vma() - Execute SFENCE.VMA instructions on given remote
 *			     harts for the specified virtual address range.
 * @hart_mask: A cpu mask containing all the target harts.
 * @cpu_mask: A cpu mask containing all the target harts.
 * @start: Start of the virtual address
 * @size: Total size of the virtual address range.
 *
 * Return: 0 on success, appropriate linux error code otherwise.
 */
int sbi_remote_sfence_vma(const unsigned long *hart_mask,
int sbi_remote_sfence_vma(const struct cpumask *cpu_mask,
			   unsigned long start,
			   unsigned long size)
{
	return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA,
			    hart_mask, start, size, 0, 0);
			    cpu_mask, start, size, 0, 0);
}
EXPORT_SYMBOL(sbi_remote_sfence_vma);

@@ -406,38 +433,38 @@ EXPORT_SYMBOL(sbi_remote_sfence_vma);
 * sbi_remote_sfence_vma_asid() - Execute SFENCE.VMA instructions on given
 * remote harts for a virtual address range belonging to a specific ASID.
 *
 * @hart_mask: A cpu mask containing all the target harts.
 * @cpu_mask: A cpu mask containing all the target harts.
 * @start: Start of the virtual address
 * @size: Total size of the virtual address range.
 * @asid: The value of address space identifier (ASID).
 *
 * Return: 0 on success, appropriate linux error code otherwise.
 */
int sbi_remote_sfence_vma_asid(const unsigned long *hart_mask,
int sbi_remote_sfence_vma_asid(const struct cpumask *cpu_mask,
				unsigned long start,
				unsigned long size,
				unsigned long asid)
{
	return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID,
			    hart_mask, start, size, asid, 0);
			    cpu_mask, start, size, asid, 0);
}
EXPORT_SYMBOL(sbi_remote_sfence_vma_asid);

/**
 * sbi_remote_hfence_gvma() - Execute HFENCE.GVMA instructions on given remote
 *			   harts for the specified guest physical address range.
 * @hart_mask: A cpu mask containing all the target harts.
 * @cpu_mask: A cpu mask containing all the target harts.
 * @start: Start of the guest physical address
 * @size: Total size of the guest physical address range.
 *
 * Return: None
 */
int sbi_remote_hfence_gvma(const unsigned long *hart_mask,
int sbi_remote_hfence_gvma(const struct cpumask *cpu_mask,
			   unsigned long start,
			   unsigned long size)
{
	return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA,
			    hart_mask, start, size, 0, 0);
			    cpu_mask, start, size, 0, 0);
}
EXPORT_SYMBOL_GPL(sbi_remote_hfence_gvma);

@@ -445,38 +472,38 @@ EXPORT_SYMBOL_GPL(sbi_remote_hfence_gvma);
 * sbi_remote_hfence_gvma_vmid() - Execute HFENCE.GVMA instructions on given
 * remote harts for a guest physical address range belonging to a specific VMID.
 *
 * @hart_mask: A cpu mask containing all the target harts.
 * @cpu_mask: A cpu mask containing all the target harts.
 * @start: Start of the guest physical address
 * @size: Total size of the guest physical address range.
 * @vmid: The value of guest ID (VMID).
 *
 * Return: 0 if success, Error otherwise.
 */
int sbi_remote_hfence_gvma_vmid(const unsigned long *hart_mask,
int sbi_remote_hfence_gvma_vmid(const struct cpumask *cpu_mask,
				unsigned long start,
				unsigned long size,
				unsigned long vmid)
{
	return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID,
			    hart_mask, start, size, vmid, 0);
			    cpu_mask, start, size, vmid, 0);
}
EXPORT_SYMBOL(sbi_remote_hfence_gvma_vmid);

/**
 * sbi_remote_hfence_vvma() - Execute HFENCE.VVMA instructions on given remote
 *			     harts for the current guest virtual address range.
 * @hart_mask: A cpu mask containing all the target harts.
 * @cpu_mask: A cpu mask containing all the target harts.
 * @start: Start of the current guest virtual address
 * @size: Total size of the current guest virtual address range.
 *
 * Return: None
 */
int sbi_remote_hfence_vvma(const unsigned long *hart_mask,
int sbi_remote_hfence_vvma(const struct cpumask *cpu_mask,
			   unsigned long start,
			   unsigned long size)
{
	return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA,
			    hart_mask, start, size, 0, 0);
			    cpu_mask, start, size, 0, 0);
}
EXPORT_SYMBOL(sbi_remote_hfence_vvma);

@@ -485,20 +512,20 @@ EXPORT_SYMBOL(sbi_remote_hfence_vvma);
 * remote harts for current guest virtual address range belonging to a specific
 * ASID.
 *
 * @hart_mask: A cpu mask containing all the target harts.
 * @cpu_mask: A cpu mask containing all the target harts.
 * @start: Start of the current guest virtual address
 * @size: Total size of the current guest virtual address range.
 * @asid: The value of address space identifier (ASID).
 *
 * Return: None
 */
int sbi_remote_hfence_vvma_asid(const unsigned long *hart_mask,
int sbi_remote_hfence_vvma_asid(const struct cpumask *cpu_mask,
				unsigned long start,
				unsigned long size,
				unsigned long asid)
{
	return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID,
			    hart_mask, start, size, asid, 0);
			    cpu_mask, start, size, asid, 0);
}
EXPORT_SYMBOL(sbi_remote_hfence_vvma_asid);

@@ -591,11 +618,7 @@ long sbi_get_mimpid(void)

static void sbi_send_cpumask_ipi(const struct cpumask *target)
{
	struct cpumask hartid_mask;

	riscv_cpuid_to_hartid_mask(target, &hartid_mask);

	sbi_send_ipi(cpumask_bits(&hartid_mask));
	sbi_send_ipi(target);
}

static const struct riscv_ipi_ops sbi_ipi_ops = {
+0 −10
Original line number Diff line number Diff line
@@ -59,16 +59,6 @@ atomic_t hart_lottery __section(".sdata")
unsigned long boot_cpu_hartid;
static DEFINE_PER_CPU(struct cpu, cpu_devices);

void riscv_cpuid_to_hartid_mask(const struct cpumask *in, struct cpumask *out)
{
	int cpu;

	cpumask_clear(out);
	for_each_cpu(cpu, in)
		cpumask_set_cpu(cpuid_to_hartid_map(cpu), out);
}
EXPORT_SYMBOL_GPL(riscv_cpuid_to_hartid_mask);

/*
 * Place kernel memory regions on the resource tree so that
 * kexec-tools can retrieve them from /proc/iomem. While there
+1 −1
Original line number Diff line number Diff line
@@ -96,7 +96,7 @@ void __init setup_smp(void)
		if (cpuid >= NR_CPUS) {
			pr_warn("Invalid cpuid [%d] for hartid [%d]\n",
				cpuid, hart);
			break;
			continue;
		}

		cpuid_to_hartid_map(cpuid) = hart;
Loading