Commit 1687d8ac authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'x86_apic_for_6.6-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull x86 apic updates from Dave Hansen:
 "This includes a very thorough rework of the 'struct apic' handlers.
  Quite a variety of them popped up over the years, especially in the
  32-bit days when odd apics were much more in vogue.

  The end result speaks for itself, which is a removal of a ton of code
  and static calls to replace indirect calls.

  If there's any breakage here, it's likely to be around the 32-bit
  museum pieces that get light to no testing these days.

  Summary:

   - Rework apic callbacks, getting rid of unnecessary ones and
     coalescing lots of silly duplicates.

   - Use static_calls() instead of indirect calls for apic->foo()

   - Tons of cleanups an crap removal along the way"

* tag 'x86_apic_for_6.6-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (64 commits)
  x86/apic: Turn on static calls
  x86/apic: Provide static call infrastructure for APIC callbacks
  x86/apic: Wrap IPI calls into helper functions
  x86/apic: Mark all hotpath APIC callback wrappers __always_inline
  x86/xen/apic: Mark apic __ro_after_init
  x86/apic: Convert other overrides to apic_update_callback()
  x86/apic: Replace acpi_wake_cpu_handler_update() and apic_set_eoi_cb()
  x86/apic: Provide apic_update_callback()
  x86/xen/apic: Use standard apic driver mechanism for Xen PV
  x86/apic: Provide common init infrastructure
  x86/apic: Wrap apic->native_eoi() into a helper
  x86/apic: Nuke ack_APIC_irq()
  x86/apic: Remove pointless arguments from [native_]eoi_write()
  x86/apic/noop: Tidy up the code
  x86/apic: Remove pointless NULL initializations
  x86/apic: Sanitize APIC ID range validation
  x86/apic: Prepare x2APIC for using apic::max_apic_id
  x86/apic: Simplify X2APIC ID validation
  x86/apic: Add max_apic_id member
  x86/apic: Wrap APIC ID validation into an inline
  ...
parents 87fa732d f8542a55

arch/x86/hyperv/hv_apic.c

+13 −13
Original line number Diff line number Diff line
@@ -86,14 +86,14 @@ static void hv_apic_write(u32 reg, u32 val) 
	}

}



static void hv_apic_eoi_write(u32 reg, u32 val)

static void hv_apic_eoi_write(void)

{

	struct hv_vp_assist_page *hvp = hv_vp_assist_page[smp_processor_id()];



	if (hvp && (xchg(&hvp->apic_assist, 0) & 0x1))

		return;



	wrmsr(HV_X64_MSR_EOI, val, 0);

	wrmsr(HV_X64_MSR_EOI, APIC_EOI_ACK, 0);

}



static bool cpu_is_self(int cpu)
@@ -286,12 +286,12 @@ void __init hv_apic_init(void) 
		 */

		orig_apic = *apic;



		apic->send_IPI = hv_send_ipi;

		apic->send_IPI_mask = hv_send_ipi_mask;

		apic->send_IPI_mask_allbutself = hv_send_ipi_mask_allbutself;

		apic->send_IPI_allbutself = hv_send_ipi_allbutself;

		apic->send_IPI_all = hv_send_ipi_all;

		apic->send_IPI_self = hv_send_ipi_self;

		apic_update_callback(send_IPI, hv_send_ipi);

		apic_update_callback(send_IPI_mask, hv_send_ipi_mask);

		apic_update_callback(send_IPI_mask_allbutself, hv_send_ipi_mask_allbutself);

		apic_update_callback(send_IPI_allbutself, hv_send_ipi_allbutself);

		apic_update_callback(send_IPI_all, hv_send_ipi_all);

		apic_update_callback(send_IPI_self, hv_send_ipi_self);

	}



	if (ms_hyperv.hints & HV_X64_APIC_ACCESS_RECOMMENDED) {
@@ -308,12 +308,12 @@ void __init hv_apic_init(void) 
		 * lazy EOI when available, but the same accessor works for

		 * both xapic and x2apic because the field layout is the same.

		 */

		apic_set_eoi_write(hv_apic_eoi_write);

		apic_update_callback(eoi, hv_apic_eoi_write);

		if (!x2apic_enabled()) {

			apic->read      = hv_apic_read;

			apic->write     = hv_apic_write;

			apic->icr_write = hv_apic_icr_write;

			apic->icr_read  = hv_apic_icr_read;

			apic_update_callback(read, hv_apic_read);

			apic_update_callback(write, hv_apic_write);

			apic_update_callback(icr_write, hv_apic_icr_write);

			apic_update_callback(icr_read, hv_apic_icr_read);

		}

	}

}

arch/x86/hyperv/hv_init.c

+1 −1
Original line number Diff line number Diff line
@@ -162,7 +162,7 @@ static inline bool hv_reenlightenment_available(void) 


DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_reenlightenment)

{

	ack_APIC_irq();

	apic_eoi();

	inc_irq_stat(irq_hv_reenlightenment_count);

	schedule_delayed_work(&hv_reenlightenment_work, HZ/10);

}

arch/x86/hyperv/hv_spinlock.c

+1 −1
Original line number Diff line number Diff line
@@ -20,7 +20,7 @@ static bool __initdata hv_pvspin = true; 


static void hv_qlock_kick(int cpu)

{

	apic->send_IPI(cpu, X86_PLATFORM_IPI_VECTOR);

	__apic_send_IPI(cpu, X86_PLATFORM_IPI_VECTOR);

}



static void hv_qlock_wait(u8 *byte, u8 val)

arch/x86/hyperv/hv_vtl.c

+1 −1
Original line number Diff line number Diff line
@@ -226,7 +226,7 @@ static int __init hv_vtl_early_init(void) 
			  "Please add 'noxsave' to the kernel command line.\n");



	real_mode_header = &hv_vtl_real_mode_header;

	apic->wakeup_secondary_cpu_64 = hv_vtl_wakeup_secondary_cpu;

	apic_update_callback(wakeup_secondary_cpu_64, hv_vtl_wakeup_secondary_cpu);



	return 0;

}

arch/x86/include/asm/apic.h

+140 −99
Original line number Diff line number Diff line
@@ -3,6 +3,7 @@ 
#define _ASM_X86_APIC_H



#include <linux/cpumask.h>

#include <linux/static_call.h>



#include <asm/alternative.h>

#include <asm/cpufeature.h>
@@ -40,11 +41,9 @@ 




#if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86_32)

extern void generic_apic_probe(void);

extern void x86_32_probe_apic(void);

#else

static inline void generic_apic_probe(void)

{

}

static inline void x86_32_probe_apic(void) { }

#endif



#ifdef CONFIG_X86_LOCAL_APIC
@@ -52,7 +51,7 @@ static inline void generic_apic_probe(void) 
extern int apic_verbosity;

extern int local_apic_timer_c2_ok;



extern int disable_apic;

extern bool apic_is_disabled;

extern unsigned int lapic_timer_period;



extern int cpuid_to_apicid[];
@@ -66,20 +65,6 @@ enum apic_intr_mode_id { 
	APIC_SYMMETRIC_IO_NO_ROUTING

};



#ifdef CONFIG_SMP

extern void __inquire_remote_apic(int apicid);

#else /* CONFIG_SMP */

static inline void __inquire_remote_apic(int apicid)

{

}

#endif /* CONFIG_SMP */



static inline void default_inquire_remote_apic(int apicid)

{

	if (apic_verbosity >= APIC_DEBUG)

		__inquire_remote_apic(apicid);

}



/*

 * With 82489DX we can't rely on apic feature bit

 * retrieved via cpuid but still have to deal with
@@ -90,7 +75,7 @@ static inline void default_inquire_remote_apic(int apicid) 
 */

static inline bool apic_from_smp_config(void)

{

	return smp_found_config && !disable_apic;

	return smp_found_config && !apic_is_disabled;

}



/*
@@ -114,8 +99,11 @@ static inline u32 native_apic_mem_read(u32 reg) 
	return *((volatile u32 *)(APIC_BASE + reg));

}



extern void native_apic_wait_icr_idle(void);

extern u32 native_safe_apic_wait_icr_idle(void);

static inline void native_apic_mem_eoi(void)

{

	native_apic_mem_write(APIC_EOI, APIC_EOI_ACK);

}



extern void native_apic_icr_write(u32 low, u32 id);

extern u64 native_apic_icr_read(void);
@@ -149,12 +137,12 @@ extern void setup_secondary_APIC_clock(void); 
extern void lapic_update_tsc_freq(void);



#ifdef CONFIG_X86_64

static inline int apic_force_enable(unsigned long addr)

static inline bool apic_force_enable(unsigned long addr)

{

	return -1;

	return false;

}

#else

extern int apic_force_enable(unsigned long addr);

extern bool apic_force_enable(unsigned long addr);

#endif



extern void apic_ap_setup(void);
@@ -207,7 +195,7 @@ static inline void native_apic_msr_write(u32 reg, u32 v) 
	wrmsr(APIC_BASE_MSR + (reg >> 4), v, 0);

}



static inline void native_apic_msr_eoi_write(u32 reg, u32 v)

static inline void native_apic_msr_eoi(void)

{

	__wrmsr(APIC_BASE_MSR + (APIC_EOI >> 4), APIC_EOI_ACK, 0);

}
@@ -223,18 +211,6 @@ static inline u32 native_apic_msr_read(u32 reg) 
	return (u32)msr;

}



static inline void native_x2apic_wait_icr_idle(void)

{

	/* no need to wait for icr idle in x2apic */

	return;

}



static inline u32 native_safe_x2apic_wait_icr_idle(void)

{

	/* no need to wait for icr idle in x2apic */

	return 0;

}



static inline void native_x2apic_icr_write(u32 low, u32 id)

{

	wrmsrl(APIC_BASE_MSR + (APIC_ICR >> 4), ((__u64) id) << 32 | low);
@@ -261,7 +237,7 @@ static inline int x2apic_enabled(void) 
#else /* !CONFIG_X86_X2APIC */

static inline void x2apic_setup(void) { }

static inline int x2apic_enabled(void) { return 0; }



static inline u32 native_apic_msr_read(u32 reg) { BUG(); }

#define x2apic_mode		(0)

#define	x2apic_supported()	(0)

#endif /* !CONFIG_X86_X2APIC */
@@ -280,8 +256,8 @@ struct irq_data; 
 */

struct apic {

	/* Hotpath functions first */

	void	(*eoi_write)(u32 reg, u32 v);

	void	(*native_eoi_write)(u32 reg, u32 v);

	void	(*eoi)(void);

	void	(*native_eoi)(void);

	void	(*write)(u32 reg, u32 v);

	u32	(*read)(u32 reg);
@@ -296,10 +272,11 @@ struct apic { 
	void	(*send_IPI_all)(int vector);

	void	(*send_IPI_self)(int vector);



	u32	disable_esr;



	enum apic_delivery_modes delivery_mode;

	bool	dest_mode_logical;



	u32	disable_esr		: 1,

		dest_mode_logical	: 1,

		x2apic_set_max_apicid	: 1;



	u32	(*calc_dest_apicid)(unsigned int cpu);
@@ -307,19 +284,18 @@ struct apic { 
	u64	(*icr_read)(void);

	void	(*icr_write)(u32 low, u32 high);



	/* The limit of the APIC ID space. */

	u32	max_apic_id;



	/* Probe, setup and smpboot functions */

	int	(*probe)(void);

	int	(*acpi_madt_oem_check)(char *oem_id, char *oem_table_id);

	int	(*apic_id_valid)(u32 apicid);

	int	(*apic_id_registered)(void);

	bool	(*apic_id_registered)(void);



	bool	(*check_apicid_used)(physid_mask_t *map, int apicid);

	void	(*init_apic_ldr)(void);

	void	(*ioapic_phys_id_map)(physid_mask_t *phys_map, physid_mask_t *retmap);

	void	(*setup_apic_routing)(void);

	int	(*cpu_present_to_apicid)(int mps_cpu);

	void	(*apicid_to_cpu_present)(int phys_apicid, physid_mask_t *retmap);

	int	(*check_phys_apicid_present)(int phys_apicid);

	int	(*phys_pkg_id)(int cpuid_apic, int index_msb);



	u32	(*get_apic_id)(unsigned long x);
@@ -330,24 +306,26 @@ struct apic { 
	/* wakeup secondary CPU using 64-bit wakeup point */

	int	(*wakeup_secondary_cpu_64)(int apicid, unsigned long start_eip);



	void	(*inquire_remote_apic)(int apicid);



#ifdef CONFIG_X86_32

	/*

	 * Called very early during boot from get_smp_config().  It should

	 * return the logical apicid.  x86_[bios]_cpu_to_apicid is

	 * initialized before this function is called.

	 *

	 * If logical apicid can't be determined that early, the function

	 * may return BAD_APICID.  Logical apicid will be configured after

	 * init_apic_ldr() while bringing up CPUs.  Note that NUMA affinity

	 * won't be applied properly during early boot in this case.

	 */

	int (*x86_32_early_logical_apicid)(int cpu);

#endif

	char	*name;

};



struct apic_override {

	void	(*eoi)(void);

	void	(*native_eoi)(void);

	void	(*write)(u32 reg, u32 v);

	u32	(*read)(u32 reg);

	void	(*send_IPI)(int cpu, int vector);

	void	(*send_IPI_mask)(const struct cpumask *mask, int vector);

	void	(*send_IPI_mask_allbutself)(const struct cpumask *msk, int vec);

	void	(*send_IPI_allbutself)(int vector);

	void	(*send_IPI_all)(int vector);

	void	(*send_IPI_self)(int vector);

	u64	(*icr_read)(void);

	void	(*icr_write)(u32 low, u32 high);

	int	(*wakeup_secondary_cpu)(int apicid, unsigned long start_eip);

	int	(*wakeup_secondary_cpu_64)(int apicid, unsigned long start_eip);

};



/*

 * Pointer to the local APIC driver in use on this system (there's

 * always just one such driver in use - the kernel decides via an
@@ -383,43 +361,111 @@ extern int lapic_can_unplug_cpu(void); 
#endif



#ifdef CONFIG_X86_LOCAL_APIC

extern struct apic_override __x86_apic_override;



void __init apic_setup_apic_calls(void);

void __init apic_install_driver(struct apic *driver);



#define apic_update_callback(_callback, _fn) {					\

		__x86_apic_override._callback = _fn;				\

		apic->_callback = _fn;						\

		static_call_update(apic_call_##_callback, _fn);			\

		pr_info("APIC: %s() replaced with %ps()\n", #_callback, _fn);	\

}



#define DECLARE_APIC_CALL(__cb)							\

	DECLARE_STATIC_CALL(apic_call_##__cb, *apic->__cb)



DECLARE_APIC_CALL(eoi);

DECLARE_APIC_CALL(native_eoi);

DECLARE_APIC_CALL(icr_read);

DECLARE_APIC_CALL(icr_write);

DECLARE_APIC_CALL(read);

DECLARE_APIC_CALL(send_IPI);

DECLARE_APIC_CALL(send_IPI_mask);

DECLARE_APIC_CALL(send_IPI_mask_allbutself);

DECLARE_APIC_CALL(send_IPI_allbutself);

DECLARE_APIC_CALL(send_IPI_all);

DECLARE_APIC_CALL(send_IPI_self);

DECLARE_APIC_CALL(wait_icr_idle);

DECLARE_APIC_CALL(wakeup_secondary_cpu);

DECLARE_APIC_CALL(wakeup_secondary_cpu_64);

DECLARE_APIC_CALL(write);



static __always_inline u32 apic_read(u32 reg)

{

	return static_call(apic_call_read)(reg);

}



static __always_inline void apic_write(u32 reg, u32 val)

{

	static_call(apic_call_write)(reg, val);

}



static __always_inline void apic_eoi(void)

{

	static_call(apic_call_eoi)();

}



static __always_inline void apic_native_eoi(void)

{

	static_call(apic_call_native_eoi)();

}



static __always_inline u64 apic_icr_read(void)

{

	return static_call(apic_call_icr_read)();

}



static __always_inline void apic_icr_write(u32 low, u32 high)

{

	static_call(apic_call_icr_write)(low, high);

}



static inline u32 apic_read(u32 reg)

static __always_inline void __apic_send_IPI(int cpu, int vector)

{

	return apic->read(reg);

	static_call(apic_call_send_IPI)(cpu, vector);

}



static inline void apic_write(u32 reg, u32 val)

static __always_inline void __apic_send_IPI_mask(const struct cpumask *mask, int vector)

{

	apic->write(reg, val);

	static_call_mod(apic_call_send_IPI_mask)(mask, vector);

}



static inline void apic_eoi(void)

static __always_inline void __apic_send_IPI_mask_allbutself(const struct cpumask *mask, int vector)

{

	apic->eoi_write(APIC_EOI, APIC_EOI_ACK);

	static_call(apic_call_send_IPI_mask_allbutself)(mask, vector);

}



static inline u64 apic_icr_read(void)

static __always_inline void __apic_send_IPI_allbutself(int vector)

{

	return apic->icr_read();

	static_call(apic_call_send_IPI_allbutself)(vector);

}



static inline void apic_icr_write(u32 low, u32 high)

static __always_inline void __apic_send_IPI_all(int vector)

{

	apic->icr_write(low, high);

	static_call(apic_call_send_IPI_all)(vector);

}



static inline void apic_wait_icr_idle(void)

static __always_inline void __apic_send_IPI_self(int vector)

{

	apic->wait_icr_idle();

	static_call_mod(apic_call_send_IPI_self)(vector);

}



static inline u32 safe_apic_wait_icr_idle(void)

static __always_inline void apic_wait_icr_idle(void)

{

	return apic->safe_wait_icr_idle();

	static_call_cond(apic_call_wait_icr_idle)();

}



extern void __init apic_set_eoi_write(void (*eoi_write)(u32 reg, u32 v));

static __always_inline u32 safe_apic_wait_icr_idle(void)

{

	return apic->safe_wait_icr_idle ? apic->safe_wait_icr_idle() : 0;

}



static __always_inline bool apic_id_valid(u32 apic_id)

{

	return apic_id <= apic->max_apic_id;

}



#else /* CONFIG_X86_LOCAL_APIC */
@@ -430,22 +476,16 @@ static inline u64 apic_icr_read(void) { return 0; } 
static inline void apic_icr_write(u32 low, u32 high) { }

static inline void apic_wait_icr_idle(void) { }

static inline u32 safe_apic_wait_icr_idle(void) { return 0; }

static inline void apic_set_eoi_write(void (*eoi_write)(u32 reg, u32 v)) {}

static inline void apic_set_eoi_cb(void (*eoi)(void)) {}

static inline void apic_native_eoi(void) { WARN_ON_ONCE(1); }

static inline void apic_setup_apic_calls(void) { }



#define apic_update_callback(_callback, _fn) do { } while (0)



#endif /* CONFIG_X86_LOCAL_APIC */



extern void apic_ack_irq(struct irq_data *data);



static inline void ack_APIC_irq(void)

{

	/*

	 * ack_APIC_irq() actually gets compiled as a single instruction

	 * ... yummie.

	 */

	apic_eoi();

}





static inline bool lapic_vector_set_in_irr(unsigned int vector)

{

	u32 irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
@@ -475,10 +515,6 @@ extern void generic_bigsmp_probe(void); 


#include <asm/smp.h>



#define APIC_DFR_VALUE	(APIC_DFR_FLAT)



DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid);



extern struct apic apic_noop;



static inline unsigned int read_apic_id(void)
@@ -490,12 +526,14 @@ static inline unsigned int read_apic_id(void) 


#ifdef CONFIG_X86_64

typedef int (*wakeup_cpu_handler)(int apicid, unsigned long start_eip);

extern void acpi_wake_cpu_handler_update(wakeup_cpu_handler handler);

extern int default_acpi_madt_oem_check(char *, char *);

extern void x86_64_probe_apic(void);

#else

static inline int default_acpi_madt_oem_check(char *a, char *b) { return 0; }

static inline void x86_64_probe_apic(void) { }

#endif



extern int default_apic_id_valid(u32 apicid);

extern int default_acpi_madt_oem_check(char *, char *);

extern void default_setup_apic_routing(void);



extern u32 apic_default_calc_apicid(unsigned int cpu);

extern u32 apic_flat_calc_apicid(unsigned int cpu);
@@ -503,9 +541,12 @@ extern u32 apic_flat_calc_apicid(unsigned int cpu); 
extern bool default_check_apicid_used(physid_mask_t *map, int apicid);

extern void default_ioapic_phys_id_map(physid_mask_t *phys_map, physid_mask_t *retmap);

extern int default_cpu_present_to_apicid(int mps_cpu);

extern int default_check_phys_apicid_present(int phys_apicid);



#endif /* CONFIG_X86_LOCAL_APIC */

#else /* CONFIG_X86_LOCAL_APIC */



static inline unsigned int read_apic_id(void) { return 0; }



#endif /* !CONFIG_X86_LOCAL_APIC */



#ifdef CONFIG_SMP

void apic_smt_update(void);