Commit dc163a41 authored by Ingo Molnar's avatar Ingo Molnar
Browse files

SGI UV: TLB shootdown using broadcast assist unit 



TLB shootdown for SGI UV.

v5: 6/12 corrections/improvements per Ingo's second review

Signed-off-by: default avatarCliff Wickman <cpw@sgi.com>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent b194b120

arch/x86/kernel/tlb_uv.c

+85 −98
Original line number Diff line number Diff line
@@ -25,15 +25,8 @@ static int uv_bau_retry_limit __read_mostly; 
static int uv_nshift __read_mostly; /* position of pnode (which is nasid>>1) */

static unsigned long uv_mmask __read_mostly;



char *status_table[] = {

	"IDLE",

	"ACTIVE",

	"DESTINATION TIMEOUT",

	"SOURCE TIMEOUT"

};



DEFINE_PER_CPU(struct ptc_stats, ptcstats);

DEFINE_PER_CPU(struct bau_control, bau_control);

static DEFINE_PER_CPU(struct ptc_stats, ptcstats);

static DEFINE_PER_CPU(struct bau_control, bau_control);



/*

 * Free a software acknowledge hardware resource by clearing its Pending
@@ -55,7 +48,6 @@ static void uv_reply_to_message(int resource, 
	if (msp)

		msp->seen_by.bits = 0;

	uv_write_local_mmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, dw);

	return;

}



/*
@@ -73,7 +65,7 @@ static void uv_bau_process_message(struct bau_payload_queue_entry *msg, 
	cpu = uv_blade_processor_id();

	msg->number_of_cpus =

	    uv_blade_nr_online_cpus(uv_node_to_blade_id(numa_node_id()));

	this_cpu_mask = (unsigned long)1 << cpu;

	this_cpu_mask = 1UL << cpu;

	if (msp->seen_by.bits & this_cpu_mask)

		return;

	atomic_or_long(&msp->seen_by.bits, this_cpu_mask);
@@ -94,53 +86,60 @@ static void uv_bau_process_message(struct bau_payload_queue_entry *msg, 
	atomic_inc_short(&msg->acknowledge_count);

	if (msg->number_of_cpus == msg->acknowledge_count)

		uv_reply_to_message(sw_ack_slot, msg, msp);

	return;

}



/*

 * Examine the payload queue on all the distribution nodes to see

 * Examine the payload queue on one distribution node to see

 * which messages have not been seen, and which cpu(s) have not seen them.

 *

 * Returns the number of cpu's that have not responded.

 */

static int uv_examine_destinations(struct bau_target_nodemask *distribution)

static int uv_examine_destination(struct bau_control *bau_tablesp, int sender)

{

	int sender;

	int i;

	int j;

	int k;

	int count = 0;

	struct bau_control *bau_tablesp;

	struct bau_payload_queue_entry *msg;

	struct bau_msg_status *msp;



	sender = smp_processor_id();

	for (i = 0; i < (sizeof(struct bau_target_nodemask) * BITSPERBYTE);

	     i++) {

		if (!bau_node_isset(i, distribution))

			continue;

		bau_tablesp = uv_bau_table_bases[i];

		for (msg = bau_tablesp->va_queue_first, j = 0;

		     j < DESTINATION_PAYLOAD_QUEUE_SIZE; msg++, j++) {

			if ((msg->sending_cpu == sender) &&

			    (!msg->replied_to)) {

				msp = bau_tablesp->msg_statuses + j;

	for (msg = bau_tablesp->va_queue_first, i = 0; i < DEST_Q_SIZE;

	     msg++, i++) {

		if ((msg->sending_cpu == sender) && (!msg->replied_to)) {

			msp = bau_tablesp->msg_statuses + i;

			printk(KERN_DEBUG

			       "blade %d: address:%#lx %d of %d, not cpu(s): ",

				       i, msg->address,

				       msg->acknowledge_count,

			       i, msg->address, msg->acknowledge_count,

			       msg->number_of_cpus);

				for (k = 0; k < msg->number_of_cpus;

				     k++) {

					if (!((long)1 << k & msp->

					      seen_by.bits)) {

			for (j = 0; j < msg->number_of_cpus; j++) {

				if (!((long)1 << j & msp-> seen_by.bits)) {

					count++;

						printk("%d ", k);

					printk("%d ", j);

				}

			}

			printk("\n");

		}

	}

	return count;

}



/*

 * Examine the payload queue on all the distribution nodes to see

 * which messages have not been seen, and which cpu(s) have not seen them.

 *

 * Returns the number of cpu's that have not responded.

 */

static int uv_examine_destinations(struct bau_target_nodemask *distribution)

{

	int sender;

	int i;

	int count = 0;



	sender = smp_processor_id();

	for (i = 0; i < (sizeof(struct bau_target_nodemask) * BITSPERBYTE);

	     i++) {

		if (!bau_node_isset(i, distribution))

			continue;

		count += uv_examine_destination(uv_bau_table_bases[i], sender);

	}

	return count;

}
@@ -150,7 +149,7 @@ static int uv_examine_destinations(struct bau_target_nodemask *distribution) 
 *

 * return COMPLETE, RETRY or GIVEUP

 */

static int uv_wait_completion(struct bau_activation_descriptor *bau_desc,

static int uv_wait_completion(struct bau_desc *bau_desc,

			      unsigned long mmr_offset, int right_shift)

{

	int exams = 0;
@@ -213,8 +212,8 @@ static int uv_wait_completion(struct bau_activation_descriptor *bau_desc, 
 * Returns 0 if some remote flushing remains to be done. The mask is left

 * unchanged.

 */

int uv_flush_send_and_wait(int cpu, int this_blade,

	struct bau_activation_descriptor *bau_desc, cpumask_t *cpumaskp)

int uv_flush_send_and_wait(int cpu, int this_blade, struct bau_desc *bau_desc,

			   cpumask_t *cpumaskp)

{

	int completion_status = 0;

	int right_shift;
@@ -237,8 +236,8 @@ int uv_flush_send_and_wait(int cpu, int this_blade, 
	time1 = get_cycles();

	do {

		tries++;

		index = ((unsigned long)

			1 << UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT) | cpu;

		index = (1UL << UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT) |

			cpu;

		uv_write_local_mmr(UVH_LB_BAU_SB_ACTIVATION_CONTROL, index);

		completion_status = uv_wait_completion(bau_desc, mmr_offset,

					right_shift);
@@ -303,7 +302,7 @@ int uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm, 
	int cpu;

	int this_blade;

	int locals = 0;

	struct bau_activation_descriptor *bau_desc;

	struct bau_desc *bau_desc;



	cpu = uv_blade_processor_id();

	this_blade = uv_numa_blade_id();
@@ -315,8 +314,7 @@ int uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm, 
	i = 0;

	for_each_cpu_mask(bit, *cpumaskp) {

		blade = uv_cpu_to_blade_id(bit);

		if (blade > (UV_DISTRIBUTION_SIZE - 1))

			BUG();

		BUG_ON(blade > (UV_DISTRIBUTION_SIZE - 1));

		if (blade == this_blade) {

			locals++;

			continue;
@@ -360,6 +358,8 @@ void uv_bau_message_interrupt(struct pt_regs *regs) 
{

	struct bau_payload_queue_entry *pqp;

	struct bau_payload_queue_entry *msg;

	struct bau_payload_queue_entry *va_queue_first;

	struct bau_payload_queue_entry *va_queue_last;

	struct pt_regs *old_regs = set_irq_regs(regs);

	cycles_t time1, time2;

	int msg_slot;
@@ -376,7 +376,8 @@ void uv_bau_message_interrupt(struct pt_regs *regs) 


	local_pnode = uv_blade_to_pnode(uv_numa_blade_id());



	pqp = __get_cpu_var(bau_control).va_queue_first;

	pqp = va_queue_first = __get_cpu_var(bau_control).va_queue_first;

	va_queue_last = __get_cpu_var(bau_control).va_queue_last;

	msg = __get_cpu_var(bau_control).bau_msg_head;

	while (msg->sw_ack_vector) {

		count++;
@@ -387,8 +388,8 @@ void uv_bau_message_interrupt(struct pt_regs *regs) 
		uv_bau_process_message(msg, msg_slot, sw_ack_slot);



		msg++;

		if (msg > __get_cpu_var(bau_control).va_queue_last)

			msg = __get_cpu_var(bau_control).va_queue_first;

		if (msg > va_queue_last)

			msg = va_queue_first;

		__get_cpu_var(bau_control).bau_msg_head = msg;

	}

	if (!count)
@@ -401,7 +402,6 @@ void uv_bau_message_interrupt(struct pt_regs *regs) 


	irq_exit();

	set_irq_regs(old_regs);

	return;

}



static void uv_enable_timeouts(void)
@@ -423,7 +423,6 @@ static void uv_enable_timeouts(void) 
		pnode = uv_blade_to_pnode(blade);

		cur_cpu += uv_blade_nr_possible_cpus(i);

	}

	return;

}



static void *uv_ptc_seq_start(struct seq_file *file, loff_t *offset)
@@ -568,6 +567,7 @@ static int __init uv_ptc_init(void) 
	if (!proc_uv_ptc) {

		printk(KERN_ERR "unable to create %s proc entry\n",

		       UV_PTC_BASENAME);

		remove_proc_entry("sgi_uv", NULL);

		return -EINVAL;

	}

	proc_uv_ptc->proc_fops = &proc_uv_ptc_operations;
@@ -582,33 +582,26 @@ static struct bau_control * __init uv_table_bases_init(int blade, int node) 
	int i;

	int *ip;

	struct bau_msg_status *msp;

	struct bau_control *bau_tablesp;

	struct bau_control *bau_tabp;



	bau_tablesp =

	bau_tabp =

	    kmalloc_node(sizeof(struct bau_control), GFP_KERNEL, node);

	if (!bau_tablesp)

		BUG();

	bau_tablesp->msg_statuses =

	BUG_ON(!bau_tabp);

	bau_tabp->msg_statuses =

	    kmalloc_node(sizeof(struct bau_msg_status) *

			 DESTINATION_PAYLOAD_QUEUE_SIZE, GFP_KERNEL, node);

	if (!bau_tablesp->msg_statuses)

		BUG();

	for (i = 0, msp = bau_tablesp->msg_statuses;

	     i < DESTINATION_PAYLOAD_QUEUE_SIZE; i++, msp++) {

			 DEST_Q_SIZE, GFP_KERNEL, node);

	BUG_ON(!bau_tabp->msg_statuses);

	for (i = 0, msp = bau_tabp->msg_statuses; i < DEST_Q_SIZE; i++, msp++)

		bau_cpubits_clear(&msp->seen_by, (int)

				  uv_blade_nr_possible_cpus(blade));

	}

	bau_tablesp->watching =

	    kmalloc_node(sizeof(int) * DESTINATION_NUM_RESOURCES,

			 GFP_KERNEL, node);

	if (!bau_tablesp->watching)

		BUG();

	for (i = 0, ip = bau_tablesp->watching;

	     i < DESTINATION_PAYLOAD_QUEUE_SIZE; i++, ip++) {

	bau_tabp->watching =

	    kmalloc_node(sizeof(int) * DEST_NUM_RESOURCES, GFP_KERNEL, node);

	BUG_ON(!bau_tabp->watching);

	for (i = 0, ip = bau_tabp->watching; i < DEST_Q_SIZE; i++, ip++) {

		*ip = 0;

	}

	uv_bau_table_bases[blade] = bau_tablesp;

	return bau_tablesp;

	uv_bau_table_bases[blade] = bau_tabp;

	return bau_tabsp;

}



/*
@@ -616,7 +609,7 @@ static struct bau_control * __init uv_table_bases_init(int blade, int node) 
 */

static void __init uv_table_bases_finish(int blade, int node, int cur_cpu,

				  struct bau_control *bau_tablesp,

				  struct bau_activation_descriptor *adp)

				  struct bau_desc *adp)

{

	int i;

	struct bau_control *bcp;
@@ -636,7 +629,7 @@ static void __init uv_table_bases_finish(int blade, int node, int cur_cpu, 
/*

 * initialize the sending side's sending buffers

 */

static struct bau_activation_descriptor * __init

static struct bau_desc * __init

uv_activation_descriptor_init(int node, int pnode)

{

	int i;
@@ -644,13 +637,12 @@ uv_activation_descriptor_init(int node, int pnode) 
	unsigned long m;

	unsigned long n;

	unsigned long mmr_image;

	struct bau_activation_descriptor *adp;

	struct bau_activation_descriptor *ad2;

	struct bau_desc *adp;

	struct bau_desc *ad2;



	adp = (struct bau_activation_descriptor *)

	adp = (struct bau_desc *)

	    kmalloc_node(16384, GFP_KERNEL, node);

	if (!adp)

		BUG();

	BUG_ON(!adp);

	pa = __pa((unsigned long)adp);

	n = pa >> uv_nshift;

	m = pa & uv_mmask;
@@ -660,7 +652,7 @@ uv_activation_descriptor_init(int node, int pnode) 
				      UVH_LB_BAU_SB_DESCRIPTOR_BASE,

				      (n << UV_DESC_BASE_PNODE_SHIFT | m));

	for (i = 0, ad2 = adp; i < UV_ACTIVATION_DESCRIPTOR_SIZE; i++, ad2++) {

		memset(ad2, 0, sizeof(struct bau_activation_descriptor));

		memset(ad2, 0, sizeof(struct bau_desc));

		ad2->header.sw_ack_flag = 1;

		ad2->header.base_dest_nodeid =

		    uv_blade_to_pnode(uv_cpu_to_blade_id(0));
@@ -683,12 +675,10 @@ static struct bau_payload_queue_entry * __init uv_payload_queue_init(int node, 
	char *cp;

	struct bau_payload_queue_entry *pqp;



	pqp = (struct bau_payload_queue_entry *)

	    kmalloc_node((DESTINATION_PAYLOAD_QUEUE_SIZE + 1) *

			 sizeof(struct bau_payload_queue_entry),

	pqp = (struct bau_payload_queue_entry *) kmalloc_node(

		(DEST_Q_SIZE + 1) * sizeof(struct bau_payload_queue_entry),

		GFP_KERNEL, node);

	if (!pqp)

		BUG();

	BUG_ON(!pqp);

	cp = (char *)pqp + 31;

	pqp = (struct bau_payload_queue_entry *)(((unsigned long)cp >> 5) << 5);

	bau_tablesp->va_queue_first = pqp;
@@ -699,13 +689,11 @@ static struct bau_payload_queue_entry * __init uv_payload_queue_init(int node, 
			      uv_physnodeaddr(pqp));

	uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL,

			      uv_physnodeaddr(pqp));

	bau_tablesp->va_queue_last =

	    pqp + (DESTINATION_PAYLOAD_QUEUE_SIZE - 1);

	bau_tablesp->va_queue_last = pqp + (DEST_Q_SIZE - 1);

	uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST,

			      (unsigned long)

			      uv_physnodeaddr(bau_tablesp->va_queue_last));

	memset(pqp, 0, sizeof(struct bau_payload_queue_entry) *

	       DESTINATION_PAYLOAD_QUEUE_SIZE);

	memset(pqp, 0, sizeof(struct bau_payload_queue_entry) * DEST_Q_SIZE);

	return pqp;

}
@@ -717,7 +705,7 @@ static int __init uv_init_blade(int blade, int node, int cur_cpu) 
	int pnode;

	unsigned long pa;

	unsigned long apicid;

	struct bau_activation_descriptor *adp;

	struct bau_desc *adp;

	struct bau_payload_queue_entry *pqp;

	struct bau_control *bau_tablesp;
@@ -755,7 +743,7 @@ static int __init uv_bau_init(void) 


	uv_bau_retry_limit = 1;

	uv_nshift = uv_hub_info->n_val;

	uv_mmask = ((unsigned long)1 << uv_hub_info->n_val) - 1;

	uv_mmask = (1UL << uv_hub_info->n_val) - 1;

	nblades = 0;

	last_blade = -1;

	for_each_online_node(node) {
@@ -767,8 +755,7 @@ static int __init uv_bau_init(void) 
	}

	uv_bau_table_bases = (struct bau_control **)

	    kmalloc(nblades * sizeof(struct bau_control *), GFP_KERNEL);

	if (!uv_bau_table_bases)

		BUG();

	BUG_ON(!uv_bau_table_bases);

	last_blade = -1;

	for_each_online_node(node) {

		blade = uv_node_to_blade_id(node);

include/asm-x86/uv/uv_bau.h

+11 −10
Original line number Diff line number Diff line
@@ -68,11 +68,11 @@ 
/*

 * number of entries in the destination side payload queue

 */

#define DESTINATION_PAYLOAD_QUEUE_SIZE  17

#define DEST_Q_SIZE			17

/*

 * number of destination side software ack resources

 */

#define DESTINATION_NUM_RESOURCES       8

#define DEST_NUM_RESOURCES		8

#define MAX_CPUS_PER_NODE		32

/*

 * completion statuses for sending a TLB flush message
@@ -197,10 +197,11 @@ struct bau_msg_header { 
};



/*

 * The activation descriptor:

 * The format of the message to send, plus all accompanying control

 * Should be 64 bytes

 */

struct bau_activation_descriptor {

struct bau_desc {

	struct bau_target_nodemask distribution;

	/*

	 * message template, consisting of header and payload:
@@ -281,7 +282,7 @@ struct bau_sw_ack_status { 
 * one on every node and per-cpu; to locate the software tables

 */

struct bau_control {

	struct bau_activation_descriptor *descriptor_base;

	struct bau_desc *descriptor_base;

	struct bau_payload_queue_entry *bau_msg_head;

	struct bau_payload_queue_entry *va_queue_first;

	struct bau_payload_queue_entry *va_queue_last;