Commit 0a67c46d authored by Bob Pearson's avatar Bob Pearson Committed by Jason Gunthorpe
Browse files

RDMA/rxe: Protect user space index loads/stores 

Modify the queue APIs to protect all user space index loads with
smp_load_acquire() and all user space index stores with
smp_store_release(). Base this on the types of the queues which can be one
of ..KERNEL, ..FROM_USER, ..TO_USER. Kernel space indices are protected by
locks which also provide memory barriers.

Link: https://lore.kernel.org/r/20210527194748.662636-3-rpearsonhpe@gmail.com


Signed-off-by: default avatarBob Pearson <rpearsonhpe@gmail.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@nvidia.com>
parent 59daff49

drivers/infiniband/sw/rxe/rxe_queue.h

+117 −51
Original line number Diff line number Diff line
@@ -66,12 +66,22 @@ static inline int queue_empty(struct rxe_queue *q) 
	u32 prod;

	u32 cons;



	/* make sure all changes to queue complete before

	 * testing queue empty

	 */

	switch (q->type) {

	case QUEUE_TYPE_FROM_USER:

		/* protect user space index */

		prod = smp_load_acquire(&q->buf->producer_index);

	/* same */

		cons = q->buf->consumer_index;

		break;

	case QUEUE_TYPE_TO_USER:

		prod = q->buf->producer_index;

		/* protect user space index */

		cons = smp_load_acquire(&q->buf->consumer_index);

		break;

	case QUEUE_TYPE_KERNEL:

		prod = q->buf->producer_index;

		cons = q->buf->consumer_index;

		break;

	}



	return ((prod - cons) & q->index_mask) == 0;

}
@@ -81,79 +91,141 @@ static inline int queue_full(struct rxe_queue *q) 
	u32 prod;

	u32 cons;



	/* make sure all changes to queue complete before

	 * testing queue full

	 */

	switch (q->type) {

	case QUEUE_TYPE_FROM_USER:

		/* protect user space index */

		prod = smp_load_acquire(&q->buf->producer_index);

	/* same */

		cons = q->buf->consumer_index;

		break;

	case QUEUE_TYPE_TO_USER:

		prod = q->buf->producer_index;

		/* protect user space index */

		cons = smp_load_acquire(&q->buf->consumer_index);

		break;

	case QUEUE_TYPE_KERNEL:

		prod = q->buf->producer_index;

		cons = q->buf->consumer_index;

		break;

	}



	return ((prod + 1 - cons) & q->index_mask) == 0;

}



static inline void advance_producer(struct rxe_queue *q)

static inline unsigned int queue_count(const struct rxe_queue *q)

{

	u32 prod;

	u32 cons;



	switch (q->type) {

	case QUEUE_TYPE_FROM_USER:

		/* protect user space index */

		prod = smp_load_acquire(&q->buf->producer_index);

		cons = q->buf->consumer_index;

		break;

	case QUEUE_TYPE_TO_USER:

		prod = q->buf->producer_index;

		/* protect user space index */

		cons = smp_load_acquire(&q->buf->consumer_index);

		break;

	case QUEUE_TYPE_KERNEL:

		prod = q->buf->producer_index;

		cons = q->buf->consumer_index;

		break;

	}



	prod = (q->buf->producer_index + 1) & q->index_mask;

	return (prod - cons) & q->index_mask;

}



	/* make sure all changes to queue complete before

	 * changing producer index

	 */

static inline void advance_producer(struct rxe_queue *q)

{

	u32 prod;



	if (q->type == QUEUE_TYPE_FROM_USER) {

		/* protect user space index */

		prod = smp_load_acquire(&q->buf->producer_index);

		prod = (prod + 1) & q->index_mask;

		/* same */

		smp_store_release(&q->buf->producer_index, prod);

	} else {

		prod = q->buf->producer_index;

		q->buf->producer_index = (prod + 1) & q->index_mask;

	}

}



static inline void advance_consumer(struct rxe_queue *q)

{

	u32 cons;



	cons = (q->buf->consumer_index + 1) & q->index_mask;



	/* make sure all changes to queue complete before

	 * changing consumer index

	 */

	if (q->type == QUEUE_TYPE_TO_USER) {

		/* protect user space index */

		cons = smp_load_acquire(&q->buf->consumer_index);

		cons = (cons + 1) & q->index_mask;

		/* same */

		smp_store_release(&q->buf->consumer_index, cons);

	} else {

		cons = q->buf->consumer_index;

		q->buf->consumer_index = (cons + 1) & q->index_mask;

	}

}



static inline void *producer_addr(struct rxe_queue *q)

{

	return q->buf->data + ((q->buf->producer_index & q->index_mask)

				<< q->log2_elem_size);

	u32 prod;



	if (q->type == QUEUE_TYPE_FROM_USER)

		/* protect user space index */

		prod = smp_load_acquire(&q->buf->producer_index);

	else

		prod = q->buf->producer_index;



	return q->buf->data + ((prod & q->index_mask) << q->log2_elem_size);

}



static inline void *consumer_addr(struct rxe_queue *q)

{

	return q->buf->data + ((q->buf->consumer_index & q->index_mask)

				<< q->log2_elem_size);

	u32 cons;



	if (q->type == QUEUE_TYPE_TO_USER)

		/* protect user space index */

		cons = smp_load_acquire(&q->buf->consumer_index);

	else

		cons = q->buf->consumer_index;



	return q->buf->data + ((cons & q->index_mask) << q->log2_elem_size);

}



static inline unsigned int producer_index(struct rxe_queue *q)

{

	u32 index;

	u32 prod;



	/* make sure all changes to queue

	 * complete before getting producer index

	 */

	index = smp_load_acquire(&q->buf->producer_index);

	index &= q->index_mask;

	if (q->type == QUEUE_TYPE_FROM_USER)

		/* protect user space index */

		prod = smp_load_acquire(&q->buf->producer_index);

	else

		prod = q->buf->producer_index;



	prod &= q->index_mask;



	return index;

	return prod;

}



static inline unsigned int consumer_index(struct rxe_queue *q)

{

	u32 index;

	u32 cons;



	/* make sure all changes to queue

	 * complete before getting consumer index

	 */

	index = smp_load_acquire(&q->buf->consumer_index);

	index &= q->index_mask;

	if (q->type == QUEUE_TYPE_TO_USER)

		/* protect user space index */

		cons = smp_load_acquire(&q->buf->consumer_index);

	else

		cons = q->buf->consumer_index;



	return index;

	cons &= q->index_mask;



	return cons;

}



static inline void *addr_from_index(struct rxe_queue *q, unsigned int index)

static inline void *addr_from_index(struct rxe_queue *q,

				unsigned int index)

{

	return q->buf->data + ((index & q->index_mask)

				<< q->buf->log2_elem_size);
@@ -166,12 +238,6 @@ static inline unsigned int index_from_addr(const struct rxe_queue *q, 
				& q->index_mask;

}



static inline unsigned int queue_count(const struct rxe_queue *q)

{

	return (q->buf->producer_index - q->buf->consumer_index)

		& q->index_mask;

}



static inline void *queue_head(struct rxe_queue *q)

{

	return queue_empty(q) ? NULL : consumer_addr(q);