Commit 30a33669 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring/poll: fix double poll req->flags races 



io_poll_double_prepare()            | io_poll_wake()
                                    | poll->head = NULL
smp_load(&poll->head); /* NULL */   |
flags = req->flags;                 |
                                    | req->flags &= ~SINGLE_POLL;
req->flags = flags | DOUBLE_POLL    |

The idea behind io_poll_double_prepare() is to serialise with the
first poll entry by taking the wq lock. However, it's not safe to assume
that io_poll_wake() is not running when we can't grab the lock and so we
may race modifying req->flags.

Skip double poll setup if that happens. It's ok because the first poll
entry will only be removed when it's definitely completing, e.g.
pollfree or oneshot with a valid mask.

Fixes: 49f1c68e ("io_uring: optimise submission side poll_refs")
Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/b7fab2d502f6121a7d7b199fe4d914a43ca9cdfd.1668184658.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 3851d25c

io_uring/poll.c

+17 −12
Original line number Diff line number Diff line
@@ -394,7 +394,8 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, 
	return 1;

}



static void io_poll_double_prepare(struct io_kiocb *req)

/* fails only when polling is already completing by the first entry */

static bool io_poll_double_prepare(struct io_kiocb *req)

{

	struct wait_queue_head *head;

	struct io_poll *poll = io_poll_get_single(req);
@@ -403,20 +404,20 @@ static void io_poll_double_prepare(struct io_kiocb *req) 
	rcu_read_lock();

	head = smp_load_acquire(&poll->head);

	/*

	 * poll arm may not hold ownership and so race with

	 * io_poll_wake() by modifying req->flags. There is only one

	 * poll entry queued, serialise with it by taking its head lock.

	 * poll arm might not hold ownership and so race for req->flags with

	 * io_poll_wake(). There is only one poll entry queued, serialise with

	 * it by taking its head lock. As we're still arming the tw hanlder

	 * is not going to be run, so there are no races with it.

	 */

	if (head)

	if (head) {

		spin_lock_irq(&head->lock);



		req->flags |= REQ_F_DOUBLE_POLL;

		if (req->opcode == IORING_OP_POLL_ADD)

			req->flags |= REQ_F_ASYNC_DATA;



	if (head)

		spin_unlock_irq(&head->lock);

	}

	rcu_read_unlock();

	return !!head;

}



static void __io_queue_proc(struct io_poll *poll, struct io_poll_table *pt,
@@ -454,7 +455,11 @@ static void __io_queue_proc(struct io_poll *poll, struct io_poll_table *pt, 
		/* mark as double wq entry */

		wqe_private |= IO_WQE_F_DOUBLE;

		io_init_poll_iocb(poll, first->events, first->wait.func);

		io_poll_double_prepare(req);

		if (!io_poll_double_prepare(req)) {

			/* the request is completing, just back off */

			kfree(poll);

			return;

		}

		*poll_ptr = poll;

	} else {

		/* fine to modify, there is no poll queued to race with us */