Commit 2b7196a2 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'io_uring-5.16-2021-11-13' of git://git.kernel.dk/linux-block 

Pull io_uring fix from Jens Axboe:
 "Just a single fix here for a buffered write hash stall, which is also
  affecting stable"

* tag 'io_uring-5.16-2021-11-13' of git://git.kernel.dk/linux-block:
  io-wq: serialize hash clear with wakeup
parents c8103c27 d3e3c102

fs/io-wq.c

+15 −2
Original line number Diff line number Diff line
@@ -423,9 +423,10 @@ static inline unsigned int io_get_work_hash(struct io_wq_work *work) 
	return work->flags >> IO_WQ_HASH_SHIFT;

}



static void io_wait_on_hash(struct io_wqe *wqe, unsigned int hash)

static bool io_wait_on_hash(struct io_wqe *wqe, unsigned int hash)

{

	struct io_wq *wq = wqe->wq;

	bool ret = false;



	spin_lock_irq(&wq->hash->wait.lock);

	if (list_empty(&wqe->wait.entry)) {
@@ -433,9 +434,11 @@ static void io_wait_on_hash(struct io_wqe *wqe, unsigned int hash) 
		if (!test_bit(hash, &wq->hash->map)) {

			__set_current_state(TASK_RUNNING);

			list_del_init(&wqe->wait.entry);

			ret = true;

		}

	}

	spin_unlock_irq(&wq->hash->wait.lock);

	return ret;

}



static struct io_wq_work *io_get_next_work(struct io_wqe_acct *acct,
@@ -475,14 +478,21 @@ static struct io_wq_work *io_get_next_work(struct io_wqe_acct *acct, 
	}



	if (stall_hash != -1U) {

		bool unstalled;



		/*

		 * Set this before dropping the lock to avoid racing with new

		 * work being added and clearing the stalled bit.

		 */

		set_bit(IO_ACCT_STALLED_BIT, &acct->flags);

		raw_spin_unlock(&wqe->lock);

		io_wait_on_hash(wqe, stall_hash);

		unstalled = io_wait_on_hash(wqe, stall_hash);

		raw_spin_lock(&wqe->lock);

		if (unstalled) {

			clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);

			if (wq_has_sleeper(&wqe->wq->hash->wait))

				wake_up(&wqe->wq->hash->wait);

		}

	}



	return NULL;
@@ -564,8 +574,11 @@ static void io_worker_handle_work(struct io_worker *worker) 
				io_wqe_enqueue(wqe, linked);



			if (hash != -1U && !next_hashed) {

				/* serialize hash clear with wake_up() */

				spin_lock_irq(&wq->hash->wait.lock);

				clear_bit(hash, &wq->hash->map);

				clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);

				spin_unlock_irq(&wq->hash->wait.lock);

				if (wq_has_sleeper(&wq->hash->wait))

					wake_up(&wq->hash->wait);

				raw_spin_lock(&wqe->lock);