Commit 6a9ce66f authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring/net: make page accounting more consistent 



Make network page accounting more consistent with how buffer
registration is working, i.e. account all memory to ctx->user.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/4aacfe64bbb81b27f9ecf5d5c219c69a07e5aa56.1658742118.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 2e32ba56

io_uring/net.c

+1 −1
Original line number Diff line number Diff line
@@ -985,7 +985,7 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) 
					  &msg.msg_iter);

		if (unlikely(ret))

			return ret;

		ret = mm_account_pinned_pages(&notif->uarg.mmp, zc->len);

		ret = io_notif_account_mem(notif, zc->len);

		if (unlikely(ret))

			return ret;

	}

io_uring/notif.c

+4 −5
Original line number Diff line number Diff line
@@ -14,12 +14,10 @@ static void __io_notif_complete_tw(struct callback_head *cb) 
	struct io_notif *notif = container_of(cb, struct io_notif, task_work);

	struct io_rsrc_node *rsrc_node = notif->rsrc_node;

	struct io_ring_ctx *ctx = notif->ctx;

	struct mmpin *mmp = &notif->uarg.mmp;



	if (mmp->user) {

		atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm);

		free_uid(mmp->user);

		mmp->user = NULL;

	if (notif->account_pages && ctx->user) {

		__io_unaccount_mem(ctx->user, notif->account_pages);

		notif->account_pages = 0;

	}

	if (likely(notif->task)) {

		io_put_task(notif->task, 1);
@@ -121,6 +119,7 @@ struct io_notif *io_alloc_notif(struct io_ring_ctx *ctx, 
		notif->ctx = ctx;

		notif->uarg.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN;

		notif->uarg.callback = io_uring_tx_zerocopy_callback;

		notif->account_pages = 0;

	}



	notif->seq = slot->seq++;

io_uring/notif.h

+19 −0
Original line number Diff line number Diff line
@@ -5,6 +5,8 @@ 
#include <net/sock.h>

#include <linux/nospec.h>



#include "rsrc.h"



#define IO_NOTIF_SPLICE_BATCH	32

#define IORING_MAX_NOTIF_SLOTS (1U << 10)
@@ -23,6 +25,8 @@ struct io_notif { 
	/* hook into ctx->notif_list and ctx->notif_list_locked */

	struct list_head	cache_node;



	unsigned long		account_pages;



	union {

		struct callback_head	task_work;

		struct work_struct	commit_work;
@@ -85,3 +89,18 @@ static inline void io_notif_slot_flush_submit(struct io_notif_slot *slot, 
	}

	io_notif_slot_flush(slot);

}



static inline int io_notif_account_mem(struct io_notif *notif, unsigned len)

{

	struct io_ring_ctx *ctx = notif->ctx;

	unsigned nr_pages = (len >> PAGE_SHIFT) + 2;

	int ret;



	if (ctx->user) {

		ret = __io_account_mem(ctx->user, nr_pages);

		if (ret)

			return ret;

		notif->account_pages += nr_pages;

	}

	return 0;

}

io_uring/rsrc.c

+4 −8
Original line number Diff line number Diff line
@@ -44,17 +44,13 @@ void io_rsrc_refs_drop(struct io_ring_ctx *ctx) 
	}

}



static inline void __io_unaccount_mem(struct user_struct *user,

				      unsigned long nr_pages)

{

	atomic_long_sub(nr_pages, &user->locked_vm);

}



static inline int __io_account_mem(struct user_struct *user,

				   unsigned long nr_pages)

int __io_account_mem(struct user_struct *user, unsigned long nr_pages)

{

	unsigned long page_limit, cur_pages, new_pages;



	if (!nr_pages)

		return 0;



	/* Don't allow more pages than we can safely lock */

	page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;

io_uring/rsrc.h

+9 −0
Original line number Diff line number Diff line
@@ -169,4 +169,13 @@ static inline u64 *io_get_tag_slot(struct io_rsrc_data *data, unsigned int idx) 


int io_rsrc_update(struct io_kiocb *req, unsigned int issue_flags);

int io_rsrc_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);



int __io_account_mem(struct user_struct *user, unsigned long nr_pages);



static inline void __io_unaccount_mem(struct user_struct *user,

				      unsigned long nr_pages)

{

	atomic_long_sub(nr_pages, &user->locked_vm);

}



#endif