Commit fd0928df authored by Jens Axboe's avatar Jens Axboe
Browse files

ioprio: move io priority from task_struct to io_context 



This is where it belongs and then it doesn't take up space for a
process that doesn't do IO.

Signed-off-by: default avatarJens Axboe <jens.axboe@oracle.com>
parent 91525300

block/cfq-iosched.c

+16 −18
Original line number Diff line number Diff line
@@ -199,7 +199,7 @@ CFQ_CFQQ_FNS(sync); 


static void cfq_dispatch_insert(struct request_queue *, struct request *);

static struct cfq_queue *cfq_get_queue(struct cfq_data *, int,

				       struct task_struct *, gfp_t);

				       struct io_context *, gfp_t);

static struct cfq_io_context *cfq_cic_rb_lookup(struct cfq_data *,

						struct io_context *);
@@ -1273,7 +1273,7 @@ cfq_alloc_io_context(struct cfq_data *cfqd, gfp_t gfp_mask) 
	return cic;

}



static void cfq_init_prio_data(struct cfq_queue *cfqq)

static void cfq_init_prio_data(struct cfq_queue *cfqq, struct io_context *ioc)

{

	struct task_struct *tsk = current;

	int ioprio_class;
@@ -1281,7 +1281,7 @@ static void cfq_init_prio_data(struct cfq_queue *cfqq) 
	if (!cfq_cfqq_prio_changed(cfqq))

		return;



	ioprio_class = IOPRIO_PRIO_CLASS(tsk->ioprio);

	ioprio_class = IOPRIO_PRIO_CLASS(ioc->ioprio);

	switch (ioprio_class) {

		default:

			printk(KERN_ERR "cfq: bad prio %x\n", ioprio_class);
@@ -1293,11 +1293,11 @@ static void cfq_init_prio_data(struct cfq_queue *cfqq) 
			cfqq->ioprio_class = IOPRIO_CLASS_BE;

			break;

		case IOPRIO_CLASS_RT:

			cfqq->ioprio = task_ioprio(tsk);

			cfqq->ioprio = task_ioprio(ioc);

			cfqq->ioprio_class = IOPRIO_CLASS_RT;

			break;

		case IOPRIO_CLASS_BE:

			cfqq->ioprio = task_ioprio(tsk);

			cfqq->ioprio = task_ioprio(ioc);

			cfqq->ioprio_class = IOPRIO_CLASS_BE;

			break;

		case IOPRIO_CLASS_IDLE:
@@ -1330,8 +1330,7 @@ static inline void changed_ioprio(struct cfq_io_context *cic) 
	cfqq = cic->cfqq[ASYNC];

	if (cfqq) {

		struct cfq_queue *new_cfqq;

		new_cfqq = cfq_get_queue(cfqd, ASYNC, cic->ioc->task,

					 GFP_ATOMIC);

		new_cfqq = cfq_get_queue(cfqd, ASYNC, cic->ioc, GFP_ATOMIC);

		if (new_cfqq) {

			cic->cfqq[ASYNC] = new_cfqq;

			cfq_put_queue(cfqq);
@@ -1363,13 +1362,13 @@ static void cfq_ioc_set_ioprio(struct io_context *ioc) 


static struct cfq_queue *

cfq_find_alloc_queue(struct cfq_data *cfqd, int is_sync,

		     struct task_struct *tsk, gfp_t gfp_mask)

		     struct io_context *ioc, gfp_t gfp_mask)

{

	struct cfq_queue *cfqq, *new_cfqq = NULL;

	struct cfq_io_context *cic;



retry:

	cic = cfq_cic_rb_lookup(cfqd, tsk->io_context);

	cic = cfq_cic_rb_lookup(cfqd, ioc);

	/* cic always exists here */

	cfqq = cic_to_cfqq(cic, is_sync);
@@ -1412,7 +1411,7 @@ cfq_find_alloc_queue(struct cfq_data *cfqd, int is_sync, 
		cfq_mark_cfqq_prio_changed(cfqq);

		cfq_mark_cfqq_queue_new(cfqq);



		cfq_init_prio_data(cfqq);

		cfq_init_prio_data(cfqq, ioc);

	}



	if (new_cfqq)
@@ -1439,11 +1438,11 @@ cfq_async_queue_prio(struct cfq_data *cfqd, int ioprio_class, int ioprio) 
}



static struct cfq_queue *

cfq_get_queue(struct cfq_data *cfqd, int is_sync, struct task_struct *tsk,

cfq_get_queue(struct cfq_data *cfqd, int is_sync, struct io_context *ioc,

	      gfp_t gfp_mask)

{

	const int ioprio = task_ioprio(tsk);

	const int ioprio_class = task_ioprio_class(tsk);

	const int ioprio = task_ioprio(ioc);

	const int ioprio_class = task_ioprio_class(ioc);

	struct cfq_queue **async_cfqq = NULL;

	struct cfq_queue *cfqq = NULL;
@@ -1453,7 +1452,7 @@ cfq_get_queue(struct cfq_data *cfqd, int is_sync, struct task_struct *tsk, 
	}



	if (!cfqq) {

		cfqq = cfq_find_alloc_queue(cfqd, is_sync, tsk, gfp_mask);

		cfqq = cfq_find_alloc_queue(cfqd, is_sync, ioc, gfp_mask);

		if (!cfqq)

			return NULL;

	}
@@ -1793,7 +1792,7 @@ static void cfq_insert_request(struct request_queue *q, struct request *rq) 
	struct cfq_data *cfqd = q->elevator->elevator_data;

	struct cfq_queue *cfqq = RQ_CFQQ(rq);



	cfq_init_prio_data(cfqq);

	cfq_init_prio_data(cfqq, RQ_CIC(rq)->ioc);



	cfq_add_rq_rb(rq);
@@ -1900,7 +1899,7 @@ static int cfq_may_queue(struct request_queue *q, int rw) 


	cfqq = cic_to_cfqq(cic, rw & REQ_RW_SYNC);

	if (cfqq) {

		cfq_init_prio_data(cfqq);

		cfq_init_prio_data(cfqq, cic->ioc);

		cfq_prio_boost(cfqq);



		return __cfq_may_queue(cfqq);
@@ -1938,7 +1937,6 @@ static int 
cfq_set_request(struct request_queue *q, struct request *rq, gfp_t gfp_mask)

{

	struct cfq_data *cfqd = q->elevator->elevator_data;

	struct task_struct *tsk = current;

	struct cfq_io_context *cic;

	const int rw = rq_data_dir(rq);

	const int is_sync = rq_is_sync(rq);
@@ -1956,7 +1954,7 @@ cfq_set_request(struct request_queue *q, struct request *rq, gfp_t gfp_mask) 


	cfqq = cic_to_cfqq(cic, is_sync);

	if (!cfqq) {

		cfqq = cfq_get_queue(cfqd, is_sync, tsk, gfp_mask);

		cfqq = cfq_get_queue(cfqd, is_sync, cic->ioc, gfp_mask);



		if (!cfqq)

			goto queue_fail;

block/ll_rw_blk.c

+21 −9
Original line number Diff line number Diff line
@@ -3904,6 +3904,26 @@ void exit_io_context(void) 
	put_io_context(ioc);

}



struct io_context *alloc_io_context(gfp_t gfp_flags, int node)

{

	struct io_context *ret;



	ret = kmem_cache_alloc_node(iocontext_cachep, gfp_flags, node);

	if (ret) {

		atomic_set(&ret->refcount, 1);

		ret->task = current;

		ret->ioprio_changed = 0;

		ret->ioprio = 0;

		ret->last_waited = jiffies; /* doesn't matter... */

		ret->nr_batch_requests = 0; /* because this is 0 */

		ret->aic = NULL;

		ret->cic_root.rb_node = NULL;

		ret->ioc_data = NULL;

	}



	return ret;

}



/*

 * If the current task has no IO context then create one and initialise it.

 * Otherwise, return its existing IO context.
@@ -3921,16 +3941,8 @@ static struct io_context *current_io_context(gfp_t gfp_flags, int node) 
	if (likely(ret))

		return ret;



	ret = kmem_cache_alloc_node(iocontext_cachep, gfp_flags, node);

	ret = alloc_io_context(gfp_flags, node);

	if (ret) {

		atomic_set(&ret->refcount, 1);

		ret->task = current;

		ret->ioprio_changed = 0;

		ret->last_waited = jiffies; /* doesn't matter... */

		ret->nr_batch_requests = 0; /* because this is 0 */

		ret->aic = NULL;

		ret->cic_root.rb_node = NULL;

		ret->ioc_data = NULL;

		/* make sure set_task_ioprio() sees the settings above */

		smp_wmb();

		tsk->io_context = ret;

fs/ioprio.c

+21 −8
Original line number Diff line number Diff line
@@ -41,18 +41,29 @@ static int set_task_ioprio(struct task_struct *task, int ioprio) 
		return err;



	task_lock(task);



	task->ioprio = ioprio;



	do {

		ioc = task->io_context;

		/* see wmb() in current_io_context() */

		smp_read_barrier_depends();



		if (ioc)

			break;



		ioc = alloc_io_context(GFP_ATOMIC, -1);

		if (!ioc) {

			err = -ENOMEM;

			break;

		}

		task->io_context = ioc;

		ioc->task = task;

	} while (1);



	if (!err) {

		ioc->ioprio = ioprio;

		ioc->ioprio_changed = 1;

	}



	task_unlock(task);

	return 0;

	return err;

}



asmlinkage long sys_ioprio_set(int which, int who, int ioprio)
@@ -148,7 +159,9 @@ static int get_task_ioprio(struct task_struct *p) 
	ret = security_task_getioprio(p);

	if (ret)

		goto out;

	ret = p->ioprio;

	ret = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, IOPRIO_NORM);

	if (p->io_context)

		ret = p->io_context->ioprio;

out:

	return ret;

}

include/linux/blkdev.h

+7 −74
Original line number Diff line number Diff line
@@ -34,83 +34,10 @@ struct sg_io_hdr; 
#define BLKDEV_MIN_RQ	4

#define BLKDEV_MAX_RQ	128	/* Default maximum */



/*

 * This is the per-process anticipatory I/O scheduler state.

 */

struct as_io_context {

	spinlock_t lock;



	void (*dtor)(struct as_io_context *aic); /* destructor */

	void (*exit)(struct as_io_context *aic); /* called on task exit */



	unsigned long state;

	atomic_t nr_queued; /* queued reads & sync writes */

	atomic_t nr_dispatched; /* number of requests gone to the drivers */



	/* IO History tracking */

	/* Thinktime */

	unsigned long last_end_request;

	unsigned long ttime_total;

	unsigned long ttime_samples;

	unsigned long ttime_mean;

	/* Layout pattern */

	unsigned int seek_samples;

	sector_t last_request_pos;

	u64 seek_total;

	sector_t seek_mean;

};



struct cfq_queue;

struct cfq_io_context {

	struct rb_node rb_node;

	void *key;



	struct cfq_queue *cfqq[2];



	struct io_context *ioc;



	unsigned long last_end_request;

	sector_t last_request_pos;



	unsigned long ttime_total;

	unsigned long ttime_samples;

	unsigned long ttime_mean;



	unsigned int seek_samples;

	u64 seek_total;

	sector_t seek_mean;



	struct list_head queue_list;



	void (*dtor)(struct io_context *); /* destructor */

	void (*exit)(struct io_context *); /* called on task exit */

};



/*

 * This is the per-process I/O subsystem state.  It is refcounted and

 * kmalloc'ed. Currently all fields are modified in process io context

 * (apart from the atomic refcount), so require no locking.

 */

struct io_context {

	atomic_t refcount;

	struct task_struct *task;



	unsigned int ioprio_changed;



	/*

	 * For request batching

	 */

	unsigned long last_waited; /* Time last woken after wait for request */

	int nr_batch_requests;     /* Number of requests left in the batch */



	struct as_io_context *aic;

	struct rb_root cic_root;

	void *ioc_data;

};



void put_io_context(struct io_context *ioc);

void exit_io_context(void);

struct io_context *get_io_context(gfp_t gfp_flags, int node);

struct io_context *alloc_io_context(gfp_t gfp_flags, int node);

void copy_io_context(struct io_context **pdst, struct io_context **psrc);

void swap_io_context(struct io_context **ioc1, struct io_context **ioc2);
@@ -894,6 +821,12 @@ static inline void exit_io_context(void) 
{

}



static inline int put_io_context(struct io_context *ioc)

{

	return 1;

}





#endif /* CONFIG_BLOCK */



#endif

include/linux/init_task.h

+0 −1
Original line number Diff line number Diff line
@@ -137,7 +137,6 @@ extern struct group_info init_groups; 
		.time_slice	= HZ, 					\

		.nr_cpus_allowed = NR_CPUS,				\

	},								\

	.ioprio		= 0,						\

	.tasks		= LIST_HEAD_INIT(tsk.tasks),			\

	.ptrace_children= LIST_HEAD_INIT(tsk.ptrace_children),		\

	.ptrace_list	= LIST_HEAD_INIT(tsk.ptrace_list),		\