Commit f5a79d7c authored by Yajun Deng's avatar Yajun Deng Committed by Andrew Morton
Browse files

mm/damon: introduce struct damos_access_pattern 

damon_new_scheme() has too many parameters, so introduce struct
damos_access_pattern to simplify it.

In additon, we can't use a bpf trace kprobe that has more than 5
parameters.

Link: https://lkml.kernel.org/r/20220908191443.129534-1-sj@kernel.org


Signed-off-by: default avatarYajun Deng <yajun.deng@linux.dev>
Signed-off-by: default avatarSeongJae Park <sj@kernel.org>
Reviewed-by: default avatarSeongJae Park <sj@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 679d7f69

include/linux/damon.h

+20 −17
Original line number Diff line number Diff line
@@ -216,13 +216,26 @@ struct damos_stat { 
};



/**

 * struct damos - Represents a Data Access Monitoring-based Operation Scheme.

 * struct damos_access_pattern - Target access pattern of the given scheme.

 * @min_sz_region:	Minimum size of target regions.

 * @max_sz_region:	Maximum size of target regions.

 * @min_nr_accesses:	Minimum ``->nr_accesses`` of target regions.

 * @max_nr_accesses:	Maximum ``->nr_accesses`` of target regions.

 * @min_age_region:	Minimum age of target regions.

 * @max_age_region:	Maximum age of target regions.

 */

struct damos_access_pattern {

	unsigned long min_sz_region;

	unsigned long max_sz_region;

	unsigned int min_nr_accesses;

	unsigned int max_nr_accesses;

	unsigned int min_age_region;

	unsigned int max_age_region;

};



/**

 * struct damos - Represents a Data Access Monitoring-based Operation Scheme.

 * @pattern:		Access pattern of target regions.

 * @action:		&damo_action to be applied to the target regions.

 * @quota:		Control the aggressiveness of this scheme.

 * @wmarks:		Watermarks for automated (in)activation of this scheme.
@@ -230,10 +243,8 @@ struct damos_stat { 
 * @list:		List head for siblings.

 *

 * For each aggregation interval, DAMON finds regions which fit in the

 * condition (&min_sz_region, &max_sz_region, &min_nr_accesses,

 * &max_nr_accesses, &min_age_region, &max_age_region) and applies &action to

 * those.  To avoid consuming too much CPU time or IO resources for the

 * &action, &quota is used.

 * &pattern and applies &action to those. To avoid consuming too much

 * CPU time or IO resources for the &action, &quota is used.

 *

 * To do the work only when needed, schemes can be activated for specific

 * system situations using &wmarks.  If all schemes that registered to the
@@ -248,12 +259,7 @@ struct damos_stat { 
 * &action is applied.

 */

struct damos {

	unsigned long min_sz_region;

	unsigned long max_sz_region;

	unsigned int min_nr_accesses;

	unsigned int max_nr_accesses;

	unsigned int min_age_region;

	unsigned int max_age_region;

	struct damos_access_pattern pattern;

	enum damos_action action;

	struct damos_quota quota;

	struct damos_watermarks wmarks;
@@ -509,10 +515,7 @@ void damon_destroy_region(struct damon_region *r, struct damon_target *t); 
int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges,

		unsigned int nr_ranges);



struct damos *damon_new_scheme(

		unsigned long min_sz_region, unsigned long max_sz_region,

		unsigned int min_nr_accesses, unsigned int max_nr_accesses,

		unsigned int min_age_region, unsigned int max_age_region,

struct damos *damon_new_scheme(struct damos_access_pattern *pattern,

			enum damos_action action, struct damos_quota *quota,

			struct damos_watermarks *wmarks);

void damon_add_scheme(struct damon_ctx *ctx, struct damos *s);

mm/damon/core.c

+15 −16
Original line number Diff line number Diff line
@@ -231,10 +231,7 @@ int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, 
	return 0;

}



struct damos *damon_new_scheme(

		unsigned long min_sz_region, unsigned long max_sz_region,

		unsigned int min_nr_accesses, unsigned int max_nr_accesses,

		unsigned int min_age_region, unsigned int max_age_region,

struct damos *damon_new_scheme(struct damos_access_pattern *pattern,

			enum damos_action action, struct damos_quota *quota,

			struct damos_watermarks *wmarks)

{
@@ -243,12 +240,12 @@ struct damos *damon_new_scheme( 
	scheme = kmalloc(sizeof(*scheme), GFP_KERNEL);

	if (!scheme)

		return NULL;

	scheme->min_sz_region = min_sz_region;

	scheme->max_sz_region = max_sz_region;

	scheme->min_nr_accesses = min_nr_accesses;

	scheme->max_nr_accesses = max_nr_accesses;

	scheme->min_age_region = min_age_region;

	scheme->max_age_region = max_age_region;

	scheme->pattern.min_sz_region = pattern->min_sz_region;

	scheme->pattern.max_sz_region = pattern->max_sz_region;

	scheme->pattern.min_nr_accesses = pattern->min_nr_accesses;

	scheme->pattern.max_nr_accesses = pattern->max_nr_accesses;

	scheme->pattern.min_age_region = pattern->min_age_region;

	scheme->pattern.max_age_region = pattern->max_age_region;

	scheme->action = action;

	scheme->stat = (struct damos_stat){};

	INIT_LIST_HEAD(&scheme->list);
@@ -667,10 +664,12 @@ static bool __damos_valid_target(struct damon_region *r, struct damos *s) 
	unsigned long sz;



	sz = r->ar.end - r->ar.start;

	return s->min_sz_region <= sz && sz <= s->max_sz_region &&

		s->min_nr_accesses <= r->nr_accesses &&

		r->nr_accesses <= s->max_nr_accesses &&

		s->min_age_region <= r->age && r->age <= s->max_age_region;

	return s->pattern.min_sz_region <= sz &&

		sz <= s->pattern.max_sz_region &&

		s->pattern.min_nr_accesses <= r->nr_accesses &&

		r->nr_accesses <= s->pattern.max_nr_accesses &&

		s->pattern.min_age_region <= r->age &&

		r->age <= s->pattern.max_age_region;

}



static bool damos_valid_target(struct damon_ctx *c, struct damon_target *t,

mm/damon/dbgfs.c

+17 −10
Original line number Diff line number Diff line
@@ -131,9 +131,12 @@ static ssize_t sprint_schemes(struct damon_ctx *c, char *buf, ssize_t len) 
	damon_for_each_scheme(s, c) {

		rc = scnprintf(&buf[written], len - written,

				"%lu %lu %u %u %u %u %d %lu %lu %lu %u %u %u %d %lu %lu %lu %lu %lu %lu %lu %lu %lu\n",

				s->min_sz_region, s->max_sz_region,

				s->min_nr_accesses, s->max_nr_accesses,

				s->min_age_region, s->max_age_region,

				s->pattern.min_sz_region,

				s->pattern.max_sz_region,

				s->pattern.min_nr_accesses,

				s->pattern.max_nr_accesses,

				s->pattern.min_age_region,

				s->pattern.max_age_region,

				damos_action_to_dbgfs_scheme_action(s->action),

				s->quota.ms, s->quota.sz,

				s->quota.reset_interval,
@@ -221,8 +224,6 @@ static struct damos **str_to_schemes(const char *str, ssize_t len, 
	struct damos *scheme, **schemes;

	const int max_nr_schemes = 256;

	int pos = 0, parsed, ret;

	unsigned long min_sz, max_sz;

	unsigned int min_nr_a, max_nr_a, min_age, max_age;

	unsigned int action_input;

	enum damos_action action;
@@ -233,13 +234,18 @@ static struct damos **str_to_schemes(const char *str, ssize_t len, 


	*nr_schemes = 0;

	while (pos < len && *nr_schemes < max_nr_schemes) {

		struct damos_access_pattern pattern = {};

		struct damos_quota quota = {};

		struct damos_watermarks wmarks;



		ret = sscanf(&str[pos],

				"%lu %lu %u %u %u %u %u %lu %lu %lu %u %u %u %u %lu %lu %lu %lu%n",

				&min_sz, &max_sz, &min_nr_a, &max_nr_a,

				&min_age, &max_age, &action_input, &quota.ms,

				&pattern.min_sz_region, &pattern.max_sz_region,

				&pattern.min_nr_accesses,

				&pattern.max_nr_accesses,

				&pattern.min_age_region,

				&pattern.max_age_region,

				&action_input, &quota.ms,

				&quota.sz, &quota.reset_interval,

				&quota.weight_sz, &quota.weight_nr_accesses,

				&quota.weight_age, &wmarks.metric,
@@ -251,7 +257,9 @@ static struct damos **str_to_schemes(const char *str, ssize_t len, 
		if ((int)action < 0)

			goto fail;



		if (min_sz > max_sz || min_nr_a > max_nr_a || min_age > max_age)

		if (pattern.min_sz_region > pattern.max_sz_region ||

		    pattern.min_nr_accesses > pattern.max_nr_accesses ||

		    pattern.min_age_region > pattern.max_age_region)

			goto fail;



		if (wmarks.high < wmarks.mid || wmarks.high < wmarks.low ||
@@ -259,8 +267,7 @@ static struct damos **str_to_schemes(const char *str, ssize_t len, 
			goto fail;



		pos += parsed;

		scheme = damon_new_scheme(min_sz, max_sz, min_nr_a, max_nr_a,

				min_age, max_age, action, &quota, &wmarks);

		scheme = damon_new_scheme(&pattern, action, &quota, &wmarks);

		if (!scheme)

			goto fail;

mm/damon/lru_sort.c

+28 −18
Original line number Diff line number Diff line
@@ -293,6 +293,17 @@ static bool get_monitoring_region(unsigned long *start, unsigned long *end) 
/* Create a DAMON-based operation scheme for hot memory regions */

static struct damos *damon_lru_sort_new_hot_scheme(unsigned int hot_thres)

{

	struct damos_access_pattern pattern = {

		/* Find regions having PAGE_SIZE or larger size */

		.min_sz_region = PAGE_SIZE,

		.max_sz_region = ULONG_MAX,

		/* and accessed for more than the threshold */

		.min_nr_accesses = hot_thres,

		.max_nr_accesses = UINT_MAX,

		/* no matter its age */

		.min_age_region = 0,

		.max_age_region = UINT_MAX,

	};

	struct damos_watermarks wmarks = {

		.metric = DAMOS_WMARK_FREE_MEM_RATE,

		.interval = wmarks_interval,
@@ -313,26 +324,31 @@ static struct damos *damon_lru_sort_new_hot_scheme(unsigned int hot_thres) 
		.weight_nr_accesses = 1,

		.weight_age = 0,

	};

	struct damos *scheme = damon_new_scheme(

			/* Find regions having PAGE_SIZE or larger size */

			PAGE_SIZE, ULONG_MAX,

			/* and accessed for more than the threshold */

			hot_thres, UINT_MAX,

			/* no matter its age */

			0, UINT_MAX,



	return damon_new_scheme(

			&pattern,

			/* prioritize those on LRU lists, as soon as found */

			DAMOS_LRU_PRIO,

			/* under the quota. */

			&quota,

			/* (De)activate this according to the watermarks. */

			&wmarks);



	return scheme;

}



/* Create a DAMON-based operation scheme for cold memory regions */

static struct damos *damon_lru_sort_new_cold_scheme(unsigned int cold_thres)

{

	struct damos_access_pattern pattern = {

		/* Find regions having PAGE_SIZE or larger size */

		.min_sz_region = PAGE_SIZE,

		.max_sz_region = ULONG_MAX,

		/* and not accessed at all */

		.min_nr_accesses = 0,

		.max_nr_accesses = 0,

		/* for min_age or more micro-seconds */

		.min_age_region = cold_thres,

		.max_age_region = UINT_MAX,

	};

	struct damos_watermarks wmarks = {

		.metric = DAMOS_WMARK_FREE_MEM_RATE,

		.interval = wmarks_interval,
@@ -354,21 +370,15 @@ static struct damos *damon_lru_sort_new_cold_scheme(unsigned int cold_thres) 
		.weight_nr_accesses = 0,

		.weight_age = 1,

	};

	struct damos *scheme = damon_new_scheme(

			/* Find regions having PAGE_SIZE or larger size */

			PAGE_SIZE, ULONG_MAX,

			/* and not accessed at all */

			0, 0,

			/* for cold_thres or more micro-seconds, and */

			cold_thres, UINT_MAX,



	return damon_new_scheme(

			&pattern,

			/* mark those as not accessed, as soon as found */

			DAMOS_LRU_DEPRIO,

			/* under the quota. */

			&quota,

			/* (De)activate this according to the watermarks. */

			&wmarks);



	return scheme;

}



static int damon_lru_sort_apply_parameters(void)

mm/damon/reclaim.c

+14 −9
Original line number Diff line number Diff line
@@ -264,6 +264,17 @@ static bool get_monitoring_region(unsigned long *start, unsigned long *end) 


static struct damos *damon_reclaim_new_scheme(void)

{

	struct damos_access_pattern pattern = {

		/* Find regions having PAGE_SIZE or larger size */

		.min_sz_region = PAGE_SIZE,

		.max_sz_region = ULONG_MAX,

		/* and not accessed at all */

		.min_nr_accesses = 0,

		.max_nr_accesses = 0,

		/* for min_age or more micro-seconds */

		.min_age_region = min_age / aggr_interval,

		.max_age_region = UINT_MAX,

	};

	struct damos_watermarks wmarks = {

		.metric = DAMOS_WMARK_FREE_MEM_RATE,

		.interval = wmarks_interval,
@@ -284,21 +295,15 @@ static struct damos *damon_reclaim_new_scheme(void) 
		.weight_nr_accesses = 0,

		.weight_age = 1

	};

	struct damos *scheme = damon_new_scheme(

			/* Find regions having PAGE_SIZE or larger size */

			PAGE_SIZE, ULONG_MAX,

			/* and not accessed at all */

			0, 0,

			/* for min_age or more micro-seconds, and */

			min_age / aggr_interval, UINT_MAX,



	return damon_new_scheme(

			&pattern,

			/* page out those, as soon as found */

			DAMOS_PAGEOUT,

			/* under the quota. */

			&quota,

			/* (De)activate this according to the watermarks. */

			&wmarks);



	return scheme;

}



static int damon_reclaim_apply_parameters(void)