!4449 memory tiering: calculate abstract distance based on ACPI HMAT (f3ae85d1) · Commits · EulixOS / Software / Kernel

drivers/acpi/numa/hmat.c

+93 −53

Original line number	Diff line number	Diff line
		@@ -24,6 +24,7 @@
		#include <linux/node.h>
		#include <linux/sysfs.h>
		#include <linux/dax.h>
		#include <linux/memory-tiers.h>

		static u8 hmat_revision;
		static int hmat_disable __initdata;
		@@ -582,28 +583,25 @@ static int initiators_to_nodemask(unsigned long *p_nodes)
		return 0;
		}

		static void hmat_register_target_initiators(struct memory_target *target)
		static void hmat_update_target_attrs(struct memory_target *target,
		unsigned long *p_nodes, int access)
		{
		static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
		struct memory_initiator *initiator;
		unsigned int mem_nid, cpu_nid;
		unsigned int cpu_nid;
		struct memory_locality *loc = NULL;
		u32 best = 0;
		bool access0done = false;
		int i;

		mem_nid = pxm_to_node(target->memory_pxm);
		bitmap_zero(p_nodes, MAX_NUMNODES);
		/*
		* If the Address Range Structure provides a local processor pxm, link
		* If the Address Range Structure provides a local processor pxm, set
		* only that one. Otherwise, find the best performance attributes and
		* register all initiators that match.
		* collect all initiators that match.
		*/
		if (target->processor_pxm != PXM_INVAL) {
		cpu_nid = pxm_to_node(target->processor_pxm);
		register_memory_node_under_compute_node(mem_nid, cpu_nid, 0);
		access0done = true;
		if (node_state(cpu_nid, N_CPU)) {
		register_memory_node_under_compute_node(mem_nid, cpu_nid, 1);
		if (access == 0 \|\| node_state(cpu_nid, N_CPU)) {
		set_bit(target->processor_pxm, p_nodes);
		return;
		}
		}
		@@ -617,47 +615,10 @@ static void hmat_register_target_initiators(struct memory_target *target)
		* We'll also use the sorting to prime the candidate nodes with known
		* initiators.
		*/
		bitmap_zero(p_nodes, MAX_NUMNODES);
		list_sort(NULL, &initiators, initiator_cmp);
		if (initiators_to_nodemask(p_nodes) < 0)
		return;

		if (!access0done) {
		for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) {
		loc = localities_types[i];
		if (!loc)
		continue;

		best = 0;
		list_for_each_entry(initiator, &initiators, node) {
		u32 value;

		if (!test_bit(initiator->processor_pxm, p_nodes))
		continue;

		value = hmat_initiator_perf(target, initiator,
		loc->hmat_loc);
		if (hmat_update_best(loc->hmat_loc->data_type, value, &best))
		bitmap_clear(p_nodes, 0, initiator->processor_pxm);
		if (value != best)
		clear_bit(initiator->processor_pxm, p_nodes);
		}
		if (best)
		hmat_update_target_access(target, loc->hmat_loc->data_type,
		best, 0);
		}

		for_each_set_bit(i, p_nodes, MAX_NUMNODES) {
		cpu_nid = pxm_to_node(i);
		register_memory_node_under_compute_node(mem_nid, cpu_nid, 0);
		}
		}

		/* Access 1 ignores Generic Initiators */
		bitmap_zero(p_nodes, MAX_NUMNODES);
		if (initiators_to_nodemask(p_nodes) < 0)
		return;

		for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) {
		loc = localities_types[i];
		if (!loc)
		@@ -667,7 +628,7 @@ static void hmat_register_target_initiators(struct memory_target *target)
		list_for_each_entry(initiator, &initiators, node) {
		u32 value;

		if (!initiator->has_cpu) {
		if (access == 1 && !initiator->has_cpu) {
		clear_bit(initiator->processor_pxm, p_nodes);
		continue;
		}
		@@ -681,14 +642,33 @@ static void hmat_register_target_initiators(struct memory_target *target)
		clear_bit(initiator->processor_pxm, p_nodes);
		}
		if (best)
		hmat_update_target_access(target, loc->hmat_loc->data_type, best, 1);
		hmat_update_target_access(target, loc->hmat_loc->data_type, best, access);
		}
		}

		static void __hmat_register_target_initiators(struct memory_target *target,
		unsigned long *p_nodes,
		int access)
		{
		unsigned int mem_nid, cpu_nid;
		int i;

		mem_nid = pxm_to_node(target->memory_pxm);
		hmat_update_target_attrs(target, p_nodes, access);
		for_each_set_bit(i, p_nodes, MAX_NUMNODES) {
		cpu_nid = pxm_to_node(i);
		register_memory_node_under_compute_node(mem_nid, cpu_nid, 1);
		register_memory_node_under_compute_node(mem_nid, cpu_nid, access);
		}
		}

		static void hmat_register_target_initiators(struct memory_target *target)
		{
		static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);

		__hmat_register_target_initiators(target, p_nodes, 0);
		__hmat_register_target_initiators(target, p_nodes, 1);
		}

		static void hmat_register_target_cache(struct memory_target *target)
		{
		unsigned mem_nid = pxm_to_node(target->memory_pxm);
		@@ -780,6 +760,61 @@ static int hmat_callback(struct notifier_block *self,
		return NOTIFY_OK;
		}

		static int hmat_set_default_dram_perf(void)
		{
		int rc;
		int nid, pxm;
		struct memory_target *target;
		struct node_hmem_attrs *attrs;

		if (!default_dram_type)
		return -EIO;

		for_each_node_mask(nid, default_dram_type->nodes) {
		pxm = node_to_pxm(nid);
		target = find_mem_target(pxm);
		if (!target)
		continue;
		attrs = &target->hmem_attrs[1];
		rc = mt_set_default_dram_perf(nid, attrs, "ACPI HMAT");
		if (rc)
		return rc;
		}

		return 0;
		}

		static int hmat_calculate_adistance(struct notifier_block *self,
		unsigned long nid, void *data)
		{
		static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
		struct memory_target *target;
		struct node_hmem_attrs *perf;
		int *adist = data;
		int pxm;

		pxm = node_to_pxm(nid);
		target = find_mem_target(pxm);
		if (!target)
		return NOTIFY_OK;

		mutex_lock(&target_lock);
		hmat_update_target_attrs(target, p_nodes, 1);
		mutex_unlock(&target_lock);

		perf = &target->hmem_attrs[1];

		if (mt_perf_to_adistance(perf, adist))
		return NOTIFY_OK;

		return NOTIFY_STOP;
		}

		static struct notifier_block hmat_adist_nb __meminitdata = {
		.notifier_call = hmat_calculate_adistance,
		.priority = 100,
		};

		static __init void hmat_free_structures(void)
		{
		struct memory_target target, tnext;
		@@ -862,7 +897,12 @@ static __init int hmat_init(void)
		hmat_register_targets();

		/* Keep the table and structures if the notifier may use them */
		if (!hotplug_memory_notifier(hmat_callback, HMAT_CALLBACK_PRI))
		if (hotplug_memory_notifier(hmat_callback, HMAT_CALLBACK_PRI))
		goto out_put;

		if (!hmat_set_default_dram_perf())
		register_mt_adistance_algorithm(&hmat_adist_nb);

		return 0;
		out_put:
		hmat_free_structures();

drivers/dax/kmem.c

+49 −13

Original line number	Diff line number	Diff line
		@@ -49,14 +49,52 @@ struct dax_kmem_data {
		struct resource *res[];
		};

		static struct memory_dev_type *dax_slowmem_type;
		static DEFINE_MUTEX(kmem_memory_type_lock);
		static LIST_HEAD(kmem_memory_types);

		static struct memory_dev_type *kmem_find_alloc_memory_type(int adist)
		{
		bool found = false;
		struct memory_dev_type *mtype;

		mutex_lock(&kmem_memory_type_lock);
		list_for_each_entry(mtype, &kmem_memory_types, list) {
		if (mtype->adistance == adist) {
		found = true;
		break;
		}
		}
		if (!found) {
		mtype = alloc_memory_type(adist);
		if (!IS_ERR(mtype))
		list_add(&mtype->list, &kmem_memory_types);
		}
		mutex_unlock(&kmem_memory_type_lock);

		return mtype;
		}

		static void kmem_put_memory_types(void)
		{
		struct memory_dev_type mtype, mtn;

		mutex_lock(&kmem_memory_type_lock);
		list_for_each_entry_safe(mtype, mtn, &kmem_memory_types, list) {
		list_del(&mtype->list);
		put_memory_type(mtype);
		}
		mutex_unlock(&kmem_memory_type_lock);
		}

		static int dev_dax_kmem_probe(struct dev_dax *dev_dax)
		{
		struct device *dev = &dev_dax->dev;
		unsigned long total_len = 0;
		struct dax_kmem_data *data;
		struct memory_dev_type *mtype;
		int i, rc, mapped = 0;
		int numa_node;
		int adist = MEMTIER_DEFAULT_DAX_ADISTANCE;

		/*
		* Ensure good NUMA information for the persistent memory.
		@@ -71,6 +109,11 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax)
		return -EINVAL;
		}

		mt_calc_adistance(numa_node, &adist);
		mtype = kmem_find_alloc_memory_type(adist);
		if (IS_ERR(mtype))
		return PTR_ERR(mtype);

		for (i = 0; i < dev_dax->nr_range; i++) {
		struct range range;

		@@ -88,7 +131,7 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax)
		return -EINVAL;
		}

		init_node_memory_type(numa_node, dax_slowmem_type);
		init_node_memory_type(numa_node, mtype);

		rc = -ENOMEM;
		data = kzalloc(struct_size(data, res, dev_dax->nr_range), GFP_KERNEL);
		@@ -167,7 +210,7 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax)
		err_res_name:
		kfree(data);
		err_dax_kmem_data:
		clear_node_memory_type(numa_node, dax_slowmem_type);
		clear_node_memory_type(numa_node, mtype);
		return rc;
		}

		@@ -219,7 +262,7 @@ static void dev_dax_kmem_remove(struct dev_dax *dev_dax)
		* for that. This implies this reference will be around
		* till next reboot.
		*/
		clear_node_memory_type(node, dax_slowmem_type);
		clear_node_memory_type(node, NULL);
		}
		}
		#else
		@@ -251,12 +294,6 @@ static int __init dax_kmem_init(void)
		if (!kmem_name)
		return -ENOMEM;

		dax_slowmem_type = alloc_memory_type(MEMTIER_DEFAULT_DAX_ADISTANCE);
		if (IS_ERR(dax_slowmem_type)) {
		rc = PTR_ERR(dax_slowmem_type);
		goto err_dax_slowmem_type;
		}

		rc = dax_driver_register(&device_dax_kmem_driver);
		if (rc)
		goto error_dax_driver;
		@@ -264,8 +301,7 @@ static int __init dax_kmem_init(void)
		return rc;

		error_dax_driver:
		put_memory_type(dax_slowmem_type);
		err_dax_slowmem_type:
		kmem_put_memory_types();
		kfree_const(kmem_name);
		return rc;
		}
		@@ -275,7 +311,7 @@ static void __exit dax_kmem_exit(void)
		dax_driver_unregister(&device_dax_kmem_driver);
		if (!any_hotremove_failed)
		kfree_const(kmem_name);
		put_memory_type(dax_slowmem_type);
		kmem_put_memory_types();
		}

		MODULE_AUTHOR("Intel Corporation");

include/linux/memory-tiers.h

+39 −0

Original line number	Diff line number	Diff line
		@@ -6,6 +6,7 @@
		#include <linux/nodemask.h>
		#include <linux/kref.h>
		#include <linux/mmzone.h>
		#include <linux/notifier.h>
		/*
		* Each tier cover a abstrace distance chunk size of 128
		*/
		@@ -23,6 +24,8 @@ struct memory_tier;
		struct memory_dev_type {
		/* list of memory types that are part of same tier as this type */
		struct list_head tier_sibiling;
		/* list of memory types that are managed by one driver */
		struct list_head list;
		/* abstract distance for this specific memory type */
		int adistance;
		/* Nodes of same abstract distance */
		@@ -30,12 +33,21 @@ struct memory_dev_type {
		struct kref kref;
		};

		struct node_hmem_attrs;

		#ifdef CONFIG_NUMA
		extern bool numa_demotion_enabled;
		extern struct memory_dev_type *default_dram_type;
		struct memory_dev_type *alloc_memory_type(int adistance);
		void put_memory_type(struct memory_dev_type *memtype);
		void init_node_memory_type(int node, struct memory_dev_type *default_type);
		void clear_node_memory_type(int node, struct memory_dev_type *memtype);
		int register_mt_adistance_algorithm(struct notifier_block *nb);
		int unregister_mt_adistance_algorithm(struct notifier_block *nb);
		int mt_calc_adistance(int node, int *adist);
		int mt_set_default_dram_perf(int nid, struct node_hmem_attrs *perf,
		const char *source);
		int mt_perf_to_adistance(struct node_hmem_attrs perf, int adist);
		#ifdef CONFIG_MIGRATION
		int next_demotion_node(int node);
		void node_get_allowed_targets(pg_data_t pgdat, nodemask_t targets);
		@@ -60,6 +72,7 @@ static inline bool node_is_toptier(int node)
		#else

		#define numa_demotion_enabled false
		#define default_dram_type NULL
		/*
		* CONFIG_NUMA implementation returns non NULL error.
		*/
		@@ -97,5 +110,31 @@ static inline bool node_is_toptier(int node)
		{
		return true;
		}

		static inline int register_mt_adistance_algorithm(struct notifier_block *nb)
		{
		return 0;
		}

		static inline int unregister_mt_adistance_algorithm(struct notifier_block *nb)
		{
		return 0;
		}

		static inline int mt_calc_adistance(int node, int *adist)
		{
		return NOTIFY_DONE;
		}

		static inline int mt_set_default_dram_perf(int nid, struct node_hmem_attrs *perf,
		const char *source)
		{
		return -EIO;
		}

		static inline int mt_perf_to_adistance(struct node_hmem_attrs perf, int adist)
		{
		return -EIO;
		}
		#endif /* CONFIG_NUMA */
		#endif /* _LINUX_MEMORY_TIERS_H */

mm/memory-tiers.c

+163 −2

Original line number	Diff line number	Diff line
		@@ -5,6 +5,7 @@
		#include <linux/kobject.h>
		#include <linux/memory.h>
		#include <linux/memory-tiers.h>
		#include <linux/notifier.h>

		#include "internal.h"

		@@ -36,7 +37,7 @@ struct node_memory_type_map {
		static DEFINE_MUTEX(memory_tier_lock);
		static LIST_HEAD(memory_tiers);
		static struct node_memory_type_map node_memory_types[MAX_NUMNODES];
		static struct memory_dev_type *default_dram_type;
		struct memory_dev_type *default_dram_type;

		static struct bus_type memory_tier_subsys = {
		.name = "memory_tiering",
		@@ -105,6 +106,13 @@ static int top_tier_adistance;
		static struct demotion_nodes *node_demotion __read_mostly;
		#endif /* CONFIG_MIGRATION */

		static BLOCKING_NOTIFIER_HEAD(mt_adistance_algorithms);

		static bool default_dram_perf_error;
		static struct node_hmem_attrs default_dram_perf;
		static int default_dram_perf_ref_nid = NUMA_NO_NODE;
		static const char *default_dram_perf_ref_source;

		static inline struct memory_tier to_memory_tier(struct device device)
		{
		return container_of(device, struct memory_tier, dev);
		@@ -578,13 +586,14 @@ EXPORT_SYMBOL_GPL(init_node_memory_type);
		void clear_node_memory_type(int node, struct memory_dev_type *memtype)
		{
		mutex_lock(&memory_tier_lock);
		if (node_memory_types[node].memtype == memtype)
		if (node_memory_types[node].memtype == memtype \|\| !memtype)
		node_memory_types[node].map_count--;
		/*
		* If we umapped all the attached devices to this node,
		* clear the node memory type.
		*/
		if (!node_memory_types[node].map_count) {
		memtype = node_memory_types[node].memtype;
		node_memory_types[node].memtype = NULL;
		put_memory_type(memtype);
		}
		@@ -592,6 +601,158 @@ void clear_node_memory_type(int node, struct memory_dev_type *memtype)
		}
		EXPORT_SYMBOL_GPL(clear_node_memory_type);

		static void dump_hmem_attrs(struct node_hmem_attrs attrs, const char prefix)
		{
		pr_info(
		"%sread_latency: %u, write_latency: %u, read_bandwidth: %u, write_bandwidth: %u\n",
		prefix, attrs->read_latency, attrs->write_latency,
		attrs->read_bandwidth, attrs->write_bandwidth);
		}

		int mt_set_default_dram_perf(int nid, struct node_hmem_attrs *perf,
		const char *source)
		{
		int rc = 0;

		mutex_lock(&memory_tier_lock);
		if (default_dram_perf_error) {
		rc = -EIO;
		goto out;
		}

		if (perf->read_latency + perf->write_latency == 0 \|\|
		perf->read_bandwidth + perf->write_bandwidth == 0) {
		rc = -EINVAL;
		goto out;
		}

		if (default_dram_perf_ref_nid == NUMA_NO_NODE) {
		default_dram_perf = *perf;
		default_dram_perf_ref_nid = nid;
		default_dram_perf_ref_source = kstrdup(source, GFP_KERNEL);
		goto out;
		}

		/*
		* The performance of all default DRAM nodes is expected to be
		* same (that is, the variation is less than 10%). And it
		* will be used as base to calculate the abstract distance of
		* other memory nodes.
		*/
		if (abs(perf->read_latency - default_dram_perf.read_latency) * 10 >
		default_dram_perf.read_latency \|\|
		abs(perf->write_latency - default_dram_perf.write_latency) * 10 >
		default_dram_perf.write_latency \|\|
		abs(perf->read_bandwidth - default_dram_perf.read_bandwidth) * 10 >
		default_dram_perf.read_bandwidth \|\|
		abs(perf->write_bandwidth - default_dram_perf.write_bandwidth) * 10 >
		default_dram_perf.write_bandwidth) {
		pr_info(
		"memory-tiers: the performance of DRAM node %d mismatches that of the reference\n"
		"DRAM node %d.\n", nid, default_dram_perf_ref_nid);
		pr_info(" performance of reference DRAM node %d:\n",
		default_dram_perf_ref_nid);
		dump_hmem_attrs(&default_dram_perf, " ");
		pr_info(" performance of DRAM node %d:\n", nid);
		dump_hmem_attrs(perf, " ");
		pr_info(
		" disable default DRAM node performance based abstract distance algorithm.\n");
		default_dram_perf_error = true;
		rc = -EINVAL;
		}

		out:
		mutex_unlock(&memory_tier_lock);
		return rc;
		}

		int mt_perf_to_adistance(struct node_hmem_attrs perf, int adist)
		{
		if (default_dram_perf_error)
		return -EIO;

		if (default_dram_perf_ref_nid == NUMA_NO_NODE)
		return -ENOENT;

		if (perf->read_latency + perf->write_latency == 0 \|\|
		perf->read_bandwidth + perf->write_bandwidth == 0)
		return -EINVAL;

		mutex_lock(&memory_tier_lock);
		/*
		* The abstract distance of a memory node is in direct proportion to
		* its memory latency (read + write) and inversely proportional to its
		* memory bandwidth (read + write). The abstract distance, memory
		* latency, and memory bandwidth of the default DRAM nodes are used as
		* the base.
		*/
		adist = MEMTIER_ADISTANCE_DRAM
		(perf->read_latency + perf->write_latency) /
		(default_dram_perf.read_latency + default_dram_perf.write_latency) *
		(default_dram_perf.read_bandwidth + default_dram_perf.write_bandwidth) /
		(perf->read_bandwidth + perf->write_bandwidth);
		mutex_unlock(&memory_tier_lock);

		return 0;
		}
		EXPORT_SYMBOL_GPL(mt_perf_to_adistance);

		/**
		* register_mt_adistance_algorithm() - Register memory tiering abstract distance algorithm
		* @nb: The notifier block which describe the algorithm
		*
		* Return: 0 on success, errno on error.
		*
		* Every memory tiering abstract distance algorithm provider needs to
		* register the algorithm with register_mt_adistance_algorithm(). To
		* calculate the abstract distance for a specified memory node, the
		* notifier function will be called unless some high priority
		* algorithm has provided result. The prototype of the notifier
		* function is as follows,
		*
		* int (algorithm_notifier)(struct notifier_block nb,
		* unsigned long nid, void *data);
		*
		* Where "nid" specifies the memory node, "data" is the pointer to the
		* returned abstract distance (that is, "int *adist"). If the
		* algorithm provides the result, NOTIFY_STOP should be returned.
		* Otherwise, return_value & %NOTIFY_STOP_MASK == 0 to allow the next
		* algorithm in the chain to provide the result.
		*/
		int register_mt_adistance_algorithm(struct notifier_block *nb)
		{
		return blocking_notifier_chain_register(&mt_adistance_algorithms, nb);
		}
		EXPORT_SYMBOL_GPL(register_mt_adistance_algorithm);

		/**
		* unregister_mt_adistance_algorithm() - Unregister memory tiering abstract distance algorithm
		* @nb: the notifier block which describe the algorithm
		*
		* Return: 0 on success, errno on error.
		*/
		int unregister_mt_adistance_algorithm(struct notifier_block *nb)
		{
		return blocking_notifier_chain_unregister(&mt_adistance_algorithms, nb);
		}
		EXPORT_SYMBOL_GPL(unregister_mt_adistance_algorithm);

		/**
		* mt_calc_adistance() - Calculate abstract distance with registered algorithms
		* @node: the node to calculate abstract distance for
		* @adist: the returned abstract distance
		*
		* Return: if return_value & %NOTIFY_STOP_MASK != 0, then some
		* abstract distance algorithm provides the result, and return it via
		* @adist. Otherwise, no algorithm can provide the result and @adist
		* will be kept as it is.
		*/
		int mt_calc_adistance(int node, int *adist)
		{
		return blocking_notifier_call_chain(&mt_adistance_algorithms, node, adist);
		}
		EXPORT_SYMBOL_GPL(mt_calc_adistance);

		static int __meminit memtier_hotplug_callback(struct notifier_block *self,
		unsigned long action, void *_arg)
		{