Commit 58d208de authored by Aaron Tomlin's avatar Aaron Tomlin Committed by Luis Chamberlain
Browse files

module: Move latched RB-tree support to a separate file 



No functional change.

This patch migrates module latched RB-tree support
(e.g. see __module_address()) from core module code
into kernel/module/tree_lookup.c.

Reviewed-by: default avatarChristophe Leroy <christophe.leroy@csgroup.eu>
Signed-off-by: default avatarAaron Tomlin <atomlin@redhat.com>
Signed-off-by: default avatarLuis Chamberlain <mcgrof@kernel.org>
parent 1be9473e

kernel/module/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -11,3 +11,4 @@ obj-y += main.o 
obj-$(CONFIG_MODULE_DECOMPRESS) += decompress.o

obj-$(CONFIG_MODULE_SIG) += signing.o

obj-$(CONFIG_LIVEPATCH) += livepatch.o

obj-$(CONFIG_MODULES_TREE_LOOKUP) += tree_lookup.o

kernel/module/internal.h

+33 −0
Original line number Diff line number Diff line
@@ -9,6 +9,7 @@ 
#include <linux/compiler.h>

#include <linux/module.h>

#include <linux/mutex.h>

#include <linux/rculist.h>



#ifndef ARCH_SHF_SMALL

#define ARCH_SHF_SMALL 0
@@ -93,3 +94,35 @@ static inline void module_decompress_cleanup(struct load_info *info) 
{

}

#endif



#ifdef CONFIG_MODULES_TREE_LOOKUP

struct mod_tree_root {

	struct latch_tree_root root;

	unsigned long addr_min;

	unsigned long addr_max;

};



extern struct mod_tree_root mod_tree;



void mod_tree_insert(struct module *mod);

void mod_tree_remove_init(struct module *mod);

void mod_tree_remove(struct module *mod);

struct module *mod_find(unsigned long addr);

#else /* !CONFIG_MODULES_TREE_LOOKUP */



static inline void mod_tree_insert(struct module *mod) { }

static inline void mod_tree_remove_init(struct module *mod) { }

static inline void mod_tree_remove(struct module *mod) { }

static inline struct module *mod_find(unsigned long addr)

{

	struct module *mod;



	list_for_each_entry_rcu(mod, &modules, list,

				lockdep_is_held(&module_mutex)) {

		if (within_module(addr, mod))

			return mod;

	}



	return NULL;

}

#endif /* CONFIG_MODULES_TREE_LOOKUP */

kernel/module/main.c

+4 −126
Original line number Diff line number Diff line
@@ -90,138 +90,16 @@ static DECLARE_WORK(init_free_wq, do_free_init); 
static LLIST_HEAD(init_free_list);



#ifdef CONFIG_MODULES_TREE_LOOKUP



/*

 * Use a latched RB-tree for __module_address(); this allows us to use

 * RCU-sched lookups of the address from any context.

 *

 * This is conditional on PERF_EVENTS || TRACING because those can really hit

 * __module_address() hard by doing a lot of stack unwinding; potentially from

 * NMI context.

 */



static __always_inline unsigned long __mod_tree_val(struct latch_tree_node *n)

{

	struct module_layout *layout = container_of(n, struct module_layout, mtn.node);



	return (unsigned long)layout->base;

}



static __always_inline unsigned long __mod_tree_size(struct latch_tree_node *n)

{

	struct module_layout *layout = container_of(n, struct module_layout, mtn.node);



	return (unsigned long)layout->size;

}



static __always_inline bool

mod_tree_less(struct latch_tree_node *a, struct latch_tree_node *b)

{

	return __mod_tree_val(a) < __mod_tree_val(b);

}



static __always_inline int

mod_tree_comp(void *key, struct latch_tree_node *n)

{

	unsigned long val = (unsigned long)key;

	unsigned long start, end;



	start = __mod_tree_val(n);

	if (val < start)

		return -1;



	end = start + __mod_tree_size(n);

	if (val >= end)

		return 1;



	return 0;

}



static const struct latch_tree_ops mod_tree_ops = {

	.less = mod_tree_less,

	.comp = mod_tree_comp,

};



static struct mod_tree_root {

	struct latch_tree_root root;

	unsigned long addr_min;

	unsigned long addr_max;

} mod_tree __cacheline_aligned = {

struct mod_tree_root mod_tree __cacheline_aligned = {

	.addr_min = -1UL,

};



#define module_addr_min mod_tree.addr_min

#define module_addr_max mod_tree.addr_max



static noinline void __mod_tree_insert(struct mod_tree_node *node)

{

	latch_tree_insert(&node->node, &mod_tree.root, &mod_tree_ops);

}



static void __mod_tree_remove(struct mod_tree_node *node)

{

	latch_tree_erase(&node->node, &mod_tree.root, &mod_tree_ops);

}



/*

 * These modifications: insert, remove_init and remove; are serialized by the

 * module_mutex.

 */

static void mod_tree_insert(struct module *mod)

{

	mod->core_layout.mtn.mod = mod;

	mod->init_layout.mtn.mod = mod;



	__mod_tree_insert(&mod->core_layout.mtn);

	if (mod->init_layout.size)

		__mod_tree_insert(&mod->init_layout.mtn);

}



static void mod_tree_remove_init(struct module *mod)

{

	if (mod->init_layout.size)

		__mod_tree_remove(&mod->init_layout.mtn);

}



static void mod_tree_remove(struct module *mod)

{

	__mod_tree_remove(&mod->core_layout.mtn);

	mod_tree_remove_init(mod);

}



static struct module *mod_find(unsigned long addr)

{

	struct latch_tree_node *ltn;



	ltn = latch_tree_find((void *)addr, &mod_tree.root, &mod_tree_ops);

	if (!ltn)

		return NULL;



	return container_of(ltn, struct mod_tree_node, node)->mod;

}



#else /* MODULES_TREE_LOOKUP */



static unsigned long module_addr_min = -1UL, module_addr_max = 0;



static void mod_tree_insert(struct module *mod) { }

static void mod_tree_remove_init(struct module *mod) { }

static void mod_tree_remove(struct module *mod) { }



static struct module *mod_find(unsigned long addr)

{

	struct module *mod;



	list_for_each_entry_rcu(mod, &modules, list,

				lockdep_is_held(&module_mutex)) {

		if (within_module(addr, mod))

			return mod;

	}



	return NULL;

}



#endif /* MODULES_TREE_LOOKUP */

#else /* !CONFIG_MODULES_TREE_LOOKUP */

static unsigned long module_addr_min = -1UL, module_addr_max;

#endif /* CONFIG_MODULES_TREE_LOOKUP */



/*

 * Bounds of module text, for speeding up __module_address.

kernel/module/tree_lookup.c

0 → 100644
+109 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0-or-later

/*

 * Modules tree lookup

 *

 * Copyright (C) 2015 Peter Zijlstra

 * Copyright (C) 2015 Rusty Russell

 */



#include <linux/module.h>

#include <linux/rbtree_latch.h>

#include "internal.h"



/*

 * Use a latched RB-tree for __module_address(); this allows us to use

 * RCU-sched lookups of the address from any context.

 *

 * This is conditional on PERF_EVENTS || TRACING because those can really hit

 * __module_address() hard by doing a lot of stack unwinding; potentially from

 * NMI context.

 */



static __always_inline unsigned long __mod_tree_val(struct latch_tree_node *n)

{

	struct module_layout *layout = container_of(n, struct module_layout, mtn.node);



	return (unsigned long)layout->base;

}



static __always_inline unsigned long __mod_tree_size(struct latch_tree_node *n)

{

	struct module_layout *layout = container_of(n, struct module_layout, mtn.node);



	return (unsigned long)layout->size;

}



static __always_inline bool

mod_tree_less(struct latch_tree_node *a, struct latch_tree_node *b)

{

	return __mod_tree_val(a) < __mod_tree_val(b);

}



static __always_inline int

mod_tree_comp(void *key, struct latch_tree_node *n)

{

	unsigned long val = (unsigned long)key;

	unsigned long start, end;



	start = __mod_tree_val(n);

	if (val < start)

		return -1;



	end = start + __mod_tree_size(n);

	if (val >= end)

		return 1;



	return 0;

}



static const struct latch_tree_ops mod_tree_ops = {

	.less = mod_tree_less,

	.comp = mod_tree_comp,

};



static noinline void __mod_tree_insert(struct mod_tree_node *node)

{

	latch_tree_insert(&node->node, &mod_tree.root, &mod_tree_ops);

}



static void __mod_tree_remove(struct mod_tree_node *node)

{

	latch_tree_erase(&node->node, &mod_tree.root, &mod_tree_ops);

}



/*

 * These modifications: insert, remove_init and remove; are serialized by the

 * module_mutex.

 */

void mod_tree_insert(struct module *mod)

{

	mod->core_layout.mtn.mod = mod;

	mod->init_layout.mtn.mod = mod;



	__mod_tree_insert(&mod->core_layout.mtn);

	if (mod->init_layout.size)

		__mod_tree_insert(&mod->init_layout.mtn);

}



void mod_tree_remove_init(struct module *mod)

{

	if (mod->init_layout.size)

		__mod_tree_remove(&mod->init_layout.mtn);

}



void mod_tree_remove(struct module *mod)

{

	__mod_tree_remove(&mod->core_layout.mtn);

	mod_tree_remove_init(mod);

}



struct module *mod_find(unsigned long addr)

{

	struct latch_tree_node *ltn;



	ltn = latch_tree_find((void *)addr, &mod_tree.root, &mod_tree_ops);

	if (!ltn)

		return NULL;



	return container_of(ltn, struct mod_tree_node, node)->mod;

}