Commit a5cd1177 authored by Xu Kuohai's avatar Xu Kuohai Committed by Pu Lehui
Browse files

arm64: ftrace: Support direct call for no literal module functions

hulk inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/I9FGRE



--------------------------------

For kernel modules copmiled without DIRECT_CALL, there is no literal
space reserved on the function entry, this patch add support for these
functions.

Signed-off-by: default avatarXu Kuohai <xukuohai@huawei.com>
Signed-off-by: default avatarPu Lehui <pulehui@huawei.com>
parent d7a6aa35
Loading
Loading
Loading
Loading
+62 −12
Original line number Diff line number Diff line
@@ -117,6 +117,9 @@ enum ftrace_callsite_action {

static unsigned long ftrace_literal_call_addr(struct dyn_ftrace *rec)
{
	if (rec->arch.func == 0UL)
		return 0UL;

	return rec->arch.func - 2 * AARCH64_INSN_SIZE;
}

@@ -125,6 +128,9 @@ static unsigned long ftrace_literal_addr(struct dyn_ftrace *rec)
	unsigned long addr = 0;

	addr = ftrace_literal_call_addr(rec);
	if (addr == 0UL)
		return 0UL;

	if (addr % sizeof(long))
		addr -= 3 * AARCH64_INSN_SIZE;
	else
@@ -154,6 +160,9 @@ static int ftrace_init_literal(struct module *mod, struct dyn_ftrace *rec)
	old = aarch64_insn_gen_nop();

	addr = ftrace_literal_addr(rec);
	if (addr == 0UL)
		return 0UL;

	ftrace_update_literal(addr, 0, FC_INIT);

	pc = ftrace_literal_call_addr(rec);
@@ -222,10 +231,12 @@ static bool ftrace_find_callable_addr(struct dyn_ftrace *rec,
		unsigned long literal_addr;

		literal_addr = ftrace_literal_addr(rec);
		if (literal_addr != 0UL) {
			ftrace_update_literal(literal_addr, *addr, action);
			*addr = ftrace_literal_call_addr(rec);
			return true;
		}
	}

	/*
	 * When the target is outside of the range of a 'BL' instruction, we
@@ -343,27 +354,51 @@ int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec)
}
#endif

static int ftrace_nop_count(unsigned long addr)
{
	u32 insn;
	u32 nop = aarch64_insn_gen_nop();
	int count = 0;

	for (;;) {
		if (aarch64_insn_read((void *)addr, &insn))
			return -1;

		if (insn != nop)
			break;

		count++;
		addr += AARCH64_INSN_SIZE;
	}

	return count;
}

unsigned long ftrace_call_adjust(unsigned long addr)
{
	if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS)) {
		u32 insn;
		u32 nop = aarch64_insn_gen_nop();
		int count = ftrace_nop_count(addr);

		if (count != 5 && count != 7 && count != 2)
			return 0;

		if (count == 5 || count == 7) {
			/* Skip the first 5 NOPS */
			addr += 5 * AARCH64_INSN_SIZE;

			/* Skip bti c */
			if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) {
				if (aarch64_insn_read((void *)addr, &insn))
					return 0;

		if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) {
			if (insn != nop) {
				if (insn != nop)
					addr += AARCH64_INSN_SIZE;
				if (aarch64_insn_read((void *)addr, &insn))
					return 0;
			}
		}

		if (WARN_ON_ONCE(insn != nop))
		if (ftrace_nop_count(addr) != 2)
			return 0;

		return addr + AARCH64_INSN_SIZE;
@@ -421,10 +456,25 @@ void arch_ftrace_update_code(int command)
	ftrace_modify_all_code(command);
}

bool ftrace_directable(struct dyn_ftrace *rec)
{
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
	return rec->arch.func != 0UL;
#else
	return false;
#endif
}

void ftrace_rec_arch_init(struct dyn_ftrace *rec, unsigned long func)
{
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
	int count;

	count = ftrace_nop_count(func);
	if (count == 5 || count == 7)
		rec->arch.func = func + 5 * AARCH64_INSN_SIZE;
	else
		rec->arch.func = 0UL;
#endif
}

+2 −0
Original line number Diff line number Diff line
@@ -280,6 +280,8 @@ struct ftrace_func_entry {

struct dyn_ftrace;

bool ftrace_directable(struct dyn_ftrace *rec);

#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
extern int ftrace_direct_func_count;
int register_ftrace_direct(unsigned long ip, unsigned long addr);
+8 −0
Original line number Diff line number Diff line
@@ -5092,6 +5092,11 @@ static struct ftrace_direct_func *ftrace_alloc_direct_func(unsigned long addr)
	return direct;
}

bool __weak ftrace_directable(struct dyn_ftrace *rec)
{
	return true;
}

/**
 * register_ftrace_direct - Call a custom trampoline directly
 * @ip: The address of the nop at the beginning of a function
@@ -5133,6 +5138,9 @@ int register_ftrace_direct(unsigned long ip, unsigned long addr)
	if (!rec)
		goto out_unlock;

	if (!ftrace_directable(rec))
		goto out_unlock;

	/*
	 * Check if the rec says it has a direct call but we didn't
	 * find one earlier?