I have asked a question about hooking functions in Linux kernel. Now I'm learning about ftrace.
I wrote a sample program that you can see below, it worked fine on Linux 5.x and 6.x 66-bit as well as Aarch64 on a Raspberry Pi 4 and it worked as expected.
Now I'm testing it on an older kernel, 4.x running inside Qemu as it's Aarch64. But I keep getting EINVAL from ftrace_set_filter() and ftrace_set_filter_ip().
This code is inspired by xcellerator examples https://github.com/xcellerator/linux_kernel_hacking, I modified the source code a lot based on anything I could find online to fix my issue, so any problem with the code is probably mine mistake and not the original authors.
Header,
/*
* Helper library for ftrace hooking kernel functions
* Author: Harvey Phillips (xcellerator@gmx.com)
* License: GPL
* */
#include <linux/ftrace.h>
#include <linux/linkage.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/version.h>
#if LINUX_VERSION_CODE < KERNEL_VERSION(5,11,0)
#define FTRACE_OPS_FL_RECURSION FTRACE_OPS_FL_RECURSION_SAFE
#endif
#if LINUX_VERSION_CODE < KERNEL_VERSION(5,11,0)
#define ftrace_regs pt_regs
static __always_inline struct pt_regs *ftrace_get_regs(struct ftrace_regs *fregs)
{
return fregs;
}
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,7,0)
#define KPROBE_LOOKUP 1
#include <linux/kprobes.h>
static struct kprobe kp = {
.symbol_name = "kallsyms_lookup_name"
};
#endif
#define HOOK(_name, _hook, _orig) \
{ \
.name = (_name), \
.function = (_hook), \
.original = (_orig), \
}
#define USE_FENTRY_OFFSET 0
#if !USE_FENTRY_OFFSET
#pragma GCC optimize("-fno-optimize-sibling-calls")
#endif
struct ftrace_hook {
const char *name;
void *function;
void *original;
unsigned long address;
struct ftrace_ops ops;
};
static int fh_resolve_hook_address(struct ftrace_hook *hook)
{
#ifdef KPROBE_LOOKUP
typedef unsigned long (*kallsyms_lookup_name_t)(const char *name);
kallsyms_lookup_name_t kallsyms_lookup_name;
int ret;
ret = register_kprobe(&kp);
if (ret < 0) {
printk("register_kprobe failed: %d\n", ret);
return ret;
}
kallsyms_lookup_name = (kallsyms_lookup_name_t) kp.addr;
unregister_kprobe(&kp);
#endif
hook->address = kallsyms_lookup_name(hook->name);
if (!hook->address) {
printk("unresolved symbol: %s\n", hook->name);
return -ENOENT;
}
#if USE_FENTRY_OFFSET
*((unsigned long*) hook->original) = hook->address + MCOUNT_INSN_SIZE;
#else
*((unsigned long*) hook->original) = hook->address;
#endif
printk("Resolved address of %s: %lx\n", hook->name, hook->address);
return 0;
}
static void notrace fh_ftrace_thunk(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *ops, struct ftrace_regs *fregs)
{
printk("fh_ftrace_thunk - Init\n");
struct pt_regs *regs = ftrace_get_regs(fregs);
printk("ftrace_get_regs - After\n");
struct ftrace_hook *hook = container_of(ops, struct ftrace_hook, ops);
#if USE_FENTRY_OFFSET
regs->pc = (unsigned long)hook->function;
#else
if (!within_module(parent_ip, THIS_MODULE))
regs->pc = (unsigned long)hook->function;
#endif
}
int fh_install_hook(struct ftrace_hook *hook)
{
int err;
printk("fh_install_hook - prologue\n");
err = fh_resolve_hook_address(hook);
printk("fh_resolve_hook - calling fh_resolve_hook_address\n");
if (err) {
printk("fh_resolve_hook_address() failed: %d\n", err);
return err;
}
hook->ops.func = fh_ftrace_thunk;
hook->ops.flags = FTRACE_OPS_FL_SAVE_REGS
| FTRACE_OPS_FL_RECURSION
| FTRACE_OPS_FL_IPMODIFY;
printk("fh_install_hook - BEFORE ftrace_set_filter_ip\n");
err = ftrace_set_filter_ip(&hook->ops, hook->address, 0, 0);
printk("fh_install_hook - AFTER ftrace_set_filter_ip\n");
if (err) {
printk("ftrace_set_filter_ip() failed: %d\n", err);
return err;
}
printk("register_ftrace_function - before\n");
err = register_ftrace_function(&hook->ops);
printk("register_ftrace_function - after\n");
if (err) {
printk("register_ftrace_function() failed: %d\n", err);
ftrace_set_filter_ip(&hook->ops, hook->address, 1, 0);
return err;
}
printk("register_ftrace_function - return\n");
return 0;
}
/**
* fh_remove_hooks() - disable and unregister a single hook
* @hook: a hook to remove
*/
void fh_remove_hook(struct ftrace_hook *hook)
{
int err;
err = unregister_ftrace_function(&hook->ops);
if (err) {
printk("unregister_ftrace_function() failed: %d\n", err);
}
err = ftrace_set_filter_ip(&hook->ops, hook->address, 1, 0);
if (err) {
printk("ftrace_set_filter_ip() failed: %d\n", err);
}
}
/**
* fh_install_hooks() - register and enable multiple hooks
* @hooks: array of hooks to install
* @count: number of hooks to install
*
* If some hooks fail to install then all hooks will be removed.
*
* Returns: zero on success, negative error code otherwise.
*/
int fh_install_hooks(struct ftrace_hook *hooks, size_t count)
{
int err;
size_t i;
for (i = 0; i < count; i++) {
printk("fh_install_hooks - %lu\n", i);
err = fh_install_hook(&hooks[i]);
if (err)
goto error;
}
return 0;
error:
printk("fh_install_hooks - error statement\n");
while (i != 0) {
fh_remove_hook(&hooks[--i]);
}
return err;
}
/**
* fh_remove_hooks() - disable and unregister multiple hooks
* @hooks: array of hooks to remove
* @count: number of hooks to remove
*/
void fh_remove_hooks(struct ftrace_hook *hooks, size_t count)
{
size_t i;
for (i = 0; i < count; i++)
fh_remove_hook(&hooks[i]);
}
Kernel module,
#include <linux/init.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/ftrace.h>
#include <linux/ptrace.h>
#include <linux/sched.h>
#include <linux/uaccess.h>
#include <linux/version.h>
#include "ftrace_helper.h"
MODULE_LICENSE("GPL");
MODULE_AUTHOR("jelal");
MODULE_DESCRIPTION("Hook into execve syscall");
MODULE_VERSION("0.01");
#ifdef CONFIG_ARM64
static asmlinkage long (*orig_sys_execve)(const char __user *filename,
const char __user *const __user *argv,
const char __user *const __user *envp);
static asmlinkage long hook_sys_execve(const char __user *filename,
const char __user *const __user *argv,
const char __user *const __user *envp) {
char fname[256];
if (strncpy_from_user(fname, filename, sizeof(fname)) > 0) {
printk(KERN_INFO "execve called with filename: %s\n", fname);
} else {
printk(KERN_INFO "execve called with an unknown filename\n");
}
return orig_sys_execve(filename, argv, envp);
}
static struct ftrace_hook hooks[] = {
HOOK("__arm64_sys_execve", hook_sys_execve, &orig_sys_execve),
};
static int __init testftrace_init(void) {
int err;
printk(KERN_INFO "testftrace: Init\n");
err = fh_install_hooks(hooks, ARRAY_SIZE(hooks));
if (err) {
printk(KERN_ERR "fh_install_hooks() failed: %d\n", err);
return err;
}
printk(KERN_INFO "testftrace: Loaded >:-)\n");
return 0;
}
static void __exit testftrace_exit(void) {
fh_remove_hooks(hooks, ARRAY_SIZE(hooks));
printk(KERN_INFO "testftrace: Unloaded :-(\n");
}
module_init(testftrace_init);
module_exit(testftrace_exit);
#else
static int __init testftrace_init(void) {
printk(KERN_INFO "testftrace: Only supported on ARM64 architecture\n");
return -ENODEV;
}
static void __exit testftrace_exit(void) {
printk(KERN_INFO "testftrace: Unloaded\n");
}
module_init(testftrace_init);
module_exit(testftrace_exit);
#endif
The Makefile,
obj-m += testftrace.o
all:
make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules
clean:
make -C /lib/modules/$(shell uname -r)/build M=$(PWD) clean
I am root user, running make followed by insmod testftrace.ko and I'm getting the following error,
insmod: ERROR: could not insert module testftrace.ko: Invalid parameters
and in dmesg -w,
19855.397220] testftrace: Init
[19855.400273] Resolved address of __arm64_sys_execve: ffff00000834a260
[19855.400536] ftrace_set_filter_ip() failed: -22
[19855.400563] fh_install_hooks - error statement
[19855.400581] fh_install_hooks() failed: -22
Error -22 or EINVAL could happen when the address for the function you want to hook is invalid, I checked kallsyms to check the address,
root@localhost execvehook]# cat /proc/kallsyms | grep _sys_exec
ffff00000834a260 T __arm64_sys_execve
...
As you can see, the address for this function seem to be correct. I tried with system call functions and none-system call function but I'm getting the same error.
I tried to use ftrace_set_filter instead of ftrace_set_filter_ip but I got the same EINVAL error.
This is also the kernel config of the Linux that I'm running under Qemu,
CONFIG_HAVE_LIVEPATCH_WO_FTRACE=y
CONFIG_LIVEPATCH_WO_FTRACE=y
# CONFIG_PSTORE_FTRACE is not set
CONFIG_HAVE_DYNAMIC_FTRACE=y
CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y
CONFIG_FTRACE=y
CONFIG_FTRACE_SYSCALLS=y
CONFIG_DYNAMIC_FTRACE=y
CONFIG_FTRACE_MCOUNT_RECORD=y
# CONFIG_FTRACE_STARTUP_TEST is not set
At this point, I ran out of ideas to test and my searches didn't lead to anything helpful. I hope to get some help from here.
The problem I'm trying to solve is, to be able to use ftrace on kernel 4.x and older on Arm and Intel.
Thanks, Jelal
Thanks everyone for your comments. Here I post the answer for the initial question after some testing. I followed some of the comments and I confirm the issue was related to QEMU, when I tested on a physical machine and a hypervisor it worked fine.