path: root/SOURCE/module/kernel/load.c

/*
 * Linux内核诊断工具--内核态load-monitor功能
 *
 * Copyright (C) 2020 Alibaba Ltd.
 *
 * 作者: Baoyou Xie <[email protected]>
 *
 * License terms: GNU General Public License (GPL) version 3
 *
 */

#include <linux/hrtimer.h>
#include <linux/kernel.h>
#include <linux/kallsyms.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/timex.h>
#include <linux/tracepoint.h>
#include <trace/events/irq.h>
#include <linux/proc_fs.h>
#include <linux/init.h>
#include <linux/sysctl.h>
#include <trace/events/napi.h>
#include <linux/rtc.h>
#include <linux/time.h>
#include <linux/rbtree.h>
#include <linux/cpu.h>
#include <linux/syscalls.h>
#include <asm-generic/rwonce.h> // for READ_ONCE(x)

#include <asm/irq_regs.h>

#include "internal.h"
#include "mm_tree.h"
#include "pub/trace_file.h"
#include "pub/variant_buffer.h"
#include "pub/trace_point.h"

#include "uapi/load_monitor.h"

static atomic64_t diag_nr_running = ATOMIC64_INIT(0);

static struct diag_load_monitor_settings load_monitor_settings;

static unsigned int load_monitor_alloced;

static struct mm_tree mm_tree;

unsigned long *orig_avenrun_r;
unsigned long *orig_avenrun;

static struct load_monitor_cpu_run ld_mon_cpu_run[NR_CPUS];
static struct diag_variant_buffer load_monitor_variant_buffer;

static ktime_t last_dump;

static void __maybe_unused clean_data(void)
{
	cleanup_mm_tree(&mm_tree);
}

#ifndef FSHIFT
#define FSHIFT		11		/* nr of bits of precision */
#endif
#ifndef FIXED_1
#define FIXED_1		(1<<FSHIFT)	/* 1.0 as fixed-point */
#endif
#ifndef LOAD_INT
#define LOAD_INT(x) ((x) >> FSHIFT)
#endif
#ifndef LOAD_FRAC
#define LOAD_FRAC(x) LOAD_INT(((x) & (FIXED_1-1)) * 100)
#endif

#if defined(UPSTREAM_4_19_32)
void diag_load_timer(struct diag_percpu_context *context)
{
	return;
}
#else

// https://www.spinics.net/lists/kernel/msg3582022.html
// fun remove from 5.8.rc3, just for build to add it here
#define task_contributes_to_load2(task)	((READ_ONCE(task->__state)  & TASK_UNINTERRUPTIBLE) != 0 && \
					 (task->flags & PF_FROZEN) == 0 && \
					 (READ_ONCE(task->__state) & TASK_NOLOAD) == 0)

static void load_monitor_ipi(void *ignore)
{
	struct load_monitor_cpu_run *cpu_run;
	struct task_struct *tsk;
	unsigned long flags;
	int cpu;

	tsk = current;
	cpu = smp_processor_id();
	if (cpu >= NR_CPUS)
		return;

	cpu_run = &ld_mon_cpu_run[cpu];

	cpu_run->id = get_cycles();
	cpu_run->et_type = et_load_monitor_cpu_run;
	cpu_run->cpu = cpu;
	do_diag_gettimeofday(&cpu_run->tv);

	diag_task_brief(tsk, &cpu_run->task);
	diag_task_kern_stack(tsk, &cpu_run->kern_stack);
	diag_task_user_stack(tsk, &cpu_run->user_stack);

	diag_variant_buffer_spin_lock(&load_monitor_variant_buffer, flags);
	diag_variant_buffer_reserve(&load_monitor_variant_buffer, sizeof(struct load_monitor_cpu_run));
	diag_variant_buffer_write_nolock(&load_monitor_variant_buffer,
					 cpu_run, sizeof(struct load_monitor_cpu_run));
	diag_variant_buffer_seal(&load_monitor_variant_buffer);
	diag_variant_buffer_spin_unlock(&load_monitor_variant_buffer, flags);
}

void diag_load_timer(struct diag_percpu_context *context)
{
	u64 ms;
	bool scare = false;
	unsigned long load_d;
	struct task_struct *g, *p;

	if (!load_monitor_settings.activated)
		return;
	if (!load_monitor_settings.threshold_load && !load_monitor_settings.threshold_load_r && !load_monitor_settings.threshold_load_d
		&& !load_monitor_settings.threshold_task_d)
		return;
	if (smp_processor_id() != 0)
		return;

	if (load_monitor_settings.threshold_load && orig_avenrun
			&& LOAD_INT(orig_avenrun[0]) >= load_monitor_settings.threshold_load)
		scare = true;
	if (load_monitor_settings.threshold_load_r && orig_avenrun_r
			&& LOAD_INT(orig_avenrun_r[0]) >= load_monitor_settings.threshold_load_r)
		scare = true;
	if (load_monitor_settings.threshold_load_d && orig_avenrun && orig_avenrun_r) {
		load_d = LOAD_INT(orig_avenrun[0] - orig_avenrun_r[0]);

		if (load_d >= load_monitor_settings.threshold_load_d && load_d < 999999)
			scare = true;
	}
	if (load_monitor_settings.threshold_task_d) {
		int nr_uninterrupt = 0;
		struct task_struct *g, *p;

		rcu_read_lock();

		do_each_thread(g, p) {
			if (task_contributes_to_load2(p))
				nr_uninterrupt++;
		} while_each_thread(g, p);

		rcu_read_unlock();

		if (nr_uninterrupt >= load_monitor_settings.threshold_task_d)
			scare = true;
	}

	if (scare) {
		unsigned long flags;
		static struct load_monitor_detail detail;
		static struct load_monitor_task tsk_info;
		unsigned long event_id;

		ms = ktime_to_ms(ktime_sub(ktime_get(), last_dump));
		if (!load_monitor_settings.mass && ms < 10 * 1000)
			return;

		last_dump = ktime_get();
		
		if (orig_avenrun) {
			detail.load_1_1 = LOAD_INT(orig_avenrun[0]);
			detail.load_1_2 = LOAD_FRAC(orig_avenrun[0]);
			detail.load_5_1 = LOAD_INT(orig_avenrun[1]);
			detail.load_5_2 = LOAD_FRAC(orig_avenrun[1]);
			detail.load_15_1 = LOAD_INT(orig_avenrun[2]);
			detail.load_15_2 = LOAD_FRAC(orig_avenrun[2]);
		}
		if (orig_avenrun && orig_avenrun_r) {
			unsigned long l1, l2, l3;

			detail.load_r_1_1 = LOAD_INT(orig_avenrun_r[0]);
			detail.load_r_1_2 = LOAD_FRAC(orig_avenrun_r[0]);
			detail.load_r_5_1 = LOAD_INT(orig_avenrun_r[1]);
			detail.load_r_5_2 = LOAD_FRAC(orig_avenrun_r[1]);
			detail.load_r_15_1 = LOAD_INT(orig_avenrun_r[2]);
			detail.load_r_15_2 = LOAD_FRAC(orig_avenrun_r[2]);
			l1 = orig_avenrun[0] - orig_avenrun_r[0];
			l2 = orig_avenrun[1] - orig_avenrun_r[1];
			l3 = orig_avenrun[2] - orig_avenrun_r[2];
			detail.load_d_1_1 = LOAD_INT(l1);
			detail.load_d_1_2 = LOAD_FRAC(l1);
			detail.load_d_5_1 = LOAD_INT(l2);
			detail.load_d_5_2 = LOAD_FRAC(l2);
			detail.load_d_15_1 = LOAD_INT(l3);
			detail.load_d_15_2 = LOAD_FRAC(l3);
		}

		event_id = get_cycles();
		detail.id = event_id;
		detail.et_type = et_load_monitor_detail;
		do_diag_gettimeofday(&detail.tv);

		rcu_read_lock();
		diag_variant_buffer_spin_lock(&load_monitor_variant_buffer, flags);
		diag_variant_buffer_reserve(&load_monitor_variant_buffer, sizeof(struct load_monitor_detail));
		diag_variant_buffer_write_nolock(&load_monitor_variant_buffer, &detail, sizeof(struct load_monitor_detail));
		diag_variant_buffer_seal(&load_monitor_variant_buffer);
		diag_variant_buffer_spin_unlock(&load_monitor_variant_buffer, flags);
		do_each_thread(g, p) {
// [[PATCH v2 7/7] sched: Change task_struct::state](https://lore.kernel.org/all/[email protected]/#r)
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 13, 0)
			if ((p->state == TASK_RUNNING) || task_contributes_to_load2(p)) {
#else
			if ((p->__state == TASK_RUNNING) || task_contributes_to_load2(p)) {
#endif
				tsk_info.et_type = et_load_monitor_task;
				tsk_info.id = event_id;
				tsk_info.tv = detail.tv;
				diag_task_brief(p, &tsk_info.task);
				diag_task_kern_stack(p, &tsk_info.kern_stack);
				dump_proc_chains_argv(load_monitor_settings.style, &mm_tree, p, &tsk_info.proc_chains);
				diag_variant_buffer_spin_lock(&load_monitor_variant_buffer, flags);
				diag_variant_buffer_reserve(&load_monitor_variant_buffer, sizeof(struct load_monitor_task));
				diag_variant_buffer_write_nolock(&load_monitor_variant_buffer,
					&tsk_info, sizeof(struct load_monitor_task));
				diag_variant_buffer_seal(&load_monitor_variant_buffer);
				diag_variant_buffer_spin_unlock(&load_monitor_variant_buffer, flags);
			}
		} while_each_thread(g, p);
		rcu_read_unlock();

		if (!load_monitor_settings.mass && load_monitor_settings.cpu_run) {
			smp_call_function(load_monitor_ipi, NULL, 1);
			load_monitor_ipi(NULL);
		}
	}
}
#endif

#if KERNEL_VERSION(4, 9, 0) <= LINUX_VERSION_CODE
__maybe_unused static void trace_sched_process_exec_hit(void *__data,
	struct task_struct *tsk,
	pid_t old_pid,
	struct linux_binprm *bprm)
#elif KERNEL_VERSION(3, 10, 0) <= LINUX_VERSION_CODE
__maybe_unused static void trace_sched_process_exec_hit(void *__data,
	struct task_struct *tsk,
	pid_t old_pid,
	struct linux_binprm *bprm)
#endif
#if KERNEL_VERSION(3, 10, 0) <= LINUX_VERSION_CODE
{
	atomic64_inc_return(&diag_nr_running);
	diag_hook_exec(bprm, &mm_tree);
	atomic64_dec_return(&diag_nr_running);
}
#endif

#if KERNEL_VERSION(4, 9, 0) <= LINUX_VERSION_CODE
static void trace_sched_process_exit_hit(void *__data, struct task_struct *tsk)
#elif KERNEL_VERSION(3, 10, 0) <= LINUX_VERSION_CODE
static void trace_sched_process_exit_hit(void *__data, struct task_struct *tsk)
#else
static void trace_sched_process_exit_hit(struct task_struct *tsk)
#endif
{
	diag_hook_process_exit_exec(tsk, &mm_tree);
}

static int __activate_load_monitor(void)
{
	int ret = 0;

	clean_data();

	ret = alloc_diag_variant_buffer(&load_monitor_variant_buffer);
	if (ret)
		goto out_variant_buffer;
	load_monitor_alloced = 1;

	if (load_monitor_settings.style == 1) {
#if KERNEL_VERSION(3, 10, 0) <= LINUX_VERSION_CODE
		hook_tracepoint("sched_process_exec", trace_sched_process_exec_hit, NULL);
#endif
		hook_tracepoint("sched_process_exit", trace_sched_process_exit_hit, NULL);
	}
	//get_argv_processes(&mm_tree);

	return 1;
out_variant_buffer:
	return 0;
}

int activate_load_monitor(void)
{
	if (!load_monitor_settings.activated)
		load_monitor_settings.activated = __activate_load_monitor();

	return load_monitor_settings.activated;
}

static void __deactivate_load_monitor(void)
{
	if (load_monitor_settings.style == 1) {
#if KERNEL_VERSION(3, 10, 0) <= LINUX_VERSION_CODE
		unhook_tracepoint("sched_process_exec", trace_sched_process_exec_hit, NULL);
#endif
		unhook_tracepoint("sched_process_exit", trace_sched_process_exit_hit, NULL);
	}

	synchronize_sched();
	msleep(10);
	while (atomic64_read(&diag_nr_running) > 0) {
		msleep(10);
	}

	clean_data();

	load_monitor_settings.verbose = 0;
	load_monitor_settings.threshold_load = 0;
	load_monitor_settings.threshold_load_r = 0;
	load_monitor_settings.threshold_load_d = 0;
	load_monitor_settings.threshold_task_d = 0;
	last_dump = ktime_set(0, 0);
}

int deactivate_load_monitor(void)
{
	if (load_monitor_settings.activated)
		__deactivate_load_monitor();
	load_monitor_settings.activated = 0;

	return load_monitor_settings.activated;
}

int load_monitor_syscall(struct pt_regs *regs, long id)
{
	int __user *user_ptr_len;
	size_t __user user_buf_len;
	void __user *user_buf;
	int ret = 0;
	struct diag_load_monitor_settings settings;

	switch (id) {
	case DIAG_LOAD_MONITOR_SET:
		user_buf = (void __user *)SYSCALL_PARAM1(regs);
		user_buf_len = (size_t)SYSCALL_PARAM2(regs);

		if (user_buf_len != sizeof(struct diag_load_monitor_settings)) {
			ret = -EINVAL;
		} else if (load_monitor_settings.activated) {
			ret = -EBUSY;
		} else {
			ret = copy_from_user(&settings, user_buf, user_buf_len);
			if (!ret) {
				load_monitor_settings = settings;
			}
		}
		break;
	case DIAG_LOAD_MONITOR_SETTINGS:
		user_buf = (void __user *)SYSCALL_PARAM1(regs);
		user_buf_len = (size_t)SYSCALL_PARAM2(regs);

		if (user_buf_len != sizeof(struct diag_load_monitor_settings)) {
			ret = -EINVAL;
		} else {
			settings = load_monitor_settings;
			ret = copy_to_user(user_buf, &settings, user_buf_len);
		}
		break;
	case DIAG_LOAD_MONITOR_DUMP:
		user_ptr_len = (void __user *)SYSCALL_PARAM1(regs);
		user_buf = (void __user *)SYSCALL_PARAM2(regs);
		user_buf_len = (size_t)SYSCALL_PARAM3(regs);

		if (!load_monitor_alloced) {
			ret = -EINVAL;
		} else {
			ret = copy_to_user_variant_buffer(&load_monitor_variant_buffer,
					user_ptr_len, user_buf, user_buf_len);
			record_dump_cmd("load-monitor");
		}
		break;
	default:
		ret = -ENOSYS;
		break;
	}

	return ret;
}

long diag_ioctl_load_monitor(unsigned int cmd, unsigned long arg)
{
	int ret = 0;
	struct diag_load_monitor_settings settings;
	struct diag_ioctl_dump_param dump_param;

	switch (cmd) {
	case CMD_LOAD_MONITOR_SET:
		if (load_monitor_settings.activated) {
			ret = -EBUSY;
		} else {
			ret = copy_from_user(&settings, (void *)arg, sizeof(struct diag_load_monitor_settings));
			if (!ret) {
				load_monitor_settings = settings;
			}
		}
		break;
	case CMD_LOAD_MONITOR_SETTINGS:
		settings = load_monitor_settings;
		ret = copy_to_user((void *)arg, &settings, sizeof(struct diag_load_monitor_settings));
		break;
	case CMD_LOAD_MONITOR_DUMP:
		ret = copy_from_user(&dump_param, (void *)arg, sizeof(struct diag_ioctl_dump_param));
		if (!load_monitor_alloced) {
			ret = -EINVAL;
		} else if (!ret) {
			ret = copy_to_user_variant_buffer(&load_monitor_variant_buffer,
					dump_param.user_ptr_len, dump_param.user_buf, dump_param.user_buf_len);
			record_dump_cmd("load-monitor");
		}
		break;
	default:
		ret = -ENOSYS;
		break;
	}

	return ret;
}

int diag_load_init(void)
{
	LOOKUP_SYMS_NORET(avenrun_r);
	LOOKUP_SYMS_NORET(avenrun);

	init_mm_tree(&mm_tree);
	init_diag_variant_buffer(&load_monitor_variant_buffer, 50 * 1024 * 1024);
	if (load_monitor_settings.activated)
		load_monitor_settings.activated = __activate_load_monitor();

	return 0;
}

void diag_load_exit(void)
{
	if (load_monitor_settings.activated)
		deactivate_load_monitor();
	load_monitor_settings.activated = 0;
	destroy_diag_variant_buffer(&load_monitor_variant_buffer);
}