Merge tag 'modules-6.2-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/mcgrof/linux

Pull modules updates from Luis Chamberlain: "Tux gets for xmas an improvement to the average lookup performance of kallsyms_lookup_name() by 715x thanks to the work by Zhen Lei, which upgraded our old implementation from being O(n) to O(log(n)), while also retaining the old implementation support on /proc/kallsyms. The only penalty was increasing the memory footprint by 3 * kallsyms_num_syms. Folks who want to improve this further now also have a dedicated selftest facility through KALLSYMS_SELFTEST. Stephen Boyd added zstd in-kernel decompression support, but the only users of this would be folks using the load-pin LSM because otherwise we do module decompression in userspace. The only other thing with mentioning is a minor boot time optimization by Rasmus Villemoes which deferes param_sysfs_init() to late init. The rest is cleanups and minor fixes" * tag 'modules-6.2-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/mcgrof/linux: livepatch: Call klp_match_callback() in klp_find_callback() to avoid code duplication module/decompress: Support zstd in-kernel decompression kallsyms: Remove unneeded semicolon kallsyms: Add self-test facility livepatch: Use kallsyms_on_each_match_symbol() to improve performance kallsyms: Add helper kallsyms_on_each_match_symbol() kallsyms: Reduce the memory occupied by kallsyms_seqs_of_names[] kallsyms: Correctly sequence symbols when CONFIG_LTO_CLANG=y kallsyms: Improve the performance of kallsyms_lookup_name() scripts/kallsyms: rename build_initial_tok_table() module: Fix NULL vs IS_ERR checking for module_get_next_page kernel/params.c: defer most of param_sysfs_init() to late_initcall time module: Remove unused macros module_addr_min/max module: remove redundant module_sysfs_initialized variable

Linus Torvalds committed 3 years ago

Commit 3ba2c3f2 parents 0015edd 4f1354d16 files · +840 −43Browse files

Unified · Split

16 files

include/linux/kallsyms.h +9 −0
include/linux/module.h +0 −1
init/Kconfig +13 −0
kernel/Makefile +1 −0
kernel/kallsyms.c +104 −12
kernel/kallsyms_internal.h +1 −0
kernel/kallsyms_selftest.c +485 −0
kernel/kallsyms_selftest.h +13 −0
kernel/livepatch/core.c +19 −12
kernel/module/Kconfig +2 −1
kernel/module/decompress.c +94 −6
kernel/module/main.c +0 −3
kernel/module/sysfs.c +1 −1
kernel/params.c +19 −4
scripts/kallsyms.c +75 −3
scripts/link-vmlinux.sh +4 −0

include/linux/kallsyms.h+9−0

@@ -66,9 +66,12 @@ static inline void dereference_symbol_descriptor(void ptr) · 65 unmodified lines
66	66	}
67	67
68	68	#ifdef CONFIG_KALLSYMS
	69	+unsigned long kallsyms_sym_address(int idx);
69	70	int kallsyms_on_each_symbol(int (fn)(void , const char , struct module ,
70	71	unsigned long),
71	72	void *data);
	73	+int kallsyms_on_each_match_symbol(int (fn)(void , unsigned long),
	74	+ const char name, void data);
72	75
73	76	/* Lookup the address for a symbol. Returns 0 if not found. */
74	77	unsigned long kallsyms_lookup_name(const char *name);
@@ -168,6 +171,12 @@ static inline int kallsyms_on_each_symbol(int (fn)(void , const char *, struct · 93 unmodified lines
168	171	{
169	172	return -EOPNOTSUPP;
170	173	}
	174	+
	175	+static inline int kallsyms_on_each_match_symbol(int (fn)(void , unsigned long),
	176	+ const char name, void data)
	177	+{
	178	+ return -EOPNOTSUPP;
	179	+}
171	180	#endif /CONFIG_KALLSYMS/
172	181
173	182	static inline void print_ip_sym(const char *loglvl, unsigned long ip)

include/linux/module.h+0−1

@@ -827,7 +827,6 @@ void dereference_module_function_descriptor(struct module mod, void *ptr) · 826 unmodified lines
827	827	#ifdef CONFIG_SYSFS
828	828	extern struct kset *module_kset;
829	829	extern struct kobj_type module_ktype;
830		−extern int module_sysfs_initialized;
831	830	#endif /* CONFIG_SYSFS */
832	831
833	832	#define symbol_request(x) try_then_request_module(symbol_get(x), "symbol:" #x)

init/Kconfig+13−0

@@ -1723,6 +1723,19 @@ config KALLSYMS · 1722 unmodified lines
1723	1723	symbolic stack backtraces. This increases the size of the kernel
1724	1724	somewhat, as all symbols have to be loaded into the kernel image.
1725	1725
	1726	+config KALLSYMS_SELFTEST
	1727	+ bool "Test the basic functions and performance of kallsyms"
	1728	+ depends on KALLSYMS
	1729	+ default n
	1730	+ help
	1731	+ Test the basic functions and performance of some interfaces, such as
	1732	+ kallsyms_lookup_name. It also calculates the compression rate of the
	1733	+ kallsyms compression algorithm for the current symbol set.
	1734	+
	1735	+ Start self-test automatically after system startup. Suggest executing
	1736	+ "dmesg \| grep kallsyms_selftest" to collect test results. "finish" is
	1737	+ displayed in the last line, indicating that the test is complete.
	1738	+
1726	1739	config KALLSYMS_ALL
1727	1740	bool "Include all symbols in kallsyms"
1728	1741	depends on DEBUG_KERNEL && KALLSYMS

kernel/Makefile+1−0

@@ -69,6 +69,7 @@ endif · 68 unmodified lines
69	69	obj-$(CONFIG_UID16) += uid16.o
70	70	obj-$(CONFIG_MODULE_SIG_FORMAT) += module_signature.o
71	71	obj-$(CONFIG_KALLSYMS) += kallsyms.o
	72	+obj-$(CONFIG_KALLSYMS_SELFTEST) += kallsyms_selftest.o
72	73	obj-$(CONFIG_BSD_PROCESS_ACCT) += acct.o
73	74	obj-$(CONFIG_CRASH_CORE) += crash_core.o
74	75	obj-$(CONFIG_KEXEC_CORE) += kexec_core.o

kernel/kallsyms.c+104−12

@@ -146,7 +146,7 @@ static unsigned int get_symbol_offset(unsigned long pos) · 145 unmodified lines
146	146	return name - kallsyms_names;
147	147	}
148	148
149		−static unsigned long kallsyms_sym_address(int idx)
	149	+unsigned long kallsyms_sym_address(int idx)
150	150	{
151	151	if (!IS_ENABLED(CONFIG_KALLSYMS_BASE_RELATIVE))
152	152	return kallsyms_addresses[idx];
@@ -187,26 +187,100 @@ static bool cleanup_symbol_name(char *s) · 34 unmodified lines
187	187	return false;
188	188	}
189	189
	190	+static int compare_symbol_name(const char name, char namebuf)
	191	+{
	192	+ int ret;
	193	+
	194	+ ret = strcmp(name, namebuf);
	195	+ if (!ret)
	196	+ return ret;
	197	+
	198	+ if (cleanup_symbol_name(namebuf) && !strcmp(name, namebuf))
	199	+ return 0;
	200	+
	201	+ return ret;
	202	+}
	203	+
	204	+static unsigned int get_symbol_seq(int index)
	205	+{
	206	+ unsigned int i, seq = 0;
	207	+
	208	+ for (i = 0; i < 3; i++)
	209	+ seq = (seq << 8) \| kallsyms_seqs_of_names[3 * index + i];
	210	+
	211	+ return seq;
	212	+}
	213	+
	214	+static int kallsyms_lookup_names(const char *name,
	215	+ unsigned int *start,
	216	+ unsigned int *end)
	217	+{
	218	+ int ret;
	219	+ int low, mid, high;
	220	+ unsigned int seq, off;
	221	+ char namebuf[KSYM_NAME_LEN];
	222	+
	223	+ low = 0;
	224	+ high = kallsyms_num_syms - 1;
	225	+
	226	+ while (low <= high) {
	227	+ mid = low + (high - low) / 2;
	228	+ seq = get_symbol_seq(mid);
	229	+ off = get_symbol_offset(seq);
	230	+ kallsyms_expand_symbol(off, namebuf, ARRAY_SIZE(namebuf));
	231	+ ret = compare_symbol_name(name, namebuf);
	232	+ if (ret > 0)
	233	+ low = mid + 1;
	234	+ else if (ret < 0)
	235	+ high = mid - 1;
	236	+ else
	237	+ break;
	238	+ }
	239	+
	240	+ if (low > high)
	241	+ return -ESRCH;
	242	+
	243	+ low = mid;
	244	+ while (low) {
	245	+ seq = get_symbol_seq(low - 1);
	246	+ off = get_symbol_offset(seq);
	247	+ kallsyms_expand_symbol(off, namebuf, ARRAY_SIZE(namebuf));
	248	+ if (compare_symbol_name(name, namebuf))
	249	+ break;
	250	+ low--;
	251	+ }
	252	+ *start = low;
	253	+
	254	+ if (end) {
	255	+ high = mid;
	256	+ while (high < kallsyms_num_syms - 1) {
	257	+ seq = get_symbol_seq(high + 1);
	258	+ off = get_symbol_offset(seq);
	259	+ kallsyms_expand_symbol(off, namebuf, ARRAY_SIZE(namebuf));
	260	+ if (compare_symbol_name(name, namebuf))
	261	+ break;
	262	+ high++;
	263	+ }
	264	+ *end = high;
	265	+ }
	266	+
	267	+ return 0;
	268	+}
	269	+
190	270	/* Lookup the address for this symbol. Returns 0 if not found. */
191	271	unsigned long kallsyms_lookup_name(const char *name)
192	272	{
193		− char namebuf[KSYM_NAME_LEN];
194		− unsigned long i;
195		− unsigned int off;
	273	+ int ret;
	274	+ unsigned int i;
196	275
197	276	/* Skip the search for empty string. */
198	277	if (!*name)
199	278	return 0;
200	279
201		− for (i = 0, off = 0; i < kallsyms_num_syms; i++) {
202		− off = kallsyms_expand_symbol(off, namebuf, ARRAY_SIZE(namebuf));
203		−
204		− if (strcmp(namebuf, name) == 0)
205		− return kallsyms_sym_address(i);
	280	+ ret = kallsyms_lookup_names(name, &i, NULL);
	281	+ if (!ret)
	282	+ return kallsyms_sym_address(get_symbol_seq(i));
206	283
207		− if (cleanup_symbol_name(namebuf) && strcmp(namebuf, name) == 0)
208		− return kallsyms_sym_address(i);
209		− }
210	284	return module_kallsyms_lookup_name(name);
211	285	}
212	286
@@ -233,6 +307,24 @@ int kallsyms_on_each_symbol(int (fn)(void , const char , struct module , · 20 unmodified lines
233	307	return 0;
234	308	}
235	309
	310	+int kallsyms_on_each_match_symbol(int (fn)(void , unsigned long),
	311	+ const char name, void data)
	312	+{
	313	+ int ret;
	314	+ unsigned int i, start, end;
	315	+
	316	+ ret = kallsyms_lookup_names(name, &start, &end);
	317	+ if (ret)
	318	+ return 0;
	319	+
	320	+ for (i = start; !ret && i <= end; i++) {
	321	+ ret = fn(data, kallsyms_sym_address(get_symbol_seq(i)));
	322	+ cond_resched();
	323	+ }
	324	+
	325	+ return ret;
	326	+}
	327	+
236	328	static unsigned long get_symbol_pos(unsigned long addr,
237	329	unsigned long *symbolsize,
238	330	unsigned long *offset)

kernel/kallsyms_internal.h+1−0

@@ -26,5 +26,6 @@ extern const char kallsyms_token_table[] __weak; · 25 unmodified lines
26	26	extern const u16 kallsyms_token_index[] __weak;
27	27
28	28	extern const unsigned int kallsyms_markers[] __weak;
	29	+extern const u8 kallsyms_seqs_of_names[] __weak;
29	30
30	31	#endif // LINUX_KALLSYMS_INTERNAL_H_

kernel/kallsyms_selftest.c+485−0

@@ -0,0 +1,485 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
	2	+/*
	3	+ * Test the function and performance of kallsyms
	4	+ *
	5	+ * Copyright (C) Huawei Technologies Co., Ltd., 2022
	6	+ *
	7	+ * Authors: Zhen Lei <thunder.leizhen@huawei.com> Huawei
	8	+ */
	9	+
	10	+#define pr_fmt(fmt) "kallsyms_selftest: " fmt
	11	+
	12	+#include <linux/init.h>
	13	+#include <linux/module.h>
	14	+#include <linux/kallsyms.h>
	15	+#include <linux/random.h>
	16	+#include <linux/sched/clock.h>
	17	+#include <linux/kthread.h>
	18	+#include <linux/vmalloc.h>
	19	+
	20	+#include "kallsyms_internal.h"
	21	+#include "kallsyms_selftest.h"
	22	+
	23	+
	24	+#define MAX_NUM_OF_RECORDS 64
	25	+
	26	+struct test_stat {
	27	+ int min;
	28	+ int max;
	29	+ int save_cnt;
	30	+ int real_cnt;
	31	+ int perf;
	32	+ u64 sum;
	33	+ char *name;
	34	+ unsigned long addr;
	35	+ unsigned long addrs[MAX_NUM_OF_RECORDS];
	36	+};
	37	+
	38	+struct test_item {
	39	+ char *name;
	40	+ unsigned long addr;
	41	+};
	42	+
	43	+#define ITEM_FUNC(s) \
	44	+ { \
	45	+ .name = #s, \
	46	+ .addr = (unsigned long)s, \
	47	+ }
	48	+
	49	+#define ITEM_DATA(s) \
	50	+ { \
	51	+ .name = #s, \
	52	+ .addr = (unsigned long)&s, \
	53	+ }
	54	+
	55	+
	56	+static int kallsyms_test_var_bss_static;
	57	+static int kallsyms_test_var_data_static = 1;
	58	+int kallsyms_test_var_bss;
	59	+int kallsyms_test_var_data = 1;
	60	+
	61	+static int kallsyms_test_func_static(void)
	62	+{
	63	+ kallsyms_test_var_bss_static++;
	64	+ kallsyms_test_var_data_static++;
	65	+
	66	+ return 0;
	67	+}
	68	+
	69	+int kallsyms_test_func(void)
	70	+{
	71	+ return kallsyms_test_func_static();
	72	+}
	73	+
	74	+__weak int kallsyms_test_func_weak(void)
	75	+{
	76	+ kallsyms_test_var_bss++;
	77	+ kallsyms_test_var_data++;
	78	+ return 0;
	79	+}
	80	+
	81	+static struct test_item test_items[] = {
	82	+ ITEM_FUNC(kallsyms_test_func_static),
	83	+ ITEM_FUNC(kallsyms_test_func),
	84	+ ITEM_FUNC(kallsyms_test_func_weak),
	85	+ ITEM_FUNC(vmalloc),
	86	+ ITEM_FUNC(vfree),
	87	+#ifdef CONFIG_KALLSYMS_ALL
	88	+ ITEM_DATA(kallsyms_test_var_bss_static),
	89	+ ITEM_DATA(kallsyms_test_var_data_static),
	90	+ ITEM_DATA(kallsyms_test_var_bss),
	91	+ ITEM_DATA(kallsyms_test_var_data),
	92	+ ITEM_DATA(vmap_area_list),
	93	+#endif
	94	+};
	95	+
	96	+static char stub_name[KSYM_NAME_LEN];
	97	+
	98	+static int stat_symbol_len(void data, const char name, struct module *mod, unsigned long addr)
	99	+{
	100	+ (u32 )data += strlen(name);
	101	+
	102	+ return 0;
	103	+}
	104	+
	105	+static void test_kallsyms_compression_ratio(void)
	106	+{
	107	+ u32 pos, off, len, num;
	108	+ u32 ratio, total_size, total_len = 0;
	109	+
	110	+ kallsyms_on_each_symbol(stat_symbol_len, &total_len);
	111	+
	112	+ /*
	113	+ * A symbol name cannot start with a number. This stub name helps us
	114	+ * traverse the entire symbol table without finding a match. It's used
	115	+ * for subsequent performance tests, and its length is the average
	116	+ * length of all symbol names.
	117	+ */
	118	+ memset(stub_name, '4', sizeof(stub_name));
	119	+ pos = total_len / kallsyms_num_syms;
	120	+ stub_name[pos] = 0;
	121	+
	122	+ pos = 0;
	123	+ num = 0;
	124	+ off = 0;
	125	+ while (pos < kallsyms_num_syms) {
	126	+ len = kallsyms_names[off];
	127	+ num++;
	128	+ off++;
	129	+ pos++;
	130	+ if ((len & 0x80) != 0) {
	131	+ len = (len & 0x7f) \| (kallsyms_names[off] << 7);
	132	+ num++;
	133	+ off++;
	134	+ }
	135	+ off += len;
	136	+ }
	137	+
	138	+ /*
	139	+ * 1. The length fields is not counted
	140	+ * 2. The memory occupied by array kallsyms_token_table[] and
	141	+ * kallsyms_token_index[] needs to be counted.
	142	+ */
	143	+ total_size = off - num;
	144	+ pos = kallsyms_token_index[0xff];
	145	+ total_size += pos + strlen(&kallsyms_token_table[pos]) + 1;
	146	+ total_size += 0x100 * sizeof(u16);
	147	+
	148	+ pr_info(" ---------------------------------------------------------\n");
	149	+ pr_info("\| nr_symbols \| compressed size \| original size \| ratio(%%) \|\n");
	150	+ pr_info("\|---------------------------------------------------------\|\n");
	151	+ ratio = (u32)div_u64(10000ULL * total_size, total_len);
	152	+ pr_info("\| %10d \| %10d \| %10d \| %2d.%-2d \|\n",
	153	+ kallsyms_num_syms, total_size, total_len, ratio / 100, ratio % 100);
	154	+ pr_info(" ---------------------------------------------------------\n");
	155	+}
	156	+
	157	+static int lookup_name(void data, const char name, struct module *mod, unsigned long addr)
	158	+{
	159	+ u64 t0, t1, t;
	160	+ unsigned long flags;
	161	+ struct test_stat stat = (struct test_stat )data;
	162	+
	163	+ local_irq_save(flags);
	164	+ t0 = sched_clock();
	165	+ (void)kallsyms_lookup_name(name);
	166	+ t1 = sched_clock();
	167	+ local_irq_restore(flags);
	168	+
	169	+ t = t1 - t0;
	170	+ if (t < stat->min)
	171	+ stat->min = t;
	172	+
	173	+ if (t > stat->max)
	174	+ stat->max = t;
	175	+
	176	+ stat->real_cnt++;
	177	+ stat->sum += t;
	178	+
	179	+ return 0;
	180	+}
	181	+
	182	+static void test_perf_kallsyms_lookup_name(void)
	183	+{
	184	+ struct test_stat stat;
	185	+
	186	+ memset(&stat, 0, sizeof(stat));
	187	+ stat.min = INT_MAX;
	188	+ kallsyms_on_each_symbol(lookup_name, &stat);
	189	+ pr_info("kallsyms_lookup_name() looked up %d symbols\n", stat.real_cnt);
	190	+ pr_info("The time spent on each symbol is (ns): min=%d, max=%d, avg=%lld\n",
	191	+ stat.min, stat.max, div_u64(stat.sum, stat.real_cnt));
	192	+}
	193	+
	194	+static bool match_cleanup_name(const char s, const char name)
	195	+{
	196	+ char *p;
	197	+ int len;
	198	+
	199	+ if (!IS_ENABLED(CONFIG_LTO_CLANG))
	200	+ return false;
	201	+
	202	+ p = strchr(s, '.');
	203	+ if (!p)
	204	+ return false;
	205	+
	206	+ len = strlen(name);
	207	+ if (p - s != len)
	208	+ return false;
	209	+
	210	+ return !strncmp(s, name, len);
	211	+}
	212	+
	213	+static int find_symbol(void data, const char name, struct module *mod, unsigned long addr)
	214	+{
	215	+ struct test_stat stat = (struct test_stat )data;
	216	+
	217	+ if (strcmp(name, stat->name) == 0 \|\|
	218	+ (!stat->perf && match_cleanup_name(name, stat->name))) {
	219	+ stat->real_cnt++;
	220	+ stat->addr = addr;
	221	+
	222	+ if (stat->save_cnt < MAX_NUM_OF_RECORDS) {
	223	+ stat->addrs[stat->save_cnt] = addr;
	224	+ stat->save_cnt++;
	225	+ }
	226	+
	227	+ if (stat->real_cnt == stat->max)
	228	+ return 1;
	229	+ }
	230	+
	231	+ return 0;
	232	+}
	233	+
	234	+static void test_perf_kallsyms_on_each_symbol(void)
	235	+{
	236	+ u64 t0, t1;
	237	+ unsigned long flags;
	238	+ struct test_stat stat;
	239	+
	240	+ memset(&stat, 0, sizeof(stat));
	241	+ stat.max = INT_MAX;
	242	+ stat.name = stub_name;
	243	+ stat.perf = 1;
	244	+ local_irq_save(flags);
	245	+ t0 = sched_clock();
	246	+ kallsyms_on_each_symbol(find_symbol, &stat);
	247	+ t1 = sched_clock();
	248	+ local_irq_restore(flags);
	249	+ pr_info("kallsyms_on_each_symbol() traverse all: %lld ns\n", t1 - t0);
	250	+}
	251	+
	252	+static int match_symbol(void *data, unsigned long addr)
	253	+{
	254	+ struct test_stat stat = (struct test_stat )data;
	255	+
	256	+ stat->real_cnt++;
	257	+ stat->addr = addr;
	258	+
	259	+ if (stat->save_cnt < MAX_NUM_OF_RECORDS) {
	260	+ stat->addrs[stat->save_cnt] = addr;
	261	+ stat->save_cnt++;
	262	+ }
	263	+
	264	+ if (stat->real_cnt == stat->max)
	265	+ return 1;
	266	+
	267	+ return 0;
	268	+}
	269	+
	270	+static void test_perf_kallsyms_on_each_match_symbol(void)
	271	+{
	272	+ u64 t0, t1;
	273	+ unsigned long flags;
	274	+ struct test_stat stat;
	275	+
	276	+ memset(&stat, 0, sizeof(stat));
	277	+ stat.max = INT_MAX;
	278	+ stat.name = stub_name;
	279	+ local_irq_save(flags);
	280	+ t0 = sched_clock();
	281	+ kallsyms_on_each_match_symbol(match_symbol, stat.name, &stat);
	282	+ t1 = sched_clock();
	283	+ local_irq_restore(flags);
	284	+ pr_info("kallsyms_on_each_match_symbol() traverse all: %lld ns\n", t1 - t0);
	285	+}
	286	+
	287	+static int test_kallsyms_basic_function(void)
	288	+{
	289	+ int i, j, ret;
	290	+ int next = 0, nr_failed = 0;
	291	+ char *prefix;
	292	+ unsigned short rand;
	293	+ unsigned long addr, lookup_addr;
	294	+ char namebuf[KSYM_NAME_LEN];
	295	+ struct test_stat stat, stat2;
	296	+
	297	+ stat = kmalloc(sizeof(stat) 2, GFP_KERNEL);
	298	+ if (!stat)
	299	+ return -ENOMEM;
	300	+ stat2 = stat + 1;
	301	+
	302	+ prefix = "kallsyms_lookup_name() for";
	303	+ for (i = 0; i < ARRAY_SIZE(test_items); i++) {
	304	+ addr = kallsyms_lookup_name(test_items[i].name);
	305	+ if (addr != test_items[i].addr) {
	306	+ nr_failed++;
	307	+ pr_info("%s %s failed: addr=%lx, expect %lx\n",
	308	+ prefix, test_items[i].name, addr, test_items[i].addr);
	309	+ }
	310	+ }
	311	+
	312	+ prefix = "kallsyms_on_each_symbol() for";
	313	+ for (i = 0; i < ARRAY_SIZE(test_items); i++) {
	314	+ memset(stat, 0, sizeof(*stat));
	315	+ stat->max = INT_MAX;
	316	+ stat->name = test_items[i].name;
	317	+ kallsyms_on_each_symbol(find_symbol, stat);
	318	+ if (stat->addr != test_items[i].addr \|\| stat->real_cnt != 1) {
	319	+ nr_failed++;
	320	+ pr_info("%s %s failed: count=%d, addr=%lx, expect %lx\n",
	321	+ prefix, test_items[i].name,
	322	+ stat->real_cnt, stat->addr, test_items[i].addr);
	323	+ }
	324	+ }
	325	+
	326	+ prefix = "kallsyms_on_each_match_symbol() for";
	327	+ for (i = 0; i < ARRAY_SIZE(test_items); i++) {
	328	+ memset(stat, 0, sizeof(*stat));
	329	+ stat->max = INT_MAX;
	330	+ stat->name = test_items[i].name;
	331	+ kallsyms_on_each_match_symbol(match_symbol, test_items[i].name, stat);
	332	+ if (stat->addr != test_items[i].addr \|\| stat->real_cnt != 1) {
	333	+ nr_failed++;
	334	+ pr_info("%s %s failed: count=%d, addr=%lx, expect %lx\n",
	335	+ prefix, test_items[i].name,
	336	+ stat->real_cnt, stat->addr, test_items[i].addr);
	337	+ }
	338	+ }
	339	+
	340	+ if (nr_failed) {
	341	+ kfree(stat);
	342	+ return -ESRCH;
	343	+ }
	344	+
	345	+ for (i = 0; i < kallsyms_num_syms; i++) {
	346	+ addr = kallsyms_sym_address(i);
	347	+ if (!is_ksym_addr(addr))
	348	+ continue;
	349	+
	350	+ ret = lookup_symbol_name(addr, namebuf);
	351	+ if (unlikely(ret)) {
	352	+ namebuf[0] = 0;
	353	+ goto failed;
	354	+ }
	355	+
	356	+ /*
	357	+ * The first '.' may be the initial letter, in which case the
	358	+ * entire symbol name will be truncated to an empty string in
	359	+ * cleanup_symbol_name(). Do not test these symbols.
	360	+ *
	361	+ * For example:
	362	+ * cat /proc/kallsyms \| awk '{print $3}' \| grep -E "^\." \| head
	363	+ * .E_read_words
	364	+ * .E_leading_bytes
	365	+ * .E_trailing_bytes
	366	+ * .E_write_words
	367	+ * .E_copy
	368	+ * .str.292.llvm.12122243386960820698
	369	+ * .str.24.llvm.12122243386960820698
	370	+ * .str.29.llvm.12122243386960820698
	371	+ * .str.75.llvm.12122243386960820698
	372	+ * .str.99.llvm.12122243386960820698
	373	+ */
	374	+ if (IS_ENABLED(CONFIG_LTO_CLANG) && !namebuf[0])
	375	+ continue;
	376	+
	377	+ lookup_addr = kallsyms_lookup_name(namebuf);
	378	+
	379	+ memset(stat, 0, sizeof(*stat));
	380	+ stat->max = INT_MAX;
	381	+ kallsyms_on_each_match_symbol(match_symbol, namebuf, stat);
	382	+
	383	+ /*
	384	+ * kallsyms_on_each_symbol() is too slow, randomly select some
	385	+ * symbols for test.
	386	+ */
	387	+ if (i >= next) {
	388	+ memset(stat2, 0, sizeof(*stat2));
	389	+ stat2->max = INT_MAX;
	390	+ stat2->name = namebuf;
	391	+ kallsyms_on_each_symbol(find_symbol, stat2);
	392	+
	393	+ /*
	394	+ * kallsyms_on_each_symbol() and kallsyms_on_each_match_symbol()
	395	+ * need to get the same traversal result.
	396	+ */
	397	+ if (stat->addr != stat2->addr \|\|
	398	+ stat->real_cnt != stat2->real_cnt \|\|
	399	+ memcmp(stat->addrs, stat2->addrs,
	400	+ stat->save_cnt * sizeof(stat->addrs[0])))
	401	+ goto failed;
	402	+
	403	+ /*
	404	+ * The average of random increments is 128, that is, one of
	405	+ * them is tested every 128 symbols.
	406	+ */
	407	+ get_random_bytes(&rand, sizeof(rand));
	408	+ next = i + (rand & 0xff) + 1;
	409	+ }
	410	+
	411	+ /* Need to be found at least once */
	412	+ if (!stat->real_cnt)
	413	+ goto failed;
	414	+
	415	+ /*
	416	+ * kallsyms_lookup_name() returns the address of the first
	417	+ * symbol found and cannot be NULL.
	418	+ */
	419	+ if (!lookup_addr \|\| lookup_addr != stat->addrs[0])
	420	+ goto failed;
	421	+
	422	+ /*
	423	+ * If the addresses of all matching symbols are recorded, the
	424	+ * target address needs to be exist.
	425	+ */
	426	+ if (stat->real_cnt <= MAX_NUM_OF_RECORDS) {
	427	+ for (j = 0; j < stat->save_cnt; j++) {
	428	+ if (stat->addrs[j] == addr)
	429	+ break;
	430	+ }
	431	+
	432	+ if (j == stat->save_cnt)
	433	+ goto failed;
	434	+ }
	435	+ }
	436	+
	437	+ kfree(stat);
	438	+
	439	+ return 0;
	440	+
	441	+failed:
	442	+ pr_info("Test for %dth symbol failed: (%s) addr=%lx", i, namebuf, addr);
	443	+ kfree(stat);
	444	+ return -ESRCH;
	445	+}
	446	+
	447	+static int test_entry(void *p)
	448	+{
	449	+ int ret;
	450	+
	451	+ do {
	452	+ schedule_timeout(5 * HZ);
	453	+ } while (system_state != SYSTEM_RUNNING);
	454	+
	455	+ pr_info("start\n");
	456	+ ret = test_kallsyms_basic_function();
	457	+ if (ret) {
	458	+ pr_info("abort\n");
	459	+ return 0;
	460	+ }
	461	+
	462	+ test_kallsyms_compression_ratio();
	463	+ test_perf_kallsyms_lookup_name();
	464	+ test_perf_kallsyms_on_each_symbol();
	465	+ test_perf_kallsyms_on_each_match_symbol();
	466	+ pr_info("finish\n");
	467	+
	468	+ return 0;
	469	+}
	470	+
	471	+static int __init kallsyms_test_init(void)
	472	+{
	473	+ struct task_struct *t;
	474	+
	475	+ t = kthread_create(test_entry, NULL, "kallsyms_test");
	476	+ if (IS_ERR(t)) {
	477	+ pr_info("Create kallsyms selftest task failed\n");
	478	+ return PTR_ERR(t);
	479	+ }
	480	+ kthread_bind(t, 0);
	481	+ wake_up_process(t);
	482	+
	483	+ return 0;
	484	+}
	485	+late_initcall(kallsyms_test_init);

kernel/kallsyms_selftest.h+13−0

@@ -0,0 +1,13 @@
	1	+/* SPDX-License-Identifier: GPL-2.0-or-later */
	2	+#ifndef LINUX_KALLSYMS_SELFTEST_H_
	3	+#define LINUX_KALLSYMS_SELFTEST_H_
	4	+
	5	+#include <linux/types.h>
	6	+
	7	+extern int kallsyms_test_var_bss;
	8	+extern int kallsyms_test_var_data;
	9	+
	10	+extern int kallsyms_test_func(void);
	11	+extern int kallsyms_test_func_weak(void);
	12	+
	13	+#endif // LINUX_KALLSYMS_SELFTEST_H_

kernel/livepatch/core.c+19−12

@@ -125,20 +125,10 @@ struct klp_find_arg { · 124 unmodified lines
125	125	unsigned long pos;
126	126	};
127	127
128		−static int klp_find_callback(void data, const char name,
129		− struct module *mod, unsigned long addr)
	128	+static int klp_match_callback(void *data, unsigned long addr)
130	129	{
131	130	struct klp_find_arg *args = data;
132	131
133		− if ((mod && !args->objname) \|\| (!mod && args->objname))
134		− return 0;
135		−
136		− if (strcmp(args->name, name))
137		− return 0;
138		−
139		− if (args->objname && strcmp(args->objname, mod->name))
140		− return 0;
141		−
142	132	args->addr = addr;
143	133	args->count++;
144	134
@@ -153,6 +143,23 @@ static int klp_find_callback(void data, const char name, · 8 unmodified lines
153	143	return 0;
154	144	}
155	145
	146	+static int klp_find_callback(void data, const char name,
	147	+ struct module *mod, unsigned long addr)
	148	+{
	149	+ struct klp_find_arg *args = data;
	150	+
	151	+ if ((mod && !args->objname) \|\| (!mod && args->objname))
	152	+ return 0;
	153	+
	154	+ if (strcmp(args->name, name))
	155	+ return 0;
	156	+
	157	+ if (args->objname && strcmp(args->objname, mod->name))
	158	+ return 0;
	159	+
	160	+ return klp_match_callback(data, addr);
	161	+}
	162	+
156	163	static int klp_find_object_symbol(const char objname, const char name,
157	164	unsigned long sympos, unsigned long *addr)
158	165	{
@@ -167,7 +174,7 @@ static int klp_find_object_symbol(const char objname, const char name, · 8 unmodified lines
167	174	if (objname)
168	175	module_kallsyms_on_each_symbol(klp_find_callback, &args);
169	176	else
170		− kallsyms_on_each_symbol(klp_find_callback, &args);
	177	+ kallsyms_on_each_match_symbol(klp_match_callback, name, &args);
171	178
172	179	/*
173	180	* Ensure an address was found. If sympos is 0, ensure symbol is unique;

kernel/module/Kconfig+2−1

@@ -221,9 +221,10 @@ endchoice · 220 unmodified lines
221	221
222	222	config MODULE_DECOMPRESS
223	223	bool "Support in-kernel module decompression"
224		− depends on MODULE_COMPRESS_GZIP \|\| MODULE_COMPRESS_XZ
	224	+ depends on MODULE_COMPRESS_GZIP \|\| MODULE_COMPRESS_XZ \|\| MODULE_COMPRESS_ZSTD
225	225	select ZLIB_INFLATE if MODULE_COMPRESS_GZIP
226	226	select XZ_DEC if MODULE_COMPRESS_XZ
	227	+ select ZSTD_DECOMPRESS if MODULE_COMPRESS_ZSTD
227	228	help
228	229
229	230	Support for decompressing kernel modules by the kernel itself

kernel/module/decompress.c+94−6

@@ -50,7 +50,7 @@ static struct page module_get_next_page(struct load_info info) · 49 unmodified lines
50	50	return page;
51	51	}
52	52
53		−#ifdef CONFIG_MODULE_COMPRESS_GZIP
	53	+#if defined(CONFIG_MODULE_COMPRESS_GZIP)
54	54	#include <linux/zlib.h>
55	55	#define MODULE_COMPRESSION gzip
56	56	#define MODULE_DECOMPRESS_FN module_gzip_decompress
@@ -114,8 +114,8 @@ static ssize_t module_gzip_decompress(struct load_info *info, · 57 unmodified lines
114	114	do {
115	115	struct page *page = module_get_next_page(info);
116	116
117		− if (!page) {
118		− retval = -ENOMEM;
	117	+ if (IS_ERR(page)) {
	118	+ retval = PTR_ERR(page);
119	119	goto out_inflate_end;
120	120	}
121	121
@@ -141,7 +141,7 @@ out: · 19 unmodified lines
141	141	kfree(s.workspace);
142	142	return retval;
143	143	}
144		−#elif CONFIG_MODULE_COMPRESS_XZ
	144	+#elif defined(CONFIG_MODULE_COMPRESS_XZ)
145	145	#include <linux/xz.h>
146	146	#define MODULE_COMPRESSION xz
147	147	#define MODULE_DECOMPRESS_FN module_xz_decompress
@@ -173,8 +173,8 @@ static ssize_t module_xz_decompress(struct load_info *info, · 25 unmodified lines
173	173	do {
174	174	struct page *page = module_get_next_page(info);
175	175
176		− if (!page) {
177		− retval = -ENOMEM;
	176	+ if (IS_ERR(page)) {
	177	+ retval = PTR_ERR(page);
178	178	goto out;
179	179	}
180	180
@@ -199,6 +199,94 @@ static ssize_t module_xz_decompress(struct load_info *info, · 18 unmodified lines
199	199	xz_dec_end(xz_dec);
200	200	return retval;
201	201	}
	202	+#elif defined(CONFIG_MODULE_COMPRESS_ZSTD)
	203	+#include <linux/zstd.h>
	204	+#define MODULE_COMPRESSION zstd
	205	+#define MODULE_DECOMPRESS_FN module_zstd_decompress
	206	+
	207	+static ssize_t module_zstd_decompress(struct load_info *info,
	208	+ const void *buf, size_t size)
	209	+{
	210	+ static const u8 signature[] = { 0x28, 0xb5, 0x2f, 0xfd };
	211	+ ZSTD_outBuffer zstd_dec;
	212	+ ZSTD_inBuffer zstd_buf;
	213	+ zstd_frame_header header;
	214	+ size_t wksp_size;
	215	+ void *wksp = NULL;
	216	+ ZSTD_DStream *dstream;
	217	+ size_t ret;
	218	+ size_t new_size = 0;
	219	+ int retval;
	220	+
	221	+ if (size < sizeof(signature) \|\|
	222	+ memcmp(buf, signature, sizeof(signature))) {
	223	+ pr_err("not a zstd compressed module\n");
	224	+ return -EINVAL;
	225	+ }
	226	+
	227	+ zstd_buf.src = buf;
	228	+ zstd_buf.pos = 0;
	229	+ zstd_buf.size = size;
	230	+
	231	+ ret = zstd_get_frame_header(&header, zstd_buf.src, zstd_buf.size);
	232	+ if (ret != 0) {
	233	+ pr_err("ZSTD-compressed data has an incomplete frame header\n");
	234	+ retval = -EINVAL;
	235	+ goto out;
	236	+ }
	237	+ if (header.windowSize > (1 << ZSTD_WINDOWLOG_MAX)) {
	238	+ pr_err("ZSTD-compressed data has too large a window size\n");
	239	+ retval = -EINVAL;
	240	+ goto out;
	241	+ }
	242	+
	243	+ wksp_size = zstd_dstream_workspace_bound(header.windowSize);
	244	+ wksp = kmalloc(wksp_size, GFP_KERNEL);
	245	+ if (!wksp) {
	246	+ retval = -ENOMEM;
	247	+ goto out;
	248	+ }
	249	+
	250	+ dstream = zstd_init_dstream(header.windowSize, wksp, wksp_size);
	251	+ if (!dstream) {
	252	+ pr_err("Can't initialize ZSTD stream\n");
	253	+ retval = -ENOMEM;
	254	+ goto out;
	255	+ }
	256	+
	257	+ do {
	258	+ struct page *page = module_get_next_page(info);
	259	+
	260	+ if (!IS_ERR(page)) {
	261	+ retval = PTR_ERR(page);
	262	+ goto out;
	263	+ }
	264	+
	265	+ zstd_dec.dst = kmap_local_page(page);
	266	+ zstd_dec.pos = 0;
	267	+ zstd_dec.size = PAGE_SIZE;
	268	+
	269	+ ret = zstd_decompress_stream(dstream, &zstd_dec, &zstd_buf);
	270	+ kunmap(page);
	271	+ retval = zstd_get_error_code(ret);
	272	+ if (retval)
	273	+ break;
	274	+
	275	+ new_size += zstd_dec.pos;
	276	+ } while (zstd_dec.pos == PAGE_SIZE && ret != 0);
	277	+
	278	+ if (retval) {
	279	+ pr_err("ZSTD-decompression failed with status %d\n", retval);
	280	+ retval = -EINVAL;
	281	+ goto out;
	282	+ }
	283	+
	284	+ retval = new_size;
	285	+
	286	+ out:
	287	+ kfree(wksp);
	288	+ return retval;
	289	+}
202	290	#else
203	291	#error "Unexpected configuration for CONFIG_MODULE_DECOMPRESS"
204	292	#endif

kernel/module/main.c+0−3

@@ -85,9 +85,6 @@ struct mod_tree_root mod_data_tree __cacheline_aligned = { · 84 unmodified lines
85	85	};
86	86	#endif
87	87
88		−#define module_addr_min mod_tree.addr_min
89		−#define module_addr_max mod_tree.addr_max
90		−
91	88	struct symsearch {
92	89	const struct kernel_symbol start, stop;
93	90	const s32 *crcs;

kernel/module/sysfs.c+1−1

@@ -340,7 +340,7 @@ static int mod_sysfs_init(struct module *mod) · 339 unmodified lines
340	340	int err;
341	341	struct kobject *kobj;
342	342
343		− if (!module_sysfs_initialized) {
	343	+ if (!module_kset) {
344	344	pr_err("%s: module sysfs not initialized\n", mod->name);
345	345	err = -EINVAL;
346	346	goto out;

kernel/params.c+19−4

@@ -940,7 +940,6 @@ static const struct kset_uevent_ops module_uevent_ops = { · 939 unmodified lines
940	940	};
941	941
942	942	struct kset *module_kset;
943		−int module_sysfs_initialized;
944	943
945	944	static void module_kobj_release(struct kobject *kobj)
946	945	{
@@ -954,7 +953,11 @@ struct kobj_type module_ktype = { · 7 unmodified lines
954	953	};
955	954
956	955	/*
957		− * param_sysfs_init - wrapper for built-in params support
	956	+ * param_sysfs_init - create "module" kset
	957	+ *
	958	+ * This must be done before the initramfs is unpacked and
	959	+ * request_module() thus becomes possible, because otherwise the
	960	+ * module load would fail in mod_sysfs_init.
958	961	*/
959	962	static int __init param_sysfs_init(void)
960	963	{
@@ -964,13 +967,25 @@ static int __init param_sysfs_init(void) · 3 unmodified lines
964	967	__FILE__, __LINE__);
965	968	return -ENOMEM;
966	969	}
967		− module_sysfs_initialized = 1;
	970	+
	971	+ return 0;
	972	+}
	973	+subsys_initcall(param_sysfs_init);
	974	+
	975	+/*
	976	+ * param_sysfs_builtin_init - add sysfs version and parameter
	977	+ * attributes for built-in modules
	978	+ */
	979	+static int __init param_sysfs_builtin_init(void)
	980	+{
	981	+ if (!module_kset)
	982	+ return -ENOMEM;
968	983
969	984	version_sysfs_builtin();
970	985	param_sysfs_builtin();
971	986
972	987	return 0;
973	988	}
974		−subsys_initcall(param_sysfs_init);
	989	+late_initcall(param_sysfs_builtin_init);
975	990
976	991	#endif /* CONFIG_SYSFS */

scripts/kallsyms.c+75−3

@@ -49,6 +49,7 @@ _Static_assert( · 48 unmodified lines
49	49	struct sym_entry {
50	50	unsigned long long addr;
51	51	unsigned int len;
	52	+ unsigned int seq;
52	53	unsigned int start_pos;
53	54	unsigned int percpu_absolute;
54	55	unsigned char sym[];
@@ -77,6 +78,7 @@ static unsigned int table_size, table_cnt; · 22 unmodified lines
77	78	static int all_symbols;
78	79	static int absolute_percpu;
79	80	static int base_relative;
	81	+static int lto_clang;
80	82
81	83	static int token_profit[0x10000];
82	84
@@ -88,7 +90,7 @@ static unsigned char best_table_len[256]; · 5 unmodified lines
88	90	static void usage(void)
89	91	{
90	92	fprintf(stderr, "Usage: kallsyms [--all-symbols] [--absolute-percpu] "
91		− "[--base-relative] in.map > out.S\n");
	93	+ "[--base-relative] [--lto-clang] in.map > out.S\n");
92	94	exit(1);
93	95	}
94	96
@@ -410,6 +412,65 @@ static int symbol_absolute(const struct sym_entry *s) · 315 unmodified lines
410	412	return s->percpu_absolute;
411	413	}
412	414
	415	+static char * s_name(char *buf)
	416	+{
	417	+ /* Skip the symbol type */
	418	+ return buf + 1;
	419	+}
	420	+
	421	+static void cleanup_symbol_name(char *s)
	422	+{
	423	+ char *p;
	424	+
	425	+ if (!lto_clang)
	426	+ return;
	427	+
	428	+ /*
	429	+ * ASCII[.] = 2e
	430	+ * ASCII[0-9] = 30,39
	431	+ * ASCII[A-Z] = 41,5a
	432	+ * ASCII[_] = 5f
	433	+ * ASCII[a-z] = 61,7a
	434	+ *
	435	+ * As above, replacing '.' with '\0' does not affect the main sorting,
	436	+ * but it helps us with subsorting.
	437	+ */
	438	+ p = strchr(s, '.');
	439	+ if (p)
	440	+ *p = '\0';
	441	+}
	442	+
	443	+static int compare_names(const void a, const void b)
	444	+{
	445	+ int ret;
	446	+ char sa_namebuf[KSYM_NAME_LEN];
	447	+ char sb_namebuf[KSYM_NAME_LEN];
	448	+ const struct sym_entry sa = (const struct sym_entry **)a;
	449	+ const struct sym_entry sb = (const struct sym_entry **)b;
	450	+
	451	+ expand_symbol(sa->sym, sa->len, sa_namebuf);
	452	+ expand_symbol(sb->sym, sb->len, sb_namebuf);
	453	+ cleanup_symbol_name(s_name(sa_namebuf));
	454	+ cleanup_symbol_name(s_name(sb_namebuf));
	455	+ ret = strcmp(s_name(sa_namebuf), s_name(sb_namebuf));
	456	+ if (!ret) {
	457	+ if (sa->addr > sb->addr)
	458	+ return 1;
	459	+ else if (sa->addr < sb->addr)
	460	+ return -1;
	461	+
	462	+ /* keep old order */
	463	+ return (int)(sa->seq - sb->seq);
	464	+ }
	465	+
	466	+ return ret;
	467	+}
	468	+
	469	+static void sort_symbols_by_name(void)
	470	+{
	471	+ qsort(table, table_cnt, sizeof(table[0]), compare_names);
	472	+}
	473	+
413	474	static void write_src(void)
414	475	{
415	476	unsigned int i, k, off;
@@ -495,6 +556,7 @@ static void write_src(void) · 79 unmodified lines
495	556	for (i = 0; i < table_cnt; i++) {
496	557	if ((i & 0xFF) == 0)
497	558	markers[i >> 8] = off;
	559	+ table[i]->seq = i;
498	560
499	561	/* There cannot be any symbol of length zero. */
500	562	if (table[i]->len == 0) {
@@ -535,6 +597,15 @@ static void write_src(void) · 34 unmodified lines
535	597
536	598	free(markers);
537	599
	600	+ sort_symbols_by_name();
	601	+ output_label("kallsyms_seqs_of_names");
	602	+ for (i = 0; i < table_cnt; i++)
	603	+ printf("\t.byte 0x%02x, 0x%02x, 0x%02x\n",
	604	+ (unsigned char)(table[i]->seq >> 16),
	605	+ (unsigned char)(table[i]->seq >> 8),
	606	+ (unsigned char)(table[i]->seq >> 0));
	607	+ printf("\n");
	608	+
538	609	output_label("kallsyms_token_table");
539	610	off = 0;
540	611	for (i = 0; i < 256; i++) {
@@ -573,7 +644,7 @@ static void forget_symbol(const unsigned char *symbol, int len) · 32 unmodified lines
573	644	}
574	645
575	646	/* do the initial token count */
576		−static void build_initial_tok_table(void)
	647	+static void build_initial_token_table(void)
577	648	{
578	649	unsigned int i;
579	650
@@ -698,7 +769,7 @@ static void insert_real_symbols_in_table(void) · 118 unmodified lines
698	769
699	770	static void optimize_token_table(void)
700	771	{
701		− build_initial_tok_table();
	772	+ build_initial_token_table();
702	773
703	774	insert_real_symbols_in_table();
704	775
@@ -818,6 +889,7 @@ int main(int argc, char **argv) · 113 unmodified lines
818	889	{"all-symbols", no_argument, &all_symbols, 1},
819	890	{"absolute-percpu", no_argument, &absolute_percpu, 1},
820	891	{"base-relative", no_argument, &base_relative, 1},
	892	+ {"lto-clang", no_argument, &lto_clang, 1},
821	893	{},
822	894	};
823	895

scripts/link-vmlinux.sh+4−0

@@ -156,6 +156,10 @@ kallsyms() · 155 unmodified lines
156	156	kallsymopt="${kallsymopt} --base-relative"
157	157	fi
158	158
	159	+ if is_enabled CONFIG_LTO_CLANG; then
	160	+ kallsymopt="${kallsymopt} --lto-clang"
	161	+ fi
	162	+
159	163	info KSYMS ${2}
160	164	scripts/kallsyms ${kallsymopt} ${1} > ${2}
161	165	}