History

Merge tag 'modules-6.2-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/mcgrof/linux

Pull modules updates from Luis Chamberlain: "Tux gets for xmas an improvement to the average lookup performance of kallsyms_lookup_name() by 715x thanks to the work by Zhen Lei, which upgraded our old implementation from being O(n) to O(log(n)), while also retaining the old implementation support on /proc/kallsyms. The only penalty was increasing the memory footprint by 3 * kallsyms_num_syms. Folks who want to improve this further now also have a dedicated selftest facility through KALLSYMS_SELFTEST. Stephen Boyd added zstd in-kernel decompression support, but the only users of this would be folks using the load-pin LSM because otherwise we do module decompression in userspace. The only other thing with mentioning is a minor boot time optimization by Rasmus Villemoes which deferes param_sysfs_init() to late init. The rest is cleanups and minor fixes" * tag 'modules-6.2-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/mcgrof/linux: livepatch: Call klp_match_callback() in klp_find_callback() to avoid code duplication module/decompress: Support zstd in-kernel decompression kallsyms: Remove unneeded semicolon kallsyms: Add self-test facility livepatch: Use kallsyms_on_each_match_symbol() to improve performance kallsyms: Add helper kallsyms_on_each_match_symbol() kallsyms: Reduce the memory occupied by kallsyms_seqs_of_names[] kallsyms: Correctly sequence symbols when CONFIG_LTO_CLANG=y kallsyms: Improve the performance of kallsyms_lookup_name() scripts/kallsyms: rename build_initial_tok_table() module: Fix NULL vs IS_ERR checking for module_get_next_page kernel/params.c: defer most of param_sysfs_init() to late_initcall time module: Remove unused macros module_addr_min/max module: remove redundant module_sysfs_initialized variable

Linus Torvalds committed 3 years ago

Commit 3ba2c3f2 parents 0015edd 4f1354d16 files · +840 −43Browse files

Unified · Split

16 files
  1. include/linux/kallsyms.h +9 −0
  2. include/linux/module.h +0 −1
  3. init/Kconfig +13 −0
  4. kernel/Makefile +1 −0
  5. kernel/kallsyms.c +104 −12
  6. kernel/kallsyms_internal.h +1 −0
  7. kernel/kallsyms_selftest.c +485 −0
  8. kernel/kallsyms_selftest.h +13 −0
  9. kernel/livepatch/core.c +19 −12
  10. kernel/module/Kconfig +2 −1
  11. kernel/module/decompress.c +94 −6
  12. kernel/module/main.c +0 −3
  13. kernel/module/sysfs.c +1 −1
  14. kernel/params.c +19 −4
  15. scripts/kallsyms.c +75 −3
  16. scripts/link-vmlinux.sh +4 −0
include/linux/kallsyms.h+9−0
@@ -66,9 +66,12 @@ static inline void *dereference_symbol_descriptor(void *ptr) · 65 unmodified lines
66}66}
67 67
68#ifdef CONFIG_KALLSYMS68#ifdef CONFIG_KALLSYMS
69unsigned long kallsyms_sym_address(int idx);
69int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *,70int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *,
70 unsigned long),71 unsigned long),
71 void *data);72 void *data);
73int kallsyms_on_each_match_symbol(int (*fn)(void *, unsigned long),
74 const char *name, void *data);
72 75
73/* Lookup the address for a symbol. Returns 0 if not found. */76/* Lookup the address for a symbol. Returns 0 if not found. */
74unsigned long kallsyms_lookup_name(const char *name);77unsigned long kallsyms_lookup_name(const char *name);
@@ -168,6 +171,12 @@ static inline int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct · 93 unmodified lines
168{171{
169 return -EOPNOTSUPP;172 return -EOPNOTSUPP;
170}173}
174
175static inline int kallsyms_on_each_match_symbol(int (*fn)(void *, unsigned long),
176 const char *name, void *data)
177{
178 return -EOPNOTSUPP;
179}
171#endif /*CONFIG_KALLSYMS*/180#endif /*CONFIG_KALLSYMS*/
172 181
173static inline void print_ip_sym(const char *loglvl, unsigned long ip)182static inline void print_ip_sym(const char *loglvl, unsigned long ip)
include/linux/module.h+0−1
@@ -827,7 +827,6 @@ void *dereference_module_function_descriptor(struct module *mod, void *ptr) · 826 unmodified lines
827#ifdef CONFIG_SYSFS827#ifdef CONFIG_SYSFS
828extern struct kset *module_kset;828extern struct kset *module_kset;
829extern struct kobj_type module_ktype;829extern struct kobj_type module_ktype;
830extern int module_sysfs_initialized;
831#endif /* CONFIG_SYSFS */830#endif /* CONFIG_SYSFS */
832 831
833#define symbol_request(x) try_then_request_module(symbol_get(x), "symbol:" #x)832#define symbol_request(x) try_then_request_module(symbol_get(x), "symbol:" #x)
init/Kconfig+13−0
@@ -1723,6 +1723,19 @@ config KALLSYMS · 1722 unmodified lines
1723 symbolic stack backtraces. This increases the size of the kernel1723 symbolic stack backtraces. This increases the size of the kernel
1724 somewhat, as all symbols have to be loaded into the kernel image.1724 somewhat, as all symbols have to be loaded into the kernel image.
1725 1725
1726config KALLSYMS_SELFTEST
1727 bool "Test the basic functions and performance of kallsyms"
1728 depends on KALLSYMS
1729 default n
1730 help
1731 Test the basic functions and performance of some interfaces, such as
1732 kallsyms_lookup_name. It also calculates the compression rate of the
1733 kallsyms compression algorithm for the current symbol set.
1734
1735 Start self-test automatically after system startup. Suggest executing
1736 "dmesg | grep kallsyms_selftest" to collect test results. "finish" is
1737 displayed in the last line, indicating that the test is complete.
1738
1726config KALLSYMS_ALL1739config KALLSYMS_ALL
1727 bool "Include all symbols in kallsyms"1740 bool "Include all symbols in kallsyms"
1728 depends on DEBUG_KERNEL && KALLSYMS1741 depends on DEBUG_KERNEL && KALLSYMS
kernel/Makefile+1−0
@@ -69,6 +69,7 @@ endif · 68 unmodified lines
69obj-$(CONFIG_UID16) += uid16.o69obj-$(CONFIG_UID16) += uid16.o
70obj-$(CONFIG_MODULE_SIG_FORMAT) += module_signature.o70obj-$(CONFIG_MODULE_SIG_FORMAT) += module_signature.o
71obj-$(CONFIG_KALLSYMS) += kallsyms.o71obj-$(CONFIG_KALLSYMS) += kallsyms.o
72obj-$(CONFIG_KALLSYMS_SELFTEST) += kallsyms_selftest.o
72obj-$(CONFIG_BSD_PROCESS_ACCT) += acct.o73obj-$(CONFIG_BSD_PROCESS_ACCT) += acct.o
73obj-$(CONFIG_CRASH_CORE) += crash_core.o74obj-$(CONFIG_CRASH_CORE) += crash_core.o
74obj-$(CONFIG_KEXEC_CORE) += kexec_core.o75obj-$(CONFIG_KEXEC_CORE) += kexec_core.o
kernel/kallsyms.c+104−12
@@ -146,7 +146,7 @@ static unsigned int get_symbol_offset(unsigned long pos) · 145 unmodified lines
146 return name - kallsyms_names;146 return name - kallsyms_names;
147}147}
148 148
149static unsigned long kallsyms_sym_address(int idx)149unsigned long kallsyms_sym_address(int idx)
150{150{
151 if (!IS_ENABLED(CONFIG_KALLSYMS_BASE_RELATIVE))151 if (!IS_ENABLED(CONFIG_KALLSYMS_BASE_RELATIVE))
152 return kallsyms_addresses[idx];152 return kallsyms_addresses[idx];
@@ -187,26 +187,100 @@ static bool cleanup_symbol_name(char *s) · 34 unmodified lines
187 return false;187 return false;
188}188}
189 189
190static int compare_symbol_name(const char *name, char *namebuf)
191{
192 int ret;
193
194 ret = strcmp(name, namebuf);
195 if (!ret)
196 return ret;
197
198 if (cleanup_symbol_name(namebuf) && !strcmp(name, namebuf))
199 return 0;
200
201 return ret;
202}
203
204static unsigned int get_symbol_seq(int index)
205{
206 unsigned int i, seq = 0;
207
208 for (i = 0; i < 3; i++)
209 seq = (seq << 8) | kallsyms_seqs_of_names[3 * index + i];
210
211 return seq;
212}
213
214static int kallsyms_lookup_names(const char *name,
215 unsigned int *start,
216 unsigned int *end)
217{
218 int ret;
219 int low, mid, high;
220 unsigned int seq, off;
221 char namebuf[KSYM_NAME_LEN];
222
223 low = 0;
224 high = kallsyms_num_syms - 1;
225
226 while (low <= high) {
227 mid = low + (high - low) / 2;
228 seq = get_symbol_seq(mid);
229 off = get_symbol_offset(seq);
230 kallsyms_expand_symbol(off, namebuf, ARRAY_SIZE(namebuf));
231 ret = compare_symbol_name(name, namebuf);
232 if (ret > 0)
233 low = mid + 1;
234 else if (ret < 0)
235 high = mid - 1;
236 else
237 break;
238 }
239
240 if (low > high)
241 return -ESRCH;
242
243 low = mid;
244 while (low) {
245 seq = get_symbol_seq(low - 1);
246 off = get_symbol_offset(seq);
247 kallsyms_expand_symbol(off, namebuf, ARRAY_SIZE(namebuf));
248 if (compare_symbol_name(name, namebuf))
249 break;
250 low--;
251 }
252 *start = low;
253
254 if (end) {
255 high = mid;
256 while (high < kallsyms_num_syms - 1) {
257 seq = get_symbol_seq(high + 1);
258 off = get_symbol_offset(seq);
259 kallsyms_expand_symbol(off, namebuf, ARRAY_SIZE(namebuf));
260 if (compare_symbol_name(name, namebuf))
261 break;
262 high++;
263 }
264 *end = high;
265 }
266
267 return 0;
268}
269
190/* Lookup the address for this symbol. Returns 0 if not found. */270/* Lookup the address for this symbol. Returns 0 if not found. */
191unsigned long kallsyms_lookup_name(const char *name)271unsigned long kallsyms_lookup_name(const char *name)
192{272{
193 char namebuf[KSYM_NAME_LEN];273 int ret;
194 unsigned long i;274 unsigned int i;
195 unsigned int off;
196 275
197 /* Skip the search for empty string. */276 /* Skip the search for empty string. */
198 if (!*name)277 if (!*name)
199 return 0;278 return 0;
200 279
201 for (i = 0, off = 0; i < kallsyms_num_syms; i++) {280 ret = kallsyms_lookup_names(name, &i, NULL);
202 off = kallsyms_expand_symbol(off, namebuf, ARRAY_SIZE(namebuf));281 if (!ret)
203 282 return kallsyms_sym_address(get_symbol_seq(i));
204 if (strcmp(namebuf, name) == 0)
205 return kallsyms_sym_address(i);
206 283
207 if (cleanup_symbol_name(namebuf) && strcmp(namebuf, name) == 0)
208 return kallsyms_sym_address(i);
209 }
210 return module_kallsyms_lookup_name(name);284 return module_kallsyms_lookup_name(name);
211}285}
212 286
@@ -233,6 +307,24 @@ int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, · 20 unmodified lines
233 return 0;307 return 0;
234}308}
235 309
310int kallsyms_on_each_match_symbol(int (*fn)(void *, unsigned long),
311 const char *name, void *data)
312{
313 int ret;
314 unsigned int i, start, end;
315
316 ret = kallsyms_lookup_names(name, &start, &end);
317 if (ret)
318 return 0;
319
320 for (i = start; !ret && i <= end; i++) {
321 ret = fn(data, kallsyms_sym_address(get_symbol_seq(i)));
322 cond_resched();
323 }
324
325 return ret;
326}
327
236static unsigned long get_symbol_pos(unsigned long addr,328static unsigned long get_symbol_pos(unsigned long addr,
237 unsigned long *symbolsize,329 unsigned long *symbolsize,
238 unsigned long *offset)330 unsigned long *offset)
kernel/kallsyms_internal.h+1−0
@@ -26,5 +26,6 @@ extern const char kallsyms_token_table[] __weak; · 25 unmodified lines
26extern const u16 kallsyms_token_index[] __weak;26extern const u16 kallsyms_token_index[] __weak;
27 27
28extern const unsigned int kallsyms_markers[] __weak;28extern const unsigned int kallsyms_markers[] __weak;
29extern const u8 kallsyms_seqs_of_names[] __weak;
29 30
30#endif // LINUX_KALLSYMS_INTERNAL_H_31#endif // LINUX_KALLSYMS_INTERNAL_H_
kernel/kallsyms_selftest.c+485−0
@@ -0,0 +1,485 @@
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Test the function and performance of kallsyms
4 *
5 * Copyright (C) Huawei Technologies Co., Ltd., 2022
6 *
7 * Authors: Zhen Lei <thunder.leizhen@huawei.com> Huawei
8 */
9
10#define pr_fmt(fmt) "kallsyms_selftest: " fmt
11
12#include <linux/init.h>
13#include <linux/module.h>
14#include <linux/kallsyms.h>
15#include <linux/random.h>
16#include <linux/sched/clock.h>
17#include <linux/kthread.h>
18#include <linux/vmalloc.h>
19
20#include "kallsyms_internal.h"
21#include "kallsyms_selftest.h"
22
23
24#define MAX_NUM_OF_RECORDS 64
25
26struct test_stat {
27 int min;
28 int max;
29 int save_cnt;
30 int real_cnt;
31 int perf;
32 u64 sum;
33 char *name;
34 unsigned long addr;
35 unsigned long addrs[MAX_NUM_OF_RECORDS];
36};
37
38struct test_item {
39 char *name;
40 unsigned long addr;
41};
42
43#define ITEM_FUNC(s) \
44 { \
45 .name = #s, \
46 .addr = (unsigned long)s, \
47 }
48
49#define ITEM_DATA(s) \
50 { \
51 .name = #s, \
52 .addr = (unsigned long)&s, \
53 }
54
55
56static int kallsyms_test_var_bss_static;
57static int kallsyms_test_var_data_static = 1;
58int kallsyms_test_var_bss;
59int kallsyms_test_var_data = 1;
60
61static int kallsyms_test_func_static(void)
62{
63 kallsyms_test_var_bss_static++;
64 kallsyms_test_var_data_static++;
65
66 return 0;
67}
68
69int kallsyms_test_func(void)
70{
71 return kallsyms_test_func_static();
72}
73
74__weak int kallsyms_test_func_weak(void)
75{
76 kallsyms_test_var_bss++;
77 kallsyms_test_var_data++;
78 return 0;
79}
80
81static struct test_item test_items[] = {
82 ITEM_FUNC(kallsyms_test_func_static),
83 ITEM_FUNC(kallsyms_test_func),
84 ITEM_FUNC(kallsyms_test_func_weak),
85 ITEM_FUNC(vmalloc),
86 ITEM_FUNC(vfree),
87#ifdef CONFIG_KALLSYMS_ALL
88 ITEM_DATA(kallsyms_test_var_bss_static),
89 ITEM_DATA(kallsyms_test_var_data_static),
90 ITEM_DATA(kallsyms_test_var_bss),
91 ITEM_DATA(kallsyms_test_var_data),
92 ITEM_DATA(vmap_area_list),
93#endif
94};
95
96static char stub_name[KSYM_NAME_LEN];
97
98static int stat_symbol_len(void *data, const char *name, struct module *mod, unsigned long addr)
99{
100 *(u32 *)data += strlen(name);
101
102 return 0;
103}
104
105static void test_kallsyms_compression_ratio(void)
106{
107 u32 pos, off, len, num;
108 u32 ratio, total_size, total_len = 0;
109
110 kallsyms_on_each_symbol(stat_symbol_len, &total_len);
111
112 /*
113 * A symbol name cannot start with a number. This stub name helps us
114 * traverse the entire symbol table without finding a match. It's used
115 * for subsequent performance tests, and its length is the average
116 * length of all symbol names.
117 */
118 memset(stub_name, '4', sizeof(stub_name));
119 pos = total_len / kallsyms_num_syms;
120 stub_name[pos] = 0;
121
122 pos = 0;
123 num = 0;
124 off = 0;
125 while (pos < kallsyms_num_syms) {
126 len = kallsyms_names[off];
127 num++;
128 off++;
129 pos++;
130 if ((len & 0x80) != 0) {
131 len = (len & 0x7f) | (kallsyms_names[off] << 7);
132 num++;
133 off++;
134 }
135 off += len;
136 }
137
138 /*
139 * 1. The length fields is not counted
140 * 2. The memory occupied by array kallsyms_token_table[] and
141 * kallsyms_token_index[] needs to be counted.
142 */
143 total_size = off - num;
144 pos = kallsyms_token_index[0xff];
145 total_size += pos + strlen(&kallsyms_token_table[pos]) + 1;
146 total_size += 0x100 * sizeof(u16);
147
148 pr_info(" ---------------------------------------------------------\n");
149 pr_info("| nr_symbols | compressed size | original size | ratio(%%) |\n");
150 pr_info("|---------------------------------------------------------|\n");
151 ratio = (u32)div_u64(10000ULL * total_size, total_len);
152 pr_info("| %10d | %10d | %10d | %2d.%-2d |\n",
153 kallsyms_num_syms, total_size, total_len, ratio / 100, ratio % 100);
154 pr_info(" ---------------------------------------------------------\n");
155}
156
157static int lookup_name(void *data, const char *name, struct module *mod, unsigned long addr)
158{
159 u64 t0, t1, t;
160 unsigned long flags;
161 struct test_stat *stat = (struct test_stat *)data;
162
163 local_irq_save(flags);
164 t0 = sched_clock();
165 (void)kallsyms_lookup_name(name);
166 t1 = sched_clock();
167 local_irq_restore(flags);
168
169 t = t1 - t0;
170 if (t < stat->min)
171 stat->min = t;
172
173 if (t > stat->max)
174 stat->max = t;
175
176 stat->real_cnt++;
177 stat->sum += t;
178
179 return 0;
180}
181
182static void test_perf_kallsyms_lookup_name(void)
183{
184 struct test_stat stat;
185
186 memset(&stat, 0, sizeof(stat));
187 stat.min = INT_MAX;
188 kallsyms_on_each_symbol(lookup_name, &stat);
189 pr_info("kallsyms_lookup_name() looked up %d symbols\n", stat.real_cnt);
190 pr_info("The time spent on each symbol is (ns): min=%d, max=%d, avg=%lld\n",
191 stat.min, stat.max, div_u64(stat.sum, stat.real_cnt));
192}
193
194static bool match_cleanup_name(const char *s, const char *name)
195{
196 char *p;
197 int len;
198
199 if (!IS_ENABLED(CONFIG_LTO_CLANG))
200 return false;
201
202 p = strchr(s, '.');
203 if (!p)
204 return false;
205
206 len = strlen(name);
207 if (p - s != len)
208 return false;
209
210 return !strncmp(s, name, len);
211}
212
213static int find_symbol(void *data, const char *name, struct module *mod, unsigned long addr)
214{
215 struct test_stat *stat = (struct test_stat *)data;
216
217 if (strcmp(name, stat->name) == 0 ||
218 (!stat->perf && match_cleanup_name(name, stat->name))) {
219 stat->real_cnt++;
220 stat->addr = addr;
221
222 if (stat->save_cnt < MAX_NUM_OF_RECORDS) {
223 stat->addrs[stat->save_cnt] = addr;
224 stat->save_cnt++;
225 }
226
227 if (stat->real_cnt == stat->max)
228 return 1;
229 }
230
231 return 0;
232}
233
234static void test_perf_kallsyms_on_each_symbol(void)
235{
236 u64 t0, t1;
237 unsigned long flags;
238 struct test_stat stat;
239
240 memset(&stat, 0, sizeof(stat));
241 stat.max = INT_MAX;
242 stat.name = stub_name;
243 stat.perf = 1;
244 local_irq_save(flags);
245 t0 = sched_clock();
246 kallsyms_on_each_symbol(find_symbol, &stat);
247 t1 = sched_clock();
248 local_irq_restore(flags);
249 pr_info("kallsyms_on_each_symbol() traverse all: %lld ns\n", t1 - t0);
250}
251
252static int match_symbol(void *data, unsigned long addr)
253{
254 struct test_stat *stat = (struct test_stat *)data;
255
256 stat->real_cnt++;
257 stat->addr = addr;
258
259 if (stat->save_cnt < MAX_NUM_OF_RECORDS) {
260 stat->addrs[stat->save_cnt] = addr;
261 stat->save_cnt++;
262 }
263
264 if (stat->real_cnt == stat->max)
265 return 1;
266
267 return 0;
268}
269
270static void test_perf_kallsyms_on_each_match_symbol(void)
271{
272 u64 t0, t1;
273 unsigned long flags;
274 struct test_stat stat;
275
276 memset(&stat, 0, sizeof(stat));
277 stat.max = INT_MAX;
278 stat.name = stub_name;
279 local_irq_save(flags);
280 t0 = sched_clock();
281 kallsyms_on_each_match_symbol(match_symbol, stat.name, &stat);
282 t1 = sched_clock();
283 local_irq_restore(flags);
284 pr_info("kallsyms_on_each_match_symbol() traverse all: %lld ns\n", t1 - t0);
285}
286
287static int test_kallsyms_basic_function(void)
288{
289 int i, j, ret;
290 int next = 0, nr_failed = 0;
291 char *prefix;
292 unsigned short rand;
293 unsigned long addr, lookup_addr;
294 char namebuf[KSYM_NAME_LEN];
295 struct test_stat *stat, *stat2;
296
297 stat = kmalloc(sizeof(*stat) * 2, GFP_KERNEL);
298 if (!stat)
299 return -ENOMEM;
300 stat2 = stat + 1;
301
302 prefix = "kallsyms_lookup_name() for";
303 for (i = 0; i < ARRAY_SIZE(test_items); i++) {
304 addr = kallsyms_lookup_name(test_items[i].name);
305 if (addr != test_items[i].addr) {
306 nr_failed++;
307 pr_info("%s %s failed: addr=%lx, expect %lx\n",
308 prefix, test_items[i].name, addr, test_items[i].addr);
309 }
310 }
311
312 prefix = "kallsyms_on_each_symbol() for";
313 for (i = 0; i < ARRAY_SIZE(test_items); i++) {
314 memset(stat, 0, sizeof(*stat));
315 stat->max = INT_MAX;
316 stat->name = test_items[i].name;
317 kallsyms_on_each_symbol(find_symbol, stat);
318 if (stat->addr != test_items[i].addr || stat->real_cnt != 1) {
319 nr_failed++;
320 pr_info("%s %s failed: count=%d, addr=%lx, expect %lx\n",
321 prefix, test_items[i].name,
322 stat->real_cnt, stat->addr, test_items[i].addr);
323 }
324 }
325
326 prefix = "kallsyms_on_each_match_symbol() for";
327 for (i = 0; i < ARRAY_SIZE(test_items); i++) {
328 memset(stat, 0, sizeof(*stat));
329 stat->max = INT_MAX;
330 stat->name = test_items[i].name;
331 kallsyms_on_each_match_symbol(match_symbol, test_items[i].name, stat);
332 if (stat->addr != test_items[i].addr || stat->real_cnt != 1) {
333 nr_failed++;
334 pr_info("%s %s failed: count=%d, addr=%lx, expect %lx\n",
335 prefix, test_items[i].name,
336 stat->real_cnt, stat->addr, test_items[i].addr);
337 }
338 }
339
340 if (nr_failed) {
341 kfree(stat);
342 return -ESRCH;
343 }
344
345 for (i = 0; i < kallsyms_num_syms; i++) {
346 addr = kallsyms_sym_address(i);
347 if (!is_ksym_addr(addr))
348 continue;
349
350 ret = lookup_symbol_name(addr, namebuf);
351 if (unlikely(ret)) {
352 namebuf[0] = 0;
353 goto failed;
354 }
355
356 /*
357 * The first '.' may be the initial letter, in which case the
358 * entire symbol name will be truncated to an empty string in
359 * cleanup_symbol_name(). Do not test these symbols.
360 *
361 * For example:
362 * cat /proc/kallsyms | awk '{print $3}' | grep -E "^\." | head
363 * .E_read_words
364 * .E_leading_bytes
365 * .E_trailing_bytes
366 * .E_write_words
367 * .E_copy
368 * .str.292.llvm.12122243386960820698
369 * .str.24.llvm.12122243386960820698
370 * .str.29.llvm.12122243386960820698
371 * .str.75.llvm.12122243386960820698
372 * .str.99.llvm.12122243386960820698
373 */
374 if (IS_ENABLED(CONFIG_LTO_CLANG) && !namebuf[0])
375 continue;
376
377 lookup_addr = kallsyms_lookup_name(namebuf);
378
379 memset(stat, 0, sizeof(*stat));
380 stat->max = INT_MAX;
381 kallsyms_on_each_match_symbol(match_symbol, namebuf, stat);
382
383 /*
384 * kallsyms_on_each_symbol() is too slow, randomly select some
385 * symbols for test.
386 */
387 if (i >= next) {
388 memset(stat2, 0, sizeof(*stat2));
389 stat2->max = INT_MAX;
390 stat2->name = namebuf;
391 kallsyms_on_each_symbol(find_symbol, stat2);
392
393 /*
394 * kallsyms_on_each_symbol() and kallsyms_on_each_match_symbol()
395 * need to get the same traversal result.
396 */
397 if (stat->addr != stat2->addr ||
398 stat->real_cnt != stat2->real_cnt ||
399 memcmp(stat->addrs, stat2->addrs,
400 stat->save_cnt * sizeof(stat->addrs[0])))
401 goto failed;
402
403 /*
404 * The average of random increments is 128, that is, one of
405 * them is tested every 128 symbols.
406 */
407 get_random_bytes(&rand, sizeof(rand));
408 next = i + (rand & 0xff) + 1;
409 }
410
411 /* Need to be found at least once */
412 if (!stat->real_cnt)
413 goto failed;
414
415 /*
416 * kallsyms_lookup_name() returns the address of the first
417 * symbol found and cannot be NULL.
418 */
419 if (!lookup_addr || lookup_addr != stat->addrs[0])
420 goto failed;
421
422 /*
423 * If the addresses of all matching symbols are recorded, the
424 * target address needs to be exist.
425 */
426 if (stat->real_cnt <= MAX_NUM_OF_RECORDS) {
427 for (j = 0; j < stat->save_cnt; j++) {
428 if (stat->addrs[j] == addr)
429 break;
430 }
431
432 if (j == stat->save_cnt)
433 goto failed;
434 }
435 }
436
437 kfree(stat);
438
439 return 0;
440
441failed:
442 pr_info("Test for %dth symbol failed: (%s) addr=%lx", i, namebuf, addr);
443 kfree(stat);
444 return -ESRCH;
445}
446
447static int test_entry(void *p)
448{
449 int ret;
450
451 do {
452 schedule_timeout(5 * HZ);
453 } while (system_state != SYSTEM_RUNNING);
454
455 pr_info("start\n");
456 ret = test_kallsyms_basic_function();
457 if (ret) {
458 pr_info("abort\n");
459 return 0;
460 }
461
462 test_kallsyms_compression_ratio();
463 test_perf_kallsyms_lookup_name();
464 test_perf_kallsyms_on_each_symbol();
465 test_perf_kallsyms_on_each_match_symbol();
466 pr_info("finish\n");
467
468 return 0;
469}
470
471static int __init kallsyms_test_init(void)
472{
473 struct task_struct *t;
474
475 t = kthread_create(test_entry, NULL, "kallsyms_test");
476 if (IS_ERR(t)) {
477 pr_info("Create kallsyms selftest task failed\n");
478 return PTR_ERR(t);
479 }
480 kthread_bind(t, 0);
481 wake_up_process(t);
482
483 return 0;
484}
485late_initcall(kallsyms_test_init);
kernel/kallsyms_selftest.h+13−0
@@ -0,0 +1,13 @@
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2#ifndef LINUX_KALLSYMS_SELFTEST_H_
3#define LINUX_KALLSYMS_SELFTEST_H_
4
5#include <linux/types.h>
6
7extern int kallsyms_test_var_bss;
8extern int kallsyms_test_var_data;
9
10extern int kallsyms_test_func(void);
11extern int kallsyms_test_func_weak(void);
12
13#endif // LINUX_KALLSYMS_SELFTEST_H_
kernel/livepatch/core.c+19−12
@@ -125,20 +125,10 @@ struct klp_find_arg { · 124 unmodified lines
125 unsigned long pos;125 unsigned long pos;
126};126};
127 127
128static int klp_find_callback(void *data, const char *name,128static int klp_match_callback(void *data, unsigned long addr)
129 struct module *mod, unsigned long addr)
130{129{
131 struct klp_find_arg *args = data;130 struct klp_find_arg *args = data;
132 131
133 if ((mod && !args->objname) || (!mod && args->objname))
134 return 0;
135
136 if (strcmp(args->name, name))
137 return 0;
138
139 if (args->objname && strcmp(args->objname, mod->name))
140 return 0;
141
142 args->addr = addr;132 args->addr = addr;
143 args->count++;133 args->count++;
144 134
@@ -153,6 +143,23 @@ static int klp_find_callback(void *data, const char *name, · 8 unmodified lines
153 return 0;143 return 0;
154}144}
155 145
146static int klp_find_callback(void *data, const char *name,
147 struct module *mod, unsigned long addr)
148{
149 struct klp_find_arg *args = data;
150
151 if ((mod && !args->objname) || (!mod && args->objname))
152 return 0;
153
154 if (strcmp(args->name, name))
155 return 0;
156
157 if (args->objname && strcmp(args->objname, mod->name))
158 return 0;
159
160 return klp_match_callback(data, addr);
161}
162
156static int klp_find_object_symbol(const char *objname, const char *name,163static int klp_find_object_symbol(const char *objname, const char *name,
157 unsigned long sympos, unsigned long *addr)164 unsigned long sympos, unsigned long *addr)
158{165{
@@ -167,7 +174,7 @@ static int klp_find_object_symbol(const char *objname, const char *name, · 8 unmodified lines
167 if (objname)174 if (objname)
168 module_kallsyms_on_each_symbol(klp_find_callback, &args);175 module_kallsyms_on_each_symbol(klp_find_callback, &args);
169 else176 else
170 kallsyms_on_each_symbol(klp_find_callback, &args);177 kallsyms_on_each_match_symbol(klp_match_callback, name, &args);
171 178
172 /*179 /*
173 * Ensure an address was found. If sympos is 0, ensure symbol is unique;180 * Ensure an address was found. If sympos is 0, ensure symbol is unique;
kernel/module/Kconfig+2−1
@@ -221,9 +221,10 @@ endchoice · 220 unmodified lines
221 221
222config MODULE_DECOMPRESS222config MODULE_DECOMPRESS
223 bool "Support in-kernel module decompression"223 bool "Support in-kernel module decompression"
224 depends on MODULE_COMPRESS_GZIP || MODULE_COMPRESS_XZ224 depends on MODULE_COMPRESS_GZIP || MODULE_COMPRESS_XZ || MODULE_COMPRESS_ZSTD
225 select ZLIB_INFLATE if MODULE_COMPRESS_GZIP225 select ZLIB_INFLATE if MODULE_COMPRESS_GZIP
226 select XZ_DEC if MODULE_COMPRESS_XZ226 select XZ_DEC if MODULE_COMPRESS_XZ
227 select ZSTD_DECOMPRESS if MODULE_COMPRESS_ZSTD
227 help228 help
228 229
229 Support for decompressing kernel modules by the kernel itself230 Support for decompressing kernel modules by the kernel itself
kernel/module/decompress.c+94−6
@@ -50,7 +50,7 @@ static struct page *module_get_next_page(struct load_info *info) · 49 unmodified lines
50 return page;50 return page;
51}51}
52 52
53#ifdef CONFIG_MODULE_COMPRESS_GZIP53#if defined(CONFIG_MODULE_COMPRESS_GZIP)
54#include <linux/zlib.h>54#include <linux/zlib.h>
55#define MODULE_COMPRESSION gzip55#define MODULE_COMPRESSION gzip
56#define MODULE_DECOMPRESS_FN module_gzip_decompress56#define MODULE_DECOMPRESS_FN module_gzip_decompress
@@ -114,8 +114,8 @@ static ssize_t module_gzip_decompress(struct load_info *info, · 57 unmodified lines
114 do {114 do {
115 struct page *page = module_get_next_page(info);115 struct page *page = module_get_next_page(info);
116 116
117 if (!page) {117 if (IS_ERR(page)) {
118 retval = -ENOMEM;118 retval = PTR_ERR(page);
119 goto out_inflate_end;119 goto out_inflate_end;
120 }120 }
121 121
@@ -141,7 +141,7 @@ out: · 19 unmodified lines
141 kfree(s.workspace);141 kfree(s.workspace);
142 return retval;142 return retval;
143}143}
144#elif CONFIG_MODULE_COMPRESS_XZ144#elif defined(CONFIG_MODULE_COMPRESS_XZ)
145#include <linux/xz.h>145#include <linux/xz.h>
146#define MODULE_COMPRESSION xz146#define MODULE_COMPRESSION xz
147#define MODULE_DECOMPRESS_FN module_xz_decompress147#define MODULE_DECOMPRESS_FN module_xz_decompress
@@ -173,8 +173,8 @@ static ssize_t module_xz_decompress(struct load_info *info, · 25 unmodified lines
173 do {173 do {
174 struct page *page = module_get_next_page(info);174 struct page *page = module_get_next_page(info);
175 175
176 if (!page) {176 if (IS_ERR(page)) {
177 retval = -ENOMEM;177 retval = PTR_ERR(page);
178 goto out;178 goto out;
179 }179 }
180 180
@@ -199,6 +199,94 @@ static ssize_t module_xz_decompress(struct load_info *info, · 18 unmodified lines
199 xz_dec_end(xz_dec);199 xz_dec_end(xz_dec);
200 return retval;200 return retval;
201}201}
202#elif defined(CONFIG_MODULE_COMPRESS_ZSTD)
203#include <linux/zstd.h>
204#define MODULE_COMPRESSION zstd
205#define MODULE_DECOMPRESS_FN module_zstd_decompress
206
207static ssize_t module_zstd_decompress(struct load_info *info,
208 const void *buf, size_t size)
209{
210 static const u8 signature[] = { 0x28, 0xb5, 0x2f, 0xfd };
211 ZSTD_outBuffer zstd_dec;
212 ZSTD_inBuffer zstd_buf;
213 zstd_frame_header header;
214 size_t wksp_size;
215 void *wksp = NULL;
216 ZSTD_DStream *dstream;
217 size_t ret;
218 size_t new_size = 0;
219 int retval;
220
221 if (size < sizeof(signature) ||
222 memcmp(buf, signature, sizeof(signature))) {
223 pr_err("not a zstd compressed module\n");
224 return -EINVAL;
225 }
226
227 zstd_buf.src = buf;
228 zstd_buf.pos = 0;
229 zstd_buf.size = size;
230
231 ret = zstd_get_frame_header(&header, zstd_buf.src, zstd_buf.size);
232 if (ret != 0) {
233 pr_err("ZSTD-compressed data has an incomplete frame header\n");
234 retval = -EINVAL;
235 goto out;
236 }
237 if (header.windowSize > (1 << ZSTD_WINDOWLOG_MAX)) {
238 pr_err("ZSTD-compressed data has too large a window size\n");
239 retval = -EINVAL;
240 goto out;
241 }
242
243 wksp_size = zstd_dstream_workspace_bound(header.windowSize);
244 wksp = kmalloc(wksp_size, GFP_KERNEL);
245 if (!wksp) {
246 retval = -ENOMEM;
247 goto out;
248 }
249
250 dstream = zstd_init_dstream(header.windowSize, wksp, wksp_size);
251 if (!dstream) {
252 pr_err("Can't initialize ZSTD stream\n");
253 retval = -ENOMEM;
254 goto out;
255 }
256
257 do {
258 struct page *page = module_get_next_page(info);
259
260 if (!IS_ERR(page)) {
261 retval = PTR_ERR(page);
262 goto out;
263 }
264
265 zstd_dec.dst = kmap_local_page(page);
266 zstd_dec.pos = 0;
267 zstd_dec.size = PAGE_SIZE;
268
269 ret = zstd_decompress_stream(dstream, &zstd_dec, &zstd_buf);
270 kunmap(page);
271 retval = zstd_get_error_code(ret);
272 if (retval)
273 break;
274
275 new_size += zstd_dec.pos;
276 } while (zstd_dec.pos == PAGE_SIZE && ret != 0);
277
278 if (retval) {
279 pr_err("ZSTD-decompression failed with status %d\n", retval);
280 retval = -EINVAL;
281 goto out;
282 }
283
284 retval = new_size;
285
286 out:
287 kfree(wksp);
288 return retval;
289}
202#else290#else
203#error "Unexpected configuration for CONFIG_MODULE_DECOMPRESS"291#error "Unexpected configuration for CONFIG_MODULE_DECOMPRESS"
204#endif292#endif
kernel/module/main.c+0−3
@@ -85,9 +85,6 @@ struct mod_tree_root mod_data_tree __cacheline_aligned = { · 84 unmodified lines
85};85};
86#endif86#endif
87 87
88#define module_addr_min mod_tree.addr_min
89#define module_addr_max mod_tree.addr_max
90
91struct symsearch {88struct symsearch {
92 const struct kernel_symbol *start, *stop;89 const struct kernel_symbol *start, *stop;
93 const s32 *crcs;90 const s32 *crcs;
kernel/module/sysfs.c+1−1
@@ -340,7 +340,7 @@ static int mod_sysfs_init(struct module *mod) · 339 unmodified lines
340 int err;340 int err;
341 struct kobject *kobj;341 struct kobject *kobj;
342 342
343 if (!module_sysfs_initialized) {343 if (!module_kset) {
344 pr_err("%s: module sysfs not initialized\n", mod->name);344 pr_err("%s: module sysfs not initialized\n", mod->name);
345 err = -EINVAL;345 err = -EINVAL;
346 goto out;346 goto out;
kernel/params.c+19−4
@@ -940,7 +940,6 @@ static const struct kset_uevent_ops module_uevent_ops = { · 939 unmodified lines
940};940};
941 941
942struct kset *module_kset;942struct kset *module_kset;
943int module_sysfs_initialized;
944 943
945static void module_kobj_release(struct kobject *kobj)944static void module_kobj_release(struct kobject *kobj)
946{945{
@@ -954,7 +953,11 @@ struct kobj_type module_ktype = { · 7 unmodified lines
954};953};
955 954
956/*955/*
957 * param_sysfs_init - wrapper for built-in params support956 * param_sysfs_init - create "module" kset
957 *
958 * This must be done before the initramfs is unpacked and
959 * request_module() thus becomes possible, because otherwise the
960 * module load would fail in mod_sysfs_init.
958 */961 */
959static int __init param_sysfs_init(void)962static int __init param_sysfs_init(void)
960{963{
@@ -964,13 +967,25 @@ static int __init param_sysfs_init(void) · 3 unmodified lines
964 __FILE__, __LINE__);967 __FILE__, __LINE__);
965 return -ENOMEM;968 return -ENOMEM;
966 }969 }
967 module_sysfs_initialized = 1;970
971 return 0;
972}
973subsys_initcall(param_sysfs_init);
974
975/*
976 * param_sysfs_builtin_init - add sysfs version and parameter
977 * attributes for built-in modules
978 */
979static int __init param_sysfs_builtin_init(void)
980{
981 if (!module_kset)
982 return -ENOMEM;
968 983
969 version_sysfs_builtin();984 version_sysfs_builtin();
970 param_sysfs_builtin();985 param_sysfs_builtin();
971 986
972 return 0;987 return 0;
973}988}
974subsys_initcall(param_sysfs_init);989late_initcall(param_sysfs_builtin_init);
975 990
976#endif /* CONFIG_SYSFS */991#endif /* CONFIG_SYSFS */
scripts/kallsyms.c+75−3
@@ -49,6 +49,7 @@ _Static_assert( · 48 unmodified lines
49struct sym_entry {49struct sym_entry {
50 unsigned long long addr;50 unsigned long long addr;
51 unsigned int len;51 unsigned int len;
52 unsigned int seq;
52 unsigned int start_pos;53 unsigned int start_pos;
53 unsigned int percpu_absolute;54 unsigned int percpu_absolute;
54 unsigned char sym[];55 unsigned char sym[];
@@ -77,6 +78,7 @@ static unsigned int table_size, table_cnt; · 22 unmodified lines
77static int all_symbols;78static int all_symbols;
78static int absolute_percpu;79static int absolute_percpu;
79static int base_relative;80static int base_relative;
81static int lto_clang;
80 82
81static int token_profit[0x10000];83static int token_profit[0x10000];
82 84
@@ -88,7 +90,7 @@ static unsigned char best_table_len[256]; · 5 unmodified lines
88static void usage(void)90static void usage(void)
89{91{
90 fprintf(stderr, "Usage: kallsyms [--all-symbols] [--absolute-percpu] "92 fprintf(stderr, "Usage: kallsyms [--all-symbols] [--absolute-percpu] "
91 "[--base-relative] in.map > out.S\n");93 "[--base-relative] [--lto-clang] in.map > out.S\n");
92 exit(1);94 exit(1);
93}95}
94 96
@@ -410,6 +412,65 @@ static int symbol_absolute(const struct sym_entry *s) · 315 unmodified lines
410 return s->percpu_absolute;412 return s->percpu_absolute;
411}413}
412 414
415static char * s_name(char *buf)
416{
417 /* Skip the symbol type */
418 return buf + 1;
419}
420
421static void cleanup_symbol_name(char *s)
422{
423 char *p;
424
425 if (!lto_clang)
426 return;
427
428 /*
429 * ASCII[.] = 2e
430 * ASCII[0-9] = 30,39
431 * ASCII[A-Z] = 41,5a
432 * ASCII[_] = 5f
433 * ASCII[a-z] = 61,7a
434 *
435 * As above, replacing '.' with '\0' does not affect the main sorting,
436 * but it helps us with subsorting.
437 */
438 p = strchr(s, '.');
439 if (p)
440 *p = '\0';
441}
442
443static int compare_names(const void *a, const void *b)
444{
445 int ret;
446 char sa_namebuf[KSYM_NAME_LEN];
447 char sb_namebuf[KSYM_NAME_LEN];
448 const struct sym_entry *sa = *(const struct sym_entry **)a;
449 const struct sym_entry *sb = *(const struct sym_entry **)b;
450
451 expand_symbol(sa->sym, sa->len, sa_namebuf);
452 expand_symbol(sb->sym, sb->len, sb_namebuf);
453 cleanup_symbol_name(s_name(sa_namebuf));
454 cleanup_symbol_name(s_name(sb_namebuf));
455 ret = strcmp(s_name(sa_namebuf), s_name(sb_namebuf));
456 if (!ret) {
457 if (sa->addr > sb->addr)
458 return 1;
459 else if (sa->addr < sb->addr)
460 return -1;
461
462 /* keep old order */
463 return (int)(sa->seq - sb->seq);
464 }
465
466 return ret;
467}
468
469static void sort_symbols_by_name(void)
470{
471 qsort(table, table_cnt, sizeof(table[0]), compare_names);
472}
473
413static void write_src(void)474static void write_src(void)
414{475{
415 unsigned int i, k, off;476 unsigned int i, k, off;
@@ -495,6 +556,7 @@ static void write_src(void) · 79 unmodified lines
495 for (i = 0; i < table_cnt; i++) {556 for (i = 0; i < table_cnt; i++) {
496 if ((i & 0xFF) == 0)557 if ((i & 0xFF) == 0)
497 markers[i >> 8] = off;558 markers[i >> 8] = off;
559 table[i]->seq = i;
498 560
499 /* There cannot be any symbol of length zero. */561 /* There cannot be any symbol of length zero. */
500 if (table[i]->len == 0) {562 if (table[i]->len == 0) {
@@ -535,6 +597,15 @@ static void write_src(void) · 34 unmodified lines
535 597
536 free(markers);598 free(markers);
537 599
600 sort_symbols_by_name();
601 output_label("kallsyms_seqs_of_names");
602 for (i = 0; i < table_cnt; i++)
603 printf("\t.byte 0x%02x, 0x%02x, 0x%02x\n",
604 (unsigned char)(table[i]->seq >> 16),
605 (unsigned char)(table[i]->seq >> 8),
606 (unsigned char)(table[i]->seq >> 0));
607 printf("\n");
608
538 output_label("kallsyms_token_table");609 output_label("kallsyms_token_table");
539 off = 0;610 off = 0;
540 for (i = 0; i < 256; i++) {611 for (i = 0; i < 256; i++) {
@@ -573,7 +644,7 @@ static void forget_symbol(const unsigned char *symbol, int len) · 32 unmodified lines
573}644}
574 645
575/* do the initial token count */646/* do the initial token count */
576static void build_initial_tok_table(void)647static void build_initial_token_table(void)
577{648{
578 unsigned int i;649 unsigned int i;
579 650
@@ -698,7 +769,7 @@ static void insert_real_symbols_in_table(void) · 118 unmodified lines
698 769
699static void optimize_token_table(void)770static void optimize_token_table(void)
700{771{
701 build_initial_tok_table();772 build_initial_token_table();
702 773
703 insert_real_symbols_in_table();774 insert_real_symbols_in_table();
704 775
@@ -818,6 +889,7 @@ int main(int argc, char **argv) · 113 unmodified lines
818 {"all-symbols", no_argument, &all_symbols, 1},889 {"all-symbols", no_argument, &all_symbols, 1},
819 {"absolute-percpu", no_argument, &absolute_percpu, 1},890 {"absolute-percpu", no_argument, &absolute_percpu, 1},
820 {"base-relative", no_argument, &base_relative, 1},891 {"base-relative", no_argument, &base_relative, 1},
892 {"lto-clang", no_argument, &lto_clang, 1},
821 {},893 {},
822 };894 };
823 895
scripts/link-vmlinux.sh+4−0
@@ -156,6 +156,10 @@ kallsyms() · 155 unmodified lines
156 kallsymopt="${kallsymopt} --base-relative"156 kallsymopt="${kallsymopt} --base-relative"
157 fi157 fi
158 158
159 if is_enabled CONFIG_LTO_CLANG; then
160 kallsymopt="${kallsymopt} --lto-clang"
161 fi
162
159 info KSYMS ${2}163 info KSYMS ${2}
160 scripts/kallsyms ${kallsymopt} ${1} > ${2}164 scripts/kallsyms ${kallsymopt} ${1} > ${2}
161}165}