Merge tag 'x86-boot-2020-08-03' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 boot updates from Ingo Molnar: "The main change in this cycle was to add support for ZSTD-compressed kernel and initrd images. ZSTD has a very fast decompressor, yet it compresses better than gzip" * tag 'x86-boot-2020-08-03' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: Documentation: dontdiff: Add zstd compressed files .gitignore: Add ZSTD-compressed files x86: Add support for ZSTD compressed kernel x86: Bump ZO_z_extra_bytes margin for zstd usr: Add support for zstd compressed initramfs init: Add support for zstd compressed kernel lib: Add zstd support to decompress lib: Prepare zstd for preboot environment, improve performance

Linus Torvalds committed 6 years ago

Commit c0dfadf2 parents ba77c56 1ac1efa21 files · +469 −26Browse files

Unified · Split

21 files

.gitignore +1 −0
Documentation/dontdiff +1 −0
Documentation/x86/boot.rst +3 −3
Makefile +2 −1
arch/x86/Kconfig +1 −0
arch/x86/boot/compressed/Makefile +5 −1
arch/x86/boot/compressed/kaslr.c +0 −7
arch/x86/boot/compressed/misc.c +4 −0
arch/x86/boot/header.S +7 −1
arch/x86/include/asm/boot.h +9 −2
include/linux/decompress/unzstd.h +11 −0
init/Kconfig +14 −1
lib/Kconfig +4 −0
lib/Makefile +1 −0
lib/decompress.c +5 −0
lib/decompress_unzstd.c +345 −0
lib/zstd/fse_decompress.c +1 −8
lib/zstd/zstd_internal.h +12 −2
scripts/Makefile.lib +22 −0
usr/Kconfig +20 −0
usr/Makefile +1 −0

.gitignore+1−0

@@ -44,6 +44,7 @@ · 43 unmodified lines
44	44	*.tab.[ch]
45	45	*.tar
46	46	*.xz
	47	+*.zst
47	48	Module.symvers
48	49	modules.builtin
49	50	modules.order

Documentation/dontdiff+1−0

@@ -55,6 +55,7 @@ · 54 unmodified lines
55	55	*.ver
56	56	*.xml
57	57	*.xz
	58	+*.zst
58	59	*_MODULES
59	60	*_vga16.c
60	61	*~

Documentation/x86/boot.rst+3−3

@@ -782,9 +782,9 @@ Protocol: 2.08+ · 781 unmodified lines
782	782	uncompressed data should be determined using the standard magic
783	783	numbers. The currently supported compression formats are gzip
784	784	(magic numbers 1F 8B or 1F 9E), bzip2 (magic number 42 5A), LZMA
785		− (magic number 5D 00), XZ (magic number FD 37), and LZ4 (magic number
786		− 02 21). The uncompressed payload is currently always ELF (magic
787		− number 7F 45 4C 46).
	785	+ (magic number 5D 00), XZ (magic number FD 37), LZ4 (magic number
	786	+ 02 21) and ZSTD (magic number 28 B5). The uncompressed payload is
	787	+ currently always ELF (magic number 7F 45 4C 46).
788	788
789	789	============ ==============
790	790	Field name: payload_length

Makefile+2−1

@@ -464,6 +464,7 @@ KLZOP = lzop · 463 unmodified lines
464	464	LZMA = lzma
465	465	LZ4 = lz4c
466	466	XZ = xz
	467	+ZSTD = zstd
467	468
468	469	CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \
469	470	-Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF)
@@ -512,7 +513,7 @@ CLANG_FLAGS := · 42 unmodified lines
512	513	export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC
513	514	export CPP AR NM STRIP OBJCOPY OBJDUMP OBJSIZE READELF PAHOLE LEX YACC AWK INSTALLKERNEL
514	515	export PERL PYTHON PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
515		−export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ
	516	+export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ ZSTD
516	517	export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE
517	518
518	519	export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS

arch/x86/Kconfig+1−0

@@ -188,6 +188,7 @@ config X86 · 187 unmodified lines
188	188	select HAVE_KERNEL_LZMA
189	189	select HAVE_KERNEL_LZO
190	190	select HAVE_KERNEL_XZ
	191	+ select HAVE_KERNEL_ZSTD
191	192	select HAVE_KPROBES
192	193	select HAVE_KPROBES_ON_FTRACE
193	194	select HAVE_FUNCTION_ERROR_INJECTION

arch/x86/boot/compressed/Makefile+5−1

@@ -26,7 +26,7 @@ OBJECT_FILES_NON_STANDARD := y · 25 unmodified lines
26	26	KCOV_INSTRUMENT := n
27	27
28	28	targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma \
29		− vmlinux.bin.xz vmlinux.bin.lzo vmlinux.bin.lz4
	29	+ vmlinux.bin.xz vmlinux.bin.lzo vmlinux.bin.lz4 vmlinux.bin.zst
30	30
31	31	KBUILD_CFLAGS := -m$(BITS) -O2
32	32	KBUILD_CFLAGS += -fno-strict-aliasing $(call cc-option, -fPIE, -fPIC)
@@ -42,6 +42,7 @@ KBUILD_CFLAGS += $(call cc-disable-warning, gnu) · 9 unmodified lines
42	42	KBUILD_CFLAGS += -Wno-pointer-sign
43	43	KBUILD_CFLAGS += $(call cc-option,-fmacro-prefix-map=$(srctree)/=)
44	44	KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
	45	+KBUILD_CFLAGS += -D__DISABLE_EXPORTS
45	46
46	47	KBUILD_AFLAGS := $(KBUILD_CFLAGS) -D__ASSEMBLY__
47	48	GCOV_PROFILE := n
@@ -145,6 +146,8 @@ $(obj)/vmlinux.bin.lzo: $(vmlinux.bin.all-y) FORCE · 97 unmodified lines
145	146	$(call if_changed,lzo)
146	147	$(obj)/vmlinux.bin.lz4: $(vmlinux.bin.all-y) FORCE
147	148	$(call if_changed,lz4)
	149	+$(obj)/vmlinux.bin.zst: $(vmlinux.bin.all-y) FORCE
	150	+ $(call if_changed,zstd22)
148	151
149	152	suffix-$(CONFIG_KERNEL_GZIP) := gz
150	153	suffix-$(CONFIG_KERNEL_BZIP2) := bz2
@@ -152,6 +155,7 @@ suffix-$(CONFIG_KERNEL_LZMA) := lzma · 1 unmodified line
152	155	suffix-$(CONFIG_KERNEL_XZ) := xz
153	156	suffix-$(CONFIG_KERNEL_LZO) := lzo
154	157	suffix-$(CONFIG_KERNEL_LZ4) := lz4
	158	+suffix-$(CONFIG_KERNEL_ZSTD) := zst
155	159
156	160	quiet_cmd_mkpiggy = MKPIGGY $@
157	161	cmd_mkpiggy = $(obj)/mkpiggy $< > $@

arch/x86/boot/compressed/kaslr.c+0−7

@@ -19,13 +19,6 @@ · 18 unmodified lines
19	19	*/
20	20	#define BOOT_CTYPE_H
21	21
22		−/*
23		− * _ctype[] in lib/ctype.c is needed by isspace() of linux/ctype.h.
24		− * While both lib/ctype.c and lib/cmdline.c will bring EXPORT_SYMBOL
25		− * which is meaningless and will cause compiling error in some cases.
26		− */
27		−#define __DISABLE_EXPORTS
28		−
29	22	#include "misc.h"
30	23	#include "error.h"
31	24	#include "../string.h"

arch/x86/boot/compressed/misc.c+4−0

@@ -77,6 +77,10 @@ static int lines, cols; · 76 unmodified lines
77	77	#ifdef CONFIG_KERNEL_LZ4
78	78	#include "../../../../lib/decompress_unlz4.c"
79	79	#endif
	80	+
	81	+#ifdef CONFIG_KERNEL_ZSTD
	82	+#include "../../../../lib/decompress_unzstd.c"
	83	+#endif
80	84	/*
81	85	* NOTE: When adding a new decompressor, please update the analysis in
82	86	* ../header.S.

arch/x86/boot/header.S+7−1

@@ -539,8 +539,14 @@ pref_address: .quad LOAD_PHYSICAL_ADDR # preferred load addr · 538 unmodified lines
539	539	# the size-dependent part now grows so fast.
540	540	#
541	541	# extra_bytes = (uncompressed_size >> 8) + 65536
	542	+#
	543	+# ZSTD compressed data grows by at most 3 bytes per 128K, and only has a 22
	544	+# byte fixed overhead but has a maximum block size of 128K, so it needs a
	545	+# larger margin.
	546	+#
	547	+# extra_bytes = (uncompressed_size >> 8) + 131072
542	548
543		−#define ZO_z_extra_bytes ((ZO_z_output_len >> 8) + 65536)
	549	+#define ZO_z_extra_bytes ((ZO_z_output_len >> 8) + 131072)
544	550	#if ZO_z_output_len > ZO_z_input_len
545	551	# define ZO_z_extract_offset (ZO_z_output_len + ZO_z_extra_bytes - \
546	552	ZO_z_input_len)

arch/x86/include/asm/boot.h+9−2

@@ -24,9 +24,16 @@ · 23 unmodified lines
24	24	# error "Invalid value for CONFIG_PHYSICAL_ALIGN"
25	25	#endif
26	26
27		−#ifdef CONFIG_KERNEL_BZIP2
	27	+#if defined(CONFIG_KERNEL_BZIP2)
28	28	# define BOOT_HEAP_SIZE 0x400000
29		−#else /* !CONFIG_KERNEL_BZIP2 */
	29	+#elif defined(CONFIG_KERNEL_ZSTD)
	30	+/*
	31	+ * Zstd needs to allocate the ZSTD_DCtx in order to decompress the kernel.
	32	+ * The ZSTD_DCtx is ~160KB, so set the heap size to 192KB because it is a
	33	+ * round number and to allow some slack.
	34	+ */
	35	+# define BOOT_HEAP_SIZE 0x30000
	36	+#else
30	37	# define BOOT_HEAP_SIZE 0x10000
31	38	#endif
32	39

include/linux/decompress/unzstd.h+11−0

@@ -0,0 +1,11 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+#ifndef LINUX_DECOMPRESS_UNZSTD_H
	3	+#define LINUX_DECOMPRESS_UNZSTD_H
	4	+
	5	+int unzstd(unsigned char *inbuf, long len,
	6	+ long (fill)(void, unsigned long),
	7	+ long (flush)(void, unsigned long),
	8	+ unsigned char *output,
	9	+ long *pos,
	10	+ void (error_fn)(char x));
	11	+#endif

init/Kconfig+14−1

@@ -191,13 +191,16 @@ config HAVE_KERNEL_LZO · 190 unmodified lines
191	191	config HAVE_KERNEL_LZ4
192	192	bool
193	193
	194	+config HAVE_KERNEL_ZSTD
	195	+ bool
	196	+
194	197	config HAVE_KERNEL_UNCOMPRESSED
195	198	bool
196	199
197	200	choice
198	201	prompt "Kernel compression mode"
199	202	default KERNEL_GZIP
200		− depends on HAVE_KERNEL_GZIP \|\| HAVE_KERNEL_BZIP2 \|\| HAVE_KERNEL_LZMA \|\| HAVE_KERNEL_XZ \|\| HAVE_KERNEL_LZO \|\| HAVE_KERNEL_LZ4 \|\| HAVE_KERNEL_UNCOMPRESSED
	203	+ depends on HAVE_KERNEL_GZIP \|\| HAVE_KERNEL_BZIP2 \|\| HAVE_KERNEL_LZMA \|\| HAVE_KERNEL_XZ \|\| HAVE_KERNEL_LZO \|\| HAVE_KERNEL_LZ4 \|\| HAVE_KERNEL_ZSTD \|\| HAVE_KERNEL_UNCOMPRESSED
201	204	help
202	205	The linux kernel is a kind of self-extracting executable.
203	206	Several compression algorithms are available, which differ
@@ -276,6 +279,16 @@ config KERNEL_LZ4 · 72 unmodified lines
276	279	is about 8% bigger than LZO. But the decompression speed is
277	280	faster than LZO.
278	281
	282	+config KERNEL_ZSTD
	283	+ bool "ZSTD"
	284	+ depends on HAVE_KERNEL_ZSTD
	285	+ help
	286	+ ZSTD is a compression algorithm targeting intermediate compression
	287	+ with fast decompression speed. It will compress better than GZIP and
	288	+ decompress around the same speed as LZO, but slower than LZ4. You
	289	+ will need at least 192 KB RAM or more for booting. The zstd command
	290	+ line tool is required for compression.
	291	+
279	292	config KERNEL_UNCOMPRESSED
280	293	bool "None"
281	294	depends on HAVE_KERNEL_UNCOMPRESSED

lib/Kconfig+4−0

@@ -342,6 +342,10 @@ config DECOMPRESS_LZ4 · 341 unmodified lines
342	342	select LZ4_DECOMPRESS
343	343	tristate
344	344
	345	+config DECOMPRESS_ZSTD
	346	+ select ZSTD_DECOMPRESS
	347	+ tristate
	348	+
345	349	#
346	350	# Generic allocator support is selected if needed
347	351	#

lib/Makefile+1−0

@@ -170,6 +170,7 @@ lib-$(CONFIG_DECOMPRESS_LZMA) += decompress_unlzma.o · 169 unmodified lines
170	170	lib-$(CONFIG_DECOMPRESS_XZ) += decompress_unxz.o
171	171	lib-$(CONFIG_DECOMPRESS_LZO) += decompress_unlzo.o
172	172	lib-$(CONFIG_DECOMPRESS_LZ4) += decompress_unlz4.o
	173	+lib-$(CONFIG_DECOMPRESS_ZSTD) += decompress_unzstd.o
173	174
174	175	obj-$(CONFIG_TEXTSEARCH) += textsearch.o
175	176	obj-$(CONFIG_TEXTSEARCH_KMP) += ts_kmp.o

lib/decompress.c+5−0

@@ -13,6 +13,7 @@ · 12 unmodified lines
13	13	#include <linux/decompress/inflate.h>
14	14	#include <linux/decompress/unlzo.h>
15	15	#include <linux/decompress/unlz4.h>
	16	+#include <linux/decompress/unzstd.h>
16	17
17	18	#include <linux/types.h>
18	19	#include <linux/string.h>
@@ -37,6 +38,9 @@ · 18 unmodified lines
37	38	#ifndef CONFIG_DECOMPRESS_LZ4
38	39	# define unlz4 NULL
39	40	#endif
	41	+#ifndef CONFIG_DECOMPRESS_ZSTD
	42	+# define unzstd NULL
	43	+#endif
40	44
41	45	struct compress_format {
42	46	unsigned char magic[2];
@@ -52,6 +56,7 @@ static const struct compress_format compressed_formats[] __initconst = { · 9 unmodified lines
52	56	{ {0xfd, 0x37}, "xz", unxz },
53	57	{ {0x89, 0x4c}, "lzo", unlzo },
54	58	{ {0x02, 0x21}, "lz4", unlz4 },
	59	+ { {0x28, 0xb5}, "zstd", unzstd },
55	60	{ {0, 0}, NULL, NULL }
56	61	};
57	62

lib/decompress_unzstd.c+345−0

@@ -0,0 +1,345 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+
	3	+/*
	4	+ * Important notes about in-place decompression
	5	+ *
	6	+ * At least on x86, the kernel is decompressed in place: the compressed data
	7	+ * is placed to the end of the output buffer, and the decompressor overwrites
	8	+ * most of the compressed data. There must be enough safety margin to
	9	+ * guarantee that the write position is always behind the read position.
	10	+ *
	11	+ * The safety margin for ZSTD with a 128 KB block size is calculated below.
	12	+ * Note that the margin with ZSTD is bigger than with GZIP or XZ!
	13	+ *
	14	+ * The worst case for in-place decompression is that the beginning of
	15	+ * the file is compressed extremely well, and the rest of the file is
	16	+ * uncompressible. Thus, we must look for worst-case expansion when the
	17	+ * compressor is encoding uncompressible data.
	18	+ *
	19	+ * The structure of the .zst file in case of a compresed kernel is as follows.
	20	+ * Maximum sizes (as bytes) of the fields are in parenthesis.
	21	+ *
	22	+ * Frame Header: (18)
	23	+ * Blocks: (N)
	24	+ * Checksum: (4)
	25	+ *
	26	+ * The frame header and checksum overhead is at most 22 bytes.
	27	+ *
	28	+ * ZSTD stores the data in blocks. Each block has a header whose size is
	29	+ * a 3 bytes. After the block header, there is up to 128 KB of payload.
	30	+ * The maximum uncompressed size of the payload is 128 KB. The minimum
	31	+ * uncompressed size of the payload is never less than the payload size
	32	+ * (excluding the block header).
	33	+ *
	34	+ * The assumption, that the uncompressed size of the payload is never
	35	+ * smaller than the payload itself, is valid only when talking about
	36	+ * the payload as a whole. It is possible that the payload has parts where
	37	+ * the decompressor consumes more input than it produces output. Calculating
	38	+ * the worst case for this would be tricky. Instead of trying to do that,
	39	+ * let's simply make sure that the decompressor never overwrites any bytes
	40	+ * of the payload which it is currently reading.
	41	+ *
	42	+ * Now we have enough information to calculate the safety margin. We need
	43	+ * - 22 bytes for the .zst file format headers;
	44	+ * - 3 bytes per every 128 KiB of uncompressed size (one block header per
	45	+ * block); and
	46	+ * - 128 KiB (biggest possible zstd block size) to make sure that the
	47	+ * decompressor never overwrites anything from the block it is currently
	48	+ * reading.
	49	+ *
	50	+ * We get the following formula:
	51	+ *
	52	+ * safety_margin = 22 + uncompressed_size * 3 / 131072 + 131072
	53	+ * <= 22 + (uncompressed_size >> 15) + 131072
	54	+ */
	55	+
	56	+/*
	57	+ * Preboot environments #include "path/to/decompress_unzstd.c".
	58	+ * All of the source files we depend on must be #included.
	59	+ * zstd's only source dependeny is xxhash, which has no source
	60	+ * dependencies.
	61	+ *
	62	+ * When UNZSTD_PREBOOT is defined we declare __decompress(), which is
	63	+ * used for kernel decompression, instead of unzstd().
	64	+ *
	65	+ * Define __DISABLE_EXPORTS in preboot environments to prevent symbols
	66	+ * from xxhash and zstd from being exported by the EXPORT_SYMBOL macro.
	67	+ */
	68	+#ifdef STATIC
	69	+# define UNZSTD_PREBOOT
	70	+# include "xxhash.c"
	71	+# include "zstd/entropy_common.c"
	72	+# include "zstd/fse_decompress.c"
	73	+# include "zstd/huf_decompress.c"
	74	+# include "zstd/zstd_common.c"
	75	+# include "zstd/decompress.c"
	76	+#endif
	77	+
	78	+#include <linux/decompress/mm.h>
	79	+#include <linux/kernel.h>
	80	+#include <linux/zstd.h>
	81	+
	82	+/* 128MB is the maximum window size supported by zstd. */
	83	+#define ZSTD_WINDOWSIZE_MAX (1 << ZSTD_WINDOWLOG_MAX)
	84	+/*
	85	+ * Size of the input and output buffers in multi-call mode.
	86	+ * Pick a larger size because it isn't used during kernel decompression,
	87	+ * since that is single pass, and we have to allocate a large buffer for
	88	+ * zstd's window anyway. The larger size speeds up initramfs decompression.
	89	+ */
	90	+#define ZSTD_IOBUF_SIZE (1 << 17)
	91	+
	92	+static int INIT handle_zstd_error(size_t ret, void (error)(char x))
	93	+{
	94	+ const int err = ZSTD_getErrorCode(ret);
	95	+
	96	+ if (!ZSTD_isError(ret))
	97	+ return 0;
	98	+
	99	+ switch (err) {
	100	+ case ZSTD_error_memory_allocation:
	101	+ error("ZSTD decompressor ran out of memory");
	102	+ break;
	103	+ case ZSTD_error_prefix_unknown:
	104	+ error("Input is not in the ZSTD format (wrong magic bytes)");
	105	+ break;
	106	+ case ZSTD_error_dstSize_tooSmall:
	107	+ case ZSTD_error_corruption_detected:
	108	+ case ZSTD_error_checksum_wrong:
	109	+ error("ZSTD-compressed data is corrupt");
	110	+ break;
	111	+ default:
	112	+ error("ZSTD-compressed data is probably corrupt");
	113	+ break;
	114	+ }
	115	+ return -1;
	116	+}
	117	+
	118	+/*
	119	+ * Handle the case where we have the entire input and output in one segment.
	120	+ * We can allocate less memory (no circular buffer for the sliding window),
	121	+ * and avoid some memcpy() calls.
	122	+ */
	123	+static int INIT decompress_single(const u8 in_buf, long in_len, u8 out_buf,
	124	+ long out_len, long *in_pos,
	125	+ void (error)(char x))
	126	+{
	127	+ const size_t wksp_size = ZSTD_DCtxWorkspaceBound();
	128	+ void *wksp = large_malloc(wksp_size);
	129	+ ZSTD_DCtx *dctx = ZSTD_initDCtx(wksp, wksp_size);
	130	+ int err;
	131	+ size_t ret;
	132	+
	133	+ if (dctx == NULL) {
	134	+ error("Out of memory while allocating ZSTD_DCtx");
	135	+ err = -1;
	136	+ goto out;
	137	+ }
	138	+ /*
	139	+ * Find out how large the frame actually is, there may be junk at
	140	+ * the end of the frame that ZSTD_decompressDCtx() can't handle.
	141	+ */
	142	+ ret = ZSTD_findFrameCompressedSize(in_buf, in_len);
	143	+ err = handle_zstd_error(ret, error);
	144	+ if (err)
	145	+ goto out;
	146	+ in_len = (long)ret;
	147	+
	148	+ ret = ZSTD_decompressDCtx(dctx, out_buf, out_len, in_buf, in_len);
	149	+ err = handle_zstd_error(ret, error);
	150	+ if (err)
	151	+ goto out;
	152	+
	153	+ if (in_pos != NULL)
	154	+ *in_pos = in_len;
	155	+
	156	+ err = 0;
	157	+out:
	158	+ if (wksp != NULL)
	159	+ large_free(wksp);
	160	+ return err;
	161	+}
	162	+
	163	+static int INIT __unzstd(unsigned char *in_buf, long in_len,
	164	+ long (fill)(void, unsigned long),
	165	+ long (flush)(void, unsigned long),
	166	+ unsigned char *out_buf, long out_len,
	167	+ long *in_pos,
	168	+ void (error)(char x))
	169	+{
	170	+ ZSTD_inBuffer in;
	171	+ ZSTD_outBuffer out;
	172	+ ZSTD_frameParams params;
	173	+ void *in_allocated = NULL;
	174	+ void *out_allocated = NULL;
	175	+ void *wksp = NULL;
	176	+ size_t wksp_size;
	177	+ ZSTD_DStream *dstream;
	178	+ int err;
	179	+ size_t ret;
	180	+
	181	+ if (out_len == 0)
	182	+ out_len = LONG_MAX; /* no limit */
	183	+
	184	+ if (fill == NULL && flush == NULL)
	185	+ /*
	186	+ * We can decompress faster and with less memory when we have a
	187	+ * single chunk.
	188	+ */
	189	+ return decompress_single(in_buf, in_len, out_buf, out_len,
	190	+ in_pos, error);
	191	+
	192	+ /*
	193	+ * If in_buf is not provided, we must be using fill(), so allocate
	194	+ * a large enough buffer. If it is provided, it must be at least
	195	+ * ZSTD_IOBUF_SIZE large.
	196	+ */
	197	+ if (in_buf == NULL) {
	198	+ in_allocated = large_malloc(ZSTD_IOBUF_SIZE);
	199	+ if (in_allocated == NULL) {
	200	+ error("Out of memory while allocating input buffer");
	201	+ err = -1;
	202	+ goto out;
	203	+ }
	204	+ in_buf = in_allocated;
	205	+ in_len = 0;
	206	+ }
	207	+ /* Read the first chunk, since we need to decode the frame header. */
	208	+ if (fill != NULL)
	209	+ in_len = fill(in_buf, ZSTD_IOBUF_SIZE);
	210	+ if (in_len < 0) {
	211	+ error("ZSTD-compressed data is truncated");
	212	+ err = -1;
	213	+ goto out;
	214	+ }
	215	+ /* Set the first non-empty input buffer. */
	216	+ in.src = in_buf;
	217	+ in.pos = 0;
	218	+ in.size = in_len;
	219	+ /* Allocate the output buffer if we are using flush(). */
	220	+ if (flush != NULL) {
	221	+ out_allocated = large_malloc(ZSTD_IOBUF_SIZE);
	222	+ if (out_allocated == NULL) {
	223	+ error("Out of memory while allocating output buffer");
	224	+ err = -1;
	225	+ goto out;
	226	+ }
	227	+ out_buf = out_allocated;
	228	+ out_len = ZSTD_IOBUF_SIZE;
	229	+ }
	230	+ /* Set the output buffer. */
	231	+ out.dst = out_buf;
	232	+ out.pos = 0;
	233	+ out.size = out_len;
	234	+
	235	+ /*
	236	+ * We need to know the window size to allocate the ZSTD_DStream.
	237	+ * Since we are streaming, we need to allocate a buffer for the sliding
	238	+ * window. The window size varies from 1 KB to ZSTD_WINDOWSIZE_MAX
	239	+ * (8 MB), so it is important to use the actual value so as not to
	240	+ * waste memory when it is smaller.
	241	+ */
	242	+ ret = ZSTD_getFrameParams(&params, in.src, in.size);
	243	+ err = handle_zstd_error(ret, error);
	244	+ if (err)
	245	+ goto out;
	246	+ if (ret != 0) {
	247	+ error("ZSTD-compressed data has an incomplete frame header");
	248	+ err = -1;
	249	+ goto out;
	250	+ }
	251	+ if (params.windowSize > ZSTD_WINDOWSIZE_MAX) {
	252	+ error("ZSTD-compressed data has too large a window size");
	253	+ err = -1;
	254	+ goto out;
	255	+ }
	256	+
	257	+ /*
	258	+ * Allocate the ZSTD_DStream now that we know how much memory is
	259	+ * required.
	260	+ */
	261	+ wksp_size = ZSTD_DStreamWorkspaceBound(params.windowSize);
	262	+ wksp = large_malloc(wksp_size);
	263	+ dstream = ZSTD_initDStream(params.windowSize, wksp, wksp_size);
	264	+ if (dstream == NULL) {
	265	+ error("Out of memory while allocating ZSTD_DStream");
	266	+ err = -1;
	267	+ goto out;
	268	+ }
	269	+
	270	+ /*
	271	+ * Decompression loop:
	272	+ * Read more data if necessary (error if no more data can be read).
	273	+ * Call the decompression function, which returns 0 when finished.
	274	+ * Flush any data produced if using flush().
	275	+ */
	276	+ if (in_pos != NULL)
	277	+ *in_pos = 0;
	278	+ do {
	279	+ /*
	280	+ * If we need to reload data, either we have fill() and can
	281	+ * try to get more data, or we don't and the input is truncated.
	282	+ */
	283	+ if (in.pos == in.size) {
	284	+ if (in_pos != NULL)
	285	+ *in_pos += in.pos;
	286	+ in_len = fill ? fill(in_buf, ZSTD_IOBUF_SIZE) : -1;
	287	+ if (in_len < 0) {
	288	+ error("ZSTD-compressed data is truncated");
	289	+ err = -1;
	290	+ goto out;
	291	+ }
	292	+ in.pos = 0;
	293	+ in.size = in_len;
	294	+ }
	295	+ /* Returns zero when the frame is complete. */
	296	+ ret = ZSTD_decompressStream(dstream, &out, &in);
	297	+ err = handle_zstd_error(ret, error);
	298	+ if (err)
	299	+ goto out;
	300	+ /* Flush all of the data produced if using flush(). */
	301	+ if (flush != NULL && out.pos > 0) {
	302	+ if (out.pos != flush(out.dst, out.pos)) {
	303	+ error("Failed to flush()");
	304	+ err = -1;
	305	+ goto out;
	306	+ }
	307	+ out.pos = 0;
	308	+ }
	309	+ } while (ret != 0);
	310	+
	311	+ if (in_pos != NULL)
	312	+ *in_pos += in.pos;
	313	+
	314	+ err = 0;
	315	+out:
	316	+ if (in_allocated != NULL)
	317	+ large_free(in_allocated);
	318	+ if (out_allocated != NULL)
	319	+ large_free(out_allocated);
	320	+ if (wksp != NULL)
	321	+ large_free(wksp);
	322	+ return err;
	323	+}
	324	+
	325	+#ifndef UNZSTD_PREBOOT
	326	+STATIC int INIT unzstd(unsigned char *buf, long len,
	327	+ long (fill)(void, unsigned long),
	328	+ long (flush)(void, unsigned long),
	329	+ unsigned char *out_buf,
	330	+ long *pos,
	331	+ void (error)(char x))
	332	+{
	333	+ return __unzstd(buf, len, fill, flush, out_buf, 0, pos, error);
	334	+}
	335	+#else
	336	+STATIC int INIT __decompress(unsigned char *buf, long len,
	337	+ long (fill)(void, unsigned long),
	338	+ long (flush)(void, unsigned long),
	339	+ unsigned char *out_buf, long out_len,
	340	+ long *pos,
	341	+ void (error)(char x))
	342	+{
	343	+ return __unzstd(buf, len, fill, flush, out_buf, out_len, pos, error);
	344	+}
	345	+#endif

lib/zstd/fse_decompress.c+1−8

@@ -47,6 +47,7 @@ · 46 unmodified lines
47	47	****************************************************************/
48	48	#include "bitstream.h"
49	49	#include "fse.h"
	50	+#include "zstd_internal.h"
50	51	#include <linux/compiler.h>
51	52	#include <linux/kernel.h>
52	53	#include <linux/string.h> /* memcpy, memset */
@@ -60,14 +61,6 @@ · 7 unmodified lines
60	61	enum { FSE_static_assert = 1 / (int)(!!(c)) }; \
61	62	} /* use only after variable declarations */
62	63
63		−/* check and forward error code */
64		−#define CHECK_F(f) \
65		− { \
66		− size_t const e = f; \
67		− if (FSE_isError(e)) \
68		− return e; \
69		− }
70		−
71	64	/* **************************************************************
72	65	* Templates
73	66	****************************************************************/

lib/zstd/zstd_internal.h+12−2

@@ -127,7 +127,14 @@ static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; · 126 unmodified lines
127	127	* Shared functions to include for inlining
128	128	*********************************************/
129	129	ZSTD_STATIC void ZSTD_copy8(void dst, const void src) {
130		− memcpy(dst, src, 8);
	130	+ /*
	131	+ * zstd relies heavily on gcc being able to analyze and inline this
	132	+ * memcpy() call, since it is called in a tight loop. Preboot mode
	133	+ * is compiled in freestanding mode, which stops gcc from analyzing
	134	+ * memcpy(). Use __builtin_memcpy() to tell gcc to analyze this as a
	135	+ * regular memcpy().
	136	+ */
	137	+ __builtin_memcpy(dst, src, 8);
131	138	}
132	139	/*! ZSTD_wildcopy() :
133	140	* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
@@ -137,13 +144,16 @@ ZSTD_STATIC void ZSTD_wildcopy(void dst, const void src, ptrdiff_t length) · 3 unmodified lines
137	144	const BYTE* ip = (const BYTE*)src;
138	145	BYTE* op = (BYTE*)dst;
139	146	BYTE* const oend = op + length;
140		− /* Work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388.
	147	+#if defined(GCC_VERSION) && GCC_VERSION >= 70000 && GCC_VERSION < 70200
	148	+ /*
	149	+ * Work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388.
141	150	* Avoid the bad case where the loop only runs once by handling the
142	151	* special case separately. This doesn't trigger the bug because it
143	152	* doesn't involve pointer/integer overflow.
144	153	*/
145	154	if (length <= 8)
146	155	return ZSTD_copy8(dst, src);
	156	+#endif
147	157	do {
148	158	ZSTD_copy8(op, ip);
149	159	op += 8;

scripts/Makefile.lib+22−0

@@ -413,6 +413,28 @@ quiet_cmd_xzkern = XZKERN $@ · 412 unmodified lines
413	413	quiet_cmd_xzmisc = XZMISC $@
414	414	cmd_xzmisc = cat $(real-prereqs) \| $(XZ) --check=crc32 --lzma2=dict=1MiB > $@
415	415
	416	+# ZSTD
	417	+# ---------------------------------------------------------------------------
	418	+# Appends the uncompressed size of the data using size_append. The .zst
	419	+# format has the size information available at the beginning of the file too,
	420	+# but it's in a more complex format and it's good to avoid changing the part
	421	+# of the boot code that reads the uncompressed size.
	422	+#
	423	+# Note that the bytes added by size_append will make the zstd tool think that
	424	+# the file is corrupt. This is expected.
	425	+#
	426	+# zstd uses a maximum window size of 8 MB. zstd22 uses a maximum window size of
	427	+# 128 MB. zstd22 is used for kernel compression because it is decompressed in a
	428	+# single pass, so zstd doesn't need to allocate a window buffer. When streaming
	429	+# decompression is used, like initramfs decompression, zstd22 should likely not
	430	+# be used because it would require zstd to allocate a 128 MB buffer.
	431	+
	432	+quiet_cmd_zstd = ZSTD $@
	433	+ cmd_zstd = { cat $(real-prereqs) \| $(ZSTD) -19; $(size_append); } > $@
	434	+
	435	+quiet_cmd_zstd22 = ZSTD22 $@
	436	+ cmd_zstd22 = { cat $(real-prereqs) \| $(ZSTD) -22 --ultra; $(size_append); } > $@
	437	+
416	438	# ASM offsets
417	439	# ---------------------------------------------------------------------------
418	440

usr/Kconfig+20−0

@@ -100,6 +100,15 @@ config RD_LZ4 · 99 unmodified lines
100	100	Support loading of a LZ4 encoded initial ramdisk or cpio buffer
101	101	If unsure, say N.
102	102
	103	+config RD_ZSTD
	104	+ bool "Support initial ramdisk/ramfs compressed using ZSTD"
	105	+ default y
	106	+ depends on BLK_DEV_INITRD
	107	+ select DECOMPRESS_ZSTD
	108	+ help
	109	+ Support loading of a ZSTD encoded initial ramdisk or cpio buffer.
	110	+ If unsure, say N.
	111	+
103	112	choice
104	113	prompt "Built-in initramfs compression mode"
105	114	depends on INITRAMFS_SOURCE != ""
@@ -196,6 +205,17 @@ config INITRAMFS_COMPRESSION_LZ4 · 90 unmodified lines
196	205	If you choose this, keep in mind that most distros don't provide lz4
197	206	by default which could cause a build failure.
198	207
	208	+config INITRAMFS_COMPRESSION_ZSTD
	209	+ bool "ZSTD"
	210	+ depends on RD_ZSTD
	211	+ help
	212	+ ZSTD is a compression algorithm targeting intermediate compression
	213	+ with fast decompression speed. It will compress better than GZIP and
	214	+ decompress around the same speed as LZO, but slower than LZ4.
	215	+
	216	+ If you choose this, keep in mind that you may need to install the zstd
	217	+ tool to be able to compress the initram.
	218	+
199	219	config INITRAMFS_COMPRESSION_NONE
200	220	bool "None"
201	221	help

usr/Makefile+1−0

@@ -15,6 +15,7 @@ compress-$(CONFIG_INITRAMFS_COMPRESSION_LZMA) := lzma · 14 unmodified lines
15	15	compress-$(CONFIG_INITRAMFS_COMPRESSION_XZ) := xzmisc
16	16	compress-$(CONFIG_INITRAMFS_COMPRESSION_LZO) := lzo
17	17	compress-$(CONFIG_INITRAMFS_COMPRESSION_LZ4) := lz4
	18	+compress-$(CONFIG_INITRAMFS_COMPRESSION_ZSTD) := zstd
18	19
19	20	obj-$(CONFIG_BLK_DEV_INITRD) := initramfs_data.o
20	21