Merge tag 'x86-boot-2020-08-03' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 boot updates from Ingo Molnar: "The main change in this cycle was to add support for ZSTD-compressed kernel and initrd images. ZSTD has a very fast decompressor, yet it compresses better than gzip" * tag 'x86-boot-2020-08-03' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: Documentation: dontdiff: Add zstd compressed files .gitignore: Add ZSTD-compressed files x86: Add support for ZSTD compressed kernel x86: Bump ZO_z_extra_bytes margin for zstd usr: Add support for zstd compressed initramfs init: Add support for zstd compressed kernel lib: Add zstd support to decompress lib: Prepare zstd for preboot environment, improve performance

Linus Torvalds committed 6 years ago

Commit c0dfadf2 parents ba77c56 1ac1efa21 files · +469 −26Browse files

Unified · Split

21 files

.gitignore +1 −0
Documentation/dontdiff +1 −0
Documentation/x86/boot.rst +3 −3
Makefile +2 −1
arch/x86/Kconfig +1 −0
arch/x86/boot/compressed/Makefile +5 −1
arch/x86/boot/compressed/kaslr.c +0 −7
arch/x86/boot/compressed/misc.c +4 −0
arch/x86/boot/header.S +7 −1
arch/x86/include/asm/boot.h +9 −2
include/linux/decompress/unzstd.h +11 −0
init/Kconfig +14 −1
lib/Kconfig +4 −0
lib/Makefile +1 −0
lib/decompress.c +5 −0
lib/decompress_unzstd.c +345 −0
lib/zstd/fse_decompress.c +1 −8
lib/zstd/zstd_internal.h +12 −2
scripts/Makefile.lib +22 −0
usr/Kconfig +20 −0
usr/Makefile +1 −0

.gitignore+1−0

@@ -44,6 +44,7 @@ · 43 unmodified lines
44	*.tab.[ch]	44	*.tab.[ch]
45	*.tar	45	*.tar
46	*.xz	46	*.xz
		47	*.zst
47	Module.symvers	48	Module.symvers
48	modules.builtin	49	modules.builtin
49	modules.order	50	modules.order

Documentation/dontdiff+1−0

@@ -55,6 +55,7 @@ · 54 unmodified lines
55	*.ver	55	*.ver
56	*.xml	56	*.xml
57	*.xz	57	*.xz
		58	*.zst
58	*_MODULES	59	*_MODULES
59	*_vga16.c	60	*_vga16.c
60	*~	61	*~

Documentation/x86/boot.rst+3−3

@@ -782,9 +782,9 @@ Protocol: 2.08+ · 781 unmodified lines
782	uncompressed data should be determined using the standard magic	782	uncompressed data should be determined using the standard magic
783	numbers. The currently supported compression formats are gzip	783	numbers. The currently supported compression formats are gzip
784	(magic numbers 1F 8B or 1F 9E), bzip2 (magic number 42 5A), LZMA	784	(magic numbers 1F 8B or 1F 9E), bzip2 (magic number 42 5A), LZMA
785	(magic number 5D 00), XZ (magic number FD 37), and LZ4 (magic number	785	(magic number 5D 00), XZ (magic number FD 37), LZ4 (magic number
786	02 21). The uncompressed payload is currently always ELF (magic	786	02 21) and ZSTD (magic number 28 B5). The uncompressed payload is
787	number 7F 45 4C 46).	787	currently always ELF (magic number 7F 45 4C 46).
788		788
789	============ ==============	789	============ ==============
790	Field name: payload_length	790	Field name: payload_length

Makefile+2−1

 LZMA		= lzma
 LZ4		= lz4c
 XZ		= xz
+ZSTD		= zstd
 CHECKFLAGS     := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \
 		  -Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF)
 export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC
 export CPP AR NM STRIP OBJCOPY OBJDUMP OBJSIZE READELF PAHOLE LEX YACC AWK INSTALLKERNEL
 export PERL PYTHON PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
-export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ
+export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ ZSTD
 export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE
 export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS

arch/x86/Kconfig+1−0

@@ -188,6 +188,7 @@ config X86 · 187 unmodified lines
188	select HAVE_KERNEL_LZMA	188	select HAVE_KERNEL_LZMA
189	select HAVE_KERNEL_LZO	189	select HAVE_KERNEL_LZO
190	select HAVE_KERNEL_XZ	190	select HAVE_KERNEL_XZ
		191	select HAVE_KERNEL_ZSTD
191	select HAVE_KPROBES	192	select HAVE_KPROBES
192	select HAVE_KPROBES_ON_FTRACE	193	select HAVE_KPROBES_ON_FTRACE
193	select HAVE_FUNCTION_ERROR_INJECTION	194	select HAVE_FUNCTION_ERROR_INJECTION

arch/x86/boot/compressed/Makefile+5−1

 KCOV_INSTRUMENT		:= n
 targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma \
-	vmlinux.bin.xz vmlinux.bin.lzo vmlinux.bin.lz4
+	vmlinux.bin.xz vmlinux.bin.lzo vmlinux.bin.lz4 vmlinux.bin.zst
 KBUILD_CFLAGS := -m$(BITS) -O2
 KBUILD_CFLAGS += -fno-strict-aliasing $(call cc-option, -fPIE, -fPIC)
 KBUILD_CFLAGS += -Wno-pointer-sign
 KBUILD_CFLAGS += $(call cc-option,-fmacro-prefix-map=$(srctree)/=)
 KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
+KBUILD_CFLAGS += -D__DISABLE_EXPORTS
 KBUILD_AFLAGS  := $(KBUILD_CFLAGS) -D__ASSEMBLY__
 GCOV_PROFILE := n
 	$(call if_changed,lzo)
 $(obj)/vmlinux.bin.lz4: $(vmlinux.bin.all-y) FORCE
 	$(call if_changed,lz4)
+$(obj)/vmlinux.bin.zst: $(vmlinux.bin.all-y) FORCE
+	$(call if_changed,zstd22)
 suffix-$(CONFIG_KERNEL_GZIP)	:= gz
 suffix-$(CONFIG_KERNEL_BZIP2)	:= bz2
 suffix-$(CONFIG_KERNEL_XZ)	:= xz
 suffix-$(CONFIG_KERNEL_LZO) 	:= lzo
 suffix-$(CONFIG_KERNEL_LZ4) 	:= lz4
+suffix-$(CONFIG_KERNEL_ZSTD)	:= zst
 quiet_cmd_mkpiggy = MKPIGGY $@
       cmd_mkpiggy = $(obj)/mkpiggy $< > $@

arch/x86/boot/compressed/kaslr.c+0−7

@@ -19,13 +19,6 @@ · 18 unmodified lines
19	*/	19	*/
20	#define BOOT_CTYPE_H	20	#define BOOT_CTYPE_H
21		21
22	/*
23	* _ctype[] in lib/ctype.c is needed by isspace() of linux/ctype.h.
24	* While both lib/ctype.c and lib/cmdline.c will bring EXPORT_SYMBOL
25	* which is meaningless and will cause compiling error in some cases.
26	*/
27	#define __DISABLE_EXPORTS
28
29	#include "misc.h"	22	#include "misc.h"
30	#include "error.h"	23	#include "error.h"
31	#include "../string.h"	24	#include "../string.h"

arch/x86/boot/compressed/misc.c+4−0

 #ifdef CONFIG_KERNEL_LZ4
 #include "../../../../lib/decompress_unlz4.c"
 #endif
+#ifdef CONFIG_KERNEL_ZSTD
+#include "../../../../lib/decompress_unzstd.c"
+#endif
 /*
  * NOTE: When adding a new decompressor, please update the analysis in
  * ../header.S.

arch/x86/boot/header.S+7−1

 # the size-dependent part now grows so fast.
 #
 # extra_bytes = (uncompressed_size >> 8) + 65536
+#
+# ZSTD compressed data grows by at most 3 bytes per 128K, and only has a 22
+# byte fixed overhead but has a maximum block size of 128K, so it needs a
+# larger margin.
+#
+# extra_bytes = (uncompressed_size >> 8) + 131072
-#define ZO_z_extra_bytes	((ZO_z_output_len >> 8) + 65536)
+#define ZO_z_extra_bytes	((ZO_z_output_len >> 8) + 131072)
 #if ZO_z_output_len > ZO_z_input_len
 # define ZO_z_extract_offset	(ZO_z_output_len + ZO_z_extra_bytes - \
 				 ZO_z_input_len)

arch/x86/include/asm/boot.h+9−2

 # error "Invalid value for CONFIG_PHYSICAL_ALIGN"
 #endif
-#ifdef CONFIG_KERNEL_BZIP2
+#if defined(CONFIG_KERNEL_BZIP2)
 # define BOOT_HEAP_SIZE		0x400000
-#else /* !CONFIG_KERNEL_BZIP2 */
+#elif defined(CONFIG_KERNEL_ZSTD)
+/*
+ * Zstd needs to allocate the ZSTD_DCtx in order to decompress the kernel.
+ * The ZSTD_DCtx is ~160KB, so set the heap size to 192KB because it is a
+ * round number and to allow some slack.
+ */
+# define BOOT_HEAP_SIZE		 0x30000
+#else
 # define BOOT_HEAP_SIZE		 0x10000
 #endif

include/linux/decompress/unzstd.h+11−0

@@ -0,0 +1,11 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef LINUX_DECOMPRESS_UNZSTD_H
	3	#define LINUX_DECOMPRESS_UNZSTD_H
	4
	5	int unzstd(unsigned char *inbuf, long len,
	6	long (fill)(void, unsigned long),
	7	long (flush)(void, unsigned long),
	8	unsigned char *output,
	9	long *pos,
	10	void (error_fn)(char x));
	11	#endif

init/Kconfig+14−1

 config HAVE_KERNEL_LZ4
 	bool
+config HAVE_KERNEL_ZSTD
+	bool
 config HAVE_KERNEL_UNCOMPRESSED
 	bool
 choice
 	prompt "Kernel compression mode"
 	default KERNEL_GZIP
-	depends on HAVE_KERNEL_GZIP || HAVE_KERNEL_BZIP2 || HAVE_KERNEL_LZMA || HAVE_KERNEL_XZ || HAVE_KERNEL_LZO || HAVE_KERNEL_LZ4 || HAVE_KERNEL_UNCOMPRESSED
+	depends on HAVE_KERNEL_GZIP || HAVE_KERNEL_BZIP2 || HAVE_KERNEL_LZMA || HAVE_KERNEL_XZ || HAVE_KERNEL_LZO || HAVE_KERNEL_LZ4 || HAVE_KERNEL_ZSTD || HAVE_KERNEL_UNCOMPRESSED
 	help
 	  The linux kernel is a kind of self-extracting executable.
 	  Several compression algorithms are available, which differ
 	  is about 8% bigger than LZO. But the decompression speed is
 	  faster than LZO.
+config KERNEL_ZSTD
+	bool "ZSTD"
+	depends on HAVE_KERNEL_ZSTD
+	help
+	  ZSTD is a compression algorithm targeting intermediate compression
+	  with fast decompression speed. It will compress better than GZIP and
+	  decompress around the same speed as LZO, but slower than LZ4. You
+	  will need at least 192 KB RAM or more for booting. The zstd command
+	  line tool is required for compression.
 config KERNEL_UNCOMPRESSED
 	bool "None"
 	depends on HAVE_KERNEL_UNCOMPRESSED

lib/Kconfig+4−0

 	select LZ4_DECOMPRESS
 	tristate
+config DECOMPRESS_ZSTD
+	select ZSTD_DECOMPRESS
+	tristate
 #
 # Generic allocator support is selected if needed
 #

lib/Makefile+1−0

@@ -170,6 +170,7 @@ lib-$(CONFIG_DECOMPRESS_LZMA) += decompress_unlzma.o · 169 unmodified lines
170	lib-$(CONFIG_DECOMPRESS_XZ) += decompress_unxz.o	170	lib-$(CONFIG_DECOMPRESS_XZ) += decompress_unxz.o
171	lib-$(CONFIG_DECOMPRESS_LZO) += decompress_unlzo.o	171	lib-$(CONFIG_DECOMPRESS_LZO) += decompress_unlzo.o
172	lib-$(CONFIG_DECOMPRESS_LZ4) += decompress_unlz4.o	172	lib-$(CONFIG_DECOMPRESS_LZ4) += decompress_unlz4.o
		173	lib-$(CONFIG_DECOMPRESS_ZSTD) += decompress_unzstd.o
173		174
174	obj-$(CONFIG_TEXTSEARCH) += textsearch.o	175	obj-$(CONFIG_TEXTSEARCH) += textsearch.o
175	obj-$(CONFIG_TEXTSEARCH_KMP) += ts_kmp.o	176	obj-$(CONFIG_TEXTSEARCH_KMP) += ts_kmp.o

lib/decompress.c+5−0

 #include <linux/decompress/inflate.h>
 #include <linux/decompress/unlzo.h>
 #include <linux/decompress/unlz4.h>
+#include <linux/decompress/unzstd.h>
 #include <linux/types.h>
 #include <linux/string.h>
 #ifndef CONFIG_DECOMPRESS_LZ4
 # define unlz4 NULL
 #endif
+#ifndef CONFIG_DECOMPRESS_ZSTD
+# define unzstd NULL
+#endif
 struct compress_format {
 	unsigned char magic[2];
 	{ {0xfd, 0x37}, "xz", unxz },
 	{ {0x89, 0x4c}, "lzo", unlzo },
 	{ {0x02, 0x21}, "lz4", unlz4 },
+	{ {0x28, 0xb5}, "zstd", unzstd },
 	{ {0, 0}, NULL, NULL }
 };

lib/decompress_unzstd.c+345−0

@@ -0,0 +1,345 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	/*
	4	* Important notes about in-place decompression
	5	*
	6	* At least on x86, the kernel is decompressed in place: the compressed data
	7	* is placed to the end of the output buffer, and the decompressor overwrites
	8	* most of the compressed data. There must be enough safety margin to
	9	* guarantee that the write position is always behind the read position.
	10	*
	11	* The safety margin for ZSTD with a 128 KB block size is calculated below.
	12	* Note that the margin with ZSTD is bigger than with GZIP or XZ!
	13	*
	14	* The worst case for in-place decompression is that the beginning of
	15	* the file is compressed extremely well, and the rest of the file is
	16	* uncompressible. Thus, we must look for worst-case expansion when the
	17	* compressor is encoding uncompressible data.
	18	*
	19	* The structure of the .zst file in case of a compresed kernel is as follows.
	20	* Maximum sizes (as bytes) of the fields are in parenthesis.
	21	*
	22	* Frame Header: (18)
	23	* Blocks: (N)
	24	* Checksum: (4)
	25	*
	26	* The frame header and checksum overhead is at most 22 bytes.
	27	*
	28	* ZSTD stores the data in blocks. Each block has a header whose size is
	29	* a 3 bytes. After the block header, there is up to 128 KB of payload.
	30	* The maximum uncompressed size of the payload is 128 KB. The minimum
	31	* uncompressed size of the payload is never less than the payload size
	32	* (excluding the block header).
	33	*
	34	* The assumption, that the uncompressed size of the payload is never
	35	* smaller than the payload itself, is valid only when talking about
	36	* the payload as a whole. It is possible that the payload has parts where
	37	* the decompressor consumes more input than it produces output. Calculating
	38	* the worst case for this would be tricky. Instead of trying to do that,
	39	* let's simply make sure that the decompressor never overwrites any bytes
	40	* of the payload which it is currently reading.
	41	*
	42	* Now we have enough information to calculate the safety margin. We need
	43	* - 22 bytes for the .zst file format headers;
	44	* - 3 bytes per every 128 KiB of uncompressed size (one block header per
	45	* block); and
	46	* - 128 KiB (biggest possible zstd block size) to make sure that the
	47	* decompressor never overwrites anything from the block it is currently
	48	* reading.
	49	*
	50	* We get the following formula:
	51	*
	52	* safety_margin = 22 + uncompressed_size * 3 / 131072 + 131072
	53	* <= 22 + (uncompressed_size >> 15) + 131072
	54	*/
	55
	56	/*
	57	* Preboot environments #include "path/to/decompress_unzstd.c".
	58	* All of the source files we depend on must be #included.
	59	* zstd's only source dependeny is xxhash, which has no source
	60	* dependencies.
	61	*
	62	* When UNZSTD_PREBOOT is defined we declare __decompress(), which is
	63	* used for kernel decompression, instead of unzstd().
	64	*
	65	* Define __DISABLE_EXPORTS in preboot environments to prevent symbols
	66	* from xxhash and zstd from being exported by the EXPORT_SYMBOL macro.
	67	*/
	68	#ifdef STATIC
	69	# define UNZSTD_PREBOOT
	70	# include "xxhash.c"
	71	# include "zstd/entropy_common.c"
	72	# include "zstd/fse_decompress.c"
	73	# include "zstd/huf_decompress.c"
	74	# include "zstd/zstd_common.c"
	75	# include "zstd/decompress.c"
	76	#endif
	77
	78	#include <linux/decompress/mm.h>
	79	#include <linux/kernel.h>
	80	#include <linux/zstd.h>
	81
	82	/* 128MB is the maximum window size supported by zstd. */
	83	#define ZSTD_WINDOWSIZE_MAX (1 << ZSTD_WINDOWLOG_MAX)
	84	/*
	85	* Size of the input and output buffers in multi-call mode.
	86	* Pick a larger size because it isn't used during kernel decompression,
	87	* since that is single pass, and we have to allocate a large buffer for
	88	* zstd's window anyway. The larger size speeds up initramfs decompression.
	89	*/
	90	#define ZSTD_IOBUF_SIZE (1 << 17)
	91
	92	static int INIT handle_zstd_error(size_t ret, void (error)(char x))
	93	{
	94	const int err = ZSTD_getErrorCode(ret);
	95
	96	if (!ZSTD_isError(ret))
	97	return 0;
	98
	99	switch (err) {
	100	case ZSTD_error_memory_allocation:
	101	error("ZSTD decompressor ran out of memory");
	102	break;
	103	case ZSTD_error_prefix_unknown:
	104	error("Input is not in the ZSTD format (wrong magic bytes)");
	105	break;
	106	case ZSTD_error_dstSize_tooSmall:
	107	case ZSTD_error_corruption_detected:
	108	case ZSTD_error_checksum_wrong:
	109	error("ZSTD-compressed data is corrupt");
	110	break;
	111	default:
	112	error("ZSTD-compressed data is probably corrupt");
	113	break;
	114	}
	115	return -1;
	116	}
	117
	118	/*
	119	* Handle the case where we have the entire input and output in one segment.
	120	* We can allocate less memory (no circular buffer for the sliding window),
	121	* and avoid some memcpy() calls.
	122	*/
	123	static int INIT decompress_single(const u8 in_buf, long in_len, u8 out_buf,
	124	long out_len, long *in_pos,
	125	void (error)(char x))
	126	{
	127	const size_t wksp_size = ZSTD_DCtxWorkspaceBound();
	128	void *wksp = large_malloc(wksp_size);
	129	ZSTD_DCtx *dctx = ZSTD_initDCtx(wksp, wksp_size);
	130	int err;
	131	size_t ret;
	132
	133	if (dctx == NULL) {
	134	error("Out of memory while allocating ZSTD_DCtx");
	135	err = -1;
	136	goto out;
	137	}
	138	/*
	139	* Find out how large the frame actually is, there may be junk at
	140	* the end of the frame that ZSTD_decompressDCtx() can't handle.
	141	*/
	142	ret = ZSTD_findFrameCompressedSize(in_buf, in_len);
	143	err = handle_zstd_error(ret, error);
	144	if (err)
	145	goto out;
	146	in_len = (long)ret;
	147
	148	ret = ZSTD_decompressDCtx(dctx, out_buf, out_len, in_buf, in_len);
	149	err = handle_zstd_error(ret, error);
	150	if (err)
	151	goto out;
	152
	153	if (in_pos != NULL)
	154	*in_pos = in_len;
	155
	156	err = 0;
	157	out:
	158	if (wksp != NULL)
	159	large_free(wksp);
	160	return err;
	161	}
	162
	163	static int INIT __unzstd(unsigned char *in_buf, long in_len,
	164	long (fill)(void, unsigned long),
	165	long (flush)(void, unsigned long),
	166	unsigned char *out_buf, long out_len,
	167	long *in_pos,
	168	void (error)(char x))
	169	{
	170	ZSTD_inBuffer in;
	171	ZSTD_outBuffer out;
	172	ZSTD_frameParams params;
	173	void *in_allocated = NULL;
	174	void *out_allocated = NULL;
	175	void *wksp = NULL;
	176	size_t wksp_size;
	177	ZSTD_DStream *dstream;
	178	int err;
	179	size_t ret;
	180
	181	if (out_len == 0)
	182	out_len = LONG_MAX; /* no limit */
	183
	184	if (fill == NULL && flush == NULL)
	185	/*
	186	* We can decompress faster and with less memory when we have a
	187	* single chunk.
	188	*/
	189	return decompress_single(in_buf, in_len, out_buf, out_len,
	190	in_pos, error);
	191
	192	/*
	193	* If in_buf is not provided, we must be using fill(), so allocate
	194	* a large enough buffer. If it is provided, it must be at least
	195	* ZSTD_IOBUF_SIZE large.
	196	*/
	197	if (in_buf == NULL) {
	198	in_allocated = large_malloc(ZSTD_IOBUF_SIZE);
	199	if (in_allocated == NULL) {
	200	error("Out of memory while allocating input buffer");
	201	err = -1;
	202	goto out;
	203	}
	204	in_buf = in_allocated;
	205	in_len = 0;
	206	}
	207	/* Read the first chunk, since we need to decode the frame header. */
	208	if (fill != NULL)
	209	in_len = fill(in_buf, ZSTD_IOBUF_SIZE);
	210	if (in_len < 0) {
	211	error("ZSTD-compressed data is truncated");
	212	err = -1;
	213	goto out;
	214	}
	215	/* Set the first non-empty input buffer. */
	216	in.src = in_buf;
	217	in.pos = 0;
	218	in.size = in_len;
	219	/* Allocate the output buffer if we are using flush(). */
	220	if (flush != NULL) {
	221	out_allocated = large_malloc(ZSTD_IOBUF_SIZE);
	222	if (out_allocated == NULL) {
	223	error("Out of memory while allocating output buffer");
	224	err = -1;
	225	goto out;
	226	}
	227	out_buf = out_allocated;
	228	out_len = ZSTD_IOBUF_SIZE;
	229	}
	230	/* Set the output buffer. */
	231	out.dst = out_buf;
	232	out.pos = 0;
	233	out.size = out_len;
	234
	235	/*
	236	* We need to know the window size to allocate the ZSTD_DStream.
	237	* Since we are streaming, we need to allocate a buffer for the sliding
	238	* window. The window size varies from 1 KB to ZSTD_WINDOWSIZE_MAX
	239	* (8 MB), so it is important to use the actual value so as not to
	240	* waste memory when it is smaller.
	241	*/
	242	ret = ZSTD_getFrameParams(&params, in.src, in.size);
	243	err = handle_zstd_error(ret, error);
	244	if (err)
	245	goto out;
	246	if (ret != 0) {
	247	error("ZSTD-compressed data has an incomplete frame header");
	248	err = -1;
	249	goto out;
	250	}
	251	if (params.windowSize > ZSTD_WINDOWSIZE_MAX) {
	252	error("ZSTD-compressed data has too large a window size");
	253	err = -1;
	254	goto out;
	255	}
	256
	257	/*
	258	* Allocate the ZSTD_DStream now that we know how much memory is
	259	* required.
	260	*/
	261	wksp_size = ZSTD_DStreamWorkspaceBound(params.windowSize);
	262	wksp = large_malloc(wksp_size);
	263	dstream = ZSTD_initDStream(params.windowSize, wksp, wksp_size);
	264	if (dstream == NULL) {
	265	error("Out of memory while allocating ZSTD_DStream");
	266	err = -1;
	267	goto out;
	268	}
	269
	270	/*
	271	* Decompression loop:
	272	* Read more data if necessary (error if no more data can be read).
	273	* Call the decompression function, which returns 0 when finished.
	274	* Flush any data produced if using flush().
	275	*/
	276	if (in_pos != NULL)
	277	*in_pos = 0;
	278	do {
	279	/*
	280	* If we need to reload data, either we have fill() and can
	281	* try to get more data, or we don't and the input is truncated.
	282	*/
	283	if (in.pos == in.size) {
	284	if (in_pos != NULL)
	285	*in_pos += in.pos;
	286	in_len = fill ? fill(in_buf, ZSTD_IOBUF_SIZE) : -1;
	287	if (in_len < 0) {
	288	error("ZSTD-compressed data is truncated");
	289	err = -1;
	290	goto out;
	291	}
	292	in.pos = 0;
	293	in.size = in_len;
	294	}
	295	/* Returns zero when the frame is complete. */
	296	ret = ZSTD_decompressStream(dstream, &out, &in);
	297	err = handle_zstd_error(ret, error);
	298	if (err)
	299	goto out;
	300	/* Flush all of the data produced if using flush(). */
	301	if (flush != NULL && out.pos > 0) {
	302	if (out.pos != flush(out.dst, out.pos)) {
	303	error("Failed to flush()");
	304	err = -1;
	305	goto out;
	306	}
	307	out.pos = 0;
	308	}
	309	} while (ret != 0);
	310
	311	if (in_pos != NULL)
	312	*in_pos += in.pos;
	313
	314	err = 0;
	315	out:
	316	if (in_allocated != NULL)
	317	large_free(in_allocated);
	318	if (out_allocated != NULL)
	319	large_free(out_allocated);
	320	if (wksp != NULL)
	321	large_free(wksp);
	322	return err;
	323	}
	324
	325	#ifndef UNZSTD_PREBOOT
	326	STATIC int INIT unzstd(unsigned char *buf, long len,
	327	long (fill)(void, unsigned long),
	328	long (flush)(void, unsigned long),
	329	unsigned char *out_buf,
	330	long *pos,
	331	void (error)(char x))
	332	{
	333	return __unzstd(buf, len, fill, flush, out_buf, 0, pos, error);
	334	}
	335	#else
	336	STATIC int INIT __decompress(unsigned char *buf, long len,
	337	long (fill)(void, unsigned long),
	338	long (flush)(void, unsigned long),
	339	unsigned char *out_buf, long out_len,
	340	long *pos,
	341	void (error)(char x))
	342	{
	343	return __unzstd(buf, len, fill, flush, out_buf, out_len, pos, error);
	344	}
	345	#endif

lib/zstd/fse_decompress.c+1−8

 ****************************************************************/
 #include "bitstream.h"
 #include "fse.h"
+#include "zstd_internal.h"
 #include <linux/compiler.h>
 #include <linux/kernel.h>
 #include <linux/string.h> /* memcpy, memset */
 		enum { FSE_static_assert = 1 / (int)(!!(c)) }; \
 	} /* use only *after* variable declarations */
-/* check and forward error code */
-#define CHECK_F(f)                  \
-	{                           \
-		size_t const e = f; \
-		if (FSE_isError(e)) \
-			return e;   \
-	}
 /* **************************************************************
 *  Templates
 ****************************************************************/

lib/zstd/zstd_internal.h+12−2

 *  Shared functions to include for inlining
 *********************************************/
 ZSTD_STATIC void ZSTD_copy8(void *dst, const void *src) {
-	memcpy(dst, src, 8);
+	/*
+	 * zstd relies heavily on gcc being able to analyze and inline this
+	 * memcpy() call, since it is called in a tight loop. Preboot mode
+	 * is compiled in freestanding mode, which stops gcc from analyzing
+	 * memcpy(). Use __builtin_memcpy() to tell gcc to analyze this as a
+	 * regular memcpy().
+	 */
+	__builtin_memcpy(dst, src, 8);
 }
 /*! ZSTD_wildcopy() :
 *   custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
 	const BYTE* ip = (const BYTE*)src;
 	BYTE* op = (BYTE*)dst;
 	BYTE* const oend = op + length;
-	/* Work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388.
+#if defined(GCC_VERSION) && GCC_VERSION >= 70000 && GCC_VERSION < 70200
+	/*
+	 * Work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388.
 	 * Avoid the bad case where the loop only runs once by handling the
 	 * special case separately. This doesn't trigger the bug because it
 	 * doesn't involve pointer/integer overflow.
 	 */
 	if (length <= 8)
 		return ZSTD_copy8(dst, src);
+#endif
 	do {
 		ZSTD_copy8(op, ip);
 		op += 8;

scripts/Makefile.lib+22−0

@@ -413,6 +413,28 @@ quiet_cmd_xzkern = XZKERN $@ · 412 unmodified lines
413	quiet_cmd_xzmisc = XZMISC $@	413	quiet_cmd_xzmisc = XZMISC $@
414	cmd_xzmisc = cat $(real-prereqs) \| $(XZ) --check=crc32 --lzma2=dict=1MiB > $@	414	cmd_xzmisc = cat $(real-prereqs) \| $(XZ) --check=crc32 --lzma2=dict=1MiB > $@
415		415
		416	# ZSTD
		417	# ---------------------------------------------------------------------------
		418	# Appends the uncompressed size of the data using size_append. The .zst
		419	# format has the size information available at the beginning of the file too,
		420	# but it's in a more complex format and it's good to avoid changing the part
		421	# of the boot code that reads the uncompressed size.
		422	#
		423	# Note that the bytes added by size_append will make the zstd tool think that
		424	# the file is corrupt. This is expected.
		425	#
		426	# zstd uses a maximum window size of 8 MB. zstd22 uses a maximum window size of
		427	# 128 MB. zstd22 is used for kernel compression because it is decompressed in a
		428	# single pass, so zstd doesn't need to allocate a window buffer. When streaming
		429	# decompression is used, like initramfs decompression, zstd22 should likely not
		430	# be used because it would require zstd to allocate a 128 MB buffer.
		431
		432	quiet_cmd_zstd = ZSTD $@
		433	cmd_zstd = { cat $(real-prereqs) \| $(ZSTD) -19; $(size_append); } > $@
		434
		435	quiet_cmd_zstd22 = ZSTD22 $@
		436	cmd_zstd22 = { cat $(real-prereqs) \| $(ZSTD) -22 --ultra; $(size_append); } > $@
		437
416	# ASM offsets	438	# ASM offsets
417	# ---------------------------------------------------------------------------	439	# ---------------------------------------------------------------------------
418		440

usr/Kconfig+20−0

@@ -100,6 +100,15 @@ config RD_LZ4 · 99 unmodified lines
100	Support loading of a LZ4 encoded initial ramdisk or cpio buffer	100	Support loading of a LZ4 encoded initial ramdisk or cpio buffer
101	If unsure, say N.	101	If unsure, say N.
102		102
		103	config RD_ZSTD
		104	bool "Support initial ramdisk/ramfs compressed using ZSTD"
		105	default y
		106	depends on BLK_DEV_INITRD
		107	select DECOMPRESS_ZSTD
		108	help
		109	Support loading of a ZSTD encoded initial ramdisk or cpio buffer.
		110	If unsure, say N.
		111
103	choice	112	choice
104	prompt "Built-in initramfs compression mode"	113	prompt "Built-in initramfs compression mode"
105	depends on INITRAMFS_SOURCE != ""	114	depends on INITRAMFS_SOURCE != ""
@@ -196,6 +205,17 @@ config INITRAMFS_COMPRESSION_LZ4 · 90 unmodified lines
196	If you choose this, keep in mind that most distros don't provide lz4	205	If you choose this, keep in mind that most distros don't provide lz4
197	by default which could cause a build failure.	206	by default which could cause a build failure.
198		207
		208	config INITRAMFS_COMPRESSION_ZSTD
		209	bool "ZSTD"
		210	depends on RD_ZSTD
		211	help
		212	ZSTD is a compression algorithm targeting intermediate compression
		213	with fast decompression speed. It will compress better than GZIP and
		214	decompress around the same speed as LZO, but slower than LZ4.
		215
		216	If you choose this, keep in mind that you may need to install the zstd
		217	tool to be able to compress the initram.
		218
199	config INITRAMFS_COMPRESSION_NONE	219	config INITRAMFS_COMPRESSION_NONE
200	bool "None"	220	bool "None"
201	help	221	help

usr/Makefile+1−0

@@ -15,6 +15,7 @@ compress-$(CONFIG_INITRAMFS_COMPRESSION_LZMA) := lzma · 14 unmodified lines
15	compress-$(CONFIG_INITRAMFS_COMPRESSION_XZ) := xzmisc	15	compress-$(CONFIG_INITRAMFS_COMPRESSION_XZ) := xzmisc
16	compress-$(CONFIG_INITRAMFS_COMPRESSION_LZO) := lzo	16	compress-$(CONFIG_INITRAMFS_COMPRESSION_LZO) := lzo
17	compress-$(CONFIG_INITRAMFS_COMPRESSION_LZ4) := lz4	17	compress-$(CONFIG_INITRAMFS_COMPRESSION_LZ4) := lz4
		18	compress-$(CONFIG_INITRAMFS_COMPRESSION_ZSTD) := zstd
18		19
19	obj-$(CONFIG_BLK_DEV_INITRD) := initramfs_data.o	20	obj-$(CONFIG_BLK_DEV_INITRD) := initramfs_data.o
20		21