Explode zstd 1.4.5 into separate upstream files

It's much nicer to import from upstream this way, and compiles faster too. Everything in lib/ is unmodified 1.4.5. Reviewed-by: Brian Behlendorf <[email protected]> Signed-off-by: Rich Ercolani <[email protected]> Closes openzfs#12978
andrewc12 · Sep 23, 2022 · 2617a0d · 2617a0d
1 parent 674f1a2
commit 2617a0d
Show file tree

Hide file tree

Showing 58 changed files with 23,917 additions and 27,895 deletions.
diff --git a/lib/libzstd/Makefile.am b/lib/libzstd/Makefile.am
@@ -8,14 +8,33 @@ AM_CFLAGS += -fno-tree-vectorize
 
 noinst_LTLIBRARIES = libzstd.la
 
-KERNEL_C = \
-	lib/zstd.c \
+KERNEL_C = lib/common/entropy_common.c \
+	lib/common/error_private.c \
+	lib/common/fse_decompress.c \
+	lib/common/pool.c \
+	lib/common/zstd_common.c \
+	lib/compress/fse_compress.c \
+	lib/compress/hist.c \
+	lib/compress/huf_compress.c \
+	lib/compress/zstd_compress_literals.c \
+	lib/compress/zstd_compress_sequences.c \
+	lib/compress/zstd_compress_superblock.c \
+	lib/compress/zstd_compress.c \
+	lib/compress/zstd_double_fast.c \
+	lib/compress/zstd_fast.c \
+	lib/compress/zstd_lazy.c \
+	lib/compress/zstd_ldm.c \
+	lib/compress/zstd_opt.c \
+	lib/decompress/huf_decompress.c \
+	lib/decompress/zstd_ddict.c \
+	lib/decompress/zstd_decompress.c \
+	lib/decompress/zstd_decompress_block.c \
 	zfs_zstd.c
 
 nodist_libzstd_la_SOURCES = $(KERNEL_C)
 
-lib/zstd.$(OBJEXT):  CFLAGS += -fno-tree-vectorize -include $(top_srcdir)/module/zstd/include/zstd_compat_wrapper.h -Wp,-w
-lib/zstd.l$(OBJEXT): CFLAGS += -fno-tree-vectorize -include $(top_srcdir)/module/zstd/include/zstd_compat_wrapper.h -Wp,-w
+%.$(OBJEXT):  CFLAGS += -fno-tree-vectorize -include $(top_srcdir)/module/zstd/include/zstd_compat_wrapper.h -Wp,-w
+%.l$(OBJEXT): CFLAGS += -fno-tree-vectorize -include $(top_srcdir)/module/zstd/include/zstd_compat_wrapper.h -Wp,-w
 
 zfs_zstd.$(OBJEXT):  CFLAGS += -include $(top_srcdir)/module/zstd/include/zstd_compat_wrapper.h
 zfs_zstd.l$(OBJEXT): CFLAGS += -include $(top_srcdir)/module/zstd/include/zstd_compat_wrapper.h

diff --git a/module/Makefile.bsd b/module/Makefile.bsd
@@ -19,7 +19,9 @@ KMOD=	openzfs
 	${SRCDIR}/zcommon \
 	${SRCDIR}/zfs \
 	${SRCDIR}/zstd \
-	${SRCDIR}/zstd/lib
+	${SRCDIR}/zstd/lib/common \
+	${SRCDIR}/zstd/lib/compress \
+	${SRCDIR}/zstd/lib/decompress
 
 
 
@@ -313,7 +315,29 @@ SRCS+=	abd.c \
 
 #zstd
 SRCS+=	zfs_zstd.c \
-	zstd.c
+	entropy_common.c \
+	error_private.c \
+	fse_decompress.c \
+	pool.c \
+	zstd_common.c \
+	fse_compress.c \
+	hist.c \
+	huf_compress.c \
+	zstd_compress.c \
+	zstd_compress_literals.c \
+	zstd_compress_sequences.c \
+	zstd_compress_superblock.c \
+	zstd_double_fast.c \
+	zstd_fast.c \
+	zstd_lazy.c \
+	zstd_ldm.c \
+	zstd_opt.c \
+	huf_decompress.c \
+	zstd_ddict.c \
+	zstd_decompress.c \
+	zstd_decompress_block.c
+
+
 
 beforeinstall:
 .if ${MK_DEBUG_FILES} != "no"
@@ -373,4 +397,25 @@ CFLAGS.zil.c= -Wno-cast-qual
 CFLAGS.zio.c= -Wno-cast-qual
 CFLAGS.zrlock.c= -Wno-cast-qual
 CFLAGS.zfs_zstd.c= -Wno-cast-qual -Wno-pointer-arith
-CFLAGS.zstd.c= -fno-tree-vectorize -U__BMI__
+CFLAGS.entropy_common.c= -fno-tree-vectorize -U__BMI__
+CFLAGS.error_private.c= -fno-tree-vectorize -U__BMI__
+CFLAGS.fse_decompress.c= -fno-tree-vectorize -U__BMI__
+CFLAGS.pool.c= -fno-tree-vectorize -U__BMI__
+CFLAGS.xxhash.c= -fno-tree-vectorize -U__BMI__
+CFLAGS.zstd_common.c= -fno-tree-vectorize -U__BMI__
+CFLAGS.fse_compress.c= -fno-tree-vectorize -U__BMI__
+CFLAGS.hist.c= -fno-tree-vectorize -U__BMI__
+CFLAGS.huf_compress.c= -fno-tree-vectorize -U__BMI__
+CFLAGS.zstd_compress.c= -fno-tree-vectorize -U__BMI__
+CFLAGS.zstd_compress_literals.c= -fno-tree-vectorize -U__BMI__
+CFLAGS.zstd_compress_sequences.c= -fno-tree-vectorize -U__BMI__
+CFLAGS.zstd_compress_superblock.c= -fno-tree-vectorize -U__BMI__
+CFLAGS.zstd_double_fast.c= -fno-tree-vectorize -U__BMI__
+CFLAGS.zstd_fast.c= -fno-tree-vectorize -U__BMI__
+CFLAGS.zstd_lazy.c= -fno-tree-vectorize -U__BMI__
+CFLAGS.zstd_ldm.c= -fno-tree-vectorize -U__BMI__
+CFLAGS.zstd_opt.c= -fno-tree-vectorize -U__BMI__
+CFLAGS.huf_decompress.c= -fno-tree-vectorize -U__BMI__
+CFLAGS.zstd_ddict.c= -fno-tree-vectorize -U__BMI__
+CFLAGS.zstd_decompress.c= -fno-tree-vectorize -U__BMI__
+CFLAGS.zstd_decompress_block.c= -fno-tree-vectorize -U__BMI__
diff --git a/module/Makefile.in b/module/Makefile.in
@@ -128,6 +128,7 @@ cppcheck-Linux:
 		--include=@LINUX_OBJ@/include/generated/autoconf.h \
 		--include=@top_srcdir@/zfs_config.h \
 		--config-exclude=@LINUX_OBJ@/include \
+		--config-exclude=zstd/lib \
 		-I @LINUX_OBJ@/include \
 		-I @top_srcdir@/include/os/linux/kernel \
 		-I @top_srcdir@/include/os/linux/spl \

diff --git a/module/zstd/Makefile.in b/module/zstd/Makefile.in
@@ -18,22 +18,47 @@ ccflags-y += -O3
 
 # -fno-tree-vectorize gets set for gcc in zstd/common/compiler.h
 # Set it for other compilers, too.
-$(obj)/lib/zstd.o: c_flags += -fno-tree-vectorize
+common_flags := -fno-tree-vectorize
 
 # SSE register return with SSE disabled if -march=znverX is passed
-$(obj)/lib/zstd.o: c_flags += -U__BMI__
+common_flags += -U__BMI__
 
 # Quiet warnings about frame size due to unused code in unmodified zstd lib
-$(obj)/lib/zstd.o: c_flags += -Wframe-larger-than=20480
+common_flags += -Wframe-larger-than=20480
+
+ccflags-y += $(common_flags)
+
+vanilla-objs := lib/common/entropy_common.o \
+		lib/common/error_private.o \
+		lib/common/fse_decompress.o \
+		lib/common/pool.o \
+		lib/common/zstd_common.o \
+		lib/compress/fse_compress.o \
+		lib/compress/hist.o \
+		lib/compress/huf_compress.o \
+		lib/compress/zstd_compress_literals.o \
+		lib/compress/zstd_compress_sequences.o \
+		lib/compress/zstd_compress_superblock.o \
+		lib/compress/zstd_compress.o \
+		lib/compress/zstd_double_fast.o \
+		lib/compress/zstd_fast.o \
+		lib/compress/zstd_lazy.o \
+		lib/compress/zstd_ldm.o \
+		lib/compress/zstd_opt.o \
+		lib/decompress/huf_decompress.o \
+		lib/decompress/zstd_ddict.o \
+		lib/decompress/zstd_decompress.o \
+		lib/decompress/zstd_decompress_block.o
 
 # Disable aarch64 neon SIMD instructions for kernel mode
-$(obj)/lib/zstd.o: c_flags += -include $(zstd_include)/aarch64_compat.h -include $(zstd_include)/zstd_compat_wrapper.h -Wp,-w
+$(addprefix $(obj)/,$(vanilla-objs)) : ccflags-y += -include $(zstd_include)/aarch64_compat.h -include $(zstd_include)/zstd_compat_wrapper.h -Wp,-w $(common_flags)
 
-$(obj)/zfs_zstd.o: c_flags += -include $(zstd_include)/zstd_compat_wrapper.h
+$(obj)/zfs_zstd.o: ccflags-y += -include $(zstd_include)/zstd_compat_wrapper.h $(common_flags)
 
 $(MODULE)-objs += zfs_zstd.o
-$(MODULE)-objs += lib/zstd.o
 $(MODULE)-objs += zstd_sparc.o
+$(MODULE)-objs += $(vanilla-objs)
 
 all:
-	mkdir -p lib
+	mkdir -p lib/common lib/compress lib/decompress
+
diff --git a/module/zstd/README.md b/module/zstd/README.md
@@ -10,9 +10,8 @@ library, besides upgrading to a newer ZSTD release.
 Tree structure:
 
 * `zfs_zstd.c` is the actual `zzstd` kernel module.
-* `lib/` contains the unmodified, [_"amalgamated"_](https://github.com/facebook/zstd/blob/dev/contrib/single_file_libs/README.md)
-  version of the `Zstandard` library, generated from our template file
-* `zstd-in.c` is our template file for generating the library
+* `lib/` contains the unmodified version of the `Zstandard` library
+* `zstd-in.c` is our template file for generating the single-file library
 * `include/`: This directory contains supplemental includes for platform
   compatibility, which are not expected to be used by ZFS elsewhere in the
   future. Thus we keep them private to ZSTD.
@@ -22,36 +21,19 @@ Tree structure:
 To update ZSTD the following steps need to be taken:
 
 1. Grab the latest release of [ZSTD](https://github.com/facebook/zstd/releases).
-2. Update `module/zstd/zstd-in.c` if required. (see
-   `zstd/contrib/single_file_libs/zstd-in.c` in the zstd repository)
-3. Generate the "single-file-library" and put it to `module/zstd/lib/`.
-4. Copy the following files to `module/zstd/lib/`:
-   - `zstd/lib/zstd.h`
-   - `zstd/lib/common/zstd_errors.h`
+2. Copy the files output by the following script to `module/zstd/lib/`:
+`grep include [path to zstd]/contrib/single_file_libs/zstd-in.c  | awk '{ print $2 }'`
+3. Remove debug.c, threading.c, and zstdmt_compress.c.
+4. Update Makefiles with resulting file lists.
 
-This can be done using a few shell commands from inside the zfs repo:
-
-~~~sh
-cd PATH/TO/ZFS
-
-url="https://github.com/facebook/zstd"
-release="$(curl -s "${url}"/releases/latest | grep -oP '(?<=v)[\d\.]+')"
-zstd="/tmp/zstd-${release}/"
-
-wget -O /tmp/zstd.tar.gz \
-    "${url}/releases/download/v${release}/zstd-${release}.tar.gz"
-tar -C /tmp -xzf /tmp/zstd.tar.gz
-
-cp ${zstd}/lib/zstd.h module/zstd/lib/
-cp ${zstd}/lib/zstd_errors.h module/zstd/lib/
-${zstd}/contrib/single_file_libs/combine.sh \
-    -r ${zstd}/lib -o module/zstd/lib/zstd.c module/zstd/zstd-in.c
 ~~~
 
 Note: if the zstd library for zfs is updated to a newer version,
 the macro list in include/zstd_compat_wrapper.h usually needs to be updated.
 this can be done with some hand crafting of the output of the following
-script: nm zstd.o | awk '{print "#define "$3 " zfs_" $3}' > macrotable
+script (on the object file generated from the "single-file library" script in zstd's
+contrib/single_file_libs):
+`nm zstd.o | awk '{print "#define "$3 " zfs_" $3}' > macrotable`
 
 
 ## Altering ZSTD and breaking changes

diff --git a/module/zstd/include/string.h b/module/zstd/include/string.h
@@ -44,6 +44,7 @@ extern "C" {
 #ifdef _KERNEL
 
 #if defined(__FreeBSD__)
+#include <sys/types.h>    /* u_int, u_char */
 #include <sys/systm.h>    /* memcpy, memset */
 #elif defined(__linux__)
 #include <linux/string.h> /* memcpy, memset */

diff --git a/module/zstd/include/zstd_compat_wrapper.h b/module/zstd/include/zstd_compat_wrapper.h
@@ -45,6 +45,20 @@
  * nm zstd.o | awk '{print "#define "$3 " zfs_" $3}' > macrotable
  */
 
+#define	MEM_MODULE
+#define	XXH_NAMESPACE ZSTD_
+#define	XXH_PRIVATE_API
+#define	XXH_INLINE_ALL
+#define	ZSTD_LEGACY_SUPPORT 0
+#define	ZSTD_LIB_DICTBUILDER 0
+#define	ZSTD_LIB_DEPRECATED 0
+#define	ZSTD_NOBENCH
+#define	DEBUGLEVEL 0
+#ifdef _KERNEL
+#define	ZSTD_DEPS_ASSERT
+#endif
+
+
 #define	BIT_initDStream zfs_BIT_initDStream
 #define	BIT_mask zfs_BIT_mask
 #define	BIT_reloadDStream zfs_BIT_reloadDStream
@@ -406,7 +420,6 @@
 	zfs_ZSTD_insertAndFindFirstIndex_internal
 #define	ZSTD_insertBlock zfs_ZSTD_insertBlock
 #define	ZSTD_invalidateRepCodes zfs_ZSTD_invalidateRepCodes
-#define	ZSTD_isError zfs_ZSTD_isError
 #define	ZSTD_isFrame zfs_ZSTD_isFrame
 #define	ZSTD_ldm_adjustParameters zfs_ZSTD_ldm_adjustParameters
 #define	ZSTD_ldm_blockCompress zfs_ZSTD_ldm_blockCompress