diff --git a/ChangeLog b/ChangeLog index 03269a3..bdfa66c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,20 @@ +erofs-utils 1.8.3 + + * Another maintenance release includes the following fixes: + - (mkfs.erofs) Fix multi-threaded compression with `-Eall-fragments`; + - (mkfs.erofs) Fix large chunk-based image generation; + - (mkfs.erofs) Avoid large arrays on the stack (Jianan Huang); + - (mkfs.erofs) Fix PAX format parsing in headerball mode (Mike Baynton); + - (mkfs.erofs) Several fixes for incremental builds (Hongzhen Luo); + - (mkfs.erofs) Fix reproducible builds due to `i_ino` (Jooyung Han); + - Use pkg-config for liblz4 configuration; + - Get rid of pthread_cancel() dependencies; + - (mkfs.erofs) Add `-U ` support; + - (mkfs.erofs) Add `--hard-dereference` for NixOS reproducibility (Paul Meyer); + - Several minor random fixes. + + -- Gao Xiang Sat, 14 Dec 2024 00:00:00 +0800 + erofs-utils 1.8.2 * Another maintenance release includes the following fixes: diff --git a/VERSION b/VERSION index 168fdfd..205d56c 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -1.8.2 -2024-09-24 +1.8.3 +2024-12-14 diff --git a/configure.ac b/configure.ac index 9c1657b..45a7d33 100644 --- a/configure.ac +++ b/configure.ac @@ -123,8 +123,8 @@ AC_ARG_ENABLE([fuzzing], [enable_fuzzing="no"]) AC_ARG_ENABLE(lz4, - [AS_HELP_STRING([--disable-lz4], [disable LZ4 compression support @<:@default=enabled@:>@])], - [enable_lz4="$enableval"], [enable_lz4="yes"]) + [AS_HELP_STRING([--disable-lz4], [disable LZ4 compression support @<:@default=auto@:>@])], + [enable_lz4="$enableval"]) AC_ARG_ENABLE(lzma, [AS_HELP_STRING([--disable-lzma], [disable LZMA compression support @<:@default=auto@:>@])], @@ -172,17 +172,6 @@ AC_ARG_WITH(selinux, esac], [with_selinux=no]) # Checks for libraries. -# Use customized LZ4 library path when specified. -AC_ARG_WITH(lz4-incdir, - [AS_HELP_STRING([--with-lz4-incdir=DIR], [LZ4 include directory])], [ - EROFS_UTILS_PARSE_DIRECTORY(["$withval"],[withval])]) - -AC_ARG_WITH(lz4-libdir, - [AS_HELP_STRING([--with-lz4-libdir=DIR], [LZ4 lib directory])], [ - EROFS_UTILS_PARSE_DIRECTORY(["$withval"],[withval])]) - -AC_ARG_VAR([LZ4_CFLAGS], [C compiler flags for lz4]) -AC_ARG_VAR([LZ4_LIBS], [linker flags for lz4]) # Checks for header files. AC_CHECK_HEADERS(m4_flatten([ @@ -396,36 +385,35 @@ AS_IF([test "x$enable_fuse" != "xno"], [ CPPFLAGS="${saved_CPPFLAGS}"], [have_fuse="no"]) # Configure lz4 -test -z $LZ4_LIBS && LZ4_LIBS='-llz4' - -if test "x$enable_lz4" = "xyes"; then - test -z "${with_lz4_incdir}" || LZ4_CFLAGS="-I$with_lz4_incdir $LZ4_CFLAGS" - +AS_IF([test "x$enable_lz4" != "xno"], [ saved_CPPFLAGS=${CPPFLAGS} - CPPFLAGS="${LZ4_CFLAGS} ${CPPFLAGS}" - - AC_CHECK_HEADERS([lz4.h],[have_lz4h="yes"], []) - - if test "x${have_lz4h}" = "xyes" ; then + PKG_CHECK_MODULES([liblz4], [liblz4], [ + # Paranoia: don't trust the result reported by pkgconfig before trying out saved_LIBS="$LIBS" - saved_LDFLAGS=${LDFLAGS} - test -z "${with_lz4_libdir}" || LDFLAGS="-L$with_lz4_libdir ${LDFLAGS}" - AC_CHECK_LIB(lz4, LZ4_compress_destSize, [ - have_lz4="yes" - have_lz4hc="yes" - AC_CHECK_LIB(lz4, LZ4_compress_HC_destSize, [], [ - AC_CHECK_DECL(LZ4_compress_HC_destSize, [lz4_force_static="yes"], - [have_lz4hc="no"], [[ -#define LZ4_HC_STATIC_LINKING_ONLY (1) + saved_CPPFLAGS=${CPPFLAGS} + CPPFLAGS="${liblz4_CFLAGS} ${CPPFLAGS}" + LIBS="${liblz4_LIBS} $LIBS" + AC_CHECK_HEADERS([lz4.h],[ + AC_CHECK_LIB(lz4, LZ4_compress_destSize, [ + AC_CHECK_DECL(LZ4_compress_destSize, [have_lz4="yes"], + [], [[ +#include + ]]) + ]) + AC_CHECK_LIB(lz4, LZ4_compress_HC_destSize, [ + AC_CHECK_DECL(LZ4_compress_HC_destSize, [have_lz4hc="yes"], + [], [[ #include ]]) ]) - ], [AC_MSG_ERROR([Cannot find proper lz4 version (>= 1.8.0)])]) - LDFLAGS=${saved_LDFLAGS} + ]) LIBS="${saved_LIBS}" - fi - CPPFLAGS=${saved_CPPFLAGS} -fi + CPPFLAGS="${saved_CPPFLAGS}" + ], [[]]) + AS_IF([test "x$enable_lz4" = "xyes" -a "x$have_lz4" != "xyes"], [ + AC_MSG_ERROR([Cannot find a proper liblz4 version]) + ]) +]) # Configure liblzma have_liblzma="no" @@ -581,16 +569,7 @@ if test "x${have_lz4}" = "xyes"; then if test "x${have_lz4hc}" = "xyes"; then AC_DEFINE([LZ4HC_ENABLED], [1], [Define to 1 if lz4hc is enabled.]) fi - - if test "x${lz4_force_static}" = "xyes"; then - LZ4_LIBS="-Wl,-Bstatic -Wl,-whole-archive -Xlinker ${LZ4_LIBS} -Wl,-no-whole-archive -Wl,-Bdynamic" - test -z "${with_lz4_libdir}" || LZ4_LIBS="-L${with_lz4_libdir} $LZ4_LIBS" - else - test -z "${with_lz4_libdir}" || LZ4_LIBS="-L${with_lz4_libdir} -R${with_lz4_libdir} $LZ4_LIBS" - fi - liblz4_LIBS="${LZ4_LIBS}" fi -AC_SUBST([liblz4_LIBS]) if test "x${have_liblzma}" = "xyes"; then AC_DEFINE([HAVE_LIBLZMA], [1], [Define to 1 if liblzma is enabled.]) diff --git a/debian/changelog b/debian/changelog index 4d24443..21e8186 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,9 @@ -erofs-utils (1.8.2-2deepin1) unstable; urgency=medium +erofs-utils (1.8.3-1) unstable; urgency=medium - * Add deepin patch. + * New upstream release 1.8.3. + * Use FUSE3 for erofsfuse, thanks to наб (closes: #1084471). - -- xiangzelong Fri, 13 Dec 2024 16:27:30 +0800 + -- Gao Xiang Sun, 15 Dec 2024 00:00:00 +0800 erofs-utils (1.8.2-2) unstable; urgency=medium diff --git a/debian/control b/debian/control index 235396c..6e0a8d5 100644 --- a/debian/control +++ b/debian/control @@ -11,7 +11,7 @@ Build-Depends: pkgconf, uuid-dev, zlib1g-dev, - libfuse-dev [linux-any] + libfuse3-dev [linux-any] Section: admin Standards-Version: 4.7.0 Homepage: http://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git @@ -33,9 +33,10 @@ Description: Utilities for EROFS File System performance for embedded devices with limited memory. Package: erofsfuse +Architecture: linux-any Build-Profiles: Depends: ${shlibs:Depends}, ${misc:Depends} -Architecture: linux-any +Recommends: fuse3 Description: FUSE Mount Utility for EROFS File System EROFS (Enhanced Read-Only File System) is a lightweight read-only file system with modern designs (eg. page-sized diff --git a/debian/patches/0002-fix-undifined-symbols-duing-linking-liberofsfuse.patch b/debian/patches/0002-fix-undifined-symbols-duing-linking-liberofsfuse.patch deleted file mode 100644 index 83df831..0000000 --- a/debian/patches/0002-fix-undifined-symbols-duing-linking-liberofsfuse.patch +++ /dev/null @@ -1,34 +0,0 @@ -From: ComixHe @ 2024-12-13 6:32 UTC (permalink / raw) - -Some of the symbols required by erofsfuse are provided by liberofs. -When option 'enable-static-fuse' is set, all these object file should be -exported to liberofsfuse.a - -https://lore.kernel.org/linux-erofs/398413dc-95ad-4ab9-a387-e44aacd68aa1@linux.alibaba.com/T/#mc3a64520a9365157dfa4ed2b5de9320757000d9e - -Signed-off-by: ComixHe ---- - fuse/Makefile.am | 10 +++++----- - 1 file changed, 5 insertions(+), 5 deletions(-) - -diff --git a/fuse/Makefile.am b/fuse/Makefile.am -index 1062b73..50186da 100644 ---- a/fuse/Makefile.am -+++ b/fuse/Makefile.am -@@ -11,9 +11,9 @@ erofsfuse_LDADD = $(top_builddir)/lib/liberofs.la ${libfuse2_LIBS} ${libfuse3_LI - ${libqpl_LIBS} - - if ENABLE_STATIC_FUSE --lib_LIBRARIES = liberofsfuse.a --liberofsfuse_a_SOURCES = main.c --liberofsfuse_a_CFLAGS = -Wall -I$(top_srcdir)/include --liberofsfuse_a_CFLAGS += -Dmain=erofsfuse_main ${libfuse2_CFLAGS} ${libfuse3_CFLAGS} ${libselinux_CFLAGS} --liberofsfuse_a_LIBADD = $(top_builddir)/lib/liberofs.la -+lib_LTLIBRARIES = liberofsfuse.la -+liberofsfuse_la_SOURCES = main.c -+liberofsfuse_la_CFLAGS = -Wall -I$(top_srcdir)/include -+liberofsfuse_la_CFLAGS += -Dmain=erofsfuse_main ${libfuse2_CFLAGS} ${libfuse3_CFLAGS} ${libselinux_CFLAGS} -+liberofsfuse_la_LIBADD = $(top_builddir)/lib/liberofs.la - endif --- -2.47.1 diff --git a/debian/patches/series b/debian/patches/series deleted file mode 100644 index 72f6d6c..0000000 --- a/debian/patches/series +++ /dev/null @@ -1 +0,0 @@ -0002-fix-undifined-symbols-duing-linking-liberofsfuse.patch diff --git a/fuse/Makefile.am b/fuse/Makefile.am index 1062b73..50186da 100644 --- a/fuse/Makefile.am +++ b/fuse/Makefile.am @@ -11,9 +11,9 @@ erofsfuse_LDADD = $(top_builddir)/lib/liberofs.la ${libfuse2_LIBS} ${libfuse3_LI ${libqpl_LIBS} if ENABLE_STATIC_FUSE -lib_LIBRARIES = liberofsfuse.a -liberofsfuse_a_SOURCES = main.c -liberofsfuse_a_CFLAGS = -Wall -I$(top_srcdir)/include -liberofsfuse_a_CFLAGS += -Dmain=erofsfuse_main ${libfuse2_CFLAGS} ${libfuse3_CFLAGS} ${libselinux_CFLAGS} -liberofsfuse_a_LIBADD = $(top_builddir)/lib/liberofs.la +lib_LTLIBRARIES = liberofsfuse.la +liberofsfuse_la_SOURCES = main.c +liberofsfuse_la_CFLAGS = -Wall -I$(top_srcdir)/include +liberofsfuse_la_CFLAGS += -Dmain=erofsfuse_main ${libfuse2_CFLAGS} ${libfuse3_CFLAGS} ${libselinux_CFLAGS} +liberofsfuse_la_LIBADD = $(top_builddir)/lib/liberofs.la endif diff --git a/include/erofs/block_list.h b/include/erofs/block_list.h index 7db4d0c..8cc87d7 100644 --- a/include/erofs/block_list.h +++ b/include/erofs/block_list.h @@ -17,7 +17,7 @@ int erofs_blocklist_open(FILE *fp, bool srcmap); FILE *erofs_blocklist_close(void); void tarerofs_blocklist_write(erofs_blk_t blkaddr, erofs_blk_t nblocks, - erofs_off_t srcoff); + erofs_off_t srcoff, unsigned int zeroedlen); #ifdef WITH_ANDROID void erofs_droid_blocklist_write(struct erofs_inode *inode, erofs_blk_t blk_start, erofs_blk_t nblocks); diff --git a/include/erofs/config.h b/include/erofs/config.h index ae366c1..bb03e70 100644 --- a/include/erofs/config.h +++ b/include/erofs/config.h @@ -46,10 +46,6 @@ struct erofs_configure { int c_dbg_lvl; bool c_dry_run; bool c_legacy_compress; -#ifndef NDEBUG - bool c_random_pclusterblks; - bool c_random_algorithms; -#endif char c_timeinherit; char c_chunkbits; bool c_inline_data; @@ -62,6 +58,7 @@ struct erofs_configure { bool c_extra_ea_name_prefixes; bool c_xattr_name_filter; bool c_ovlfs_strip; + bool c_hard_dereference; #ifdef HAVE_LIBSELINUX struct selabel_handle *sehnd; @@ -94,6 +91,10 @@ struct erofs_configure { char *fs_config_file; char *block_list_file; #endif +#ifndef NDEBUG + bool c_random_pclusterblks; + bool c_random_algorithms; +#endif }; extern struct erofs_configure cfg; diff --git a/include/erofs_fs.h b/include/erofs_fs.h index fc21915..9c69aac 100644 --- a/include/erofs_fs.h +++ b/include/erofs_fs.h @@ -414,8 +414,7 @@ enum { Z_EROFS_LCLUSTER_TYPE_MAX }; -#define Z_EROFS_LI_LCLUSTER_TYPE_BITS 2 -#define Z_EROFS_LI_LCLUSTER_TYPE_BIT 0 +#define Z_EROFS_LI_LCLUSTER_TYPE_MASK (Z_EROFS_LCLUSTER_TYPE_MAX - 1) /* (noncompact only, HEAD) This pcluster refers to partial decompressed data */ #define Z_EROFS_LI_PARTIAL_REF (1 << 15) @@ -474,8 +473,6 @@ static inline void erofs_check_ondisk_layout_definitions(void) sizeof(struct z_erofs_lcluster_index)); BUILD_BUG_ON(sizeof(struct erofs_deviceslot) != 128); - BUILD_BUG_ON(BIT(Z_EROFS_LI_LCLUSTER_TYPE_BITS) < - Z_EROFS_LCLUSTER_TYPE_MAX - 1); #ifndef __cplusplus /* exclude old compiler versions like gcc 7.5.0 */ BUILD_BUG_ON(__builtin_constant_p(fmh.v) ? diff --git a/lib/Makefile.am b/lib/Makefile.am index 2cb4cab..9c0604d 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -38,7 +38,7 @@ liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \ liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include if ENABLE_LZ4 -liberofs_la_CFLAGS += ${LZ4_CFLAGS} +liberofs_la_CFLAGS += ${liblz4_CFLAGS} liberofs_la_SOURCES += compressor_lz4.c if ENABLE_LZ4HC liberofs_la_SOURCES += compressor_lz4hc.c @@ -51,9 +51,11 @@ endif liberofs_la_SOURCES += kite_deflate.c compressor_deflate.c if ENABLE_LIBDEFLATE +liberofs_la_CFLAGS += ${libdeflate_CFLAGS} liberofs_la_SOURCES += compressor_libdeflate.c endif if ENABLE_LIBZSTD +liberofs_la_CFLAGS += ${libzstd_CFLAGS} liberofs_la_SOURCES += compressor_libzstd.c endif if ENABLE_EROFS_MT diff --git a/lib/blobchunk.c b/lib/blobchunk.c index 6c2ea0e..119dd82 100644 --- a/lib/blobchunk.c +++ b/lib/blobchunk.c @@ -133,12 +133,13 @@ static int erofs_blob_hashmap_cmp(const void *a, const void *b, int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, erofs_off_t off) { - erofs_blk_t remaining_blks = BLK_ROUND_UP(inode->sbi, inode->i_size); + struct erofs_sb_info *sbi = inode->sbi; + erofs_blk_t remaining_blks = BLK_ROUND_UP(sbi, inode->i_size); struct erofs_inode_chunk_index idx = {0}; erofs_blk_t extent_start = EROFS_NULL_ADDR; erofs_blk_t extent_end, chunkblks; erofs_off_t source_offset; - unsigned int dst, src, unit; + unsigned int dst, src, unit, zeroedlen; bool first_extent = true; if (inode->u.chunkformat & EROFS_CHUNK_FORMAT_INDEXES) @@ -169,7 +170,7 @@ int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, remaining_blks -= extent_end - extent_start; tarerofs_blocklist_write(extent_start, extent_end - extent_start, - source_offset); + source_offset, 0); erofs_droid_blocklist_write_extent(inode, extent_start, extent_end - extent_start, @@ -190,9 +191,13 @@ int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, } off = roundup(off, unit); extent_end = min(extent_end, extent_start + remaining_blks); - if (extent_start != EROFS_NULL_ADDR) + if (extent_start != EROFS_NULL_ADDR) { + zeroedlen = inode->i_size & (erofs_blksiz(sbi) - 1); + if (zeroedlen) + zeroedlen = erofs_blksiz(sbi) - zeroedlen; tarerofs_blocklist_write(extent_start, extent_end - extent_start, - source_offset); + source_offset, zeroedlen); + } erofs_droid_blocklist_write_extent(inode, extent_start, extent_start == EROFS_NULL_ADDR ? 0 : extent_end - extent_start, @@ -480,9 +485,8 @@ int tarerofs_write_chunkes(struct erofs_inode *inode, erofs_off_t data_offset) int erofs_mkfs_dump_blobs(struct erofs_sb_info *sbi) { struct erofs_buffer_head *bh; - ssize_t length; + ssize_t length, ret; u64 pos_in, pos_out; - ssize_t ret; if (blobfile) { fflush(blobfile); @@ -532,9 +536,21 @@ int erofs_mkfs_dump_blobs(struct erofs_sb_info *sbi) pos_out += sbi->bdev.offset; if (blobfile) { pos_in = 0; - ret = erofs_copy_file_range(fileno(blobfile), &pos_in, - sbi->bdev.fd, &pos_out, datablob_size); - ret = ret < datablob_size ? -EIO : 0; + do { + length = min_t(erofs_off_t, datablob_size, SSIZE_MAX); + ret = erofs_copy_file_range(fileno(blobfile), &pos_in, + sbi->bdev.fd, &pos_out, length); + } while (ret > 0 && (datablob_size -= ret)); + + if (ret >= 0) { + if (datablob_size) { + erofs_err("failed to append the remaining %llu-byte chunk data", + datablob_size); + ret = -EIO; + } else { + ret = 0; + } + } } else { ret = erofs_io_ftruncate(&sbi->bdev, pos_out + datablob_size); } diff --git a/lib/block_list.c b/lib/block_list.c index 261e9ff..6bbe4ec 100644 --- a/lib/block_list.c +++ b/lib/block_list.c @@ -32,13 +32,17 @@ FILE *erofs_blocklist_close(void) /* XXX: really need to be cleaned up */ void tarerofs_blocklist_write(erofs_blk_t blkaddr, erofs_blk_t nblocks, - erofs_off_t srcoff) + erofs_off_t srcoff, unsigned int zeroedlen) { if (!block_list_fp || !nblocks || !srcmap_enabled) return; - fprintf(block_list_fp, "%08x %8x %08" PRIx64 "\n", - blkaddr, nblocks, srcoff); + if (zeroedlen) + fprintf(block_list_fp, "%08x %8x %08" PRIx64 " %08u\n", + blkaddr, nblocks, srcoff, zeroedlen); + else + fprintf(block_list_fp, "%08x %8x %08" PRIx64 "\n", + blkaddr, nblocks, srcoff); } #ifdef WITH_ANDROID diff --git a/lib/compress.c b/lib/compress.c index 17e7112..65edd00 100644 --- a/lib/compress.c +++ b/lib/compress.c @@ -8,6 +8,9 @@ #ifndef _LARGEFILE64_SOURCE #define _LARGEFILE64_SOURCE #endif +#ifdef EROFS_MT_ENABLED +#include +#endif #include #include #include @@ -126,7 +129,7 @@ static void z_erofs_write_indexes_final(struct z_erofs_compress_ictx *ctx) di.di_clusterofs = cpu_to_le16(ctx->clusterofs); di.di_u.blkaddr = 0; - di.di_advise = cpu_to_le16(type << Z_EROFS_LI_LCLUSTER_TYPE_BIT); + di.di_advise = cpu_to_le16(type); memcpy(ctx->metacur, &di, sizeof(di)); ctx->metacur += sizeof(di); @@ -156,8 +159,7 @@ static void z_erofs_write_extent(struct z_erofs_compress_ictx *ctx, DBG_BUGON(e->partial); type = e->raw ? Z_EROFS_LCLUSTER_TYPE_PLAIN : Z_EROFS_LCLUSTER_TYPE_HEAD1; - advise = type << Z_EROFS_LI_LCLUSTER_TYPE_BIT; - di.di_advise = cpu_to_le16(advise); + di.di_advise = cpu_to_le16(type); if (inode->datalayout == EROFS_INODE_COMPRESSED_FULL && !e->compressedblks) @@ -215,8 +217,7 @@ static void z_erofs_write_extent(struct z_erofs_compress_ictx *ctx, advise |= Z_EROFS_LI_PARTIAL_REF; } } - advise |= type << Z_EROFS_LI_LCLUSTER_TYPE_BIT; - di.di_advise = cpu_to_le16(advise); + di.di_advise = cpu_to_le16(advise | type); memcpy(ctx->metacur, &di, sizeof(di)); ctx->metacur += sizeof(di); @@ -448,31 +449,39 @@ static int z_erofs_fill_inline_data(struct erofs_inode *inode, void *data, return len; } -static void tryrecompress_trailing(struct z_erofs_compress_sctx *ctx, - struct erofs_compress *ec, - void *in, unsigned int *insize, - void *out, unsigned int *compressedsize) +static int tryrecompress_trailing(struct z_erofs_compress_sctx *ctx, + struct erofs_compress *ec, + void *in, unsigned int *insize, + void *out, unsigned int *compressedsize) { struct erofs_sb_info *sbi = ctx->ictx->inode->sbi; - char tmp[Z_EROFS_PCLUSTER_MAX_SIZE]; + char *tmp; unsigned int count; int ret = *compressedsize; /* no need to recompress */ if (!(ret & (erofs_blksiz(sbi) - 1))) - return; + return 0; + + tmp = malloc(Z_EROFS_PCLUSTER_MAX_SIZE); + if (!tmp) + return -ENOMEM; count = *insize; ret = erofs_compress_destsize(ec, in, &count, (void *)tmp, rounddown(ret, erofs_blksiz(sbi))); if (ret <= 0 || ret + (*insize - count) >= roundup(*compressedsize, erofs_blksiz(sbi))) - return; + goto out; /* replace the original compressed data if any gain */ memcpy(out, tmp, ret); *insize = count; *compressedsize = ret; + +out: + free(tmp); + return 0; } static bool z_erofs_fixup_deduped_fragment(struct z_erofs_compress_sctx *ctx, @@ -628,9 +637,14 @@ static int __z_erofs_compress_one(struct z_erofs_compress_sctx *ctx, goto fix_dedupedfrag; } - if (may_inline && len == e->length) - tryrecompress_trailing(ctx, h, ctx->queue + ctx->head, - &e->length, dst, &compressedsize); + if (may_inline && len == e->length) { + ret = tryrecompress_trailing(ctx, h, + ctx->queue + ctx->head, + &e->length, dst, + &compressedsize); + if (ret) + return ret; + } e->compressedblks = BLK_ROUND_UP(sbi, compressedsize); DBG_BUGON(e->compressedblks * blksz >= e->length); @@ -742,8 +756,7 @@ static void *parse_legacy_indexes(struct z_erofs_compressindex_vec *cv, struct z_erofs_lcluster_index *const di = db + i; const unsigned int advise = le16_to_cpu(di->di_advise); - cv->clustertype = (advise >> Z_EROFS_LI_LCLUSTER_TYPE_BIT) & - ((1 << Z_EROFS_LI_LCLUSTER_TYPE_BITS) - 1); + cv->clustertype = advise & Z_EROFS_LI_LCLUSTER_TYPE_MASK; cv->clusterofs = le16_to_cpu(di->di_clusterofs); if (cv->clustertype == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { @@ -971,10 +984,8 @@ void z_erofs_drop_inline_pcluster(struct erofs_inode *inode) struct z_erofs_lcluster_index *di = (inode->compressmeta + inode->extent_isize) - sizeof(struct z_erofs_lcluster_index); - __le16 advise = - cpu_to_le16(type << Z_EROFS_LI_LCLUSTER_TYPE_BIT); - di->di_advise = advise; + di->di_advise = cpu_to_le16(type); } else if (inode->datalayout == EROFS_INODE_COMPRESSED_COMPACT) { /* handle the last compacted 4B pack */ unsigned int eofs, base, pos, v, lo; @@ -1453,12 +1464,8 @@ void *erofs_begin_compressed_file(struct erofs_inode *inode, int fd, u64 fpos) inode->idata_size = 0; inode->fragment_size = 0; - if (z_erofs_mt_enabled) { - ictx = malloc(sizeof(*ictx)); - if (!ictx) - return ERR_PTR(-ENOMEM); - ictx->fd = dup(fd); - } else { + if (!z_erofs_mt_enabled || + (cfg.c_all_fragments && !erofs_is_packed_inode(inode))) { #ifdef EROFS_MT_ENABLED pthread_mutex_lock(&g_ictx.mutex); if (g_ictx.seg_num) @@ -1468,6 +1475,11 @@ void *erofs_begin_compressed_file(struct erofs_inode *inode, int fd, u64 fpos) #endif ictx = &g_ictx; ictx->fd = fd; + } else { + ictx = malloc(sizeof(*ictx)); + if (!ictx) + return ERR_PTR(-ENOMEM); + ictx->fd = dup(fd); } ictx->ccfg = &erofs_ccfg[inode->z_algorithmtype[0]]; @@ -1778,7 +1790,9 @@ int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *s cfg.c_mt_workers << 2, z_erofs_mt_wq_tls_alloc, z_erofs_mt_wq_tls_free); - z_erofs_mt_enabled = !ret; + if (ret) + return ret; + z_erofs_mt_enabled = true; } pthread_mutex_init(&g_ictx.mutex, NULL); pthread_cond_init(&g_ictx.cond, NULL); diff --git a/lib/compressor_lz4hc.c b/lib/compressor_lz4hc.c index 1e1ccc7..9955c0d 100644 --- a/lib/compressor_lz4hc.c +++ b/lib/compressor_lz4hc.c @@ -4,7 +4,6 @@ * http://www.huawei.com/ * Created by Gao Xiang */ -#define LZ4_HC_STATIC_LINKING_ONLY (1) #include #include "erofs/internal.h" #include "erofs/print.h" diff --git a/lib/inode.c b/lib/inode.c index 7958d43..0404a8d 100644 --- a/lib/inode.c +++ b/lib/inode.c @@ -6,6 +6,9 @@ * with heavy changes by Gao Xiang */ #define _GNU_SOURCE +#ifdef EROFS_MT_ENABLED +#include +#endif #include #include #include @@ -818,6 +821,7 @@ static int erofs_prepare_inode_buffer(struct erofs_inode *inode) bh->fsprivate = erofs_igrab(inode); bh->op = &erofs_write_inode_bhops; inode->bh = bh; + inode->i_ino[0] = ++inode->sbi->inos; /* inode serial number */ return 0; } @@ -1111,7 +1115,6 @@ struct erofs_inode *erofs_new_inode(struct erofs_sb_info *sbi) return ERR_PTR(-ENOMEM); inode->sbi = sbi; - inode->i_ino[0] = sbi->inos++; /* inode serial number */ inode->i_count = 1; inode->datalayout = EROFS_INODE_FLAT_PLAIN; @@ -1138,7 +1141,7 @@ static struct erofs_inode *erofs_iget_from_srcpath(struct erofs_sb_info *sbi, * hard-link, just return it. Also don't lookup for directories * since hard-link directory isn't allowed. */ - if (!S_ISDIR(st.st_mode)) { + if (!S_ISDIR(st.st_mode) && (!cfg.c_hard_dereference)) { inode = erofs_iget(st.st_dev, st.st_ino); if (inode) return inode; @@ -1195,7 +1198,8 @@ static int erofs_inode_reserve_data_blocks(struct erofs_inode *inode) erofs_bdrop(bh, false); inode->datalayout = EROFS_INODE_FLAT_PLAIN; - tarerofs_blocklist_write(inode->u.i_blkaddr, nblocks, inode->i_ino[1]); + tarerofs_blocklist_write(inode->u.i_blkaddr, nblocks, inode->i_ino[1], + alignedsz - inode->i_size); return 0; } @@ -1702,7 +1706,7 @@ static int erofs_mkfs_dump_tree(struct erofs_inode *root, bool rebuild, { struct erofs_sb_info *sbi = root->sbi; struct erofs_inode *dumpdir = erofs_igrab(root); - int err; + int err, err2; erofs_mark_parent_inode(root, root); /* rootdir mark */ root->next_dirwrite = NULL; @@ -1713,6 +1717,12 @@ static int erofs_mkfs_dump_tree(struct erofs_inode *root, bool rebuild, list_del(&root->i_hash); erofs_insert_ihash(root); } else if (cfg.c_root_xattr_isize) { + if (cfg.c_root_xattr_isize > EROFS_XATTR_ALIGN( + UINT16_MAX - sizeof(struct erofs_xattr_entry))) { + erofs_err("Invalid configuration for c_root_xattr_isize: %u (too large)", + cfg.c_root_xattr_isize); + return -EINVAL; + } root->xattr_isize = cfg.c_root_xattr_isize; } @@ -1729,7 +1739,6 @@ static int erofs_mkfs_dump_tree(struct erofs_inode *root, bool rebuild, } do { - int err; struct erofs_inode *dir = dumpdir; /* used for adding sub-directories in reverse order due to FIFO */ struct erofs_inode *head, **last = &head; @@ -1766,10 +1775,10 @@ static int erofs_mkfs_dump_tree(struct erofs_inode *root, bool rebuild, } *last = dumpdir; /* fixup the last (or the only) one */ dumpdir = head; - err = erofs_mkfs_go(sbi, EROFS_MKFS_JOB_DIR_BH, + err2 = erofs_mkfs_go(sbi, EROFS_MKFS_JOB_DIR_BH, &dir, sizeof(dir)); - if (err) - return err; + if (err || err2) + return err ? err : err2; } while (dumpdir); return err; diff --git a/lib/rebuild.c b/lib/rebuild.c index 08c1b86..3e58f00 100644 --- a/lib/rebuild.c +++ b/lib/rebuild.c @@ -46,6 +46,7 @@ static struct erofs_dentry *erofs_rebuild_mkdir(struct erofs_inode *dir, inode->i_gid = getgid(); inode->i_mtime = inode->sbi->build_time; inode->i_mtime_nsec = inode->sbi->build_time_nsec; + inode->dev = dir->dev; erofs_init_empty_dir(inode); d = erofs_d_alloc(dir, s); @@ -465,7 +466,9 @@ static int erofs_rebuild_basedir_dirent_iter(struct erofs_dir_context *ctx) struct erofs_inode *inode = d->inode; /* update sub-directories only for recursively loading */ - if (S_ISDIR(inode->i_mode)) { + if (S_ISDIR(inode->i_mode) && + (ctx->de_ftype == EROFS_FT_DIR || + ctx->de_ftype == EROFS_FT_UNKNOWN)) { list_del(&inode->i_hash); inode->dev = dir->sbi->dev; inode->i_ino[1] = ctx->de_nid; @@ -497,6 +500,15 @@ int erofs_rebuild_load_basedir(struct erofs_inode *dir) if (__erofs_unlikely(IS_ROOT(dir))) dir->xattr_isize = fakeinode.xattr_isize; + /* + * May be triggered if ftype == EROFS_FT_UNKNOWN, which is impossible + * with the current mkfs. + */ + if (__erofs_unlikely(!S_ISDIR(fakeinode.i_mode))) { + DBG_BUGON(1); + return 0; + } + ctx = (struct erofs_rebuild_dir_context) { .ctx.dir = &fakeinode, .ctx.cb = erofs_rebuild_basedir_dirent_iter, diff --git a/lib/tar.c b/lib/tar.c index b32abd4..0dd990e 100644 --- a/lib/tar.c +++ b/lib/tar.c @@ -667,6 +667,7 @@ int tarerofs_parse_tar(struct erofs_inode *root, struct erofs_tarfile *tar) unsigned int j, csum, cksum; int ckksum, ret, rem; + root->dev = tar->dev; if (eh.path) eh.path = strdup(eh.path); if (eh.link) @@ -808,13 +809,14 @@ int tarerofs_parse_tar(struct erofs_inode *root, struct erofs_tarfile *tar) } dataoff = tar->offset; - if (!(tar->headeronly_mode || tar->ddtaridx_mode)) - tar->offset += st.st_size; + tar->offset += st.st_size; switch(th->typeflag) { case '0': case '7': case '1': st.st_mode |= S_IFREG; + if (tar->headeronly_mode || tar->ddtaridx_mode) + tar->offset -= st.st_size; break; case '2': st.st_mode |= S_IFLNK; diff --git a/lib/workqueue.c b/lib/workqueue.c index 47cec9b..18ee0f9 100644 --- a/lib/workqueue.c +++ b/lib/workqueue.c @@ -15,9 +15,9 @@ static void *worker_thread(void *arg) while (true) { pthread_mutex_lock(&wq->lock); - while (wq->job_count == 0 && !wq->shutdown) + while (!wq->job_count && !wq->shutdown) pthread_cond_wait(&wq->cond_empty, &wq->lock); - if (wq->job_count == 0 && wq->shutdown) { + if (!wq->job_count && wq->shutdown) { pthread_mutex_unlock(&wq->lock); break; } @@ -40,6 +40,30 @@ static void *worker_thread(void *arg) return NULL; } +int erofs_destroy_workqueue(struct erofs_workqueue *wq) +{ + if (!wq) + return -EINVAL; + + pthread_mutex_lock(&wq->lock); + wq->shutdown = true; + pthread_cond_broadcast(&wq->cond_empty); + pthread_mutex_unlock(&wq->lock); + + while (wq->nworker) { + int ret = -pthread_join(wq->workers[wq->nworker - 1], NULL); + + if (ret) + return ret; + --wq->nworker; + } + free(wq->workers); + pthread_mutex_destroy(&wq->lock); + pthread_cond_destroy(&wq->cond_empty); + pthread_cond_destroy(&wq->cond_full); + return 0; +} + int erofs_alloc_workqueue(struct erofs_workqueue *wq, unsigned int nworker, unsigned int max_jobs, erofs_wq_func_t on_start, erofs_wq_func_t on_exit) @@ -51,7 +75,6 @@ int erofs_alloc_workqueue(struct erofs_workqueue *wq, unsigned int nworker, return -EINVAL; wq->head = wq->tail = NULL; - wq->nworker = nworker; wq->max_jobs = max_jobs; wq->job_count = 0; wq->shutdown = false; @@ -66,15 +89,14 @@ int erofs_alloc_workqueue(struct erofs_workqueue *wq, unsigned int nworker, return -ENOMEM; for (i = 0; i < nworker; i++) { - ret = pthread_create(&wq->workers[i], NULL, worker_thread, wq); - if (ret) { - while (i) - pthread_cancel(wq->workers[--i]); - free(wq->workers); - return ret; - } + ret = -pthread_create(&wq->workers[i], NULL, worker_thread, wq); + if (ret) + break; } - return 0; + wq->nworker = i; + if (ret) + erofs_destroy_workqueue(wq); + return ret; } int erofs_queue_work(struct erofs_workqueue *wq, struct erofs_work *work) @@ -99,25 +121,3 @@ int erofs_queue_work(struct erofs_workqueue *wq, struct erofs_work *work) pthread_mutex_unlock(&wq->lock); return 0; } - -int erofs_destroy_workqueue(struct erofs_workqueue *wq) -{ - unsigned int i; - - if (!wq) - return -EINVAL; - - pthread_mutex_lock(&wq->lock); - wq->shutdown = true; - pthread_cond_broadcast(&wq->cond_empty); - pthread_mutex_unlock(&wq->lock); - - for (i = 0; i < wq->nworker; i++) - pthread_join(wq->workers[i], NULL); - - free(wq->workers); - pthread_mutex_destroy(&wq->lock); - pthread_cond_destroy(&wq->cond_empty); - pthread_cond_destroy(&wq->cond_full); - return 0; -} diff --git a/lib/xattr.c b/lib/xattr.c index 7fbd24b..e420775 100644 --- a/lib/xattr.c +++ b/lib/xattr.c @@ -169,6 +169,7 @@ static unsigned int put_xattritem(struct xattr_item *item) { if (item->count > 1) return --item->count; + hash_del(&item->node); free(item); return 0; } diff --git a/lib/zmap.c b/lib/zmap.c index a5c5b00..f1cdc66 100644 --- a/lib/zmap.c +++ b/lib/zmap.c @@ -142,8 +142,8 @@ static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m, return 0; } -static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m, - unsigned long lcn) +static int z_erofs_load_full_lcluster(struct z_erofs_maprecorder *m, + unsigned long lcn) { struct erofs_inode *const vi = m->inode; struct erofs_sb_info *sbi = vi->sbi; @@ -152,7 +152,7 @@ static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m, vi->inode_isize + vi->xattr_isize) + lcn * sizeof(struct z_erofs_lcluster_index); struct z_erofs_lcluster_index *di; - unsigned int advise, type; + unsigned int advise; int err; err = z_erofs_reload_indexes(m, erofs_blknr(sbi, pos)); @@ -164,10 +164,8 @@ static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m, di = m->kaddr + erofs_blkoff(sbi, pos); advise = le16_to_cpu(di->di_advise); - type = (advise >> Z_EROFS_LI_LCLUSTER_TYPE_BIT) & - ((1 << Z_EROFS_LI_LCLUSTER_TYPE_BITS) - 1); - switch (type) { - case Z_EROFS_LCLUSTER_TYPE_NONHEAD: + m->type = advise & Z_EROFS_LI_LCLUSTER_TYPE_MASK; + if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { m->clusterofs = 1 << vi->z_logical_clusterbits; m->delta[0] = le16_to_cpu(di->di_u.delta[0]); if (m->delta[0] & Z_EROFS_LI_D0_CBLKCNT) { @@ -180,19 +178,11 @@ static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m, m->delta[0] = 1; } m->delta[1] = le16_to_cpu(di->di_u.delta[1]); - break; - case Z_EROFS_LCLUSTER_TYPE_PLAIN: - case Z_EROFS_LCLUSTER_TYPE_HEAD1: - if (advise & Z_EROFS_LI_PARTIAL_REF) - m->partialref = true; + } else { + m->partialref = !!(advise & Z_EROFS_LI_PARTIAL_REF); m->clusterofs = le16_to_cpu(di->di_clusterofs); m->pblk = le32_to_cpu(di->di_u.blkaddr); - break; - default: - DBG_BUGON(1); - return -EOPNOTSUPP; } - m->type = type; return 0; } @@ -337,8 +327,8 @@ static int unpack_compacted_index(struct z_erofs_maprecorder *m, return 0; } -static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m, - unsigned long lcn, bool lookahead) +static int z_erofs_load_compact_lcluster(struct z_erofs_maprecorder *m, + unsigned long lcn, bool lookahead) { struct erofs_inode *const vi = m->inode; struct erofs_sb_info *sbi = vi->sbi; @@ -389,18 +379,17 @@ static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m, return unpack_compacted_index(m, amortizedshift, pos, lookahead); } -static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m, - unsigned int lcn, bool lookahead) +static int z_erofs_load_lcluster_from_disk(struct z_erofs_maprecorder *m, + unsigned int lcn, bool lookahead) { - const unsigned int datamode = m->inode->datalayout; - - if (datamode == EROFS_INODE_COMPRESSED_FULL) - return legacy_load_cluster_from_disk(m, lcn); - - if (datamode == EROFS_INODE_COMPRESSED_COMPACT) - return compacted_load_cluster_from_disk(m, lcn, lookahead); - - return -EINVAL; + switch (m->inode->datalayout) { + case EROFS_INODE_COMPRESSED_FULL: + return z_erofs_load_full_lcluster(m, lcn); + case EROFS_INODE_COMPRESSED_COMPACT: + return z_erofs_load_compact_lcluster(m, lcn, lookahead); + default: + return -EINVAL; + } } static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m, @@ -421,7 +410,7 @@ static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m, /* load extent head logical cluster if needed */ lcn -= lookback_distance; - err = z_erofs_load_cluster_from_disk(m, lcn, false); + err = z_erofs_load_lcluster_from_disk(m, lcn, false); if (err) return err; @@ -471,7 +460,7 @@ static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, if (m->compressedblks) goto out; - err = z_erofs_load_cluster_from_disk(m, lcn, false); + err = z_erofs_load_lcluster_from_disk(m, lcn, false); if (err) return err; @@ -532,7 +521,7 @@ static int z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m) return 0; } - err = z_erofs_load_cluster_from_disk(m, lcn, true); + err = z_erofs_load_lcluster_from_disk(m, lcn, true); if (err) return err; @@ -581,7 +570,7 @@ static int z_erofs_do_map_blocks(struct erofs_inode *vi, initial_lcn = ofs >> lclusterbits; endoff = ofs & ((1 << lclusterbits) - 1); - err = z_erofs_load_cluster_from_disk(&m, initial_lcn, false); + err = z_erofs_load_lcluster_from_disk(&m, initial_lcn, false); if (err) goto out; diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1 index abdd9b9..0093839 100644 --- a/man/mkfs.erofs.1 +++ b/man/mkfs.erofs.1 @@ -110,6 +110,17 @@ Set the universally unique identifier (UUID) of the filesystem to .IR UUID . The format of the UUID is a series of hex digits separated by hyphens, like this: "c1b9d5a2-f162-11cf-9ece-0020afc76f16". +The +.I UUID +parameter may also be one of the following: +.RS 1.2i +.TP +.I clear +clear the file system UUID +.TP +.I random +generate a new randomly-generated UUID +.RE .TP .B \-\-all-root Make all files owned by root. diff --git a/mkfs/main.c b/mkfs/main.c index d422787..9ca7dad 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -85,6 +85,7 @@ static struct option long_options[] = { {"mkfs-time", no_argument, NULL, 525}, {"all-time", no_argument, NULL, 526}, {"sort", required_argument, NULL, 527}, + {"hard-dereference", no_argument, NULL, 528}, {0, 0, 0, 0}, }; @@ -174,6 +175,7 @@ static void usage(int argc, char **argv) " --force-gid=# set all file gids to # (# = GID)\n" " --uid-offset=# add offset # to all file uids (# = id offset)\n" " --gid-offset=# add offset # to all file gids (# = id offset)\n" + " --hard-dereference dereference hardlinks, add links as separate inodes\n" " --ignore-mtime use build time instead of strict per-file modification time\n" " --max-extent-bytes=# set maximum decompressed extent size # in bytes\n" " --mount-point=X X=prefix of target fs path (default: /)\n" @@ -617,7 +619,12 @@ static int mkfs_parse_options_cfg(int argc, char *argv[]) has_timestamp = true; break; case 'U': - if (erofs_uuid_parse(optarg, fixeduuid)) { + if (!strcmp(optarg, "clear")) { + memset(fixeduuid, 0, 16); + } else if (!strcmp(optarg, "random")) { + valid_fixeduuid = false; + break; + } else if (erofs_uuid_parse(optarg, fixeduuid)) { erofs_err("invalid UUID %s", optarg); return -EINVAL; } @@ -846,6 +853,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[]) if (!strcmp(optarg, "none")) erofstar.try_no_reorder = true; break; + case 528: + cfg.c_hard_dereference = true; + break; case 'V': version(); exit(0);