From 084228cae38c1c11b9a3e17804debb74c0548374 Mon Sep 17 00:00:00 2001 From: Mark Harmstone Date: Tue, 15 Oct 2024 10:59:04 +0100 Subject: [PATCH] btrfs-progs: mkfs: add lzo to --compress option Allows --compress to work with lzo. Signed-off-by: Mark Harmstone --- Documentation/mkfs.btrfs.rst | 2 +- mkfs/main.c | 2 +- mkfs/rootdir.c | 192 ++++++++++++++++++++++++++++++++++- 3 files changed, 190 insertions(+), 6 deletions(-) diff --git a/Documentation/mkfs.btrfs.rst b/Documentation/mkfs.btrfs.rst index f70b6ebe2..2f0a06a20 100644 --- a/Documentation/mkfs.btrfs.rst +++ b/Documentation/mkfs.btrfs.rst @@ -214,7 +214,7 @@ OPTIONS --compress : Try to compress files when using *--rootdir*. Supported values for *algo* are - *no* (the default), *zlib*, and *zstd*. The optional value *level* is a + *no* (the default), *zlib*, *lzo*, and *zstd*. The optional value *level* is a compression level, from 1 to 9 for ZLIB and from 1 to 15 for ZSTD. As with the kernel, :command:`mkfs.btrfs` won't write compressed extents when diff --git a/mkfs/main.c b/mkfs/main.c index 91b338db4..46e72460b 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -443,7 +443,7 @@ static const char * const mkfs_usage[] = { OPTLINE("-u|--subvol TYPE:SUBDIR", "create SUBDIR as subvolume rather than normal directory, can be specified multiple times"), OPTLINE("--shrink", "(with --rootdir) shrink the filled filesystem to minimal size"), OPTLINE("-K|--nodiscard", "do not perform whole device TRIM"), - OPTLINE("--compress ALGO:LEVEL", "compression algorithm and level to use; ALGO can be no (default), zlib, zstd"), + OPTLINE("--compress ALGO:LEVEL", "compression algorithm and level to use; ALGO can be no (default), zlib, lzo, zstd"), OPTLINE("-f|--force", "force overwrite of existing filesystem"), "General:", OPTLINE("-q|--quiet", "no messages except errors"), diff --git a/mkfs/rootdir.c b/mkfs/rootdir.c index 4b1f6d6ee..83dc86101 100644 --- a/mkfs/rootdir.c +++ b/mkfs/rootdir.c @@ -32,6 +32,7 @@ #include #endif #include +#include #include "kernel-lib/sizes.h" #include "kernel-shared/accessors.h" #include "kernel-shared/uapi/btrfs_tree.h" @@ -56,6 +57,8 @@ #define ZSTD_BTRFS_DEFAULT_LEVEL 3 #define ZSTD_BTRFS_MAX_LEVEL 15 +#define LZO_LEN 4 + static u32 fs_block_size; /* @@ -445,6 +448,77 @@ static ssize_t zlib_compress_extent(struct btrfs_inode_item *btrfs_inode, return 0; } +static ssize_t lzo_compress_extent(struct btrfs_inode_item *btrfs_inode, + u32 sectorsize, const void *in_buf, + size_t in_size, void *out_buf, char *wrkmem) +{ + int ret; + unsigned int sectors; + u32 total_size, out_pos; + u64 flags; + + out_pos = LZO_LEN; + total_size = LZO_LEN; + sectors = DIV_ROUND_UP(in_size, sectorsize); + + for (unsigned int i = 0; i < sectors; i++) { + size_t in_len, out_len, new_pos; + u32 padding; + + in_len = min((size_t)sectorsize, in_size - (i * sectorsize)); + + ret = lzo1x_1_compress(in_buf + (i * sectorsize), in_len, + out_buf + out_pos + LZO_LEN, &out_len, + wrkmem); + if (ret) { + error("lzo1x_1_compress returned %i", ret); + return -EINVAL; + } + + put_unaligned_le32(out_len, out_buf + out_pos); + + new_pos = out_pos + LZO_LEN + out_len; + + /* Make sure that our header doesn't cross a sector boundary. */ + if (new_pos / sectorsize != (new_pos + LZO_LEN - 1) / sectorsize) + padding = round_up(new_pos, LZO_LEN) - new_pos; + else + padding = 0; + + out_pos += out_len + LZO_LEN + padding; + total_size += out_len + LZO_LEN + padding; + + /* Follow kernel in trying to compress the first three sectors, + * then giving up if the output isn't any smaller. */ + if (i >= 3 && total_size > i * sectorsize) { + total_size = 0; + goto out; + } + } + + if (total_size > in_size) { + total_size = 0; + goto out; + } + + put_unaligned_le32(total_size, out_buf); + +out: + flags = btrfs_stack_inode_flags(btrfs_inode); + + if (total_size == 0) { + if (!(flags & BTRFS_INODE_COMPRESS)) { + flags |= BTRFS_INODE_NOCOMPRESS; + btrfs_set_stack_inode_flags(btrfs_inode, flags); + } + } else { + flags |= BTRFS_INODE_COMPRESS; + btrfs_set_stack_inode_flags(btrfs_inode, flags); + } + + return total_size; +} + #if COMPRESSION_ZSTD static ssize_t zstd_compress_extent(struct btrfs_inode_item *btrfs_inode, u32 sectorsize, const void *in_buf, @@ -557,7 +631,7 @@ static int read_and_write_extent(struct btrfs_trans_handle *trans, struct btrfs_inode_item *btrfs_inode, u64 objectid, int fd, u64 file_pos, char *buf, u64 size, const char *path_name, - char *comp_buf) + char *comp_buf, char *wrkmem) { int ret; u32 sectorsize = root->fs_info->sectorsize; @@ -601,6 +675,11 @@ static int read_and_write_extent(struct btrfs_trans_handle *trans, buf, bytes_read, comp_buf); break; + case BTRFS_COMPRESS_LZO: + comp_ret = lzo_compress_extent(btrfs_inode, sectorsize, + buf, bytes_read, + comp_buf, wrkmem); + break; #if COMPRESSION_ZSTD case BTRFS_COMPRESS_ZSTD: comp_ret = zstd_compress_extent(btrfs_inode, sectorsize, @@ -658,6 +737,11 @@ static int read_and_write_extent(struct btrfs_trans_handle *trans, features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD; btrfs_set_super_incompat_flags(trans->fs_info->super_copy, features); + } else if (g_compression == BTRFS_COMPRESS_LZO) { + features = btrfs_super_incompat_flags(trans->fs_info->super_copy); + features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO; + btrfs_set_super_incompat_flags(trans->fs_info->super_copy, + features); } } else { to_write = round_up(to_read, sectorsize); @@ -754,6 +838,56 @@ static int zlib_compress_inline_extent(char *buf, u64 size, char **comp_buf, return ret; } +static u32 lzo_max_outlen(u32 inlen) { + /* Return the worst-case output length for LZO. Formula comes from + * LZO.FAQ. */ + return inlen + (inlen / 16) + 64 + 3; +} + +static int lzo_compress_inline_extent(void *buf, u64 size, char **comp_buf, + u64 *comp_size, char *wrkmem) +{ + int ret; + size_t out_len, out_size; + void *out = NULL; + + out_size = LZO_LEN + LZO_LEN + lzo_max_outlen(size); + + out = malloc(out_size); + if (!out) { + error_msg(ERROR_MSG_MEMORY, NULL); + ret = -ENOMEM; + goto out; + } + + ret = lzo1x_1_compress(buf, size, out + LZO_LEN + LZO_LEN, &out_len, + wrkmem); + if (ret) { + error("lzo1x_1_compress returned %i", ret); + ret = -EINVAL; + goto out; + } + + if (out_len + LZO_LEN + LZO_LEN >= size) { + ret = 0; + goto out; + } + + put_unaligned_le32(out_len + LZO_LEN + LZO_LEN, out); + put_unaligned_le32(out_len, out + LZO_LEN); + + *comp_buf = out; + *comp_size = out_len + LZO_LEN + LZO_LEN; + + ret = 1; + +out: + if (ret != 1) + free(out); + + return ret; +} + #if COMPRESSION_ZSTD static int zstd_compress_inline_extent(char *buf, u64 size, char **comp_buf, u64 *comp_size) @@ -840,7 +974,7 @@ static int add_file_items(struct btrfs_trans_handle *trans, ssize_t ret_read; u32 sectorsize = fs_info->sectorsize; u64 file_pos = 0; - char *buf = NULL, *comp_buf = NULL; + char *buf = NULL, *comp_buf = NULL, *wrkmem = NULL; int fd; if (st->st_size == 0) @@ -852,6 +986,14 @@ static int add_file_items(struct btrfs_trans_handle *trans, return ret; } + if (g_compression == BTRFS_COMPRESS_LZO) { + wrkmem = malloc(LZO1X_1_MEM_COMPRESS); + if (!wrkmem) { + ret = -ENOMEM; + goto end; + } + } + if (st->st_size <= BTRFS_MAX_INLINE_DATA_SIZE(fs_info) && st->st_size < sectorsize) { char *buffer = malloc(st->st_size); @@ -877,6 +1019,13 @@ static int add_file_items(struct btrfs_trans_handle *trans, if (ret < 0) goto end; break; + case BTRFS_COMPRESS_LZO: + ret = lzo_compress_inline_extent(buffer, st->st_size, + &comp_buf, &comp_size, + wrkmem); + if (ret < 0) + goto end; + break; #if COMPRESSION_ZSTD case BTRFS_COMPRESS_ZSTD: ret = zstd_compress_inline_extent(buffer, st->st_size, @@ -913,7 +1062,40 @@ static int add_file_items(struct btrfs_trans_handle *trans, goto end; } - if (g_compression != BTRFS_COMPRESS_NONE) { + if (g_compression == BTRFS_COMPRESS_LZO) { + unsigned int sectors; + size_t comp_buf_len; + + /* LZO helpfully doesn't provide a way to specify the output + * buffer size, so we need to allocate for the worst-case + * scenario to avoid buffer overruns. + * + * 4 bytes for the total size + * And for each sector: + * - 4 bytes for the compressed sector size + * - the worst-case output size + * - 3 bytes for possible padding + */ + + sectors = BTRFS_MAX_COMPRESSED / sectorsize; + + comp_buf_len = LZO_LEN; + comp_buf_len += (LZO_LEN + lzo_max_outlen(sectorsize) + + LZO_LEN - 1) * sectors; + + comp_buf = malloc(comp_buf_len); + if (!comp_buf) { + ret = -ENOMEM; + goto end; + } + + ret = lzo_init(); + if (ret) { + error("lzo_init returned %i", ret); + ret = -EINVAL; + goto end; + } + } else if (g_compression != BTRFS_COMPRESS_NONE) { comp_buf = malloc(BTRFS_MAX_COMPRESSED); if (!comp_buf) { ret = -ENOMEM; @@ -924,7 +1106,7 @@ static int add_file_items(struct btrfs_trans_handle *trans, while (file_pos < st->st_size) { ret = read_and_write_extent(trans, root, btrfs_inode, objectid, fd, file_pos, buf, st->st_size, - path_name, comp_buf); + path_name, comp_buf, wrkmem); if (ret < 0) break; @@ -932,6 +1114,7 @@ static int add_file_items(struct btrfs_trans_handle *trans, } end: + free(wrkmem); free(comp_buf); free(buf); close(fd); @@ -1352,6 +1535,7 @@ int btrfs_mkfs_fill_dir(struct btrfs_trans_handle *trans, const char *source_dir switch (compression) { case BTRFS_COMPRESS_NONE: + case BTRFS_COMPRESS_LZO: break; case BTRFS_COMPRESS_ZLIB: if (compression_level > ZLIB_BTRFS_MAX_LEVEL)