output_file.c

/*
 * Copyright (C) 2010 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#define _FILE_OFFSET_BITS 64
#define _LARGEFILE64_SOURCE 1

#include <fcntl.h>
#include <inttypes.h>
#include <limits.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <zlib.h>

#include "defs.h"
#include "output_file.h"
#include "sparse_crc32.h"
#include "sparse_format.h"

#ifndef USE_MINGW
#include <sys/mman.h>
#define O_BINARY 0
#else
#define ftruncate64 ftruncate
#endif

#if defined(__APPLE__) && defined(__MACH__)
#define lseek64 lseek
#define ftruncate64 ftruncate
#define mmap64 mmap
#define off64_t off_t
#endif

#define min(a, b) \
	({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; })

#define SPARSE_HEADER_MAJOR_VER 1
#define SPARSE_HEADER_MINOR_VER 0
#define SPARSE_HEADER_LEN       (sizeof(sparse_header_t))
#define CHUNK_HEADER_LEN (sizeof(chunk_header_t))

#define container_of(inner, outer_t, elem) \
	((outer_t *)((char *)(inner) - offsetof(outer_t, elem)))

struct output_file_ops {
    int (*open) (struct output_file *, int fd);
    int (*skip) (struct output_file *, int64_t);
    int (*pad) (struct output_file *, int64_t);
    int (*write) (struct output_file *, void *, size_t);
    void (*close) (struct output_file *);
};

struct sparse_file_ops {
    int (*write_data_chunk) (struct output_file * out, unsigned int len, void *data);
    int (*write_fill_chunk) (struct output_file * out, unsigned int len, uint32_t fill_val);
    int (*write_skip_chunk) (struct output_file * out, int64_t len);
    int (*write_end_chunk) (struct output_file * out);
};

struct output_file {
    int64_t cur_out_ptr;
    unsigned int chunk_cnt;
    uint32_t crc32;
    struct output_file_ops *ops;
    struct sparse_file_ops *sparse_ops;
    int use_crc;
    unsigned int block_size;
    int64_t len;
    char *zero_buf;
    uint32_t *fill_buf;
    char *buf;
};

struct output_file_gz {
    struct output_file out;
    gzFile gz_fd;
};

#define to_output_file_gz(_o) \
	container_of((_o), struct output_file_gz, out)

struct output_file_normal {
    struct output_file out;
    int fd;
};

#define to_output_file_normal(_o) \
	container_of((_o), struct output_file_normal, out)

struct output_file_callback {
    struct output_file out;
    void *priv;
    int (*write) (void *priv, const void *buf, int len);
};

#define to_output_file_callback(_o) \
	container_of((_o), struct output_file_callback, out)

static int file_open(struct output_file *out, int fd)
{
    struct output_file_normal *outn = to_output_file_normal(out);

    outn->fd = fd;
    return 0;
}

static int file_skip(struct output_file *out, int64_t cnt)
{
    off64_t ret;
    struct output_file_normal *outn = to_output_file_normal(out);

    ret = lseek64(outn->fd, cnt, SEEK_CUR);
    if (ret < 0) {
        error_errno("lseek64");
        return -1;
    }
    return 0;
}

static int file_pad(struct output_file *out, int64_t len)
{
    int ret;
    struct output_file_normal *outn = to_output_file_normal(out);

    ret = ftruncate64(outn->fd, len);
    if (ret < 0) {
        return -errno;
    }

    return 0;
}

static int file_write(struct output_file *out, void *data, size_t len)
{
    ssize_t ret;
    struct output_file_normal *outn = to_output_file_normal(out);

    while (len > 0) {
        ret = write(outn->fd, data, len);
        if (ret < 0) {
            if (errno == EINTR) {
                continue;
            }
            error_errno("write");
            return -1;
        }

        data = (char *)data + ret;
        len -= ret;
    }

    return 0;
}

static void file_close(struct output_file *out)
{
    struct output_file_normal *outn = to_output_file_normal(out);

    free(outn);
}

static struct output_file_ops file_ops = {
    .open = file_open,
    .skip = file_skip,
    .pad = file_pad,
    .write = file_write,
    .close = file_close,
};

static int gz_file_open(struct output_file *out, int fd)
{
    struct output_file_gz *outgz = to_output_file_gz(out);

    outgz->gz_fd = gzdopen(fd, "wb9");
    if (!outgz->gz_fd) {
        error_errno("gzopen");
        return -errno;
    }

    return 0;
}

static int gz_file_skip(struct output_file *out, int64_t cnt)
{
    off64_t ret;
    struct output_file_gz *outgz = to_output_file_gz(out);

    ret = gzseek(outgz->gz_fd, cnt, SEEK_CUR);
    if (ret < 0) {
        error_errno("gzseek");
        return -1;
    }
    return 0;
}

static int gz_file_pad(struct output_file *out, int64_t len)
{
    off64_t ret;
    struct output_file_gz *outgz = to_output_file_gz(out);

    ret = gztell(outgz->gz_fd);
    if (ret < 0) {
        return -1;
    }

    if (ret >= len) {
        return 0;
    }

    ret = gzseek(outgz->gz_fd, len - 1, SEEK_SET);
    if (ret < 0) {
        return -1;
    }

    gzwrite(outgz->gz_fd, "", 1);

    return 0;
}

static int gz_file_write(struct output_file *out, void *data, size_t len)
{
    int ret;
    struct output_file_gz *outgz = to_output_file_gz(out);

    while (len > 0) {
        ret = gzwrite(outgz->gz_fd, data,
                  min(len, (unsigned int)INT_MAX));
        if (ret == 0) {
            error("gzwrite %s", gzerror(outgz->gz_fd, NULL));
            return -1;
        }
        len -= ret;
        data = (char *)data + ret;
    }

    return 0;
}

static void gz_file_close(struct output_file *out)
{
    struct output_file_gz *outgz = to_output_file_gz(out);

    gzclose(outgz->gz_fd);
    free(outgz);
}

static struct output_file_ops gz_file_ops = {
    .open = gz_file_open,
    .skip = gz_file_skip,
    .pad = gz_file_pad,
    .write = gz_file_write,
    .close = gz_file_close,
};

static int callback_file_open(struct output_file *out __unused, int fd __unused)
{
    return 0;
}

static int callback_file_skip(struct output_file *out, int64_t off)
{
    struct output_file_callback *outc = to_output_file_callback(out);
    int to_write;
    int ret;

    while (off > 0) {
        to_write = min(off, (int64_t) INT_MAX);
        ret = outc->write(outc->priv, NULL, to_write);
        if (ret < 0) {
            return ret;
        }
        off -= to_write;
    }

    return 0;
}

static int callback_file_pad(struct output_file *out __unused, int64_t len __unused)
{
    return -1;
}

static int callback_file_write(struct output_file *out, void *data, size_t len)
{
    struct output_file_callback *outc = to_output_file_callback(out);

    return outc->write(outc->priv, data, len);
}

static void callback_file_close(struct output_file *out)
{
    struct output_file_callback *outc = to_output_file_callback(out);

    free(outc);
}

static struct output_file_ops callback_file_ops = {
    .open = callback_file_open,
    .skip = callback_file_skip,
    .pad = callback_file_pad,
    .write = callback_file_write,
    .close = callback_file_close,
};

int read_all(int fd, void *buf, size_t len)
{
    size_t total = 0;
    int ret;
    char *ptr = buf;

    while (total < len) {
        ret = read(fd, ptr, len - total);

        if (ret < 0)
            return -errno;

        if (ret == 0)
            return -EINVAL;

        ptr += ret;
        total += ret;
    }

    return 0;
}

static int write_sparse_skip_chunk(struct output_file *out, int64_t skip_len)
{
    chunk_header_t chunk_header;
    int ret;

    if (skip_len % out->block_size) {
        error("don't care size %" PRIi64 " is not a multiple of the block size %u",
              skip_len, out->block_size);
        return -1;
    }

    /* We are skipping data, so emit a don't care chunk. */
    chunk_header.chunk_type = CHUNK_TYPE_DONT_CARE;
    chunk_header.reserved1 = 0;
    chunk_header.chunk_sz = skip_len / out->block_size;
    chunk_header.total_sz = CHUNK_HEADER_LEN;
    ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
    if (ret < 0)
        return -1;

    out->cur_out_ptr += skip_len;
    out->chunk_cnt++;

    return 0;
}

static int write_sparse_fill_chunk(struct output_file *out, unsigned int len, uint32_t fill_val)
{
    chunk_header_t chunk_header;
    int rnd_up_len, count;
    int ret;

    /* Round up the fill length to a multiple of the block size */
    rnd_up_len = ALIGN(len, out->block_size);

    /* Finally we can safely emit a chunk of data */
    chunk_header.chunk_type = CHUNK_TYPE_FILL;
    chunk_header.reserved1 = 0;
    chunk_header.chunk_sz = rnd_up_len / out->block_size;
    chunk_header.total_sz = CHUNK_HEADER_LEN + sizeof(fill_val);
    ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));

    if (ret < 0)
        return -1;
    ret = out->ops->write(out, &fill_val, sizeof(fill_val));
    if (ret < 0)
        return -1;

    if (out->use_crc) {
        count = out->block_size / sizeof(uint32_t);
        while (count--)
            out->crc32 = sparse_crc32(out->crc32, &fill_val, sizeof(uint32_t));
    }

    out->cur_out_ptr += rnd_up_len;
    out->chunk_cnt++;

    return 0;
}

static int write_sparse_data_chunk(struct output_file *out, unsigned int len, void *data)
{
    chunk_header_t chunk_header;
    int rnd_up_len, zero_len;
    int ret;

    /* Round up the data length to a multiple of the block size */
    rnd_up_len = ALIGN(len, out->block_size);
    zero_len = rnd_up_len - len;

    /* Finally we can safely emit a chunk of data */
    chunk_header.chunk_type = CHUNK_TYPE_RAW;
    chunk_header.reserved1 = 0;
    chunk_header.chunk_sz = rnd_up_len / out->block_size;
    chunk_header.total_sz = CHUNK_HEADER_LEN + rnd_up_len;
    ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));

    if (ret < 0)
        return -1;
    ret = out->ops->write(out, data, len);
    if (ret < 0)
        return -1;
    if (zero_len) {
        ret = out->ops->write(out, out->zero_buf, zero_len);
        if (ret < 0)
            return -1;
    }

    if (out->use_crc) {
        out->crc32 = sparse_crc32(out->crc32, data, len);
        if (zero_len)
            out->crc32 = sparse_crc32(out->crc32, out->zero_buf, zero_len);
    }

    out->cur_out_ptr += rnd_up_len;
    out->chunk_cnt++;

    return 0;
}

int write_sparse_end_chunk(struct output_file *out)
{
    chunk_header_t chunk_header;
    int ret;

    if (out->use_crc) {
        chunk_header.chunk_type = CHUNK_TYPE_CRC32;
        chunk_header.reserved1 = 0;
        chunk_header.chunk_sz = 0;
        chunk_header.total_sz = CHUNK_HEADER_LEN + 4;

        ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
        if (ret < 0) {
            return ret;
        }
        out->ops->write(out, &out->crc32, 4);
        if (ret < 0) {
            return ret;
        }

        out->chunk_cnt++;
    }

    return 0;
}

static struct sparse_file_ops sparse_file_ops = {
    .write_data_chunk = write_sparse_data_chunk,
    .write_fill_chunk = write_sparse_fill_chunk,
    .write_skip_chunk = write_sparse_skip_chunk,
    .write_end_chunk = write_sparse_end_chunk,
};

static int write_normal_data_chunk(struct output_file *out, unsigned int len, void *data)
{
    int ret;
    unsigned int rnd_up_len = ALIGN(len, out->block_size);

    ret = out->ops->write(out, data, len);
    if (ret < 0) {
        return ret;
    }

    if (rnd_up_len > len) {
        ret = out->ops->skip(out, rnd_up_len - len);
    }

    return ret;
}

static int write_normal_fill_chunk(struct output_file *out, unsigned int len, uint32_t fill_val)
{
    int ret;
    unsigned int i;
    unsigned int write_len;

    /* Initialize fill_buf with the fill_val */
    for (i = 0; i < out->block_size / sizeof(uint32_t); i++) {
        out->fill_buf[i] = fill_val;
    }

    while (len) {
        write_len = min(len, out->block_size);
        ret = out->ops->write(out, out->fill_buf, write_len);
        if (ret < 0) {
            return ret;
        }

        len -= write_len;
    }

    return 0;
}

static int write_normal_skip_chunk(struct output_file *out, int64_t len)
{
    return out->ops->skip(out, len);
}

int write_normal_end_chunk(struct output_file *out)
{
    return out->ops->pad(out, out->len);
}

static struct sparse_file_ops normal_file_ops = {
    .write_data_chunk = write_normal_data_chunk,
    .write_fill_chunk = write_normal_fill_chunk,
    .write_skip_chunk = write_normal_skip_chunk,
    .write_end_chunk = write_normal_end_chunk,
};

void output_file_close(struct output_file *out)
{
    out->sparse_ops->write_end_chunk(out);
    out->ops->close(out);
}

static int output_file_init(struct output_file *out, int block_size,
                            int64_t len, bool sparse, int chunks, bool crc)
{
    int ret;

    out->len = len;
    out->block_size = block_size;
    out->cur_out_ptr = 0ll;
    out->chunk_cnt = 0;
    out->crc32 = 0;
    out->use_crc = crc;

    out->zero_buf = calloc(block_size, 1);
    if (!out->zero_buf) {
        error_errno("malloc zero_buf");
        return -ENOMEM;
    }

    out->fill_buf = calloc(block_size, 1);
    if (!out->fill_buf) {
        error_errno("malloc fill_buf");
        ret = -ENOMEM;
        goto err_fill_buf;
    }

    if (sparse) {
        out->sparse_ops = &sparse_file_ops;
    } else {
        out->sparse_ops = &normal_file_ops;
    }

    if (sparse) {
        sparse_header_t sparse_header = {
            .magic = SPARSE_HEADER_MAGIC,
            .major_version = SPARSE_HEADER_MAJOR_VER,
            .minor_version = SPARSE_HEADER_MINOR_VER,
            .file_hdr_sz = SPARSE_HEADER_LEN,
            .chunk_hdr_sz = CHUNK_HEADER_LEN,
            .blk_sz = out->block_size,
            .total_blks = out->len / out->block_size,
            .total_chunks = chunks,
            .image_checksum = 0
        };

        if (out->use_crc) {
            sparse_header.total_chunks++;
        }

        ret = out->ops->write(out, &sparse_header, sizeof(sparse_header));
        if (ret < 0) {
            goto err_write;
        }
    }

    return 0;

 err_write:
    free(out->fill_buf);
 err_fill_buf:
    free(out->zero_buf);
    return ret;
}

static struct output_file *output_file_new_gz(void)
{
    struct output_file_gz *outgz = calloc(1, sizeof(struct output_file_gz));
    if (!outgz) {
        error_errno("malloc struct outgz");
        return NULL;
    }

    outgz->out.ops = &gz_file_ops;

    return &outgz->out;
}

static struct output_file *output_file_new_normal(void)
{
    struct output_file_normal *outn = calloc(1, sizeof(struct output_file_normal));
    if (!outn) {
        error_errno("malloc struct outn");
        return NULL;
    }

    outn->out.ops = &file_ops;

    return &outn->out;
}

struct output_file *output_file_open_callback(int (*write) (void *, const void *, int),
                                              void *priv, unsigned int block_size, int64_t len,
                                              int gz __unused, int sparse, int chunks, int crc)
{
    int ret;
    struct output_file_callback *outc;

    outc = calloc(1, sizeof(struct output_file_callback));
    if (!outc) {
        error_errno("malloc struct outc");
        return NULL;
    }

    outc->out.ops = &callback_file_ops;
    outc->priv = priv;
    outc->write = write;

    ret = output_file_init(&outc->out, block_size, len, sparse, chunks, crc);
    if (ret < 0) {
        free(outc);
        return NULL;
    }

    return &outc->out;
}

struct output_file *output_file_open_fd(int fd, unsigned int block_size, int64_t len,
                                        int gz, int sparse, int chunks, int crc)
{
    int ret;
    struct output_file *out;

    if (gz) {
        out = output_file_new_gz();
    } else {
        out = output_file_new_normal();
    }
    if (!out) {
        return NULL;
    }

    out->ops->open(out, fd);

    ret = output_file_init(out, block_size, len, sparse, chunks, crc);
    if (ret < 0) {
        free(out);
        return NULL;
    }

    return out;
}

/* Write a contiguous region of data blocks from a memory buffer */
int write_data_chunk(struct output_file *out, unsigned int len, void *data)
{
    return out->sparse_ops->write_data_chunk(out, len, data);
}

/* Write a contiguous region of data blocks with a fill value */
int write_fill_chunk(struct output_file *out, unsigned int len, uint32_t fill_val)
{
    return out->sparse_ops->write_fill_chunk(out, len, fill_val);
}

int write_fd_chunk(struct output_file *out, unsigned int len, int fd, int64_t offset)
{
    int ret;
    int64_t aligned_offset;
    int aligned_diff;
    uint64_t buffer_size;
    char *ptr;

    aligned_offset = offset & ~(sysconf(_SC_PAGESIZE) - 1);
    aligned_diff = offset - aligned_offset;
    buffer_size = (uint64_t)len + (uint64_t)aligned_diff;

#ifndef USE_MINGW
    if (buffer_size > SIZE_MAX)
        return -E2BIG;
    char *data = mmap64(NULL, buffer_size, PROT_READ, MAP_SHARED, fd,
                        aligned_offset);
    if (data == MAP_FAILED) {
        return -errno;
    }
    ptr = data + aligned_diff;
#else
    off64_t pos;
    char *data = malloc(len);
    if (!data) {
        return -errno;
    }
    pos = lseek64(fd, offset, SEEK_SET);
    if (pos < 0) {
        free(data);
        return -errno;
    }
    ret = read_all(fd, data, len);
    if (ret < 0) {
        free(data);
        return ret;
    }
    ptr = data;
#endif

    ret = out->sparse_ops->write_data_chunk(out, len, ptr);

#ifndef USE_MINGW
    munmap(data, buffer_size);
#else
    free(data);
#endif

    return ret;
}

/* Write a contiguous region of data blocks from a file */
int write_file_chunk(struct output_file *out, unsigned int len, const char *file, int64_t offset)
{
    int ret;

    int file_fd = open(file, O_RDONLY | O_BINARY);
    if (file_fd < 0) {
        return -errno;
    }

    ret = write_fd_chunk(out, len, file_fd, offset);

    close(file_fd);

    return ret;
}

int write_skip_chunk(struct output_file *out, int64_t len)
{
    return out->sparse_ops->write_skip_chunk(out, len);
}