Add Deflate algorithm to Sprite Compression using ZLIB-NG

Co-authored-by: ivan-mogilko <[email protected]>

.cirrus.yml

@@ -13,8 +13,10 @@ build_windows_task:
     AGS_LIBVORBIS_LIB: C:\Lib\Xiph\x86
     AGS_SDL_INCLUDE: C:\Lib\SDL2\include
     AGS_SDL_SOUND_INCLUDE: C:\Lib\SDL_sound\src
+    AGS_ZLIB_INCLUDE: C:\Lib\zlib\include
     AGS_SDL_LIB: C:\Lib\SDL2\lib\x86
     AGS_SDL_SOUND_LIB: C:\Lib\SDL_sound\lib\x86
+    AGS_ZLIB_LIB: C:\Lib\zlib\lib\x86
   build_debug_script: >
     "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86 &&
     cd Solutions &&
@@ -277,7 +279,9 @@ build_editor_task:
     os_version: 2019
   env:
     AGS_SDL_INCLUDE: C:\Lib\SDL2\include
-    AGS_SDL_LIB: C:\Lib\SDL2\lib\x86
+    AGS_SDL_LIB: C:\Lib\SDL2\lib\x86x86
+    AGS_ZLIB_INCLUDE: C:\Lib\zlib\include
+    AGS_ZLIB_LIB: C:\Lib\zlib\lib\x86
   nuget_packages_cache:
     folder: Solutions\packages
     fingerprint_script: type Editor\AGS.Editor\packages.config

CMake/FetchZLIB.cmake

@@ -0,0 +1,22 @@
+FetchContent_Declare(
+  zlibng_content
+  URL https://github.com/zlib-ng/zlib-ng/archive/refs/tags/2.1.3.tar.gz
+  URL_HASH MD5=f2ddef7b10e24cdcc4f17285575a6895
+)
+
+FetchContent_GetProperties(zlibng_content)
+if(NOT zlibng_content_POPULATED)
+  FetchContent_Populate(zlibng_content)
+
+  set(ZLIB_ENABLE_TESTS  OFF CACHE BOOL "")
+  set(ZLIBNG_ENABLE_TESTS  OFF CACHE BOOL "")
+  set(WITH_GTEST  OFF CACHE BOOL "")
+  set(ZLIB_COMPAT ON CACHE BOOL "")
+  add_subdirectory(${zlibng_content_SOURCE_DIR} ${zlibng_content_BINARY_DIR} EXCLUDE_FROM_ALL)
+
+  add_library(zlib-interface INTERFACE)
+  target_link_libraries(zlib-interface INTERFACE zlibstatic)
+  add_library(ZLIB::ZLIB ALIAS zlib-interface)
+
+endif()
+set(ZLIB_LIBRARIES ZLIB::ZLIB)

CMakeLists.txt

@@ -16,6 +16,7 @@ option(AGS_USE_LOCAL_SDL2_SOUND "Use a locally installed SDL2 Sound" ${AGS_USE_L
 option(AGS_USE_LOCAL_OGG "Use a locally installed OGG" ${AGS_USE_LOCAL_ALL_LIBRARIES})
 option(AGS_USE_LOCAL_THEORA "Use a locally installed Theora" ${AGS_USE_LOCAL_ALL_LIBRARIES})
 option(AGS_USE_LOCAL_VORBIS "Use a locally installed Vorbis" ${AGS_USE_LOCAL_ALL_LIBRARIES})
+option(AGS_USE_LOCAL_ZLIB "Use a locally installed ZLib" ${AGS_USE_LOCAL_ALL_LIBRARIES})
 
 option(AGS_TESTS "Build tests" OFF)
 option(AGS_BUILD_ENGINE "Build Engine" ON)
@@ -35,6 +36,7 @@ message(" AGS_USE_LOCAL_SDL2_SOUND: ${AGS_USE_LOCAL_SDL2_SOUND}")
 message(" AGS_USE_LOCAL_OGG: ${AGS_USE_LOCAL_OGG}")
 message(" AGS_USE_LOCAL_THEORA: ${AGS_USE_LOCAL_THEORA}")
 message(" AGS_USE_LOCAL_VORBIS: ${AGS_USE_LOCAL_VORBIS}")
+message(" AGS_USE_LOCAL_ZLIB: ${AGS_USE_LOCAL_ZLIB}")
 message("------ AGS selected CMake options ------")
 message(" AGS_TESTS: ${AGS_TESTS}")
 message(" AGS_BUILD_ENGINE: ${AGS_BUILD_ENGINE}")
@@ -146,7 +148,7 @@ if(EMSCRIPTEN)
     set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/CMake/Emscripten")
 
     set(AGS_OPENGLES2 TRUE)
-    set(USE_FLAGS " -sUSE_SDL=2  -sUSE_VORBIS=1  -sUSE_OGG=1 ")
+    set(USE_FLAGS " -sUSE_SDL=2  -sUSE_VORBIS=1  -sUSE_OGG=1 -sUSE_ZLIB=1")
     set(AGS_DISABLE_THREADS TRUE)
     if(AGS_DISABLE_THREADS)
         add_compile_definitions("AGS_DISABLE_THREADS=1")
@@ -303,6 +305,14 @@ else()
     find_package(Theora REQUIRED)
 endif()
 
+if(EMSCRIPTEN)
+    # do nothing
+elseif (NOT AGS_USE_LOCAL_ZLIB)
+    include(FetchZLIB)
+else()
+    find_package(ZLIB REQUIRED)
+endif()
+
 ###############################################################################
 # subdirectories of this project
 

Common/CMakeLists.txt

@@ -190,7 +190,8 @@ target_link_libraries(common PUBLIC
         Allegro::Allegro
         AlFont::AlFont
         AAStr::AAStr
-        glm::glm)
+        glm::glm
+        ${ZLIB_LIBRARIES})
 
 if (WIN32)
     target_link_libraries(common PUBLIC shlwapi)

Common/ac/spritefile.cpp

@@ -445,6 +445,8 @@ HError SpriteFile::LoadSprite(sprkey_t index, Common::Bitmap *&sprite)
             break;
         case kSprCompress_LZW: lzw_decompress(im_data.Buf, im_data.Size, im_data.BPP, _stream.get(), in_data_size);
             break;
+        case kSprCompress_Deflate: inflate_decompress(im_data.Buf, im_data.Size, im_data.BPP, _stream.get(), in_data_size);
+            break;
         default: assert(!"Unsupported compression type!"); break;
         }
         // TODO: test that not more than data_size was read!
@@ -699,6 +701,8 @@ void SpriteFileWriter::WriteBitmap(Bitmap *image)
             break;
         case kSprCompress_LZW: lzw_compress(im_data.Buf, im_data.Size, im_data.BPP, &mems);
             break;
+        case kSprCompress_Deflate: deflate_compress(im_data.Buf, im_data.Size, im_data.BPP, &mems);
+            break;
         default: assert(!"Unsupported compression type!"); break;
         }
         // mark to write as a plain byte array

Common/ac/spritefile.h

@@ -83,7 +83,8 @@ enum SpriteCompression
 {
     kSprCompress_None = 0,
     kSprCompress_RLE,
-    kSprCompress_LZW
+    kSprCompress_LZW,
+    kSprCompress_Deflate
 };
 
 typedef int32_t sprkey_t;

Common/util/compress.cpp

@@ -11,14 +11,11 @@
 // http://www.opensource.org/licenses/artistic-license-2.0.php
 //
 //=============================================================================
-#ifdef _MANAGED
-// ensure this doesn't get compiled to .NET IL
-#pragma unmanaged
-#endif
 
 #include "util/compress.h"
 #include <stdlib.h>
 #include <stdio.h>
+#include <iostream>
 #include <vector>
 #include "ac/common.h"	// quit, update_polled_stuff
 #include "gfx/bitmap.h"
@@ -27,6 +24,7 @@
 #if AGS_PLATFORM_ENDIAN_BIG
 #include "util/bbop.h"
 #endif
+#include <zlib.h>
 
 using namespace AGS::Common;
 
@@ -449,3 +447,97 @@ std::unique_ptr<Bitmap> load_lzw(Stream *in, int dst_bpp, RGB (*pal)[256])
 
   return bmm;
 }
+
+//-----------------------------------------------------------------------------
+// Deflate
+//-----------------------------------------------------------------------------
+
+bool z_deflate(Stream* input, Stream* output) {
+    z_stream stream;
+    memset(&stream, 0, sizeof(stream));
+
+    int ret = deflateInit(&stream, Z_DEFAULT_COMPRESSION);
+    if (ret != Z_OK) {
+        std::cerr << "Error initializing compression" << std::endl;
+        return false;
+    }
+
+    stream.data_type = 0;
+
+    char inbuf[1024];
+    char outbuf[1024];
+
+    stream.avail_in = input->Read(inbuf, sizeof(inbuf));
+    while (stream.avail_in > 0) {
+        stream.next_in = (Bytef*)inbuf;
+        int flush = input->EOS() ? Z_FINISH : Z_NO_FLUSH;
+
+        do {
+            stream.next_out = (Bytef*)outbuf;
+            stream.avail_out = sizeof(outbuf);
+            ret = deflate(&stream, flush);
+            assert(ret != Z_STREAM_ERROR);
+            int have = sizeof(outbuf) - stream.avail_out;
+            output->Write(outbuf, have);
+        } while (stream.avail_out == 0);
+
+        stream.avail_in = input->Read(inbuf, sizeof(inbuf));
+    }
+
+    (void)deflateEnd(&stream);
+
+    return true;
+}
+
+bool z_inflate(const uint8_t* src, size_t src_sz, uint8_t* dst, size_t dst_sz) {
+    z_stream stream;
+    stream.zalloc = Z_NULL;
+    stream.zfree = Z_NULL;
+    stream.opaque = Z_NULL;
+    stream.avail_in = 0;
+    stream.next_in = Z_NULL;
+
+    int ret = inflateInit(&stream);
+    if (ret != Z_OK) {
+        std::cerr << "Error initializing decompression" << std::endl;
+        return false;
+    }
+
+    stream.next_in = (Bytef*)src;
+    stream.avail_in = src_sz;
+    stream.next_out = dst;
+    stream.avail_out = dst_sz;
+
+    do {
+        ret = inflate(&stream, Z_FINISH);
+        switch (ret) {
+            case Z_NEED_DICT:
+            case Z_DATA_ERROR:
+            case Z_MEM_ERROR:
+            case Z_BUF_ERROR:
+                std::cerr << "Error decompressing data" << std::endl;
+                (void)inflateEnd(&stream);
+                return false;
+            default:
+                stream.next_out = dst + (dst_sz - stream.avail_out);
+                break;
+        }
+    } while (stream.avail_out > 0);
+
+    (void)inflateEnd(&stream);
+
+    return true;
+}
+
+void deflate_compress(const uint8_t* data, size_t data_sz, int /*image_bpp*/, Stream* out)
+{
+    MemoryStream mem_in(data, data_sz);
+    z_deflate(&mem_in, out);
+}
+
+void inflate_decompress(uint8_t* data, size_t data_sz, int /*image_bpp*/, Stream* in, size_t in_sz)
+{
+    std::vector<uint8_t> in_buf(in_sz);
+    in->Read(in_buf.data(), in_sz);
+    z_inflate(in_buf.data(), in_sz, data, data_sz);
+}

Common/util/compress.h

@@ -41,4 +41,8 @@ void save_lzw(Common::Stream *out, const Common::Bitmap *bmpp, const RGB (*pal)[
 // Loads bitmap decompressing
 std::unique_ptr<Common::Bitmap> load_lzw(Common::Stream *in, int dst_bpp, RGB (*pal)[256] = nullptr);
 
+// Deflate compression
+void deflate_compress(const uint8_t* data, size_t data_sz, int image_bpp, Common::Stream* out);
+void inflate_decompress(uint8_t* data, size_t data_sz, int image_bpp, Common::Stream* in, size_t in_sz);
+
 #endif // __AC_COMPRESS_H

Common/util/lzw.cpp

@@ -22,11 +22,6 @@
 
 using namespace AGS::Common;
 
-#ifdef _MANAGED
-// ensure this doesn't get compiled to .NET IL
-#pragma unmanaged
-#endif
-
 int insert(int, int);
 void _delete(int);
 

Common/util/lzw.h

@@ -19,11 +19,9 @@
 #define __AGS_CN_UTIL__LZW_H
 
 #include "core/types.h"
+#include "util/stream.h"
 
-namespace AGS { namespace Common { class Stream; } }
-using namespace AGS; // FIXME later
-
-bool lzwcompress(Common::Stream *lzw_in, Common::Stream *out);
+bool lzwcompress(AGS::Common::Stream *lzw_in, AGS::Common::Stream *out);
 // Expands lzw-compressed data from src to dst.
 // the dst buffer should be large enough, or the uncompression will not be complete.
 bool lzwexpand(const uint8_t *src, size_t src_sz, uint8_t *dst, size_t dst_sz);

Editor/AGS.Types/Enums/SpriteCompression.cs

@@ -6,6 +6,7 @@ public enum SpriteCompression
     {
         None,
         RLE,
-        LZW
+        LZW,
+        Deflate
     }
 }

Engine/Makefile-defs.linux

@@ -22,6 +22,7 @@ CXXFLAGS := -std=c++11 -Werror=delete-non-virtual-dtor $(CXXFLAGS)
 LIBS += $(FT_LDFLAGS)
 LIBS += $(shell pkg-config --libs ogg)
 LIBS += $(shell pkg-config --libs theora)
+LIBS += $(shell pkg-config --libs zlib)
 
 ifeq ($(USE_TREMOR), 1)
   LIBS += -lvorbisidec

Engine/Makefile-defs.osx

@@ -36,6 +36,7 @@ LIBS := -framework Cocoa \
 LIBS += $(FT_LDFLAGS)
 LIBS += $(shell pkg-config --libs ogg)
 LIBS += $(shell pkg-config --libs theora)
+LIBS += $(shell pkg-config --libs zlib)
 
 ifeq ($(USE_TREMOR), 1)
 LIBS += -lvorbisidec

Solutions/Common.Lib/Common.Lib.vcxproj

@@ -162,6 +162,9 @@
   <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug_XP|x64'" Label="PropertySheets">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Project="..\zlib.props" />
+  </ImportGroup>
   <PropertyGroup Label="UserMacros" />
   <PropertyGroup>
     <OutDir>$(SolutionDir)\.lib\$(Configuration)\</OutDir>

Solutions/zlib.props

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ImportGroup Label="PropertySheets" />
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup />
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <AdditionalIncludeDirectories>$(ZLIB_INCLUDE);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Platform)'=='Win32'">
+    <Link>
+      <AdditionalLibraryDirectories>$(AGS_ZLIB_LIB);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalDependencies>zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+    <Lib>
+      <AdditionalLibraryDirectories>$(AGS_ZLIB_LIB);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalDependencies>zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Lib>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Platform)'=='x64'">
+    <Link>
+      <AdditionalLibraryDirectories>$(AGS_ZLIB_LIB_X64);$(AGS_ZLIB_LIB)\..\x64;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalDependencies>zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+    <Lib>
+      <AdditionalLibraryDirectories>$(AGS_ZLIB_LIB_X64);$(AGS_ZLIB_LIB)\..\x64;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalDependencies>zlib.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Lib>
+  </ItemDefinitionGroup>
+  <ItemGroup />
+</Project>

ci/emscripten/Dockerfile

@@ -108,8 +108,8 @@ RUN echo "## Create emscripten user (1000:1000)" \
 
 # ------------------------------------------------------------------------------
 
-RUN embuilder.py build --lto libstubs libc libc++ libc++abi sdl2 ogg vorbis
-RUN embuilder.py build libstubs libc libc++ libc++abi sdl2 ogg vorbis
+RUN embuilder.py build --lto libstubs libc libc++ libc++abi sdl2 ogg vorbis zlib
+RUN embuilder.py build libstubs libc libc++ libc++abi sdl2 ogg vorbis zlib
 
 # ------------------------------------------------------------------------------
 # Use commonly used /src as working directory

ci/linux/Dockerfile

@@ -69,6 +69,19 @@ RUN curl -fLsS "https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSI
   ./configure --system-curl && make -j$(getconf _NPROCESSORS_ONLN) && make install && \
   rm -r /tmp/cmake-$CMAKE_VERSION
 
+# Build and install zlib-ng
+ARG ZLIB_VERSION=2.1.3
+RUN cd /tmp && \
+  curl -fLsS "https://github.com/zlib-ng/zlib-ng/archive/refs/tags/$ZLIB_VERSION.tar.gz" --output zlib.tar.gz && \
+  tar -xvzf zlib.tar.gz && \
+  mv zlib-ng-$ZLIB_VERSION zlib &&  \
+  cd /tmp/zlib  && \
+  mkdir /tmp/zlib/build && \
+  cd /tmp/zlib/build && \
+  cmake -DZLIB_ENABLE_TESTS=0 -DZLIBNG_ENABLE_TESTS=0 -DWITH_GTEST=0 -DZLIB_COMPAT=1 ..  && make -j$(getconf _NPROCESSORS_ONLN) && make install && \
+  rm /usr/local/lib/libz.*so* && \
+  rm -r /tmp/zlib
+
 # Build newer libogg
 ARG LIBOGG_VERSION=1.3.5
 RUN curl -fLsS "https://github.com/xiph/ogg/releases/download/v${LIBOGG_VERSION}/libogg-${LIBOGG_VERSION}.tar.xz" --output /tmp/libogg-${LIBOGG_VERSION}.tar.xz && \