build_test.yml

# Copyright (c) the JPEG XL Project Authors. All rights reserved.
#
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

# Workflow for building and running tests.

name: Build/Test
on:
  push:
    branches:
      - main
      - v*.*.x
  pull_request:
    types: [opened, reopened, labeled, synchronize]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  ubuntu_build:
    name: Ubuntu Build ${{ matrix.name }}
    runs-on: ${{ matrix.os || 'ubuntu-latest' }}
    strategy:
      matrix:
        # We have one job per "name" in the matrix. Attributes are set on the
        # specific job names.
        name: [release, debug, asan, msan, scalar]
        include:
          - name: release
            test_in_pr: true
            # Track static stack size on build and check it doesn't exceed 3 kB.
            env_stack_size: 1
            max_stack: 3000
            # Conformance tooling test requires numpy.
            apt_pkgs: graphviz python3-numpy
          - name: lowprecision
            mode: release
            test_in_pr: true
            cmake_args: -DCMAKE_CXX_FLAGS=-DJXL_HIGH_PRECISION=0
          - name: debug
            # Runs on AVX3 CPUs require more stack than others. Make sure to
            # test on AVX3-enabled CPUs when changing this value.
            env_test_stack_size: 4000
          # Build scalar-only hwy instructions.
          - name: scalar
            mode: release
            cxxflags: -DHWY_COMPILE_ONLY_SCALAR
          # Disabling optional features to speed up msan build a little bit.
          - name: msan
            skip_install: true
            cmake_args: >-
              -DJPEGXL_ENABLE_DEVTOOLS=OFF -DJPEGXL_ENABLE_PLUGINS=OFF
              -DJPEGXL_ENABLE_VIEWERS=OFF
          - name: asan
            skip_install: true
          - name: coverage
            apt_pkgs: gcovr
            # Coverage builds require a bit more RAM.
            env_test_stack_size: 2048
          # Build with support for decoding to JPEG bytes disabled. Produces a
          # smaller build if only decoding to pixels is needed.
          - name: release-nojpeg
            mode: release
            cxxflags: -DJXL_DEBUG_ON_ABORT=0
            cmake_args: >-
              -DJPEGXL_ENABLE_TRANSCODE_JPEG=OFF
              -DJPEGXL_ENABLE_PLUGINS=OFF
              -DJPEGXL_ENABLE_VIEWERS=OFF
          # Builds with gcc in release mode
          - name: release:gcc8
            mode: release
            apt_pkgs: gcc-8 g++-8
            cmake_args: >-
              -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8
          # Builds with clang-5 in release mode
          - name: release:clang-5
            os: ubuntu-18.04
            mode: release
            # TODO(eustas): investigate, why static brotli library is not found.
            skip_install: true
            apt_pkgs: clang-5.0
            cmake_args: >-
              -DCMAKE_C_COMPILER=clang-5.0 -DCMAKE_CXX_COMPILER=clang++-5.0
              -DJPEGXL_ENABLE_PLUGINS=OFF

    env:
      CCACHE_DIR: ${{ github.workspace }}/.ccache
      # Whether we track the stack size.
      STACK_SIZE: ${{ matrix.env_stack_size }}
      TEST_STACK_LIMIT: ${{ matrix.env_test_stack_size }}
      WILL_RUN_TESTS: ${{ github.event_name == 'push' || (github.event_name == 'pull_request' && (matrix.test_in_pr || contains(github.event.pull_request.labels.*.names, 'CI:full'))) }}

    steps:
    - name: Install build deps
      run: |
        sudo apt update
        sudo apt install -y \
          ccache \
          clang-7 \
          cmake \
          doxygen \
          libbenchmark-dev \
          libbenchmark-tools \
          libbrotli-dev \
          libgdk-pixbuf2.0-dev \
          libgflags-dev \
          libgif-dev \
          libgtest-dev \
          libgtk2.0-dev  \
          libjpeg-dev \
          libopenexr-dev \
          libpng-dev \
          libwebp-dev \
          ninja-build \
          pkg-config \
          xvfb \
          ${{ matrix.apt_pkgs }} \
        #
        echo "CC=clang-7" >> $GITHUB_ENV
        echo "CXX=clang++-7" >> $GITHUB_ENV
    - name: Checkout the source
      uses: actions/checkout@v2
      with:
        submodules: true
        fetch-depth: 2
    - name: Sphinx dependencies
      # Dependencies for sphinx HTML documentation
      if: matrix.name == 'release'
      run: |
        pip3 install -r doc/sphinx/requirements.txt
    - name: Git environment
      id: git-env
      run: |
        echo "::set-output name=parent::$(git rev-parse ${{ github.sha }}^)"
      shell: bash
    - name: ccache
      uses: actions/cache@v2
      with:
        path: ${{ env.CCACHE_DIR }}
        # When the cache hits the key it is not updated, so if this is a rebuild
        # of the same Pull Request it will reuse the cache if still around. For
        # either Pull Requests or new pushes to main, this will use the parent
        # hash as the starting point from the restore-keys entry.
        key: ${{ runner.os }}-${{ github.sha }}-${{ matrix.name }}
        restore-keys: |
          ${{ runner.os }}-${{ steps.git-env.outputs.parent }}-${{ matrix.name }}
    - name: Build
      if: matrix.name != 'coverage' || env.WILL_RUN_TESTS == 'true'
      run: |
        mkdir -p ${CCACHE_DIR}
        echo "max_size = 200M" > ${CCACHE_DIR}/ccache.conf
        mode="${{ matrix.mode }}"
        build_tests=$([ "$WILL_RUN_TESTS" == "true" ] && echo "ON" || echo "OFF")
        [[ -n "${mode}" ]] || mode="${{ matrix.name }}"
        ./ci.sh ${mode} -DJPEGXL_FORCE_SYSTEM_BROTLI=ON \
          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
          -DCMAKE_C_COMPILER_LAUNCHER=ccache \
          -DBUILD_TESTING=${build_tests} \
          ${{ matrix.cmake_args }}
      env:
        SKIP_TEST: 1
        CMAKE_CXX_FLAGS: ${{ matrix.cxxflags }}
    - name: ccache stats
      run: ccache --show-stats
    - name: Build stats ${{ matrix.name }}
      if: matrix.mode == 'release' || matrix.name == 'release'
      run: |
        tools/build_stats.py --save build/stats.json \
          --max-stack ${{ matrix.max_stack || '0' }} \
          cjxl djxl libjxl.so libjxl_dec.so
    # Check that we can build the example project against the installed libs.
    - name: Install and build examples
      if: |
        (matrix.mode == 'release' || matrix.name == 'release') &&
        !matrix.skip_install
      run: |
        set -x
        sudo cmake --build build -- install
        cmake -Bbuild-example -Hexamples -G Ninja
        cmake --build build-example
        if ldd build-example/decode_oneshot_static | grep libjxl; then
          echo "decode_oneshot_static is not using the static lib" >&2
          exit 1
        fi
        # Test that the built binaries run.
        echo -e -n "PF\n1 1\n-1.0\nrrrrggggbbbb" > test.pfm
        build-example/encode_oneshot test.pfm test.jxl
        build-example/encode_oneshot_static test.pfm test-static.jxl
        build-example/decode_oneshot test.jxl dec.pfm dec.icc
        build-example/decode_oneshot_static test.jxl dec-static.pfm dec-static.icc
    # Run the tests on push and when requested in pull_request.
    - name: Test ${{ matrix.mode }}
      if: env.WILL_RUN_TESTS == 'true'
      run: |
        ./ci.sh test ${{ matrix.ctest_args }}
    # Print the running time summary for the slowest tests.
    - name: Test runtime stats
      run: |
        sort build/Testing/Temporary/CTestCostData.txt -k 3 -n | tail -n 20 || true
    - name: Build HTML documentation (sphinx/readthetdocs)
      if: matrix.name == 'release'
      run: |
        cmake --build build -- rtd-html
    - name: Coverage report
      if: github.event_name == 'push' && matrix.name == 'coverage'
      run: |
        ./ci.sh coverage_report
    - name: Coverage upload to Codecov
      if: github.event_name == 'push' && matrix.name == 'coverage'
      uses: codecov/codecov-action@v2
      with:
        flags: unittests
        files: build/coverage.xml
    - name: Fast benchmark ${{ matrix.mode }}
      if: |
        matrix.name != 'coverage' && (github.event_name == 'push' ||
        (github.event_name == 'pull_request' && (
         matrix.test_in_pr ||
         contains(github.event.pull_request.labels.*.names, 'CI:full'))))
      run: |
        STORE_IMAGES=0 ./ci.sh fast_benchmark
    # Run gbench once, just to make sure it runs, not for actual benchmarking.
    # This doesn't work on msan because we use gbench library from the system
    # which is not instrumented by msan.
    - name: gbench check
      if: |
        matrix.name == 'release' || (
          github.event_name == 'push' && matrix.name != 'msan')
      run: |
        ./ci.sh gbench --benchmark_min_time=0

  cross_compile_ubuntu:
    name: Cross-compiling ${{ matrix.build_target }} ${{ matrix.lowprecision }}
    runs-on: [ubuntu-22.04]
    strategy:
      matrix:
        include:
          - arch: i386
            lowprecision:
            test_in_pr: true
            build_target: i686-linux-gnu

    env:
      BUILD_DIR: build
      WILL_RUN_TESTS: ${{ github.event_name == 'push' || (github.event_name == 'pull_request' && (matrix.test_in_pr || contains(github.event.pull_request.labels.*.names, 'CI:full'))) }}

    steps:
    - name: Setup apt
      shell: bash
      run: |
        set -x
        sudo apt-get update -y
        sudo apt-get install -y curl gnupg ca-certificates
        sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 1E9377A2BA9EF27F

        if [[ "${{ matrix.arch }}" != "amd64" ]]; then
          sudo dpkg --add-architecture "${{ matrix.arch }}"

          # Update the sources.list with the split of supported architectures.
          bkplist="/etc/apt/sources.list.bkp"
          sudo mv /etc/apt/sources.list "${bkplist}"

          newlist="/etc/apt/sources.list"
          sudo rm -f "${newlist}"

          main_list="amd64"
          port_list=""
          if [[ "${{ matrix.arch }}" == "i386" ]]; then
            main_list="amd64,i386"
          else
            port_list="${{ matrix.arch }}"
          fi

          if [[ -n "${port_list}" ]]; then
            port_url="http://ports.ubuntu.com/ubuntu-ports/"
            grep -v -E '^#' "${bkplist}" |
              sed -E "s;^deb (http[^ ]+) (.*)\$;deb [arch=${{ matrix.arch }}] ${port_url} \\2;" \
              | sudo tee -a "${newlist}"
          fi
          grep -v -E '^#' "${bkplist}" |
            sed -E "s;^deb (http[^ ]+) (.*)\$;deb [arch=${main_list}] \\1 \\2\ndeb-src [arch=${main_list}] \\1 \\2;" \
            | sudo tee -a "${newlist}"
        fi

    - name: Install build deps
      shell: bash
      run: |
        set -x
        sudo apt update
        pkgs=(
          # Build dependencies
          cmake
          doxygen
          graphviz
          ninja-build
          pkg-config
          qemu-user-static
          xvfb

          # Toolchain for cross-compiling.
          clang-14
          # libclang-common-7-dev:${{ matrix.arch }}
          libc6-dev-${{ matrix.arch }}-cross
          libstdc++-11-dev-${{ matrix.arch }}-cross
          libstdc++-11-dev:${{ matrix.arch }}

          # Dependencies
          libbrotli-dev:${{ matrix.arch }}
          libgif-dev:${{ matrix.arch }}
          libjpeg-dev:${{ matrix.arch }}
          libpng-dev:${{ matrix.arch }}
          libwebp-dev:${{ matrix.arch }}

          # For OpenEXR:
          libilmbase-dev:${{ matrix.arch }}
          libopenexr-dev:${{ matrix.arch }}

          # GTK plugins
          libgdk-pixbuf2.0-dev:${{ matrix.arch }}
          libgtk2.0-dev:${{ matrix.arch }}

          # QT
          libqt5x11extras5-dev:${{ matrix.arch }}
          qtbase5-dev:${{ matrix.arch }}
        )
        if [[ "${{ matrix.build_target }}" != "x86_64-linux-gnu" ]]; then
          pkgs+=(
            binutils-${{ matrix.build_target }}
            gcc-${{ matrix.build_target }}
          )
        fi
        if [[ "${{ matrix.arch }}" != "i386" ]]; then
          pkgs+=(
            # TCMalloc
            libgoogle-perftools-dev:${{ matrix.arch }}
            libgoogle-perftools4:${{ matrix.arch }}
            libtcmalloc-minimal4:${{ matrix.arch }}
            libunwind-dev:${{ matrix.arch }}
          )
        fi
        DEBIAN_FRONTEND=noninteractive sudo apt install -y "${pkgs[@]}"
        echo "CC=clang-14" >> $GITHUB_ENV
        echo "CXX=clang++-14" >> $GITHUB_ENV
    - name: Checkout the source
      uses: actions/checkout@v2
      with:
        submodules: true
        fetch-depth: 1
    - name: Build
      run: |
        ./ci.sh release \
          -DJPEGXL_FORCE_SYSTEM_BROTLI=ON \
          -DJPEGXL_BUNDLE_GFLAGS=ON \
          -DJPEGXL_ENABLE_JNI=OFF \
          ${{ matrix.cmake_args }}
      env:
        SKIP_TEST: 1
        BUILD_TARGET: ${{ matrix.build_target }}
    - name: Build stats ${{ matrix.build_target }}
      run: |
        tools/build_stats.py --save build/stats.json \
          --binutils ${{ matrix.build_target }}- \
          --max-stack ${{ matrix.max_stack || '0' }} \
          cjxl djxl libjxl.so libjxl_dec.so
    - uses: actions/upload-artifact@v2
      with:
        name: ans_test_assembly
        path: |
          build/ans_test.s
    # Run the tests on push and when requested in pull_request.
    - name: Test
      if: env.WILL_RUN_TESTS == 'true'
      run: |
        ./ci.sh test
      env:
        BUILD_TARGET: ${{ matrix.build_target }}

  windows_msys:
    name: Windows MSYS2 / ${{ matrix.arch }}
    runs-on: windows-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - arch: x86_64
            msystem: mingw64
          - arch: i686
            msystem: mingw32
    defaults:
      run:
        shell: msys2 {0}
    steps:
      - name: Checkout the source
        uses: actions/checkout@v2
        with:
          submodules: true
          fetch-depth: 1
      - uses: msys2/setup-msys2@v2
        with:
          msystem: ${{ matrix.msystem }}
          update: true
          path-type: inherit
          install: >-
            base-devel
            git
            mingw-w64-${{ matrix.arch }}-brotli
            mingw-w64-${{ matrix.arch }}-cmake
            mingw-w64-${{ matrix.arch }}-gcc
            mingw-w64-${{ matrix.arch }}-gflags
            mingw-w64-${{ matrix.arch }}-giflib
            mingw-w64-${{ matrix.arch }}-gtest
            mingw-w64-${{ matrix.arch }}-libavif
            mingw-w64-${{ matrix.arch }}-libjpeg-turbo
            mingw-w64-${{ matrix.arch }}-libpng
            mingw-w64-${{ matrix.arch }}-libwebp
            mingw-w64-${{ matrix.arch }}-ninja

      - name: CMake configure
        # AVX2 tests fail with segfault when built in MSYS2.
        run: |
          cmake \
            -DCMAKE_BUILD_TYPE=Release \
            -DCMAKE_CXX_FLAGS="-DHWY_DISABLED_TARGETS=\"HWY_AVX2|HWY_AVX3\"" \
            -DJPEGXL_ENABLE_JNI=OFF \
            -DJPEGXL_ENABLE_MANPAGES=OFF \
            -DJPEGXL_FORCE_SYSTEM_BROTLI=ON \
            -DJPEGXL_FORCE_SYSTEM_GTEST=ON \
            -B build \
            -G Ninja
      - name: CMake build
        run: cmake --build build
      - name: Test
        if: |
          github.event_name == 'push' ||
          (github.event_name == 'pull_request' &&
           contains(github.event.pull_request.labels.*.names, 'CI:full'))
        # LibraryCLinkageTest doesn't work in here because it needs the DLL
        # which is not installed.
        run: ctest --test-dir build --parallel 2 --output-on-failure -E LibraryCLinkageTest

  wasm32_build:
    name: WASM wasm32/${{ matrix.variant }}
    runs-on: ubuntu-latest
    env:
      CCACHE_DIR: ${{ github.workspace }}/.ccache
      BUILD_TARGET: wasm32
      EM_VERSION: 3.1.1
      NODE_VERSION: 18

    strategy:
      matrix:
        include:
          - variant: scalar
          - variant: simd

    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
        fetch-depth: 1
    - name: Install build deps
      shell: bash
      run: |
        set -x
        sudo apt update
        pkgs=(
          # Build dependencies
          ccache
          cmake
          doxygen
          ninja-build
          pkg-config
        )
        DEBIAN_FRONTEND=noninteractive sudo apt install -y "${pkgs[@]}"

    - name: Git environment
      id: git-env
      run: |
        echo "::set-output name=parent::$(git rev-parse ${{ github.sha }}^)"
      shell: bash
    - name: ccache
      uses: actions/cache@v2
      with:
        path: ${{ env.CCACHE_DIR }}
        key: ${{ runner.os }}-${{ github.sha }}-${{ matrix.variant }}
        restore-keys: |
          ${{ runner.os }}-${{ steps.git-env.outputs.parent }}-${{ matrix.variant }}

    - name: Install node
      uses: actions/setup-node@v3
      with:
        node-version: ${{env.NODE_VERSION}}

    - name: Get non-EMSDK node path
      run: which node >> $HOME/.base_node_path

    - name: Install emsdk
      uses: mymindstorm/setup-emsdk@v11
      # TODO(deymo): We could cache this action but it doesn't work when running
      # in a matrix.
      with:
        version: ${{env.EM_VERSION}}
        no-cache: true

    - name: Set EMSDK node version
      run: |
        echo "NODE_JS='$(cat $HOME/.base_node_path)'" >> $EM_CONFIG
        emsdk construct_env

    # TODO(deymo): Build and install other dependencies like libpng, libjpeg,
    # etc.
    - name: Build
      run: |
        mkdir -p ${CCACHE_DIR}
        echo "max_size = 200M" > ${CCACHE_DIR}/ccache.conf
        if [[ "${{ matrix.variant }}" == "simd" ]]; then
          export ENABLE_WASM_SIMD=1
        fi
        ./ci.sh release \
          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
          -DCMAKE_C_COMPILER_LAUNCHER=ccache
      env:
        SKIP_TEST: 1
    - name: ccache stats
      run: ccache --show-stats

    - name: Test
      if: |
        github.event_name == 'push' ||
        (github.event_name == 'pull_request' &&
         contains(github.event.pull_request.labels.*.names, 'CI:full'))
      run: |
        ./ci.sh test