Merge branch 'dev' into gcs-max-direct-upload-size

.clang-format

@@ -7,7 +7,9 @@ AlignConsecutiveAssignments: false
 AlignConsecutiveDeclarations: false
 AlignEscapedNewlinesLeft: true
 AlignOperands:   true
-AlignTrailingComments: true
+AlignTrailingComments:
+  Kind: Always
+  OverEmptyLines: 0
 AllowAllParametersOfDeclarationOnNextLine: true
 AllowShortBlocksOnASingleLine: false
 AllowShortCaseLabelsOnASingleLine: false
@@ -59,6 +61,7 @@ IncludeCategories:
     Priority:        3
 IncludeIsMainRegex: '([-_](test|unittest))?$'
 IndentCaseLabels: true
+IndentRequiresClause: true
 IndentWidth:     2
 IndentWrappedFunctionNames: false
 JavaScriptQuotes: Leave
@@ -76,6 +79,8 @@ PenaltyExcessCharacter: 1000000
 PenaltyReturnTypeOnItsOwnLine: 200
 PointerAlignment: Left
 ReflowComments:  true
+RequiresClausePosition: OwnLine
+RequiresExpressionIndentation: OuterScope
 SortIncludes:    true
 SpaceAfterCStyleCast: false
 SpaceAfterTemplateKeyword: true

.dockerignore

@@ -1,5 +1,5 @@
 build
 builds
+**/buildtrees/*
 scripts
 !scripts/capnpupdate.sh
-.git

.github/workflows/backport.yml

@@ -1,6 +1,6 @@
 name: Backport
 on:
-  pull_request:
+  pull_request_target:
     types:
       - closed
       - labeled
@@ -11,10 +11,20 @@ on:
 
 jobs:
   backport:
-    runs-on: ubuntu-22.04
     name: Backport
+    runs-on: ubuntu-22.04
+    if: >
+      github.event.pull_request.merged &&
+      (
+        github.event.action == 'closed' ||
+        (
+          github.event.action == 'labeled' &&
+          contains(github.event.label.name, 'backport')
+        )
+      )
     steps:
       - name: Backport
-        uses: tibdex/backport@v1
+        uses: zephyrproject-rtos/action-backport@7e74f601d11eaca577742445e87775b5651a965f #tag=v2.0.3-3
         with:
+          issue_labels: Backport
           github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/build-docs.yml

@@ -39,7 +39,7 @@ jobs:
         shell: bash
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
 
       - name: Install Doxygen (linux only)
         run: |

.github/workflows/build-rtools40.yml

@@ -8,8 +8,6 @@ jobs:
     strategy:
       matrix:
         include: [
-          { msystem: MINGW64, toolchain: x86_64 },
-          { msystem: MINGW32, toolchain: i686 },
           { msystem: ucrt64, toolchain: "ucrt-x86_64" }
         ]
       fail-fast: false
@@ -19,17 +17,28 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 1
+      - name: Install rtools40 if needed
+        uses: r-windows/install-rtools@master
       - name: Building TileDB with Rtools40
         run: |
           cd ${GITHUB_WORKSPACE}/.github/workflows/mingw-w64-tiledb
           pacman -Sy
           makepkg-mingw --noconfirm --syncdeps
         env:
           TILEDB_HOME: ${{ github.workspace }}
-          MINGW_INSTALLS: ${{ matrix.msystem }}
+          MINGW_ARCH: ${{ matrix.msystem }}
         shell: c:\rtools40\usr\bin\bash.exe --login {0}
       - name: "Upload binaries"
         uses: actions/upload-artifact@v3
         with:
           name: mingw-w64-${{ matrix.msystem }}-tiledb
           path: .github/workflows/mingw-w64-tiledb/*.pkg.tar.*
+      - name: "Print log files (failed build only)"
+        shell: bash
+        run: |
+          source $GITHUB_WORKSPACE/scripts/ci/print_logs.sh
+          echo "# ------------------------------------"
+          find $BUILD_BUILDDIRECTORY/dist
+          echo "# --------vcpkg----------------------------"
+          find $BUILD_BUILDDIRECTORY/vcpkg_installed
+        if: failure() # only run this job if the build step failed

.github/workflows/build-ubuntu20.04-backwards-compatibility.yml

@@ -3,7 +3,8 @@ on:
   workflow_call:
 
 env:
-  CXX: g++
+  CXX: g++-10
+  CC: gcc-10
 
 jobs:
   build:
@@ -23,7 +24,7 @@ jobs:
         shell: bash
 
       # Need this for virtualenv and arrow tests if enabled
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: '3.8'
       - run: |
@@ -58,7 +59,7 @@ jobs:
           - ubuntu-20.04
         # Note: v2_1_0 arrays were never created so its currently skipped
         # Note: This matrix is used to set the value of TILEDB_COMPATIBILITY_VERSION
-        tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_0", "v2_13_0", "v2_14_0", "v2_15_0"]
+        tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_0", "v2_13_0", "v2_14_0", "v2_15_0", "v2_16_0"]
     timeout-minutes: 30
     name: ${{ matrix.tiledb_version }}
     steps:

.github/workflows/build-windows.yml

@@ -11,21 +11,20 @@ env:
   TILEDB_SERIALIZATION: OFF
   TILEDB_STATIC: OFF
   TILEDB_TOOLS: ON
-  TILEDB_BASE_BOOTSTRAP_OPTIONS: '-EnableVerbose -EnableStaticTileDB -EnableBuildDeps'
+  TILEDB_BASE_BOOTSTRAP_OPTIONS: '-EnableVerbose -EnableStaticTileDB'
 
 jobs:
   build:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [windows-2019]
+        os: [windows-2019, windows-2022]
         #considering https://stackoverflow.com/questions/65035256/how-to-access-matrix-variables-in-github-actions
-        environ: [azure, s3, serialization]
+        environ: [azure, s3, gcs, serialization]
         include:
           - environ: 'azure'
             TILEDB_AZURE: ON
-            TILEDB_USE_CUSTOM_NODEJS: OFF
             TILEDB_ARROW_TESTS: ON
             TILEDB_SERIALIZATION: OFF
             TILEDB_S3: OFF
@@ -36,20 +35,33 @@ jobs:
             TILEDB_SERIALIZATION: OFF
             TILEDB_AZURE: OFF
             TILEDB_WEBP: OFF
+          - environ: 'gcs'
+            TILEDB_GCS: ON
+            TILEDB_ARROW_TESTS: ON
+            TILEDB_SERIALIZATION: OFF
+            TILEDB_AZURE: OFF
+            TILEDB_S3: OFF
+            TILEDB_WEBP: OFF
           - environ: 'serialization'
             TILEDB_S3: OFF
             TILEDB_AZURE: OFF
             TILEDB_ARROW_TESTS: OFF
             TILEDB_SERIALIZATION: ON
+            TILEDB_WEBP: ON
+
+    name: ${{ matrix.os }} - ${{ matrix.environ }}
+
     env:
       TILEDB_HOME: ${{ github.workspace }}
       TILEDB_GA_IMAGE_NAME: ${{ matrix.os }}
       TILEDB_S3: ${{ matrix.TILEDB_S3 }}
       TILEDB_AZURE: ${{ matrix.TILEDB_AZURE }} #azure }}
+      TILEDB_GCS: ${{ matrix.TILEDB_GCS }} #gcs }}
       TILEDB_SERIALIZATION: ${{ matrix.TILEDB_SERIALIZATION }} #serialization }}
       TILEDB_ARROW_TESTS: ${{ matrix.TILEDB_ARROW_TESTS }}
       TILEDB_WEBP: ${{ matrix.TILEDB_WEBP }}
-      TILEDB_CMAKE_BUILD_TYPE: 'RelWithDebInfo'
+      TILEDB_CMAKE_BUILD_TYPE: 'Release'
+      VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite'
     steps:
       - name: 'tiledb env prep'
         run: |
@@ -66,6 +78,14 @@ jobs:
           write-host "BUILD_BUILDDIRECTORY=$env:BUILD_BUILDDIRECTORY" *>> $env:GITHUB_ENV
           write-host "BUILD_SOURCESDIRECTORY=$env:BUILD_SOURCESDIRECTORY" *>> $env:GITHUB_ENV
 
+      # Configure required environment variables for vcpkg to use
+      # GitHub's Action Cache
+      - uses: actions/github-script@v6
+        with:
+          script: |
+            core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+
       - name: 'Print env'
         shell: cmd
         run: |
@@ -99,28 +119,46 @@ jobs:
           if [[ "$TILEDB_ARROW_TESTS" == "ON" ]]; then
             pip install pyarrow pybind11 numpy
           fi
+
       - name: Prepare git
         run: git config --global core.autocrlf false
+
       - uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+
+      - name: Prevent vcpkg from building debug variants
+        shell: python
+        run: |
+          import os
+
+          workspace = os.environ["GITHUB_WORKSPACE"]
+          triplet_paths = [os.path.join(workspace, "external", "vcpkg", "triplets"),
+                          os.path.join(workspace, "ports", "triplets")]
+
+          for triplet_path in triplet_paths:
+            for path, dnames, fnames in os.walk(triplet_path):
+              for fname in fnames:
+                fname = os.path.join(path, fname)
+                print(fname)
+                with open(fname, "rb") as handle:
+                  contents = handle.read()
+                # If a file is already patched we skip patching it again because
+                # this affects the hashes calculated by vcpkg for caching
+                if b"VCPKG_BUILD_TYPE release" in contents:
+                  continue
+                contents += b"\nset(VCPKG_BUILD_TYPE release)\n"
+                with open(fname, "wb") as handle:
+                  handle.write(contents)
+
       - name: core tiledb windows build
         run: |
           $ErrorView = "NormalView"
 
           #directory $env:BUILD_BUILDDIRECTORY was created in step 'tiledb env prep'
           cd $env:BUILD_BUILDDIRECTORY
 
-          $VSCategory = "Enterprise" # alternate 'Community'
-          if ($env:TILEDB_GA_IMAGE_NAME -eq "windows-2019") {
-            if (!(Test-Path "C:\Program Files (x86)\Microsoft Visual Studio\2019\${VSCategory}\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin")) {
-              Write-Host "ERROR***: Missing C:\Program Files (x86)\Microsoft Visual Studio\2019\${VSCategory}\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin"
-              exit $LastExitCode
-            }
-            $env:Path += ";C:\Program Files (x86)\Microsoft Visual Studio\2019\${VSCategory}\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin"
-          } else {
-            Write-Host "Unknown image name: '$($env:TILEDB_GA_IMAGE_NAME)'"
-            exit $LastExitCode
-          }
-
           # allow double-checking path
           cmd /c "echo $PATH"
 
@@ -135,8 +173,7 @@ jobs:
             $bootstrapOptions = $bootstrapOptions + " -_EnableCrc32"
           }
           if ($env:TILEDB_GCS -eq "ON") {
-            $bootstrapOptions = "-EnableGCS " + $bootstrapOptions
-            #NOTE: GCS simulator not yet actually in place.
+            $bootstrapOptions = "-EnableGcs " + $bootstrapOptions
           }
           if ($env:TILEDB_SERIALIZATION -eq "ON") {
             $bootstrapOptions = "-EnableSerialization " + $bootstrapOptions
@@ -256,6 +293,10 @@ jobs:
             tasklist | findstr /i node.exe
           }
 
+          if ($env:TILEDB_GCS -eq "ON") {
+            & "$env:BUILD_SOURCESDIRECTORY\scripts\install-run-gcs-emu.ps1"
+          }
+
           # Actually run tests
 
           $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\$CMakeBuildType\tiledb_unit.exe -d=yes"
@@ -285,7 +326,7 @@ jobs:
             }
           }
           # Build the examples
-          cmake --build $env:BUILD_BUILDDIRECTORY --target examples --config $CMakeBuildType -- /verbosity:minimal
+          cmake --build $env:BUILD_BUILDDIRECTORY --target examples --config $CMakeBuildType -v
 
           if ($LastExitCode -ne 0) {
             Write-Host "Examples failed to build."
@@ -368,9 +409,9 @@ jobs:
           cd build
 
           # Build zip artifact
-          cmake -A X64 -DCMAKE_PREFIX_PATH="$env:BUILD_BUILDDIRECTORY\dist" ..
+          cmake -A X64 -DCMAKE_PREFIX_PATH="$env:BUILD_BUILDDIRECTORY\dist;$env:BUILD_BUILD_DIRECTORY\vcpkg_installed\x64-windows-mod" ..
 
-          cmake --build . --config $CMakeBuildType -- /verbosity:minimal
+          cmake --build . --config $CMakeBuildType -v
 
           #.\$CMakeBuildType\ExampleExe.exe
           $cmd = ".\$CMakeBuildType\ExampleExe.exe"
@@ -423,3 +464,13 @@ jobs:
             echo TILEDB_CI_SUCCESS != 1, exiting.
             exit 1;
           fi
+
+      - name: "Print log files (failed build only)"
+        shell: bash
+        run: |
+          source $GITHUB_WORKSPACE/scripts/ci/print_logs.sh
+          echo "# ------------------------------------"
+          find $BUILD_BUILDDIRECTORY/dist
+          echo "# --------vcpkg----------------------------"
+          find $BUILD_BUILDDIRECTORY/vcpkg_installed
+        if: ${{ failure() }} # only run this job if the build step failed

.github/workflows/ci-linux_mac.yml

@@ -33,14 +33,19 @@ on:
         description: 'Job timeout (minutes)'
         required: false
         type: number
+      asan:
+        default: false
+        description: 'Enable Address Sanitizer'
+        required: false
+        type: boolean
 
 env:
   BACKWARDS_COMPATIBILITY_ARRAYS: OFF
   TILEDB_CI_BACKEND: ${{ inputs.ci_backend }}
   TILEDB_CI_OS: runner.os
   CXX: ${{ inputs.matrix_compiler_cxx }}
   CC: ${{ inputs.matrix_compiler_cc }}
-  bootstrap_args: "--enable-ccache ${{ inputs.bootstrap_args }}"
+  bootstrap_args: "--enable-ccache ${{ inputs.bootstrap_args }} ${{ inputs.asan && '--enable-sanitizer=address' || '' }}"
   VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite'
   SCCACHE_GHA_ENABLED: "true"
 
@@ -140,6 +145,9 @@ jobs:
       - name: 'Test libtiledb'
         id: test
         shell: bash
+        env:
+          ASAN_OPTIONS: ${{ inputs.asan && 'detect_leaks=0' || '' }}
+          LD_PRELOAD: ${{ inputs.asan && '/usr/lib/x86_64-linux-gnu/libasan.so.6' || '' }}
         run: |
           set -e pipefail
 
@@ -172,6 +180,11 @@ jobs:
 
           source $GITHUB_WORKSPACE/scripts/ci/posix/build-services-stop.sh
 
+          # Skip examples and benchmarks if ASAN is enabled
+          if [[ "${{ inputs.asan }}" == "true" ]]; then
+            exit 0
+          fi
+
           ###################################################
           # Build and run Examples
           make -C $GITHUB_WORKSPACE/build -j3 examples
@@ -198,7 +211,7 @@ jobs:
 
       - name: 'Upload failure artifacts (macOS)' # https://github.com/actions/upload-artifact#where-does-the-upload-go
         if: ${{ failure() == true && startsWith(matrix.os, 'macos-') == true }} # only run this job if the build step failed
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           retention-days: 10
           name: "${{ matrix.os }}.coredumps.${{ github.job }}.${{ github.run_number }}.${{github.run_id}}.${{github.run_attempt}}"