From 6dff6fae0b09c16177fe85ec9b62c128bd721ad7 Mon Sep 17 00:00:00 2001
From: Robert Bindar <robertbindar@gmail.com>
Date: Tue, 16 Apr 2024 20:37:43 +0300
Subject: [PATCH] Add end-to-end cloud test workflow with Microsoft Entra ID.
 (#4841)

This PR adds the foundations for testing connectivity of TileDB to the
real clouds by adding a new workflow named `test-cloud-e2e`. Currently
only Azure with Microsoft Entra ID authentication is provided.

Authentication happens [with OpenID
Connect](https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-azure).
A new environment was created that has read-only access to the `core`
container of the `tiledbci` storage account, and we test that the file
`test.txt` in that container exists. The test is configured with
environment variables, making it reusable for subsequent tests with
other cloud providers.

---
TYPE: NO_HISTORY

---------

Co-authored-by: Theodore Tsirpanis <theodore.tsirpanis@tiledb.com>
---
 .github/workflows/test-cloud-e2e.yml | 107 +++++++++++++++++++++++++++
 test/src/unit-vfs.cc                 |  20 +++++
 2 files changed, 127 insertions(+)
 create mode 100644 .github/workflows/test-cloud-e2e.yml

diff --git a/.github/workflows/test-cloud-e2e.yml b/.github/workflows/test-cloud-e2e.yml
new file mode 100644
index 00000000000..845389c52b6
--- /dev/null
+++ b/.github/workflows/test-cloud-e2e.yml
@@ -0,0 +1,107 @@
+name: End-to-End cloud service tests
+on:
+  workflow_dispatch:
+    inputs:
+      run_azure:
+        description: 'Run Azure tests'
+        required: true
+        default: true
+        type: boolean
+  push:
+    branches:
+      - dev
+      - release-*
+
+env:
+  VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite'
+  SCCACHE_GHA_ENABLED: "true"
+
+jobs:
+  azure:
+    runs-on: ubuntu-latest
+    if: inputs.run_azure != 'false'
+    environment: azure-e2e-test
+    env:
+      bootstrap_args: --enable-azure --enable-ccache
+    permissions:
+      id-token: write # Get OIDC token for authentication to Azure
+    name: Azure
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      # Configure required environment variables for vcpkg to use
+      # GitHub's Action Cache
+      - uses: actions/github-script@v7
+        with:
+          script: |
+            core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+
+      - name: Prevent vpckg from building debug variants
+        run: python ./scripts/ci/patch_vcpkg_triplets.py
+
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.3
+
+      - name: 'Configure libtiledb'
+        id: configure
+        shell: bash
+        run: |
+          set -e pipefail
+
+          # Show CMake Version
+          cmake --version
+
+          source $GITHUB_WORKSPACE/scripts/ci/bootstrap_libtiledb.sh
+
+      - name: 'Build libtiledb'
+        id: build
+        shell: bash
+        run: |
+          set -e pipefail
+
+          #####################################################
+          # Build libtiledb using previous bootstrap
+
+          source $GITHUB_WORKSPACE/scripts/ci/build_libtiledb.sh
+
+      - name: 'Az CLI login'
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+      - name: 'Test libtiledb'
+        id: test
+        shell: bash
+        env:
+          # Allow forks to specify different values.
+          AZURE_STORAGE_ACCOUNT: ${{ vars.AZURE_STORAGE_ACCOUNT || 'tiledbci' }}
+          TILEDB_VFS_E2E_TEST_FILE: ${{ vars.AZURE_E2E_TEST_FILE || 'azure://tiledb/test.txt' }}
+        run: |
+          set -e pipefail
+
+          cd $GITHUB_WORKSPACE/build
+
+          ###################################################
+          # Run tests
+
+          # Bypass Catch2 Framework stdout interception with awk on test output
+          ./tiledb/test/tiledb_unit -d yes "[vfs-e2e]" | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
+
+      - name: 'Test status check'
+        run: |
+          # tiledb_unit is configured to set a  variable TILEDB_CI_SUCCESS=1
+          # following the test run. If this variable is not set, the build should fail.
+          # see https://github.com/TileDB-Inc/TileDB/pull/1400 (5f0623f4d3)
+          if [[ "${{ steps.test.outputs.TILEDB_CI_SUCCESS }}" -ne 1 ]]; then
+            exit 1;
+          fi
+
+      - name: "Print log files (failed build only)"
+        run: |
+          source $GITHUB_WORKSPACE/scripts/ci/print_logs.sh
+        if: ${{ failure() }} # only run this job if the build step failed
diff --git a/test/src/unit-vfs.cc b/test/src/unit-vfs.cc
index 95008baf51a..7ba6cc42207 100644
--- a/test/src/unit-vfs.cc
+++ b/test/src/unit-vfs.cc
@@ -552,6 +552,26 @@ TEMPLATE_LIST_TEST_CASE("VFS: File I/O", "[vfs][uri][file_io]", AllBackends) {
   }
 }
 
+TEST_CASE("VFS: Test end-to-end", "[.vfs-e2e]") {
+  auto test_file_ptr = getenv("TILEDB_VFS_E2E_TEST_FILE");
+  if (test_file_ptr == nullptr) {
+    FAIL("TILEDB_VFS_E2E_TEST_FILE variable is not specified");
+  }
+  URI test_file{test_file_ptr};
+
+  ThreadPool compute_tp(1);
+  ThreadPool io_tp(1);
+  // Will be configured from environment variables.
+  Config config;
+
+  VFS vfs{&g_helper_stats, &compute_tp, &io_tp, config};
+  REQUIRE(vfs.supports_uri_scheme(test_file));
+
+  uint64_t nbytes = 0;
+  require_tiledb_ok(vfs.file_size(test_file, &nbytes));
+  CHECK(nbytes > 0);
+}
+
 TEST_CASE("VFS: test ls_with_sizes", "[vfs][ls-with-sizes]") {
   ThreadPool compute_tp(4);
   ThreadPool io_tp(4);