Merge branch 'main' into fix/improve_iceberg_retries_test

.github/workflows/integration-tests.yml

@@ -56,7 +56,7 @@ jobs:
         with:
           python-version: ${{ vars.DEFAULT_PYTHON_VERSION }}
       - uses: pypa/hatch@install
-      - uses: aws-actions/configure-aws-credentials@v2
+      - uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.ASSUMABLE_ROLE_NAME }}
           aws-region: ${{ vars.DBT_TEST_ATHENA_REGION_NAME }}

.github/workflows/publish-internal.yml

@@ -0,0 +1,94 @@
+name: "Publish Internally"
+
+on:
+  workflow_call:
+    inputs:
+      package:
+        description: "Choose the package to publish"
+        type: string
+        default: "dbt-athena"
+      deploy-to:
+        description: "Choose whether to publish to test or prod"
+        type: string
+        default: "prod"
+      branch:
+        description: "Choose the branch to publish"
+        type: string
+        default: "main"
+  workflow_dispatch:
+    inputs:
+      package:
+        description: "Choose the package to publish"
+        type: string
+        default: "dbt-athena"
+      deploy-to:
+        description: "Choose whether to publish to test or prod"
+        type: string
+        default: "test"
+      branch:
+        description: "Choose the branch to publish"
+        type: string
+        default: "main"
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ inputs.deploy-to }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.branch }}
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ vars.DEFAULT_PYTHON_VERSION }}
+      - uses: pypa/hatch@install
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ vars.AWS_REGION }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      - id: package
+        run: echo "version=$(hatch version)" >> $GITHUB_OUTPUT
+        working-directory: ./${{ inputs.package }}
+      - id: published
+        run: |
+          versions_published="$(aws codeartifact list-package-versions \
+            --domain ${{ vars.AWS_DOMAIN }} \
+            --repository ${{ vars.AWS_REPOSITORY }} \
+            --format pypi \
+            --package ${{ inputs.package }} \
+            --output json \
+            --query 'versions[*].version' | jq -r '.[]' | grep "^${{ steps.package.outputs.version }}" || true )"  # suppress pipefail only here
+          echo "versions=$(echo "${versions_published[*]}"| tr '\n' ',')" >> $GITHUB_OUTPUT
+      - id: next
+        uses: dbt-labs/dbt-release/.github/actions/next-cloud-release-version@main
+        with:
+          version_number: ${{ steps.package.outputs.version }}
+          versions_published: ${{ steps.published.outputs.versions }}
+      - run: |
+          hatch version ${{ steps.next.outputs.internal_release_version }}+$(git rev-parse HEAD)
+          hatch build --clean
+          hatch run build:check-all
+        working-directory: ./${{ inputs.package }}
+      - run: |
+          export HATCH_INDEX_USER=${{ secrets.AWS_USER }}
+
+          export HATCH_INDEX_AUTH=$(aws codeartifact get-authorization-token \
+            --domain ${{ vars.AWS_DOMAIN }} \
+            --output text \
+            --query authorizationToken)
+
+          export HATCH_INDEX_REPO=$(aws codeartifact get-repository-endpoint \
+            --domain ${{ vars.AWS_DOMAIN }} \
+            --repository ${{ vars.AWS_REPOSITORY }} \
+            --format pypi \
+            --output text \
+            --query repositoryEndpoint)
+
+          hatch publish
+        working-directory: ./${{ inputs.package }}

.github/workflows/publish-pypi.yml

@@ -15,6 +15,20 @@ on:
         description: "Choose the branch to publish"
         type: string
         default: "main"
+  workflow_dispatch:
+    inputs:
+      package:
+        description: "Choose the package to publish"
+        type: string
+        default: "dbt-athena"
+      deploy-to:
+        description: "Choose whether to publish to test or prod"
+        type: string
+        default: "test"
+      branch:
+        description: "Choose the branch to publish"
+        type: string
+        default: "main"
 
 permissions:
   contents: read

.github/workflows/publish.yml

@@ -11,6 +11,14 @@ on:
         description: "Choose the branch to release from"
         type: string
         default: "main"
+      pypi-internal:
+        description: "Publish Internally"
+        type: boolean
+        default: true
+      pypi-public:
+        description: "Publish to PyPI"
+        type: boolean
+        default: false
 
 # don't attempt to release the same target in parallel
 concurrency:
@@ -30,17 +38,27 @@ jobs:
       repository: ${{ github.repository }}
     secrets: inherit
 
-  publish-dbt-athena:
+  publish-internal:
+    if: ${{ inputs.pypi-internal == true }}
+    needs: [unit-tests, integration-tests]
+    uses: ./.github/workflows/publish-internal.yml
+    with:
+      deploy-to: ${{ inputs.deploy-to }}
+      branch: ${{ inputs.branch }}
+
+  publish-pypi:
+    if: ${{ inputs.pypi-public == true }}
     needs: [unit-tests, integration-tests]
     uses: ./.github/workflows/publish-pypi.yml
     with:
       deploy-to: ${{ inputs.deploy-to }}
       branch: ${{ inputs.branch }}
 
-  publish-dbt-athena-community:
+  publish-pypi-dbt-athena-community:
+    if: ${{ inputs.pypi-public == true }}
     # dbt-athena-community is hard pinned to dbt-athena to ensure they are the same
     # this means we need to finish publishing dbt-athena before starting to build dbt-athena-community
-    needs: publish-dbt-athena
+    needs: publish-pypi
     uses: ./.github/workflows/publish-pypi.yml
     with:
       package: "dbt-athena-community"