From 86f3d9d9b82de4eecdcd779e5d348d83617a0416 Mon Sep 17 00:00:00 2001 From: Oscar Salazar Date: Wed, 26 Jun 2024 13:48:58 -0600 Subject: [PATCH 01/16] Add pipeline for CLI + NA integration tests --- .github/actions/modified_apps/action.yml | 30 ++++++++++ .../actions/modified_apps/modified_files.sh | 5 ++ .github/workflows/ci-integration.yml | 59 +++++++++++++++++++ .github/workflows/ci.yml | 44 +++++++------- account-privileges/app/setup_script.sql | 2 +- account-privileges/ci.sh | 3 + config.toml | 9 +++ data-mapping/app/manifest.yml | 1 - data-mapping/ci.sh | 3 + reference-usage/ci.sh | 3 + spcs-three-tier/ci.sh | 3 + spcs-three-tier/setup.sh | 12 +++- 12 files changed, 148 insertions(+), 26 deletions(-) create mode 100644 .github/actions/modified_apps/action.yml create mode 100755 .github/actions/modified_apps/modified_files.sh create mode 100644 .github/workflows/ci-integration.yml create mode 100644 account-privileges/ci.sh create mode 100644 config.toml create mode 100644 data-mapping/ci.sh create mode 100644 reference-usage/ci.sh create mode 100644 spcs-three-tier/ci.sh diff --git a/.github/actions/modified_apps/action.yml b/.github/actions/modified_apps/action.yml new file mode 100644 index 0000000..a73c328 --- /dev/null +++ b/.github/actions/modified_apps/action.yml @@ -0,0 +1,30 @@ +name: 'Modified Apps' +description: 'Gets the modified apps root folder' +runs: + using: 'composite' + steps: + - name: Get changed files + id: changed-files-step + run: $GITHUB_ACTION_PATH/modified_files.sh + shell: bash + env: + GITHUB_ACTION_PATH: ${{ github.action_path }} + GITHUB_EVENT_NAME: ${{ github.event_name }} + GITHUB_EVENT_BEFORE: ${{ github.event.before }} + GITHUB_EVENT_AFTER: ${{ github.event.after }} + - name: Get changed apps + uses: actions/github-script@v7 + id: changed-apps-step + env: + CHANGED_FILES: ${{ steps.changed-files-step.outputs.changed_files }} + with: + script: | + const { CHANGED_FILES } = process.env; + const paths = new Set(CHANGED_FILES.split(" ") + .map(x => x.substring(0, x.indexOf("/") + 1)) + .filter(x => x.length > 0 && !x.startsWith('.'))); + core.setOutput('changedApps', [...paths].join(',')); +outputs: + modified_apps: + description: 'The modified apps folders.' + value: ${{ steps.changed-apps-step.outputs.changedApps }} \ No newline at end of file diff --git a/.github/actions/modified_apps/modified_files.sh b/.github/actions/modified_apps/modified_files.sh new file mode 100755 index 0000000..ff82e64 --- /dev/null +++ b/.github/actions/modified_apps/modified_files.sh @@ -0,0 +1,5 @@ +if [ $GITHUB_EVENT_NAME == 'pull_request' ]; then + echo "changed_files=$(git diff --name-only -r HEAD^1 HEAD | xargs)" >> $GITHUB_OUTPUT +else + echo "changed_files=$(git diff --name-only $GITHUB_EVENT_BEFORE $GITHUB_EVENT_AFTER | xargs)" >> $GITHUB_OUTPUT +fi \ No newline at end of file diff --git a/.github/workflows/ci-integration.yml b/.github/workflows/ci-integration.yml new file mode 100644 index 0000000..5ba25a5 --- /dev/null +++ b/.github/workflows/ci-integration.yml @@ -0,0 +1,59 @@ +name: native-app-examples-integration + +on: + pull_request: + types: + - opened + - edited + - labeled + - unlabeled + - synchronize + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + env: + SNOWFLAKE_CONNECTIONS_DEFAULT_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + SNOWFLAKE_CONNECTIONS_DEFAULT_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_CONNECTIONS_DEFAULT_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_CONNECTIONS_DEFAULT_ROLE: ${{ secrets.SNOWFLAKE_ROLE }} + SNOWFLAKE_CONNECTIONS_DEFAULT_WAREHOSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }} + defaults: + run: + shell: bash -l {0} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }} + - name: Set PIPX_BIN_DIR # This step will be removed after fixing a bug in snowflake-cli-action@v1 + run: echo "PIPX_BIN_DIR=$HOME/.local/bin" >> $GITHUB_ENV + - name: Get modified apps + id: modified-apps-step + uses: ./.github/actions/modified_apps + - name: Set up Snowflake CLI + uses: Snowflake-Labs/snowflake-cli-action@v1.1 + with: + cli-version: "latest" + default-config-file-path: "config.toml" + - name: Verify Snowflake Connection + run: | + snow --version + snow connection test + - name: Run Integration + run: | + set -e + modified_apps=${{ steps.modified-apps-step.outputs.modified_apps }} + IFS=',' read -ra modified_apps_array <<< "$modified_apps" + for app in "${modified_apps_array[@]}"; do + cd $app + if [ -f ci.sh ]; then + sh ci.sh + else + snow app run + snow app teardown + fi + cd .. + done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d0594f..59b45d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,24 +30,20 @@ jobs: uses: actions/setup-python@v3 with: python-version: "3.10" - - name: Get changed files - id: changed-files - run: | - if ${{ github.event_name == 'pull_request' }}; then - echo "changed_files=$(git diff --name-only -r HEAD^1 HEAD | xargs)" >> $GITHUB_OUTPUT - else - echo "changed_files=$(git diff --name-only ${{ github.event.before }} ${{ github.event.after }} | xargs)" >> $GITHUB_OUTPUT - fi + - name: Get modified apps + id: modified-apps-step + uses: ./.github/actions/modified_apps - name: Determine tests to run uses: actions/github-script@v7 id: tests_to_run env: - CHANGED_FILES: ${{ steps.changed-files.outputs.changed_files }} + MODIFIED_APPS: ${{ steps.modified-apps-step.outputs.modified_apps }} with: script: | - const { CHANGED_FILES } = process.env; + const { MODIFIED_APPS } = process.env; const fs = require('fs'); const path = require('path'); + var has_python_tests = false; function getPytestPaths(dir, callback) { @@ -63,25 +59,29 @@ jobs: extension = path.extname(file.name); if (extension == '.py') - { - callback(dir); + { + callback(dir, file.name); } } } } - const paths = new Set(CHANGED_FILES.split(" ") - .map(x => x.substring(0, x.indexOf("/") + 1)) - .filter(x => x.length > 0 && !x.startsWith('.'))); + const paths = MODIFIED_APPS.length > 0 ? MODIFIED_APPS.split(",") : [] const pytestPaths = new Set() const pytestArgs = new Set() for (const rootPath of paths) { let subFoldersWithPythonFiles = 0; - getPytestPaths(rootPath, x => + getPytestPaths(rootPath, (folder, fileName) => { - pytestPaths.add(x); + pytestPaths.add(folder); + + if (fileName.startsWith('test')) + { + has_python_tests = true + } + subFoldersWithPythonFiles++ }) @@ -91,12 +91,13 @@ jobs: } } - core.setOutput('pytestPaths', [...pytestPaths].join(' ')); - core.setOutput('pytestArgs', [...pytestArgs].join(' ')); + core.setOutput('pytestPaths', has_tests ? [...pytestPaths].join(' ') : ""); + core.setOutput('pytestArgs', has_tests ? [...pytestArgs].join(' ') : ""); - name: Setup test environment uses: conda-incubator/setup-miniconda@v2 with: + miniconda-version: "latest" environment-file: ${{ matrix.environment-file }} - name: Install dependencies run: | @@ -104,8 +105,9 @@ jobs: python -m pip install pytest - name: Run tests run: | - args=${{ steps.tests_to_run.outputs.pytestArgs }} - pythonpath=${{ steps.tests_to_run.outputs.pytestPaths }} + args="${{ steps.tests_to_run.outputs.pytestArgs }}" + pythonpath="${{ steps.tests_to_run.outputs.pytestPaths }}" + if [ -z "${args}" ] || [ -z "${pythonpath}" ]; then echo “Nothing to test” else diff --git a/account-privileges/app/setup_script.sql b/account-privileges/app/setup_script.sql index d60455d..cbb712a 100644 --- a/account-privileges/app/setup_script.sql +++ b/account-privileges/app/setup_script.sql @@ -67,7 +67,7 @@ GRANT USAGE ON PROCEDURE core.app_update_table(NUMBER) TO APPLICATION ROLE app_p -- 5. Create a streamlit object using the code you wrote in you wrote in src/module-ui, as shown below. -- The `from` value is derived from the stage path described in snowflake.yml -CREATE STREAMLIT core.ui +CREATE OR REPLACE STREAMLIT core.ui FROM '/streamlit/' MAIN_FILE = 'ui.py'; diff --git a/account-privileges/ci.sh b/account-privileges/ci.sh new file mode 100644 index 0000000..c760c0e --- /dev/null +++ b/account-privileges/ci.sh @@ -0,0 +1,3 @@ +snow sql -f 'prepare/references.sql' +snow app run +snow app teardown \ No newline at end of file diff --git a/config.toml b/config.toml new file mode 100644 index 0000000..1b979b9 --- /dev/null +++ b/config.toml @@ -0,0 +1,9 @@ +default_connection_name = "default" + +[connections] +[connections.default] +account = "" +user = "" +password = "" +role = "" +warehouse = "" \ No newline at end of file diff --git a/data-mapping/app/manifest.yml b/data-mapping/app/manifest.yml index e34e45e..afa02f5 100644 --- a/data-mapping/app/manifest.yml +++ b/data-mapping/app/manifest.yml @@ -1,6 +1,5 @@ manifest_version: 1 artifacts: - readme: README.md setup_script: setup_script.sql default_streamlit: ui."Dashboard" diff --git a/data-mapping/ci.sh b/data-mapping/ci.sh new file mode 100644 index 0000000..57bd727 --- /dev/null +++ b/data-mapping/ci.sh @@ -0,0 +1,3 @@ +sh prepare_data.sh +snow app run +snow app teardown \ No newline at end of file diff --git a/reference-usage/ci.sh b/reference-usage/ci.sh new file mode 100644 index 0000000..a520752 --- /dev/null +++ b/reference-usage/ci.sh @@ -0,0 +1,3 @@ +snow sql -f 'prepare/provider.sql' +snow app run +snow app teardown \ No newline at end of file diff --git a/spcs-three-tier/ci.sh b/spcs-three-tier/ci.sh new file mode 100644 index 0000000..f9ee598 --- /dev/null +++ b/spcs-three-tier/ci.sh @@ -0,0 +1,3 @@ +sh setup.sh +sh deploy.sh +sh cleanup.sh \ No newline at end of file diff --git a/spcs-three-tier/setup.sh b/spcs-three-tier/setup.sh index fb80d0a..450f5cd 100755 --- a/spcs-three-tier/setup.sh +++ b/spcs-three-tier/setup.sh @@ -19,8 +19,14 @@ cp $frontend_yaml_template $frontend_yaml cp $backend_yaml_template $backend_yaml # Replace placeholders in Makefile file using | as delimiter -sed -i "" "s|<>|$repository_url|g" $makefile -sed -i "" "s|<>|$repository_url|g" $frontend_yaml -sed -i "" "s|<>|$repository_url|g" $backend_yaml +if [[ "$OSTYPE" == "linux-gnu"* ]]; then + sed -i "s|<>|$repository_url|g" $makefile + sed -i "s|<>|$repository_url|g" $frontend_yaml + sed -i "s|<>|$repository_url|g" $backend_yaml +else + sed -i "" "s|<>|$repository_url|g" $makefile + sed -i "" "s|<>|$repository_url|g" $frontend_yaml + sed -i "" "s|<>|$repository_url|g" $backend_yaml +fi make all \ No newline at end of file From 42ac46603e9d769765fdfd13a6091af974d8995d Mon Sep 17 00:00:00 2001 From: Oscar Salazar Date: Wed, 26 Jun 2024 14:00:37 -0600 Subject: [PATCH 02/16] Update ci-integration.yml --- .github/workflows/ci-integration.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-integration.yml b/.github/workflows/ci-integration.yml index 5ba25a5..d9bb4a2 100644 --- a/.github/workflows/ci-integration.yml +++ b/.github/workflows/ci-integration.yml @@ -28,8 +28,8 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }} - - name: Set PIPX_BIN_DIR # This step will be removed after fixing a bug in snowflake-cli-action@v1 - run: echo "PIPX_BIN_DIR=$HOME/.local/bin" >> $GITHUB_ENV + # - name: Set PIPX_BIN_DIR # This step will be removed after fixing a bug in snowflake-cli-action@v1 + # run: echo "PIPX_BIN_DIR=$HOME/.local/bin" >> $GITHUB_ENV - name: Get modified apps id: modified-apps-step uses: ./.github/actions/modified_apps From 7f782960c0f94991bc9e12f0b9f381845e47c595 Mon Sep 17 00:00:00 2001 From: Oscar Salazar Date: Wed, 26 Jun 2024 14:05:10 -0600 Subject: [PATCH 03/16] adding fixes --- .github/workflows/ci-integration.yml | 4 ++-- .github/workflows/ci.yml | 4 ++-- spcs-three-tier/setup.sh | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-integration.yml b/.github/workflows/ci-integration.yml index d9bb4a2..5ba25a5 100644 --- a/.github/workflows/ci-integration.yml +++ b/.github/workflows/ci-integration.yml @@ -28,8 +28,8 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }} - # - name: Set PIPX_BIN_DIR # This step will be removed after fixing a bug in snowflake-cli-action@v1 - # run: echo "PIPX_BIN_DIR=$HOME/.local/bin" >> $GITHUB_ENV + - name: Set PIPX_BIN_DIR # This step will be removed after fixing a bug in snowflake-cli-action@v1 + run: echo "PIPX_BIN_DIR=$HOME/.local/bin" >> $GITHUB_ENV - name: Get modified apps id: modified-apps-step uses: ./.github/actions/modified_apps diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59b45d3..aaf4b5e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,8 +91,8 @@ jobs: } } - core.setOutput('pytestPaths', has_tests ? [...pytestPaths].join(' ') : ""); - core.setOutput('pytestArgs', has_tests ? [...pytestArgs].join(' ') : ""); + core.setOutput('pytestPaths', has_python_tests ? [...pytestPaths].join(' ') : ""); + core.setOutput('pytestArgs', has_python_tests ? [...pytestArgs].join(' ') : ""); - name: Setup test environment uses: conda-incubator/setup-miniconda@v2 diff --git a/spcs-three-tier/setup.sh b/spcs-three-tier/setup.sh index 450f5cd..49fced3 100755 --- a/spcs-three-tier/setup.sh +++ b/spcs-three-tier/setup.sh @@ -19,14 +19,14 @@ cp $frontend_yaml_template $frontend_yaml cp $backend_yaml_template $backend_yaml # Replace placeholders in Makefile file using | as delimiter -if [[ "$OSTYPE" == "linux-gnu"* ]]; then +if [[ "$OSTYPE" == "darwin"* ]]; then + sed -i "" "s|<>|$repository_url|g" $makefile + sed -i "" "s|<>|$repository_url|g" $frontend_yaml + sed -i "" "s|<>|$repository_url|g" $backend_yaml +else sed -i "s|<>|$repository_url|g" $makefile sed -i "s|<>|$repository_url|g" $frontend_yaml sed -i "s|<>|$repository_url|g" $backend_yaml -else - sed -i "" "s|<>|$repository_url|g" $makefile - sed -i "" "s|<>|$repository_url|g" $frontend_yaml - sed -i "" "s|<>|$repository_url|g" $backend_yaml fi make all \ No newline at end of file From 56ec6694453a4e9ef4db23c38485831ccc46151e Mon Sep 17 00:00:00 2001 From: Oscar Salazar Date: Wed, 26 Jun 2024 14:14:08 -0600 Subject: [PATCH 04/16] Update setup.sh --- spcs-three-tier/setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/spcs-three-tier/setup.sh b/spcs-three-tier/setup.sh index 49fced3..8f2561f 100755 --- a/spcs-three-tier/setup.sh +++ b/spcs-three-tier/setup.sh @@ -1,3 +1,4 @@ +#!/bin/bash set -e snow sql -f "prepare/spcs_setup.sql" snow sql -f "prepare/provider_setup.sql" From 81dceafc1c364b6764be164af0da08c85abf3d93 Mon Sep 17 00:00:00 2001 From: Oscar Salazar Date: Wed, 26 Jun 2024 14:17:27 -0600 Subject: [PATCH 05/16] Update ci.sh --- spcs-three-tier/ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spcs-three-tier/ci.sh b/spcs-three-tier/ci.sh index f9ee598..8948d4d 100644 --- a/spcs-three-tier/ci.sh +++ b/spcs-three-tier/ci.sh @@ -1,3 +1,3 @@ -sh setup.sh +bash setup.sh sh deploy.sh sh cleanup.sh \ No newline at end of file From 545146a2bac1f53ed1c8f814e8578ef3859771b6 Mon Sep 17 00:00:00 2001 From: Oscar Salazar Date: Wed, 26 Jun 2024 14:21:00 -0600 Subject: [PATCH 06/16] Update ci-integration.yml --- .github/workflows/ci-integration.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci-integration.yml b/.github/workflows/ci-integration.yml index 5ba25a5..bcc5344 100644 --- a/.github/workflows/ci-integration.yml +++ b/.github/workflows/ci-integration.yml @@ -28,8 +28,6 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }} - - name: Set PIPX_BIN_DIR # This step will be removed after fixing a bug in snowflake-cli-action@v1 - run: echo "PIPX_BIN_DIR=$HOME/.local/bin" >> $GITHUB_ENV - name: Get modified apps id: modified-apps-step uses: ./.github/actions/modified_apps From 07ad9ed64a3ca7dfc2d5f4536dbcdb2e177c6127 Mon Sep 17 00:00:00 2001 From: Oscar Salazar Date: Wed, 26 Jun 2024 14:49:45 -0600 Subject: [PATCH 07/16] Adding set -e to all bash scripts for CI --- account-privileges/ci.sh | 1 + data-mapping/ci.sh | 1 + reference-usage/ci.sh | 1 + spcs-three-tier/ci.sh | 1 + 4 files changed, 4 insertions(+) diff --git a/account-privileges/ci.sh b/account-privileges/ci.sh index c760c0e..743ae87 100644 --- a/account-privileges/ci.sh +++ b/account-privileges/ci.sh @@ -1,3 +1,4 @@ +set -e snow sql -f 'prepare/references.sql' snow app run snow app teardown \ No newline at end of file diff --git a/data-mapping/ci.sh b/data-mapping/ci.sh index 57bd727..ac34cc7 100644 --- a/data-mapping/ci.sh +++ b/data-mapping/ci.sh @@ -1,3 +1,4 @@ +set -e sh prepare_data.sh snow app run snow app teardown \ No newline at end of file diff --git a/reference-usage/ci.sh b/reference-usage/ci.sh index a520752..82ab163 100644 --- a/reference-usage/ci.sh +++ b/reference-usage/ci.sh @@ -1,3 +1,4 @@ +set -e snow sql -f 'prepare/provider.sql' snow app run snow app teardown \ No newline at end of file diff --git a/spcs-three-tier/ci.sh b/spcs-three-tier/ci.sh index 8948d4d..422edd4 100644 --- a/spcs-three-tier/ci.sh +++ b/spcs-three-tier/ci.sh @@ -1,3 +1,4 @@ +set -e bash setup.sh sh deploy.sh sh cleanup.sh \ No newline at end of file From a026ed6da64c31bf23101c7736c020cd25b32c1a Mon Sep 17 00:00:00 2001 From: Oscar Salazar Date: Tue, 2 Jul 2024 09:43:11 -0600 Subject: [PATCH 08/16] Adding CONTRIBUTING.md file --- CONTRIBUTING.md | 41 +++++++++++++++++++++++++++++++++++++++++ README.md | 2 ++ 2 files changed, 43 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..54e65b9 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,41 @@ +# Continuous Integration (CI) + +By default, GitHub disables any workflows (or CI/CD pipelines) when forking a repository. The forked repository contains a CI/CD workflow to deploy your data pipeline to dev and prod environments. Enable the workflows by opening your forked repository in GitHub, clicking on the `Actions` tab near the top middle of the page, and then clicking on the "I understand my workflows, go ahead and enable them" green button. + +## Python Tests Workflow (Unit Tests) + +Unit tests only runs when there are changes or additions in native apps containing python files. The [pipeline](./.github/workflows/ci.yml) detects which apps were changed and only the tests for those apps are going to be executed. + +If you add a new app that requires a new python package, you have to specify that in the [environment file](./shared_python_ci_env.yml) used to run python tests. + +*Only apps with at least one python test are going to be tested in the pipeline. We recommend to add testing to changes/additions in order to improve the general quality* + +## Integration Tests Workflow + +Integration tests purpose is to do automatic integration/deployment into Snowflake by running the `snow` commands from the [Snowflake CLI](https://docs.snowflake.com/developer-guide/snowflake-cli-v2/index). + +Same as *Unit Tests*, Integration Tests runs only for apps that were added/modified. Integration testing pipeline is defined in the [ci.integration.yml](./.github/workflows/ci-integration.yml) file. + +### Configuration + +Integration Tests depends on the `Snowflake CLI` to connect to your Snowflake account, so we need to set up some credentials before start contributing the repository. Action Secrets in GitHub are used to securely store values/variables for use in CI/CD pipelines. + +In GitHub, click on the `Settings` tab near the top of the page. From the Settings page, click on the `Secrets and variables` then `Actions` tab in the left hand navigation. The "Secrets" tab should be selected. For each secret listed below click on the green `New repository secret` and enter the name given below along with the appropriate value (adjusting as appropriate). + +| Secret name | Secret value | +| --- | --- | +| SNOWFLAKE_ACCOUNT | myaccount | +| SNOWFLAKE_USER | myusername | +| SNOWFLAKE_PASSWORD | mypassword | +| SNOWFLAKE_ROLE | myrole | +| SNOWFLAKE_WAREHOUSE | mywarehouse | + +Once the GitHub secrets are set, you are able to contribute in the repo. + +### Usage + +Integration Tests Workflow uses the [snowflake-cli-action](https://github.com/Snowflake-Labs/snowflake-cli-action), that basically installs the `Snowflake CLI` in our workflow environment. + +For each modify app, we test that `snow app run` and `snow app teardown` commands run successfully, that means that we are verify that the app were deployed and removed successfully from our Snowflake account. + +For apps that requieres some data before deployment, we provide a capability to define a `ci.sh` file in the root of the app, that executes all the steps required to deploy the app successfully. \ No newline at end of file diff --git a/README.md b/README.md index 9e8aa01..7eb22a6 100644 --- a/README.md +++ b/README.md @@ -32,3 +32,5 @@ Some applications require other account-level setup before they can be properly ## Contributing Contributions are welcome and encouraged under the [Apache 2.0 License](./LICENSE.txt). Please feel free to open issues or pull requests. + +For more information about contributing, see [CONTRIBUTING.md](./CONTRIBUTING.md) From 05540614b72ea8406bdd4d63aebed9884b4028e9 Mon Sep 17 00:00:00 2001 From: Oscar Salazar Date: Tue, 2 Jul 2024 10:50:44 -0600 Subject: [PATCH 09/16] Update CONTRIBUTING.md Co-authored-by: Cam Gorrie --- CONTRIBUTING.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 54e65b9..d015df6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,6 +2,7 @@ By default, GitHub disables any workflows (or CI/CD pipelines) when forking a repository. The forked repository contains a CI/CD workflow to deploy your data pipeline to dev and prod environments. Enable the workflows by opening your forked repository in GitHub, clicking on the `Actions` tab near the top middle of the page, and then clicking on the "I understand my workflows, go ahead and enable them" green button. +You must also add the appropriate secrets to your fork to enable integration tests; see the configuration section below. ## Python Tests Workflow (Unit Tests) Unit tests only runs when there are changes or additions in native apps containing python files. The [pipeline](./.github/workflows/ci.yml) detects which apps were changed and only the tests for those apps are going to be executed. From ab95f5582895d6ce5deac371a50651bc0eec7a86 Mon Sep 17 00:00:00 2001 From: Oscar Salazar Date: Tue, 2 Jul 2024 10:50:51 -0600 Subject: [PATCH 10/16] Update CONTRIBUTING.md Co-authored-by: Cam Gorrie --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d015df6..ef107f2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,7 +19,7 @@ Same as *Unit Tests*, Integration Tests runs only for apps that were added/modif ### Configuration -Integration Tests depends on the `Snowflake CLI` to connect to your Snowflake account, so we need to set up some credentials before start contributing the repository. Action Secrets in GitHub are used to securely store values/variables for use in CI/CD pipelines. +Integration Tests depends on `Snowflake CLI` to connect to your Snowflake account; we need to set up some credentials before these tests will run. GitHub Secrets are used to securely store values/variables for use in CI/CD pipelines. In GitHub, click on the `Settings` tab near the top of the page. From the Settings page, click on the `Secrets and variables` then `Actions` tab in the left hand navigation. The "Secrets" tab should be selected. For each secret listed below click on the green `New repository secret` and enter the name given below along with the appropriate value (adjusting as appropriate). From 0e4bab9a70fb59f7619f2c2a4ece3f1d15e8f1f8 Mon Sep 17 00:00:00 2001 From: Oscar Salazar Date: Tue, 2 Jul 2024 10:54:43 -0600 Subject: [PATCH 11/16] Update spcs-three-tier/ci.sh Co-authored-by: Cam Gorrie --- spcs-three-tier/ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spcs-three-tier/ci.sh b/spcs-three-tier/ci.sh index 422edd4..bbd8d2c 100644 --- a/spcs-three-tier/ci.sh +++ b/spcs-three-tier/ci.sh @@ -1,4 +1,4 @@ set -e bash setup.sh sh deploy.sh -sh cleanup.sh \ No newline at end of file +./cleanup.sh \ No newline at end of file From d0b29096509cf4f85f505ce77c2fed8a2534d183 Mon Sep 17 00:00:00 2001 From: Oscar Salazar Date: Tue, 2 Jul 2024 10:54:57 -0600 Subject: [PATCH 12/16] Update spcs-three-tier/ci.sh Co-authored-by: Cam Gorrie --- spcs-three-tier/ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spcs-three-tier/ci.sh b/spcs-three-tier/ci.sh index bbd8d2c..9a98ca1 100644 --- a/spcs-three-tier/ci.sh +++ b/spcs-three-tier/ci.sh @@ -1,4 +1,4 @@ set -e bash setup.sh -sh deploy.sh +./deploy.sh ./cleanup.sh \ No newline at end of file From de409bb5d7154ece7447d9b12697678c78941d78 Mon Sep 17 00:00:00 2001 From: Oscar Salazar Date: Tue, 2 Jul 2024 10:55:05 -0600 Subject: [PATCH 13/16] Update data-mapping/ci.sh Co-authored-by: Cam Gorrie --- data-mapping/ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-mapping/ci.sh b/data-mapping/ci.sh index ac34cc7..98ee66a 100644 --- a/data-mapping/ci.sh +++ b/data-mapping/ci.sh @@ -1,4 +1,4 @@ set -e -sh prepare_data.sh +./prepare_data.sh snow app run snow app teardown \ No newline at end of file From 630c84568aa74843e9a55d67043638c4e010c1e1 Mon Sep 17 00:00:00 2001 From: Oscar Salazar Date: Tue, 2 Jul 2024 11:02:04 -0600 Subject: [PATCH 14/16] changing workflows triggers --- .github/workflows/ci-integration.yml | 9 +-------- .github/workflows/ci.yml | 9 +-------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci-integration.yml b/.github/workflows/ci-integration.yml index bcc5344..5173e6b 100644 --- a/.github/workflows/ci-integration.yml +++ b/.github/workflows/ci-integration.yml @@ -1,13 +1,6 @@ name: native-app-examples-integration -on: - pull_request: - types: - - opened - - edited - - labeled - - unlabeled - - synchronize +on: [ pull_request, push ] permissions: contents: read diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aaf4b5e..beaa2c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,13 +1,6 @@ name: native-app-examples -on: - pull_request: - types: - - opened - - edited - - labeled - - unlabeled - - synchronize +on: [ pull_request, push ] permissions: contents: read From 35dbeb4b87f45582fbc1c232ed0bb47e62729d5c Mon Sep 17 00:00:00 2001 From: Oscar Salazar Date: Tue, 2 Jul 2024 11:18:33 -0600 Subject: [PATCH 15/16] Update CONTRIBUTING.md --- CONTRIBUTING.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ef107f2..d2baeeb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,6 +37,19 @@ Once the GitHub secrets are set, you are able to contribute in the repo. Integration Tests Workflow uses the [snowflake-cli-action](https://github.com/Snowflake-Labs/snowflake-cli-action), that basically installs the `Snowflake CLI` in our workflow environment. -For each modify app, we test that `snow app run` and `snow app teardown` commands run successfully, that means that we are verify that the app were deployed and removed successfully from our Snowflake account. +#### Apps that depends on existing data. -For apps that requieres some data before deployment, we provide a capability to define a `ci.sh` file in the root of the app, that executes all the steps required to deploy the app successfully. \ No newline at end of file +For apps that requieres some data before deployment, we provide a capability to define a `ci.sh` file in the root of the app, that executes all the steps required to deploy the app successfully. E.g. + +```bash +# ci.yml + +set -e +bash setup.sh +./deploy.sh +./cleanup.sh +``` + +#### Default behavior + +If there are no `ci.yml` file, we just execute the `snow app run` and `snow app teardown` commands. That means that we are verify that the app were deployed and removed successfully from our Snowflake account. \ No newline at end of file From 19688c08a2a2eadea5affd8b856910e0c40eac8e Mon Sep 17 00:00:00 2001 From: Oscar Salazar Date: Tue, 2 Jul 2024 15:34:20 -0600 Subject: [PATCH 16/16] Update CONTRIBUTING.md --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d2baeeb..c9619f1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,7 +42,7 @@ Integration Tests Workflow uses the [snowflake-cli-action](https://github.com/Sn For apps that requieres some data before deployment, we provide a capability to define a `ci.sh` file in the root of the app, that executes all the steps required to deploy the app successfully. E.g. ```bash -# ci.yml +# ci.sh set -e bash setup.sh @@ -52,4 +52,4 @@ bash setup.sh #### Default behavior -If there are no `ci.yml` file, we just execute the `snow app run` and `snow app teardown` commands. That means that we are verify that the app were deployed and removed successfully from our Snowflake account. \ No newline at end of file +If there are no `ci.sh` file, we just execute the `snow app run` and `snow app teardown` commands. That means that we are verify that the app were deployed and removed successfully from our Snowflake account. \ No newline at end of file