diff --git a/.github/actions/modified_apps/action.yml b/.github/actions/modified_apps/action.yml new file mode 100644 index 0000000..a73c328 --- /dev/null +++ b/.github/actions/modified_apps/action.yml @@ -0,0 +1,30 @@ +name: 'Modified Apps' +description: 'Gets the modified apps root folder' +runs: + using: 'composite' + steps: + - name: Get changed files + id: changed-files-step + run: $GITHUB_ACTION_PATH/modified_files.sh + shell: bash + env: + GITHUB_ACTION_PATH: ${{ github.action_path }} + GITHUB_EVENT_NAME: ${{ github.event_name }} + GITHUB_EVENT_BEFORE: ${{ github.event.before }} + GITHUB_EVENT_AFTER: ${{ github.event.after }} + - name: Get changed apps + uses: actions/github-script@v7 + id: changed-apps-step + env: + CHANGED_FILES: ${{ steps.changed-files-step.outputs.changed_files }} + with: + script: | + const { CHANGED_FILES } = process.env; + const paths = new Set(CHANGED_FILES.split(" ") + .map(x => x.substring(0, x.indexOf("/") + 1)) + .filter(x => x.length > 0 && !x.startsWith('.'))); + core.setOutput('changedApps', [...paths].join(',')); +outputs: + modified_apps: + description: 'The modified apps folders.' + value: ${{ steps.changed-apps-step.outputs.changedApps }} \ No newline at end of file diff --git a/.github/actions/modified_apps/modified_files.sh b/.github/actions/modified_apps/modified_files.sh new file mode 100755 index 0000000..ff82e64 --- /dev/null +++ b/.github/actions/modified_apps/modified_files.sh @@ -0,0 +1,5 @@ +if [ $GITHUB_EVENT_NAME == 'pull_request' ]; then + echo "changed_files=$(git diff --name-only -r HEAD^1 HEAD | xargs)" >> $GITHUB_OUTPUT +else + echo "changed_files=$(git diff --name-only $GITHUB_EVENT_BEFORE $GITHUB_EVENT_AFTER | xargs)" >> $GITHUB_OUTPUT +fi \ No newline at end of file diff --git a/.github/workflows/ci-integration.yml b/.github/workflows/ci-integration.yml new file mode 100644 index 0000000..5173e6b --- /dev/null +++ b/.github/workflows/ci-integration.yml @@ -0,0 +1,50 @@ +name: native-app-examples-integration + +on: [ pull_request, push ] + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + env: + SNOWFLAKE_CONNECTIONS_DEFAULT_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + SNOWFLAKE_CONNECTIONS_DEFAULT_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_CONNECTIONS_DEFAULT_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_CONNECTIONS_DEFAULT_ROLE: ${{ secrets.SNOWFLAKE_ROLE }} + SNOWFLAKE_CONNECTIONS_DEFAULT_WAREHOSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }} + defaults: + run: + shell: bash -l {0} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }} + - name: Get modified apps + id: modified-apps-step + uses: ./.github/actions/modified_apps + - name: Set up Snowflake CLI + uses: Snowflake-Labs/snowflake-cli-action@v1.1 + with: + cli-version: "latest" + default-config-file-path: "config.toml" + - name: Verify Snowflake Connection + run: | + snow --version + snow connection test + - name: Run Integration + run: | + set -e + modified_apps=${{ steps.modified-apps-step.outputs.modified_apps }} + IFS=',' read -ra modified_apps_array <<< "$modified_apps" + for app in "${modified_apps_array[@]}"; do + cd $app + if [ -f ci.sh ]; then + sh ci.sh + else + snow app run + snow app teardown + fi + cd .. + done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5bfb31f..d1a53a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,13 +1,6 @@ name: native-app-examples -on: - pull_request: - types: - - opened - - edited - - labeled - - unlabeled - - synchronize +on: [ pull_request, push ] permissions: contents: read @@ -30,24 +23,20 @@ jobs: uses: actions/setup-python@v3 with: python-version: "3.10" - - name: Get changed files - id: changed-files - run: | - if ${{ github.event_name == 'pull_request' }}; then - echo "changed_files=$(git diff --name-only -r HEAD^1 HEAD | xargs)" >> $GITHUB_OUTPUT - else - echo "changed_files=$(git diff --name-only ${{ github.event.before }} ${{ github.event.after }} | xargs)" >> $GITHUB_OUTPUT - fi + - name: Get modified apps + id: modified-apps-step + uses: ./.github/actions/modified_apps - name: Determine tests to run uses: actions/github-script@v7 id: tests_to_run env: - CHANGED_FILES: ${{ steps.changed-files.outputs.changed_files }} + MODIFIED_APPS: ${{ steps.modified-apps-step.outputs.modified_apps }} with: script: | - const { CHANGED_FILES } = process.env; + const { MODIFIED_APPS } = process.env; const fs = require('fs'); const path = require('path'); + var has_python_tests = false; function getPytestPaths(dir, callback) { @@ -63,25 +52,29 @@ jobs: extension = path.extname(file.name); if (extension == '.py') - { - callback(dir); + { + callback(dir, file.name); } } } } - const paths = new Set(CHANGED_FILES.split(" ") - .map(x => x.substring(0, x.indexOf("/") + 1)) - .filter(x => x.length > 0 && !x.startsWith('.'))); + const paths = MODIFIED_APPS.length > 0 ? MODIFIED_APPS.split(",") : [] const pytestPaths = new Set() const pytestArgs = new Set() for (const rootPath of paths) { let subFoldersWithPythonFiles = 0; - getPytestPaths(rootPath, x => + getPytestPaths(rootPath, (folder, fileName) => { - pytestPaths.add(x); + pytestPaths.add(folder); + + if (fileName.startsWith('test')) + { + has_python_tests = true + } + subFoldersWithPythonFiles++ }) @@ -91,12 +84,13 @@ jobs: } } - core.setOutput('pytestPaths', [...pytestPaths].join(' ')); - core.setOutput('pytestArgs', [...pytestArgs].join(' ')); + core.setOutput('pytestPaths', has_python_tests ? [...pytestPaths].join(' ') : ""); + core.setOutput('pytestArgs', has_python_tests ? [...pytestArgs].join(' ') : ""); - name: Setup test environment uses: conda-incubator/setup-miniconda@v2 with: + miniconda-version: "latest" environment-file: ${{ matrix.environment-file }} - name: Install dependencies run: | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..c9619f1 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,55 @@ +# Continuous Integration (CI) + +By default, GitHub disables any workflows (or CI/CD pipelines) when forking a repository. The forked repository contains a CI/CD workflow to deploy your data pipeline to dev and prod environments. Enable the workflows by opening your forked repository in GitHub, clicking on the `Actions` tab near the top middle of the page, and then clicking on the "I understand my workflows, go ahead and enable them" green button. + +You must also add the appropriate secrets to your fork to enable integration tests; see the configuration section below. +## Python Tests Workflow (Unit Tests) + +Unit tests only runs when there are changes or additions in native apps containing python files. The [pipeline](./.github/workflows/ci.yml) detects which apps were changed and only the tests for those apps are going to be executed. + +If you add a new app that requires a new python package, you have to specify that in the [environment file](./shared_python_ci_env.yml) used to run python tests. + +*Only apps with at least one python test are going to be tested in the pipeline. We recommend to add testing to changes/additions in order to improve the general quality* + +## Integration Tests Workflow + +Integration tests purpose is to do automatic integration/deployment into Snowflake by running the `snow` commands from the [Snowflake CLI](https://docs.snowflake.com/developer-guide/snowflake-cli-v2/index). + +Same as *Unit Tests*, Integration Tests runs only for apps that were added/modified. Integration testing pipeline is defined in the [ci.integration.yml](./.github/workflows/ci-integration.yml) file. + +### Configuration + +Integration Tests depends on `Snowflake CLI` to connect to your Snowflake account; we need to set up some credentials before these tests will run. GitHub Secrets are used to securely store values/variables for use in CI/CD pipelines. + +In GitHub, click on the `Settings` tab near the top of the page. From the Settings page, click on the `Secrets and variables` then `Actions` tab in the left hand navigation. The "Secrets" tab should be selected. For each secret listed below click on the green `New repository secret` and enter the name given below along with the appropriate value (adjusting as appropriate). + +| Secret name | Secret value | +| --- | --- | +| SNOWFLAKE_ACCOUNT | myaccount | +| SNOWFLAKE_USER | myusername | +| SNOWFLAKE_PASSWORD | mypassword | +| SNOWFLAKE_ROLE | myrole | +| SNOWFLAKE_WAREHOUSE | mywarehouse | + +Once the GitHub secrets are set, you are able to contribute in the repo. + +### Usage + +Integration Tests Workflow uses the [snowflake-cli-action](https://github.com/Snowflake-Labs/snowflake-cli-action), that basically installs the `Snowflake CLI` in our workflow environment. + +#### Apps that depends on existing data. + +For apps that requieres some data before deployment, we provide a capability to define a `ci.sh` file in the root of the app, that executes all the steps required to deploy the app successfully. E.g. + +```bash +# ci.sh + +set -e +bash setup.sh +./deploy.sh +./cleanup.sh +``` + +#### Default behavior + +If there are no `ci.sh` file, we just execute the `snow app run` and `snow app teardown` commands. That means that we are verify that the app were deployed and removed successfully from our Snowflake account. \ No newline at end of file diff --git a/README.md b/README.md index f9c0d29..65a6d27 100644 --- a/README.md +++ b/README.md @@ -33,3 +33,5 @@ Some applications require other account-level setup before they can be properly ## Contributing Contributions are welcome and encouraged under the [Apache 2.0 License](./LICENSE.txt). Please feel free to open issues or pull requests. + +For more information about contributing, see [CONTRIBUTING.md](./CONTRIBUTING.md) diff --git a/account-privileges/app/setup_script.sql b/account-privileges/app/setup_script.sql index d60455d..cbb712a 100644 --- a/account-privileges/app/setup_script.sql +++ b/account-privileges/app/setup_script.sql @@ -67,7 +67,7 @@ GRANT USAGE ON PROCEDURE core.app_update_table(NUMBER) TO APPLICATION ROLE app_p -- 5. Create a streamlit object using the code you wrote in you wrote in src/module-ui, as shown below. -- The `from` value is derived from the stage path described in snowflake.yml -CREATE STREAMLIT core.ui +CREATE OR REPLACE STREAMLIT core.ui FROM '/streamlit/' MAIN_FILE = 'ui.py'; diff --git a/account-privileges/ci.sh b/account-privileges/ci.sh new file mode 100644 index 0000000..743ae87 --- /dev/null +++ b/account-privileges/ci.sh @@ -0,0 +1,4 @@ +set -e +snow sql -f 'prepare/references.sql' +snow app run +snow app teardown \ No newline at end of file diff --git a/config.toml b/config.toml new file mode 100644 index 0000000..1b979b9 --- /dev/null +++ b/config.toml @@ -0,0 +1,9 @@ +default_connection_name = "default" + +[connections] +[connections.default] +account = "" +user = "" +password = "" +role = "" +warehouse = "" \ No newline at end of file diff --git a/data-mapping/app/manifest.yml b/data-mapping/app/manifest.yml index e34e45e..afa02f5 100644 --- a/data-mapping/app/manifest.yml +++ b/data-mapping/app/manifest.yml @@ -1,6 +1,5 @@ manifest_version: 1 artifacts: - readme: README.md setup_script: setup_script.sql default_streamlit: ui."Dashboard" diff --git a/data-mapping/ci.sh b/data-mapping/ci.sh new file mode 100644 index 0000000..98ee66a --- /dev/null +++ b/data-mapping/ci.sh @@ -0,0 +1,4 @@ +set -e +./prepare_data.sh +snow app run +snow app teardown \ No newline at end of file diff --git a/reference-usage/ci.sh b/reference-usage/ci.sh new file mode 100644 index 0000000..82ab163 --- /dev/null +++ b/reference-usage/ci.sh @@ -0,0 +1,4 @@ +set -e +snow sql -f 'prepare/provider.sql' +snow app run +snow app teardown \ No newline at end of file diff --git a/spcs-three-tier/ci.sh b/spcs-three-tier/ci.sh new file mode 100644 index 0000000..9a98ca1 --- /dev/null +++ b/spcs-three-tier/ci.sh @@ -0,0 +1,4 @@ +set -e +bash setup.sh +./deploy.sh +./cleanup.sh \ No newline at end of file diff --git a/spcs-three-tier/setup.sh b/spcs-three-tier/setup.sh index fb80d0a..8f2561f 100755 --- a/spcs-three-tier/setup.sh +++ b/spcs-three-tier/setup.sh @@ -1,3 +1,4 @@ +#!/bin/bash set -e snow sql -f "prepare/spcs_setup.sql" snow sql -f "prepare/provider_setup.sql" @@ -19,8 +20,14 @@ cp $frontend_yaml_template $frontend_yaml cp $backend_yaml_template $backend_yaml # Replace placeholders in Makefile file using | as delimiter -sed -i "" "s|<>|$repository_url|g" $makefile -sed -i "" "s|<>|$repository_url|g" $frontend_yaml -sed -i "" "s|<>|$repository_url|g" $backend_yaml +if [[ "$OSTYPE" == "darwin"* ]]; then + sed -i "" "s|<>|$repository_url|g" $makefile + sed -i "" "s|<>|$repository_url|g" $frontend_yaml + sed -i "" "s|<>|$repository_url|g" $backend_yaml +else + sed -i "s|<>|$repository_url|g" $makefile + sed -i "s|<>|$repository_url|g" $frontend_yaml + sed -i "s|<>|$repository_url|g" $backend_yaml +fi make all \ No newline at end of file