SNOW-1449792: Adding pipeline to integrate modified apps into Snowflake

.github/actions/modified_apps/action.yml

@@ -0,0 +1,30 @@
+name: 'Modified Apps'
+description: 'Gets the modified apps root folder'
+runs:
+  using: 'composite'
+  steps:
+    - name: Get changed files
+      id: changed-files-step
+      run: $GITHUB_ACTION_PATH/modified_files.sh
+      shell: bash
+      env:
+        GITHUB_ACTION_PATH: ${{ github.action_path }}
+        GITHUB_EVENT_NAME:  ${{ github.event_name }}
+        GITHUB_EVENT_BEFORE: ${{ github.event.before }}
+        GITHUB_EVENT_AFTER: ${{ github.event.after }}
+    - name: Get changed apps
+      uses: actions/github-script@v7
+      id: changed-apps-step
+      env:
+        CHANGED_FILES: ${{ steps.changed-files-step.outputs.changed_files }}
+      with:
+        script: |
+          const { CHANGED_FILES } = process.env;
+          const paths = new Set(CHANGED_FILES.split(" ")
+                        .map(x => x.substring(0, x.indexOf("/") + 1))
+                        .filter(x => x.length > 0 && !x.startsWith('.'))); 
+          core.setOutput('changedApps', [...paths].join(','));
+outputs:
+  modified_apps:
+    description: 'The modified apps folders.'
+    value: ${{ steps.changed-apps-step.outputs.changedApps }}

.github/actions/modified_apps/modified_files.sh

@@ -0,0 +1,5 @@
+if [ $GITHUB_EVENT_NAME == 'pull_request' ]; then
+    echo "changed_files=$(git diff --name-only -r HEAD^1 HEAD | xargs)" >> $GITHUB_OUTPUT
+else
+    echo "changed_files=$(git diff --name-only $GITHUB_EVENT_BEFORE $GITHUB_EVENT_AFTER | xargs)" >> $GITHUB_OUTPUT
+fi

.github/workflows/ci-integration.yml

@@ -0,0 +1,50 @@
+name: native-app-examples-integration
+
+on: [ pull_request, push ]
+
+permissions:
+  contents: read
+
+jobs:          
+  build:
+    runs-on: ubuntu-latest
+    env:
+        SNOWFLAKE_CONNECTIONS_DEFAULT_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
+        SNOWFLAKE_CONNECTIONS_DEFAULT_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+        SNOWFLAKE_CONNECTIONS_DEFAULT_USER: ${{ secrets.SNOWFLAKE_USER }}
+        SNOWFLAKE_CONNECTIONS_DEFAULT_ROLE: ${{ secrets.SNOWFLAKE_ROLE }}
+        SNOWFLAKE_CONNECTIONS_DEFAULT_WAREHOSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }}
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }}
+      - name: Get modified apps
+        id: modified-apps-step
+        uses: ./.github/actions/modified_apps
+      - name: Set up Snowflake CLI
+        uses: Snowflake-Labs/[email protected]
+        with:
+          cli-version: "latest"
+          default-config-file-path: "config.toml"
+      - name: Verify Snowflake Connection
+        run: |
+            snow --version
+            snow connection test
+      - name: Run Integration
+        run: |
+            set -e
+            modified_apps=${{ steps.modified-apps-step.outputs.modified_apps }}
+            IFS=',' read -ra modified_apps_array <<< "$modified_apps"
+            for app in "${modified_apps_array[@]}"; do
+                cd $app
+                if [ -f ci.sh ]; then 
+                    sh ci.sh
+                else
+                   snow app run
+                   snow app teardown
+                fi
+                cd ..
+            done

.github/workflows/ci.yml

@@ -1,13 +1,6 @@
 name: native-app-examples
 
-on:
-  pull_request:
-    types:
-      - opened
-      - edited
-      - labeled
-      - unlabeled
-      - synchronize
+on: [ pull_request, push ]
 
 permissions:
   contents: read
@@ -30,24 +23,20 @@ jobs:
         uses: actions/setup-python@v3
         with:
           python-version: "3.10"
-      - name: Get changed files
-        id: changed-files
-        run: |
-            if ${{ github.event_name == 'pull_request' }}; then
-                echo "changed_files=$(git diff --name-only -r HEAD^1 HEAD | xargs)" >> $GITHUB_OUTPUT
-            else
-                echo "changed_files=$(git diff --name-only ${{ github.event.before }} ${{ github.event.after }} | xargs)" >> $GITHUB_OUTPUT
-            fi
+      - name: Get modified apps
+        id: modified-apps-step
+        uses: ./.github/actions/modified_apps
       - name: Determine tests to run
         uses: actions/github-script@v7
         id: tests_to_run
         env:
-          CHANGED_FILES: ${{ steps.changed-files.outputs.changed_files }}
+          MODIFIED_APPS: ${{ steps.modified-apps-step.outputs.modified_apps }}
         with:
           script: |
-            const { CHANGED_FILES } = process.env;
+            const { MODIFIED_APPS } = process.env;
             const fs = require('fs');
             const path = require('path');
+            var has_python_tests = false;
 
             function getPytestPaths(dir, callback) 
             {
@@ -63,25 +52,29 @@ jobs:
                   extension = path.extname(file.name);
 
                   if (extension == '.py')
-                  {
-                    callback(dir);
+                  {                    
+                    callback(dir, file.name);
                   }
                 }
               }
             }
 
-            const paths = new Set(CHANGED_FILES.split(" ")
-                          .map(x => x.substring(0, x.indexOf("/") + 1))
-                          .filter(x => x.length > 0 && !x.startsWith('.')));
+            const paths = MODIFIED_APPS.length > 0 ? MODIFIED_APPS.split(",") : []
 
             const pytestPaths = new Set()
             const pytestArgs = new Set()
             for (const rootPath of paths) 
             { 
               let subFoldersWithPythonFiles = 0;
-              getPytestPaths(rootPath, x => 
+              getPytestPaths(rootPath, (folder, fileName) => 
               {
-                pytestPaths.add(x);
+                pytestPaths.add(folder);
+
+                if (fileName.startsWith('test'))
+                {
+                  has_python_tests = true
+                }
+
                 subFoldersWithPythonFiles++
               })
 
@@ -91,12 +84,13 @@ jobs:
               }
             }
 
-            core.setOutput('pytestPaths', [...pytestPaths].join(' '));
-            core.setOutput('pytestArgs', [...pytestArgs].join(' '));
+            core.setOutput('pytestPaths', has_python_tests ? [...pytestPaths].join(' ') : "");
+            core.setOutput('pytestArgs', has_python_tests ? [...pytestArgs].join(' ') : "");
 
       - name: Setup test environment
         uses: conda-incubator/setup-miniconda@v2
         with:
+            miniconda-version: "latest"
             environment-file: ${{ matrix.environment-file }}
       - name: Install dependencies
         run: |

CONTRIBUTING.md

@@ -0,0 +1,55 @@
+# Continuous Integration (CI)
+
+By default, GitHub disables any workflows (or CI/CD pipelines) when forking a repository. The forked repository contains a CI/CD workflow to deploy your data pipeline to dev and prod environments. Enable the workflows by opening your forked repository in GitHub, clicking on the `Actions` tab near the top middle of the page, and then clicking on the "I understand my workflows, go ahead and enable them" green button.
+
+You must also add the appropriate secrets to your fork to enable integration tests; see the configuration section below.
+## Python Tests Workflow (Unit Tests)
+
+Unit tests only runs when there are changes or additions in native apps containing python files. The [pipeline](./.github/workflows/ci.yml) detects which apps were changed and only the tests for those apps are going to be executed.
+
+If you add a new app that requires a new python package, you have to specify that in the [environment file](./shared_python_ci_env.yml) used to run python tests.
+
+*Only apps with at least one python test are going to be tested in the pipeline. We recommend to add testing to changes/additions in order to improve the general quality*
+
+## Integration Tests Workflow
+
+Integration tests purpose is to do automatic integration/deployment into Snowflake by running the `snow` commands from the [Snowflake CLI](https://docs.snowflake.com/developer-guide/snowflake-cli-v2/index).
+
+Same as *Unit Tests*, Integration Tests runs only for apps that were added/modified. Integration testing pipeline is defined in the [ci.integration.yml](./.github/workflows/ci-integration.yml) file.
+
+### Configuration
+
+Integration Tests depends on `Snowflake CLI` to connect to your Snowflake account; we need to set up some credentials before these tests will run. GitHub Secrets are used to securely store values/variables for use in CI/CD pipelines.
+
+In GitHub, click on the `Settings` tab near the top of the page. From the Settings page, click on the `Secrets and variables` then `Actions` tab in the left hand navigation. The "Secrets" tab should be selected. For each secret listed below click on the green `New repository secret` and enter the name given below along with the appropriate value (adjusting as appropriate).
+
+| Secret name | Secret value |
+| --- | --- |
+| SNOWFLAKE_ACCOUNT | myaccount |
+| SNOWFLAKE_USER | myusername |
+| SNOWFLAKE_PASSWORD | mypassword |
+| SNOWFLAKE_ROLE | myrole |
+| SNOWFLAKE_WAREHOUSE | mywarehouse |
+
+Once the GitHub secrets are set, you are able to contribute in the repo.
+
+### Usage
+
+Integration Tests Workflow uses the [snowflake-cli-action](https://github.com/Snowflake-Labs/snowflake-cli-action), that basically installs the `Snowflake CLI` in our workflow environment.
+
+#### Apps that depends on existing data.
+
+For apps that requieres some data before deployment, we provide a capability to define a `ci.sh` file in the root of the app, that executes all the steps required to deploy the app successfully. E.g.
+
+```bash
+# ci.sh
+
+set -e
+bash setup.sh
+./deploy.sh
+./cleanup.sh
+```
+
+#### Default behavior
+
+If there are no `ci.sh` file, we just execute the `snow app run` and `snow app teardown` commands. That means that we are verify that the app were deployed and removed successfully from our Snowflake account.

README.md

@@ -33,3 +33,5 @@ Some applications require other account-level setup before they can be properly
 ## Contributing
 
 Contributions are welcome and encouraged under the [Apache 2.0 License](./LICENSE.txt). Please feel free to open issues or pull requests.
+
+For more information about contributing, see [CONTRIBUTING.md](./CONTRIBUTING.md)

account-privileges/app/setup_script.sql

@@ -67,7 +67,7 @@ GRANT USAGE ON PROCEDURE core.app_update_table(NUMBER) TO APPLICATION ROLE app_p
 
 -- 5. Create a streamlit object using the code you wrote in you wrote in src/module-ui, as shown below. 
 -- The `from` value is derived from the stage path described in snowflake.yml
-CREATE STREAMLIT core.ui
+CREATE OR REPLACE STREAMLIT core.ui
      FROM '/streamlit/'
      MAIN_FILE = 'ui.py';
 

account-privileges/ci.sh

@@ -0,0 +1,4 @@
+set -e
+snow sql -f 'prepare/references.sql'
+snow app run
+snow app teardown

config.toml

@@ -0,0 +1,9 @@
+default_connection_name = "default"
+
+[connections]
+[connections.default]
+account = ""
+user = ""
+password = ""
+role = ""
+warehouse = ""

data-mapping/app/manifest.yml

@@ -1,6 +1,5 @@
 manifest_version: 1
 artifacts:
-  readme: README.md
   setup_script: setup_script.sql
   default_streamlit: ui."Dashboard"
 

data-mapping/ci.sh

@@ -0,0 +1,4 @@
+set -e
+./prepare_data.sh
+snow app run
+snow app teardown

reference-usage/ci.sh

@@ -0,0 +1,4 @@
+set -e
+snow sql -f 'prepare/provider.sql'
+snow app run
+snow app teardown

spcs-three-tier/ci.sh

@@ -0,0 +1,4 @@
+set -e
+bash setup.sh
+./deploy.sh
+./cleanup.sh

spcs-three-tier/setup.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 set -e
 snow sql -f "prepare/spcs_setup.sql"
 snow sql -f "prepare/provider_setup.sql"
@@ -19,8 +20,14 @@ cp $frontend_yaml_template $frontend_yaml
 cp $backend_yaml_template $backend_yaml
 
 # Replace placeholders in Makefile file using | as delimiter
-sed -i "" "s|<<REPOSITORY>>|$repository_url|g" $makefile
-sed -i "" "s|<<REPOSITORY>>|$repository_url|g" $frontend_yaml
-sed -i "" "s|<<REPOSITORY>>|$repository_url|g" $backend_yaml
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    sed -i "" "s|<<REPOSITORY>>|$repository_url|g" $makefile
+    sed -i "" "s|<<REPOSITORY>>|$repository_url|g" $frontend_yaml
+    sed -i "" "s|<<REPOSITORY>>|$repository_url|g" $backend_yaml   
+else
+    sed -i "s|<<REPOSITORY>>|$repository_url|g" $makefile
+    sed -i "s|<<REPOSITORY>>|$repository_url|g" $frontend_yaml
+    sed -i "s|<<REPOSITORY>>|$repository_url|g" $backend_yaml
+fi
 
 make all