From 0149fb9869be75dfaaf56f948cf58097a18c2fe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Arroyo=20Torrens?= Date: Tue, 10 Aug 2021 14:30:05 +0200 Subject: [PATCH] Add Redshift tooling/structure (#129) Co-authored-by: vdelacruzb --- .github/workflows/ci.yml | 102 ++++++++++- .github/workflows/delete-staging.yml | 2 +- .github/workflows/deploy-production-old.yml | 2 +- .github/workflows/deploy-production.yml | 31 +++- .github/workflows/deploy-staging.yml | 34 +++- .gitignore | 7 +- Makefile | 4 +- common/redshift/Makefile | 166 ++++++++++++++++++ common/redshift/dev_env.sh.sample | 16 ++ common/redshift/python2_requirements.txt | 1 + common/redshift/python3_requirements.txt | 5 + common/redshift/test_utils/__init__.py | 15 ++ common/snowflake/dev_env.sh.sample | 1 - modules/quadkey/redshift/Makefile | 3 + modules/quadkey/redshift/README.md | 3 + modules/quadkey/redshift/doc/VERSION.md | 20 +++ modules/quadkey/redshift/doc/_INTRO.md | 5 + modules/quadkey/redshift/lib/__init__.py | 1 + modules/quadkey/redshift/sql/VERSION.sql | 12 ++ .../redshift/test/integration/__init__.py | 20 +++ .../redshift/test/integration/test_VERSION.py | 7 + .../quadkey/redshift/test/unit/__init__.py | 7 + .../redshift/test/unit/test_version.py | 5 + tox.ini | 5 + 24 files changed, 454 insertions(+), 20 deletions(-) create mode 100644 common/redshift/Makefile create mode 100644 common/redshift/dev_env.sh.sample create mode 100644 common/redshift/python2_requirements.txt create mode 100644 common/redshift/python3_requirements.txt create mode 100644 common/redshift/test_utils/__init__.py create mode 100644 modules/quadkey/redshift/Makefile create mode 100644 modules/quadkey/redshift/README.md create mode 100644 modules/quadkey/redshift/doc/VERSION.md create mode 100644 modules/quadkey/redshift/doc/_INTRO.md create mode 100644 modules/quadkey/redshift/lib/__init__.py create mode 100644 modules/quadkey/redshift/sql/VERSION.sql create mode 100644 modules/quadkey/redshift/test/integration/__init__.py create mode 100644 modules/quadkey/redshift/test/integration/test_VERSION.py create mode 100644 modules/quadkey/redshift/test/unit/__init__.py create mode 100644 modules/quadkey/redshift/test/unit/test_version.py create mode 100644 tox.ini diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c4de7643..2ed2b2950 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,8 @@ on: env: NODE_VERSION: 14 + PYTHON2_VERSION: 2.7.18 + PYTHON3_VERSION: 3.8.10 jobs: @@ -41,7 +43,7 @@ jobs: with: PATTERNS: '**/*.+(js|sql)' FILES: 'Makefile' - - name: Install node + - name: Setup node uses: actions/setup-node@v1 with: node-version: ${{ env.NODE_VERSION }} @@ -66,11 +68,11 @@ jobs: with: PATTERNS: '**/*.+(js|sql)' FILES: 'Makefile' - - name: Install node + - name: Setup node uses: actions/setup-node@v1 with: node-version: ${{ env.NODE_VERSION }} - - name: Install gcloud + - name: Setup gcloud uses: google-github-actions/setup-gcloud@master with: version: ${{ env.GCLOUD_VERSION }} @@ -109,13 +111,13 @@ jobs: with: PATTERNS: '**/*.+(js|sql)' FILES: 'Makefile' - - name: Install node + - name: Setup node uses: actions/setup-node@v1 with: node-version: ${{ env.NODE_VERSION }} - name: Run unit tests run: make test-unit CLOUD=snowflake - + test-integration-sf: needs: lint-sf runs-on: ubuntu-20.04 @@ -137,13 +139,97 @@ jobs: with: PATTERNS: '**/*.+(js|sql)' FILES: 'Makefile' - - name: Install node + - name: Setup node uses: actions/setup-node@v1 with: node-version: ${{ env.NODE_VERSION }} - - name: Install snowsql + - name: Setup snowsql run: | curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-${{env.SNOWSQL_VERSION}}-linux_x86_64.bash SNOWSQL_DEST=~/snowflake SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-${{env.SNOWSQL_VERSION}}-linux_x86_64.bash - name: Run integration tests - run: make test-integration-full CLOUD=snowflake \ No newline at end of file + run: make test-integration-full CLOUD=snowflake + + lint-rs: + if: | + ( github.event_name == 'push' ) || + ( github.event_name == 'pull_request_target' && github.event.label.name == 'run_ci' ) + runs-on: ubuntu-20.04 + timeout-minutes: 5 + steps: + - name: Checkout repo + uses: actions/checkout@v2 + - name: Check diff + uses: technote-space/get-diff-action@v4 + with: + PATTERNS: '**/*.+(py)' + FILES: 'Makefile' + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: ${{ env.PYTHON3_VERSION }} + - name: Setup virtualenv + run: pip install virtualenv + - name: Check linter + run: make lint CLOUD=redshift + + test-unit-rs: + needs: lint-rs + runs-on: ubuntu-20.04 + timeout-minutes: 5 + env: + RS_SCHEMA_PREFIX: ci_${{ github.sha }}_${{ github.run_id }}_ + steps: + - name: Checkout repo + uses: actions/checkout@v2 + - name: Check diff + uses: technote-space/get-diff-action@v4 + with: + PATTERNS: '**/*.+(py)' + FILES: 'Makefile' + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: ${{ env.PYTHON2_VERSION }} + - name: Setup virtualenv + run: pip install virtualenv + - name: Run unit tests + run: make test-unit CLOUD=redshift + + test-integration-rs: + needs: lint-rs + runs-on: ubuntu-20.04 + timeout-minutes: 10 + env: + RS_REGION: us-east-2 + RS_HOST: redshift-cluster-1.c2gsqdockj5a.us-east-2.redshift.amazonaws.com + RS_CLUSTER_ID: redshift-cluster-1 + RS_DATABASE: rscartoci + RS_SCHEMA_PREFIX: ci_${{ github.sha }}_${{ github.run_id }}_ + RS_USER: ${{ secrets.RS_USER }} + RS_PASSWORD: ${{ secrets.RS_PASSWORD }} + AWS_ACCESS_KEY_ID: ${{ secrets.RS_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.RS_AWS_SECRET_ACCESS_KEY }} + AWS_S3_BUCKET: s3://rscartoci/ + steps: + - name: Checkout repo + uses: actions/checkout@v2 + - name: Check diff + uses: technote-space/get-diff-action@v4 + with: + PATTERNS: '**/*.+(py)' + FILES: 'Makefile' + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: ${{ env.PYTHON3_VERSION }} + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.RS_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.RS_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Setup virtualenv + run: pip install virtualenv + - name: Run integration tests + run: make test-integration-full CLOUD=redshift \ No newline at end of file diff --git a/.github/workflows/delete-staging.yml b/.github/workflows/delete-staging.yml index c03b59920..7f6a5762d 100644 --- a/.github/workflows/delete-staging.yml +++ b/.github/workflows/delete-staging.yml @@ -13,7 +13,7 @@ jobs: GCLOUD_VERSION: 290.0.1 BQ_PROJECT: bqcartost-core-${{ github.event.pull_request.number }} steps: - - name: Install gcloud + - name: Setup gcloud uses: google-github-actions/setup-gcloud@master with: version: ${{ env.GCLOUD_VERSION }} diff --git a/.github/workflows/deploy-production-old.yml b/.github/workflows/deploy-production-old.yml index 4f99bead5..6ce6fdf46 100644 --- a/.github/workflows/deploy-production-old.yml +++ b/.github/workflows/deploy-production-old.yml @@ -39,7 +39,7 @@ jobs: with: PATTERNS: '**/*.+(js|sql)' FILES: 'Makefile' - - name: Install gcloud + - name: Setup gcloud uses: google-github-actions/setup-gcloud@master with: version: ${{ env.GCLOUD_VERSION }} diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index b7dcbe4cf..f3ba73c64 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -85,7 +85,7 @@ jobs: with: PATTERNS: '**/*.+(js|sql)' FILES: 'Makefile' - - name: Install gcloud + - name: Setup gcloud uses: google-github-actions/setup-gcloud@master with: version: ${{ env.GCLOUD_VERSION }} @@ -115,9 +115,34 @@ jobs: with: PATTERNS: '**/*.+(js|sql)' FILES: 'Makefile' - - name: Install snowsql + - name: Setup snowsql run: | curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-${{env.SNOWSQL_VERSION}}-linux_x86_64.bash SNOWSQL_DEST=~/snowflake SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-${{env.SNOWSQL_VERSION}}-linux_x86_64.bash - name: Deploy to production - run: make deploy CLOUD=snowflake \ No newline at end of file + run: make deploy CLOUD=snowflake + + deploy-rs: + runs-on: ubuntu-20.04 + timeout-minutes: 15 + env: + RS_REGION: us-east-2 + RS_HOST: redshift-cluster-1.c2gsqdockj5a.us-east-2.redshift.amazonaws.com + RS_CLUSTER_ID: redshift-cluster-1 + RS_DATABASE: rscarto + RS_USER: ${{ secrets.RS_USER }} + RS_PASSWORD: ${{ secrets.RS_PASSWORD }} + AWS_ACCESS_KEY_ID: ${{ secrets.RS_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.RS_AWS_SECRET_ACCESS_KEY }} + AWS_S3_BUCKET: s3://rscarto/ + steps: + - name: Checkout repo + uses: actions/checkout@v2 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.RS_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.RS_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Deploy to production + run: make deploy CLOUD=redshift \ No newline at end of file diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index a2c011f20..c78ad949d 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -22,7 +22,7 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v2 - - name: Install gcloud + - name: Setup gcloud uses: google-github-actions/setup-gcloud@master with: version: ${{ env.GCLOUD_VERSION }} @@ -62,9 +62,37 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v2 - - name: Install snowsql + - name: Setup snowsql run: | curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-${{env.SNOWSQL_VERSION}}-linux_x86_64.bash SNOWSQL_DEST=~/snowflake SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-${{env.SNOWSQL_VERSION}}-linux_x86_64.bash - name: Deploy to staging - run: make deploy CLOUD=snowflake \ No newline at end of file + run: make deploy CLOUD=snowflake + + deploy-rs: + if: | + ( github.event_name == 'pull_request_review' && github.event.review.state == 'approved' ) || + ( github.event_name == 'pull_request_target' && github.event.label.name == 'deploy_st' ) + runs-on: ubuntu-20.04 + timeout-minutes: 15 + env: + RS_REGION: us-east-2 + RS_HOST: redshift-cluster-1.c2gsqdockj5a.us-east-2.redshift.amazonaws.com + RS_CLUSTER_ID: redshift-cluster-1 + RS_DATABASE: rscartost + RS_USER: ${{ secrets.RS_USER }} + RS_PASSWORD: ${{ secrets.RS_PASSWORD }} + AWS_ACCESS_KEY_ID: ${{ secrets.RS_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.RS_AWS_SECRET_ACCESS_KEY }} + AWS_S3_BUCKET: s3://rscartost/ + steps: + - name: Checkout repo + uses: actions/checkout@v2 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.RS_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.RS_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Deploy to staging + run: make deploy CLOUD=redshift \ No newline at end of file diff --git a/.gitignore b/.gitignore index c4dfbfe29..91fda4c06 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,14 @@ dist/ +*venv*/ node_modules/ +__pycache__/ +.pytest_cache/ *.log +*.pyc snowsql_rt.* .DS_STORE dev_env.sh package-lock.json -yarn.lock \ No newline at end of file +yarn.lock +MANIFEST \ No newline at end of file diff --git a/Makefile b/Makefile index 3ce68e298..352d8263b 100644 --- a/Makefile +++ b/Makefile @@ -8,13 +8,13 @@ help: echo "Please choose one of the following targets: lint, lint-fix, build, test-unit, test-integration, test-integration-full, deploy, clean, clean-deploy" lint lint-fix build test-unit test-integration deploy clean clean-deploy: - if [ "$(CLOUD)" = "bigquery" ] || [ "$(CLOUD)" = "snowflake" ]; then \ + if [ "$(CLOUD)" = "bigquery" ] || [ "$(CLOUD)" = "snowflake" ] || [ "$(CLOUD)" = "redshift" ]; then \ for module in `node scripts/modulesort.js`; do \ echo "> Module $${module}/$(CLOUD)"; \ $(MAKE) -C modules/$${module}/$(CLOUD) $@ || exit 1; \ done; \ else \ - echo "CLOUD is undefined. Please set one of the following values: bigquery, snowflake"; \ + echo "CLOUD is undefined. Please set one of the following values: bigquery, snowflake, redshift"; \ fi test-integration-full: diff --git a/common/redshift/Makefile b/common/redshift/Makefile new file mode 100644 index 000000000..6f6dd8c63 --- /dev/null +++ b/common/redshift/Makefile @@ -0,0 +1,166 @@ +# Makefile for Redshift modules + +SED ?= sed +AWS ?= aws +PYTHON2_VERSION ?= 2.7 +PYTHON3_VERSION ?= 3 + +RS_SCHEMA = $(RS_SCHEMA_PREFIX)$(MODULE) +RS_PREFIX = $(RS_SCHEMA_PREFIX) + +SHARE_CREATE_FILE = ./sql/_SHARE_CREATE.sql +SHARE_REMOVE_FILE = ./sql/_SHARE_REMOVE.sql + +COMMON_DIR = ../../../common +SCRIPTS_DIR = ../../../scripts +VENV2_DIR = $(COMMON_DIR)/redshift/venv2 +VENV3_DIR = $(COMMON_DIR)/redshift/venv3 +VENV2_ACTIVATE = $(VENV2_DIR)/bin/activate +VENV3_ACTIVATE = $(VENV3_DIR)/bin/activate + +REPLACEMENTS = -e 's!@@RS_PREFIX@@!$(RS_PREFIX)!g' + +AWS_PRINT = tr '\r\n' ' ' | jq -M 'if has("Error") then .["QueryString"],.["Error"] else .["QueryString"] end' +AWS_CHECK = tr '\r\n' ' ' | jq -e 'has("Error") | not' 1>/dev/null +AWS_PARSE_OUTPUT = echo $$AWS_OUTPUT | $(AWS_PRINT); echo $$AWS_OUTPUT | $(AWS_CHECK) +AWS_RUN_STATEMENT_SQL = \ + STATEMENT_ID=`$(AWS) redshift-data execute-statement --region $(RS_REGION) --cluster-identifier $(RS_CLUSTER_ID) --database $(RS_DATABASE) --db-user $(RS_USER) --sql "$$STATEMENT_SQL" --output text --query 'Id'`; \ + AWS_OUTPUT=`$(AWS) redshift-data describe-statement --id $$STATEMENT_ID --region $(RS_REGION) --no-cli-pager`; $(AWS_PARSE_OUTPUT) + +.SILENT: + +.PHONY: help lint lint-fix build test-unit test-integration test-integration-full deploy clean clean-deploy + +help: + echo "Please choose one of the following targets: lint, lint-fix, build, test-unit, test-integration, test-integration-full, deploy, clean, clean-deploy" + +lint: venv3 + . $(VENV3_ACTIVATE) && \ + flake8 lib/ test/ --enable-extensions Q0 && \ + deactivate || exit 0 + +lint-fix: venv3 + . $(VENV3_ACTIVATE) && \ + brunette lib/ test/ --line-length=88 --single-quotes --quiet && \ + flake8 lib/ test/ --enable-extensions Q0 && \ + deactivate || exit 0 + +build: + rm -rf dist + mkdir -p dist/$(RS_SCHEMA)Lib + cp `find lib -name "*.py"` dist/$(RS_SCHEMA)Lib/ + cd dist && zip -r $(RS_SCHEMA)Lib * + rm -rf dist/$(RS_SCHEMA)Lib + +test-unit: venv2 + rm -rf lib/$(MODULE)Lib + mkdir -p lib/$(MODULE)Lib + cp `find lib -name "*.py"` lib/$(MODULE)Lib/ + . $(VENV2_ACTIVATE) && \ + pytest test/unit && \ + deactivate + rm -rf lib/$(MODULE)Lib + +test-integration: check check-extra venv3 + . $(VENV3_ACTIVATE) && \ + pytest test/integration && \ + deactivate + +test-integration-full: + $(MAKE) deploy + $(MAKE) test-integration || ($(MAKE) clean-deploy && exit 1) + $(MAKE) clean-deploy + +deploy: check + $(MAKE) storage-upload + $(MAKE) schema-create + $(MAKE) schema-deploy + $(MAKE) share-create + +clean: + rm -rf dist + rm -rf $(VENV2_DIR) $(VENV3_DIR) + +clean-deploy: check + $(MAKE) storage-remove + $(MAKE) share-remove + $(MAKE) schema-remove || ((sleep 5 && $(MAKE) schema-remove) || exit 1) + +storage-upload: build + for f in $(wildcard dist/*.zip); do \ + $(AWS) s3 cp $$f $(AWS_S3_BUCKET) || exit 1; \ + done + for f in $(notdir $(wildcard dist/*.zip)); do \ + STATEMENT_SQL="CREATE OR REPLACE LIBRARY $(RS_SCHEMA) LANGUAGE plpythonu FROM '$(AWS_S3_BUCKET)$(notdir $$f)' CREDENTIALS 'aws_access_key_id=$(AWS_ACCESS_KEY_ID);aws_secret_access_key=$(AWS_SECRET_ACCESS_KEY)';"; $(AWS_RUN_STATEMENT_SQL); \ + done + +storage-remove: + for f in $(notdir $(wildcard dist/*.zip)); do \ + $(AWS) s3 rm $(AWS_S3_BUCKET)$$f || exit 1; \ + done + +schema-create: + STATEMENT_SQL="CREATE SCHEMA IF NOT EXISTS $(RS_SCHEMA)"; $(AWS_RUN_STATEMENT_SQL) + +schema-remove: + STATEMENT_SQL="DROP SCHEMA IF EXISTS $(RS_SCHEMA) CASCADE"; $(AWS_RUN_STATEMENT_SQL) + +schema-deploy: + for n in `IGNORE="_SHARE_CREATE _SHARE_REMOVE" node $(SCRIPTS_DIR)/sqlsort.js`; do \ + STATEMENT_SQL=`$(SED) $(REPLACEMENTS) $$n`; $(AWS_RUN_STATEMENT_SQL); \ + done + +share-create: +ifeq ($(RS_SHARE_ENABLED), 1) + STATEMENT_SQL=`$(SED) $(REPLACEMENTS) $(SHARE_CREATE_FILE)`; $(AWS_RUN_STATEMENT_SQL) +endif + +share-remove: +ifeq ($(RS_SHARE_ENABLED), 1) + STATEMENT_SQL=`$(SED) $(REPLACEMENTS) $(SHARE_REMOVE_FILE)`; $(AWS_RUN_STATEMENT_SQL) +endif + +venv2: + virtualenv -p python$(PYTHON2_VERSION) $(VENV2_DIR) -q + . $(VENV2_ACTIVATE) && \ + python -m pip install -U pip -q 2>/dev/null && \ + pip install -r $(COMMON_DIR)/redshift/python2_requirements.txt -q 2>/dev/null && \ + deactivate + +venv3: + virtualenv -p python$(PYTHON3_VERSION) $(VENV3_DIR) -q + . $(VENV3_ACTIVATE) && \ + python -m pip install -U pip -q && \ + pip install -r $(COMMON_DIR)/redshift/python3_requirements.txt -q && \ + deactivate + +check: +ifndef RS_REGION + $(error RS_REGION is undefined) +endif +ifndef RS_CLUSTER_ID + $(error RS_CLUSTER_ID is undefined) +endif +ifndef RS_DATABASE + $(error RS_DATABASE is undefined) +endif +ifndef RS_USER + $(error RS_USER is undefined) +endif +ifndef AWS_ACCESS_KEY_ID + $(error AWS_ACCESS_KEY_ID is undefined) +endif +ifndef AWS_SECRET_ACCESS_KEY + $(error AWS_SECRET_ACCESS_KEY is undefined) +endif +ifndef AWS_S3_BUCKET + $(error AWS_S3_BUCKET is undefined) +endif + +check-extra: +ifndef RS_HOST + $(error RS_HOST is undefined) +endif +ifndef RS_PASSWORD + $(error RS_PASSWORD is undefined) +endif diff --git a/common/redshift/dev_env.sh.sample b/common/redshift/dev_env.sh.sample new file mode 100644 index 000000000..8d5c946a5 --- /dev/null +++ b/common/redshift/dev_env.sh.sample @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# Development environment variables for Redshift +# This file populates the necessary environment variables to run integration tests for the project +# You can edit this file and rename it into dev_env.sh to avoid pushing local changes to the repo + +export RS_REGION="" +export RS_HOST="" +export RS_CLUSTER_ID="" +export RS_DATABASE="" +export RS_SCHEMA_PREFIX="" +export RS_USER="" +export RS_PASSWORD="" +export AWS_ACCESS_KEY_ID="" +export AWS_SECRET_ACCESS_KEY="" +export AWS_S3_BUCKET="s3:///" diff --git a/common/redshift/python2_requirements.txt b/common/redshift/python2_requirements.txt new file mode 100644 index 000000000..406177bab --- /dev/null +++ b/common/redshift/python2_requirements.txt @@ -0,0 +1 @@ +pytest==4.6.11 \ No newline at end of file diff --git a/common/redshift/python3_requirements.txt b/common/redshift/python3_requirements.txt new file mode 100644 index 000000000..2784b828f --- /dev/null +++ b/common/redshift/python3_requirements.txt @@ -0,0 +1,5 @@ +flake8==3.9.2 +flake8-quotes==3.2.0 +brunette==0.2.0 +pytest==6.2.4 +redshift-connector==2.0.884 \ No newline at end of file diff --git a/common/redshift/test_utils/__init__.py b/common/redshift/test_utils/__init__.py new file mode 100644 index 000000000..d5de723ea --- /dev/null +++ b/common/redshift/test_utils/__init__.py @@ -0,0 +1,15 @@ +import os +import redshift_connector + +conn = redshift_connector.connect( + host=os.environ['RS_HOST'], + database=os.environ['RS_DATABASE'], + user=os.environ['RS_USER'], + password=os.environ['RS_PASSWORD'], +) + + +def run_query(query): + cursor = conn.cursor() + cursor.execute(query.replace('@@RS_PREFIX@@', os.environ['RS_SCHEMA_PREFIX'])) + return cursor.fetchall() diff --git a/common/snowflake/dev_env.sh.sample b/common/snowflake/dev_env.sh.sample index c7928543c..52c48c158 100644 --- a/common/snowflake/dev_env.sh.sample +++ b/common/snowflake/dev_env.sh.sample @@ -8,7 +8,6 @@ export SF_DATABASE="" export SF_SCHEMA_PREFIX="" export SF_SHARE_PREFIX="" export SF_SHARE_ENABLED=0 - export SNOWSQL_ACCOUNT="" export SNOWSQL_USER="" export SNOWSQL_PWD="" \ No newline at end of file diff --git a/modules/quadkey/redshift/Makefile b/modules/quadkey/redshift/Makefile new file mode 100644 index 000000000..867582bea --- /dev/null +++ b/modules/quadkey/redshift/Makefile @@ -0,0 +1,3 @@ +MODULE = quadkey + +include ../../../common/redshift/Makefile \ No newline at end of file diff --git a/modules/quadkey/redshift/README.md b/modules/quadkey/redshift/README.md new file mode 100644 index 000000000..be2d88040 --- /dev/null +++ b/modules/quadkey/redshift/README.md @@ -0,0 +1,3 @@ +# Quadkey module for Redshift + +You can learn more about quadkeys and quandints in the [Overview section](/spatial-extension-rs/overview/spatial-indexes/#quadkey) of the documentation. diff --git a/modules/quadkey/redshift/doc/VERSION.md b/modules/quadkey/redshift/doc/VERSION.md new file mode 100644 index 000000000..f60540019 --- /dev/null +++ b/modules/quadkey/redshift/doc/VERSION.md @@ -0,0 +1,20 @@ +### VERSION + +{{% bannerNote type="code" %}} +quadkey.VERSION() +{{%/ bannerNote %}} + +**Description** + +Returns the current version of the quadkey module. + +**Return type** + +`VARCHAR` + +**Example** + +```sql +SELECT quadkey.VERSION(); +-- 1.0.0 +``` \ No newline at end of file diff --git a/modules/quadkey/redshift/doc/_INTRO.md b/modules/quadkey/redshift/doc/_INTRO.md new file mode 100644 index 000000000..665fba4d4 --- /dev/null +++ b/modules/quadkey/redshift/doc/_INTRO.md @@ -0,0 +1,5 @@ +## quadkey + +
+ +You can learn more about quadkeys and quandints in the [Overview section](/spatial-extension-rs/overview/spatial-indexes/#quadkey) of the documentation. diff --git a/modules/quadkey/redshift/lib/__init__.py b/modules/quadkey/redshift/lib/__init__.py new file mode 100644 index 000000000..1f356cc57 --- /dev/null +++ b/modules/quadkey/redshift/lib/__init__.py @@ -0,0 +1 @@ +__version__ = '1.0.0' diff --git a/modules/quadkey/redshift/sql/VERSION.sql b/modules/quadkey/redshift/sql/VERSION.sql new file mode 100644 index 000000000..20a263d7b --- /dev/null +++ b/modules/quadkey/redshift/sql/VERSION.sql @@ -0,0 +1,12 @@ +---------------------------- +-- Copyright (C) 2021 CARTO +---------------------------- + +CREATE OR REPLACE FUNCTION @@RS_PREFIX@@quadkey.VERSION +() +RETURNS VARCHAR +IMMUTABLE +AS $$ + from @@RS_PREFIX@@quadkeyLib import __version__ + return __version__ +$$ LANGUAGE plpythonu; \ No newline at end of file diff --git a/modules/quadkey/redshift/test/integration/__init__.py b/modules/quadkey/redshift/test/integration/__init__.py new file mode 100644 index 000000000..c6085a111 --- /dev/null +++ b/modules/quadkey/redshift/test/integration/__init__.py @@ -0,0 +1,20 @@ +import os +import sys + +# Include this to allow importing lib from source code +sys.path.insert( + 1, os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..') +) +sys.path.insert( + 1, + os.path.join( + os.path.dirname(os.path.realpath(__file__)), + '..', + '..', + '..', + '..', + '..', + 'common', + 'redshift', + ), +) diff --git a/modules/quadkey/redshift/test/integration/test_VERSION.py b/modules/quadkey/redshift/test/integration/test_VERSION.py new file mode 100644 index 000000000..2a0b4a15f --- /dev/null +++ b/modules/quadkey/redshift/test/integration/test_VERSION.py @@ -0,0 +1,7 @@ +from lib import __version__ +from test_utils import run_query + + +def test_version(): + result = run_query('SELECT @@RS_PREFIX@@quadkey.VERSION()') + assert result[0][0] == __version__ diff --git a/modules/quadkey/redshift/test/unit/__init__.py b/modules/quadkey/redshift/test/unit/__init__.py new file mode 100644 index 000000000..a6e5d71bf --- /dev/null +++ b/modules/quadkey/redshift/test/unit/__init__.py @@ -0,0 +1,7 @@ +import os +import sys + +# Include this to allow importing lib from source code +sys.path.insert( + 1, os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..') +) diff --git a/modules/quadkey/redshift/test/unit/test_version.py b/modules/quadkey/redshift/test/unit/test_version.py new file mode 100644 index 000000000..34fd12af1 --- /dev/null +++ b/modules/quadkey/redshift/test/unit/test_version.py @@ -0,0 +1,5 @@ +from lib import quadkeyLib, __version__ + + +def test_init(): + assert quadkeyLib.__version__ == __version__ diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..735473191 --- /dev/null +++ b/tox.ini @@ -0,0 +1,5 @@ +[flake8] +max-line-length = 88 +extend-ignore = + # See https://github.com/PyCQA/pycodestyle/issues/373 + E203, \ No newline at end of file