From 68e3630c23f76f1eb95da835c658f4f44af11919 Mon Sep 17 00:00:00 2001 From: Andresse Njeungoue Date: Tue, 4 Jun 2024 19:15:37 +0000 Subject: [PATCH] refactor: #113 remove useless code --- .envrc.gitpod | 17 ----------------- .gitignore | 3 +-- .gitpod.yml | 2 -- README.md | 3 --- etl/filter-cameroon.py | 3 +++ etl/requirements-dev.txt | 4 ---- etl/requirements.txt | 3 --- 7 files changed, 4 insertions(+), 31 deletions(-) diff --git a/.envrc.gitpod b/.envrc.gitpod index 721dddb..05f054b 100644 --- a/.envrc.gitpod +++ b/.envrc.gitpod @@ -35,20 +35,3 @@ then mv csvdiff_1.4.0_linux_64-bit.deb csvdiff.deb &&\ sudo dpkg -i csvdiff.deb fi - -if ! command -v aws &> /dev/null -then - pushd /tmp || exit - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip awscliv2.zip && sudo ./aws/install && rm awscliv2.zip && rm -rf aws/ &&\ - mkdir "$WORKPLACE"/.aws && touch "$WORKPLACE"/.aws/credentials && touch "$WORKPLACE"/.aws/config &&\ - export AWS_SHARED_CREDENTIALS_FILE=$WORKPLACE/.aws/credentials - export AWS_CONFIG_FILE=$WORKPLACE/.aws/config - export AWS_SDK_LOAD_CONFIG=1 - aws configure - popd || exit -fi - -echo "export AWS_SHARED_CREDENTIALS_FILE=$WORKPLACE/.aws/credentials" >> ~/.bashrc -echo "export AWS_CONFIG_FILE=$WORKPLACE/.aws/config" >> ~/.bashrc -echo "export AWS_SDK_LOAD_CONFIG=1" >> ~/.bashrc diff --git a/.gitignore b/.gitignore index 2e94bed..af92a4d 100644 --- a/.gitignore +++ b/.gitignore @@ -178,8 +178,7 @@ node_modules/** package-lock.json package.json -## To avoid commit of aws creds in gitpod env -.aws/ + ## To avoid commit of .envrc in gitpod because this cause gitpod to execute .envrc instead of gitpod.yml .envrc etl/experiments/.chainlit/** diff --git a/.gitpod.yml b/.gitpod.yml index bcc805c..3b40ef6 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -8,7 +8,6 @@ tasks: pushd etl/ pre-commit install pre-commit install --hook-type commit-msg - aws s3 cp s3://mongulu-files/enrich_cache.sqlite enrich_cache.sqlite curl "https://media.interieur.gouv.fr/rna/rna_waldec_20220301.zip" -o rna_waldec_20220301.zip unzip rna_waldec_20220301.zip -d rna_waldec_20220301 rm -f rna_waldec_20220301.zip @@ -22,4 +21,3 @@ vscode: - vivaxy.vscode-conventional-commits - sourcegraph.cody-ai - donjayamanne.githistory - - amazonwebservices.aws-toolkit-vscode diff --git a/README.md b/README.md index 8244466..9fb235e 100644 --- a/README.md +++ b/README.md @@ -22,9 +22,7 @@ If you are here, it means that you are interested in an in-house deployment of t ### Prerequisites -* Have a minimum of competence on the AWS and Terraform cloud * Create a Sourcegraph account and get credentials to use [CodyAI](https://www.youtube.com/watch?v=_csyHcEcxDA) -* Create a AWS Builder ID to use [Code whisperer](https://www.youtube.com/watch?v=rHNMfOK8pWI) *[Optional]* * [Devspace](https://www.devspace.sh/) installed locally * Have admin access on a [Gogocarto](https://gogocarto.fr/projects) * Go through the [Gogocarto tutorials](https://peertube.openstreetmap.fr/c/gogo_tutos/videos) @@ -38,7 +36,6 @@ If you are here, it means that you are interested in an in-house deployment of t Execute `filter-cameroon.ipynb` et `enrich-database.ipynb` notebooks : ``` pipenv shell - aws s3 cp s3://mongulu-files/enrich_cache.sqlite enrich_cache.sqlite secretsfoundry run --script 'python filter-cameroon.py' ``` diff --git a/etl/filter-cameroon.py b/etl/filter-cameroon.py index 8bd5df4..bcb1a5e 100644 --- a/etl/filter-cameroon.py +++ b/etl/filter-cameroon.py @@ -16,6 +16,9 @@ from pandarallel import pandarallel from rich.console import Console +# enrich_cache finally not used, it was store in aws but with clear params +# geocode is used now => store in local => 1000 requ free per day + # %% start = time.time() file_location = os.getcwd() + "/rna_waldec_20220301/" diff --git a/etl/requirements-dev.txt b/etl/requirements-dev.txt index 086f10c..cd1f25f 100644 --- a/etl/requirements-dev.txt +++ b/etl/requirements-dev.txt @@ -1,11 +1,7 @@ nbdime==3.1.1 -python-lsp-server[all]==1.4.1 -jupyterlab-lsp== 3.10.1 nbqa==1.3.1 -jupyterlab-tabular-data-editor==1.0.0 csvs-to-sqlite==1.3 ipython>=8.10.0 # not directly required, pinned by Snyk to avoid a vulnerability numpy>=1.22.2 # not directly required, pinned by Snyk to avoid a vulnerability setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability -jupyter-server>=2.11.2 # not directly required, pinned by Snyk to avoid a vulnerability tornado>=6.3.3 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/etl/requirements.txt b/etl/requirements.txt index 6100c48..861d108 100644 --- a/etl/requirements.txt +++ b/etl/requirements.txt @@ -2,8 +2,6 @@ colorama==0.4.4 findspark==2.0.1 geocoder==1.38.1 geopy==2.2.0 -#jupyter==1.0.0 -#jupyter-utils==1.2.6 pandarallel==1.6.1 pipdeptree==2.2.1 requests-cache==0.9.4 @@ -12,7 +10,6 @@ wheel==0.38.1 abbreviations==0.2.5 ftfy==6.1.1 rich==12.4.4 -#jupyterlab==3.6.7 cryptography>=3.2 # not directly required, pinned by Snyk to avoid a vulnerability ipython>=8.10.0 # not directly required, pinned by Snyk to avoid a vulnerability setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability