From 7cedf5b795b83af2b8d82f49698a8da40443cc6e Mon Sep 17 00:00:00 2001 From: Sarah Cheng Date: Thu, 5 May 2022 19:45:17 -0700 Subject: [PATCH 1/6] SEC-1819 Fixed python dependencies. --- Dockerfile | 4 ++++ dependencies/python/bandit.txt | 7 +++++++ dependencies/python/lintly23.txt | 23 +++++++++++++++++++++++ dependencies/python/semgrep.txt | 26 ++++++++++++++++++++++++++ 4 files changed, 60 insertions(+) create mode 100644 dependencies/python/bandit.txt create mode 100644 dependencies/python/lintly23.txt create mode 100644 dependencies/python/semgrep.txt diff --git a/Dockerfile b/Dockerfile index b4ca04df9d1..af9845f0217 100644 --- a/Dockerfile +++ b/Dockerfile @@ -407,6 +407,10 @@ ENV PATH="${PATH}:/venvs/snakemake/bin" ENV PATH="${PATH}:/venvs/sqlfluff/bin" ENV PATH="${PATH}:/venvs/yamllint/bin" ENV PATH="${PATH}:/venvs/yq/bin" +# 23andMe packages +ENV PATH="${PATH}:/venvs/bandit/bin" +ENV PATH="${PATH}:/venvs/lintly23/bin" +ENV PATH="${PATH}:/venvs/semgrep/bin" ############################# # Copy scripts to container # diff --git a/dependencies/python/bandit.txt b/dependencies/python/bandit.txt new file mode 100644 index 00000000000..be750d5c2ba --- /dev/null +++ b/dependencies/python/bandit.txt @@ -0,0 +1,7 @@ +bandit==1.7.4 +gitdb==4.0.9 +gitpython==3.1.27 +pbr==5.9.0 +pyyaml==6.0 +smmap==5.0.0 +stevedore==3.5.0 diff --git a/dependencies/python/lintly23.txt b/dependencies/python/lintly23.txt new file mode 100644 index 00000000000..c69d779a670 --- /dev/null +++ b/dependencies/python/lintly23.txt @@ -0,0 +1,23 @@ +autologging==1.3.2 +cached-property==1.5.2 +certifi==2021.10.8 +cffi==1.15.0 +charset-normalizer==2.0.12 +ci-py==1.0.0 +click==8.1.3 +deprecated==1.2.13 +idna==3.3 +jinja2==2.11.3 +lintly23==0.7.13 +markupsafe==2.1.1 +pycparser==2.21 +pygithub==1.55 +pyjwt==2.3.0 +pynacl==1.5.0 +python-gitlab==2.10.1 +requests-toolbelt==0.9.1 +requests==2.27.1 +six==1.16.0 +unidiff==0.6.0 +urllib3==1.26.9 +wrapt==1.14.1 diff --git a/dependencies/python/semgrep.txt b/dependencies/python/semgrep.txt new file mode 100644 index 00000000000..d852fa0f12b --- /dev/null +++ b/dependencies/python/semgrep.txt @@ -0,0 +1,26 @@ +attrs==21.4.0 +boltons==21.0.0 +bracex==2.2.1 +certifi==2021.10.8 +charset-normalizer==2.0.12 +click-option-group==0.5.3 +click==8.1.3 +colorama==0.4.4 +defusedxml==0.7.1 +face==20.1.1 +glom==22.1.0 +idna==3.3 +jsonschema==3.2.0 +packaging==21.3 +peewee==3.14.10 +pyparsing==3.0.8 +pyrsistent==0.18.1 +requests==2.27.1 +ruamel.yaml.clib==0.2.6 +ruamel.yaml==0.17.21 +semgrep==0.91.0 +setuptools==62.1.0 +six==1.16.0 +tqdm==4.64.0 +urllib3==1.26.9 +wcmatch==8.3 From 3dc4c209a02b142bd5a722f14eb60eb6c9245a92 Mon Sep 17 00:00:00 2001 From: Sarah Cheng Date: Thu, 5 May 2022 22:50:05 -0700 Subject: [PATCH 2/6] SEC-1819 Cleaned up traces of snakefmt that we removed in the past. --- TEMPLATES/.snakefmt.toml | 0 dependencies/python/snakefmt.txt | 11 ----------- lib/linter.sh | 2 +- 3 files changed, 1 insertion(+), 12 deletions(-) delete mode 100644 TEMPLATES/.snakefmt.toml delete mode 100644 dependencies/python/snakefmt.txt diff --git a/TEMPLATES/.snakefmt.toml b/TEMPLATES/.snakefmt.toml deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dependencies/python/snakefmt.txt b/dependencies/python/snakefmt.txt deleted file mode 100644 index d11d5805d4c..00000000000 --- a/dependencies/python/snakefmt.txt +++ /dev/null @@ -1,11 +0,0 @@ -black==22.3.0 -click==8.1.3 -importlib-metadata==1.7.0 -mypy-extensions==0.4.3 -pathspec==0.9.0 -platformdirs==2.5.2 -snakefmt==0.6.0 -toml==0.10.2 -tomli==2.0.1 -typing_extensions==4.2.0 -zipp==3.8.0 diff --git a/lib/linter.sh b/lib/linter.sh index c3fffdf91e8..3b32538c3d4 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -277,7 +277,7 @@ LANGUAGE_ARRAY=('ANSIBLE' 'ARM' 'BASH' 'BASH_EXEC' 'CLANG_FORMAT' 'PHP_PSALM' 'POWERSHELL' 'PROTOBUF' 'PYTHON_BANDIT' 'PYTHON_BLACK' 'PYTHON_PYLINT' 'PYTHON_FLAKE8' 'PYTHON_ISORT' 'PYTHON_MYPY' 'R' 'RAKU' 'RUBY' 'RUST_2015' 'RUST_2018' 'RUST_2021' 'RUST_CLIPPY' 'SCALAFMT' 'SEMGREP' 'SHELL_SHFMT' - 'SNAKEMAKE_LINT' 'SNAKEMAKE_SNAKEFMT' 'STATES' 'SQL' 'SQLFLUFF' 'TEKTON' + 'SNAKEMAKE_LINT' 'STATES' 'SQL' 'SQLFLUFF' 'TEKTON' 'TERRAFORM_TFLINT' 'TERRAFORM_TERRASCAN' 'TERRAGRUNT' 'TSX' 'TYPESCRIPT_ES' "${TYPESCRIPT_STYLE_NAME}" 'XML' 'YAML') From 1ccf4e9fdc698130f9db1af6dfe164e9464a1f3a Mon Sep 17 00:00:00 2001 From: Sarah Cheng Date: Thu, 5 May 2022 23:59:08 -0700 Subject: [PATCH 3/6] SEC-1819 Updated deploy workflow to use the new slim Dockerfile format. --- .github/workflows/deploy-ttam.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/deploy-ttam.yml b/.github/workflows/deploy-ttam.yml index 883fbc307a9..956b370dcba 100644 --- a/.github/workflows/deploy-ttam.yml +++ b/.github/workflows/deploy-ttam.yml @@ -55,24 +55,17 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - ################################################################### - # Checkout the code base (required for docker path context below) # - ################################################################### - - name: Checkout Code - uses: actions/checkout@v2 - ########################################### # Build and Push containers to registries # ########################################### - name: Build and push uses: docker/build-push-action@v2 with: - context: . - file: ./Dockerfile-slim build-args: | BUILD_DATE=${{ env.BUILD_DATE }} BUILD_REVISION=${{ github.sha }} BUILD_VERSION=${{ github.sha }} + target: final_slim push: true tags: | ghcr.io/23andme/super-linter:latest From ed7b8c411370c10dd02949f409da8ed56ca54d13 Mon Sep 17 00:00:00 2001 From: Sarah Cheng Date: Fri, 6 May 2022 02:18:49 -0700 Subject: [PATCH 4/6] SEC-1819 Temp fix for an outside dependency just to get the Dockerfile working again. --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index af9845f0217..61008d74e15 100644 --- a/Dockerfile +++ b/Dockerfile @@ -253,8 +253,8 @@ RUN apk add --no-cache rakudo zef \ ############################## # Install google-java-format # ############################## - && GOOGLE_JAVA_FORMAT_VERSION=$(curl -s https://github.com/google/google-java-format/releases/latest \ - | cut -d '"' -f 2 | cut -d '/' -f 8 | sed -e 's/v//g') \ + && GOOGLE_JAVA_FORMAT_VERSION=$(basename $(curl -s -w %{redirect_url} https://github.com/google/google-java-format/releases/latest) \ + | sed -e 's/v//g') \ && curl --retry 5 --retry-delay 5 -sSL \ "https://github.com/google/google-java-format/releases/download/v$GOOGLE_JAVA_FORMAT_VERSION/google-java-format-$GOOGLE_JAVA_FORMAT_VERSION-all-deps.jar" \ --output /usr/bin/google-java-format \ From 32a07da347f1ed95d7a3e447070949b2408dce42 Mon Sep 17 00:00:00 2001 From: Sarah Cheng Date: Fri, 6 May 2022 18:23:56 -0700 Subject: [PATCH 5/6] SEC-1819 Bumped version of lintly23 to use up-to-date jinja lib. --- dependencies/python/lintly23.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dependencies/python/lintly23.txt b/dependencies/python/lintly23.txt index c69d779a670..873dd1eca77 100644 --- a/dependencies/python/lintly23.txt +++ b/dependencies/python/lintly23.txt @@ -7,8 +7,8 @@ ci-py==1.0.0 click==8.1.3 deprecated==1.2.13 idna==3.3 -jinja2==2.11.3 -lintly23==0.7.13 +jinja2==3.1.2 +lintly23==0.7.14 markupsafe==2.1.1 pycparser==2.21 pygithub==1.55 From fc0b073cf8d535d4a9ddf0e36b7fd9a2c50f32c0 Mon Sep 17 00:00:00 2001 From: Sarah Cheng Date: Mon, 9 May 2022 03:29:55 -0700 Subject: [PATCH 6/6] SEC-1819 Silenced some of the spammier bandit warnings. --- TEMPLATES/.bandit.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TEMPLATES/.bandit.yml b/TEMPLATES/.bandit.yml index 9dc8690e6c7..de41924a04c 100644 --- a/TEMPLATES/.bandit.yml +++ b/TEMPLATES/.bandit.yml @@ -86,6 +86,8 @@ tests: # (optional) list skipped test IDs here, eg '[B101, B406]': skips: + - B101 + - B301 - B311 ### (optional) plugin settings - some test plugins require configuration data ### that may be given here, per-plugin. All bandit test plugins have a built in