From 0e5a106255cdea02b477ed9cc4692e89f69c206e Mon Sep 17 00:00:00 2001 From: Sarah Cheng Date: Thu, 5 May 2022 19:09:52 -0700 Subject: [PATCH] SEC-1819 Removed deps checker. Revert "SEC-1548 fixing output format" This reverts commit 90399d0d68a6f5dcd7ca3859c3e3f808c94bdc7b. Revert "SEC-1548 copying linter file to docker image" This reverts commit 15d7d3a1b55491104d1f945d32f1205e447f5374. Revert "SEC-1548 adding exit code to deps checker" This reverts commit d07c6eb563e3b81779363665373af1a9b6aa8ced. Revert "Fixing version error code" This reverts commit 37283f13dff43270058f877e5351178252f9a123. Revert "SEC-1548 Adding dependency checker tool" This reverts commit 5b904e8a162339749d7d55daa2a7eefc4442340d. --- Dockerfile | 5 ----- Dockerfile-slim | 5 ----- README.md | 14 +++++++------- lib/functions/buildFileList.sh | 2 -- lib/functions/linterVersions.sh | 2 +- lib/functions/lintly.sh | 1 - lib/linter.sh | 6 ++---- ttam-linters/deps-checker.sh | 34 --------------------------------- 8 files changed, 10 insertions(+), 59 deletions(-) delete mode 100755 ttam-linters/deps-checker.sh diff --git a/Dockerfile b/Dockerfile index 80803f02384..3d4d3981b5f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -439,11 +439,6 @@ ENV ARM_TTK_PSD1="${ARM_TTK_DIRECTORY}/arm-ttk-master/arm-ttk/arm-ttk.psd1" ############################# COPY lib /action/lib -#################################### -# Copy custom linters to container # -#################################### -COPY ttam-linters /action/ttam-linters - ################################## # Copy linter rules to container # ################################## diff --git a/Dockerfile-slim b/Dockerfile-slim index a40dc617758..d9a3fee2f53 100644 --- a/Dockerfile-slim +++ b/Dockerfile-slim @@ -374,11 +374,6 @@ ENV PATH="${PATH}:/node_modules/.bin" ############################# COPY lib /action/lib -#################################### -# Copy custom linters to container # -#################################### -COPY ttam-linters /action/ttam-linters - ################################## # Copy linter rules to container # ################################## diff --git a/README.md b/README.md index fccaba2ce53..a61a1d19657 100644 --- a/README.md +++ b/README.md @@ -104,12 +104,12 @@ Developers on **GitHub** can call the **GitHub Action** to lint their codebase w | **YAML** | [YamlLint](https://github.com/adrienverge/yamllint) | ### 23andMe Custom Linters -| _Language_ | _Linter_ | -| -------------------------------- | ------------------------------------------------------------------------------------------------ | -| _(All files)_ | [semgrep](https://github.com/returntocorp/semgrep) | -| _(Dependency files)_ | [deps-checker](https://github.com/23andMe/super-linter/blob/master/ttam-linters/deps-checker.sh) | -| **AWS CloudFormation templates** | [cfn-nag](https://github.com/stelligent/cfn_nag) | -| **Python3** | [bandit](https://github.com/PyCQA/bandit) | +| _Language_ | _Linter_ | +| -------------------------------- | ----------------------------------------------------| +| _(All files)_ | [semgrep](https://github.com/returntocorp/semgrep) | +| **AWS CloudFormation templates** | [cfn-nag](https://github.com/stelligent/cfn_nag) | +| **Python3** | [bandit](https://github.com/PyCQA/bandit) | + ## How to use @@ -438,7 +438,7 @@ The following is a list of supported language packs. | **Language pack** | **Included Tools** | | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | -| **RUN_SECURITY_TOOLS** | `CLOUDFORMATION_CFN_NAG`
`DEPS_CHECKER`
`DOCKERFILE_HADOLINT`
`GITLEAKS`
`PYTHON_BANDIT`
`SEMGREP`
`TERRAFORM_TERRASCAN` | +| **RUN_SECURITY_TOOLS** | `CLOUDFORMATION_CFN_NAG`
`DOCKERFILE_HADOLINT`
`GITLEAKS`
`PYTHON_BANDIT`
`SEMGREP`
`TERRAFORM_TERRASCAN` | | **RUN_CODE_QUALITY_TOOLS** | _(All other tools that are currently not security tools)_ | ### Template rules files diff --git a/lib/functions/buildFileList.sh b/lib/functions/buildFileList.sh index 6764a306dc8..a04ba524d1f 100755 --- a/lib/functions/buildFileList.sh +++ b/lib/functions/buildFileList.sh @@ -318,8 +318,6 @@ function BuildFileList() { FILE_ARRAY_JSCPD+=("${FILE}") # GitLeaks also runs an all files FILE_ARRAY_GITLEAKS+=("${FILE}") - # deps-checker should also check all the files - FILE_ARRAY_DEPS_CHECKER+=("${FILE}") # Deps-checker will filter for appropriate files ####################### # Get the shell files # diff --git a/lib/functions/linterVersions.sh b/lib/functions/linterVersions.sh index 28d6bea7ba1..d7a50725b64 100755 --- a/lib/functions/linterVersions.sh +++ b/lib/functions/linterVersions.sh @@ -68,7 +68,7 @@ BuildLinterVersions() { if [[ ${LINTER} == "arm-ttk" ]]; then # Need specific command for ARM GET_VERSION_CMD="$(grep -iE 'version' "/usr/bin/arm-ttk" | xargs 2>&1)" - elif [[ ${LINTER} == "bash-exec" ]] || [[ ${LINTER} == "deps-checker" ]] || [[ ${LINTER} == "gherkin-lint" ]] || [[ ${LINTER} == "gitleaks" ]]; then + elif [[ ${LINTER} == "bash-exec" ]] || [[ ${LINTER} == "gherkin-lint" ]] || [[ ${LINTER} == "gitleaks" ]]; then # Need specific command for Protolint and editorconfig-checker GET_VERSION_CMD="$(echo "--version not supported")" elif [[ ${LINTER} == "lintr" ]]; then diff --git a/lib/functions/lintly.sh b/lib/functions/lintly.sh index b380a0794a7..6a126e4af8a 100755 --- a/lib/functions/lintly.sh +++ b/lib/functions/lintly.sh @@ -22,7 +22,6 @@ LINTLY_SUPPORT_ARRAY['DOCKERFILE_HADOLINT']="hadolint" LINTLY_SUPPORT_ARRAY['CLOUDFORMATION']="cfn-lint" LINTLY_SUPPORT_ARRAY['CLOUDFORMATION_CFN_NAG']="cfn-nag" LINTLY_SUPPORT_ARRAY['GITLEAKS']="gitleaks" -LINTLY_SUPPORT_ARRAY['DEPS_CHECKER']='deps-checker' LINTLY_SUPPORT_ARRAY['SEMGREP']='semgrep' export LINTLY_SUPPORT_ARRAY # Workaround SC2034 diff --git a/lib/linter.sh b/lib/linter.sh index c2a15e56b2b..52436b6d147 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -235,7 +235,7 @@ RUN_CODE_QUALITY_TOOLS="${RUN_CODE_QUALITY_TOOLS:-false}" # Language array # ################## LANGUAGE_ARRAY=('ANSIBLE' 'ARM' 'BASH' 'BASH_EXEC' 'CLANG_FORMAT' - 'CLOUDFORMATION' 'CLOUDFORMATION_CFN_NAG' 'CLOJURE' 'COFFEESCRIPT' 'CPP' 'CSHARP' 'CSS' 'DART' 'DEPS_CHECKER' + 'CLOUDFORMATION' 'CLOUDFORMATION_CFN_NAG' 'CLOJURE' 'COFFEESCRIPT' 'CPP' 'CSHARP' 'CSS' 'DART' 'DOCKERFILE' 'DOCKERFILE_HADOLINT' 'EDITORCONFIG' 'ENV' 'GITHUB_ACTIONS' 'GITLEAKS' 'GHERKIN' 'GO' 'GOOGLE_JAVA_FORMAT' 'GROOVY' 'HTML' 'JAVA' 'JAVASCRIPT_ES' "${JAVASCRIPT_STYLE_NAME}" 'JSCPD' 'JSON' 'JSONC' 'JSX' @@ -264,7 +264,6 @@ LINTER_NAMES_ARRAY['CPP']="cpplint" LINTER_NAMES_ARRAY['CSHARP']="dotnet-format" LINTER_NAMES_ARRAY['CSS']="stylelint" LINTER_NAMES_ARRAY['DART']="dart" -LINTER_NAMES_ARRAY['DEPS_CHECKER']="deps-checker" LINTER_NAMES_ARRAY['DOCKERFILE']="dockerfilelint" LINTER_NAMES_ARRAY['DOCKERFILE_HADOLINT']="hadolint" LINTER_NAMES_ARRAY['EDITORCONFIG']="editorconfig-checker" @@ -336,7 +335,7 @@ LINTED_LANGUAGES_ARRAY=() # Will be filled at run time with all languages that w ################## LANGUAGE_PACKS=('SECURITY_TOOLS' 'CODE_QUALITY_TOOLS') # shellcheck disable=SC2034 # Variable is referenced indirectly -SECURITY_TOOLS=('CLOUDFORMATION_CFN_NAG' 'DOCKERFILE_HADOLINT' 'GITLEAKS' 'PYTHON_BANDIT' 'TERRAFORM_TERRASCAN' 'DEPS_CHECKER' 'SEMGREP') +SECURITY_TOOLS=('CLOUDFORMATION_CFN_NAG' 'DOCKERFILE_HADOLINT' 'GITLEAKS' 'PYTHON_BANDIT' 'TERRAFORM_TERRASCAN' 'SEMGREP') # CODE_QUALITY_TOOLS are all tools (i.e., items in LANGUAGE_ARRAY) that are not in SECURITY_TOOLS # shellcheck disable=SC2034 # Variable is referenced indirectly mapfile -t CODE_QUALITY_TOOLS < <(printf '%s\n' "${LANGUAGE_ARRAY[@]}" "${SECURITY_TOOLS[@]}" | sort | uniq -u) @@ -740,7 +739,6 @@ LINTER_COMMANDS_ARRAY['CPP']="cpplint" LINTER_COMMANDS_ARRAY['CSHARP']="dotnet-format --folder --check --exclude / --include" LINTER_COMMANDS_ARRAY['CSS']="stylelint --config ${CSS_LINTER_RULES}" LINTER_COMMANDS_ARRAY['DART']="dartanalyzer --fatal-infos --fatal-warnings --options ${DART_LINTER_RULES}" -LINTER_COMMANDS_ARRAY['DEPS_CHECKER']="/action/ttam-linters/deps-checker.sh" # NOTE: dockerfilelint's "-c" option expects the folder *containing* the DOCKER_LINTER_RULES file LINTER_COMMANDS_ARRAY['DOCKERFILE']="dockerfilelint -c $(dirname "${DOCKERFILE_LINTER_RULES}")" LINTER_COMMANDS_ARRAY['DOCKERFILE_HADOLINT']="hadolint -c ${DOCKERFILE_HADOLINT_LINTER_RULES} ${LINTER_OPTS[DOCKERFILE_HADOLINT]}" diff --git a/ttam-linters/deps-checker.sh b/ttam-linters/deps-checker.sh deleted file mode 100755 index 70246a64bad..00000000000 --- a/ttam-linters/deps-checker.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -declare -A ERROR_CODES - -# declaring an hastable for file and its codes -ERROR_CODES["requirements.txt"]="A001" -ERROR_CODES["Pipfile"]="A002" -ERROR_CODES["Dockerfile"]="A003" -ERROR_CODES["package.json"]="A004" -ERROR_CODES["production.txt"]="A005" - -# File passed by superlinter -FILE_PATH=$1 - -# This one is to get file name -FILE_NAME=$(echo "${FILE_PATH}" | rev | cut -d "/" -f1 | rev) - -# Checking if files are start with "-r" -POINTER_FILES="$(grep -iE "^-r " "${FILE_PATH}")" - -format_output_for_lintly() { - - output=${FILE_PATH}":1:1: ${ERROR_CODES[${FILE_NAME}]} does not explicitly point to an approved artifact repository manager." - echo "${output}" - -} - -if [[ -v ERROR_CODES["${FILE_NAME}"] ]] && [[ -z "${POINTER_FILES}" ]]; then - NON_COMPLIANT_FILES="$(grep -Li "$COMPLIANT_FILTER" "$FILE_PATH")" - if [[ -n "${NON_COMPLIANT_FILES}" ]]; then - format_output_for_lintly - exit 1 - fi -fi