From fa8b70745d333f9c6a8a65b11fd2367d661c1b11 Mon Sep 17 00:00:00 2001 From: Aashir Siddiqui Date: Mon, 25 Nov 2024 11:45:13 +0000 Subject: [PATCH 1/3] Added detect-secrets script to CLI Signed-off-by: Aashir Siddiqui --- .github/workflows/pr-build.yml | 6 + build-locally.sh | 2 +- detect-secrets.sh | 236 +++++++++++++++++++++++++++++++++ 3 files changed, 243 insertions(+), 1 deletion(-) create mode 100755 detect-secrets.sh diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 70424103..b82edd7d 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -94,6 +94,12 @@ jobs: run : | ./test-galasactl-local.sh --buildTool gradle + - name: Turn script into an executable + run: chmod +x detect-secrets.sh + + - name: Run the detect secrets script + run: ./detect-secrets.sh + # Commenting out for now as we cannot reach the prod1 ecosystem from GitHub Actions. # - name: Chmod ecosystem test script # run : | diff --git a/build-locally.sh b/build-locally.sh index 277cc816..a59cad10 100755 --- a/build-locally.sh +++ b/build-locally.sh @@ -680,7 +680,7 @@ build_generated_source_gradle run_test_locally_using_galasactl ${BASEDIR}/temp/local-run-log-gradle.txt -check_secrets +${BASEDIR}/detect-secrets.sh # launch_test_on_ecosystem # test_on_windows diff --git a/detect-secrets.sh b/detect-secrets.sh new file mode 100755 index 00000000..4d4c6139 --- /dev/null +++ b/detect-secrets.sh @@ -0,0 +1,236 @@ +#!/usr/bin/env bash + +# +# Copyright contributors to the Galasa project +# +# SPDX-License-Identifier: EPL-2.0 +# + +#----------------------------------------------------------------------------------------- +# +# Objectives: Detect secrets in every repo in galasa, this will prevent commiting any +# secrets to Github. If it finds any secrets, build should fail! +# +#----------------------------------------------------------------------------------------- + +# Where is this script executing from ? +BASEDIR=$(dirname "$0") +pushd $BASEDIR 2>&1 >>/dev/null +BASEDIR=$(pwd) +popd 2>&1 >>/dev/null +# echo "Running from directory ${BASEDIR}" +export ORIGINAL_DIR=$(pwd) +# cd "${BASEDIR}" + +cd "${BASEDIR}" +REPO_ROOT=$(pwd) + +#----------------------------------------------------------------------------------------- +# +# Set Colors +# +#----------------------------------------------------------------------------------------- +bold=$(tput bold) +underline=$(tput sgr 0 1) +reset=$(tput sgr0) +red=$(tput setaf 1) +green=$(tput setaf 76) +white=$(tput setaf 7) +tan=$(tput setaf 202) +blue=$(tput setaf 25) + +#----------------------------------------------------------------------------------------- +# +# Headers and Logging +# +#----------------------------------------------------------------------------------------- +underline() { printf "${underline}${bold}%s${reset}\n" "$@"; } +h1() { printf "\n${underline}${bold}${blue}%s${reset}\n" "$@"; } +h2() { printf "\n${underline}${bold}${white}%s${reset}\n" "$@"; } +debug() { printf "${white}[.] %s${reset}\n" "$@"; } +info() { printf "${white}[➜] %s${reset}\n" "$@"; } +success() { printf "${white}[${green}✔${white}] ${green}%s${reset}\n" "$@"; } +error() { printf "${white}[${red}✖${white}] ${red}%s${reset}\n" "$@"; } +warn() { printf "${white}[${tan}➜${white}] ${tan}%s${reset}\n" "$@"; } +bold() { printf "${bold}%s${reset}\n" "$@"; } +note() { printf "\n${underline}${bold}${blue}Note:${reset} ${blue}%s${reset}\n" "$@"; } + +#----------------------------------------------------------------------------------------- +# Functions +#----------------------------------------------------------------------------------------- + +function usage { + info "Syntax: detect-secrets.sh [OPTIONS]" + cat </dev/null; then + success "Python3 is already installed." + else + error "Please install Python3 to conitnue." + exit 1 + fi +} + +# Function to check if pip3 is installed +check_pip3_installed() { + + h2 "Checking if pip3 is installed" + + if ! command -v pip3 &> /dev/null; then + error "pip3 is not installed. Please install it to proceed." + exit 1 + else + success "pip3 is installed." + fi +} + +function check_if_detect_secrets_is_installed() { + h2 "Checking if detect-secrets is installed" + + # Check if detect-secrets is installed in the virtual environment + if command -v detect-secrets &> /dev/null; then + info "detect-secrets is already installed." + else + info "detect-secrets is not installed. Installing now..." + + # Install detect-secrets from IBM GitHub repository + pip3 install --upgrade "git+https://github.com/ibm/detect-secrets.git@master#egg=detect-secrets" + + # Verify if the installation was successful + if command -v detect-secrets &> /dev/null; then + info "detect-secrets was installed correctly" + else + error "Failed to install detect-secrets" + deactivate + exit 1 + fi + fi + + success "OK" +} + +function check_if_pre_commit_hook_is_installed() { + + h2 "Checking if pre-commit hook is installed" + + if command -v pre-commit &> /dev/null; then + info "pre-commit hook is already installed." + else + info "pre-commit hook is not installed. Installing now..." + + # Install pre-commit hook + pip3 install pre-commit + + if command -v pre-commit &> /dev/null; then + info "pre-commit hook was installed correctly" + info "Activating pre commit hook" + pre-commit install + else + error "Failed to install pre-commit hook" + exit 1 + fi + fi + + success "OK" +} + +function remove_timestamp_from_secrets_baseline() { + h2 "Removing the timestamp from the secrets baseline file so it doesn't always cause a git change." + + mkdir -p ${BASEDIR}/temp + rc=$? + check_exit_code $rc "Failed to create a temporary folder" + + cat ${baseline_file} | grep -v "generated_at" >${BASEDIR}/temp/.secrets.baseline.temp + rc=$? + check_exit_code $rc "Failed to create a temporary file with no timestamp inside" + + mv ${BASEDIR}/temp/.secrets.baseline.temp $1 + rc=$? + check_exit_code $rc "Failed to overwrite the secrets baseline with one containing no timestamp inside." + + success "secrets baseline timestamp content has been removed ok" +} + +function check_secrets { + h2 "updating secrets baseline" + cd $REPO_ROOT + baseline_file=".secrets.baseline" + + cmd="detect-secrets scan --update ${baseline_file}" + info "Running command $cmd" + $cmd + rc=$? + check_exit_code $rc "Failed to run detect-secrets. Please check it is installed properly" + success "updated secrets file" + + h2 "running audit for secrets" + cmd="detect-secrets audit ${baseline_file}" + info "Running command $cmd" + $cmd + rc=$? + check_exit_code $rc "Failed to audit detect-secrets." + + #Check all secrets have been audited + secrets=$(grep -c hashed_secret ${baseline_file}) + audits=$(grep -c is_secret ${baseline_file}) + if [[ "$secrets" != "$audits" ]]; then + error "Not all secrets found have been audited" + exit 1 + fi + + remove_timestamp_from_secrets_baseline ${baseline_file} + + success "secrets audit complete" + +} + +#----------------------------------------------------------------------------------------- +# Process parameters +#----------------------------------------------------------------------------------------- + +while [ "$1" != "" ]; do + case $1 in + -h | --help) + usage + exit + ;; + + *) + error "Unexpected argument $1" + usage + exit 1 + ;; + esac + shift +done + +#----------------------------------------------------------------------------------------- +# Main logic. +#----------------------------------------------------------------------------------------- + +h1 "Starting search in repos to detect secrets" +check_if_python3_is_installed +check_pip3_installed +check_if_detect_secrets_is_installed +check_if_pre_commit_hook_is_installed + +check_secrets \ No newline at end of file From 6c7ff71dc4bc6d09461f0e24b8b1c95ba40a2e34 Mon Sep 17 00:00:00 2001 From: Aashir Siddiqui Date: Mon, 25 Nov 2024 11:59:24 +0000 Subject: [PATCH 2/3] Moved detect-secrets earlier in steps Signed-off-by: Aashir Siddiqui --- .github/workflows/pr-build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index b82edd7d..b7740d35 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -23,6 +23,12 @@ jobs: - name: Checkout CLI uses: actions/checkout@v4 + - name: Turn script into an executable + run: chmod +x detect-secrets.sh + + - name: Run the detect secrets script + run: ./detect-secrets.sh + - name: Setup Gradle uses: gradle/actions/setup-gradle@v3 with: @@ -94,12 +100,6 @@ jobs: run : | ./test-galasactl-local.sh --buildTool gradle - - name: Turn script into an executable - run: chmod +x detect-secrets.sh - - - name: Run the detect secrets script - run: ./detect-secrets.sh - # Commenting out for now as we cannot reach the prod1 ecosystem from GitHub Actions. # - name: Chmod ecosystem test script # run : | From adc584d9ad4c15bfbc72e4399ff8be192a5ac2c1 Mon Sep 17 00:00:00 2001 From: Aashir Siddiqui Date: Wed, 27 Nov 2024 14:27:57 +0000 Subject: [PATCH 3/3] Empty commit to trigger rebuild Signed-off-by: Aashir Siddiqui