audit-repo

#!/bin/bash

set -eu
printf '\n'

# Colors
BOLD="$(tput bold 2>/dev/null || printf '')"
ITALIC="$(tput sitm 2>/dev/null || printf '')"
GREY="$(tput setaf 0 2>/dev/null || printf '')"
UNDERLINE="$(tput smul 2>/dev/null || printf '')"
RED="$(tput setaf 1 2>/dev/null || printf '')"
GREEN="$(tput setaf 2 2>/dev/null || printf '')"
YELLOW="$(tput setaf 3 2>/dev/null || printf '')"
BLUE="$(tput setaf 4 2>/dev/null || printf '')"
MAGENTA="$(tput setaf 5 2>/dev/null || printf '')"
NO_COLOR="$(tput sgr0 2>/dev/null || printf '')"

# RegExes
PASSWORD_IN_URL='//[^/\s:]+:[^/\s:]+@'
OUTPUTS_IN_NOTEBOOK='\"outputs\":\s+\['
AWS_URL='\.amazonaws\.com'
IP_ADDRESS='\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b'
SSH_PRIVATE_KEY='---BEGIN (RSA|OPENSSH|DSA|EC) PRIVATE KEY---'
GPG_PRIVATE_KEY='---BEGIN (PGP|GPG) PRIVATE KEY BLOCK---'
AWS_API_KEY='AKIA[0-9A-Z]{16}'

declare -A RXS=(
    "passwords in URLs"                   "$PASSWORD_IN_URL"
    "presence of outputs in notebooks"    "$OUTPUTS_IN_NOTEBOOK"
    "presence of AWS URLs"                "$AWS_URL"
    "presence of IP addresses"            "$IP_ADDRESS"
    "presence of private keys (SSH)"      "$SSH_PRIVATE_KEY"
    "presence of private keys (GPG)"      "$GPG_PRIVATE_KEY"
    "presence of AWS api keys"            "$AWS_API_KEY"
)

RUN_TRUFFLEHOG=0
RUN_GIT_HISTORY=0

info() {
  printf '%s\n' "${BOLD}${GREY}>${NO_COLOR} $*"
}

warn() {
  printf '%s\n' "${YELLOW}! $*${NO_COLOR}"
}

error() {
  printf '%s\n' "${RED}x $*${NO_COLOR}" >&2
}

completed() {
  printf '%s\n' "${GREEN}✓${NO_COLOR} $*"
}

has() {
  command -v "$1" 1>/dev/null 2>&1
}

verify_trufflehog_installed_or_exit() {
    if ! has trufflehog ; then
        error "Trufflehog is not installed, aborting audit!"
        exit 1
    else
        completed "Audit will use ${BOLD}trufflehog${NO_COLOR}"
        true  # No-op everything is ok
    fi
}

verify_bwrap_installed_or_exit() {
    if ! has bwrap ; then
        error "Bwrap is not installed, aborting audit!"
        exit 1
    else
        completed "Audit will use ${BOLD}bwrap${NO_COLOR}"
        true  # No-op everything is ok

    fi
}

verify_grep_is_installed_or_exit() {
    if has ugrep2; then
        GREP_ARGS=( -E -l --hidden --color=always -R )
        GREP="ugrep"
    elif has rg2; then
        GREP_ARGS=( -l --hidden --color always )
        GREP="rg"
    elif has grep; then
        GREP_ARGS=( -E -l -R -s --exclude-dir=.git -e)
        GREP="grep"
    else
        error "No grep installed neither some of the other supported similar tools (e.g. ugrep or ripgrep), aborting audit!"
        exit 1
    fi
    completed "Audit will be executed using ${BOLD}${GREP}${NO_COLOR}"
}

verify_git_is_installed_or_exit() {
    if ! has git ; then
        error "git is not installed, aborting audit!"
        exit 1
    else
        completed "Audit found ${BOLD}git${NO_COLOR} in the system"
        true
    fi
}

get_tmpdir() {
  git_repo="$1"
  if has mktemp; then
    printf "%s/%s" "$(mktemp -d)" "${git_repo}"
  else
    # No really good options here--let's pick a default + hope
    printf "/tmp/audit/%s" "${git_repo}"
  fi
}

usage() {
  printf "%s\n" \
    "audit-repo [-h] [--help] [${ITALIC}repository${NO_COLOR}...]" \
    "" \
    "Fetch and check for ${UNDERLINE}${BOLD}secrets${NO_COLOR} in the git repository " \
    "Plase note that this may come up with ${ITALIC}false positives${NO_COLOR} -- use your judgment on each result."

  printf "\n%s\n" "Arguments"
  printf "\t%s\n\t\t%s\n\n" \
         "repositories" "One or more repositories to be audited. ${BOLD}${YELLOW}you need to write the full URL path to it${NO_COLOR}" \
         "" "examples: https://github.com/org/test.git, git@gitlab.com:org/test.git"

  printf "\n%s\n" "Options"
  printf "\t%s\n\t\t%s\n\n" \
         "-h, --help"       "Display this help message" \
         "-t, --trufflehog" "Run trufflehog to check for passwords, keys, etc (requires trufllehog to be installed)" \
         "-g, --git-history" "Run all the checks but in the complete repository' history"
}

clone() {
    git_repo="$1"
    temp_dir="$2"
    git clone "${git_repo}" "${temp_dir}"  && completed "cloning ${git_repo} succeded" || (error "cloning ${git_repo} failed" && exit 1)
}

calculate_delta() {
   delta=$(( $(date -d "$1" +%s) - $(date -d "$2" +%s) ))
   printf '%s' $(date -d "@$delta" -u +%H:%M:%S)
}

audit_one() {
    local   initial_timestamp="$(date)"

    info "Started at $(date +'%Y-%m-%d %T')"

    git_url="$1"

    git_repo=$(basename $git_url .git)

    directory=$(get_tmpdir "$git_repo")

    info "Cloning repository $1"
    clone "${git_url}" "${directory}"

    printf "\n"

    info "${UNDERLINE}Auditing repository's working directory${NO_COLOR}"

    printf "\n"

    for rx in "${!RXS[@]}"; do
        echo "$(${GREP} ${GREP_ARGS[@]} ${RXS[$rx]} ${directory})"
        completed "Completed checks for detecting ${rx} (${RXS[$rx]})"
        printf "\n"
    done

    find "${directory}" -type f \( -name '*.csv' -o -name '*.txt' -o -name '*.zip' -o -name '*.png' \) ! -size 0
    completed "Searching for non-empty data or image files (${ITALIC}csv, txt, zip, png${NO_COLOR}) completed"

    printf "\n"

    info "${UNDERLINE}Auditing git's commits history${NO_COLOR}"

    printf "\n"

    if [ "${RUN_GIT_HISTORY}" -eq 1 ]; then

       cd "${directory}"
       for rx in "${!RXS[@]}"; do
           echo ${rx}
	   echo "$(git grep -l -E -e "${RXS[$rx]}" $(git rev-list --all))"
           completed "Completed checks for detecting  ${rx} (${RXS[$rx]})"
           printf "\n"
       done
    fi

    if [ ${RUN_TRUFFLEHOG} -eq 1 ]; then
        trufflehog git file:///"${directory}"
        completed "Running trufflehog (checking for passwords, keys, etc) completed"
    fi

    printf "\n"

    local final_timestamp="$(date)"
    delta=$(calculate_delta "${final_timestamp}" "${initial_timestamp}")
    completed "Audition for ${BOLD}${git_repo} ${NO_COLOR} completed in ${delta}"

    printf "\n"
}

audit_all() {

    info "${UNDERLINE}Audition started ${NO_COLOR}"

    for git_repo in $GIT_REPOS; do
        info "Auditing ${ITALIC}${git_repo} ${NO_COLOR}"
        audit_one ${git_repo}
    done
}

# if no arguments are provided, return usage function
if [ $# -eq 0 ]; then
    warn  You need to specify the repository to audit
    printf "\n"
    usage # run usage function
    exit 1
fi

ARGS=$(getopt --options "htg" --long "help,trufflehog,git-history" -- "$@")

eval set -- "$ARGS"

# parse argv variables
while true; do
 case "$1" in
  -h | --help)
    usage
    exit
    ;;
  -t | --trufflehog)
    RUN_TRUFFLEHOG=1
    ;;
  -g | --git-history)
    RUN_GIT_HISTORY=1
    ;;
  --)
    shift ;
    break
    ;;
  *)
    error "Unknown option: $1"
    usage
    exit 1
    ;;
 esac
 shift ;
done

GIT_REPOS="$*"

printf "  %s\n" "${UNDERLINE}Checking dependencies${NO_COLOR}"

verify_grep_is_installed_or_exit

verify_git_is_installed_or_exit

if [ "${RUN_TRUFFLEHOG}" -eq 1 ]; then
    verify_trufflehog_installed_or_exit
fi

printf '\n'

printf "  %s\n" "${UNDERLINE}Configuration${NO_COLOR}"
info "${BOLD}Repositories to be audited${NO_COLOR}: ${GREEN}${GIT_REPOS}${NO_COLOR}"
if [ "${RUN_TRUFFLEHOG}" -eq 1 ]; then
    info "${BOLD}Audit will run${NO_COLOR}: ${GREEN}trufflehog${NO_COLOR}"
fi
if [ "${RUN_GIT_HISTORY}" -eq 1 ]; then
    info "${BOLD}Audit will run${NO_COLOR}: ${GREEN}checks in repo's commits history${NO_COLOR}"
fi

printf '\n'

audit_all

completed "Audit completed!"

printf '\n'
info "Please check the recommendations (if any) (remember that potentially will be some ${ITALIC}false positives${NO_COLOR}) printed above to complete the cleaning of the repository"