fix: exclude parquet files from checks

infra/build/Makefile

@@ -53,7 +53,7 @@ OPA_VERSION := 0.52.0
 # Updated by Update Tooling Workflow
 GCRANE_VERSION := 0.16.1
 
-DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.14.3
+DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.14.4
 DOCKER_TAG_MAJOR_VERSION_DEVELOPER_TOOLS := $(firstword $(subst ., , $(DOCKER_TAG_VERSION_DEVELOPER_TOOLS)))
 DOCKER_TAG_MINOR_VERSION_DEVELOPER_TOOLS := $(shell echo "${DOCKER_TAG_VERSION_DEVELOPER_TOOLS}" | awk -F. '{print $$1"."$$2}')
 

infra/build/developer-tools/build/scripts/task_helper_functions.sh

@@ -65,7 +65,7 @@ maketemp() {
 #                            ONLY WHEN the "for_header_check" flag is passed in
 #     E.g.: EXCLUDE_HEADER_CHECK="\./config/foo_resource.yml|\./scripts/bar_script.sh"
 find_files() {
-  local pth="$1" find_path_regex="(" exclude_dirs=( ".*/\.git"
+  local pth="$1" find_path_regex="(" exclude_dirs=(".*/\.git"
     ".*/\.terraform"
     ".*/\.terraform.lock.hcl"
     ".*/\.kitchen"
@@ -75,17 +75,18 @@ find_files() {
     ".*/.*\.jpeg"
     ".*/.*\.svg"
     ".*/.*\.ico"
+    ".*/.*\.parquet"
     ".*/.*\.pb"
     ".*/.*\.index"
     "\./autogen"
     "\./test/fixtures/all_examples"
     "\./test/fixtures/shared"
     "\./cache"
-    "\./test/source\.sh" )
+    "\./test/source\.sh")
   shift
 
   # Concat all of the above dirs except the last, separated by a pipe
-  for ((index=0; index<$((${#exclude_dirs[@]}-1)); ++index)); do
+  for ((index = 0; index < $((${#exclude_dirs[@]} - 1)); ++index)); do
     find_path_regex+="${exclude_dirs[index]}|"
   done
 
@@ -144,8 +145,8 @@ function basefiles() {
 # every file named 'Dockerfile'
 function lint_docker() {
   echo "Running hadolint on Dockerfiles"
-  find_files . -name "Dockerfile" -print0 \
-    | compat_xargs -0 hadolint
+  find_files . -name "Dockerfile" -print0 |
+    compat_xargs -0 hadolint
 }
 
 # This function creates TF_PLUGIN_CACHE_DIR if TF_PLUGIN_CACHE_DIR envvar is set
@@ -175,7 +176,7 @@ function check_terraform() {
       echo "Check the output for diffs and correct using terraform fmt <dir>" >&2
       rval="$rc"
     fi
-  done <<< "$(find_files . -name "*.tf" -print)"
+  done <<<"$(find_files . -name "*.tf" -print)"
   if [[ "${rval}" -ne 0 ]]; then
     return "${rval}"
   fi
@@ -185,18 +186,18 @@ function check_terraform() {
 
   # If enable parallel, run validate in parallel
   if [[ "${ENABLE_PARALLEL:-}" -eq 1 ]]; then
-    find_files . -name "*.tf" -print \
-    | grep -v 'test/fixtures/shared' \
-    | compat_xargs -n1 dirname \
-    | sort -u \
-    | parallel --keep-order --retries 3 --joblog /tmp/lint_log terraform_validate
+    find_files . -name "*.tf" -print |
+      grep -v 'test/fixtures/shared' |
+      compat_xargs -n1 dirname |
+      sort -u |
+      parallel --keep-order --retries 3 --joblog /tmp/lint_log terraform_validate
     cat /tmp/lint_log
   else
-    find_files . -name "*.tf" -print \
-    | grep -v 'test/fixtures/shared' \
-    | compat_xargs -n1 dirname \
-    | sort -u \
-    | compat_xargs -t -n1 terraform_validate
+    find_files . -name "*.tf" -print |
+      grep -v 'test/fixtures/shared' |
+      compat_xargs -n1 dirname |
+      sort -u |
+      compat_xargs -t -n1 terraform_validate
   fi
 }
 
@@ -234,9 +235,9 @@ function check_trailing_whitespace() {
 check_whitespace() {
   local rc
   echo "Checking for trailing whitespace"
-  find_files . -print \
-    | grep -v -E '\.(pyc|png|gz|tfvars|mp4|zip|ico|pb|index)$' \
-    | compat_xargs grep -H -n '[[:blank:]]$'
+  find_files . -print |
+    grep -v -E '\.(pyc|png|gz|tfvars|mp4|zip|ico|parquet|pb|index)$' |
+    compat_xargs grep -H -n '[[:blank:]]$'
   rc=$?
   if [[ ${rc} -eq 0 ]]; then
     printf "Error: Trailing whitespace found in the lines above.\n\n"
@@ -245,10 +246,10 @@ check_whitespace() {
     rc=0
   fi
   echo "Checking for missing newline at end of file"
-  find_files . -print \
-    | grep -v -E '\.(png|gz|tfvars|mp4|zip|ico|pb|index)$' \
-    | compat_xargs check_eof_newline
-  return $((rc+$?))
+  find_files . -print |
+    grep -v -E '\.(png|gz|tfvars|mp4|zip|ico|parquet|pb|index)$' |
+    compat_xargs check_eof_newline
+  return $((rc + $?))
 }
 
 # Helper function to facilitate switch to a 0.12 compatible doc generator:
@@ -305,9 +306,9 @@ function generate_docs() {
     else
       echo "Skipping ${path} because README.md does not exist."
     fi
-  done < <(find_files . -name '*.tf' -print0 \
-    | compat_xargs -0 -n1 dirname \
-    | sort -u)
+  done < <(find_files . -name '*.tf' -print0 |
+    compat_xargs -0 -n1 dirname |
+    sort -u)
 
   # disable opt in after https://github.com/GoogleCloudPlatform/cloud-foundation-toolkit/issues/1353
   if [[ "${ENABLE_BPMETADATA:-}" -ne 1 ]]; then
@@ -329,7 +330,7 @@ function generate_metadata() {
     eval "cft blueprint metadata $arg"
   fi
 
-  if [ $? -ne 0 ]; then    
+  if [ $? -ne 0 ]; then
     echo "Warning! Unable to generate metadata."
     return 1
   fi
@@ -364,33 +365,33 @@ function check_tflint() {
   rval=0
   echo "Checking for tflint"
   local path
-    while read -r path; do
-      local tflintCfg
-      # skip any tf configs under test/
-      if [[ $path == "./test"* ]];then
-        echo "Skipping ${path}"
-        continue
-      fi
-      # load default ruleset
-      tflintCfg="/root/tflint/.tflint.example.hcl"
-      # if module, load tighter ruleset
-      if [[ $path == "." || $path == "./modules"* ]];then
-        tflintCfg="/root/tflint/.tflint.module.hcl"
-      fi
+  while read -r path; do
+    local tflintCfg
+    # skip any tf configs under test/
+    if [[ $path == "./test"* ]]; then
+      echo "Skipping ${path}"
+      continue
+    fi
+    # load default ruleset
+    tflintCfg="/root/tflint/.tflint.example.hcl"
+    # if module, load tighter ruleset
+    if [[ $path == "." || $path == "./modules"* ]]; then
+      tflintCfg="/root/tflint/.tflint.module.hcl"
+    fi
 
-      cd "${path}" && echo "Working in ${path} ..."
-      tflint --config=${tflintCfg} --no-color
-      rc=$?
-      if [[ "${rc}" -ne 0 ]]; then
-        echo "tflint failed ${path} "
-        ((rval++))
-      else
-        echo "tflint passed ${path} "
-      fi
-      cd - >/dev/null
-    done < <(find_files . -name '*.tf' -print0 \
-      | compat_xargs -0 -n1 dirname \
-      | sort -u)
+    cd "${path}" && echo "Working in ${path} ..."
+    tflint --config=${tflintCfg} --no-color
+    rc=$?
+    if [[ "${rc}" -ne 0 ]]; then
+      echo "tflint failed ${path} "
+      ((rval++))
+    else
+      echo "tflint passed ${path} "
+    fi
+    cd - >/dev/null
+  done < <(find_files . -name '*.tf' -print0 |
+    compat_xargs -0 -n1 dirname |
+    sort -u)
   return $((rval))
 }
 
@@ -448,7 +449,7 @@ function post_lint_status_pr_comment() {
   export GITHUB_PAT_TOKEN=$(gcloud secrets versions access latest --secret="gh-pat-token")
   final_message=$(/usr/local/bin/test_lint.sh --markdown --contrib-guide=../blob/master/CONTRIBUTING.md)
   if [ -z "$final_message" ]; then
-  final_message="Thanks for the PR! 🚀<br/>✅ Lint checks have passed."
+    final_message="Thanks for the PR! 🚀<br/>✅ Lint checks have passed."
   fi
   python3 /usr/local/bin/gh_lint_comment.py -r "${REPO_NAME}" -p "${_PR_NUMBER}" -c "${final_message}"
 }
@@ -513,8 +514,7 @@ function check_headers() {
 function fix_headers() {
   echo "Adding file license headers"
   YEAR=$(date +'%Y')
-  if [ $# -eq 0 ]
-  then
+  if [ $# -eq 0 ]; then
     find_files . for_header_check -type f -print0 | compat_xargs -0 addlicense -y $YEAR
   else
     addlicense -y $YEAR "$@"
@@ -542,7 +542,7 @@ init_credentials() {
   local tmpfile
   # shellcheck disable=SC2119
   tmpfile="$(maketemp)"
-  echo "${SERVICE_ACCOUNT_JSON}" > "${tmpfile}"
+  echo "${SERVICE_ACCOUNT_JSON}" >"${tmpfile}"
 
   # Terraform and most other tools respect GOOGLE_CREDENTIALS
   # https://www.terraform.io/docs/providers/google/provider_reference.html#credentials-1
@@ -560,7 +560,7 @@ init_credentials() {
 }
 
 init_credentials_if_found() {
-   if [[ -z "${SERVICE_ACCOUNT_JSON:-}" ]]; then
+  if [[ -z "${SERVICE_ACCOUNT_JSON:-}" ]]; then
     echo "Proceeding using application default credentials"
   else
     init_credentials
@@ -583,7 +583,7 @@ prepare_environment() {
   fi
 }
 
- # Destroy the setup environment
+# Destroy the setup environment
 cleanup_environment() {
   set -eu
 
@@ -627,12 +627,12 @@ kitchen_do() {
   local command="$1"
   shift
   case "$command" in
-    create | converge | destroy | setup | test | verify)
-      kitchen "$command" "$@" --test-base-path="$KITCHEN_TEST_BASE_PATH"
-      ;;
-    *)
-      kitchen "$command" "$@"
-      ;;
+  create | converge | destroy | setup | test | verify)
+    kitchen "$command" "$@" --test-base-path="$KITCHEN_TEST_BASE_PATH"
+    ;;
+  *)
+    kitchen "$command" "$@"
+    ;;
   esac
 }
 
@@ -656,7 +656,6 @@ finish_integration() {
   exit "${rv}"
 }
 
-
 # This function is called by /usr/local/bin/test_validator.sh and can be
 # overridden on a per-module basis to implement additional steps.
 run_terraform_validator() {
@@ -667,46 +666,42 @@ run_terraform_validator() {
   project="$2"
   policy_file_path="$3"
 
-
   export tf_name=$(basename -- $tf_full_path)
   export base_dir=$(pwd)
   export tmp_plan="${base_dir}/test/integration/tmp/tfvt/${tf_name}"
 
-
   echo "*************** TFV VALIDATE ************************"
   echo "      Validating $tf_name at path $tf_full_path"
   echo "      Using policy from: $policy_file_path "
   echo "      in project: $project"
   echo "*****************************************************"
 
-
   if [ ! -d "$tmp_plan" ]; then
-      mkdir -p "$tmp_plan/" || exit 1
+    mkdir -p "$tmp_plan/" || exit 1
   fi
 
   if [ -z "$policy_file_path" ]; then
-      echo "no policy repo found! Check the argument provided for policysource to this script."
-      echo "https://github.com/GoogleCloudPlatform/terraform-validator/blob/main/docs/policy_library.md"
-      exit 1
+    echo "no policy repo found! Check the argument provided for policysource to this script."
+    echo "https://github.com/GoogleCloudPlatform/terraform-validator/blob/main/docs/policy_library.md"
+    exit 1
   else
-      if [ -d "$tf_full_path" ]; then
+    if [ -d "$tf_full_path" ]; then
 
-          cd "$tf_full_path" || exit 1
+      cd "$tf_full_path" || exit 1
 
-          terraform plan -input=false -out "$tmp_plan/plan.tfplan"  || exit 1
-          terraform show -json "$tmp_plan/plan.tfplan" > "$tmp_plan/plan.json" || exit 1
+      terraform plan -input=false -out "$tmp_plan/plan.tfplan" || exit 1
+      terraform show -json "$tmp_plan/plan.tfplan" >"$tmp_plan/plan.json" || exit 1
 
-          terraform-validator validate "$tmp_plan/plan.json" --policy-path="$policy_file_path" --project="$project" || exit 1
+      terraform-validator validate "$tmp_plan/plan.json" --policy-path="$policy_file_path" --project="$project" || exit 1
 
-          cd "$base_dir" || exit
-      else
-        echo "ERROR:  $tf_full_path does not exist"
-        exit 1
-      fi
+      cd "$base_dir" || exit
+    else
+      echo "ERROR:  $tf_full_path does not exist"
+      exit 1
+    fi
   fi
 }
 
-
 # Intended to allow a module to customize a particular check or behavior.  For
 # example, the pubsub module runs "kitchen converge" twice instead of the
 # default one time.