From 46dec4f3d9be21d320b6bb912061f1b7a0c63c05 Mon Sep 17 00:00:00 2001 From: Lillian Zhang <47681745+lillianzhang331@users.noreply.github.com> Date: Wed, 5 May 2021 12:06:48 -0700 Subject: [PATCH] Flush cache with version change (#40) * nodejs- clear cache if node version changed * copy functionality to npm yarn and ts * store_node_version tests for npm nodejs and ts * fix typo * clear cache tests * fixes to clearing cache tests * dont need to mkdir * shellcheck * change version from 12 to 14 * change to use env.build instead of store.toml * use env.build PREV_NODE_VERSION var * address review comments * stub node command, mock node 14, check for PREV_NODE_VERSION var * use -n instead of ! -z --- buildpacks/nodejs/CHANGELOG.md | 2 + buildpacks/nodejs/bin/build | 6 ++ buildpacks/nodejs/lib/build.sh | 52 +++++++++++++--- buildpacks/nodejs/shpec/build_shpec.sh | 70 +++++++++++++++++++--- buildpacks/npm/bin/build | 4 ++ buildpacks/npm/lib/build.sh | 34 ++++++++--- buildpacks/npm/mocks/node/v14/bin/node | 7 +++ buildpacks/npm/shpec/build_shpec.sh | 62 ++++++++++++++++--- buildpacks/typescript/bin/build | 4 ++ buildpacks/typescript/lib/build.sh | 18 ++++++ buildpacks/typescript/shpec/build_shpec.sh | 61 ++++++++++++++++--- buildpacks/yarn/bin/build | 2 + buildpacks/yarn/lib/build.sh | 35 ++++++++--- 13 files changed, 306 insertions(+), 51 deletions(-) create mode 100644 buildpacks/npm/mocks/node/v14/bin/node diff --git a/buildpacks/nodejs/CHANGELOG.md b/buildpacks/nodejs/CHANGELOG.md index 0b907aea..35f3e8e9 100644 --- a/buildpacks/nodejs/CHANGELOG.md +++ b/buildpacks/nodejs/CHANGELOG.md @@ -3,6 +3,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +- Change node engine version from 12 to 14 ([#40](https://github.com/heroku/buildpacks-node/pull/40)) +- Clear cache when node version changes ([#40](https://github.com/heroku/buildpacks-node/pull/40)) ## [0.7.3] 2021/03/04 - Flush cache when stack image changes ([#28](https://github.com/heroku/buildpacks-node/pull/28)) diff --git a/buildpacks/nodejs/bin/build b/buildpacks/nodejs/bin/build index 9610e122..499d02aa 100755 --- a/buildpacks/nodejs/bin/build +++ b/buildpacks/nodejs/bin/build @@ -22,6 +22,8 @@ rm -rf node_modules clear_cache_on_stack_change "$layers_dir" +write_to_store_toml "$layers_dir" + set_up_environment "$layers_dir/nodejs" bootstrap_buildpack "$layers_dir/bootstrap" @@ -29,9 +31,13 @@ bootstrap_buildpack "$layers_dir/bootstrap" install_or_reuse_toolbox "$layers_dir/toolbox" export PATH=$layers_dir/toolbox/bin:$PATH +store_node_version "$layers_dir/nodejs" + install_or_reuse_node "$layers_dir/nodejs" "$build_dir" export PATH=$layers_dir/nodejs/bin:$PATH +clear_cache_on_node_version_change "$layers_dir" "$layers_dir/nodejs" + parse_package_json_engines "$layers_dir/package_manager_metadata" "$build_dir" if [[ -f "yarn.lock" ]]; then diff --git a/buildpacks/nodejs/lib/build.sh b/buildpacks/nodejs/lib/build.sh index 7ca70582..2b522e34 100755 --- a/buildpacks/nodejs/lib/build.sh +++ b/buildpacks/nodejs/lib/build.sh @@ -16,6 +16,18 @@ source "$bp_dir/lib/utils/log.sh" # shellcheck source=/dev/null source "$bp_dir/lib/utils/toml.sh" +write_to_store_toml() { + local layers_dir=$1 + + if [[ ! -f "${layers_dir}/store.toml" ]]; then + touch "${layers_dir}/store.toml" + cat <"${layers_dir}/store.toml" +[metadata] +last_stack = "$CNB_STACK_ID" +TOML + fi +} + clear_cache_on_stack_change() { local layers_dir=$1 @@ -29,14 +41,6 @@ clear_cache_on_stack_change() { rm -rf "${layers_dir:?}"/* fi fi - - if [[ ! -f "${layers_dir}/store.toml" ]]; then - touch "${layers_dir}/store.toml" - cat <"${layers_dir}/store.toml" -[metadata] -last_stack = "$CNB_STACK_ID" -TOML - fi } set_up_environment() { @@ -68,6 +72,18 @@ install_or_reuse_toolbox() { echo "launch = false" >>"${layer_dir}.toml" } +store_node_version() { + local layer_dir=$1 + local prev_node_version + # shellcheck disable=SC2002 + prev_node_version=$(cat "${layer_dir}.toml" | grep version | xargs | cut -d " " -f3) + mkdir -p "${layer_dir}/env.build" + if [[ -s "${layer_dir}/env.build/PREV_NODE_VERSION.override" ]]; then + rm -rf "${layer_dir}/env.build/PREV_NODE_VERSION.override" + fi + echo -e "$prev_node_version\c" >>"${layer_dir}/env.build/PREV_NODE_VERSION.override" +} + install_or_reuse_node() { local layer_dir=$1 local build_dir=$2 @@ -79,7 +95,7 @@ install_or_reuse_node() { status "Installing Node" info "Getting Node version" engine_node=$(json_get_key "$build_dir/package.json" ".engines.node") - node_version=${engine_node:-12.x} + node_version=${engine_node:-14.x} info "Resolving Node version" resolved_data=$(resolve-version node "$node_version") @@ -106,6 +122,24 @@ install_or_reuse_node() { fi } +clear_cache_on_node_version_change() { + local layers_dir=$1 + local layer_dir=$2 + local prev_node_version + local curr_node_version + + curr_node_version="$(node -v)" + curr_node_version=${curr_node_version:1} #to truncate the "v" that is concatedated to version in node -v + if [[ -s "${layer_dir}/env.build/PREV_NODE_VERSION" ]]; then + prev_node_version=$(cat "${layer_dir}/env.build/PREV_NODE_VERSION") + + if [[ "$curr_node_version" != "$prev_node_version" ]]; then + info "Deleting cache because node version changed from \"$prev_node_version\" to \"$curr_node_version\"" + rm -rf "${layers_dir}/yarn" "${layers_dir}/yarn.toml" + fi + fi +} + parse_package_json_engines() { local layer_dir=$1 local build_dir=$2 diff --git a/buildpacks/nodejs/shpec/build_shpec.sh b/buildpacks/nodejs/shpec/build_shpec.sh index 1a7d9902..fa8ff4c4 100644 --- a/buildpacks/nodejs/shpec/build_shpec.sh +++ b/buildpacks/nodejs/shpec/build_shpec.sh @@ -46,6 +46,7 @@ rm_binaries() { describe "lib/build.sh" stub_command "info" + stub_command "node" rm_binaries layers_dir=$(create_temp_layer_dir) @@ -55,14 +56,6 @@ describe "lib/build.sh" export CNB_STACK_ID="heroku-20" - it "creates store.toml when not present" - assert file_absent "$layers_dir/store.toml" - - clear_cache_on_stack_change "$layers_dir" - - assert file_present "$layers_dir/store.toml" - end - it "does not delete layers with same stack" assert file_present "$layers_dir/my_layer.toml" @@ -71,6 +64,8 @@ describe "lib/build.sh" assert file_present "$layers_dir/my_layer.toml" end + write_to_store_toml "$layers_dir" + it "deletes layers when stack changes" CNB_STACK_ID="heroku-22" @@ -84,6 +79,51 @@ describe "lib/build.sh" unset CNB_STACK_ID end + describe "clear_cache_on_node_version_change" + + touch "$layers_dir/yarn" + + it "does not delete layers with same node version" + mkdir "${layers_dir}/nodejs" + mkdir "${layers_dir}/nodejs/env.build" + + echo -e "$(node -v)\c" >>"${layers_dir}/nodejs/env.build/PREV_NODE_VERSION" + + assert file_present "$layers_dir/yarn" + + clear_cache_on_node_version_change "$layers_dir" "$layers_dir/nodejs" + + assert file_present "$layers_dir/yarn" + end + + it "deletes layers when node version changes" + rm -rf "${layers_dir}/nodejs/env.build/PREV_NODE_VERSION" + echo -e "different_version" >>"${layers_dir}/nodejs/env.build/PREV_NODE_VERSION" + + assert file_present "$layers_dir/yarn" + + clear_cache_on_node_version_change "$layers_dir" "$layers_dir/nodejs" + + assert file_absent "$layers_dir/yarn" + end + + end + + describe "write_to_store_toml" + + if [[ -s "$layers_dir/store.toml" ]]; then + rm -rf "$layers_dir/store.toml" + fi + + it "creates store.toml when not present" + assert file_absent "$layers_dir/store.toml" + + write_to_store_toml "$layers_dir" + + assert file_present "$layers_dir/store.toml" + end + end + describe "boostrap_buildpack" create_binaries "$layers_dir/bootstrap" @@ -150,6 +190,19 @@ describe "lib/build.sh" end end + describe "store_node_version" + layers_dir=$(create_temp_layer_dir) + + touch "${layers_dir}/nodejs.toml" + echo -e "[metadata]\nversion = \"test_version\"" > "${layers_dir}/nodejs.toml" + + it "stores node version in PREV_NODE_VERSION env" + assert file_absent "$layers_dir/nodejs/env.build/PREV_NODE_VERSION.override" + store_node_version "$layers_dir/nodejs" + assert equal "$(cat "$layers_dir/nodejs/env.build/PREV_NODE_VERSION.override")" test_version + end + end + describe "install_or_reuse_node" layers_dir=$(create_temp_layer_dir) project_dir=$(create_temp_project_dir) @@ -301,6 +354,7 @@ describe "lib/build.sh" end unstub_command "info" + unstub_command "node" rm_binaries end diff --git a/buildpacks/npm/bin/build b/buildpacks/npm/bin/build index a4be67f1..2d01522a 100755 --- a/buildpacks/npm/bin/build +++ b/buildpacks/npm/bin/build @@ -29,6 +29,10 @@ warn_prebuilt_modules "$build_dir" run_prebuild "$build_dir" +clear_cache_on_node_version_change "$layers_dir" + +write_to_store_toml "$layers_dir" + install_or_reuse_node_modules "$build_dir" "$layers_dir/node_modules" run_build "$build_dir" diff --git a/buildpacks/npm/lib/build.sh b/buildpacks/npm/lib/build.sh index 6b7d49e9..f8accee8 100755 --- a/buildpacks/npm/lib/build.sh +++ b/buildpacks/npm/lib/build.sh @@ -47,14 +47,6 @@ clear_cache_on_stack_change() { rm -rf "${layers_dir:?}"/* fi fi - - if [[ ! -f "${layers_dir}/store.toml" ]]; then - touch "${layers_dir}/store.toml" - cat <"${layers_dir}/store.toml" -[metadata] -last_stack = "$CNB_STACK_ID" -TOML - fi } detect_package_lock() { @@ -143,6 +135,32 @@ install_modules() { fi } +write_to_store_toml() { + local layers_dir=$1 + + if [[ ! -f "${layers_dir}/store.toml" ]]; then + touch "${layers_dir}/store.toml" + cat <"${layers_dir}/store.toml" +[metadata] +last_stack = "$CNB_STACK_ID" +TOML + fi +} + +clear_cache_on_node_version_change() { + local layers_dir=$1 + local curr_node_version + + curr_node_version="$(node -v)" + curr_node_version=${curr_node_version:1} #to truncate the "v" that is concatedated to version in node -v + if [[ -n "$PREV_NODE_VERSION" ]]; then + if [[ "$curr_node_version" != "$PREV_NODE_VERSION" ]]; then + info "Deleting cache because node version changed from \"$PREV_NODE_VERSION\" to \"$curr_node_version\"" + rm -rf "${layers_dir:?}"/* + fi + fi +} + install_or_reuse_node_modules() { local build_dir=$1 local layer_dir=$2 diff --git a/buildpacks/npm/mocks/node/v14/bin/node b/buildpacks/npm/mocks/node/v14/bin/node new file mode 100644 index 00000000..6a0e127d --- /dev/null +++ b/buildpacks/npm/mocks/node/v14/bin/node @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +node_version="v14.16.1" + +if [[ "$1" == "-v" ]]; then + echo "$node_version" +fi diff --git a/buildpacks/npm/shpec/build_shpec.sh b/buildpacks/npm/shpec/build_shpec.sh index 35666475..073b84d6 100755 --- a/buildpacks/npm/shpec/build_shpec.sh +++ b/buildpacks/npm/shpec/build_shpec.sh @@ -53,8 +53,13 @@ use_npm() { PATH="${shpec_dir}/../mocks/npm/v$1/bin:$CURRENT_PATH" } +use_node() { + PATH="${shpec_dir}/../mocks/node/v$1/bin:$CURRENT_PATH" +} + describe "lib/build.sh" install_tools + stub_command "node" CURRENT_PATH=$PATH layers_dir=$(create_temp_layer_dir) @@ -64,14 +69,6 @@ describe "lib/build.sh" export CNB_STACK_ID="heroku-20" - it "creates store.toml when not present" - assert file_absent "$layers_dir/store.toml" - - clear_cache_on_stack_change "$layers_dir" - - assert file_present "$layers_dir/store.toml" - end - it "does not delete layers with same stack" assert file_present "$layers_dir/my_layer.toml" @@ -80,8 +77,10 @@ describe "lib/build.sh" assert file_present "$layers_dir/my_layer.toml" end + write_to_store_toml "$layers_dir" + it "deletes layers when stack changes" - CNB_STACK_ID="heroku-22" + export CNB_STACK_ID="heroku-22" assert file_present "$layers_dir/my_layer.toml" @@ -93,6 +92,50 @@ describe "lib/build.sh" unset CNB_STACK_ID end + describe "clear_cache_on_node_version_change" + + touch "$layers_dir/node_modules" + + it "does not delete layers with same node version" + use_node 14 + version="$(node -v)" + truncated_version=${version:1} + export PREV_NODE_VERSION="$truncated_version" + + assert file_present "$layers_dir/node_modules" + + clear_cache_on_node_version_change "$layers_dir" + + assert file_present "$layers_dir/node_modules" + end + + it "deletes layers when node version changes" + export PREV_NODE_VERSION="different_version" + + assert file_present "$layers_dir/node_modules" + + clear_cache_on_node_version_change "$layers_dir" + + assert file_absent "$layers_dir/node_modules" + end + unset PREV_NODE_VERSION + end + + describe "write_to_store_toml" + + if [[ -s "$layers_dir/store.toml" ]]; then + rm -rf "$layers_dir/store.toml" + fi + + it "creates store.toml when not present" + assert file_absent "$layers_dir/store.toml" + + write_to_store_toml "$layers_dir" + + assert file_present "$layers_dir/store.toml" + end + end + describe "prune_devdependencies" project_dir=$(create_temp_project_dir) use_npm 6 @@ -319,5 +362,6 @@ describe "lib/build.sh" end unstub_command "log_info" + unstub_command "node" rm_tools_and_mocks end diff --git a/buildpacks/typescript/bin/build b/buildpacks/typescript/bin/build index 8df4aa31..fb917e84 100755 --- a/buildpacks/typescript/bin/build +++ b/buildpacks/typescript/bin/build @@ -15,6 +15,10 @@ source "$bp_dir/lib/build.sh" clear_cache_on_stack_change "$layers_dir" +clear_cache_on_node_version_change "$layers_dir" + +write_to_store_toml "$layers_dir" + if check_tsc_binary "$build_dir"; then export PATH=./node_modules/typescript/bin:$PATH fi diff --git a/buildpacks/typescript/lib/build.sh b/buildpacks/typescript/lib/build.sh index ccaf29f2..745c921d 100755 --- a/buildpacks/typescript/lib/build.sh +++ b/buildpacks/typescript/lib/build.sh @@ -25,6 +25,10 @@ clear_cache_on_stack_change() { rm -rf "${layers_dir:?}"/* fi fi +} + +write_to_store_toml() { + local layers_dir=$1 if [[ ! -f "${layers_dir}/store.toml" ]]; then touch "${layers_dir}/store.toml" @@ -35,6 +39,20 @@ TOML fi } +clear_cache_on_node_version_change() { + local layers_dir=$1 + local curr_node_version + + curr_node_version="$(node -v)" + curr_node_version=${curr_node_version:1} #to truncate the "v" that is concatedated to version in node -v + if [[ -n "$PREV_NODE_VERSION" ]]; then + if [[ "$curr_node_version" != "$PREV_NODE_VERSION" ]]; then + info "Deleting cache because node version changed from \"$PREV_NODE_VERSION\" to \"$curr_node_version\"" + rm -rf "${layers_dir:?}"/* + fi + fi +} + detect_out_dir() { local build_dir=$1 diff --git a/buildpacks/typescript/shpec/build_shpec.sh b/buildpacks/typescript/shpec/build_shpec.sh index 1bc9b90c..506e933a 100755 --- a/buildpacks/typescript/shpec/build_shpec.sh +++ b/buildpacks/typescript/shpec/build_shpec.sh @@ -20,6 +20,8 @@ create_temp_layer_dir() { describe "lib/build.sh" + stub_command "node" + layers_dir=$(create_temp_layer_dir) describe "clear_cache_on_stack_change" @@ -27,14 +29,6 @@ describe "lib/build.sh" export CNB_STACK_ID="heroku-20" - it "creates store.toml when not present" - assert file_absent "$layers_dir/store.toml" - - clear_cache_on_stack_change "$layers_dir" - - assert file_present "$layers_dir/store.toml" - end - it "does not delete layers with same stack" assert file_present "$layers_dir/my_layer.toml" @@ -43,8 +37,10 @@ describe "lib/build.sh" assert file_present "$layers_dir/my_layer.toml" end + write_to_store_toml "$layers_dir" + it "deletes layers when stack changes" - CNB_STACK_ID="heroku-22" + export CNB_STACK_ID="heroku-22" assert file_present "$layers_dir/my_layer.toml" @@ -56,6 +52,50 @@ describe "lib/build.sh" unset CNB_STACK_ID end + describe "clear_cache_on_node_version_change" + + touch "$layers_dir/node_modules" + + it "does not delete layers with same node version" + + version="$(node -v)" + truncated_version=${version:1} + export PREV_NODE_VERSION="$truncated_version" + + assert file_present "$layers_dir/node_modules" + + clear_cache_on_node_version_change "$layers_dir" + + assert file_present "$layers_dir/node_modules" + end + + it "deletes layers when node version changes" + export PREV_NODE_VERSION="different_version" + + assert file_present "$layers_dir/node_modules" + + clear_cache_on_node_version_change "$layers_dir" + + assert file_absent "$layers_dir/node_modules" + end + unset PREV_NODE_VERSION + end + + describe "write_to_store_toml" + + if [[ -s "$layers_dir/store.toml" ]]; then + rm -rf "$layers_dir/store.toml" + fi + + it "creates store.toml when not present" + assert file_absent "$layers_dir/store.toml" + + write_to_store_toml "$layers_dir" + + assert file_present "$layers_dir/store.toml" + end + end + describe "detect_out_dir" it "exits with 1 if there is no outDir directory" project_dir=$(create_temp_project_dir) @@ -81,4 +121,7 @@ describe "lib/build.sh" # assert equal "$?" 0 end end + + unstub_command "node" + end diff --git a/buildpacks/yarn/bin/build b/buildpacks/yarn/bin/build index 26e37521..a34929fd 100755 --- a/buildpacks/yarn/bin/build +++ b/buildpacks/yarn/bin/build @@ -20,6 +20,8 @@ fail_multiple_lockfiles "$build_dir" clear_cache_on_stack_change "$layers_dir" export_env "$platform_dir/env" "" "" run_prebuild "$build_dir" +clear_cache_on_node_version_change "$layers_dir" +write_to_store_toml "$layers_dir" install_or_reuse_node_modules "$build_dir" "$layers_dir/node_modules" run_build "$build_dir" write_launch_toml "$build_dir/package.json" "$layers_dir/launch.toml" diff --git a/buildpacks/yarn/lib/build.sh b/buildpacks/yarn/lib/build.sh index a3f2c03b..77b1c537 100644 --- a/buildpacks/yarn/lib/build.sh +++ b/buildpacks/yarn/lib/build.sh @@ -41,14 +41,6 @@ clear_cache_on_stack_change() { rm -rf "${layers_dir:?}"/* fi fi - - if [[ ! -f "${layers_dir}/store.toml" ]]; then - touch "${layers_dir}/store.toml" - cat <"${layers_dir}/store.toml" -[metadata] -last_stack = "$CNB_STACK_ID" -TOML - fi } run_prebuild() { @@ -74,6 +66,33 @@ install_modules() { fi } +write_to_store_toml() { + local layers_dir=$1 + + if [[ ! -f "${layers_dir}/store.toml" ]]; then + touch "${layers_dir}/store.toml" + cat <"${layers_dir}/store.toml" +[metadata] +last_stack = "$CNB_STACK_ID" +TOML + fi +} + +clear_cache_on_node_version_change() { + local layers_dir=$1 + local curr_node_version + + curr_node_version="$(node -v)" + curr_node_version=${curr_node_version:1} #to truncate the "v" that is concatedated to version in node -v + + if [[ -n "$PREV_NODE_VERSION" ]]; then + if [[ "$curr_node_version" != "$PREV_NODE_VERSION" ]]; then + info "Deleting cache because node version changed from \"$PREV_NODE_VERSION\" to \"$curr_node_version\"" + rm -rf "${layers_dir:?}"/* + fi + fi +} + install_or_reuse_node_modules() { local build_dir=$1 local layer_dir=$2