From d535a259762549736f4e1150bba05d72b84853fe Mon Sep 17 00:00:00 2001
From: Ryan Phillips <ryan.a.phill@gmail.com>
Date: Thu, 23 Jan 2020 13:11:55 -0500
Subject: [PATCH] Ensure outdated packages get removed from the cache

If a package has a new version released upstream the prior version
will remain in the cache. Similarly if a package has any
dependencies changed to a different package or removed, those
unneeded packages will remain in the cache as well. Due to this
the slug size will slowly balloon overtime as long as the stack
or Aptfile is not changed.

This tracks the packages that are downloaded during installation
and removes all debs not in that list from the cache archive.
`apt-get autoclean` is not used because it will only clean out
packages that can no longer be downloaded so not all  outdated
versions will be removed. If the cache has version 1, 2, and 3
of a package in it and version 1 is removed from upstream
then version 1 will be removed but 2 will remain.
---
 bin/compile          | 22 ++++++++++++++++++++++
 test/compile_test.sh | 18 ++++++++++++++++--
 2 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/bin/compile b/bin/compile
index af544c2..f63aa87 100755
--- a/bin/compile
+++ b/bin/compile
@@ -78,6 +78,8 @@ APT_OPTIONS="$APT_OPTIONS -o dir::etc::sourcelist=$APT_SOURCES -o dir::etc::sour
 topic "Updating apt caches"
 apt-get $APT_OPTIONS update | indent
 
+PACKAGES="$CACHE_DIR/package-list"
+
 for PACKAGE in $(cat $BUILD_DIR/Aptfile | grep -v -s -e '^#' | grep -v -s -e "^:repo:"); do
   if [[ $PACKAGE == *deb ]]; then
     PACKAGE_NAME=$(basename $PACKAGE .deb)
@@ -85,12 +87,32 @@ for PACKAGE in $(cat $BUILD_DIR/Aptfile | grep -v -s -e '^#' | grep -v -s -e "^:
 
     topic "Fetching $PACKAGE"
     curl -s -L -z $PACKAGE_FILE -o $PACKAGE_FILE $PACKAGE 2>&1 | indent
+    echo $PACKAGE_NAME >> $PACKAGES
   else
     topic "Fetching .debs for $PACKAGE"
     apt-get $APT_OPTIONS -y $APT_FORCE_YES -d install --reinstall $PACKAGE | indent
+
+    # Changes the archives location to ensure the package urls are printed.
+    # Nothing is downloaded/updated here so the archive location can be set to anywhere
+    # other than the actual archives location without issue.
+    apt-get $APT_OPTIONS -y $APT_FORCE_YES -o dir::cache::archives=$APT_CACHE_DIR \
+      install --print-uris --reinstall $PACKAGE | \
+      grep ^\' | awk '{print $2}' >> $PACKAGES
   fi
 done
 
+# Removes duplicate file names
+cat $PACKAGES | awk '!seen[$0]++' > "$PACKAGES-clean"
+mv "$PACKAGES-clean" $PACKAGES
+
+# Remove outdated and leftover packages
+topic "Cleaning up outdated and leftover packages"
+find $APT_CACHE_DIR/archives -type f -name "*.deb" | \
+  grep -vFf $PACKAGES | xargs --no-run-if-empty rm
+
+# Remove the packages list file so we have a clean file for subsequent runs
+rm $PACKAGES
+
 mkdir -p $BUILD_DIR/.apt
 
 for DEB in $(ls -1 $APT_CACHE_DIR/archives/*.deb); do
diff --git a/test/compile_test.sh b/test/compile_test.sh
index f5c4a40..9add8bd 100644
--- a/test/compile_test.sh
+++ b/test/compile_test.sh
@@ -21,7 +21,7 @@ testStackChange() {
   #Set the cached STACK value to a non-existent stack, so it is guaranteed to change.
   mkdir -p "$CACHE_DIR/.apt/"
   echo "cedar-10" > "$CACHE_DIR/.apt/STACK"
-  
+
   #Load the Aptfile into the cache, to exclusively test the stack changes
   mkdir -p "$CACHE_DIR/apt/cache"
   cp $BUILD_DIR/Aptfile "$CACHE_DIR/apt/cache"
@@ -56,4 +56,18 @@ testStackCached() {
 
 loadFixture() {
   cp -a $BUILDPACK_HOME/test/fixtures/$1/. ${BUILD_DIR}
-}
\ No newline at end of file
+}
+
+testOutdatedPackageRemoval() {
+  loadFixture "Aptfile"
+
+  fake_file="$CACHE_DIR/apt/cache/archives/wget_fake_old_version.deb"
+  # Create a fake old version of wget to test that it gets removed
+  mkdir -p "$CACHE_DIR/apt/cache/archives"
+  touch "${fake_file}"
+
+  compile
+
+  assertCaptured "Cleaning up outdated and leftover packages"
+  assertTrue 'Outdated packages removed' "[ ! -f ${fake_file} ]"
+}