From 9b55dfe187a7a25b6f67ae5f924a21fcae05ffc9 Mon Sep 17 00:00:00 2001
From: theofficialgman <28281419+theofficialgman@users.noreply.github.com>
Date: Sat, 24 Feb 2024 19:59:10 -0500
Subject: [PATCH 1/5] api: add `debug` function to `echo` input when
 `pi_apps_debug` == true

---
 api | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/api b/api
index 70128575a4..9b3b671be9 100755
--- a/api
+++ b/api
@@ -29,6 +29,10 @@ status_green() { #announce the success of a major action
   echo -e "\e[92m$1\e[0m" 1>&2
 }
 
+debug() { #an echo command that only runs when debug mode is on
+  [ "$pi_apps_debug" = true ] && echo "$1"
+}
+
 generate_logo() { #display colorized Pi-Apps logo in terminal
   #ANSI color codes: https://misc.flogisoft.com/bash/tip_colors_and_formatting
   #Search for unicode characters: https://unicode-search.net - search for "block" and "quadrant"

From 01cf9a4de8a54aa6c45925377247937bab39fa5a Mon Sep 17 00:00:00 2001
From: theofficialgman <28281419+theofficialgman@users.noreply.github.com>
Date: Sat, 24 Feb 2024 17:47:15 -0500
Subject: [PATCH 2/5] api: add `anything_installed_from_uri_suite_component` to
 fully defined apt repository for installed packages

instead of checking the apt-cache policy on each package individually, check all installed packages that exist in a repo at once. this results in a multiple factor of magnitude speedup on large repositories (eg: debian/ubuntu main repositories) from minutes to seconds
---
 api | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/api b/api
index 9b3b671be9..194a374218 100755
--- a/api
+++ b/api
@@ -224,6 +224,54 @@ anything_installed_from_repo() { #Given an apt repository URL, determine if any
   fi
 }
 
+anything_installed_from_uri_suite_component() { #Given an apt repository uri, suite, and component, determine if any packages from it are currently installed
+  [ -z "$1" ] && error "anything_installed_from_uri_suite_component: A repository uri must be specified."
+  [ -z "$2" ] && error "anything_installed_from_uri_suite_component: A repository suite must be specified."
+  
+  #component is an optional specification
+  if [ -z "$3" ]; then
+    local filepath="/var/lib/apt/lists/$(echo "$1" | sed 's+.*://++g' | sed "s,/$,," | tr '/' '_')_$(echo "$2" | sed "s,/$,," | tr '/' '_')_"
+  else
+    local filepath="/var/lib/apt/lists/$(echo "$1" | sed 's+.*://++g' | sed "s,/$,," | tr '/' '_')_dists_$(echo "$2" | sed "s,/$,," | tr '/' '_')_$(echo "$3" | sed "s,/$,," | tr '/' '_')_"
+  fi
+  debug $filepath
+  
+  #find all relevant package-lists
+  local repofiles="$(ls $filepath*Packages)"
+  debug "$repofiles"
+  
+  #for every repo-file, check if any of them have an installed file
+  local found=0
+  local IFS=$'\n'
+  local repofile
+  for repofile in $repofiles ;do
+    #search the repo-file for installed packages
+
+    local packages_in_repo="$(grep '^Package' "$repofile" | awk '{print $2}' | sort)"
+    local installed_packages="$(apt list --installed 2>/dev/null | tail -n +2 | awk -F/ '{print $1}')"
+    local apt_cache_policy_output="$(echo "$packages_in_repo" | list_intersect "$installed_packages" | tr '\n' ' ' | xargs -r apt-cache policy)"
+    
+    if [ -z "$3" ]; then
+      if echo "$apt_cache_policy_output" | grep -B1 "$(echo "$1" | sed 's+.*://++g' | sed "s,/$,,") $2" | awk '{print $1}' | grep -Fq '***' ;then
+        found=1
+        break
+      fi
+    else
+      if echo "$apt_cache_policy_output" | grep -B1 "$(echo "$1" | sed 's+.*://++g' | sed "s,/$,,") $2/$3" | awk '{print $1}' | grep -Fq '***' ;then
+        found=1
+        break
+      fi
+    fi
+  done
+  
+  #return an exit code
+  if [ $found == 1 ];then
+    return 0
+  else
+    return 1
+  fi
+}
+
 remove_repofile_if_unused() { #Given a sources.list.d file, delete it if nothing from that repository is currently installed. Deletion skipped if $2 is 'test'
   local file="$1"
   local testmode="$2"

From 7a6934b33885fda25407ad78f5e88e973ace69e4 Mon Sep 17 00:00:00 2001
From: theofficialgman <28281419+theofficialgman@users.noreply.github.com>
Date: Sat, 24 Feb 2024 17:16:22 -0500
Subject: [PATCH 3/5] api: refactor `remove_repofile_if_unused` to use
 `anything_installed_from_uri_suite_component`

handles .sources files and .list files with multiple components and multiple suites (as allowed) and filters out disabled sources.

in deb822, it is convention to follow a field separator colon by a space and then the package name but the specification allows for no space or any number of spaces or tab characters. trim all tab and space characters to a single space character for components and suites that can have multiple strings of output
---
 api | 96 +++++++++++++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 78 insertions(+), 18 deletions(-)

diff --git a/api b/api
index 194a374218..984bbf42c9 100755
--- a/api
+++ b/api
@@ -279,34 +279,94 @@ remove_repofile_if_unused() { #Given a sources.list.d file, delete it if nothing
   [ -z "$file" ] && error "remove_repo_if_unused: no sources.list.d file specified!"
   #return now if the list file does not exist
   [ -f "$file" ] || return 0
+
+  #set default to not in use
+  local in_use=0
   
   if [ "${file##*.}" == "list" ]; then
-    #determine what repo-urls are in the file: include dist information. Example value: deb.debian.org_debian_dists_bookworm
-    local urls="$(cat "$file" | grep -v '^#' | tr ' ' '\n' | grep '://' -A1  | tr '\n' ' ' | sed 's/ -- /\n/g ; s/ $/\n/g ; s/ /_dists_/g')"
+    # determine what uri, suite, and components are in a file
+    local lines="$(cat "$file" | grep "^deb " | sed 's/^deb // ; s/\[.*\]//')"
+    local IFS=$'\n'
+    for line in $lines ;do
+      local uri="$(echo "$line" | awk '{print $1}')"
+      local suite="$(echo "$line" | awk '{print $2}')"
+      local components="$(echo "$line" | awk '{$1=$2=""; print $0}')"
+      local IFS=' '
+      if [ -z "$components" ]; then
+        debug "$uri $suite"
+        if anything_installed_from_uri_suite_component "$uri" "$suite";then
+          in_use=1
+          break 1
+        fi
+      else
+        for component in $components ;do
+          debug "$uri $suite $component"
+          if anything_installed_from_uri_suite_component "$uri" "$suite" "$component";then
+            in_use=1
+            break 2
+          fi
+        done
+      fi
+      local IFS=$'\n'
+    done
   elif [ "${file##*.}" == "sources" ]; then
-    #determine what repo-urls are in the file
-    local urls="$(cat "$file" | grep -v '^#' | grep '^URIs: ' | sed 's/URIs: //g')"
+    #find empty lines (empty line, line with all spaces, or line with all tabs) in the file that separate stanzas. empty lines are not allowed between fields within a stanza
+    #https://manpages.ubuntu.com/manpages/jammy/en/man5/deb822.5.html
+    local empty_lines="$(grep -P -n '^$|^ +$|^\t+$' "$file" | awk '{print $1}' | sed 's/:*//g')"
+    #get number of lines in file
+    local num_lines=$(wc -l "$file" | awk '{print $1}')
+
+    #always add last line to empty lines
+    if [ -z "$empty_lines" ]; then
+      empty_lines="$num_lines"
+    else
+      empty_lines+=$'\n'"$num_lines"
+    fi
+    debug "$empty_lines"
+
+    #parse each stanza, starting at line 1
+    local IFS=$'\n'
+    local line_start=1
+    for line_end in $empty_lines ;do
+      # if Enabled: no, continue to next loop iteration
+      sed -n "$line_start","$line_end"p  "$file" | grep -q '^Enabled: no' && continue
+      # determine what uri, suite, and components are in a file
+      #each stanza can only have one matching section. if there are multiple matches the last one is used
+      #case should be ignored for fields
+      local uris="$(sed -n "$line_start","$line_end"p  "$file" | grep -v '^#' | grep -i '^URIs:' | sed 's/URIs://Ig' | awk '{$1=$1};1' | tail -1)"
+      local suites="$(sed -n "$line_start","$line_end"p "$file" | grep -v '^#' | grep -i '^Suites:' | sed 's/Suites://Ig' | awk '{$1=$1};1' | tail -1)"
+      local components="$(sed -n "$line_start","$line_end"p "$file" | grep -v '^#' | grep -i '^Components:' | sed 's/Components://Ig' | awk '{$1=$1};1' | tail -1)"
+      local IFS=' '
+      for uri in $uris ;do
+        for suite in $suites ;do
+          if [ -z "$components" ]; then
+            debug "$uri $suite"
+            if anything_installed_from_uri_suite_component "$uri" "$suite";then
+              in_use=1
+              break 3
+            fi
+          else
+            for component in $components ;do
+              debug "$uri $suite $component"
+              if anything_installed_from_uri_suite_component "$uri" "$suite" "$component";then
+                in_use=1
+                break 4
+              fi
+            done
+          fi
+        done
+      done
+      local IFS=$'\n'
+      line_start="$line_end"
+    done
   else
     error "$file was not of apt list or sources type"
   fi
   
-  #there could be multiple urls in one file. Check each url and set the in_use variable to 1 if any packages are found
-  local IFS=$'\n'
-  local in_use=0
-  local url
-  for url in $urls ;do
-    if anything_installed_from_repo "$url" >/dev/null;then
-      in_use=1
-      break
-    fi
-  done
-  
   if [ "$testmode" == test ] && [ "$in_use" == 0 ];then
     echo "The given repository is not in use and can be deleted:"$'\n'"$file" 1>&2
   elif [ "$testmode" == test ];then
-    #explain what package is installed from this repo
-    echo "At least this package is preventing the repo from being removed (there may be more)"
-    anything_installed_from_repo "$url" #use value of $url from earlier
+    echo "At least one package is preventing the repo from being removed"
   elif [ "$in_use" == 0 ];then
     status "Removing the $(basename "$file" | sed 's/.list$//g' | sed 's/.sources$//g') repo as it is not being used"
     sudo rm -f "$file"

From 782b893d41a4d64bd3c6ffe9c0564b51b826c9d4 Mon Sep 17 00:00:00 2001
From: theofficialgman <28281419+theofficialgman@users.noreply.github.com>
Date: Sat, 24 Feb 2024 20:10:29 -0500
Subject: [PATCH 4/5] api: `anything_installed_from_uri_suite_component`
 simplify function

improve readibility, optimize list of installed packages

Co-Authored-By: Botspot <54716352+Botspot@users.noreply.github.com>
---
 api | 47 +++++++++++++++++++----------------------------
 1 file changed, 19 insertions(+), 28 deletions(-)

diff --git a/api b/api
index 984bbf42c9..75c43b2725 100755
--- a/api
+++ b/api
@@ -225,51 +225,42 @@ anything_installed_from_repo() { #Given an apt repository URL, determine if any
 }
 
 anything_installed_from_uri_suite_component() { #Given an apt repository uri, suite, and component, determine if any packages from it are currently installed
-  [ -z "$1" ] && error "anything_installed_from_uri_suite_component: A repository uri must be specified."
-  [ -z "$2" ] && error "anything_installed_from_uri_suite_component: A repository suite must be specified."
-  
-  #component is an optional specification
-  if [ -z "$3" ]; then
-    local filepath="/var/lib/apt/lists/$(echo "$1" | sed 's+.*://++g' | sed "s,/$,," | tr '/' '_')_$(echo "$2" | sed "s,/$,," | tr '/' '_')_"
+  local uri="$1"
+  local suite="$2"
+  component="$3" #can be left blank
+  [ -z "$uri" ] && error "anything_installed_from_uri_suite_component: A repository uri must be specified."
+  [ -z "$suite" ] && error "anything_installed_from_uri_suite_component: A repository suite must be specified."
+  
+  #find part of path to apt list file(s) to search for
+  if [ -z "$component" ]; then
+    local filepath="/var/lib/apt/lists/$(echo "$1" | sed 's+.*://++g' | sed "s,/$,," | tr '/' '_')_$(echo "$suite" | sed "s,/$,," | tr '/' '_')_"
   else
-    local filepath="/var/lib/apt/lists/$(echo "$1" | sed 's+.*://++g' | sed "s,/$,," | tr '/' '_')_dists_$(echo "$2" | sed "s,/$,," | tr '/' '_')_$(echo "$3" | sed "s,/$,," | tr '/' '_')_"
+    local filepath="/var/lib/apt/lists/$(echo "$1" | sed 's+.*://++g' | sed "s,/$,," | tr '/' '_')_dists_$(echo "$suite" | sed "s,/$,," | tr '/' '_')_$(echo "$component" | sed "s,/$,," | tr '/' '_')_"
   fi
   debug $filepath
   
   #find all relevant package-lists
-  local repofiles="$(ls $filepath*Packages)"
+  local repofiles="$(ls ${filepath}*_Packages)"
   debug "$repofiles"
   
   #for every repo-file, check if any of them have an installed file
-  local found=0
   local IFS=$'\n'
   local repofile
+  local installed_packages="$(grep -xF 'Status: install ok installed' /var/lib/dpkg/status -B 2 | grep '^Package: ' | sed 's/^Package: //g' | sort)"
   for repofile in $repofiles ;do
     #search the repo-file for installed packages
-
-    local packages_in_repo="$(grep '^Package' "$repofile" | awk '{print $2}' | sort)"
-    local installed_packages="$(apt list --installed 2>/dev/null | tail -n +2 | awk -F/ '{print $1}')"
+    
+    local packages_in_repo="$(grep '^Package: ' "$repofile" | awk '{print $2}' | sort)"
     local apt_cache_policy_output="$(echo "$packages_in_repo" | list_intersect "$installed_packages" | tr '\n' ' ' | xargs -r apt-cache policy)"
     
-    if [ -z "$3" ]; then
-      if echo "$apt_cache_policy_output" | grep -B1 "$(echo "$1" | sed 's+.*://++g' | sed "s,/$,,") $2" | awk '{print $1}' | grep -Fq '***' ;then
-        found=1
-        break
-      fi
+    #check if any installed packages also found on this repo are actually installed from this repo
+    if [ -z "$component" ]; then
+      echo "$apt_cache_policy_output" | grep -B1 "$(echo "$uri" | sed 's+.*://++g' | sed "s,/$,,") $suite" | awk '{print $1}' | grep -Fq '***' && return 0
     else
-      if echo "$apt_cache_policy_output" | grep -B1 "$(echo "$1" | sed 's+.*://++g' | sed "s,/$,,") $2/$3" | awk '{print $1}' | grep -Fq '***' ;then
-        found=1
-        break
-      fi
+      echo "$apt_cache_policy_output" | grep -B1 "$(echo "$uri" | sed 's+.*://++g' | sed "s,/$,,") $suite/$component" | awk '{print $1}' | grep -Fq '***' && return 0
     fi
   done
-  
-  #return an exit code
-  if [ $found == 1 ];then
-    return 0
-  else
-    return 1
-  fi
+  return 1
 }
 
 remove_repofile_if_unused() { #Given a sources.list.d file, delete it if nothing from that repository is currently installed. Deletion skipped if $2 is 'test'

From 64c8fdd5fa43e8b017dafa043527715fa877d8c7 Mon Sep 17 00:00:00 2001
From: theofficialgman <28281419+theofficialgman@users.noreply.github.com>
Date: Sat, 24 Feb 2024 20:15:58 -0500
Subject: [PATCH 5/5] api: remove `anything_installed_from_repo` function

also replace function in brave install script with `anything_installed_from_uri_suite_component`
---
 api                   | 41 -----------------------------------------
 apps/Brave/install-64 |  2 +-
 2 files changed, 1 insertion(+), 42 deletions(-)

diff --git a/api b/api
index 75c43b2725..df03b069cd 100755
--- a/api
+++ b/api
@@ -183,47 +183,6 @@ package_is_new_enough() { #check if the $1 package has an available version grea
   fi
 }
 
-anything_installed_from_repo() { #Given an apt repository URL, determine if any packages from it are currently installed
-  [ -z "$1" ] && error "anything_installed_from_repo: A repository URL must be specified."
-  
-  #user input repo-url. Remove 'https://', and translate '/' to '_' to conform to apt file-naming standard, with trailing _ to ensure full matches
-  local url="$(echo "$1" | sed 's+.*://++g' | tr '/' '_')_"
-  
-  #find all package-lists pertaining to the url
-  local repofiles="$(ls /var/lib/apt/lists/*_Packages | grep -F "$url")"
-  
-  #for every repo-file, check if any of them have an installed file
-  local found=0
-  local IFS=$'\n'
-  local repofile
-  for repofile in $repofiles ;do
-    #search the repo-file for installed packages
-    
-    grep '^Package' "$repofile" | awk '{print $2}' | while read -r package ;do
-      if package_installed "$package" ;then
-        #this package is installed; check if the version available on this repo is the current version (prevents false positives from backports repos)
-        if [ "$(sed -n "/^Package: ${package}$/,/Version:/p" "$repofile" | grep '^Version: ' | awk '{print $2}')" == "$(package_installed_version "$package")" ];then
-          echo "Package installed: $package"
-          exit 1
-        fi
-      fi
-    done #if exit code is 1, search was successful. If exit code is 0, no packages from the repo were installed.
-    
-    found=$?
-    
-    if [ $found == 1 ];then
-      break
-    fi
-  done
-  
-  #return an exit code
-  if [ $found == 1 ];then
-    return 0
-  else
-    return 1
-  fi
-}
-
 anything_installed_from_uri_suite_component() { #Given an apt repository uri, suite, and component, determine if any packages from it are currently installed
   local uri="$1"
   local suite="$2"
diff --git a/apps/Brave/install-64 b/apps/Brave/install-64
index 2d0d557f75..deb6194fd1 100755
--- a/apps/Brave/install-64
+++ b/apps/Brave/install-64
@@ -7,7 +7,7 @@ sudo curl -fsSLo /usr/share/keyrings/brave-browser-archive-keyring.gpg https://b
 
 (install_packages brave-browser)
 if [ $? != 0 ]; then
-  if ! anything_installed_from_repo "https://brave-browser-apt-release.s3.brave.com/" ; then
+  if ! anything_installed_from_uri_suite_component "https://brave-browser-apt-release.s3.brave.com/" stable main ; then
     # nothing installed from repo, this check is to prevent removing repos which other pi-apps scripts or the user have used successfully
     # safe to remove
     sudo rm -f /etc/apt/sources.list.d/brave-browser-release.list /usr/share/keyrings/brave-browser-archive-keyring.gpg