From 00123289fb4700a9a02f5e6bba6a4c4806bf1661 Mon Sep 17 00:00:00 2001 From: Erik Moeller Date: Tue, 9 Apr 2024 10:16:41 -0700 Subject: [PATCH] Remove migrated CircleCI job and cleanup script --- .circleci/config.yml | 11 +--- scripts/clean-old-packages | 122 ------------------------------------- 2 files changed, 2 insertions(+), 131 deletions(-) delete mode 100755 scripts/clean-old-packages diff --git a/.circleci/config.yml b/.circleci/config.yml index 8dfe65b4..f712bd19 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -103,25 +103,18 @@ jobs: - run: name: Commit and push command: | - dnf install -y ca-certificates git git-lfs openssh-clients python3-rpm python3-debian rpmdevtools + dnf install -y ca-certificates git git-lfs openssh-clients git clone git@github.com:freedomofpress/securedrop-yum-test.git cd securedrop-yum-test git lfs install git config user.email "securedrop@freedom.press" git config user.name "sdcibot" mkdir -p workstation/dom0/f32-nightlies - # Copy the new packages over and cleanup the old ones + # Copy the new packages over cp -v /tmp/workspace/*.rpm workstation/dom0/f32-nightlies/ - ~/project/scripts/clean-old-packages workstation/dom0/f32-nightlies 7 git add . # If there are changes, diff-index will fail, so we commit git diff-index --quiet HEAD || git commit -m "Automated SecureDrop workstation build" - # And clean up non-nightly packages too - ~/project/scripts/clean-old-packages workstation/dom0/f32 4 - git add . - git diff-index --quiet HEAD || git commit -m "Cleanup old packages" - - git push origin main workflows: diff --git a/scripts/clean-old-packages b/scripts/clean-old-packages deleted file mode 100755 index 3e306674..00000000 --- a/scripts/clean-old-packages +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env python3 -""" -Clean up old packages, specifying how many to keep - -Example: - ./clean-old-packages securedrop-apt-test/workstation/buster-nightlies 7 - -""" -import argparse -import functools -import subprocess -from collections import defaultdict -from pathlib import Path -from typing import Tuple - -import rpm -from debian import debfile - - -def sort_deb_versions(one: Tuple[str, Path], two: Tuple[str, Path]): - """sort two Debian package versions""" - status = subprocess.run(['dpkg', '--compare-versions', one[0], 'lt', two[0]]) - if status.returncode == 1: - # false, one is bigger - return 1 - else: - # true, two is bigger - return -1 - - -def sort_rpm_versions(one: Tuple[str, Path], two: Tuple[str, Path]): - """sort two RPM package versions""" - status = subprocess.run(['rpmdev-vercmp', one[0], two[0]], stdout=subprocess.DEVNULL) - if status.returncode == 11: - # false, one is bigger - return 1 - else: # status.returncode == 12 - # true, two is bigger - return -1 - - -def fix_name(name: str) -> str: - """ - Linux packages embed the version in the name, so we'd never have multiple - packages meet the deletion threshold. Silly string manipulation to drop - the version. - E.g. "linux-image-5.15.26-grsec-securedrop" -> "linux-image-securedrop" - """ - if name.endswith(('-securedrop', '-workstation')): - suffix = name.split('-')[-1] - else: - return name - if name.startswith('linux-image-'): - return f'linux-image-{suffix}' - elif name.startswith('linux-headers-'): - return f'linux-headers-{suffix}' - return name - - -def rpm_info(path: Path) -> Tuple[str, str]: - """ - learned this incantation from - and help(headers) - """ - ts = rpm.ts() - with path.open() as f: - headers = ts.hdrFromFdno(f) - return headers[rpm.RPMTAG_NAME], headers[rpm.RPMTAG_VERSION] + '-' + headers[rpm.RPMTAG_RELEASE] - - -def cleanup(data, to_keep: int, sorter): - for name, versions in sorted(data.items()): - if len(versions) <= to_keep: - # Nothing to delete - continue - print(f'### {name}') - items = sorted(versions.items(), key=functools.cmp_to_key(sorter), reverse=True) - keeps = items[:to_keep] - print('Keeping:') - for _, keep in keeps: - print(f'* {keep.name}') - delete = items[to_keep:] - print('Deleting:') - for _, path in delete: - print(f'* {path.name}') - path.unlink() - - -def main(): - parser = argparse.ArgumentParser( - description="Cleans up old packages" - ) - parser.add_argument( - "directory", - type=Path, - help="Directory to clean up", - ) - parser.add_argument( - "keep", - type=int, - help="Number of packages to keep" - ) - args = parser.parse_args() - if not args.directory.is_dir(): - raise RuntimeError(f"Directory, {args.directory}, doesn't exist") - print(f'Only keeping the latest {args.keep} packages') - debs = defaultdict(dict) - rpms = defaultdict(dict) - for deb in args.directory.glob('*.deb'): - control = debfile.DebFile(deb).control.debcontrol() - name = fix_name(control['Package']) - debs[name][control['Version']] = deb - for rpm in args.directory.glob('*.rpm'): - name, version = rpm_info(rpm) - rpms[name][version] = rpm - - cleanup(debs, args.keep, sort_deb_versions) - cleanup(rpms, args.keep, sort_rpm_versions) - - -if __name__ == '__main__': - main()