From f306c139e21692569ad09ba072c1cae71b6b7810 Mon Sep 17 00:00:00 2001 From: Caleb Date: Fri, 15 Nov 2024 15:27:35 -0500 Subject: [PATCH] Revert "NAS-132446 / 25.04 / Initial add of option to use internal APT repo for builds (#754)" This reverts commit 5f95982024caf849323997cd47a257726a272838. --- conf/build.manifest | 14 ++++++-------- scale_build/bootstrap/bootstrapdir.py | 13 +++++-------- scale_build/bootstrap/hash.py | 7 +++---- scale_build/config.py | 2 -- scale_build/image/iso.py | 6 ++---- scale_build/image/update.py | 8 +++----- scale_build/utils/manifest.py | 20 +------------------- 7 files changed, 20 insertions(+), 50 deletions(-) diff --git a/conf/build.manifest b/conf/build.manifest index b5eef565..dc380ba4 100644 --- a/conf/build.manifest +++ b/conf/build.manifest @@ -8,25 +8,23 @@ identity_file_path_default: "~/.ssh/id_rsa" # into the build chroots, or the final system images. ############################################################################ apt-repos: - base-url: https://apt.sys.truenas.net - base-url-internal: http://apt-mirror.tn.ixsystems.net - url: /fangtooth/nightlies/debian/ + url: https://apt.sys.truenas.net/fangtooth/nightlies/debian/ distribution: bookworm components: main additional: - - url: /fangtooth/nightlies/debian-security/ + - url: https://apt.sys.truenas.net/fangtooth/nightlies/debian-security/ distribution: bookworm-security component: main - - url: /fangtooth/nightlies/debian-backports/ + - url: https://apt.sys.truenas.net/fangtooth/nightlies/debian-backports/ distribution: bookworm-backports component: "main contrib non-free non-free-firmware" - - url: /fangtooth/nightlies/debian-debug/ + - url: https://apt.sys.truenas.net/fangtooth/nightlies/debian-debug/ distribution: bookworm-debug component: main - - url: /fangtooth/nightlies/yarn/ + - url: https://apt.sys.truenas.net/fangtooth/nightlies/yarn/ distribution: stable component: main - - url: /fangtooth/nightlies/docker/ + - url: https://apt.sys.truenas.net/fangtooth/nightlies/docker/ distribution: bookworm component: stable key: keys/docker.gpg diff --git a/scale_build/bootstrap/bootstrapdir.py b/scale_build/bootstrap/bootstrapdir.py index e83cdb04..4e771196 100644 --- a/scale_build/bootstrap/bootstrapdir.py +++ b/scale_build/bootstrap/bootstrapdir.py @@ -3,7 +3,7 @@ import shutil from scale_build.clean import clean_packages -from scale_build.utils.manifest import get_apt_base_url, get_manifest +from scale_build.utils.manifest import get_manifest from scale_build.utils.paths import BUILDER_DIR, CHROOT_BASEDIR, REFERENCE_FILES, REFERENCE_FILES_DIR from scale_build.utils.run import run @@ -30,12 +30,11 @@ def setup(self): def debootstrap_debian(self): manifest = get_manifest() - apt_base_url = get_apt_base_url() run( ['debootstrap'] + self.deopts + [ '--keyring', '/etc/apt/trusted.gpg.d/debian-archive-truenas-automatic.gpg', manifest['debian_release'], - self.chroot_basedir, apt_base_url + manifest['apt-repos']['url'] + self.chroot_basedir, manifest['apt-repos']['url'] ] ) @@ -47,7 +46,6 @@ def setup_impl(self): self.add_trusted_apt_key() apt_repos = get_manifest()['apt-repos'] - apt_base_url = get_apt_base_url() self.debootstrap_debian() self.setup_mounts() @@ -63,7 +61,7 @@ def setup_impl(self): run(['chroot', self.chroot_basedir, 'apt', 'install', '-y', 'gnupg']) # Save the correct repo in sources.list - apt_sources = [f'deb {apt_base_url}{apt_repos["url"]} {apt_repos["distribution"]} {apt_repos["components"]}'] + apt_sources = [f'deb {apt_repos["url"]} {apt_repos["distribution"]} {apt_repos["components"]}'] # Add additional repos for repo in apt_repos['additional']: @@ -73,7 +71,7 @@ def setup_impl(self): run(['chroot', self.chroot_basedir, 'apt-key', 'add', '/apt.key']) os.unlink(os.path.join(self.chroot_basedir, 'apt.key')) - apt_sources.append(f'deb {apt_base_url}{repo["url"]} {repo["distribution"]} {repo["component"]}') + apt_sources.append(f'deb {repo["url"]} {repo["distribution"]} {repo["component"]}') with open(apt_sources_path, 'w') as f: f.write('\n'.join(apt_sources)) @@ -149,12 +147,11 @@ def cache_filename(self): def debootstrap_debian(self): manifest = get_manifest() - apt_base_url = get_apt_base_url() run( ['debootstrap'] + self.deopts + [ '--foreign', '--keyring', '/etc/apt/trusted.gpg.d/debian-archive-truenas-automatic.gpg', manifest['debian_release'], - self.chroot_basedir, apt_base_url + manifest['apt-repos']['url'] + self.chroot_basedir, manifest['apt-repos']['url'] ] ) for reference_file in REFERENCE_FILES: diff --git a/scale_build/bootstrap/hash.py b/scale_build/bootstrap/hash.py index d347b80c..7d521da1 100644 --- a/scale_build/bootstrap/hash.py +++ b/scale_build/bootstrap/hash.py @@ -6,7 +6,7 @@ import requests import urllib.parse -from scale_build.utils.manifest import get_apt_base_url, get_manifest +from scale_build.utils.manifest import get_manifest from scale_build.utils.run import run from scale_build.utils.paths import CACHE_DIR, HASH_DIR @@ -27,12 +27,11 @@ def get_repo_hash(repo_url, distribution): def get_all_repo_hash(): apt_repos = get_manifest()['apt-repos'] - apt_base_url = get_apt_base_url() # Start by validating the main APT repo - all_repo_hash = get_repo_hash(apt_base_url + apt_repos['url'], apt_repos['distribution']) + all_repo_hash = get_repo_hash(apt_repos['url'], apt_repos['distribution']) for repo_config in apt_repos['additional']: - all_repo_hash += get_repo_hash(apt_base_url + repo_config['url'], repo_config['distribution']) + all_repo_hash += get_repo_hash(repo_config['url'], repo_config['distribution']) all_repo_hash += hashlib.sha256(get_apt_preferences().encode()).hexdigest() diff --git a/scale_build/config.py b/scale_build/config.py index 1332ce06..fb06ac45 100644 --- a/scale_build/config.py +++ b/scale_build/config.py @@ -49,8 +49,6 @@ def get_normalized_value(value, _type, default_value=None): VERSION = get_env_variable('TRUENAS_VERSION', str, f'{_VERS}-{BUILD_TIME_OBJ.strftime("%Y%m%d-%H%M%S")}') TRUENAS_VENDOR = get_env_variable('TRUENAS_VENDOR', str) PRESERVE_ISO = get_env_variable('PRESERVE_ISO', bool, False) -APT_INTERNAL_BUILD = get_env_variable('APT_INTERNAL_BUILD', bool, False) -APT_BASE_CUSTOM = get_env_variable('APT_BASE_CUSTOM', str) # We will get branch overrides and identity file path overrides from here diff --git a/scale_build/image/iso.py b/scale_build/image/iso.py index 9f47b5d6..91e12ff4 100644 --- a/scale_build/image/iso.py +++ b/scale_build/image/iso.py @@ -10,7 +10,7 @@ import requests from scale_build.exceptions import CallError -from scale_build.utils.manifest import get_apt_base_url, get_manifest +from scale_build.utils.manifest import get_manifest from scale_build.utils.run import run from scale_build.utils.paths import CD_DIR, CD_FILES_DIR, CHROOT_BASEDIR, CONF_GRUB, PKG_DIR, RELEASE_DIR, TMP_DIR from scale_build.config import TRUENAS_VENDOR @@ -134,10 +134,8 @@ def make_iso_file(): with tempfile.NamedTemporaryFile(dir=RELEASE_DIR) as efi_img: with tempfile.NamedTemporaryFile(suffix='.tar.gz') as f: apt_repos = get_manifest()['apt-repos'] - apt_base_url = get_apt_base_url() r = requests.get( - f'{apt_base_url}{apt_repos["url"]}dists/{apt_repos["distribution"]}' - '/main/installer-amd64/current/images/cdrom/' + f'{apt_repos["url"]}dists/{apt_repos["distribution"]}/main/installer-amd64/current/images/cdrom/' 'debian-cd_info.tar.gz', timeout=10, stream=True, diff --git a/scale_build/image/update.py b/scale_build/image/update.py index 136cc42e..1e8364e6 100644 --- a/scale_build/image/update.py +++ b/scale_build/image/update.py @@ -8,7 +8,7 @@ import tempfile from scale_build.config import SIGNING_KEY, SIGNING_PASSWORD -from scale_build.utils.manifest import get_apt_base_url, get_manifest +from scale_build.utils.manifest import get_manifest from scale_build.utils.run import run from scale_build.utils.paths import CHROOT_BASEDIR, RELEASE_DIR, UPDATE_DIR @@ -111,11 +111,9 @@ def install_rootfs_packages_impl(): def get_apt_sources(): apt_repos = get_manifest()['apt-repos'] - apt_base_url = get_apt_base_url() - apt_sources = [f'deb {apt_base_url}{apt_repos["url"]} {apt_repos["distribution"]} {apt_repos["components"]}'] + apt_sources = [f'deb {apt_repos["url"]} {apt_repos["distribution"]} {apt_repos["components"]}'] for repo in apt_repos['additional']: - apt_sources.append(f'deb {apt_base_url}{repo["url"]} {repo["distribution"]} {repo["component"]}') - + apt_sources.append(f'deb {repo["url"]} {repo["distribution"]} {repo["component"]}') return apt_sources diff --git a/scale_build/utils/manifest.py b/scale_build/utils/manifest.py index f54546bf..739aadb2 100644 --- a/scale_build/utils/manifest.py +++ b/scale_build/utils/manifest.py @@ -5,7 +5,7 @@ from urllib.parse import urlparse -from scale_build.config import APT_BASE_CUSTOM, APT_INTERNAL_BUILD, SKIP_SOURCE_REPO_VALIDATION, TRAIN +from scale_build.config import SKIP_SOURCE_REPO_VALIDATION, TRAIN from scale_build.exceptions import CallError, MissingManifest from scale_build.utils.paths import MANIFEST @@ -77,8 +77,6 @@ 'code_name': {'type': 'string'}, 'debian_release': {'type': 'string'}, 'identity_file_path_default': {'type': 'string'}, - 'base-url': {'type': 'string'}, - 'base-url-internal': {'type': 'string'}, 'apt-repos': { 'type': 'object', 'properties': { @@ -254,19 +252,3 @@ def validate_manifest(): 'accepts packages from github.com/truenas organization (To skip this for dev ' 'purposes, please set "SKIP_SOURCE_REPO_VALIDATION" in your environment).' ) - - -def get_apt_base_url(): - apt_repos = get_manifest()['apt-repos'] - apt_base_url = "" - - # If the user provided their own location - if APT_BASE_CUSTOM: - return APT_BASE_CUSTOM - - # Return either the CDN or the internal build url - if APT_INTERNAL_BUILD: - apt_base_url = f'{apt_repos["base-url-internal"]}' - else: - apt_base_url = f'{apt_repos["base-url"]}' - return apt_base_url