From 11a2a20dcb5a70921ab05cac5f43729c44b0008c Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 28 Jan 2021 17:19:46 -0500 Subject: [PATCH] oscontainer: use extensions artifact, drop download-extensions Use the tarball produced by `buildextend-extensions` as the source of truth for extensions and drop `download-extensions` now that we no longer need it. --- src/cmd-upload-oscontainer | 19 +++++------ src/download-extensions | 67 -------------------------------------- src/oscontainer.py | 11 ++++++- 3 files changed, 18 insertions(+), 79 deletions(-) delete mode 100755 src/download-extensions diff --git a/src/cmd-upload-oscontainer b/src/cmd-upload-oscontainer index d684fc2924..c80d9d672c 100755 --- a/src/cmd-upload-oscontainer +++ b/src/cmd-upload-oscontainer @@ -40,6 +40,14 @@ metapath = f"{latest_build_path}/meta.json" with open(metapath) as f: meta = json.load(f) +# for backcompat, we auto-build extensions if they're missing +if os.path.exists('src/config/extensions.yaml'): + if 'extensions' not in meta: + cmdlib.run_verbose(['coreos-assembler', 'buildextend-extensions']) + with open(metapath) as f: + meta = json.load(f) + assert 'extensions' in meta + configdir = os.path.abspath('src/config') oscconfigpath = f'{configdir}/oscontainer.yaml' if os.path.exists(oscconfigpath): @@ -49,15 +57,6 @@ if os.path.exists(oscconfigpath): if base is not None: args.from_image = base -extensions_src = 'src/config/extensions.yaml' -extensions_destdir = None -if os.path.exists(extensions_src): - extensions_destdir = 'tmp/extensions' - if os.path.exists(extensions_destdir): - shutil.rmtree(extensions_destdir) - os.mkdir(extensions_destdir) - cmdlib.run_verbose(['/usr/lib/coreos-assembler/download-extensions', extensions_destdir]) - print("Preparing to upload oscontainer for build: {}".format(latest_build)) ostree_commit = meta['ostree-commit'] @@ -106,8 +105,6 @@ os.environ['REGISTRY_AUTH_FILE'] = authfile cosa_argv.extend(['/usr/lib/coreos-assembler/oscontainer.py', '--workdir=./tmp', 'build', f"--from={args.from_image}"]) for d in args.add_directory: cosa_argv.append(f"--add-directory={d}") -if extensions_destdir is not None: - cosa_argv.append(f"--add-directory={extensions_destdir}") cosa_argv.append(f"--display-name={display_name}") subprocess.check_call(cosa_argv + [f'--digestfile={digestfile}', diff --git a/src/download-extensions b/src/download-extensions deleted file mode 100755 index 74d5710806..0000000000 --- a/src/download-extensions +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -# RPMs as operating system extensions, distinct from the base ostree commit/image -# https://github.com/openshift/enhancements/blob/master/enhancements/rhcos/extensions.md - -import os -import sys -import yaml -from cosalib import cmdlib - -destdir = sys.argv[1] -tmpdir = 'tmp' -# yum wants this to be absolute -configdir = os.path.abspath('src/config') -extsrcpath = f'{configdir}/extensions.yaml' -extjson = f'{tmpdir}/extensions.json' -basearch = cmdlib.get_basearch() - -with open(extsrcpath) as f: - extensions = yaml.safe_load(f) - -# The "v2" format here is that there's an extensions/ directory, with subdirectories -# for each extension - except you should ignore "repodata/". -edestdir = f'{destdir}/extensions' -os.mkdir(edestdir) - -# Stuff that's not part of the extension -dependenciesdir = f'{edestdir}/dependencies' -os.mkdir(dependenciesdir) - - -# Downloads packages from specified repos -def yumdownload(destdir, pkgs): - # FIXME eventually use rpm-ostree for this - # shellcheck disable=SC2068 - args = ['yum', f'--setopt=reposdir={configdir}', f'--arch={basearch}', 'download'] - args.extend(pkgs) - cmdlib.run_verbose(args, cwd=destdir) - - -# Reuseable function for setting up an extension -# Assumes it is running in "${destdir}/extensions" -# 1 = extension name -# 2 = package string/glob -# 3 = OPTIONAL: dependencies string/glob -def createext(extname, pkgs): - print(f"Creating extension {extname}") - extdir = f"{edestdir}/{extname}" - os.mkdir(extdir) - primary = pkgs[0] - yumdownload(extdir, [primary]) - - deps = pkgs[1:] - if len(deps) > 0: - print(f"Downloading dependencies for {extname}") - yumdownload(dependenciesdir, deps) - - -for (name, ext) in extensions['extensions'].items(): - pkgs = ext['packages'] - extarches = ext.get('architectures') - if extarches is not None and basearch not in extarches: - print(f"Skipping extension {name} for this architecture") - continue - createext(name, pkgs) - -# Create the yum/dnf repo -cmdlib.run_verbose(['createrepo_c', '--no-database', '.'], cwd=edestdir) diff --git a/src/oscontainer.py b/src/oscontainer.py index 77a160a708..a9b4d43a87 100755 --- a/src/oscontainer.py +++ b/src/oscontainer.py @@ -169,12 +169,15 @@ def oscontainer_build(containers_storage, tmpdir, src, ref, image_name_and_tag, if ostree_version is not None: config += ['-l', 'version=' + ostree_version] + meta = {} + builddir = None if os.path.isfile('builds/builds.json'): with open('builds/builds.json') as fb: builds = json.load(fb)['builds'] latest_build = builds[0]['id'] arch = cmdlib.get_basearch() - metapath = f"builds/{latest_build}/{arch}/meta.json" + builddir = f"builds/{latest_build}/{arch}" + metapath = f"{builddir}/meta.json" with open(metapath) as f: meta = json.load(f) rhcos_commit = meta['coreos-assembler.container-config-git']['commit'] @@ -184,6 +187,12 @@ def oscontainer_build(containers_storage, tmpdir, src, ref, image_name_and_tag, config += ['-l', f"com.coreos.coreos-assembler-commit={cosa_commit}"] config += ['-l', f"com.coreos.redhat-coreos-commit={rhcos_commit}"] + if 'extensions' in meta: + tarball = os.path.abspath(os.path.join(builddir, meta['extensions']['path'])) + dest_dir = os.path.join(mnt, 'extensions') + os.makedirs(dest_dir, exist_ok=True) + run_verbose(["tar", "-xf", tarball], cwd=dest_dir) + if display_name is not None: config += ['-l', 'io.openshift.build.version-display-names=machine-os=' + display_name, '-l', 'io.openshift.build.versions=machine-os=' + ostree_version]