From 7d7fb145196e0826764eda4e4354503aa0caf952 Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Wed, 13 Nov 2024 14:30:13 +0100 Subject: [PATCH 1/5] phase1: Implement custom step ShellCommandAndSetProperty Implement custom step ShellCommandAndSetProperty, as an extension of ShellCommand with the addition of setting a bool property that is set True or False if the shell command succeeded or not. Signed-off-by: Christian Marangi --- phase1/master.cfg | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/phase1/master.cfg b/phase1/master.cfg index f869dfc..82ebc90 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -17,6 +17,7 @@ from buildbot import locks from buildbot.data import resultspec from buildbot.changes.gitpoller import GitPoller from buildbot.config import BuilderConfig +from buildbot.process import buildstep from buildbot.plugins import reporters from buildbot.plugins import schedulers from buildbot.plugins import steps @@ -758,6 +759,37 @@ c["builders"].append( ) +# CUSTOM CLASS + +# Extension of ShellCommand and sets in property: +# - True: the command succeded +# - False: the command failed +class ShellCommandAndSetProperty(buildstep.ShellMixin, buildstep.BuildStep): + name = "shellandsetproperty" + renderables = ['property'] + + def __init__( + self, + property=None, + **kwargs, + ): + kwargs = self.setupShellMixin(kwargs) + + self.property = property + + super().__init__(**kwargs) + + @defer.inlineCallbacks + def run(self): + cmd = yield self.makeRemoteShellCommand() + + yield self.runCommand(cmd) + + self.setProperty(self.property, not cmd.didFail(), "ShellCommandAndSetProperty Step") + + return cmd.results() + + # NB the phase1 build factory assumes workers are single-build only def prepareFactory(target): (target, subtarget) = target.split("/") From 437dd6fb3014c03416845c6a83aac880481b235c Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 15 Nov 2024 15:27:55 +0100 Subject: [PATCH 2/5] phase1: remove kmods in target packages if archive is enabled OPKG gets confused if kmod packages are present in both, target packages as well as kernel version specific folder. Remove them from target packages to make opkg pick the kmods from kmod archive folder only. This also affects APK that picks packages from the first entry in the repository file and doesn't support checking the same package from multiple entry. Signed-off-by: Daniel Golle [ add --remove-source-files and improve implementation ] Signed-off-by: Christian Marangi --- phase1/master.cfg | 57 ++++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/phase1/master.cfg b/phase1/master.cfg index 82ebc90..380338b 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -1250,23 +1250,6 @@ def prepareFactory(target): ) ) - factory.addStep( - ShellCommand( - name="pkgindex", - description="Indexing packages", - descriptionDone="Packages indexed", - command=[ - "make", - Interpolate("-j%(prop:nproc:-1)s"), - "package/index", - "V=s", - "CONFIG_SIGNED_PACKAGES=", - ], - env=MakeEnv(), - haltOnFailure=True, - ) - ) - factory.addStep( ShellCommand( name="images", @@ -1303,17 +1286,6 @@ def prepareFactory(target): ) ) - factory.addStep( - ShellCommand( - name="checksums", - description="Calculating checksums", - descriptionDone="Checksums calculated", - command=["make", "-j1", "checksum", "V=s"], - env=MakeEnv(), - haltOnFailure=True, - ) - ) - factory.addStep( ShellCommand( name="kmoddir", @@ -1339,6 +1311,7 @@ def prepareFactory(target): descriptionDone="Kmod archive prepared", command=[ "rsync", + "--remove-source-files", "--include=/kmod-*.ipk", "--include=/kmod-*.apk", "--exclude=*", @@ -1359,6 +1332,23 @@ def prepareFactory(target): ) ) + factory.addStep( + ShellCommand( + name="pkgindex", + description="Indexing packages", + descriptionDone="Packages indexed", + command=[ + "make", + Interpolate("-j%(prop:nproc:-1)s"), + "package/index", + "V=s", + "CONFIG_SIGNED_PACKAGES=", + ], + env=MakeEnv(), + haltOnFailure=True, + ) + ) + factory.addStep( ShellCommand( name="kmodindex", @@ -1382,6 +1372,17 @@ def prepareFactory(target): ) ) + factory.addStep( + ShellCommand( + name="checksums", + description="Calculating checksums", + descriptionDone="Checksums calculated", + command=["make", "-j1", "checksum", "V=s"], + env=MakeEnv(), + haltOnFailure=True, + ) + ) + # sign factory.addStep( MasterShellCommand( From ab85b443d82332f64895a27cbfb3c4a46772f5b7 Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Fri, 15 Nov 2024 15:30:09 +0100 Subject: [PATCH 3/5] pahse1: merge sha256sums with remote kmods entry if present Add additional logic to merge the sha256sums with the remote kmods entry (in remote sha256sums) already present. This is to produce a more consistent sha256sums that actually reflect what is present in the remote server by including also the sha of the kmods for each kernel version supported. Signed-off-by: Christian Marangi --- phase1/master.cfg | 107 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 76 insertions(+), 31 deletions(-) diff --git a/phase1/master.cfg b/phase1/master.cfg index 380338b..765852b 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -588,6 +588,10 @@ def IsKmodArchiveAndRsyncEnabled(step): return bool(IsKmodArchiveEnabled(step) and branches[branch].get("bin_url")) +def IsRemoteShaSumsAvailable(step): + return step.getProperty("have_remote_shasums") + + def GetBaseVersion(branch): if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch): return branch.split("-")[1] @@ -1383,6 +1387,78 @@ def prepareFactory(target): ) ) + # download remote sha256sums to 'target-sha256sums' + factory.addStep( + ShellCommandAndSetProperty( + name="target-sha256sums", + description="Fetching remote sha256sums for target", + descriptionDone="Remote sha256sums for target fetched", + command=["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")] + + rsync_defopts + + [ + Interpolate( + "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", + url=GetRsyncParams.withArgs("bin", "url"), + target=target, + subtarget=subtarget, + prefix=GetVersionPrefix, + ), + "target-sha256sums", + ], + env={ + "RSYNC_PASSWORD": Interpolate( + "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key") + ) + }, + property="have_remote_shasums", + logEnviron=False, + haltOnFailure=False, + flunkOnFailure=False, + warnOnFailure=False, + doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")), + ) + ) + + factory.addStep( + ShellCommand( + name="target-sha256sums_kmodsparse", + description="Extract kmods from remote sha256sums", + descriptionDone="Kmods extracted", + command="sed \"/ \\*kmods\\//! d\" target-sha256sums | tee target-sha256sums-kmods", + haltOnFailure=False, + doStepIf=IsRemoteShaSumsAvailable, + ) + ) + + factory.addStep( + ShellCommand( + name="mergesha256sum", + description="Merge sha256sums kmods with sha256sums", + descriptionDone="Sha256sums merged", + command=[ + "sort", + "-t", " ", + "-k", 2, + "-u", + Interpolate( + "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", + target=target, + subtarget=subtarget, + ), + "target-sha256sums-kmods", + "-o", + Interpolate( + "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", + target=target, + subtarget=subtarget, + ), + ], + env={"LC_ALL": "C"}, + haltOnFailure=False, + doStepIf=IsRemoteShaSumsAvailable, + ) + ) + # sign factory.addStep( MasterShellCommand( @@ -1541,37 +1617,6 @@ def prepareFactory(target): ) ) - # download remote sha256sums to 'target-sha256sums' - factory.addStep( - ShellCommand( - name="target-sha256sums", - description="Fetching remote sha256sums for target", - descriptionDone="Remote sha256sums for target fetched", - command=["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")] - + rsync_defopts - + [ - Interpolate( - "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", - url=GetRsyncParams.withArgs("bin", "url"), - target=target, - subtarget=subtarget, - prefix=GetVersionPrefix, - ), - "target-sha256sums", - ], - env={ - "RSYNC_PASSWORD": Interpolate( - "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key") - ) - }, - logEnviron=False, - haltOnFailure=False, - flunkOnFailure=False, - warnOnFailure=False, - doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")), - ) - ) - # build list of files to upload factory.addStep( FileDownload( From db5d15143dc98118ab957af2d061075c2a3bebd5 Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Fri, 15 Nov 2024 16:41:52 +0100 Subject: [PATCH 4/5] phase1: correctly exclude kmods files from rsync target upload and prune There is currently a BUG in the --exclude pattern for the targetupload step. The /kmods/ directory is actually never excluded. This was never notice as the rsynclist never actually had kmods entry as make checksum step was done before moving the kmods to the dedicated directory. This is caused by the fact that with --files-from, not only the /kmods/ directory needs to be excluded but also every content in it, hence the additional --exclude "/kmods/**" is required to correcly skip the entry present in rsynclist. Signed-off-by: Christian Marangi --- phase1/master.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/phase1/master.cfg b/phase1/master.cfg index 765852b..e3dee37 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -1664,6 +1664,7 @@ def prepareFactory(target): command=[ "../rsync.sh", "--exclude=/kmods/", + "--exclude=/kmods/**", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" % (target, subtarget), From 174e7a18ac8fb8103d7a653f211eb0b0cd352649 Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Sun, 17 Nov 2024 16:13:02 +0100 Subject: [PATCH 5/5] scripts: signall: skip updating hash of previous kmods/ Handle case where a sha256sums might contain previous kmods/. In such case packages.adb for previous kmods/ needs to be skipped as not included in the sign tar and already signed by previous runs. Skip is limited to kmods/ entry as those are the only entry expected to be present in the sha256sums but not included in sign tar. Signed-off-by: Christian Marangi --- scripts/signall.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/signall.sh b/scripts/signall.sh index 6be0ef1..3bea7be 100755 --- a/scripts/signall.sh +++ b/scripts/signall.sh @@ -90,6 +90,8 @@ if [ -n "$APKSIGNKEY" ]; then grep 'packages\.adb' sha256sums | while IFS= read -r line; do filename="${line#*' *'}" + # Skip updating hash of previous kmods/ if not found in sign tar (already signed) + [ ! -f "$filename" ] && [[ "$filename" == kmods/* ]] && continue escaped_filename="${filename//\//\\\/}" escaped_filename="${escaped_filename//&/\\&}" checksum_output=$(sha256sum --binary -- "$filename")