From 36907f50c8f1acac41b649cb1d14ec5a5d9624b6 Mon Sep 17 00:00:00 2001 From: Diego Nehab <1635557+diegonehab@users.noreply.github.com> Date: Fri, 13 Sep 2024 14:02:06 +0100 Subject: [PATCH] feat: add auto-generated license info for rootfs --- .dockerignore | 1 + .github/workflows/main.yml | 21 +- CHANGELOG.md | 1 + Makefile | 23 +- fs/.dockerignore | 1 + fs/Dockerfile | 66 +--- fs/dhrystone.patch | 311 ------------------ fs/shasumfile | 2 - .../repo-info/Dockerfile.local-dpkg | 31 ++ fs/third-party/repo-info/LICENSE | 191 +++++++++++ fs/third-party/repo-info/gather-dpkg.sh | 166 ++++++++++ fs/third-party/repo-info/scan-local.sh | 97 ++++++ 12 files changed, 542 insertions(+), 369 deletions(-) create mode 100644 fs/.dockerignore delete mode 100644 fs/dhrystone.patch delete mode 100644 fs/shasumfile create mode 100644 fs/third-party/repo-info/Dockerfile.local-dpkg create mode 100644 fs/third-party/repo-info/LICENSE create mode 100755 fs/third-party/repo-info/gather-dpkg.sh create mode 100755 fs/third-party/repo-info/scan-local.sh diff --git a/.dockerignore b/.dockerignore index 083f14e2..7e8f5c5a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,3 +5,4 @@ libcmt* .github .git *.md +fs diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b97a8b2b..83c6a44c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,14 +2,14 @@ name: Build on: [push] jobs: build: - needs: [test] +# needs: [test] runs-on: ubuntu-latest-8-cores steps: - name: Install libarchive-tools run: | export DEBIAN_FRONTEND=noninteractive sudo apt-get update - sudo apt-get install -y --no-install-recommends build-essential autoconf automake libarchive-dev libarchive-tools + sudo apt-get install -y --no-install-recommends build-essential autoconf automake libarchive-dev libarchive-tools pandoc # Building from source cause the provided debian package is for Debian Bookworm - name: Download, build and install xgenext2fs @@ -50,6 +50,9 @@ jobs: - name: Create version file run: make package.json + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Buildx setup uses: docker/setup-buildx-action@v3 @@ -76,6 +79,19 @@ jobs: - name: Build rootfs run: make fs + - name: Auto-generate rootfs license information + if: startsWith(github.ref, 'refs/tags/v') + run: make fs-license + + - name: Upload rootfs license information + if: startsWith(github.ref, 'refs/tags/v') + uses: actions/upload-artifact@v4 + with: + name: license + if-no-files-found: error + path: | + ${{ env.TOOLS_ROOTFS }}.html + - name: Build libcmt id: docker_build_libcmt uses: docker/build-push-action@v5 @@ -118,6 +134,7 @@ jobs: ${{ env.TOOLS_DEB }} ${{ env.TOOLS_DEB }}.sha512 ${{ env.TOOLS_ROOTFS }} + ${{ env.TOOLS_ROOTFS }}.html ${{ env.TOOLS_ROOTFS }}.sha512 libcmt/deb/* diff --git a/CHANGELOG.md b/CHANGELOG.md index 1469e693..5bfa86a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Changed - Bump dependencies versions +- Generate rootfs.ext2.html with licenses of all installed packages ## [0.16.1] - 2024-08-12 ### Fixed diff --git a/Makefile b/Makefile index 08aa217c..676c2464 100644 --- a/Makefile +++ b/Makefile @@ -16,13 +16,14 @@ MAJOR := 0 MINOR := 16 -PATCH := 1 -LABEL := +PATCH := 2 +LABEL := -test2 VERSION := $(MAJOR).$(MINOR).$(PATCH)$(LABEL) TOOLS_DEB := machine-emulator-tools-v$(VERSION).deb TOOLS_IMAGE := cartesi/machine-emulator-tools:$(VERSION) TOOLS_ROOTFS := rootfs-tools-v$(VERSION).ext2 +TOOLS_ROOTFS_IMAGE := cartesi/rootfs-tools:$(VERSION) IMAGE_KERNEL_VERSION ?= v0.20.0 LINUX_VERSION ?= 6.5.13-ctsi-1 @@ -53,8 +54,10 @@ control: Makefile control.in package.json: Makefile package.json.in @sed 's|ARG_VERSION|$(VERSION)|g' package.json.in > package.json -$(TOOLS_ROOTFS) fs: $(TOOLS_DEB) - @docker buildx build --platform=linux/riscv64 \ +fs: $(TOOLS_ROOTFS) + +$(TOOLS_ROOTFS): $(TOOLS_DEB) + @docker buildx build --platform linux/riscv64 \ --build-arg TOOLS_DEB=$(TOOLS_DEB) \ --output type=tar,dest=rootfs.tar \ --file fs/Dockerfile \ @@ -62,6 +65,14 @@ $(TOOLS_ROOTFS) fs: $(TOOLS_DEB) xgenext2fs -fzB 4096 -b 25600 -i 4096 -a rootfs.tar -L rootfs $(TOOLS_ROOTFS) && \ rm -f rootfs.tar +fs-license: + @docker buildx build --load --platform linux/riscv64 \ + --build-arg TOOLS_DEB=$(TOOLS_DEB) \ + -t $(TOOLS_ROOTFS_IMAGE) \ + --file fs/Dockerfile \ + . + TMPFILE=$(mktemp) && (cd fs/third-party/repo-info/; ./scan-local.sh $(TOOLS_ROOTFS_IMAGE) linux/riscv64) | tee $$TMPFILE && pandoc -s -f markdown -t html5 -o $(TOOLS_ROOTFS).html $$TMPFILE && rm -f $$TMPFILE + libcmt: @mkdir $@ @@ -84,6 +95,7 @@ env: @echo TOOLS_DEB=$(TOOLS_DEB) @echo TOOLS_ROOTFS=$(TOOLS_ROOTFS) @echo TOOLS_IMAGE=$(TOOLS_IMAGE) + @echo TOOLS_ROOTFS_IMAGE=$(TOOLS_ROOTFS_IMAGE) @echo IMAGE_KERNEL_VERSION=$(IMAGE_KERNEL_VERSION) @echo LINUX_VERSION=$(LINUX_VERSION) @echo LINUX_HEADERS_URLPATH=$(LINUX_HEADERS_URLPATH) @@ -142,10 +154,11 @@ help: @echo 'available commands:' @echo ' deb - build machine-emulator-tools .deb package' @echo ' fs - build rootfs.ext2' + @echo ' fs-and-license - build rootfs.ext2 and rootfs.ext2.md' @echo ' setup - setup riscv64 buildx' @echo ' setup-required - check if riscv64 buildx setup is required' @echo ' help - list makefile commands' @echo ' env - print useful Makefile variables as a KEY=VALUE list' @echo ' clean - remove the generated artifacts' -.PHONY: build fs deb build-libcmt env setup setup-required help distclean +.PHONY: build fs fs-license deb build-libcmt env setup setup-required help distclean diff --git a/fs/.dockerignore b/fs/.dockerignore new file mode 100644 index 00000000..3c0fee30 --- /dev/null +++ b/fs/.dockerignore @@ -0,0 +1 @@ +third-party diff --git a/fs/Dockerfile b/fs/Dockerfile index 9612257b..06f39b6e 100644 --- a/fs/Dockerfile +++ b/fs/Dockerfile @@ -1,54 +1,22 @@ -FROM --platform=$BUILDPLATFORM ubuntu:22.04 AS cross-builder -ENV BUILD_BASE=/tmp/build-extra - -# Install dependencies -RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ - apt-get install -y --no-install-recommends \ - ca-certificates \ - wget \ - patch \ - libdigest-sha-perl \ - libc6-dev-riscv64-cross \ - gcc-12-riscv64-linux-gnu \ - && \ - adduser developer -u 499 --gecos ",,," --disabled-password && \ - mkdir -p ${BUILD_BASE} && chown -R developer:developer ${BUILD_BASE} && \ - rm -rf /var/lib/apt/lists/* - -USER developer -WORKDIR ${BUILD_BASE} - -# Build benchmark binaries -COPY fs/dhrystone.patch ${BUILD_BASE}/ -COPY fs/shasumfile ${BUILD_BASE}/ -RUN mkdir benchmarks && cd benchmarks && \ - wget https://www.netlib.org/benchmark/whetstone.c https://www.netlib.org/benchmark/dhry-c && \ - shasum -ca 256 ../shasumfile &&\ - bash dhry-c && \ - patch -p1 < ../dhrystone.patch && \ - riscv64-linux-gnu-gcc-12 -O2 -o whetstone whetstone.c -lm && \ - riscv64-linux-gnu-gcc-12 -O2 -o dhrystone dhry_1.c dhry_2.c -lm - # Final image -FROM --platform=linux/riscv64 riscv64/ubuntu:22.04 -ARG TOOLS_DEB=machine-emulator-tools-v0.15.0.deb +FROM --platform=linux/riscv64 ubuntu:22.04 +ARG TOOLS_DEB=machine-emulator-tools-v0.16.2.deb ADD ${TOOLS_DEB} /tmp/ RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - busybox-static=1:1.30.1-7ubuntu3 \ - coreutils=8.32-4.1ubuntu1.2 \ - bash=5.1-6ubuntu1.1 \ - psmisc=23.4-2build3 \ - bc=1.07.1-3build1 \ - curl=7.81.0-1ubuntu1.18 \ - device-tree-compiler=1.6.1-1 \ - jq=1.6-2.1ubuntu3 \ - lua5.4=5.4.4-1 \ - lua-socket=3.0~rc1+git+ac3201d-6 \ - xxd=2:8.2.3995-1ubuntu2.18 \ - file=1:5.41-3ubuntu0.1 \ - /tmp/${TOOLS_DEB} \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + busybox-static=1:1.30.1-7ubuntu3 \ + coreutils=8.32-4.1ubuntu1.2 \ + bash=5.1-6ubuntu1.1 \ + psmisc=23.4-2build3 \ + bc=1.07.1-3build1 \ + curl=7.81.0-1ubuntu1.18 \ + device-tree-compiler=1.6.1-1 \ + jq=1.6-2.1ubuntu3 \ + lua5.4=5.4.4-1 \ + lua-socket=3.0~rc1+git+ac3201d-6 \ + xxd=2:8.2.3995-1ubuntu2.19 \ + file=1:5.41-3ubuntu0.1 \ + stress-ng=0.13.12-2ubuntu1 \ + /tmp/${TOOLS_DEB} \ && \ rm -rf /var/lib/apt/lists/* /tmp/${TOOLS_DEB} -COPY --chown=root:root --from=cross-builder /tmp/build-extra/benchmarks/whetstone /usr/bin/ -COPY --chown=root:root --from=cross-builder /tmp/build-extra/benchmarks/dhrystone /usr/bin/ diff --git a/fs/dhrystone.patch b/fs/dhrystone.patch deleted file mode 100644 index 4a604188..00000000 --- a/fs/dhrystone.patch +++ /dev/null @@ -1,311 +0,0 @@ -diff -Nru a/dhry.h b/dhry.h ---- a/dhry.h -+++ b/dhry.h -@@ -345,6 +345,9 @@ - *************************************************************************** - */ - -+#ifndef DHRY_H -+#define DHRY_H -+ - /* Compiler and system dependent definitions: */ - - #ifndef TIME -@@ -359,6 +362,10 @@ - /* for "times" */ - #endif - -+#ifndef HZ -+#include -+#endif -+ - #define Mic_secs_Per_Second 1000000.0 - /* Berkeley UNIX C returns process times in seconds/HZ */ - -@@ -384,6 +391,8 @@ - /* General definitions: */ - - #include -+#include -+#include - /* for strcpy, strcmp */ - - #define Null 0 -@@ -420,4 +429,16 @@ - } variant; - } Rec_Type, *Rec_Pointer; - -+void Proc_1(Rec_Pointer Ptr_Val_Par); -+void Proc_2(One_Fifty *Int_Par_Ref); -+void Proc_3(Rec_Pointer *Ptr_Ref_Par); -+void Proc_4(); -+void Proc_5(); -+void Proc_6(Enumeration Enum_Val_Par, Enumeration *Enum_Ref_Par); -+void Proc_7(One_Fifty Int_1_Par_Val, One_Fifty Int_2_Par_Val, One_Fifty *Int_Par_Ref); -+void Proc_8(Arr_1_Dim Arr_1_Par_Ref, Arr_2_Dim Arr_2_Par_Ref, int Int_1_Par_Val, int Int_2_Par_Val); - -+Enumeration Func_1(Capital_Letter Ch_1_Par_Val, Capital_Letter Ch_2_Par_Val); -+Boolean Func_2(Str_30 Str_1_Par_Ref, Str_30 Str_2_Par_Ref); -+Boolean Func_3(Enumeration Enum_Par_Val); -+#endif -diff -Nru a/dhry_1.c b/dhry_1.c ---- a/dhry_1.c -+++ b/dhry_1.c -@@ -17,6 +17,8 @@ - - #include "dhry.h" - -+#include -+ - /* Global Variables: */ - - Rec_Pointer Ptr_Glob, -@@ -28,9 +30,6 @@ - int Arr_1_Glob [50]; - int Arr_2_Glob [50] [50]; - --extern char *malloc (); --Enumeration Func_1 (); -- /* forward declaration necessary since Enumeration may not simply be int */ - - #ifndef REG - Boolean Reg = false; -@@ -45,13 +44,11 @@ - - #ifdef TIMES - struct tms time_info; --extern int times (); - /* see library function "times" */ - #define Too_Small_Time 120 - /* Measurements should last at least about 2 seconds */ - #endif - #ifdef TIME --extern long time(); - /* see library function "time" */ - #define Too_Small_Time 2 - /* Measurements should last at least 2 seconds */ -@@ -65,8 +62,7 @@ - - /* end of variables for time measurement */ - -- --main () -+int main (int argc, char *argv[]) - /*****/ - - /* main program, corresponds to procedures */ -@@ -101,6 +97,15 @@ - /* Warning: With 16-Bit processors and Number_Of_Runs > 32000, */ - /* overflow may occur for this array element. */ - -+ Number_Of_Runs = 0; -+ if (argc == 2) -+ { -+ if (atoi(argv[1]) > 0) -+ { -+ Number_Of_Runs = atoi(argv[1]); -+ } -+ } -+ - printf ("\n"); - printf ("Dhrystone Benchmark, Version 2.1 (Language: C)\n"); - printf ("\n"); -@@ -114,13 +119,6 @@ - printf ("Program compiled without 'register' attribute\n"); - printf ("\n"); - } -- printf ("Please give the number of runs through the benchmark: "); -- { -- int n; -- scanf ("%d", &n); -- Number_Of_Runs = n; -- } -- printf ("\n"); - - printf ("Execution starts, %d runs through Dhrystone\n", Number_Of_Runs); - -@@ -211,7 +209,7 @@ - printf ("Arr_2_Glob[8][7]: %d\n", Arr_2_Glob[8][7]); - printf (" should be: Number_Of_Runs + 10\n"); - printf ("Ptr_Glob->\n"); -- printf (" Ptr_Comp: %d\n", (int) Ptr_Glob->Ptr_Comp); -+ printf (" Ptr_Comp: %ld\n", (uintptr_t) Ptr_Glob->Ptr_Comp); - printf (" should be: (implementation-dependent)\n"); - printf (" Discr: %d\n", Ptr_Glob->Discr); - printf (" should be: %d\n", 0); -@@ -222,7 +220,7 @@ - printf (" Str_Comp: %s\n", Ptr_Glob->variant.var_1.Str_Comp); - printf (" should be: DHRYSTONE PROGRAM, SOME STRING\n"); - printf ("Next_Ptr_Glob->\n"); -- printf (" Ptr_Comp: %d\n", (int) Next_Ptr_Glob->Ptr_Comp); -+ printf (" Ptr_Comp: %ld\n", (uintptr_t) Next_Ptr_Glob->Ptr_Comp); - printf (" should be: (implementation-dependent), same as above\n"); - printf (" Discr: %d\n", Next_Ptr_Glob->Discr); - printf (" should be: %d\n", 0); -@@ -273,15 +271,14 @@ - printf ("%6.1f \n", Dhrystones_Per_Second); - printf ("\n"); - } -- -+ -+ return 0; - } - - --Proc_1 (Ptr_Val_Par) -+void Proc_1 (REG Rec_Pointer Ptr_Val_Par) - /******************/ -- --REG Rec_Pointer Ptr_Val_Par; -- /* executed once */ -+/* executed once */ - { - REG Rec_Pointer Next_Record = Ptr_Val_Par->Ptr_Comp; - /* == Ptr_Glob_Next */ -@@ -311,12 +308,10 @@ - } /* Proc_1 */ - - --Proc_2 (Int_Par_Ref) -+void Proc_2 (One_Fifty *Int_Par_Ref) - /******************/ - /* executed once */ - /* *Int_Par_Ref == 1, becomes 4 */ -- --One_Fifty *Int_Par_Ref; - { - One_Fifty Int_Loc; - Enumeration Enum_Loc; -@@ -334,13 +329,10 @@ - } /* Proc_2 */ - - --Proc_3 (Ptr_Ref_Par) -+void Proc_3 (Rec_Pointer *Ptr_Ref_Par) - /******************/ - /* executed once */ - /* Ptr_Ref_Par becomes Ptr_Glob */ -- --Rec_Pointer *Ptr_Ref_Par; -- - { - if (Ptr_Glob != Null) - /* then, executed */ -@@ -349,7 +341,7 @@ - } /* Proc_3 */ - - --Proc_4 () /* without parameters */ -+void Proc_4 () /* without parameters */ - /*******/ - /* executed once */ - { -@@ -361,7 +353,7 @@ - } /* Proc_4 */ - - --Proc_5 () /* without parameters */ -+void Proc_5 () /* without parameters */ - /*******/ - /* executed once */ - { -diff -Nru a/dhry_2.c b/dhry_2.c ---- a/dhry_2.c -+++ b/dhry_2.c -@@ -27,13 +27,10 @@ - extern char Ch_1_Glob; - - --Proc_6 (Enum_Val_Par, Enum_Ref_Par) -+void Proc_6 (Enumeration Enum_Val_Par, Enumeration *Enum_Ref_Par) - /*********************************/ - /* executed once */ - /* Enum_Val_Par == Ident_3, Enum_Ref_Par becomes Ident_2 */ -- --Enumeration Enum_Val_Par; --Enumeration *Enum_Ref_Par; - { - *Enum_Ref_Par = Enum_Val_Par; - if (! Func_3 (Enum_Val_Par)) -@@ -61,7 +58,7 @@ - } /* Proc_6 */ - - --Proc_7 (Int_1_Par_Val, Int_2_Par_Val, Int_Par_Ref) -+void Proc_7 (One_Fifty Int_1_Par_Val, One_Fifty Int_2_Par_Val, One_Fifty *Int_Par_Ref) - /**********************************************/ - /* executed three times */ - /* first call: Int_1_Par_Val == 2, Int_2_Par_Val == 3, */ -@@ -70,9 +67,6 @@ - /* Int_Par_Ref becomes 17 */ - /* third call: Int_1_Par_Val == 6, Int_2_Par_Val == 10, */ - /* Int_Par_Ref becomes 18 */ --One_Fifty Int_1_Par_Val; --One_Fifty Int_2_Par_Val; --One_Fifty *Int_Par_Ref; - { - One_Fifty Int_Loc; - -@@ -81,15 +75,11 @@ - } /* Proc_7 */ - - --Proc_8 (Arr_1_Par_Ref, Arr_2_Par_Ref, Int_1_Par_Val, Int_2_Par_Val) -+void Proc_8 (Arr_1_Dim Arr_1_Par_Ref, Arr_2_Dim Arr_2_Par_Ref, int Int_1_Par_Val, int Int_2_Par_Val) - /*********************************************************************/ - /* executed once */ - /* Int_Par_Val_1 == 3 */ - /* Int_Par_Val_2 == 7 */ --Arr_1_Dim Arr_1_Par_Ref; --Arr_2_Dim Arr_2_Par_Ref; --int Int_1_Par_Val; --int Int_2_Par_Val; - { - REG One_Fifty Int_Index; - REG One_Fifty Int_Loc; -@@ -106,15 +96,12 @@ - } /* Proc_8 */ - - --Enumeration Func_1 (Ch_1_Par_Val, Ch_2_Par_Val) -+Enumeration Func_1 (Capital_Letter Ch_1_Par_Val, Capital_Letter Ch_2_Par_Val) - /*************************************************/ - /* executed three times */ - /* first call: Ch_1_Par_Val == 'H', Ch_2_Par_Val == 'R' */ - /* second call: Ch_1_Par_Val == 'A', Ch_2_Par_Val == 'C' */ - /* third call: Ch_1_Par_Val == 'B', Ch_2_Par_Val == 'C' */ -- --Capital_Letter Ch_1_Par_Val; --Capital_Letter Ch_2_Par_Val; - { - Capital_Letter Ch_1_Loc; - Capital_Letter Ch_2_Loc; -@@ -132,14 +119,11 @@ - } /* Func_1 */ - - --Boolean Func_2 (Str_1_Par_Ref, Str_2_Par_Ref) -+Boolean Func_2 (Str_30 Str_1_Par_Ref, Str_30 Str_2_Par_Ref) - /*************************************************/ - /* executed once */ - /* Str_1_Par_Ref == "DHRYSTONE PROGRAM, 1'ST STRING" */ - /* Str_2_Par_Ref == "DHRYSTONE PROGRAM, 2'ND STRING" */ -- --Str_30 Str_1_Par_Ref; --Str_30 Str_2_Par_Ref; - { - REG One_Thirty Int_Loc; - Capital_Letter Ch_Loc; -@@ -174,11 +158,10 @@ - } /* Func_2 */ - - --Boolean Func_3 (Enum_Par_Val) -+Boolean Func_3 (Enumeration Enum_Par_Val) - /***************************/ - /* executed once */ - /* Enum_Par_Val == Ident_3 */ --Enumeration Enum_Par_Val; - { - Enumeration Enum_Loc; - diff --git a/fs/shasumfile b/fs/shasumfile deleted file mode 100644 index 0eb7588a..00000000 --- a/fs/shasumfile +++ /dev/null @@ -1,2 +0,0 @@ -333e4ceca042c146f63eec605573d16ae8b07166cbc44a17bec1ea97c6f1efbf whetstone.c -038a7e9169787125c3451a6c941f3aca5db2d2f3863871afcdce154ef17f4e3e dhry-c diff --git a/fs/third-party/repo-info/Dockerfile.local-dpkg b/fs/third-party/repo-info/Dockerfile.local-dpkg new file mode 100644 index 00000000..62d9af61 --- /dev/null +++ b/fs/third-party/repo-info/Dockerfile.local-dpkg @@ -0,0 +1,31 @@ +# Modified from the original +# --- Dockerfile.local-dpkg.orig 2024-09-18 15:36:54 +# +++ Dockerfile.local-dpkg 2024-10-02 15:37:24 +# @@ -1,4 +1,4 @@ +# -FROM debian:bullseye-slim +# +FROM ubuntu:24.04 +# +# RUN set -eux; \ +# apt-get update; \ +# @@ -9,6 +9,6 @@ +# ; \ +# rm -rf /var/lib/apt/lists/* +# +# -COPY .local-scripts/gather-dpkg.sh /usr/local/bin/ +# +COPY gather-dpkg.sh /usr/local/bin/ +# +# CMD ["gather-dpkg.sh"] +FROM ubuntu:24.04 + +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + gawk \ + wget \ + ; \ + rm -rf /var/lib/apt/lists/* + +COPY gather-dpkg.sh /usr/local/bin/ + +CMD ["gather-dpkg.sh"] diff --git a/fs/third-party/repo-info/LICENSE b/fs/third-party/repo-info/LICENSE new file mode 100644 index 00000000..27448585 --- /dev/null +++ b/fs/third-party/repo-info/LICENSE @@ -0,0 +1,191 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2014 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/fs/third-party/repo-info/gather-dpkg.sh b/fs/third-party/repo-info/gather-dpkg.sh new file mode 100755 index 00000000..6246cab0 --- /dev/null +++ b/fs/third-party/repo-info/gather-dpkg.sh @@ -0,0 +1,166 @@ +#!/bin/bash +set -e + +IFS=$'\n' +rawPackages=( $(dpkg-query --show --showformat='${db:Status-Abbrev} ${source:Package}=${source:Version} ${binary:Package}=${Version}\n' 2>/dev/null) ) +unset IFS + +declare -A packages=() +for rawPackage in "${rawPackages[@]}"; do + rawPackage=( $rawPackage ) + stat="${rawPackage[0]}" + case "$stat" in + i?|h?) + # "install" or "hold" + ;; + *) + # skip "unknown", "remove", "purge" + continue + ;; + esac + src="${rawPackage[1]}" + bin="${rawPackage[2]}" + [ -z "${packages[$src]}" ] || packages[$src]+=' ' + packages[$src]+="$bin" +done + +if [ "${#packages[@]}" -eq 0 ]; then + # not Debian-based + exit 1 +fi + +if [ -e /etc/apt/sources.list ] || [ -d /etc/apt/sources.list.d ]; then + # make sure we have "deb-src" entries for "apt-get source" + find /etc/apt/sources.list* \ + -type f \ + -exec sed -i \ + -e 'p; s/^deb /deb-src /' \ + -e 's/^Types: deb$/Types: deb deb-src/' \ + '{}' + + + # retry a few times if "apt-get update" fails + tries=5 + while ! apt-get update -qq; do + (( --tries )) || : + if [ "$tries" -le 0 ]; then + echo >&2 'error: failed to "apt-get update" after multiple attempts' + exit 1 + fi + done +fi + +IFS=$'\n' +sortedSources=( $(echo "${!packages[*]}" | sort) ) +unset IFS + +echo +echo '## `dpkg` (`.deb`-based packages)' + +# prints "$2$1$3$1...$N" +join() { + local sep="$1"; shift + local out; printf -v out "${sep//%/%%}%s" "$@" + echo "${out#$sep}" +} + +for src in "${sortedSources[@]}"; do + echo + echo '### `dpkg` source package: `'"$src"'`' + echo + echo 'Binary Packages:' + echo + for bin in ${packages[$src]}; do + echo '- `'"$bin"'`' + done + + # parse /usr/share/doc/BIN/copyright + licenses=() + licenseFiles=() + for bin in ${packages[$src]}; do + # https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ + # http://dep.debian.net/deps/dep5/ + binPkg="${bin%%=*}" + binPkgOnly="${binPkg%%:*}" + f= + for try in \ + "/usr/share/doc/$bin/copyright" \ + "/usr/share/doc/$binPkg/copyright" \ + "/usr/share/doc/$binPkgOnly/copyright" \ + ; do + if [ -f "$try" ]; then + f="$try" + break + fi + done + if [ -z "$f" ]; then + echo >&2 + echo >&2 "**WARNING:** '/usr/share/doc/$binPkgOnly/copyright' is missing!" + echo >&2 + continue + fi + IFS=$'\n' + licenses+=( $(gawk -F ':[ \t]+' '$1 == "License" && NF > 1 { gsub(/^License:[ \t]+/, ""); print }' "$f") ) + licenses+=( $(grep -oE '/usr/share/common-licenses/[0-9a-zA-Z_.+-]+' "$f" | cut -d/ -f5-) ) + unset IFS + licenseFiles+=( "$f" ) + done + if [ "${#licenses[@]}" -gt 0 ]; then + IFS=$'\n' + licenses=( $( + echo "${licenses[*]}" \ + | sed -r \ + -e 's/ (and|or) /\n/g' \ + -e 's/[.,]+$//' \ + | sort -u + ) ) + unset IFS + + echo + echo 'Licenses: (parsed from: `'"$(join '`, `' "${licenseFiles[@]}")"'`)' + echo + for lic in "${licenses[@]}"; do + echo '- `'"$lic"'`' + done + else + echo + echo '**WARNING:** unable to detect licenses! (package likely not compliant with DEP-5) ' + echo 'If source is available (seen below), check the contents of `debian/copyright` within it.' + echo + fi + + sourcesUrl="https://sources.debian.net/src/${src//=//}/" + snapshotUrl="http://snapshot.debian.org/package/${src//=//}/" + + aptSourceArgs=( apt-get source -qq --print-uris "$src" ) + if aptSource="$("${aptSourceArgs[@]}" 2>/dev/null)" && [ -n "$aptSource" ]; then + echo + echo 'Source:' + echo + echo '```console' + echo '$' "${aptSourceArgs[@]}" + echo "$aptSource" + echo '```' + case "$aptSource" in + *.debian.org/*) + # _probably_ Debian -- let's link to sources.debian.net too + echo + echo 'Other potentially useful URLs:' + echo + echo '- '"$sourcesUrl"' (for browsing the source)' + echo '- '"$sourcesUrl"'debian/copyright/ (for direct copyright/license information)' + echo '- '"$snapshotUrl"' (for access to the source package after it no longer exists in the archive)' + ;; + esac + else + echo + echo '**WARNING:** unable to find source (`apt-get source` failed or returned no results)! ' + echo 'This is *usually* due to a new package version being released and the old version being removed.' + echo + if wget --quiet --spider -O /dev/null -o /dev/null "$snapshotUrl"; then + echo 'The source package *may* still be available for download from:' + echo + echo "- $snapshotUrl" + echo + fi + fi +done diff --git a/fs/third-party/repo-info/scan-local.sh b/fs/third-party/repo-info/scan-local.sh new file mode 100755 index 00000000..eeb87ef5 --- /dev/null +++ b/fs/third-party/repo-info/scan-local.sh @@ -0,0 +1,97 @@ +#!/bin/bash +# Modified from the original +# --- scan-local.sh.orig 2024-09-18 15:37:23 +# +++ scan-local.sh 2024-10-02 15:38:39 +# @@ -4,15 +4,15 @@ +# trap 'echo >&2 Ctrl+C captured, exiting; exit 1' SIGINT +# +# image="$1"; shift +# +platform="$1"; shift +# +# -docker build --pull -t repo-info:local-apk -q -f Dockerfile.local-apk . > /dev/null +# -docker build --pull -t repo-info:local-dpkg -q -f Dockerfile.local-dpkg . > /dev/null +# -docker build --pull -t repo-info:local-rpm -q -f Dockerfile.local-rpm . > /dev/null +# +docker buildx build --platform $platform --load --pull -t repo-info:local-dpkg -f Dockerfile.local-dpkg . 1>&2 +# +# name="repo-info-local-$$-$RANDOM" +# trap "docker rm -vf '$name-data' > /dev/null || :" EXIT +# +# docker create \ +# + --platform $platform \ +# --name "$name-data" \ +# -v /etc \ +# -v /lib/apk \ +# @@ -47,7 +47,7 @@ +# +# - Image ID: `{{ .Id }}` +# - Created: `{{ .Created }}` +# -- Virtual Size: '"$size"' +# +- Virtual Size: '"$size"' +# (total size of all layers on-disk) +# - Arch: `{{ .Os }}`/`{{ .Architecture }}` +# {{ if .Config.Entrypoint }}- Entrypoint: `{{ json .Config.Entrypoint }}` +# @@ -55,6 +55,4 @@ +# {{ end }}- Environment:{{ range .Config.Env }}{{ "\n" }} - `{{ . }}`{{ end }}{{ if .Config.Labels }} +# - Labels:{{ range $k, $v := .Config.Labels }}{{ "\n" }} - `{{ $k }}={{ $v }}`{{ end }}{{ end }}' "$image" +# +# -docker run --rm --volumes-from "$name-data" -v /etc/ssl repo-info:local-apk || : +# -docker run --rm --volumes-from "$name-data" -v /etc/ssl repo-info:local-dpkg || : +# -docker run --rm --volumes-from "$name-data" -v /etc/ssl repo-info:local-rpm || : +# +docker run --platform $platform --rm --volumes-from "$name-data" -v /etc/ssl repo-info:local-dpkg || : +set -e + +trap 'echo >&2 Ctrl+C captured, exiting; exit 1' SIGINT + +image="$1"; shift +platform="$1"; shift + +docker buildx build --platform $platform --load --pull -t repo-info:local-dpkg -f Dockerfile.local-dpkg . 1>&2 + +name="repo-info-local-$$-$RANDOM" +trap "docker rm -vf '$name-data' > /dev/null || :" EXIT + +docker create \ + --platform $platform \ + --name "$name-data" \ + -v /etc \ + -v /lib/apk \ + -v /usr/lib/rpm \ + -v /usr/share/apk \ + -v /usr/share/doc \ + -v /var/lib \ + "$image" \ + bogus > /dev/null + +echo '# `'"$image"'`' + +size="$( + docker inspect -f '{{ .VirtualSize }}' "$image" | awk '{ + oneKb = 1000; + oneMb = 1000 * oneKb; + oneGb = 1000 * oneMb; + if ($1 >= oneGb) { + printf "~ %.2f Gb", $1 / oneGb + } else if ($1 >= oneMb) { + printf "~ %.2f Mb", $1 / oneMb + } else if ($1 >= oneKb) { + printf "~ %.2f Kb", $1 / oneKb + } else { + printf "%d bytes", $1 + } + }' +)" + +docker inspect -f ' +## Docker Metadata + +- Image ID: `{{ .Id }}` +- Created: `{{ .Created }}` +- Virtual Size: '"$size"' + (total size of all layers on-disk) +- Arch: `{{ .Os }}`/`{{ .Architecture }}` +{{ if .Config.Entrypoint }}- Entrypoint: `{{ json .Config.Entrypoint }}` +{{ end }}{{ if .Config.Cmd }}- Command: `{{ json .Config.Cmd }}` +{{ end }}- Environment:{{ range .Config.Env }}{{ "\n" }} - `{{ . }}`{{ end }}{{ if .Config.Labels }} +- Labels:{{ range $k, $v := .Config.Labels }}{{ "\n" }} - `{{ $k }}={{ $v }}`{{ end }}{{ end }}' "$image" + +docker run --platform $platform --rm --volumes-from "$name-data" -v /etc/ssl repo-info:local-dpkg || :