diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml new file mode 100644 index 0000000..34683e8 --- /dev/null +++ b/.github/workflows/bandit.yml @@ -0,0 +1,27 @@ +name: Security check - Bandit + +on: + push: + branches: [ "*" ] + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-22.04 + permissions: + checks: write # for bandit-report-artifacts to publish the checks + contents: read # for actions/checkout to fetch code + security-events: write # for bandit-report-artifacts to upload results + actions: read # only on private (maybe?) required to get the Action run status + + steps: + - uses: actions/checkout@v4 + + - name: Run bandit + uses: VCTLabs/bandit-report-artifacts@v2 + with: + project_path: src + ignore_failure: false + config_file: pyproject.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0d1485..8d06594 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,8 +2,10 @@ name: Test bmaptool on: - - push - - pull_request + push: + branches: [ main ] + pull_request: + workflow_dispatch: jobs: test: @@ -19,7 +21,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -40,5 +42,5 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: psf/black@stable diff --git a/.github/workflows/debs.yml b/.github/workflows/debs.yml new file mode 100644 index 0000000..7d11f61 --- /dev/null +++ b/.github/workflows/debs.yml @@ -0,0 +1,85 @@ +name: Debian packages + +on: + push: + branches: [ main ] + pull_request: + workflow_dispatch: + +jobs: + get_version: + name: Get version info + runs-on: ubuntu-22.04 + defaults: + run: + shell: bash + outputs: + version: ${{ steps.git_ver.outputs.version }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get package version + id: git_ver + run: | + version=$(git describe --tags | sed -e "s|v||" -e "s|-g|+g|") + echo "Version from git: ${version}" + echo "version=${version}" >> $GITHUB_OUTPUT + + build_debs: + name: ${{ matrix.name }} + runs-on: ubuntu-22.04 + needs: [get_version] + + strategy: + fail-fast: false + matrix: + name: [ + x64_bookworm, + x64_trixie, + x64_sid, + ] + + include: + - name: x64_bookworm + dist: bookworm + + - name: x64_trixie + dist: trixie + + - name: x64_sid + dist: sid + + steps: + - name: Check github variables + env: + VERSION: ${{ needs.get_version.outputs.version }} + run: | + echo "Package version from git: ${VERSION}" + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install deps and update debian changelog + run: | + sudo apt-get update + sudo apt-get install devscripts + debchange -v ${{ needs.get_version.outputs.version }}-${{ matrix.dist }} -b -M --distribution ${{ matrix.dist }} "ci build" + + - name: Build deb packages + uses: jtdor/build-deb-action@v1 + env: + DEB_BUILD_OPTIONS: noautodbgsym + with: + docker-image: "debian:${{ matrix.dist }}-slim" + buildpackage-opts: --build=binary --no-sign + extra-build-deps: git + + - name: Upload deb files + uses: actions/upload-artifact@v4 + with: + name: "bmaptool_${{ needs.get_version.outputs.version }}-${{ matrix.dist }}" + path: ./debian/artifacts/*.deb diff --git a/debian/bmaptool.docs b/debian/bmaptool.docs deleted file mode 100644 index d608dd0..0000000 --- a/debian/bmaptool.docs +++ /dev/null @@ -1 +0,0 @@ -docs/RELEASE_NOTES diff --git a/debian/bmaptool.install b/debian/bmaptool.install deleted file mode 100644 index 8b13789..0000000 --- a/debian/bmaptool.install +++ /dev/null @@ -1 +0,0 @@ - diff --git a/debian/changelog b/debian/changelog index b2286ec..3d36a98 100644 --- a/debian/changelog +++ b/debian/changelog @@ -7,7 +7,7 @@ bmaptool (3.8.0) unstable; urgency=low variations * switch to use an X.Y.Z versioning number scheme - -- Trevor Woerner Mon, 18 Mar 2024 23:44:10 -0400 + -- Trevor Woerner Mon, 18 Mar 2024 23:44:10 -0400 bmap-tools (3.7) unstable; urgency=low @@ -17,7 +17,7 @@ bmap-tools (3.7) unstable; urgency=low * Add functionality for copying from standard input (#99) * Switch from gpg to gpgme module (#103) - -- Artem Bityutskiy Wed, 02 Aug 2023 15:11:26 +0300 + -- Artem Bityutskiy Wed, 02 Aug 2023 15:11:26 +0300 bmap-tools (3.6) unstable; urgency=low @@ -25,14 +25,14 @@ bmap-tools (3.6) unstable; urgency=low * Added the 'zstd' compression type support. * Add '--psplash-pipe' option for interacting with psplash. - -- Artem Bityutskiy Tue, 02 Feb 2021 14:08:41 +0200 + -- Artem Bityutskiy Tue, 02 Feb 2021 14:08:41 +0200 bmap-tools (3.5) unstable; urgency=low * Fixed copying of compressed files from URLs * Python 3.x support fixes and improvements. - -- Artem Bityutskiy Thu, 23 Aug 2018 10:34:31 +0300 + -- Artem Bityutskiy Thu, 23 Aug 2018 10:34:31 +0300 bmap-tools (3.4) unstable; urgency=low diff --git a/debian/clean b/debian/clean new file mode 100644 index 0000000..71550c0 --- /dev/null +++ b/debian/clean @@ -0,0 +1 @@ +bmap_tools.egg-info/ diff --git a/debian/compat b/debian/compat deleted file mode 100644 index ec63514..0000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -9 diff --git a/debian/control b/debian/control index 4ab5fcd..659ef55 100644 --- a/debian/control +++ b/debian/control @@ -2,31 +2,47 @@ Source: bmaptool Maintainer: Trevor Woerner Section: utils Priority: optional -Build-Depends: debhelper (>= 9), - python3 (>= 3.8), - python3-setuptools, -Standards-Version: 3.8.4 -XS-Python-Version: >= 3.8 +Build-Depends: + debhelper-compat (= 10), + dh-python, + python3-all, + python3-setuptools, + pybuild-plugin-pyproject, + python3-hatchling, + python3-gpg, + python3-six, + python3-defusedxml, + python3-all-dev, + python3-pytest, +Standards-Version: 4.5.0 +Vcs-Git: https://github.com/yoctoproject/bmaptool.git +Vcs-Browser: https://github.com/yoctoproject/bmaptool +Homepage: https://github.com/yoctoproject/bmaptool +Rules-Requires-Root: no +X-Python3-Version: >= 3.8 Package: bmaptool Architecture: all -Depends: python (>=3.8), - python-gpgme, - ${misc:Depends}, - ${python:Depends}, - bzip2, - pbzip2, - gzip, - pigz, - lzop, - liblz4-tool, - xz-utils, - tar, - unzip, - zstd -Description: Tools to generate block map (AKA bmap) and flash images using - bmap. bmaptool is a generic tool for creating the block map (bmap) for a file, - and copying files using the block map. The idea is that large file containing - unused blocks, like raw system image files, can be copied or flashed a lot - faster with bmaptool than with traditional tools like "dd" or "cp". See - source.tizen.org/documentation/reference/bmaptool for more information. +Depends: + python3, + python3-gpg, + python3-defusedxml, + ${misc:Depends}, + ${python3:Depends}, +Recommends: + bzip2, + lzop, + xz-utils, +Suggests: + lz4, + pbzip2, + pigz, + unzip, +Provides: + bmaptool (= ${binary:Version}), +Description: tool to flash image files to block devices using the block map + bmaptool is a generic tool for creating the block map (bmap) for a + file, and copying files using the block map. The idea is that large + file containing unused blocks, like raw system image files, can be + copied or flashed a lot faster with bmaptool than with traditional + tools like "dd" or "cp". diff --git a/debian/copyright b/debian/copyright index 8d0fe60..5ef05b2 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,19 +1,28 @@ -Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-name: bmaptool +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: bmaptool Upstream-Contact: Trevor Woerner -Source: https://github.com/yoctoproject/bmaptool - . +Source: https://github.com/yoctoproject/bmaptool.git +Comment: The initial package was put together by Ed Bartosh on Sun Oct 27 22:32:19 EEST 2012. Files: * -Copyright: © 2012-2013 Intel, Inc. +Copyright: © 2012-2014 Intel, Inc. License: GPL-2 Files: debian/* -Copyright: © 2012-2013 Intel, Inc. +Copyright: + © 2012-2013 Intel, Inc. + © 2014-2018 Simon McVittie License: GPL-2 +Files: + debian/tests/web-server.py +Copyright: + © 2017-2019 Red Hat Inc. +License: LGPL-2.1+ +Comment: Copied from the Flatpak test suite + License: GPL-2 This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2, @@ -26,3 +35,21 @@ License: GPL-2 Comment: On Debian systems, the full text of the GPL v2 can be found in /usr/share/common-licenses/GPL-2. + +License: LGPL-2.1+ + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + . + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + . + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +Comment: + On Debian systems, the full text of the GNU Lesser General Public License + version 2.1 can be found in the file '/usr/share/common-licenses/LGPL-2.1'. diff --git a/debian/gbp.conf b/debian/gbp.conf new file mode 100644 index 0000000..b5aae9e --- /dev/null +++ b/debian/gbp.conf @@ -0,0 +1,6 @@ +[DEFAULT] +debian-branch = debian/latest +upstream-branch = upstream/latest +pristine-tar = True +upstream-vcs-tag = v%(version)s +patch-numbers = False diff --git a/debian/rules b/debian/rules index 016f8b3..49eea2c 100755 --- a/debian/rules +++ b/debian/rules @@ -1,4 +1,25 @@ #!/usr/bin/make -f +#export DH_VERBOSE = 1 +export PYBUILD_NAME = bmaptool + +PYTHON3S:=$(shell py3versions -vr) + %: - dh $@ --with=python3 + dh $@ --with python3 --buildsystem=pybuild + +override_dh_auto_clean: + dh_auto_clean + dh_clean PKG-INFO + dh_clean .pytest_cache/ src/*.egg-info/ + +override_dh_auto_configure: + dh_auto_configure -- $(CONFIGURE_ARGS) + dh_auto_configure --buildsystem=pybuild + +override_dh_auto_test: +ifeq (,$(findstring nocheck, $(DEB_BUILD_OPTIONS))) + set -e ; for pyvers in $(PYTHON3S); do \ + PYTHONPATH="./src" python$$pyvers -m pytest -k 'not TestCLI' tests ; \ + done +endif diff --git a/debian/salsa-ci.yml b/debian/salsa-ci.yml new file mode 100644 index 0000000..0c22dc4 --- /dev/null +++ b/debian/salsa-ci.yml @@ -0,0 +1,3 @@ +include: + - https://salsa.debian.org/salsa-ci-team/pipeline/raw/master/salsa-ci.yml + - https://salsa.debian.org/salsa-ci-team/pipeline/raw/master/pipeline-jobs.yml diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/debian/tests/control b/debian/tests/control new file mode 100644 index 0000000..a86c48a --- /dev/null +++ b/debian/tests/control @@ -0,0 +1,3 @@ +Tests: smoke +Depends: bmaptool, dosfstools, python3 +Restrictions: allow-stderr diff --git a/debian/tests/smoke b/debian/tests/smoke new file mode 100755 index 0000000..d8b92ce --- /dev/null +++ b/debian/tests/smoke @@ -0,0 +1,29 @@ +#!/bin/sh + +set -eu + +here="$(dirname "$0")" +here="$(readlink -f "$here")" + +cd "${AUTOPKGTEST_TMP}" + +mkdir webroot +rm -f httpd-pipe +mkfifo httpd-pipe +"$here/web-server.py" webroot 3> httpd-pipe & +httpd_pid="$!" +read ignored < httpd-pipe + +truncate -s10M webroot/filesystem.img +/sbin/mkfs.vfat webroot/filesystem.img +bmaptool create webroot/filesystem.img > webroot/filesystem.img.bmap +cat webroot/filesystem.img.bmap + +bmaptool copy webroot/filesystem.img filesystem.out +diff -s webroot/filesystem.img filesystem.out + +gzip -9n webroot/filesystem.img +bmaptool copy "http://127.0.0.1:$(cat httpd-port)/filesystem.img.gz" filesystem.out2 +diff -s filesystem.out filesystem.out2 + +kill "$httpd_pid" diff --git a/debian/tests/web-server.py b/debian/tests/web-server.py new file mode 100755 index 0000000..2f448c2 --- /dev/null +++ b/debian/tests/web-server.py @@ -0,0 +1,85 @@ +#!/usr/bin/python3 + +# Taken from Flatpak + +# Copyright 2017-2019 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# On Debian systems, the full text of the GNU Lesser General Public License +# version 2.1 can be found in the file '/usr/share/common-licenses/LGPL-2.1'. + +from wsgiref.handlers import format_date_time +from email.utils import parsedate +from calendar import timegm +import gzip +import sys +import time +import zlib +import os +from http import HTTPStatus +from urllib.parse import parse_qs +import http.server as http_server +from io import BytesIO +import sys + +class RequestHandler(http_server.SimpleHTTPRequestHandler): + def handle_tokens(self): + need_token_path = self.translate_path(self.path) + ".need_token" + if os.path.isfile(need_token_path): + with open(need_token_path, 'r') as content_file: + token_content = content_file.read() + token = None + auth = self.headers.get("Authorization") + if auth and auth.startswith("Bearer "): + token = auth[7:] + if token == None: + self.send_response(HTTPStatus.UNAUTHORIZED, "No token") + self.end_headers() + return True + if token != token_content: + self.send_response(HTTPStatus.UNAUTHORIZED, "Wrong token") + self.end_headers() + return True + return False + + def do_GET(self): + if self.handle_tokens(): + return None + return super().do_GET() + +def run(dir): + RequestHandler.protocol_version = "HTTP/1.0" + httpd = http_server.HTTPServer( ("127.0.0.1", 0), RequestHandler) + host, port = httpd.socket.getsockname()[:2] + with open("httpd-port", 'w') as file: + file.write("%d" % port) + with open("httpd-pid", 'w') as file: + file.write("%d" % os.getpid()) + try: + os.write(3, bytes("Started\n", 'utf-8')); + except: + pass + print("Serving HTTP on port %d" % port); + if dir: + os.chdir(dir) + httpd.serve_forever() + +if __name__ == '__main__': + dir = None + if len(sys.argv) >= 2 and len(sys.argv[1]) > 0: + dir = sys.argv[1] + + run(dir) diff --git a/debian/upstream/metadata b/debian/upstream/metadata new file mode 100644 index 0000000..8189e63 --- /dev/null +++ b/debian/upstream/metadata @@ -0,0 +1,4 @@ +Bug-Database: https://github.com/yoctoproject/bmaptool/issues +Bug-Submit: https://github.com/yoctoproject/bmaptool/issues/new +Repository: https://github.com/yoctoproject/bmaptool.git +Repository-Browse: https://github.com/yoctoproject/bmaptool diff --git a/debian/watch b/debian/watch new file mode 100644 index 0000000..4887e6b --- /dev/null +++ b/debian/watch @@ -0,0 +1,2 @@ +version=4 +https://github.com/yoctoproject/@PACKAGE@/tags .*/v@ANY_VERSION@@ARCHIVE_EXT@ diff --git a/pyproject.toml b/pyproject.toml index 14ae192..e3b8d7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,8 +5,8 @@ dynamic = ["version"] dependencies = [ "gpg >= 1.10.0", ] -required-python = ">= 3.8" -authors = [ +requires-python = ">= 3.8" +maintainers = [ {name = "Joshua Watt", email = "JPEWhacker@gmail.com"}, {name = "Trevor Woerner", email = "twoerner@gmail.com"}, {name = "Tim Orling", email = "ticotimo@gmail.com"}, @@ -47,3 +47,45 @@ build-backend = "hatchling.build" [tool.hatch.version] path = "src/bmaptool/CLI.py" + +[tool.hatch.build.targets.sdist] +exclude = [ + "/.github", + "/.tox", + "/contrib", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/bmaptool"] + +[tool.black] +line-length = 88 +skip-string-normalization = true +include = '\.py$' +exclude = ''' +/( + \.git + | \.mypy_cache + | \.tox + | \.venv + | build + | debian + | docs + | dist + | tests +)/ +''' + +[tool.pycln] +all = true + +[tool.isort] +line_length = 72 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true + +[tool.bandit] +exclude_dirs = ["contrib", "debian"]