From 7d2d42bda3592d5a946d9696cb1da6fb28639cee Mon Sep 17 00:00:00 2001 From: Dustin Spicuzza Date: Wed, 1 Nov 2023 00:31:48 -0400 Subject: [PATCH] Remove opkg management, only download python from github --- robotpy_installer/installer.py | 233 +++++++------------------------ robotpy_installer/opkgrepo.py | 247 --------------------------------- robotpy_installer/utils.py | 16 +-- setup.cfg | 1 - 4 files changed, 57 insertions(+), 440 deletions(-) delete mode 100644 robotpy_installer/opkgrepo.py diff --git a/robotpy_installer/installer.py b/robotpy_installer/installer.py index 4f16007..780580c 100755 --- a/robotpy_installer/installer.py +++ b/robotpy_installer/installer.py @@ -7,6 +7,7 @@ import shutil import subprocess import sys +from urllib.parse import urlparse import typing import click @@ -16,22 +17,13 @@ from .version import version as __version__ from .cacheserver import CacheServer -from .errors import Error, SshExecError, OpkgError -from .opkgrepo import OpkgRepo +from .errors import Error, SshExecError from .sshcontroller import SshController, ssh_from_cfg +from .utils import _urlretrieve _WPILIB_YEAR = "2024" _IS_BETA = True -_OPKG_ARCH = "cortexa9-vfpv3" - - -_OPKG_FEEDS = [ - f"https://www.tortall.net/~robotpy/feeds/{_WPILIB_YEAR}", - f"https://download.ni.com/ni-linux-rt/feeds/academic/2023/arm/main/{_OPKG_ARCH}", - f"https://download.ni.com/ni-linux-rt/feeds/academic/2023/arm/extra/{_OPKG_ARCH}", -] - _ROBORIO_WHEELS = f"https://wpilib.jfrog.io/artifactory/api/pypi/wpilib-python-release-{_WPILIB_YEAR}/simple" _ROBORIO_IMAGES = [ @@ -48,6 +40,8 @@ _PIP_STUB_PATH = "/home/admin/rpip" +_PYTHON_IPK = "https://github.com/robotpy/roborio-python/releases/download/2024-3.12.0-r1/python312_3.12.0-r1_cortexa9-vfpv3.ipk" + logger = logging.getLogger("robotpy.installer") @@ -63,22 +57,6 @@ def log_startup(self) -> None: logger.info("RobotPy Installer %s", __version__) logger.info("-> caching files at %s", self.cache_root) - def get_opkg(self, ssl_context) -> OpkgRepo: - opkg = OpkgRepo(self.opkg_cache, _OPKG_ARCH, ssl_context) - for feed in _OPKG_FEEDS: - opkg.add_feed(feed) - return opkg - - def get_opkg_packages(self, no_index: bool, ssl_context): - opkg = self.get_opkg(ssl_context) - if not no_index: - opkg.update_packages() - - for feed in opkg.feeds: - for _, pkgdata in feed.pkgs.items(): - for pkg in pkgdata: - yield pkg - def get_ssh(self, robot: typing.Optional[str]) -> SshController: try: return ssh_from_cfg( @@ -289,87 +267,12 @@ def rm(installer: RobotpyInstaller, force: bool): shutil.rmtree(installer.cache_root) -# -# OPkg related commands -# - - -@installer.group() -def opkg(): - """ - Advanced RoboRIO package management tools - """ - - -@opkg.command(name="download") -@option("--no-index", is_flag=True, help="Only examine local cache") -@option("--use-certifi", is_flag=True, help="Use SSL certificates from certifi") -@option( - "-r", - "--requirements", - type=click.Path(exists=True), - multiple=True, - default=[], - help="Install from the given requirements file. This option can be used multiple times.", -) -@argument("packages", nargs=-1) -@pass_obj -def opkg_download( - installer: RobotpyInstaller, - no_index: bool, - use_certifi: bool, - requirements: typing.Tuple[str], - packages: typing.Sequence[str], -): - """ - Downloads opkg package to local cache - """ - - installer.log_startup() - - try: - opkg = installer.get_opkg(_make_ssl_context(use_certifi)) - if not no_index: - opkg.update_packages() - - if requirements: - packages = list(packages) + opkg.load_opkg_from_req(*requirements) - - if not packages: - raise ClickException("must specify packages to download") - - package_list = opkg.resolve_pkg_deps(packages) - for package in package_list: - opkg.download(package) - except OpkgError as e: - raise ClickException(str(e)) from e - - -@opkg.command(name="install") -@option( - "--force-reinstall", - is_flag=True, - help="When upgrading, reinstall all packages even if they are already up-to-date.", -) -@option("--ignore-image-version", is_flag=True) -@option( - "-r", - "--requirements", - type=click.Path(exists=True), - multiple=True, - default=[], - help="Install from the given requirements file. This option can be used multiple times.", -) -@argument("packages", nargs=-1, required=True) -@_common_ssh_options -@pass_obj def opkg_install( installer: RobotpyInstaller, force_reinstall: bool, - requirements: typing.Tuple[str], robot: str, ignore_image_version: bool, - packages: typing.Sequence[str], + packages: typing.Sequence[pathlib.Path], ): """ Installs opkg package on RoboRIO @@ -377,19 +280,16 @@ def opkg_install( installer.log_startup() - opkg = installer.get_opkg(None) + for package in packages: + if package.parent != installer.opkg_cache: + raise ValueError("internal error") + if not package.exists(): + raise ClickException(f"{package.name} has not been downloaded yet") # Write out the install script # -> we use a script because opkg doesn't have a good mechanism # to only install a package if it's not already installed opkg_files = [] - if requirements: - packages = list(packages) + opkg.load_opkg_from_req(*requirements) - - try: - packages = opkg.resolve_pkg_deps(packages) - except OpkgError as e: - raise ClickException(str(e)) with installer.get_ssh(robot) as ssh: cache = installer.start_cache(ssh) @@ -416,21 +316,18 @@ def opkg_install( ) for package in packages: - try: - pkg, fname = opkg.get_cached_pkg(package) - except OpkgError as e: - raise ClickException(str(e)) + pkgname, pkgversion, _ = package.name.split("_") opkg_script += "\n" + ( opkg_script_bit % { - "fname": basename(fname), - "name": pkg["Package"], - "version": pkg["Version"], + "fname": package.name, + "name": pkgname, + "version": pkgversion, } ) - opkg_files.append(fname) + opkg_files.append(package.name) # Finish it out opkg_script += "\n" + ( @@ -468,98 +365,66 @@ def opkg_install( show_disk_space(ssh) -@opkg.command(name="list") -@option("--no-index", is_flag=True, help="Only examine local cache") -@option("--use-certifi", is_flag=True, help="Use SSL certificates from certifi") -@pass_obj -def opkg_list(installer: RobotpyInstaller, no_index: bool, use_certifi: bool): - """ - List all packages in opkg database - """ +# +# python installation +# - data = set() - for pkg in installer.get_opkg_packages(no_index, _make_ssl_context(use_certifi)): - data.add("%(Package)s - %(Version)s" % pkg) - for v in sorted(data): - print(v) +def _get_python_ipk_path(installer: RobotpyInstaller) -> pathlib.Path: + parts = urlparse(_PYTHON_IPK) + return installer.opkg_cache / pathlib.PurePosixPath(parts.path).name -@opkg.command(name="search") -@option("--no-index", is_flag=True, help="Only examine local cache") +@installer.command() @option("--use-certifi", is_flag=True, help="Use SSL certificates from certifi") -@argument("search") @pass_obj -def opkg_search( - installer: RobotpyInstaller, no_index: bool, use_certifi: bool, search: str -): +def download_python(installer: RobotpyInstaller, use_certifi: bool): """ - Search opkg database for packages + Downloads Python to a folder to be installed """ + installer.opkg_cache.mkdir(parents=True, exist_ok=True) - # TODO: make this more intelligent... - data = set() - for pkg in installer.get_opkg_packages(no_index, _make_ssl_context(use_certifi)): - if search in pkg["Package"] or search in pkg.get("Description", ""): - data.add("%(Package)s - %(Version)s" % pkg) - for v in sorted(data): - print(v) + ipk_dst = _get_python_ipk_path(installer) + _urlretrieve(_PYTHON_IPK, ipk_dst, True, _make_ssl_context(use_certifi)) -@opkg.command(name="uninstall") -@argument("packages", nargs=-1, required=True) +@installer.command() @_common_ssh_options @pass_obj -def opkg_uninstall( +def install_python( installer: RobotpyInstaller, robot: str, ignore_image_version: bool, - packages: typing.Tuple[str], ): - installer.log_startup() - - with installer.get_ssh(robot) as ssh: - roborio_checks(ssh, ignore_image_version) - - package_list = " ".join(packages) - - with catch_ssh_error("removing packages"): - ssh.exec_cmd(f"opkg remove {package_list}", check=True, print_output=True) - - show_disk_space(ssh) - - -# -# python installation -# - - -@installer.command() -@option("--use-certifi", is_flag=True, help="Use SSL certificates from certifi") -@pass_context -def download_python(ctx: click.Context, use_certifi: bool): """ - Downloads Python to a folder to be installed + Installs Python on a RoboRIO. + + Requires download-python to be executed first. """ - ctx.forward( - opkg_download, packages=[_ROBOTPY_PYTHON_VERSION], use_certifi=use_certifi - ) + ipk_dst = _get_python_ipk_path(installer) + opkg_install(installer, False, robot, ignore_image_version, [ipk_dst]) @installer.command() @_common_ssh_options -@pass_context -def install_python( - ctx: click.Context, +@pass_obj +def uninstall_python( + installer: RobotpyInstaller, robot: str, ignore_image_version: bool, ): - """ - Installs Python on a RoboRIO. + """Uninstall Python from a RoboRIO""" + installer.log_startup() - Requires download-python to be executed first. - """ - ctx.forward(opkg_install, packages=[_ROBOTPY_PYTHON_VERSION]) + with installer.get_ssh(robot) as ssh: + roborio_checks(ssh, ignore_image_version) + + with catch_ssh_error("removing packages"): + ssh.exec_cmd( + f"opkg remove {_ROBOTPY_PYTHON_VERSION}", check=True, print_output=True + ) + + show_disk_space(ssh) # diff --git a/robotpy_installer/opkgrepo.py b/robotpy_installer/opkgrepo.py deleted file mode 100644 index a88d162..0000000 --- a/robotpy_installer/opkgrepo.py +++ /dev/null @@ -1,247 +0,0 @@ -import os -import string -from collections import OrderedDict -from dataclasses import dataclass -from distutils.version import LooseVersion -from functools import reduce as _reduce -from os.path import exists, join, basename -from typing import Dict, Iterable, List, Set, Sequence, Tuple - -from robotpy_installer.errors import OpkgError -from robotpy_installer.utils import _urlretrieve, md5sum - -Package = OrderedDict - - -@dataclass -class Feed: - url: str - db_fname: str - pkgs: Dict[str, List[Package]] - loaded: bool - - -class OpkgRepo(object): - """Simplistic OPkg Manager""" - - sys_packages = ["libc6"] - - def __init__(self, opkg_cache, arch: str, ssl_context): - self.feeds: List[Feed] = [] - self.opkg_cache = opkg_cache - self.arch = arch - self.ssl_context = ssl_context - if not exists(self.opkg_cache): - os.makedirs(self.opkg_cache) - self.pkg_dbs = join(self.opkg_cache, "Packages") - if not exists(self.pkg_dbs): - os.makedirs(self.pkg_dbs) - - def add_feed(self, url: str) -> None: - # Snippet from https://gist.github.com/seanh/93666 - valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits) - safe_url = "".join(c for c in url if c in valid_chars) - safe_url = safe_url.replace(" ", "_") - feed = Feed( - url, - db_fname=join(self.pkg_dbs, safe_url), - pkgs=OrderedDict(), - loaded=False, - ) - if exists(feed.db_fname): - self.load_package_db(feed) - feed.loaded = True - - self.feeds.append(feed) - - def update_packages(self) -> None: - for feed in self.feeds: - pkgurl = feed.url + "/Packages" - _urlretrieve(pkgurl, feed.db_fname, True, self.ssl_context) - self.load_package_db(feed) - - def load_package_db(self, feed: Feed) -> None: - # dictionary of lists of packages sorted by version - pkg: Package = OrderedDict() - with open(feed.db_fname, "r", encoding="utf-8") as fp: - for line in fp.readlines(): - line = line.strip() - if len(line) == 0: - self._add_pkg(pkg, feed) - pkg = OrderedDict() - else: - if ":" in line: - k, v = [i.strip() for i in line.split(":", 1)] - if k == "Version": - pkg[k] = LooseVersion(v) - else: - pkg[k] = v - - self._add_pkg(pkg, feed) - - # Finally, make sure all the packages are sorted by version - for pkglist in feed.pkgs.values(): - pkglist.sort(key=lambda p: p["Version"]) - - def _add_pkg(self, pkg: Package, feed: Feed) -> None: - if len(pkg) == 0 or pkg.get("Architecture", None) != self.arch: - return - # Add download url and fname - if "Filename" in pkg: - pkg["url"] = "/".join((feed.url, pkg["Filename"])) - - # Only retain one version of a package - pkgs = feed.pkgs.setdefault(pkg["Package"], []) - for old_pkg in pkgs: - if old_pkg["Version"] == pkg["Version"]: - old_pkg.clear() - old_pkg.update(pkg) - break - else: - pkgs.append(pkg) - - def get_pkginfo(self, name: str): - loaded = False - for feed in self.feeds: - loaded = loaded or feed.loaded - if name in feed.pkgs: - return feed.pkgs[name][-1] - - if loaded: - msg = "Package %s is not in the package list (did you misspell it?)" % name - else: - msg = "There are no package lists, did you download %s yet?" % name - - raise OpkgError(msg) - - def _get_pkg_fname(self, pkg: Package) -> str: - return join(self.opkg_cache, basename(pkg["Filename"])) - - def _get_pkg_deps(self, name: str) -> Set[str]: - info = self.get_pkginfo(name) - if "Depends" in info: - return { - dep - for dep in [ - dep.strip().split(" ", 1)[0] for dep in info["Depends"].split(",") - ] - if dep not in self.sys_packages - } - return set() - - def get_cached_pkg(self, name: str) -> Tuple: - """Returns the pkg, filename of a cached package""" - pkg = self.get_pkginfo(name) - fname = self._get_pkg_fname(pkg) - - if not exists(fname): - raise OpkgError("Package '%s' has not been downloaded" % name) - - if not md5sum(fname) == pkg["MD5Sum"]: - raise OpkgError("md5sum of package '%s' md5sum does not match" % name) - - return pkg, fname - - def resolve_pkg_deps(self, packages: Sequence[str]) -> List[str]: - """Given a list of package(s) desired to be installed, topologically - sorts them by dependencies and returns an ordered list of packages""" - - pkgs = {} - packages = list(packages) - - for pkg in packages: - if pkg in pkgs: - continue - deps = self._get_pkg_deps(pkg) - pkgs[pkg] = deps - packages.extend(deps) - - retval: List[str] = [] - for results in self._toposort(pkgs): - retval.extend(results) - - return retval - - @classmethod - def _toposort(cls, data: Dict[str, Set[str]]) -> Iterable[Set[str]]: - # Copied from https://bitbucket.org/ericvsmith/toposort/src/25b5894c4229cb888f77cf0c077c05e2464446ac/toposort.py?at=default - # -> Apache 2.0 license, Copyright 2014 True Blade Systems, Inc. - - # Special case empty input. - if len(data) == 0: - return - - # Copy the input so as to leave it unmodified. - data = data.copy() - - # Ignore self dependencies. - for k, v in data.items(): - v.discard(k) - # Find all items that don't depend on anything. - extra_items_in_deps = _reduce(set.union, data.values()) - set(data.keys()) - # Add empty dependences where needed. - data.update({item: set() for item in extra_items_in_deps}) - while True: - ordered = {item for item, dep in data.items() if len(dep) == 0} - if not ordered: - break - yield ordered - data = { - item: (dep - ordered) - for item, dep in data.items() - if item not in ordered - } - if len(data) != 0: - yield cls._modified_dfs(data) - - @staticmethod - def _modified_dfs(nodes: Dict[str, Set[str]]): - # this is a modified depth first search that does a best effort at - # a topological sort, but ignores cycles and keeps going on despite - # that. Only used if the topological sort fails. - retval = [] - visited = set() - - def _visit(n): - if n in visited: - return - - visited.add(n) - for m in nodes[n]: - _visit(m) - - retval.append(n) - - for item in nodes: - _visit(item) - - return retval - - def download(self, name: str) -> str: - pkg = self.get_pkginfo(name) - fname = self._get_pkg_fname(pkg) - - # Only download it if necessary - if not exists(fname) or not md5sum(fname) == pkg["MD5Sum"]: - _urlretrieve(pkg["url"], fname, True, self.ssl_context) - # Validate it - if md5sum(fname) != pkg["MD5Sum"]: - raise OpkgError("Downloaded package for %s md5sum does not match" % name) - - return fname - - def load_opkg_from_req(self, *files: str) -> List[str]: - """ - Pull the list of opkgs from a requirements.txt-like file - """ - opkgs = [] - # Loop through the passed in files to support multiple requirements files - for file in files: - with open(file, "r") as f: - for row in f: - # Ignore commented lines and empty lines - stripped = row.strip() - if stripped and not stripped.startswith("#"): - # Add the package to the list of packages (and remove leading and trailing whitespace) - opkgs.append(stripped) - return opkgs diff --git a/robotpy_installer/utils.py b/robotpy_installer/utils.py index 5aa0f58..292ea66 100644 --- a/robotpy_installer/utils.py +++ b/robotpy_installer/utils.py @@ -2,10 +2,10 @@ import hashlib import json import logging +import pathlib import socket import sys import urllib.request -from os.path import exists from robotpy_installer import __version__ from robotpy_installer.errors import Error @@ -25,7 +25,7 @@ def md5sum(fname): return md5.hexdigest() -def _urlretrieve(url, fname, cache, ssl_context): +def _urlretrieve(url, fname: pathlib.Path, cache: bool, ssl_context): # Get it print("Downloading", url) @@ -36,11 +36,11 @@ def _urlretrieve(url, fname, cache, ssl_context): cache_fname = None if cache: - cache_fname = fname + ".jmd" - if exists(fname) and exists(cache_fname): + cache_fname = fname.with_suffix(".jmd") + if fname.exists() and cache_fname.exists(): try: - with open(cache_fname) as fp: - md = json.load(fp) + with open(cache_fname) as cfp: + md = json.load(cfp) if md5sum(fname) == md["md5"]: etag = md.get("etag") last_modified = md.get("last-modified") @@ -72,7 +72,7 @@ def _reporthook(read, totalsize): ) as rfp: headers = rfp.info() - with open(fname, "wb") as fp: + with open(fname, "wb") as dfp: # Deal with header stuff size = -1 read = 0 @@ -84,7 +84,7 @@ def _reporthook(read, totalsize): if not block: break read += len(block) - fp.write(block) + dfp.write(block) _reporthook(read, size) if size >= 0 and read < size: diff --git a/setup.cfg b/setup.cfg index b041293..b6d8f4b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,6 @@ packages = find: install_requires = click paramiko - dataclasses; python_version < '3.7' setup_requires = setuptools_scm > 6 python_requires = >=3.6