Merge pull request #112 from adfinis/jlf/fix-improve-toml-converter

fix: improve toml converter

.github/workflows/main.yml

@@ -60,7 +60,7 @@ jobs:
 
  - name: Run tests
  run: |
- make test
+ HYPOTHESIS_PROFILE=ci make test
 
  package:
  runs-on: ubuntu-latest

.github/workflows/pull-requests.yml

@@ -30,4 +30,4 @@ jobs:
 
  - name: Run tests
  run: |
- make test
+ HYPOTHESIS_PROFILE=ci make test

Makefile

@@ -46,7 +46,7 @@ mypy: poetry-install
 
 .PHONY: pytest
 pytest: poetry-install ## run pytest
- @docker compose exec testing poetry run pytest -vv --cov
+ @docker compose exec testing poetry run sh -c "HYPOTHESIS_PROFILE=$(HYPOTHESIS_PROFILE) pytest -vv --cov"
 
 .PHONY: format
 format: poetry-install ## format code with ruff

pyaptly/__init__.py

@@ -9,10 +9,10 @@
 def init_hypothesis():
  """Initialize hypothesis profile if hypothesis is available."""
  try: # pragma: no cover
- if "HYPOTHESIS_PROFILE" in os.environ:
+ if "HYPOTHESIS_PROFILE" in os.environ and os.environ["HYPOTHESIS_PROFILE"]:
  from hypothesis import settings
 
- settings.register_profile("ci", settings(max_examples=10000))
+ settings.register_profile("ci", settings(max_examples=500))
  settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "default"))
  except (ImportError, AttributeError): # pragma: no cover
  pass

pyaptly/cli.py

@@ -167,7 +167,7 @@ def publish(**kwargs):
  publish.publish(cfg, args=fake_args)
 
 
-@cli.command(help="convert yaml- to toml-comfig")
+@cli.command()
 @click.option("--debug/--no-debug", "-d/-nd", default=False, type=bool)
 @click.argument(
  "yaml_path",
@@ -205,3 +205,43 @@ def yaml_to_toml(yaml_path: Path, toml_path: Path, add_defaults: bool, debug: bo
  toml_path,
  add_defaults=add_defaults,
  )
+
+
+@cli.command()
+@click.option("--debug/--no-debug", "-d/-nd", default=False, type=bool)
+@click.argument(
+ "in_path",
+ type=click.Path(
+ file_okay=True,
+ dir_okay=False,
+ exists=True,
+ readable=True,
+ path_type=Path,
+ ),
+)
+@click.argument(
+ "toml_path",
+ type=click.Path(
+ file_okay=True,
+ dir_okay=False,
+ exists=False,
+ writable=True,
+ path_type=Path,
+ ),
+)
+@click.option(
+ "-a/-na",
+ "--add-defaults/--no-add-defaults",
+ type=bool,
+ default=False,
+ help="Add default values to fields if missing",
+)
+def toml_to_toml(in_path: Path, toml_path: Path, add_defaults: bool, debug: bool):
+ """Convert pyaptly config files from toml to toml."""
+ from . import config_file
+
+ config_file.toml_to_toml(
+ in_path,
+ toml_path,
+ add_defaults=add_defaults,
+ )

pyaptly/config_file.py

@@ -1,10 +1,14 @@
 """Handling pyaptly config-files."""
 
+# TODO: remove this as soon as most people have converted their config.
+
 from pathlib import Path
 
-import tomli_w
+import tomli
 import yaml
 
+from pyaptly import tomli_w
+
 
 def yaml_to_toml(yaml_path: Path, toml_path: Path, *, add_defaults: bool = False):
  """Convert pyaptly config files from yaml to toml.
@@ -19,6 +23,19 @@ def yaml_to_toml(yaml_path: Path, toml_path: Path, *, add_defaults: bool = False
  tomli_w.dump(config, tf)
 
 
+def toml_to_toml(in_path: Path, toml_path: Path, *, add_defaults: bool = False):
+ """Convert pyaptly config files from toml to toml.
+
+ Setting `add_defaults=True` will set common default during conversion.
+ """
+ with in_path.open("rb") as nf:
+ with toml_path.open("wb") as tf:
+ config = tomli.load(nf)
+ if add_defaults:
+ add_default_to_config(config)
+ tomli_w.dump(config, tf)
+
+
 def add_default_to_config(config):
  """Set common default in config if the fields are missing."""
  if "mirror" in config:

pyaptly/tests/debug.toml

@@ -1,31 +1,64 @@
 [mirror.trusty]
-components = [ "main", "multiverse", "restricted", "universe",]
-architectures = [ "amd64", "i386",]
+components = [
+ "main",
+ "multiverse",
+ "restricted",
+ "universe",
+]
+architectures = [
+ "amd64",
+ "i386",
+]
 distribution = "trusty"
 sources = true
 udeb = true
 archive = "http://ch.archive.ubuntu.com/ubuntu/"
-gpg-keys = [ "40976EAF437D05B5", "3B4FE6ACC0B21F32" ]
+gpg-keys = [
+ "40976EAF437D05B5",
+ "3B4FE6ACC0B21F32",
+]
 keyserver = "keyserver.ubuntu.com"
 
 [mirror.trusty-updates]
-components = [ "main", "multiverse", "restricted", "universe",]
-architectures = [ "amd64", "i386",]
+components = [
+ "main",
+ "multiverse",
+ "restricted",
+ "universe",
+]
+architectures = [
+ "amd64",
+ "i386",
+]
 distribution = "trusty-updates"
 sources = true
 udeb = true
 archive = "http://ch.archive.ubuntu.com/ubuntu/"
-gpg-keys = [ "40976EAF437D05B5", "3B4FE6ACC0B21F32" ]
+gpg-keys = [
+ "40976EAF437D05B5",
+ "3B4FE6ACC0B21F32",
+]
 keyserver = "keyserver.ubuntu.com"
 
 [mirror.trusty-backports]
-components = [ "main", "multiverse", "restricted", "universe",]
-architectures = [ "amd64", "i386",]
+components = [
+ "main",
+ "multiverse",
+ "restricted",
+ "universe",
+]
+architectures = [
+ "amd64",
+ "i386",
+]
 distribution = "trusty-backports"
 sources = true
 udeb = true
 archive = "http://ch.archive.ubuntu.com/ubuntu/"
-gpg-keys = [ "40976EAF437D05B5", "3B4FE6ACC0B21F32" ]
+gpg-keys = [
+ "40976EAF437D05B5",
+ "3B4FE6ACC0B21F32",
+]
 keyserver = "keyserver.ubuntu.com"
 
 [snapshot.trusty-latest]

pyaptly/tests/mirror-basic.toml

@@ -1,12 +1,16 @@
 [mirror.fakerepo01]
 max-tries = 2
 archive = "http://localhost:3123/fakerepo01"
-gpg-keys = [ "2841988729C7F3FF",]
+gpg-keys = [
+ "2841988729C7F3FF",
+]
 components = "main"
 distribution = "main"
 
 [mirror.fakerepo02]
 archive = "http://localhost:3123/fakerepo02"
-gpg-keys = [ "2841988729C7F3FF",]
+gpg-keys = [
+ "2841988729C7F3FF",
+]
 components = "main"
 distribution = "main"

pyaptly/tests/mirror-extra.toml

@@ -1,6 +1,10 @@
 [mirror.fakerepo03]
 archive = "http://localhost:3123/fakerepo03"
-gpg-keys = [ "EC54D33E5B5EBE98",]
-gpg-urls = [ "http://localhost:3123/keys/test02.key",]
+gpg-keys = [
+ "EC54D33E5B5EBE98",
+]
+gpg-urls = [
+ "http://localhost:3123/keys/test02.key",
+]
 components = "main"
 distribution = "main"

pyaptly/tests/publish-current.toml

@@ -1,13 +1,56 @@
+[[publish."fakerepo01/current"]]
+distribution = "stable"
+architectures = [
+ "amd64",
+]
+components = [
+ "main",
+]
+snapshots = [
+ "fakerepo01-current",
+]
+automatic-update = true
+
+[[publish."fakerepo02/current"]]
+distribution = "stable"
+architectures = [
+ "amd64",
+]
+components = [
+ "main",
+]
+snapshots = [
+ "fakerepo02-current",
+]
+automatic-update = true
+
+[[publish."fake/current"]]
+distribution = "stable"
+architectures = [
+ "amd64",
+]
+components = [
+ "main",
+]
+snapshots = [
+ "fake-current",
+]
+automatic-update = true
+
 [mirror.fakerepo01]
 max-tries = 2
 archive = "http://localhost:3123/fakerepo01"
-gpg-keys = [ "2841988729C7F3FF",]
+gpg-keys = [
+ "2841988729C7F3FF",
+]
 components = "main"
 distribution = "main"
 
 [mirror.fakerepo02]
 archive = "http://localhost:3123/fakerepo02"
-gpg-keys = [ "2841988729C7F3FF",]
+gpg-keys = [
+ "2841988729C7F3FF",
+]
 components = "main"
 distribution = "main"
 
@@ -18,27 +61,7 @@ mirror = "fakerepo01"
 mirror = "fakerepo02"
 
 [snapshot.fake-current]
-merge = [ "fakerepo01-current", "fakerepo02-current",]
-
-[publish]
-[[publish."fakerepo01/current"]]
-distribution = "stable"
-architectures = [ "amd64",]
-components = [ "main",]
-snapshots = [ "fakerepo01-current",]
-automatic-update = true
-
-[[publish."fakerepo02/current"]]
-distribution = "stable"
-architectures = [ "amd64",]
-components = [ "main",]
-snapshots = [ "fakerepo02-current",]
-automatic-update = true
-
-[[publish."fake/current"]]
-distribution = "stable"
-architectures = [ "amd64",]
-components = [ "main",]
-snapshots = [ "fake-current",]
-automatic-update = true
-
+merge = [
+ "fakerepo01-current",
+ "fakerepo02-current",
+]

pyaptly/tests/publish-previous.toml

@@ -1,34 +1,31 @@
 [mirror.fakerepo01]
 max-tries = 2
 archive = "http://localhost:3123/fakerepo01"
-gpg-keys = [ "2841988729C7F3FF",]
+gpg-keys = [
+ "2841988729C7F3FF",
+]
 components = "main"
 distribution = "main"
 
 [mirror.fakerepo02]
 archive = "http://localhost:3123/fakerepo02"
-gpg-keys = [ "2841988729C7F3FF",]
+gpg-keys = [
+ "2841988729C7F3FF",
+]
 components = "main"
 distribution = "main"
 
 [snapshot."fakerepo01-%T"]
 mirror = "fakerepo01"
+timestamp = { time = "00:00" }
 
 [snapshot."fakerepo02-%T"]
 mirror = "fakerepo02"
-
-[snapshot."fakerepo01-%T".timestamp]
-time = "00:00"
-
-[snapshot."fakerepo02-%T".timestamp]
-time = "00:00"
-repeat-weekly = "sat"
+timestamp = { time = "00:00", repeat-weekly = "sat" }
 
 [snapshot."superfake-%T"]
 merge = [
  { name = "fakerepo01-%T", timestamp = "previous" },
  { name = "fakerepo02-%T", timestamp = 0 },
 ]
-
-[snapshot."superfake-%T".timestamp]
-time = "00:00"
+timestamp = { time = "00:00" }