Skip to content

Commit

Permalink
fix mulled v2 hash generation from strings
Browse files Browse the repository at this point in the history
the build information that may be contained in requirement versions is considered
as part of the version in some places of the code

- mulled container resolvers
- mulled-build-tool
- [planemo](https://github.com/galaxyproject/planemo/blob/b3e12a334e3d358ce30bf2c8c3f8d5a7a7e08136/planemo/conda.py#L116)

in others it isn't:

- mulled-hash
- mulled-build-files
- mulled_build

which are the files that use the `target_str_to_targets` function.

With this change the build info in version strings is always considered
as part of the version.

`mulled-build-files` is used in the multi-package-containers repo which
led to wrong hashes in up to approx 50 tools (check with `grep "=[^,]*=" combinations/* | wc -l`).
  • Loading branch information
bernt-matthias committed Jul 10, 2024
1 parent 5b3c4bd commit e7bca83
Show file tree
Hide file tree
Showing 3 changed files with 38 additions and 17 deletions.
11 changes: 4 additions & 7 deletions lib/galaxy/tool_util/deps/mulled/mulled_build.py
Original file line number Diff line number Diff line change
Expand Up @@ -352,7 +352,7 @@ def mull_targets(
with PrintProgress():
ret = involucro_context.exec_command(involucro_args)
if singularity:
# we can not remove this folder as it contains the image wich is owned by root
# we can not remove this folder as it contains the image which is owned by root
pass
# shutil.rmtree('./singularity_import')
return ret
Expand Down Expand Up @@ -531,12 +531,7 @@ def target_str_to_targets(targets_raw):
def parse_target(target_str):
if "=" in target_str:
package_name, version = target_str.split("=", 1)
build = None
if "=" in version:
version, build = version.split("=")
elif "--" in version:
version, build = version.split("--")
target = build_target(package_name, version, build)
target = build_target(package_name, version)
else:
target = build_target(target_str)
return target
Expand Down Expand Up @@ -584,6 +579,8 @@ def args_to_mull_targets_kwds(args):
kwds["singularity_image_dir"] = args.singularity_image_dir
if hasattr(args, "invfile"):
kwds["invfile"] = args.invfile
if hasattr(args, "base_image"):
kwds["base_image"] = args.base_image

kwds["involucro_context"] = context_from_args(args)

Expand Down
24 changes: 18 additions & 6 deletions lib/galaxy/tool_util/deps/mulled/mulled_build_tool.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,23 @@
from galaxy.tool_util.deps.conda_util import CondaTarget


def _mulled_build_tool(tool, args):
"""
>>> import doctest
>>> doctest.ELLIPSIS_MARKER = '-ignore-'
>>> import argparse
>>> _mulled_build_tool("test/functional/tools/mulled_example_multi_1.xml", argparse.Namespace(dry_run=True, base_image="does-not-matter-here-but-test-is-fast", command="build", verbose=True, involucro_path="./involucro")) # doctest: +ELLIPSIS
-ignore- REPO=quay.io/biocontainers/mulled-v2-8186960447c5cb2faa697666dc1e6d919ad23f3e:a6419f25efff953fc505dbd5ee734856180bb619-0 -ignore-
>>> _mulled_build_tool("test/functional/tools/mulled_example_multi_2.xml", argparse.Namespace(dry_run=True, base_image="does-not-matter-here-but-test-is-fast", command="build", verbose=True, involucro_path="./involucro")) # doctest: +ELLIPSIS
-ignore- REPO=quay.io/biocontainers/mulled-v2-8186960447c5cb2faa697666dc1e6d919ad23f3e:8e86df67d257ce6494ae12b2c60e1b94025ea529-0 -ignore-
"""
tool_source = get_tool_source(tool)
requirements, *_ = tool_source.parse_requirements_and_containers()
targets = requirements_to_mulled_targets(requirements)
kwds = args_to_mull_targets_kwds(args)
mull_targets(targets, **kwds)


def main(argv=None) -> None:
"""Main entry-point for the CLI tool."""
parser = arg_parser(argv, globals())
Expand All @@ -35,12 +52,7 @@ def main(argv=None) -> None:
parser.add_argument("command", metavar="COMMAND", help="Command (build-and-test, build, all)")
parser.add_argument("tool", metavar="TOOL", default=None, help="Path to tool to build mulled image for.")
args = parser.parse_args()
tool_source = get_tool_source(args.tool)
requirements, *_ = tool_source.parse_requirements_and_containers()
targets = requirements_to_mulled_targets(requirements)
kwds = args_to_mull_targets_kwds(args)
mull_targets(targets, **kwds)

_mulled_build_tool(args.tool, args)

def requirements_to_mulled_targets(requirements) -> List["CondaTarget"]:
"""Convert Galaxy's representation of requirements into a list of CondaTarget objects.
Expand Down
20 changes: 16 additions & 4 deletions lib/galaxy/tool_util/deps/mulled/mulled_hash.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,29 @@
)


def _mulled_hash(hash, targets):
"""
>>> _mulled_hash(hash="v2", targets="samtools=1.3.1,bedtools=2.26.0")
'mulled-v2-8186960447c5cb2faa697666dc1e6d919ad23f3e:a6419f25efff953fc505dbd5ee734856180bb619'
>>> _mulled_hash(hash="v2", targets="samtools=1.3.1=h9071d68_10,bedtools=2.26.0=0")
'mulled-v2-8186960447c5cb2faa697666dc1e6d919ad23f3e:8e86df67d257ce6494ae12b2c60e1b94025ea529'
"""
targets = target_str_to_targets(targets)
image_name = v2_image_name if hash == "v2" else v1_image_name
return image_name(targets)


def main(argv=None):
"""Main entry-point for the CLI tool."""
"""
Main entry-point for the CLI tool.
"""
parser = arg_parser(argv, globals())
parser.add_argument(
"targets", metavar="TARGETS", default=None, help="Comma-separated packages for calculating the mulled hash."
)
parser.add_argument("--hash", dest="hash", choices=["v1", "v2"], default="v2")
args = parser.parse_args()
targets = target_str_to_targets(args.targets)
image_name = v2_image_name if args.hash == "v2" else v1_image_name
print(image_name(targets))
print(_mulled_hash(args.hash, args.targets))


__all__ = ("main",)
Expand Down

0 comments on commit e7bca83

Please sign in to comment.