Skip to content

Commit

Permalink
Fix zenodo test, drop explicit datatype mapping
Browse files Browse the repository at this point in the history
This isn't supported by the zenodo api anymore
  • Loading branch information
mvdbeek committed Oct 26, 2023
1 parent 913bf29 commit c474fbc
Show file tree
Hide file tree
Showing 11 changed files with 12 additions and 117 deletions.
2 changes: 1 addition & 1 deletion docs/commands/shed_init.rst
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ to manage repositories in a Galaxy Tool Shed.
--remote_repository_url TEXT Specify repository remote_repository_url for
.shed.yml.
--homepage_url TEXT Specify repository homepage_url for .shed.yml.
--category [Assembly|ChIP-seq|Climate Analysis|Combinatorial Selections|Computational chemistry|Constructive Solid Geometry|Convert Formats|Data Export|Data Managers|Data Source|Ecology|Entomology|Epigenetics|Fasta Manipulation|Fastq Manipulation|Flow Cytometry Analysis|Genome annotation|Genome editing|Genome-Wide Association Study|Genomic Interval Operations|GIS|Graphics|Imaging|InteractiveTools|Machine Learning|Metabolomics|Metagenomics|Micro-array Analysis|Molecular Dynamics|Nanopore|Next Gen Mappers|NLP|Ontology Manipulation|Phylogenetics|Proteomics|RNA|SAM|Sequence Analysis|Statistics|Structural Materials Analysis|Synthetic Biology|Systems Biology|Text Manipulation|Tool Dependency Packages|Tool Generators|Transcriptomics|Variant Analysis|Visualization|Web Services]
--category [Assembly|Astronomy|ChIP-seq|Climate Analysis|CLIP-seq|Combinatorial Selections|Computational chemistry|Constructive Solid Geometry|Convert Formats|Data Export|Data Managers|Data Source|Ecology|Entomology|Epigenetics|Fasta Manipulation|Fastq Manipulation|Flow Cytometry Analysis|Genome annotation|Genome editing|Genome-Wide Association Study|Genomic Interval Operations|GIS|Graphics|Imaging|InteractiveTools|Machine Learning|Materials science|Metabolomics|Metagenomics|Micro-array Analysis|Molecular Dynamics|Nanopore|Next Gen Mappers|NLP|Ontology Manipulation|Phylogenetics|Proteomics|RNA|SAM|Sequence Analysis|Statistics|Structural Materials Analysis|Synthetic Biology|Systems Biology|Text Manipulation|Tool Dependency Packages|Tool Generators|Transcriptomics|Variant Analysis|Visualization|Web Services]
Specify repository category for .shed.yml (may
specify multiple).
--owner TEXT Tool Shed repository owner (username).
Expand Down
2 changes: 0 additions & 2 deletions docs/commands/training_fill_data_library.rst
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,5 @@ Build training template from workflow.
--tutorial_name TEXT Name (directory name) of the tutorial to modify
[required]
--zenodo_link TEXT Zenodo URL with the input data
--datatypes PATH YAML file with the correspondance between Zenodo
extension and Galaxy datatypes
--help Show this message and exit.
2 changes: 0 additions & 2 deletions docs/commands/training_init.rst
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,6 @@ Build training template from workflow.
workflow
--workflow_id TEXT ID of the workflow on the Galaxy instance
--zenodo_link TEXT Zenodo URL with the input data
--datatypes PATH YAML file with the correspondance between
Zenodo extension and Galaxy datatypes
--galaxy_root DIRECTORY Root of development galaxy directory to
execute command with.
--galaxy_python_version [3|3.7|3.8|3.9|3.10|3.11]
Expand Down
12 changes: 1 addition & 11 deletions planemo/options.py
Original file line number Diff line number Diff line change
Expand Up @@ -1283,15 +1283,6 @@ def training_tutorial_name_req_option():
return planemo_option("--tutorial_name", required=True, help="Name (directory name) of the tutorial to modify")


def training_datatype_option():
return planemo_option(
"--datatypes",
type=click.Path(file_okay=True, resolve_path=True),
help="YAML file with the correspondance between Zenodo extension and Galaxy datatypes",
default="shared/datatypes.yaml",
)


def training_zenodo_option():
return planemo_option("--zenodo_link", help="Zenodo URL with the input data")

Expand Down Expand Up @@ -1322,15 +1313,14 @@ def training_tutorial_option():


def training_init_options():
return _compose(training_topic_option(), training_tutorial_option(), training_datatype_option())
return _compose(training_topic_option(), training_tutorial_option())


def training_fill_data_library_options():
return _compose(
training_topic_name_option(),
training_tutorial_name_req_option(),
training_zenodo_option(),
training_datatype_option(),
)


Expand Down
2 changes: 0 additions & 2 deletions planemo/training/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,6 @@ def check_topic_init_tuto(self):
# initiate the tutorial
self.tuto = Tutorial(training=self, topic=self.topic)
self.tuto.init_from_existing_tutorial(self.kwds["tutorial_name"])
if "datatypes" in self.kwds:
self.tuto.datatype_fp = self.kwds["datatypes"]
if "workflow" in self.kwds:
self.tuto.init_wf_fp = self.kwds["workflow"]
if "workflow_id" in self.kwds:
Expand Down
18 changes: 1 addition & 17 deletions planemo/training/tutorial.py
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,6 @@ def __init__(self, training, topic, name="new_tuto", title="The new tutorial", z
self.init_wf_id = None
self.hands_on = True
self.slides = False
self.datatype_fp = ""
self.set_dir_name()
self.init_data_lib()
self.body = templates.render(
Expand Down Expand Up @@ -309,7 +308,6 @@ def init_from_kwds(self, kwds):
self.init_wf_id = kwds["workflow_id"]
self.hands_on = kwds["hands_on"]
self.slides = kwds["slides"]
self.datatype_fp = kwds["datatypes"]
self.set_dir_name()
self.init_data_lib()

Expand Down Expand Up @@ -443,9 +441,7 @@ def get_files_from_zenodo(self):

files = []
for f in req_res["files"]:
file_dict = {"url": "", "src": "url", "ext": "", "info": self.zenodo_link}
if "type" in f:
file_dict["ext"] = get_galaxy_datatype(f["type"], self.datatype_fp)
file_dict = {"url": "", "src": "url", "ext": "auto", "info": self.zenodo_link}
if "links" not in f and "self" not in f["links"]:
raise ValueError("No link for file %s" % f)
file_dict["url"] = f["links"]["self"]
Expand Down Expand Up @@ -546,18 +542,6 @@ def create_tutorial(self, ctx):
handle.write("---\nlayout: faq-page\n---\n")


def get_galaxy_datatype(z_ext, datatype_fp):
"""Get the Galaxy datatype corresponding to a Zenodo file type."""
g_datatype = ""
datatypes = load_yaml(datatype_fp)
if z_ext in datatypes:
g_datatype = datatypes[z_ext]
if g_datatype == "":
g_datatype = "# Please add a Galaxy datatype or update the shared/datatypes.yaml file"
info(f"Get Galaxy datatypes: {z_ext} --> {g_datatype}")
return g_datatype


def get_zenodo_record(zenodo_link):
"""Get the content of a Zenodo record."""
# get the record in the Zenodo link
Expand Down
3 changes: 0 additions & 3 deletions tests/data/training_datatypes.yaml

This file was deleted.

39 changes: 1 addition & 38 deletions tests/test_cmd_training_fill_data_library.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,6 @@
"""Tests for the ``training_fill_data_library`` command."""
import os
import shutil

from .test_cmd_training_generate_from_wf import create_tutorial_dir
from .test_utils import (
CliTestCase,
TEST_DATA_DIR,
)
from .test_utils import CliTestCase


class CmdTrainingFillDataLibraryTestCase(CliTestCase):
Expand All @@ -29,20 +23,6 @@ def test_training_fill_data_library_command_tutorial_topic(self):
with self._isolate():
topic_n = "test"
tuto_n = "test"
datatype = os.path.join(TEST_DATA_DIR, "training_datatypes.yaml")
# not working
create_tutorial_dir(topic_n, tuto_n)
training_fill_data_library_command = [
"training_fill_data_library",
"--topic_name",
topic_n,
"--tutorial_name",
tuto_n,
"--datatypes",
datatype,
]
shutil.rmtree("topics")
self._check_exit_code(training_fill_data_library_command, exit_code=1)
# working
create_tutorial_dir(topic_n, tuto_n)
training_fill_data_library_command = [
Expand All @@ -51,8 +31,6 @@ def test_training_fill_data_library_command_tutorial_topic(self):
topic_n,
"--tutorial_name",
tuto_n,
"--datatypes",
datatype,
]
self._check_exit_code(training_fill_data_library_command, exit_code=0)

Expand All @@ -62,19 +40,6 @@ def test_training_fill_data_library_command_tutorial_zenodo(self):
topic_n = "test"
tuto_n = "test"
create_tutorial_dir(topic_n, tuto_n)
datatype = os.path.join(TEST_DATA_DIR, "training_datatypes.yaml")
# not working test
training_fill_data_library_command = [
"training_fill_data_library",
"--topic_name",
topic_n,
"--tutorial_name",
tuto_n,
"--zenodo_link",
"https://zenodo.org/record/1321885",
]
self._check_exit_code(training_fill_data_library_command, exit_code=1)
# working
training_fill_data_library_command = [
"training_fill_data_library",
"--topic_name",
Expand All @@ -83,7 +48,5 @@ def test_training_fill_data_library_command_tutorial_zenodo(self):
tuto_n,
"--zenodo_link",
"https://zenodo.org/record/1321885",
"--datatypes",
datatype,
]
self._check_exit_code(training_fill_data_library_command, exit_code=0)
14 changes: 0 additions & 14 deletions tests/test_cmd_training_init.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,18 +79,6 @@ def test_training_init_command_tutorial(self):
def test_training_init_command_tutorial_zenodo(self):
"""Test training_init command to create new tutorial with zenodo."""
with self._isolate():
datatype = os.path.join(TEST_DATA_DIR, "training_datatypes.yaml")
# not working test
training_init_command = [
"training_init",
"--topic_name",
"test",
"--tutorial_name",
"test",
"--zenodo_link",
"https://zenodo.org/record/1321885",
]
self._check_exit_code(training_init_command, exit_code=1)
# working
training_init_command = [
"training_init",
Expand All @@ -100,8 +88,6 @@ def test_training_init_command_tutorial_zenodo(self):
"test",
"--zenodo_link",
"https://zenodo.org/record/1321885",
"--datatypes",
datatype,
]
self._check_exit_code(training_init_command, exit_code=0)

Expand Down
3 changes: 0 additions & 3 deletions tests/test_training.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
TEST_DATA_DIR,
)

datatype_fp = os.path.join(TEST_DATA_DIR, "training_datatypes.yaml")
tuto_fp = os.path.join(TEST_DATA_DIR, "training_tutorial.md")
tuto_wo_zenodo_fp = os.path.join(TEST_DATA_DIR, "training_tutorial_wo_zenodo.md")
zenodo_link = "https://zenodo.org/record/1321885"
Expand All @@ -40,7 +39,6 @@
"workflow": None,
"workflow_id": None,
"zenodo_link": None,
"datatypes": os.path.join(TEST_DATA_DIR, "training_datatypes.yaml"),
"templates": None,
# planemo configuation
"conda_auto_init": True,
Expand Down Expand Up @@ -175,7 +173,6 @@ def test_training_check_topic_init_tuto() -> None:
create_existing_tutorial("existing_tutorial", tuto_fp, train.topic)
train.check_topic_init_tuto()
assert train.tuto.name == train.kwds["tutorial_name"]
assert train.tuto.datatype_fp
# clean after
shutil.rmtree(train.topics_dir)
shutil.rmtree("metadata")
Expand Down
32 changes: 8 additions & 24 deletions tests/test_training_tutorial.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
from planemo.training import Training
from planemo.training.topic import Topic
from planemo.training.tutorial import (
get_galaxy_datatype,
get_hands_on_boxes_from_local_galaxy,
get_hands_on_boxes_from_running_galaxy,
get_wf_inputs,
Expand All @@ -21,7 +20,6 @@
from .test_training import (
create_existing_tutorial,
CTX,
datatype_fp,
KWDS,
RUNNABLE,
tuto_fp,
Expand All @@ -36,20 +34,14 @@
training = Training(KWDS)


def test_get_galaxy_datatype() -> None:
"""Test :func:`planemo.training.tutorial.get_galaxy_datatype`."""
assert get_galaxy_datatype("csv", datatype_fp) == "csv"
assert get_galaxy_datatype("test", datatype_fp) == "strange_datatype"
assert "# Please add" in get_galaxy_datatype("unknown", datatype_fp)


def test_get_zenodo_record() -> None:
"""Test :func:`planemo.training.tutorial.get_zenodo_record`."""
z_record, req_res = get_zenodo_record(zenodo_link)
file_link_prefix = "https://zenodo.org/api/files/51a1b5db-ff05-4cda-83d4-3b46682f921f"
file_link_prefix = f"https://zenodo.org/api/records/{z_record}"
assert z_record == "1321885"
assert "files" in req_res
assert req_res["files"][0]["type"] in ["rdata", "csv"]
print(req_res["files"][0])
assert req_res["files"][0]["key"].endswith(".RData")
assert file_link_prefix in req_res["files"][0]["links"]["self"]
# check with wrong zenodo link
z_record, req_res = get_zenodo_record("https://zenodo.org/api/records/zenodooo")
Expand All @@ -59,10 +51,10 @@ def test_get_zenodo_record() -> None:
# using DOI
z_link = "https://doi.org/10.5281/zenodo.1321885"
z_record, req_res = get_zenodo_record(z_link)
file_link_prefix = "https://zenodo.org/api/files/51a1b5db-ff05-4cda-83d4-3b46682f921f"
file_link_prefix = f"https://zenodo.org/api/records/{z_record}"
assert z_record == "1321885"
assert "files" in req_res
assert req_res["files"][0]["type"] in ["rdata", "csv"]
assert req_res["files"][0]["key"].endswith(".RData")
assert file_link_prefix in req_res["files"][0]["links"]["self"]


Expand Down Expand Up @@ -135,7 +127,6 @@ def test_tutorial_init() -> None:
assert not tuto.slides
assert tuto.init_wf_id is None
assert tuto.init_wf_fp is None
assert tuto.datatype_fp == ""
assert "new_tuto" in tuto.dir
assert "## Sub-step with **My Tool**" in tuto.body
assert tuto.data_lib
Expand All @@ -157,7 +148,6 @@ def test_tutorial_init_from_kwds() -> None:
"workflow": WF_FP,
"workflow_id": "id",
"zenodo_link": None,
"datatypes": datatype_fp,
}
tuto = Tutorial(training=training, topic=topic)
tuto.init_from_kwds(kwds)
Expand All @@ -169,7 +159,6 @@ def test_tutorial_init_from_kwds() -> None:
assert tuto.slides
assert tuto.init_wf_id == "id"
assert tuto.init_wf_fp == WF_FP
assert tuto.datatype_fp == datatype_fp
assert "my_tuto" in tuto.dir


Expand Down Expand Up @@ -289,25 +278,21 @@ def test_tutorial_export_workflow_file() -> None:
def test_tutorial_get_files_from_zenodo() -> None:
"""Test :func:`planemo.training.tutorial.tutorial.get_files_from_zenodo`."""
tuto = Tutorial(training=training, topic=topic, zenodo_link=zenodo_link)
tuto.datatype_fp = datatype_fp
files, z_record = tuto.get_files_from_zenodo()
assert z_record == "1321885"
# test links
file_link_prefix = "https://zenodo.org/api/files/51a1b5db-ff05-4cda-83d4-3b46682f921f"
assert file_link_prefix in tuto.zenodo_file_links[0]
assert z_record in tuto.zenodo_file_links[0]
# test files dict
assert file_link_prefix in files[0]["url"]
assert z_record in files[0]["url"]
assert files[0]["src"] == "url"
assert files[0]["info"] == zenodo_link
assert "# Please add" in files[0]["ext"]
assert files[1]["ext"] == "csv"
assert ".csv" in files[1]["url"]


def test_tutorial_prepare_data_library_from_zenodo() -> None:
"""Test :func:`planemo.training.tutorial.tutorial.prepare_data_library_from_zenodo`."""
# without zenodo link
tuto = Tutorial(training=training, topic=topic)
tuto.datatype_fp = datatype_fp
os.makedirs(tuto.wf_dir)
tuto.prepare_data_library_from_zenodo()
assert os.path.exists(tuto.data_lib_fp)
Expand Down Expand Up @@ -383,7 +368,6 @@ def test_tutorial_create_tutorial() -> None:
"workflow": WF_FP,
"workflow_id": None,
"zenodo_link": zenodo_link,
"datatypes": datatype_fp,
}
)
tuto.create_tutorial(CTX)
Expand Down

0 comments on commit c474fbc

Please sign in to comment.