diff --git a/grand_challenge_forge/forge.py b/grand_challenge_forge/forge.py index b791a21..804c6ca 100644 --- a/grand_challenge_forge/forge.py +++ b/grand_challenge_forge/forge.py @@ -13,11 +13,11 @@ from grand_challenge_forge.generation_utils import ( ci_to_civ, create_civ_stub_file, - enrich_phase_context, + extract_slug, ) from grand_challenge_forge.schemas import validate_pack_context from grand_challenge_forge.utils import cookiecutter_context as cc -from grand_challenge_forge.utils import extract_slug, remove_j2_suffix +from grand_challenge_forge.utils import remove_j2_suffix logger = logging.getLogger(__name__) @@ -100,13 +100,15 @@ def generate_upload_to_archive_script( ): context = deepcopy(context) - enrich_phase_context(context) - - # Cannot use filters in directory names so generate it here - archive_slug = extract_slug(context["phase"]["archive"]["url"]) - context["phase"]["archive"]["slug"] = archive_slug + # Cannot always use filters in directory names so generate it here + context["phase"]["archive"]["slug"] = extract_slug( + context["phase"]["archive"]["url"] + ) - script_dir = output_directory / f"upload-to-archive-{archive_slug}" + script_dir = ( + output_directory + / f"upload-to-archive-{context['phase']['archive']['slug']}" + ) # Map the expected case, but only create after the script expected_cases, create_files_func = _gen_expected_archive_cases( @@ -161,14 +163,6 @@ def create_files(): def generate_example_algorithm( context, output_directory, quality_control_registry=None ): - context = deepcopy(context) - - enrich_phase_context(context) - - # Cannot use filters in directory names so generate it here - archive_slug = extract_slug(context["phase"]["archive"]["url"]) - context["phase"]["archive"]["slug"] = archive_slug - algorithm_dir = generate_files( repo_dir=PARTIALS_PATH / "example-algorithm", context=cc(context), @@ -201,9 +195,6 @@ def quality_check(): def generate_example_evaluation( context, output_directory, quality_control_registry=None ): - context = deepcopy(context) - enrich_phase_context(context) - evaluation_dir = generate_files( repo_dir=PARTIALS_PATH / "example-evaluation-method", context=cc(context), diff --git a/grand_challenge_forge/generation_utils.py b/grand_challenge_forge/generation_utils.py index 4afe08d..8ff7929 100644 --- a/grand_challenge_forge/generation_utils.py +++ b/grand_challenge_forge/generation_utils.py @@ -1,4 +1,5 @@ import os +import re import shutil import uuid from pathlib import Path @@ -7,33 +8,42 @@ RESOURCES_PATH = SCRIPT_PATH / "resources" -def enrich_phase_context(context): - """Enriches the "phase" value of the context to simplify templating""" - phase_context = context["phase"] +def is_json(component_interface): + return component_interface["relative_path"].endswith(".json") - for ci in [ - *phase_context["inputs"], - *phase_context["outputs"], - ]: - ci["is_json"] = ci["relative_path"].endswith(".json") - ci["is_image"] = ci["super_kind"] == "Image" - ci["is_file"] = ci["super_kind"] == "File" and not ci[ - "relative_path" - ].endswith(".json") - for _type in ["json", "image", "file"]: - for in_out in ["input", "output"]: - phase_context[f"has_{in_out}_{_type}"] = any( - ci[f"is_{_type}"] for ci in phase_context[f"{in_out}s"] - ) +def is_image(component_interface): + return component_interface["super_kind"] == "Image" + + +def is_file(component_interface): + return component_interface[ + "super_kind" + ] == "File" and not component_interface["relative_path"].endswith(".json") + + +def extract_slug(url): + # Define a regex pattern to match the slug in the URL + pattern = r"/([^/]+)/*$" + + # Use re.search to find the match + match = re.search(pattern, url) + + # If a match is found, extract and return the slug + if match: + slug = match.group(1) + return slug + else: + # Return None or handle the case where no match is found + return None def create_civ_stub_file(*, target_dir, component_interface): """Creates a stub based on a component interface""" target_dir.parent.mkdir(parents=True, exist_ok=True) - if component_interface["is_json"]: + if is_json(component_interface): src = RESOURCES_PATH / "example.json" - elif component_interface["is_image"]: + elif is_image(component_interface): target_dir = target_dir / f"{str(uuid.uuid4())}.mha" target_dir.parent.mkdir(parents=True, exist_ok=True) src = RESOURCES_PATH / "example.mha" diff --git a/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/inference.py.j2 b/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/inference.py.j2 index ca44600..7030a57 100644 --- a/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/inference.py.j2 +++ b/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/inference.py.j2 @@ -18,10 +18,10 @@ Any container that shows the same behavior will do, this is purely an example of Happy programming! """ from pathlib import Path -{% if cookiecutter.phase.has_input_json or cookiecutter.phase.has_output_json -%} +{% if cookiecutter.phase.inputs | has_json or cookiecutter.phase.outputs | has_json -%} import json {%- endif %} -{% if cookiecutter.phase.has_input_image or cookiecutter.phase.has_output_image -%} +{% if cookiecutter.phase.inputs | has_image or cookiecutter.phase.outputs | has_image -%} from glob import glob import SimpleITK import numpy @@ -37,12 +37,12 @@ def run(): # Read the input {% for ci in cookiecutter.phase.inputs -%} {% set py_slug = ci.slug | replace("-", "_") -%} - {% if ci.is_image -%} + {% if ci | is_image -%} {{ py_slug }} = load_image_file_as_array( location=INPUT_PATH / "{{ ci.relative_path }}", ) {% endif -%} - {% if ci.is_json -%} + {% if ci | is_json -%} {{ py_slug }} = load_json_file( location=INPUT_PATH / "{{ ci.relative_path }}", ) @@ -57,28 +57,28 @@ def run(): # For now, let us set make bogus predictions {%- for ci in cookiecutter.phase.outputs %} {{ ci.slug | replace("-", "_")}} = - {%- if ci.is_image %} numpy.eye(4, 2) - {%- elif ci.is_json %} {"content": "should match the required format"} - {%- elif ci.is_file %} "content: should match the required format" + {%- if ci | is_image %} numpy.eye(4, 2) + {%- elif ci | is_json %} {"content": "should match the required format"} + {%- elif ci | is_file %} "content: should match the required format" {% endif %} {%- endfor %} # Save your output {% for ci in cookiecutter.phase.outputs -%} {% set py_slug = ci.slug | replace("-", "_") -%} - {% if ci.is_image -%} + {% if ci | is_image -%} write_array_as_image_file( location=OUTPUT_PATH / "{{ ci.relative_path }}", array={{ py_slug }}, ) {% endif -%} - {% if ci.is_json -%} + {% if ci | is_json -%} write_json_file( location=OUTPUT_PATH / "{{ ci.relative_path }}", content={{ py_slug }} ) {% endif -%} - {% if ci.is_file -%} + {% if ci | is_file -%} write_file( location=OUTPUT_PATH / "{{ ci.relative_path }}", content={{ py_slug }} @@ -86,7 +86,7 @@ def run(): {% endif -%} {% endfor %} return 0 -{%- if cookiecutter.phase.has_input_json %} +{%- if cookiecutter.phase.inputs | has_json %} def load_json_file(*, location): @@ -94,7 +94,7 @@ def load_json_file(*, location): with open(location, 'r') as f: return json.loads(f.read()) {%- endif %} -{%- if cookiecutter.phase.has_output_json %} +{%- if cookiecutter.phase.outputs | has_json %} def write_json_file(*, location, content): @@ -102,7 +102,7 @@ def write_json_file(*, location, content): with open(location, 'w') as f: f.write(json.dumps(content, indent=4)) {%- endif %} -{%- if cookiecutter.phase.has_input_image %} +{%- if cookiecutter.phase.inputs | has_image %} def load_image_file_as_array(*, location): @@ -113,7 +113,7 @@ def load_image_file_as_array(*, location): # Convert it to a Numpy array return SimpleITK.GetArrayFromImage(result) {%- endif %} -{%- if cookiecutter.phase.has_output_image %} +{%- if cookiecutter.phase.outputs | has_image %} def write_array_as_image_file(*, location, array): @@ -126,7 +126,7 @@ def write_array_as_image_file(*, location, array): useCompression=True, ) {%- endif %} -{%- if cookiecutter.phase.has_input_file %} +{%- if cookiecutter.phase.inputs | has_file %} # Note to the challenge hosts: @@ -137,7 +137,7 @@ def load_file(*, location): with open(location) as f: return f.read() {%- endif %} -{%- if cookiecutter.phase.has_output_file %} +{%- if cookiecutter.phase.outputs | has_file %} # Note to the challenge hosts: diff --git a/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/requirements.txt.j2 b/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/requirements.txt.j2 index 7a2e8eb..97a0c26 100644 --- a/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/requirements.txt.j2 +++ b/grand_challenge_forge/partials/example-algorithm/example-algorithm{{cookiecutter._}}/requirements.txt.j2 @@ -1,4 +1,4 @@ -{% if cookiecutter.phase.has_input_image or cookiecutter.phase.has_output_image -%} +{% if cookiecutter.phase.inputs | has_image or cookiecutter.phase.outputs | has_image -%} SimpleITK numpy {%- endif %} diff --git a/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/evaluate.py.j2 b/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/evaluate.py.j2 index e699798..1e37c42 100644 --- a/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/evaluate.py.j2 +++ b/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/evaluate.py.j2 @@ -18,7 +18,7 @@ Any container that shows the same behavior will do, this is purely an example of Happy programming! """ import json -{% if cookiecutter.phase.has_input_image -%} +{% if cookiecutter.phase.inputs | has_image -%} from glob import glob import SimpleITK {%- endif %} @@ -81,17 +81,17 @@ def process(job): # Secondly, read the results {% for ci in cookiecutter.phase.outputs -%} {% set py_slug = ci.slug | replace("-", "_") -%} - {% if ci.is_image -%} + {% if ci | is_image -%} {{ py_slug }} = load_image_file( location={{ py_slug }}_location, ) {% endif -%} - {% if ci.is_json -%} + {% if ci | is_json -%} {{ py_slug }} = load_json_file( location={{ py_slug }}_location, ) {% endif -%} - {% if ci.is_file -%} + {% if ci | is_file -%} {{ py_slug }} = load_file( location={{ py_slug }}_location, ) @@ -102,7 +102,7 @@ def process(job): # Thirdly, retrieve the input image name to match it with an image in your ground truth {% for ci in cookiecutter.phase.inputs -%} {% set py_slug = ci.slug | replace("-", "_") -%} - {% if ci.is_image -%} + {% if ci | is_image -%} {{ py_slug }}_image_name = get_image_name( values=job["inputs"], slug="{{ ci.slug }}", @@ -138,7 +138,7 @@ def read_predictions(): # The prediction file tells us the location of the users' predictions with open(INPUT_DIRECTORY / "predictions.json") as f: return json.loads(f.read()) -{%- if cookiecutter.phase.has_input_image or cookiecutter.phase.has_output_image %} +{%- if cookiecutter.phase.inputs | has_image or cookiecutter.phase.outputs | has_image %} def get_image_name(*, values, slug): @@ -164,7 +164,7 @@ def get_file_location(*, job_pk, values, slug): # Where a job's output file will be located in the evaluation container relative_path = get_interface_relative_path(values=values, slug=slug) return INPUT_DIRECTORY / job_pk / "output" / relative_path -{%- if cookiecutter.phase.has_output_json %} +{%- if cookiecutter.phase.outputs | has_json %} def load_json_file(*, location): @@ -172,7 +172,7 @@ def load_json_file(*, location): with open(location) as f: return json.loads(f.read()) {%- endif %} -{%- if cookiecutter.phase.has_output_image %} +{%- if cookiecutter.phase.outputs | has_image %} def load_image_file(*, location): @@ -183,7 +183,7 @@ def load_image_file(*, location): # Convert it to a Numpy array return SimpleITK.GetArrayFromImage(result) {%- endif %} -{%- if cookiecutter.phase.has_output_file %} +{%- if cookiecutter.phase.outputs | has_file %} def load_file(*, location): diff --git a/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/requirements.txt.j2 b/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/requirements.txt.j2 index 7a2e8eb..97a0c26 100644 --- a/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/requirements.txt.j2 +++ b/grand_challenge_forge/partials/example-evaluation-method/example-evaluation-method{{cookiecutter._}}/requirements.txt.j2 @@ -1,4 +1,4 @@ -{% if cookiecutter.phase.has_input_image or cookiecutter.phase.has_output_image -%} +{% if cookiecutter.phase.inputs | has_image or cookiecutter.phase.outputs | has_image -%} SimpleITK numpy {%- endif %} diff --git a/grand_challenge_forge/partials/filters.py b/grand_challenge_forge/partials/filters.py index d7493b3..bd63558 100644 --- a/grand_challenge_forge/partials/filters.py +++ b/grand_challenge_forge/partials/filters.py @@ -1,8 +1,48 @@ from cookiecutter.utils import simple_filter -from grand_challenge_forge.utils import extract_slug as util_extract_slug +from grand_challenge_forge import generation_utils +custom_filters = [] -@simple_filter + +def register_simple_filter(func): + func = simple_filter(func) + custom_filters.append( + f"grand_challenge_forge.partials.filters.{func.__name__}" + ) + return func + + +@register_simple_filter def extract_slug(url): - return util_extract_slug(url) + return generation_utils.extract_slug(url) + + +@register_simple_filter +def is_json(arg): + return generation_utils.is_json(arg) + + +@register_simple_filter +def has_json(arg): + return any(generation_utils.is_json(item) for item in arg) + + +@register_simple_filter +def is_image(arg): + return generation_utils.is_image(arg) + + +@register_simple_filter +def has_image(arg): + return any(generation_utils.is_image(item) for item in arg) + + +@register_simple_filter +def is_file(arg): + return generation_utils.is_file(arg) + + +@register_simple_filter +def has_file(arg): + return any(generation_utils.is_file(item) for item in arg) diff --git a/grand_challenge_forge/partials/upload-to-archive-script/upload-to-archive-{{cookiecutter.phase.archive.slug}}/upload_files.py.j2 b/grand_challenge_forge/partials/upload-to-archive-script/upload-to-archive-{{cookiecutter.phase.archive.slug}}/upload_files.py.j2 index 5482782..ff3e8e4 100644 --- a/grand_challenge_forge/partials/upload-to-archive-script/upload-to-archive-{{cookiecutter.phase.archive.slug}}/upload_files.py.j2 +++ b/grand_challenge_forge/partials/upload-to-archive-script/upload-to-archive-{{cookiecutter.phase.archive.slug}}/upload_files.py.j2 @@ -30,7 +30,7 @@ Happy uploading! from pathlib import Path import gcapi -{% if cookiecutter.phase.has_input_json -%} +{% if cookiecutter.phase.inputs | has_json -%} import json {%- endif %} @@ -84,7 +84,7 @@ def upload_files(): def map_case_content_to_interfaces(case): return { {%- for ci in cookiecutter.phase.inputs %} - {%- if ci.super_kind == "Value" %} + {%- if ci | is_json %} "{{ ci.slug }}": json.loads(Path(case[{{ loop.index - 1}}]).read_text()), {%- else %} "{{ ci.slug }}": [Path(case[{{ loop.index - 1}}])], diff --git a/grand_challenge_forge/utils.py b/grand_challenge_forge/utils.py index 62708a3..1d0773e 100644 --- a/grand_challenge_forge/utils.py +++ b/grand_challenge_forge/utils.py @@ -1,8 +1,9 @@ import importlib import os -import re from contextlib import contextmanager +from grand_challenge_forge.partials.filters import custom_filters + def truncate_with_epsilons(value, max_length=32, epsilon="..."): if len(str(value)) > max_length: @@ -13,33 +14,16 @@ def truncate_with_epsilons(value, max_length=32, epsilon="..."): def cookiecutter_context(context): + pass return { "cookiecutter": { **context, "_": "", - "_extensions": [ - "grand_challenge_forge.partials.filters.extract_slug", - ], + "_extensions": [*custom_filters], } } -def extract_slug(url): - # Define a regex pattern to match the slug in the URL - pattern = r"/([^/]+)/*$" - - # Use re.search to find the match - match = re.search(pattern, url) - - # If a match is found, extract and return the slug - if match: - slug = match.group(1) - return slug - else: - # Return None or handle the case where no match is found - return None - - def remove_j2_suffix(directory): for path, _, files in os.walk(directory): for name in files: diff --git a/tests/test_pack_generation.py b/tests/test_pack_generation.py index e0f5b1c..bf6901d 100644 --- a/tests/test_pack_generation.py +++ b/tests/test_pack_generation.py @@ -1,5 +1,5 @@ from grand_challenge_forge.forge import generate_challenge_pack -from grand_challenge_forge.utils import extract_slug +from grand_challenge_forge.generation_utils import extract_slug from tests.utils import pack_context_factory