Prepare 0 7 5 release (#54)

* rework scripts such that it uses the jinja package instead of jinja cli

* refactor script

* Prepare for upcoming changes in the config schema

change command to generate the cli and config schemas

* Remove old intermediate .yaml files

Remove `.qmd` from the file names in the `config_pages_settings.yaml`. It's now added in the script.

* Tweak empty lines in config pages jinja template

* Add environmentVariables to the config_pages_settings.yaml

* Update a bunch of automatically generated file changes

Mostly adding the default value, some type string improvements or clearing an extra empty line
Adds the new test --setup command documentation too
config.info, functionality.argumentGroup, environment variables are documented

* Use flat list of config schema data instead of map of map

Actually now a list of list instead of map of map of list
Get title information from the `__this__` parameter
Add some minor exceptions in the title that was previously handled by manually defining map keys

* Prepare & verify removal of removed parameter documentation

Also assuming any mention of 'VDSL3' is supposed to be removed.

* Copy static reference pages from viash

- Remove all subfolders in reference
- Copy static pages
- Rebuild cli & config pages

* Don't display command arguments if they are hidden

We're not (yet?) telling the --colorize and --loglevel options to the user so don't display it in the documentation

* refactor guide, add vdsl3 reference docs

* Update the pages for the Viash 0.7.5 release.

* Add vdsl3 references (#61)

* fixes to vdsl3 references

* fix link

---------

Co-authored-by: Robrecht Cannoodt <[email protected]>

_src/automation/config_pages_settings.yaml

@@ -1,63 +1,65 @@
 structure:
- 'config/config': ./index.qmd
- 'config/project': ../project/index.qmd
+ 'Config': ./index
+ 'Project': ../project/index
+ 'Info': ./info
+ 'EnvironmentVariables': ./environmentVariables
 
- 'functionality/functionality': ./functionality/index.qmd
- 'functionality/author': ./functionality/author.qmd
- 'functionality/computationalRequirements': ./functionality/computationalRequirements.qmd
+ 'Functionality': ./functionality/index
+ 'Author': ./functionality/author
+ 'ComputationalRequirements': ./functionality/computationalRequirements
+ 'ArgumentGroup': ./functionality/argumentGroup
 
- 'platforms/platform': ./platforms/index.qmd
- 'platforms/nativePlatform': ./platforms/native/index.qmd
- 'platforms/dockerPlatform': ./platforms/docker/index.qmd
- 'platforms/nextflowVdsl3Platform': ./platforms/nextflow/index.qmd
- 'platforms/nextflowLegacyPlatform': ./platforms/nextflowLegacy/index.qmd
+ 'Platform': ./platforms/index
+ 'NativePlatform': ./platforms/native/index
+ 'DockerPlatform': ./platforms/docker/index
+ 'NextflowPlatform': ./platforms/nextflow/index
 
- 'arguments/argument': ./functionality/arguments/index.qmd
- 'arguments/boolean_false': ./functionality/arguments/boolean_false.qmd
- 'arguments/boolean': ./functionality/arguments/boolean.qmd
- 'arguments/boolean_true': ./functionality/arguments/boolean_true.qmd
- 'arguments/double': ./functionality/arguments/double.qmd
- 'arguments/file': ./functionality/arguments/file.qmd
- 'arguments/integer': ./functionality/arguments/integer.qmd
- 'arguments/long': ./functionality/arguments/long.qmd
- 'arguments/string': ./functionality/arguments/string.qmd
+ 'Argument': ./functionality/arguments/index
+ 'BooleanFalseArgument': ./functionality/arguments/boolean_false
+ 'BooleanArgument': ./functionality/arguments/boolean
+ 'BooleanTrueArgument': ./functionality/arguments/boolean_true
+ 'DoubleArgument': ./functionality/arguments/double
+ 'FileArgument': ./functionality/arguments/file
+ 'IntegerArgument': ./functionality/arguments/integer
+ 'LongArgument': ./functionality/arguments/long
+ 'StringArgument': ./functionality/arguments/string
 
- 'requirements/requirements': ./platforms/docker/setup/index.qmd
- 'requirements/apkRequirements': ./platforms/docker/setup/apkRequirements.qmd
- 'requirements/aptRequirements': ./platforms/docker/setup/aptRequirements.qmd
- 'requirements/dockerRequirements': ./platforms/docker/setup/dockerRequirements.qmd
- 'requirements/javascriptRequirements': ./platforms/docker/setup/javascriptRequirements.qmd
- 'requirements/pythonRequirements': ./platforms/docker/setup/pythonRequirements.qmd
- 'requirements/rRequirements': ./platforms/docker/setup/rRequirements.qmd
- 'requirements/rubyRequirements': ./platforms/docker/setup/rubyRequirements.qmd
- 'requirements/yumRequirements': ./platforms/docker/setup/yumRequirements.qmd
+ 'Requirements': ./platforms/docker/setup/index
+ 'ApkRequirements': ./platforms/docker/setup/apkRequirements
+ 'AptRequirements': ./platforms/docker/setup/aptRequirements
+ 'DockerRequirements': ./platforms/docker/setup/dockerRequirements
+ 'JavaScriptRequirements': ./platforms/docker/setup/javascriptRequirements
+ 'PythonRequirements': ./platforms/docker/setup/pythonRequirements
+ 'RRequirements': ./platforms/docker/setup/rRequirements
+ 'RubyRequirements': ./platforms/docker/setup/rubyRequirements
+ 'YumRequirements': ./platforms/docker/setup/yumRequirements
 
- 'resources/resource': ./functionality/resources/index.qmd
- 'resources/cSharpScript': ./functionality/resources/cSharpScript.qmd
- 'resources/pythonScript': ./functionality/resources/pythonScript.qmd
- 'resources/plainFile': ./functionality/resources/plainFile.qmd
- 'resources/bashScript': ./functionality/resources/bashScript.qmd
- 'resources/scalaScript': ./functionality/resources/scalaScript.qmd
- 'resources/nextflowScript': ./functionality/resources/nextflowScript.qmd
- 'resources/rScript': ./functionality/resources/rScript.qmd
- 'resources/executable': ./functionality/resources/executable.qmd
- 'resources/javaScriptScript': ./functionality/resources/javaScriptScript.qmd
+ 'Resource': ./functionality/resources/index
+ 'CSharpScript': ./functionality/resources/cSharpScript
+ 'PythonScript': ./functionality/resources/pythonScript
+ 'PlainFile': ./functionality/resources/plainFile
+ 'BashScript': ./functionality/resources/bashScript
+ 'ScalaScript': ./functionality/resources/scalaScript
+ 'NextflowScript': ./functionality/resources/nextflowScript
+ 'RScript': ./functionality/resources/rScript
+ 'Executable': ./functionality/resources/executable
+ 'JavaScriptScript': ./functionality/resources/javaScriptScript
 
- 'nextflowParameters/nextflowDirectives': ./platforms/nextflow/directives.qmd
- 'nextflowParameters/nextflowAuto': ./platforms/nextflow/auto.qmd
- 'nextflowParameters/nextflowConfig': ./platforms/nextflow/config.qmd
+ 'NextflowDirectives': ./platforms/nextflow/directives
+ 'NextflowAuto': ./platforms/nextflow/auto
+ 'NextflowConfig': ./platforms/nextflow/config
 
 order:
- 'config/config': 20
- 'config/project': 30
+ 'Config': 20
+ 'Project': 30
 
- 'functionality/functionality': 10
- 'platforms/platform': 20
+ 'Functionality': 10
+ 'Platform': 20
+ 'Info': 30
 
- 'platforms/nativePlatform': 10
- 'platforms/dockerPlatform': 20
- 'platforms/nextflowVdsl3Platform': 30
- 'platforms/nextflowLegacyPlatform': 40
+ 'NativePlatform': 10
+ 'DockerPlatform': 20
+ 'NextflowPlatform': 30
 
 keywords:
  mykeyword: /reference/config.html

_src/automation/generate_reference_cli_pages/script.py

@@ -1,23 +1,29 @@
+import json, re, yaml
+from pathlib import Path
+import jinja2
 
 ## VIASH START
-# The following code has been auto-generated by Viash.
 par = {
- 'input': r'/path/to/file',
- 'output': r'reference/cli'
+ 'input': 'reference/cli_schema_export.json',
+ 'output': 'reference/cli'
+}
+meta = {
+ "resources_dir": "_src/automation/generate_reference_cli_pages"
 }
 ## VIASH END
-import subprocess, json, re, yaml
-from pathlib import Path
 
+output = Path(par["output"])
 keyword_regex = r"\@\[(.*?)\]\((.*?)\)"
 
 template_file = Path(meta['resources_dir'], "template_cli_page.j2.qmd")
 config_file = Path(meta['resources_dir'], 'config_pages_settings.yaml')
 
+with open(template_file) as f:
+ template = jinja2.Template(f.read())
+
 with open(config_file, 'r') as infile:
  config_pages_settings = yaml.safe_load(infile)
 
-
 def generate_pages():
  """ Load the generated JSON file and creates pages for every command entry. """
 
@@ -26,7 +32,7 @@ def generate_pages():
 
  for entry in viash_json:
  # create_page(entry["name"], entry)
- if "bannerCommand" in entry: 
+ if "bannerCommand" in entry:
  create_page(entry["name"], entry)
  else:
  for subcommand in entry['subcommands']:
@@ -47,22 +53,14 @@ def create_page(name, json_data):
  if 'descr' in arg:
  arg['descr'] = replace_keywords(arg["descr"])
 
- render_jinja_page(par['output'], name.replace(' ', '_'), page_data)
-
-def render_jinja_page(folder: str, filename: str, data: dict):
- """ Write data to yaml file and run jinja. """
-
- full_path = Path(folder, filename)
- base_dir = full_path.parent
- yaml_file = Path(base_dir, "_" + full_path.name).with_suffix('.yaml')
- qmd_file = full_path.with_suffix('.qmd')
-
- base_dir.mkdir(parents=True, exist_ok=True) 
-
- with open(yaml_file, 'w') as outfile:
- yaml.safe_dump(data, outfile, default_flow_style=False)
+ qmd_file = output / f"{name.replace(' ', '_')}.qmd"
+ render_jinja_page(qmd_file, page_data)
 
- subprocess.run(["jinja2", template_file, yaml_file, "-o", qmd_file])
+def render_jinja_page(path: Path, data: dict):
+ """Run jinja on data and write results to file."""
+ path.parent.mkdir(parents=True, exist_ok=True) 
+ content = template.render(**data)
+ path.write_text(content)
 
 def replace_keywords(text: str) -> str:
  """ Finds all keywords in the format @[keyword](text) and returns the replacements based on the keywords settings. """

_src/automation/generate_reference_cli_pages/template_cli_page.j2.qmd

@@ -17,7 +17,7 @@ search: true
 
 | Argument | Description | Type |
 |-|:----|-:
-{% for arg in d.opts|sort(attribute="name") -%}
+{% for arg in d.opts|sort(attribute="name") if not arg.hidden -%}
 
 {% if arg.name == "config" %}| `{{arg.name}}`{%- else -%}| `--{{arg.name}}`{% endif -%}
 {%- if arg.short is defined %}, `-{{arg.short}}`{% endif %} | {{arg.descr|replace("$", "\\$")|replace("\n", " ")}}

_src/automation/generate_reference_config_pages/script.py

@@ -1,54 +1,62 @@
+import json, re, yaml
+from pathlib import Path
+import jinja2
 
 ## VIASH START
-# The following code has been auto-generated by Viash.
 par = {
- 'input': r'/path/to/file',
- 'output': r'reference/config'
+ 'input': 'reference/config_schema_export.json',
+ 'output': 'reference/cli'
+}
+meta = {
+ "resources_dir": "_src/automation/generate_reference_config_pages"
 }
 ## VIASH END
-import subprocess, json, re, yaml
-from pathlib import Path
 
+output = Path(par["output"])
 keyword_regex = r"\@\[(.*?)\]\((.*?)\)"
 
 template_file = Path(meta['resources_dir'], "template_config_page.j2.qmd")
 config_file = Path(meta['resources_dir'], 'config_pages_settings.yaml')
 
+with open(template_file) as f:
+ template = jinja2.Template(f.read())
+
 with open(config_file, 'r') as infile:
  config_pages_settings = yaml.safe_load(infile)
 
-
 def read_json_entries():
  """ Load the generated JSON file, split into logical page chunks and generate pages. """
 
  with open(par['input'], 'r') as infile:
  viash_json = json.load(infile)
 
- for topic, topic_json in viash_json.items():
- if isinstance(topic_json, dict):
- for subtopic, subtopic_json in topic_json.items():
- generate_page(topic, subtopic, subtopic_json)
- else:
- generate_page(".", topic, topic_json)
+ for topic_json in viash_json:
+ generate_page(topic_json)
 
-def generate_page(topic: str, subtopic: str, json_data: list):
+def generate_page(json_data: list):
  """ Receives JSON data, does some minor data manipulation and writes to yaml & qmd. """
 
- # if topic == 'arguments': # Keep title of argument pages as-is
- # title = subtopic
- # else:
- title = re.sub(r"(\w)([A-Z])", r"\1 \2", subtopic).title() # split words and capitalize
-
+ this_parameter = {}
  # Fix description markdown keywords to links
  for d in json_data:
  if d['name'] == '__merge__':
  d['name'] = '`__merge__' # Bump __merge__ behind __this__ when sorted
+ if d['name'] == '__this__':
+ this_parameter = d
  if 'description' in d:
  d['description'] = replace_keywords(d["description"])
 
- page_data = {"topic": topic, "title": title, "data": json_data}
+ topic = this_parameter['type']
 
- filename = f"{topic}/{subtopic}"
+ # split words and capitalize
+ # do some extra substitutions to clean things up a bit
+ title = re.sub(r"(\w)([A-Z])", r"\1 \2", topic).title() \
+ .replace("Java Script", "JavaScript") \
+ .replace("C Sharp", "C#") \
+ .replace(" Argument", "")
+ filename = topic
+
+ page_data = {"title": title, "data": json_data}
 
  try:
  page_data['order'] = config_pages_settings['order'][filename]
@@ -59,23 +67,16 @@ def generate_page(topic: str, subtopic: str, json_data: list):
  filename = config_pages_settings['structure'][filename]
  except KeyError:
  print(f"Could not find {filename} in the config pages settings structure")
-
- render_jinja_page(par['output'], filename, page_data)
-
-def render_jinja_page(folder: str, filename: str, data: dict):
- """ Write data to yaml file and run jinja. """
-
- full_path = Path(folder, filename)
- base_dir = full_path.parent
- yaml_file = Path(base_dir, "_" + full_path.name).with_suffix('.yaml')
- qmd_file = full_path.with_suffix('.qmd')
 
- base_dir.mkdir(parents=True, exist_ok=True) 
+ qmd_file = output / (filename + ".qmd")
+
+ render_jinja_page(qmd_file, page_data)
 
- with open(yaml_file, 'w') as outfile:
- yaml.safe_dump(data, outfile, default_flow_style=False)
-
- subprocess.run(["jinja2", template_file, yaml_file, "-o", qmd_file])
+def render_jinja_page(path: Path, data: dict):
+ """Run jinja on data and write results to file."""
+ path.parent.mkdir(parents=True, exist_ok=True) 
+ content = template.render(**data)
+ path.write_text(content)
 
 def replace_keywords(text: str) -> str:
  """ Finds all keywords in the format @[keyword](text) and returns the replacements based on the keywords settings. """

_src/automation/generate_reference_config_pages/template_config_page.j2.qmd

@@ -9,12 +9,15 @@ order: {{order}}
 
 {% if not d.name == "__this__" %}
 ## {{d.name|replace("`__merge__", "\_\_merge\_\_")}}{# Replace fixes __merge__ being rendered bold and coming before __this__ when sorted #}
-{% if "Option of " in d.type %}
-**Type**: `{{d.type|replace("Option of ", "")}}`
-{% elif "OneOrMore of " in d.type %}
-**Type**: `{{d.type|replace("OneOrMore of ", "")}}` / `List of {{d.type|replace("OneOrMore of ", "")}}`
+{% if "Option of " in d.niceType %}
+**Type**: `{{d.niceType|replace("Option of ", "")}}`
+{% elif "OneOrMore of " in d.niceType %}
+**Type**: `{{d.niceType|replace("OneOrMore of ", "")}}` / `List of {{d.niceType|replace("OneOrMore of ", "")}}`
 {% else %}
-**Type**: `{{d.type}}`
+**Type**: `{{d.niceType}}`
+{% endif %}
+{%- if d.default is defined %}
+**Default**: `{{d.default}}`
 {% endif %}
 {% else %}
 {% endif -%}

_src/automation/generate_version_blog_pages/script.py

@@ -1,15 +1,23 @@
-import subprocess, re, yaml
+import re
 from pathlib import Path
+import jinja2
 
 ## VIASH START
 par = {
- 'input': "_src/automation/generate_version_blog_pages/test_changelog.md",
- 'output': "output/"
+ 'input': '../viash/CHANGELOG.md',
+ 'output': 'blog/posts'
+}
+meta = {
+ "resources_dir": "_src/automation/generate_version_blog_pages"
 }
 ## VIASH END
 
+output = Path(par["output"])
 template_file = Path(meta['resources_dir'], "template_blog_page.j2.qmd")
 
+with open(template_file) as f:
+ template = jinja2.Template(f.read())
+
 def bump_md_line(string: str) -> str:
  """ if header, bump header up one level """
  if re.match(r"^#+ .+$", string):
@@ -43,22 +51,14 @@ def handle_section(lines: list[str]):
  "changes": "".join(bump_markdown_headers(lines))
  }
 
- render_jinja_page(par['output'], f'viash-{version}/index.qmd', data)
-
-def render_jinja_page(folder: str, filename: str, data: dict):
- """ Write data to yaml file and run jinja. """
-
- full_path = Path(folder, filename)
- base_dir = full_path.parent
- yaml_file = Path(base_dir, "_" + full_path.name).with_suffix('.yaml')
- qmd_file = full_path.with_suffix('.qmd')
-
- base_dir.mkdir(parents=True, exist_ok=True) 
-
- with open(yaml_file, 'w') as outfile:
- yaml.safe_dump(data, outfile, default_flow_style=False)
+ qmd_file = output / f'viash-{version}/index.qmd'
+ render_jinja_page(qmd_file, data)
 
- subprocess.run(["jinja2", template_file, yaml_file, "-o", qmd_file])
+def render_jinja_page(path: Path, data: dict):
+ """Run jinja on data and write results to file."""
+ path.parent.mkdir(parents=True, exist_ok=True) 
+ content = template.render(**data)
+ path.write_text(content)
 
 def load_log(changelog_path: str):
  """ Load changelog and split into sections """