From 224ad9494495257afb54144abcc3a18d27edab7f Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 23 Feb 2024 16:08:54 +0100 Subject: [PATCH 01/12] add script to remove admonitions and run it during pdf build --- .gitignore | 1 + mkdocs.yml | 2 + pdf_build_src/process_markdowns.py | 22 +++++--- pdf_build_src/remove_admonitions.py | 55 +++++++++++++++++++ pdf_build_src/tests/data/expected/README.md | 13 +++++ pdf_build_src/tests/data/input/README.md | 15 +++++ .../magnetic-resonance-imaging-data.md | 10 ++++ .../near-infrared-spectroscopy.md | 6 +- 8 files changed, 114 insertions(+), 10 deletions(-) create mode 100644 pdf_build_src/remove_admonitions.py create mode 100644 pdf_build_src/tests/data/expected/README.md create mode 100644 pdf_build_src/tests/data/input/README.md create mode 100644 pdf_build_src/tests/data/input/modality-specific-files/magnetic-resonance-imaging-data.md diff --git a/.gitignore b/.gitignore index 59fa1a1de3..9e8cf66a91 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ venvs pdf_build_src/bids-spec.pdf pdf_build_src/bids-spec_pandoc_log.json pdf_build_src/src_copy +pdf_build_src/tests/data/output # JS/NPM package-lock.json diff --git a/mkdocs.yml b/mkdocs.yml index 82badc4288..8e7dab5cda 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -97,6 +97,8 @@ markdown_extensions: - toc: anchorlink: true - pymdownx.superfences + - admonition + - pymdownx.details plugins: - search - branchcustomization: diff --git a/pdf_build_src/process_markdowns.py b/pdf_build_src/process_markdowns.py index 5d30aabc21..cd114b317f 100644 --- a/pdf_build_src/process_markdowns.py +++ b/pdf_build_src/process_markdowns.py @@ -17,6 +17,8 @@ import numpy as np +from remove_admonitions import remove_admonitions + sys.path.append("../tools/") # functions from module macros are called by eval() later on from mkdocs_macros_bids import macros # noqa: F401 @@ -679,33 +681,37 @@ def process_macros(duplicated_src_dir_path): duplicated_src_dir_path = "src_copy/src" - # Step 1: make a copy of the src directory in the current directory + # make a copy of the src directory in the current directory copy_src() - # Step 2: run mkdocs macros embedded in markdown files + # run mkdocs macros embedded in markdown files process_macros(duplicated_src_dir_path) - # Step 3: copy BIDS_logo to images directory of the src_copy directory + # remove mkdocs admonition + remove_admonitions(input_folder=duplicated_src_dir_path, + output_folder=duplicated_src_dir_path) + + # copy BIDS_logo to images directory of the src_copy directory copy_bids_logo() - # Step 4: copy images from subdirectories of src_copy directory + # copy images from subdirectories of src_copy directory copy_images(duplicated_src_dir_path) subprocess.call("mv src_copy/src/images/images/* src_copy/src/images/", shell=True) - # Step 5: extract the latest version number, date and title + # extract the latest version number, date and title extract_header_string() add_header() edit_titlepage() - # Step 6: modify changelog to be a level 1 heading to facilitate section + # modify changelog to be a level 1 heading to facilitate section # separation modify_changelog() - # Step 7: remove all internal links + # remove all internal links assert_no_multiline_links(duplicated_src_dir_path) remove_internal_links_inline(duplicated_src_dir_path) remove_internal_links_reference(duplicated_src_dir_path) - # Step 8: correct number of dashes and fences alignment for rendering tables in PDF + # correct number of dashes and fences alignment for rendering tables in PDF correct_tables(duplicated_src_dir_path) diff --git a/pdf_build_src/remove_admonitions.py b/pdf_build_src/remove_admonitions.py new file mode 100644 index 0000000000..2639e67c6f --- /dev/null +++ b/pdf_build_src/remove_admonitions.py @@ -0,0 +1,55 @@ +"""Script to remove all mkdocs admonition from the markdown files in a directory.""" +from __future__ import annotations +from pathlib import Path +from rich import print +import shutil + +INDENT = " " + + +def remove_admonitions(input_folder: str | Path, output_folder: str | Path, indent: str = None): + + if indent is None: + indent = INDENT + + md_files = Path(input_folder).glob("**/*.md") + + for file in md_files: + + with open(file, "r", encoding="utf8") as f: + content = f.readlines() + + output_file = Path(output_folder) / file.relative_to(input_folder) + output_file.parent.mkdir(parents=True, exist_ok=True) + print(f"processing: {file}\n to: {output_file}") + + with open(output_file, "w", encoding="utf8") as f: + + is_admonition = False + counter = 0 + for line in content: + + if line.startswith("!!!"): + is_admonition = True + counter = 0 + continue + + # skip first line after admonition + if is_admonition and counter == 0: + counter += 1 + continue + + if not line.startswith(indent): + is_admonition = False + + if is_admonition: + line = line.lstrip(indent) + + f.write(line) + + +if __name__ == "__main__": + input_folder = Path(__file__).parent / "tests" / "data" / "input" + output_folder = Path(__file__).parent / "tests" / "data" / "output" + shutil.rmtree(output_folder) + remove_admonitions(input_folder, output_folder) diff --git a/pdf_build_src/tests/data/expected/README.md b/pdf_build_src/tests/data/expected/README.md new file mode 100644 index 0000000000..fbddfe6ea4 --- /dev/null +++ b/pdf_build_src/tests/data/expected/README.md @@ -0,0 +1,13 @@ +# Test inputs + +This input directory contains data to use for testing the pdf build code of the BIDS specification. + +For example the following admonition should be removed by `pdf_build_src/remove_admonitions.py`. + +Lorem ipsum dolor sit amet, consectetur adipiscing elit. +Nulla et euismod nulla. +Curabitur feugiat, tortor non consequat finibus, justo purus auctor massa, +nec semper lorem quam in massa. + +The `expected` directory should contain the documents +as they should look like after processing. diff --git a/pdf_build_src/tests/data/input/README.md b/pdf_build_src/tests/data/input/README.md new file mode 100644 index 0000000000..207152d349 --- /dev/null +++ b/pdf_build_src/tests/data/input/README.md @@ -0,0 +1,15 @@ +# Test inputs + +This input directory contains data to use for testing the pdf build code of the BIDS specification. + +For example the following admonition should be removed by `pdf_build_src/remove_admonitions.py`. + +!!! note + + Lorem ipsum dolor sit amet, consectetur adipiscing elit. + Nulla et euismod nulla. + Curabitur feugiat, tortor non consequat finibus, justo purus auctor massa, + nec semper lorem quam in massa. + +The `expected` directory should contain the documents +as they should look like after processing. diff --git a/pdf_build_src/tests/data/input/modality-specific-files/magnetic-resonance-imaging-data.md b/pdf_build_src/tests/data/input/modality-specific-files/magnetic-resonance-imaging-data.md new file mode 100644 index 0000000000..bfc1b8509a --- /dev/null +++ b/pdf_build_src/tests/data/input/modality-specific-files/magnetic-resonance-imaging-data.md @@ -0,0 +1,10 @@ +# Magnetic Resonance Imaging + +## Common metadata fields + +!!! warning "foo bar" + + Lorem ipsum dolor sit amet, consectetur adipiscing elit. + Nulla et euismod nulla. + Curabitur feugiat, tortor non consequat finibus, justo purus auctor massa, + nec semper lorem quam in massa. diff --git a/src/modality-specific-files/near-infrared-spectroscopy.md b/src/modality-specific-files/near-infrared-spectroscopy.md index 1947836c17..d1552a0740 100644 --- a/src/modality-specific-files/near-infrared-spectroscopy.md +++ b/src/modality-specific-files/near-infrared-spectroscopy.md @@ -6,8 +6,10 @@ Please see [Citing BIDS](../introduction.md#citing-bids) on how to appropriately credit this extension when referring to it in the context of the academic literature. -Several [example NIRS datasets](https://bids-standard.github.io/bids-examples/#nirs) -have been formatted using this specification and can be used for practical guidance when curating a new dataset. +!!! example "Example datasets" + + Several [example NIRS datasets](https://bids-standard.github.io/bids-examples/#nirs) + have been formatted using this specification and can be used for practical guidance when curating a new dataset. ## NIRS recording data From b18e0e51dac17f713f769ab6ec32e13a0a552b07 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 23 Feb 2024 16:18:26 +0100 Subject: [PATCH 02/12] drop rich --- pdf_build_src/pandoc_script.py | 5 ++--- pdf_build_src/process_markdowns.py | 9 +++++---- pdf_build_src/remove_admonitions.py | 6 ++++-- pdf_build_src/test_remove_admonitions.py | 7 +++++++ 4 files changed, 18 insertions(+), 9 deletions(-) create mode 100644 pdf_build_src/test_remove_admonitions.py diff --git a/pdf_build_src/pandoc_script.py b/pdf_build_src/pandoc_script.py index b7b1842b96..e9710482ab 100644 --- a/pdf_build_src/pandoc_script.py +++ b/pdf_build_src/pandoc_script.py @@ -2,6 +2,7 @@ This is done once the duplicate src directory is processed. """ + import subprocess import yaml from pathlib import Path @@ -11,9 +12,7 @@ def _find(path, filename): return next( - parent / filename - for parent in path.parents - if Path.is_file(parent / filename) + parent / filename for parent in path.parents if Path.is_file(parent / filename) ) diff --git a/pdf_build_src/process_markdowns.py b/pdf_build_src/process_markdowns.py index cd114b317f..705daa8b22 100644 --- a/pdf_build_src/process_markdowns.py +++ b/pdf_build_src/process_markdowns.py @@ -587,9 +587,9 @@ def edit_titlepage(): data = file.readlines() data[-1] = ( - fr"\textsc{{\large {version_number}}}" + rf"\textsc{{\large {version_number}}}" r"\\[0.5cm]" - fr"{{\large {build_date}}}" + rf"{{\large {build_date}}}" r"\\[2cm]" r"\vfill" r"\end{titlepage}" @@ -688,8 +688,9 @@ def process_macros(duplicated_src_dir_path): process_macros(duplicated_src_dir_path) # remove mkdocs admonition - remove_admonitions(input_folder=duplicated_src_dir_path, - output_folder=duplicated_src_dir_path) + remove_admonitions( + input_folder=duplicated_src_dir_path, output_folder=duplicated_src_dir_path + ) # copy BIDS_logo to images directory of the src_copy directory copy_bids_logo() diff --git a/pdf_build_src/remove_admonitions.py b/pdf_build_src/remove_admonitions.py index 2639e67c6f..ea243db1d3 100644 --- a/pdf_build_src/remove_admonitions.py +++ b/pdf_build_src/remove_admonitions.py @@ -1,13 +1,15 @@ """Script to remove all mkdocs admonition from the markdown files in a directory.""" + from __future__ import annotations from pathlib import Path -from rich import print import shutil INDENT = " " -def remove_admonitions(input_folder: str | Path, output_folder: str | Path, indent: str = None): +def remove_admonitions( + input_folder: str | Path, output_folder: str | Path, indent: str = None +): if indent is None: indent = INDENT diff --git a/pdf_build_src/test_remove_admonitions.py b/pdf_build_src/test_remove_admonitions.py new file mode 100644 index 0000000000..769a006310 --- /dev/null +++ b/pdf_build_src/test_remove_admonitions.py @@ -0,0 +1,7 @@ +from pathlib import Path +from remove_admonitions import remove_admonitions + + +def test_remove_admonitions(tmp_path): + input_folder = Path(__file__).parent / "tests" / "data" / "input" + remove_admonitions(input_folder, tmp_path) From 918330443eb68412e542fff99a8a49dbd0ea3879 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 23 Feb 2024 17:19:07 +0100 Subject: [PATCH 03/12] turn off code block style check --- .remarkrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.remarkrc b/.remarkrc index fc065759c2..8d6c26487a 100644 --- a/.remarkrc +++ b/.remarkrc @@ -10,6 +10,7 @@ ["lint-maximum-heading-length", false], ["lint-no-shortcut-reference-link", false], ["lint-no-trailing-spaces"], + ["remark-lint-code-block-style", false], ["lint-no-undefined-references", false] ] } From 908e6549d4a59395c0406b1f4b3b4a35f931ece7 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 23 Feb 2024 17:46:17 +0100 Subject: [PATCH 04/12] update test data --- pdf_build_src/remove_admonitions.py | 9 +++++++-- pdf_build_src/test_remove_admonitions.py | 18 ++++++++++++++++++ .../magnetic-resonance-imaging-data.md | 8 ++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 pdf_build_src/tests/data/expected/modality-specific-files/magnetic-resonance-imaging-data.md diff --git a/pdf_build_src/remove_admonitions.py b/pdf_build_src/remove_admonitions.py index ea243db1d3..93987da8ed 100644 --- a/pdf_build_src/remove_admonitions.py +++ b/pdf_build_src/remove_admonitions.py @@ -1,8 +1,12 @@ -"""Script to remove all mkdocs admonition from the markdown files in a directory.""" +"""Script to remove all mkdocs admonition from the markdown files in a directory. + +See the pdf_build_src/tests/data/input directory to see what admonitions look like. +""" from __future__ import annotations -from pathlib import Path + import shutil +from pathlib import Path INDENT = " " @@ -51,6 +55,7 @@ def remove_admonitions( if __name__ == "__main__": + """If run as a script this will just run the main function on test data.""" input_folder = Path(__file__).parent / "tests" / "data" / "input" output_folder = Path(__file__).parent / "tests" / "data" / "output" shutil.rmtree(output_folder) diff --git a/pdf_build_src/test_remove_admonitions.py b/pdf_build_src/test_remove_admonitions.py index 769a006310..34ec2074c1 100644 --- a/pdf_build_src/test_remove_admonitions.py +++ b/pdf_build_src/test_remove_admonitions.py @@ -1,7 +1,25 @@ from pathlib import Path + from remove_admonitions import remove_admonitions def test_remove_admonitions(tmp_path): input_folder = Path(__file__).parent / "tests" / "data" / "input" + expected_folder = Path(__file__).parent / "tests" / "data" / "expected" + remove_admonitions(input_folder, tmp_path) + + generated_files = [x for x in tmp_path.glob("**/*.md")] + + for file in generated_files: + + expected = expected_folder / file.relative_to(tmp_path) + + with open(expected, "r", encoding="utf8") as f: + expected_content = f.readlines() + + with open(file, "r", encoding="utf8") as f: + generated_content = f.readlines() + + for expected_line, generated_line in zip(expected_content, generated_content): + assert generated_line == expected_line diff --git a/pdf_build_src/tests/data/expected/modality-specific-files/magnetic-resonance-imaging-data.md b/pdf_build_src/tests/data/expected/modality-specific-files/magnetic-resonance-imaging-data.md new file mode 100644 index 0000000000..d70af64a49 --- /dev/null +++ b/pdf_build_src/tests/data/expected/modality-specific-files/magnetic-resonance-imaging-data.md @@ -0,0 +1,8 @@ +# Magnetic Resonance Imaging + +## Common metadata fields + +Lorem ipsum dolor sit amet, consectetur adipiscing elit. +Nulla et euismod nulla. +Curabitur feugiat, tortor non consequat finibus, justo purus auctor massa, +nec semper lorem quam in massa. From f1b5d7682ee380eae794002577c9d137393ad512 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 23 Feb 2024 17:50:23 +0100 Subject: [PATCH 05/12] example admonition --- .../electroencephalography.md | 8 ++-- .../genetic-descriptor.md | 8 ++-- .../intracranial-electroencephalography.md | 8 ++-- .../magnetic-resonance-imaging-data.md | 44 +++++++++++-------- .../magnetoencephalography.md | 12 ++--- src/modality-specific-files/microscopy.md | 14 +++--- src/modality-specific-files/motion.md | 8 ++-- ...logical-and-other-continuous-recordings.md | 12 ++--- .../positron-emission-tomography.md | 10 +++-- 9 files changed, 74 insertions(+), 50 deletions(-) diff --git a/src/modality-specific-files/electroencephalography.md b/src/modality-specific-files/electroencephalography.md index e38f2236ef..62d215e038 100644 --- a/src/modality-specific-files/electroencephalography.md +++ b/src/modality-specific-files/electroencephalography.md @@ -6,9 +6,11 @@ Please see [Citing BIDS](../introduction.md#citing-bids) on how to appropriately credit this extension when referring to it in the context of the academic literature. -Several [example EEG datasets](https://bids-standard.github.io/bids-examples/#eeg) -have been formatted using this specification -and can be used for practical guidance when curating a new dataset. +!!! example "Example datasets" + + Several [example EEG datasets](https://bids-standard.github.io/bids-examples/#eeg) + have been formatted using this specification + and can be used for practical guidance when curating a new dataset. ## EEG recording data diff --git a/src/modality-specific-files/genetic-descriptor.md b/src/modality-specific-files/genetic-descriptor.md index 304359ed45..7d1c5cd519 100644 --- a/src/modality-specific-files/genetic-descriptor.md +++ b/src/modality-specific-files/genetic-descriptor.md @@ -12,10 +12,12 @@ A genetic descriptor links a BIDS dataset to associated genetic data, potentially in a separate repository, with details of where to find the genetic data and the type of data available. -The following example dataset with genetics data have been formatted using this specification -and can be used for practical guidance when curating a new dataset. +!!! example "Example datasets" -- [`UK biobank`](https://github.com/bids-standard/bids-examples/tree/master/genetics_ukbb) + The following example dataset with genetics data have been formatted using this specification + and can be used for practical guidance when curating a new dataset. + + - [`UK biobank`](https://github.com/bids-standard/bids-examples/tree/master/genetics_ukbb) ## Dataset Description diff --git a/src/modality-specific-files/intracranial-electroencephalography.md b/src/modality-specific-files/intracranial-electroencephalography.md index 339609699d..02fa4a189a 100644 --- a/src/modality-specific-files/intracranial-electroencephalography.md +++ b/src/modality-specific-files/intracranial-electroencephalography.md @@ -6,9 +6,11 @@ Please see [Citing BIDS](../introduction.md#citing-bids) on how to appropriately credit this extension when referring to it in the context of the academic literature. -Several [example iEEG datasets](https://bids-standard.github.io/bids-examples/#ieeg) -have been formatted using this specification -and can be used for practical guidance when curating a new dataset. +!!! example "Example datasets" + + Several [example iEEG datasets](https://bids-standard.github.io/bids-examples/#ieeg) + have been formatted using this specification + and can be used for practical guidance when curating a new dataset. ## iEEG recording data diff --git a/src/modality-specific-files/magnetic-resonance-imaging-data.md b/src/modality-specific-files/magnetic-resonance-imaging-data.md index 3b51755b1d..072410f528 100644 --- a/src/modality-specific-files/magnetic-resonance-imaging-data.md +++ b/src/modality-specific-files/magnetic-resonance-imaging-data.md @@ -635,14 +635,16 @@ A guide for using macros can be found at ## Diffusion imaging data -Several [example datasets](https://github.com/bids-standard/bids-examples) -contain diffusion imaging data formatted using this specification -and that can be used for practical guidance when curating a new dataset: +!!! example "Example datasets" -- [`genetics_ukbb`](https://github.com/bids-standard/bids-examples/tree/master/genetics_ukbb) -- [`eeg_rest_fmri`](https://github.com/bids-standard/bids-examples/tree/master/eeg_rest_fmri) -- [`ds114`](https://github.com/bids-standard/bids-examples/tree/master/ds114) -- [`ds000117`](https://github.com/bids-standard/bids-examples/tree/master/ds000117) + Several [example datasets](https://github.com/bids-standard/bids-examples) + contain diffusion imaging data formatted using this specification + and that can be used for practical guidance when curating a new dataset: + + - [`genetics_ukbb`](https://github.com/bids-standard/bids-examples/tree/master/genetics_ukbb) + - [`eeg_rest_fmri`](https://github.com/bids-standard/bids-examples/tree/master/eeg_rest_fmri) + - [`ds114`](https://github.com/bids-standard/bids-examples/tree/master/ds114) + - [`ds000117`](https://github.com/bids-standard/bids-examples/tree/master/ds000117) Diffusion-weighted imaging data acquired for a participant. Currently supported image types include: @@ -868,9 +870,11 @@ JSON example: ## Arterial Spin Labeling perfusion data -Several [example ASL datasets](https://bids-standard.github.io/bids-examples/#asl) -have been formatted using this specification -and can be used for practical guidance when curating a new dataset. +!!! example "Example datasets" + + Several [example ASL datasets](https://bids-standard.github.io/bids-examples/#asl) + have been formatted using this specification + and can be used for practical guidance when curating a new dataset.