From 62457f729cb53bdbd3140f64795dc9cb0c8752d3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 31 Oct 2022 21:12:29 +0100 Subject: [PATCH 01/23] add support for MarkDown ('md') output format for --list-easyblocks --- easybuild/tools/docs.py | 24 ++++++++++++++++-------- easybuild/tools/options.py | 4 ++-- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 2f6358c02f..b0faae19ee 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -70,8 +70,9 @@ DETAILED = 'detailed' SIMPLE = 'simple' -FORMAT_TXT = 'txt' +FORMAT_MD = 'md' FORMAT_RST = 'rst' +FORMAT_TXT = 'txt' def generate_doc(name, params): @@ -449,12 +450,12 @@ def avail_classes_tree(classes, class_names, locations, detailed, format_strings def list_easyblocks(list_easyblocks=SIMPLE, output_format=FORMAT_TXT): format_strings = { - FORMAT_TXT: { - 'det_root_templ': "%s (%s%s)", - 'root_templ': "%s", - 'zero_indent': '', - 'indent': "| ", - 'sep': "|-- ", + FORMAT_MD: { + 'det_root_templ': "- **%s** (%s%s)", + 'root_templ': "- **%s**", + 'zero_indent': INDENT_2SPACES, + 'indent': INDENT_2SPACES, + 'sep': '- ', }, FORMAT_RST: { 'det_root_templ': "* **%s** (%s%s)", @@ -463,7 +464,14 @@ def list_easyblocks(list_easyblocks=SIMPLE, output_format=FORMAT_TXT): 'indent': INDENT_2SPACES, 'newline': '', 'sep': '* ', - } + }, + FORMAT_TXT: { + 'det_root_templ': "%s (%s%s)", + 'root_templ': "%s", + 'zero_indent': '', + 'indent': "| ", + 'sep': "|-- ", + }, } return gen_list_easyblocks(list_easyblocks, format_strings[output_format]) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 6128c3e650..acaae022fa 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -73,7 +73,7 @@ from easybuild.tools.config import OUTPUT_STYLE_AUTO, OUTPUT_STYLES, WARN from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path from easybuild.tools.configobj import ConfigObj, ConfigObjError -from easybuild.tools.docs import FORMAT_TXT, FORMAT_RST +from easybuild.tools.docs import FORMAT_MD, FORMAT_RST, FORMAT_TXT from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses from easybuild.tools.docs import avail_toolchain_opts, avail_easyconfig_params, avail_easyconfig_templates from easybuild.tools.docs import list_easyblocks, list_toolchains @@ -452,7 +452,7 @@ def override_options(self): 'mpi-tests': ("Run MPI tests (when relevant)", None, 'store_true', True), 'optarch': ("Set architecture optimization, overriding native architecture optimizations", None, 'store', None), - 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, [FORMAT_TXT, FORMAT_RST]), + 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, [FORMAT_MD, FORMAT_RST, FORMAT_TXT]), 'output-style': ("Control output style; auto implies using Rich if available to produce rich output, " "with fallback to basic colored output", 'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES), From e825d780377cb0926d09a342f96feff54163dca3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 31 Oct 2022 21:19:25 +0100 Subject: [PATCH 02/23] add test for list_easyblocks function --- test/framework/docs.py | 257 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 256 insertions(+), 1 deletion(-) diff --git a/test/framework/docs.py b/test/framework/docs.py index fb3e2a2b44..70d3104d52 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -29,14 +29,241 @@ import os import re import sys +import textwrap from unittest import TextTestRunner from easybuild.tools.config import module_classes -from easybuild.tools.docs import avail_easyconfig_licenses, gen_easyblocks_overview_rst, list_software +from easybuild.tools.docs import avail_easyconfig_licenses, gen_easyblocks_overview_rst +from easybuild.tools.docs import list_easyblocks, list_software, list_toolchains from easybuild.tools.utilities import import_available_modules from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config +LIST_EASYBLOCKS_SIMPLE_TXT = """EasyBlock +|-- bar +|-- ConfigureMake +| |-- MakeCp +|-- EB_EasyBuildMeta +|-- EB_FFTW +|-- EB_foo +| |-- EB_foofoo +|-- EB_GCC +|-- EB_HPL +|-- EB_libtoy +|-- EB_OpenBLAS +|-- EB_OpenMPI +|-- EB_ScaLAPACK +|-- EB_toy_buggy +|-- ExtensionEasyBlock +| |-- DummyExtension +| |-- EB_toy +| | |-- EB_toy_eula +| | |-- EB_toytoy +| |-- Toy_Extension +|-- ModuleRC +|-- PythonBundle +|-- Toolchain +Extension +|-- ExtensionEasyBlock +| |-- DummyExtension +| |-- EB_toy +| | |-- EB_toy_eula +| | |-- EB_toytoy +| |-- Toy_Extension""" + +LIST_EASYBLOCKS_DETAILED_TXT = """EasyBlock (easybuild.framework.easyblock) +|-- bar (easybuild.easyblocks.generic.bar @ %(topdir)s/generic/bar.py) +|-- ConfigureMake (easybuild.easyblocks.generic.configuremake @ %(topdir)s/generic/configuremake.py) +| |-- MakeCp (easybuild.easyblocks.generic.makecp @ %(topdir)s/generic/makecp.py) +|-- EB_EasyBuildMeta (easybuild.easyblocks.easybuildmeta @ %(topdir)s/e/easybuildmeta.py) +|-- EB_FFTW (easybuild.easyblocks.fftw @ %(topdir)s/f/fftw.py) +|-- EB_foo (easybuild.easyblocks.foo @ %(topdir)s/f/foo.py) +| |-- EB_foofoo (easybuild.easyblocks.foofoo @ %(topdir)s/f/foofoo.py) +|-- EB_GCC (easybuild.easyblocks.gcc @ %(topdir)s/g/gcc.py) +|-- EB_HPL (easybuild.easyblocks.hpl @ %(topdir)s/h/hpl.py) +|-- EB_libtoy (easybuild.easyblocks.libtoy @ %(topdir)s/l/libtoy.py) +|-- EB_OpenBLAS (easybuild.easyblocks.openblas @ %(topdir)s/o/openblas.py) +|-- EB_OpenMPI (easybuild.easyblocks.openmpi @ %(topdir)s/o/openmpi.py) +|-- EB_ScaLAPACK (easybuild.easyblocks.scalapack @ %(topdir)s/s/scalapack.py) +|-- EB_toy_buggy (easybuild.easyblocks.toy_buggy @ %(topdir)s/t/toy_buggy.py) +|-- ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) +| |-- DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) +| |-- EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) +| | |-- EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) +| | |-- EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) +| |-- Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py) +|-- ModuleRC (easybuild.easyblocks.generic.modulerc @ %(topdir)s/generic/modulerc.py) +|-- PythonBundle (easybuild.easyblocks.generic.pythonbundle @ %(topdir)s/generic/pythonbundle.py) +|-- Toolchain (easybuild.easyblocks.generic.toolchain @ %(topdir)s/generic/toolchain.py) +Extension (easybuild.framework.extension) +|-- ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) +| |-- DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) +| |-- EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) +| | |-- EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) +| | |-- EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) +| |-- Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)""" + +LIST_EASYBLOCKS_SIMPLE_RST = """* **EasyBlock** + + * bar + * ConfigureMake + + * MakeCp + + * EB_EasyBuildMeta + * EB_FFTW + * EB_foo + + * EB_foofoo + + * EB_GCC + * EB_HPL + * EB_libtoy + * EB_OpenBLAS + * EB_OpenMPI + * EB_ScaLAPACK + * EB_toy_buggy + * ExtensionEasyBlock + + * DummyExtension + * EB_toy + + * EB_toy_eula + * EB_toytoy + + * Toy_Extension + + * ModuleRC + * PythonBundle + * Toolchain + +* **Extension** + + * ExtensionEasyBlock + + * DummyExtension + * EB_toy + + * EB_toy_eula + * EB_toytoy + + * Toy_Extension + +""" + +LIST_EASYBLOCKS_DETAILED_RST = """* **EasyBlock** (easybuild.framework.easyblock) + + * bar (easybuild.easyblocks.generic.bar @ %(topdir)s/generic/bar.py) + * ConfigureMake (easybuild.easyblocks.generic.configuremake @ %(topdir)s/generic/configuremake.py) + + * MakeCp (easybuild.easyblocks.generic.makecp @ %(topdir)s/generic/makecp.py) + + * EB_EasyBuildMeta (easybuild.easyblocks.easybuildmeta @ %(topdir)s/e/easybuildmeta.py) + * EB_FFTW (easybuild.easyblocks.fftw @ %(topdir)s/f/fftw.py) + * EB_foo (easybuild.easyblocks.foo @ %(topdir)s/f/foo.py) + + * EB_foofoo (easybuild.easyblocks.foofoo @ %(topdir)s/f/foofoo.py) + + * EB_GCC (easybuild.easyblocks.gcc @ %(topdir)s/g/gcc.py) + * EB_HPL (easybuild.easyblocks.hpl @ %(topdir)s/h/hpl.py) + * EB_libtoy (easybuild.easyblocks.libtoy @ %(topdir)s/l/libtoy.py) + * EB_OpenBLAS (easybuild.easyblocks.openblas @ %(topdir)s/o/openblas.py) + * EB_OpenMPI (easybuild.easyblocks.openmpi @ %(topdir)s/o/openmpi.py) + * EB_ScaLAPACK (easybuild.easyblocks.scalapack @ %(topdir)s/s/scalapack.py) + * EB_toy_buggy (easybuild.easyblocks.toy_buggy @ %(topdir)s/t/toy_buggy.py) + * ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) + + * DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) + * EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) + + * EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) + * EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) + + * Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py) + + * ModuleRC (easybuild.easyblocks.generic.modulerc @ %(topdir)s/generic/modulerc.py) + * PythonBundle (easybuild.easyblocks.generic.pythonbundle @ %(topdir)s/generic/pythonbundle.py) + * Toolchain (easybuild.easyblocks.generic.toolchain @ %(topdir)s/generic/toolchain.py) + +* **Extension** (easybuild.framework.extension) + + * ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) + + * DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) + * EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) + + * EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) + * EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) + + * Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py) + +""" + +LIST_EASYBLOCKS_SIMPLE_MD = """- **EasyBlock** + - bar + - ConfigureMake + - MakeCp + - EB_EasyBuildMeta + - EB_FFTW + - EB_foo + - EB_foofoo + - EB_GCC + - EB_HPL + - EB_libtoy + - EB_OpenBLAS + - EB_OpenMPI + - EB_ScaLAPACK + - EB_toy_buggy + - ExtensionEasyBlock + - DummyExtension + - EB_toy + - EB_toy_eula + - EB_toytoy + - Toy_Extension + - ModuleRC + - PythonBundle + - Toolchain +- **Extension** + - ExtensionEasyBlock + - DummyExtension + - EB_toy + - EB_toy_eula + - EB_toytoy + - Toy_Extension""" + +LIST_EASYBLOCKS_DETAILED_MD = """- **EasyBlock** (easybuild.framework.easyblock) + - bar (easybuild.easyblocks.generic.bar @ %(topdir)s/generic/bar.py) + - ConfigureMake (easybuild.easyblocks.generic.configuremake @ %(topdir)s/generic/configuremake.py) + - MakeCp (easybuild.easyblocks.generic.makecp @ %(topdir)s/generic/makecp.py) + - EB_EasyBuildMeta (easybuild.easyblocks.easybuildmeta @ %(topdir)s/e/easybuildmeta.py) + - EB_FFTW (easybuild.easyblocks.fftw @ %(topdir)s/f/fftw.py) + - EB_foo (easybuild.easyblocks.foo @ %(topdir)s/f/foo.py) + - EB_foofoo (easybuild.easyblocks.foofoo @ %(topdir)s/f/foofoo.py) + - EB_GCC (easybuild.easyblocks.gcc @ %(topdir)s/g/gcc.py) + - EB_HPL (easybuild.easyblocks.hpl @ %(topdir)s/h/hpl.py) + - EB_libtoy (easybuild.easyblocks.libtoy @ %(topdir)s/l/libtoy.py) + - EB_OpenBLAS (easybuild.easyblocks.openblas @ %(topdir)s/o/openblas.py) + - EB_OpenMPI (easybuild.easyblocks.openmpi @ %(topdir)s/o/openmpi.py) + - EB_ScaLAPACK (easybuild.easyblocks.scalapack @ %(topdir)s/s/scalapack.py) + - EB_toy_buggy (easybuild.easyblocks.toy_buggy @ %(topdir)s/t/toy_buggy.py) + - ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) + - DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) + - EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) + - EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) + - EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) + - Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py) + - ModuleRC (easybuild.easyblocks.generic.modulerc @ %(topdir)s/generic/modulerc.py) + - PythonBundle (easybuild.easyblocks.generic.pythonbundle @ %(topdir)s/generic/pythonbundle.py) + - Toolchain (easybuild.easyblocks.generic.toolchain @ %(topdir)s/generic/toolchain.py) +- **Extension** (easybuild.framework.extension) + - ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) + - DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) + - EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) + - EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) + - EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) + - Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)""" + + class DocsTest(EnhancedTestCase): def test_gen_easyblocks(self): @@ -112,6 +339,34 @@ def test_license_docs(self): regex = re.compile(r"^``GPLv3``\s*The GNU General Public License", re.M) self.assertTrue(regex.search(lic_docs), "%s found in: %s" % (regex.pattern, lic_docs)) + def test_list_easyblocks(self): + """ + Tests for list_easyblocks function + """ + topdir = os.path.dirname(os.path.abspath(__file__)) + topdir_easyblocks = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks') + + txt = list_easyblocks() + self.assertEqual(txt, LIST_EASYBLOCKS_SIMPLE_TXT) + + txt = list_easyblocks(list_easyblocks='simple', output_format='txt') + self.assertEqual(txt, LIST_EASYBLOCKS_SIMPLE_TXT) + + txt = list_easyblocks(list_easyblocks='detailed', output_format='txt') + self.assertEqual(txt, LIST_EASYBLOCKS_DETAILED_TXT % {'topdir': topdir_easyblocks}) + + txt = list_easyblocks(list_easyblocks='simple', output_format='rst') + self.assertEqual(txt, LIST_EASYBLOCKS_SIMPLE_RST) + + txt = list_easyblocks(list_easyblocks='detailed', output_format='rst') + self.assertEqual(txt, LIST_EASYBLOCKS_DETAILED_RST % {'topdir': topdir_easyblocks}) + + txt = list_easyblocks(list_easyblocks='simple', output_format='md') + self.assertEqual(txt, LIST_EASYBLOCKS_SIMPLE_MD) + + txt = list_easyblocks(list_easyblocks='detailed', output_format='md') + self.assertEqual(txt, LIST_EASYBLOCKS_DETAILED_MD % {'topdir': topdir_easyblocks}) + def test_list_software(self): """Test list_software* functions.""" build_options = { From 14f54866f1d7d30d0a91f2b39cef91d061dc86a4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 31 Oct 2022 21:21:07 +0100 Subject: [PATCH 03/23] add support for MarkDown ('md') output format for --avail-easyconfig-templates --- easybuild/tools/docs.py | 42 +++++++++++++++++++++++++++++++++++- easybuild/tools/utilities.py | 36 +++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index b0faae19ee..d335029152 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -61,7 +61,7 @@ from easybuild.tools.toolchain.toolchain import DUMMY_TOOLCHAIN_NAME, SYSTEM_TOOLCHAIN_NAME, is_system_toolchain from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.utilities import INDENT_2SPACES, INDENT_4SPACES -from easybuild.tools.utilities import import_available_modules, mk_rst_table, nub, quote_str +from easybuild.tools.utilities import import_available_modules, mk_md_table, mk_rst_table, nub, quote_str _log = fancylogger.getLogger('tools.docs') @@ -81,6 +81,18 @@ def generate_doc(name, params): return func(*params) +def md_title_and_table(title, table_titles, table_values): + """Generate table in section with title in MarkDown (.md) format.""" + doc = [] + if title is not None: + doc.extend([ + '## ' + title, + '', + ]) + doc.extend(mk_md_table(table_titles, table_values)) + return doc + + def rst_title_and_table(title, table_titles, table_values): """Generate table in section with title in .rst format.""" doc = [] @@ -211,6 +223,34 @@ def avail_easyconfig_licenses_rst(): return '\n'.join(doc) +def avail_easyconfig_params_md(title, grouped_params): + """ + Compose overview of available easyconfig parameters, in MarkDown format. + """ + # main title + doc = [ + '# ' + title, + '', + ] + + for grpname in grouped_params: + # group section title + title = "%s%s parameters" % (grpname[0].upper(), grpname[1:]) + table_titles = ["**Parameter name**", "**Description**", "**Default value**"] + keys = sorted(grouped_params[grpname].keys()) + values = [grouped_params[grpname][key] for key in keys] + table_values = [ + ['`%s`' % name for name in keys], # parameter name + [x[0] for x in values], # description + ['`' + str(quote_str(x[1])) + '`' for x in values] # default value + ] + + doc.extend(md_title_and_table(title, table_titles, table_values)) + doc.append('') + + return '\n'.join(doc) + + def avail_easyconfig_params_rst(title, grouped_params): """ Compose overview of available easyconfig parameters, in RST format. diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index 81989b9c86..2cd5fece23 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -259,6 +259,42 @@ def get_subclasses(klass, include_base_class=False): return get_subclasses_dict(klass, include_base_class=include_base_class).keys() +def mk_md_table(titles, columns): + """ + Returns a MarkDown table with given titles and columns (a nested list of string columns for each column) + """ + # take into account that passed values may be iterators produced via 'map' + titles = list(titles) + columns = list(columns) + + title_cnt, col_cnt = len(titles), len(columns) + if title_cnt != col_cnt: + msg = "Number of titles/columns should be equal, found %d titles and %d columns" % (title_cnt, col_cnt) + raise ValueError(msg) + table = [] + tmpl = [] + line = [] + + # figure out column widths + for i, title in enumerate(titles): + width = max(map(len, columns[i] + [title])) + + # make line template + tmpl.append('{%s:{c}<%s}' % (i, width)) + + line = [''] * col_cnt + line_tmpl = '|'.join(tmpl) + table_line = line_tmpl.format(*line, c='-') + + table.append(line_tmpl.format(*titles, c=' ')) + table.append(table_line) + + for row in map(list, zip(*columns)): + table.append(line_tmpl.format(*row, c=' ')) + + return table + + def mk_rst_table(titles, columns): """ Returns an rst table with given titles and columns (a nested list of string columns for each column) From 86edb74e626babb37981c985c8372a9a291175cd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 31 Oct 2022 21:42:46 +0100 Subject: [PATCH 04/23] add tests for mk_*_table and *_title_and_table functions --- easybuild/tools/utilities.py | 4 +- test/framework/docs.py | 79 +++++++++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index 2cd5fece23..c458d89b00 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -277,7 +277,7 @@ def mk_md_table(titles, columns): # figure out column widths for i, title in enumerate(titles): - width = max(map(len, columns[i] + [title])) + width = max(map(len, list(columns[i]) + [title])) # make line template tmpl.append('{%s:{c}<%s}' % (i, width)) @@ -313,7 +313,7 @@ def mk_rst_table(titles, columns): # figure out column widths for i, title in enumerate(titles): - width = max(map(len, columns[i] + [title])) + width = max(map(len, list(columns[i]) + [title])) # make line template tmpl.append('{%s:{c}<%s}' % (i, width)) diff --git a/test/framework/docs.py b/test/framework/docs.py index 70d3104d52..f327b0dc86 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -35,7 +35,8 @@ from easybuild.tools.config import module_classes from easybuild.tools.docs import avail_easyconfig_licenses, gen_easyblocks_overview_rst from easybuild.tools.docs import list_easyblocks, list_software, list_toolchains -from easybuild.tools.utilities import import_available_modules +from easybuild.tools.docs import md_title_and_table, rst_title_and_table +from easybuild.tools.utilities import import_available_modules, mk_md_table, mk_rst_table from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config @@ -541,6 +542,82 @@ def test_list_software(self): expected_found = any(lines[i:i + len(expected)] == expected for i in range(len(lines))) self.assertTrue(expected_found, "%s found in: %s" % (expected, lines)) + def test_mk_table(self): + """ + Tests for mk_*_table functions. + """ + titles = ('one', 'two', 'three') + table = [ + ('1', '11111'), + ('2222222', '2'), + ('3', '3'), + ] + expected_md = [ + 'one |two |three', + '-----|-------|-----', + '1 |2222222|3 ', + '11111|2 |3 ', + ] + expected_rst = [ + '===== ======= =====', + 'one two three', + '===== ======= =====', + '1 2222222 3 ', + '11111 2 3 ', + '===== ======= =====', + '', + ] + + res = mk_md_table(titles, table) + self.assertEqual(res, expected_md) + + res = mk_rst_table(titles, table) + self.assertEqual(res, expected_rst) + + self.assertErrorRegex(ValueError, "Number of titles/columns should be equal", mk_md_table, titles, []) + self.assertErrorRegex(ValueError, "Number of titles/columns should be equal", mk_rst_table, titles, []) + + def test_title_and_table(self): + """ + Tests for *_title_and_table functions. + """ + titles = ('one', 'two', '3 is a wide column') + table = [ + titles, + ('val 11', 'val 21'), + ('val 12', 'val 22'), + ('val 13', 'val 23'), + ] + expected_md = [ + '## test title', + '', + 'one |two |3 is a wide column', + '------|------|------------------', + 'val 11|val 12|val 13 ', + 'val 21|val 22|val 23 ', + ] + expected_rst = [ + 'test title', + '----------', + '', + '====== ====== ==================', + 'one two 3 is a wide column', + '====== ====== ==================', + 'val 11 val 12 val 13 ', + 'val 21 val 22 val 23 ', + '====== ====== ==================', + '', + ] + res = md_title_and_table('test title', table[0], table[1:]) + self.assertEqual(res, expected_md) + + res = rst_title_and_table('test title', table[0], table[1:]) + self.assertEqual(res, expected_rst) + + error_pattern = "Number of titles/columns should be equal" + self.assertErrorRegex(ValueError, error_pattern, md_title_and_table, '', titles, []) + self.assertErrorRegex(ValueError, error_pattern, rst_title_and_table, '', titles, [('val 11', 'val 12')]) + def suite(): """ returns all test cases in this module """ From 19f2a1e9ccb73b1fc75ce48df220a541a0e337a2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 31 Oct 2022 21:59:59 +0100 Subject: [PATCH 05/23] remove unused textwrap import --- test/framework/docs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/framework/docs.py b/test/framework/docs.py index f327b0dc86..6e81440480 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -29,7 +29,6 @@ import os import re import sys -import textwrap from unittest import TextTestRunner from easybuild.tools.config import module_classes From 84e87253f4a6ddff99183af18fd2531d33c8c12f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 5 Dec 2022 17:07:08 +0100 Subject: [PATCH 06/23] add support for MarkDown ('md') output format for --list-software --- easybuild/tools/docs.py | 92 +++++++++++++++ test/framework/docs.py | 250 ++++++++++++++++++++++++---------------- 2 files changed, 242 insertions(+), 100 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index d335029152..5489825780 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -634,6 +634,98 @@ def list_software(output_format=FORMAT_TXT, detailed=False, only_installed=False return generate_doc('list_software_%s' % output_format, [software, detailed]) +def list_software_md(software, detailed=True): + """ + Return overview of supported software in MarkDown format + + :param software: software information (structured like list_software does) + :param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info) + :return: multi-line string presenting requested info + """ + + lines = [ + "# List of supported software", + '', + "EasyBuild supports %d different software packages (incl. toolchains, bundles):" % len(software), + '', + ] + + # links to per-letter tables + letter_refs = '' + key_letters = nub(sorted(k[0].lower() for k in software.keys())) + letter_links = ' - '.join(['' + x + '' for x in ascii_lowercase if x in key_letters]) + lines.extend([letter_links, '']) + + letter = None + sorted_keys = sorted(software.keys(), key=lambda x: x.lower()) + for key in sorted_keys: + + # start a new subsection for each letter + if key[0].lower() != letter: + + # subsection for new letter + letter = key[0].lower() + lines.extend([ + '', + '' % letter, + "### *%s*" % letter.upper(), + '', + ]) + + if detailed: + # quick links per software package + lines.extend([ + '', + ' - '.join('%s' % (k.lower(), k) for k in sorted_keys if k[0].lower() == letter), + '', + ]) + + # append software to list, including version(suffix) & toolchain info if detailed info is requested + if detailed: + table_titles = ['version', 'toolchain'] + table_values = [[], []] + + # first determine unique pairs of version/versionsuffix + # we can't use LooseVersion yet here, since nub uses set and LooseVersion instances are not hashable + pairs = nub((x['version'], x['versionsuffix']) for x in software[key]) + + # check whether any non-empty versionsuffixes are in play + with_vsuff = any(vs for (_, vs) in pairs) + if with_vsuff: + table_titles.insert(1, 'versionsuffix') + table_values.insert(1, []) + + # sort pairs by version (and then by versionsuffix); + # we sort by LooseVersion to obtain chronological version ordering, + # but we also need to retain original string version for filtering-by-version done below + sorted_pairs = sort_looseversions((LooseVersion(v), vs, v) for v, vs in pairs) + + for _, vsuff, ver in sorted_pairs: + table_values[0].append('``%s``' % ver) + if with_vsuff: + if vsuff: + table_values[1].append('``%s``' % vsuff) + else: + table_values[1].append('') + tcs = [x['toolchain'] for x in software[key] if x['version'] == ver and x['versionsuffix'] == vsuff] + table_values[-1].append(', '.join('``%s``' % tc for tc in sorted(nub(tcs)))) + + lines.extend([ + '', + '' % key.lower(), + '### *%s*' % key, + '', + ' '.join(software[key][-1]['description'].split('\n')).lstrip(' '), + '', + "*homepage*: %s" % software[key][-1]['homepage'], + '', + ] + md_title_and_table(None, table_titles, table_values)) + else: + lines.append("* %s" % key) + + return '\n'.join(lines) + + def list_software_rst(software, detailed=False): """ Return overview of supported software in RST format diff --git a/test/framework/docs.py b/test/framework/docs.py index 6e81440480..176f5b9d28 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -263,6 +263,149 @@ - EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) - Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)""" +LIST_SOFTWARE_SIMPLE_TXT = """ +* GCC +* gzip""" + +GCC_DESCR = "The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, " +GCC_DESCR += "as well as libraries for these languages (libstdc++, libgcj,...)." +GZIP_DESCR = "gzip (GNU zip) is a popular data compression program as a replacement for compress" + +LIST_SOFTWARE_DETAILED_TXT = """ +* GCC + +%(gcc_descr)s + +homepage: http://gcc.gnu.org/ + + * GCC v4.6.3: system + +* gzip + +%(gzip_descr)s + +homepage: http://www.gzip.org/ + + * gzip v1.4: GCC/4.6.3, system + * gzip v1.5: foss/2018a, intel/2018a +""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} + +LIST_SOFTWARE_SIMPLE_RST = """List of supported software +========================== + +EasyBuild |version| supports 2 different software packages (incl. toolchains, bundles): + +:ref:`list_software_letter_g` + + +.. _list_software_letter_g: + +*G* +--- + +* GCC +* gzip""" + +LIST_SOFTWARE_DETAILED_RST = """List of supported software +========================== + +EasyBuild |version| supports 2 different software packages (incl. toolchains, bundles): + +:ref:`list_software_letter_g` + + +.. _list_software_letter_g: + +*G* +--- + + +:ref:`list_software_GCC_205` - :ref:`list_software_gzip_442` + + +.. _list_software_GCC_205: + +*GCC* ++++++ + +%(gcc_descr)s + +*homepage*: http://gcc.gnu.org/ + +========= ========== +version toolchain +========= ========== +``4.6.3`` ``system`` +========= ========== + + +.. _list_software_gzip_442: + +*gzip* +++++++ + +%(gzip_descr)s + +*homepage*: http://www.gzip.org/ + +======= =============================== +version toolchain +======= =============================== +``1.4`` ``GCC/4.6.3``, ``system`` +``1.5`` ``foss/2018a``, ``intel/2018a`` +======= =============================== +""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} + +LIST_SOFTWARE_SIMPLE_MD = """# List of supported software + +EasyBuild supports 2 different software packages (incl. toolchains, bundles): + +g + + + +### *G* + +* GCC +* gzip""" + +LIST_SOFTWARE_DETAILED_MD = """# List of supported software + +EasyBuild supports 2 different software packages (incl. toolchains, bundles): + +g + + + +### *G* + + +GCC - gzip + + + +### *GCC* + +%(gcc_descr)s + +*homepage*: http://gcc.gnu.org/ + +version |toolchain +---------|---------- +``4.6.3``|``system`` + + +### *gzip* + +%(gzip_descr)s + +*homepage*: http://www.gzip.org/ + +version|toolchain +-------|------------------------------- +``1.4``|``GCC/4.6.3``, ``system`` +``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} + class DocsTest(EnhancedTestCase): @@ -376,107 +519,14 @@ def test_list_software(self): } init_config(build_options=build_options) - expected = '\n'.join([ - '', - '* GCC', - '* gzip', - ]) - self.assertEqual(list_software(output_format='txt'), expected) - - expected = re.compile('\n'.join([ - r'', - r'\* GCC', - r'', - r"The GNU Compiler Collection .*", - r'', - r'homepage: http://gcc.gnu.org/', - r'', - r' \* GCC v4.6.3: system', - r'', - r'\* gzip', - r'', - r"gzip \(GNU zip\) is .*", - r'', - r'homepage: http://www.gzip.org/', - r'', - r" \* gzip v1.4: GCC/4.6.3, system", - r" \* gzip v1.5: foss/2018a, intel/2018a", - '', - ])) - txt = list_software(output_format='txt', detailed=True) - self.assertTrue(expected.match(txt), "Pattern '%s' found in: %s" % (expected.pattern, txt)) + self.assertEqual(list_software(output_format='txt'), LIST_SOFTWARE_SIMPLE_TXT) + self.assertEqual(list_software(output_format='txt', detailed=True), LIST_SOFTWARE_DETAILED_TXT) - expected = '\n'.join([ - "List of supported software", - "==========================", - '', - "EasyBuild |version| supports 2 different software packages (incl. toolchains, bundles):", - '', - ':ref:`list_software_letter_g`', - '', - '', - '.. _list_software_letter_g:', - '', - '*G*', - '---', - '', - '* GCC', - '* gzip', - ]) - self.assertEqual(list_software(output_format='rst'), expected) - - expected = re.compile('\n'.join([ - r"List of supported software", - r"==========================", - r'', - r"EasyBuild \|version\| supports 2 different software packages \(incl. toolchains, bundles\):", - r'', - r':ref:`list_software_letter_g`', - r'', - r'', - r'.. _list_software_letter_g:', - r'', - r'\*G\*', - r'---', - r'', - r'', - r':ref:`list_software_GCC_205` - :ref:`list_software_gzip_442`', - r'', - r'', - r'\.\. _list_software_GCC_205:', - r'', - r'\*GCC\*', - r'\+\+\+\+\+', - r'', - r'The GNU Compiler Collection .*', - r'', - r'\*homepage\*: http://gcc.gnu.org/', - r'', - r'========= ==========', - r'version toolchain ', - r'========= ==========', - r'``4.6.3`` ``system``', - r'========= ==========', - r'', - r'', - r'\.\. _list_software_gzip_442:', - r'', - r'\*gzip\*', - r'\+\+\+\+\+\+', - r'', - r'gzip \(GNU zip\) is a popular .*', - r'', - r'\*homepage\*: http://www.gzip.org/', - r'', - r'======= ===============================', - r'version toolchain ', - r'======= ===============================', - r'``1.4`` ``GCC/4.6.3``, ``system`` ', - r'``1.5`` ``foss/2018a``, ``intel/2018a``', - r'======= ===============================', - ])) - txt = list_software(output_format='rst', detailed=True) - self.assertTrue(expected.match(txt), "Pattern '%s' found in: %s" % (expected.pattern, txt)) + self.assertEqual(list_software(output_format='rst'), LIST_SOFTWARE_SIMPLE_RST) + self.assertEqual(list_software(output_format='rst', detailed=True), LIST_SOFTWARE_DETAILED_RST) + + self.assertEqual(list_software(output_format='md'), LIST_SOFTWARE_SIMPLE_MD) + self.assertEqual(list_software(output_format='md', detailed=True), LIST_SOFTWARE_DETAILED_MD) # GCC/4.6.3 is installed, no gzip module installed txt = list_software(output_format='txt', detailed=True, only_installed=True) From f0c877b6900a6764130bd0e9e1710ea425b87fee Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 6 Dec 2022 20:12:37 +0100 Subject: [PATCH 07/23] strip trailing whitespace in table rows in mk_md_table and mk_rst_table --- easybuild/tools/docs.py | 1 - easybuild/tools/utilities.py | 8 +++--- test/framework/docs.py | 48 ++++++++++++++++++------------------ 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 5489825780..4d80ea9b10 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -651,7 +651,6 @@ def list_software_md(software, detailed=True): ] # links to per-letter tables - letter_refs = '' key_letters = nub(sorted(k[0].lower() for k in software.keys())) letter_links = ' - '.join(['' + x + '' for x in ascii_lowercase if x in key_letters]) lines.extend([letter_links, '']) diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index c458d89b00..4f548c33df 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -286,11 +286,11 @@ def mk_md_table(titles, columns): line_tmpl = '|'.join(tmpl) table_line = line_tmpl.format(*line, c='-') - table.append(line_tmpl.format(*titles, c=' ')) + table.append(line_tmpl.format(*titles, c=' ').strip()) table.append(table_line) for row in map(list, zip(*columns)): - table.append(line_tmpl.format(*row, c=' ')) + table.append(line_tmpl.format(*row, c=' ').strip()) return table @@ -323,11 +323,11 @@ def mk_rst_table(titles, columns): table_line = line_tmpl.format(*line, c='=') table.append(table_line) - table.append(line_tmpl.format(*titles, c=' ')) + table.append(line_tmpl.format(*titles, c=' ').strip()) table.append(table_line) for row in map(list, zip(*columns)): - table.append(line_tmpl.format(*row, c=' ')) + table.append(line_tmpl.format(*row, c=' ').strip()) table.extend([table_line, '']) diff --git a/test/framework/docs.py b/test/framework/docs.py index 176f5b9d28..7f799cbb0e 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -333,7 +333,7 @@ *homepage*: http://gcc.gnu.org/ ========= ========== -version toolchain +version toolchain ========= ========== ``4.6.3`` ``system`` ========= ========== @@ -349,9 +349,9 @@ *homepage*: http://www.gzip.org/ ======= =============================== -version toolchain +version toolchain ======= =============================== -``1.4`` ``GCC/4.6.3``, ``system`` +``1.4`` ``GCC/4.6.3``, ``system`` ``1.5`` ``foss/2018a``, ``intel/2018a`` ======= =============================== """ % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} @@ -390,7 +390,7 @@ *homepage*: http://gcc.gnu.org/ -version |toolchain +version |toolchain ---------|---------- ``4.6.3``|``system`` @@ -401,9 +401,9 @@ *homepage*: http://www.gzip.org/ -version|toolchain +version|toolchain -------|------------------------------- -``1.4``|``GCC/4.6.3``, ``system`` +``1.4``|``GCC/4.6.3``, ``system`` ``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} @@ -438,20 +438,20 @@ def test_gen_easyblocks(self): "==================== ============ =============", "easyconfig parameter description default value", "==================== ============ =============", - '``test_123`` Test 1, 2, 3 ``""`` ', - "``test_bool`` Just a test ``False`` ", - "``test_none`` Another test ``None`` ", + '``test_123`` Test 1, 2, 3 ``""``', + "``test_bool`` Just a test ``False``", + "``test_none`` Another test ``None``", "==================== ============ =============", '', "Commonly used easyconfig parameters with ``ConfigureMake`` easyblock", "--------------------------------------------------------------------", '', "==================== ================================================================", - "easyconfig parameter description ", + "easyconfig parameter description", "==================== ================================================================", "configopts Extra options passed to configure (default already has --prefix)", - "buildopts Extra options passed to make step (default already has -j X) ", - "installopts Extra options for installation ", + "buildopts Extra options passed to make step (default already has -j X)", + "installopts Extra options for installation", "==================== ================================================================", ]) @@ -577,12 +577,12 @@ def test_list_software(self): '*homepage*: https://easybuilders.github.io/easybuild', '', '======= ============= ===========================', - 'version versionsuffix toolchain ', + 'version versionsuffix toolchain', '======= ============= ===========================', '``0.0`` ``gompi/2018a``, ``system``', - '``0.0`` ``-deps`` ``system`` ', - '``0.0`` ``-iter`` ``system`` ', - '``0.0`` ``-multiple`` ``system`` ', + '``0.0`` ``-deps`` ``system``', + '``0.0`` ``-iter`` ``system``', + '``0.0`` ``-multiple`` ``system``', '``0.0`` ``-test`` ``gompi/2018a``, ``system``', '======= ============= ===========================', ] @@ -604,15 +604,15 @@ def test_mk_table(self): expected_md = [ 'one |two |three', '-----|-------|-----', - '1 |2222222|3 ', - '11111|2 |3 ', + '1 |2222222|3', + '11111|2 |3', ] expected_rst = [ '===== ======= =====', 'one two three', '===== ======= =====', - '1 2222222 3 ', - '11111 2 3 ', + '1 2222222 3', + '11111 2 3', '===== ======= =====', '', ] @@ -642,8 +642,8 @@ def test_title_and_table(self): '', 'one |two |3 is a wide column', '------|------|------------------', - 'val 11|val 12|val 13 ', - 'val 21|val 22|val 23 ', + 'val 11|val 12|val 13', + 'val 21|val 22|val 23', ] expected_rst = [ 'test title', @@ -652,8 +652,8 @@ def test_title_and_table(self): '====== ====== ==================', 'one two 3 is a wide column', '====== ====== ==================', - 'val 11 val 12 val 13 ', - 'val 21 val 22 val 23 ', + 'val 11 val 12 val 13', + 'val 21 val 22 val 23', '====== ====== ==================', '', ] From 841ab60687fb17767e38218bc0dc137207418ea6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 9 Dec 2022 10:54:35 +0100 Subject: [PATCH 08/23] implement support for '--avail-cfgfile-constants --output-format md' and '--avail-easyconfig-constants --output-format md' --- easybuild/tools/docs.py | 61 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 4d80ea9b10..62aa541662 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -114,6 +114,7 @@ def avail_cfgfile_constants(go_cfg_constants, output_format=FORMAT_TXT): def avail_cfgfile_constants_txt(go_cfg_constants): + """Generate documentation on constants for configuration files in txt format""" doc = [ "Constants available (only) in configuration files:", "syntax: %(CONSTANT_NAME)s", @@ -129,25 +130,53 @@ def avail_cfgfile_constants_txt(go_cfg_constants): def avail_cfgfile_constants_rst(go_cfg_constants): + """Generate documentation on constants for configuration files in rst format""" title = "Constants available (only) in configuration files" - doc = [title, '-' * len(title), ''] + doc = [title, '-' * len(title)] for section in go_cfg_constants: doc.append('') if section != go_cfg_constants['DEFAULT']: - section_title = "only in '%s' section:" % section + section_title = "Only in '%s' section:" % section doc.extend([section_title, '-' * len(section_title), '']) table_titles = ["Constant name", "Constant help", "Constant value"] + sorted_names = sorted(go_cfg_constants[section].keys()) table_values = [ - ['``' + name + '``' for name in go_cfg_constants[section].keys()], - [tup[1] for tup in go_cfg_constants[section].values()], - ['``' + tup[0] + '``' for tup in go_cfg_constants[section].values()], + ['``' + x + '``' for x in sorted_names], + [go_cfg_constants[section][x][1] for x in sorted_names], + ['``' + go_cfg_constants[section][x][0] + '``' for x in sorted_names], ] doc.extend(mk_rst_table(table_titles, table_values)) return '\n'.join(doc) +def avail_cfgfile_constants_md(go_cfg_constants): + """Generate documentation on constants for configuration files in MarkDown format""" + title = "Constants available (only) in configuration files" + doc = [ + '# ' + title, + '', + ] + + for section in go_cfg_constants: + if section != go_cfg_constants['DEFAULT']: + doc.extend([ + "### Only in '%s' section:" % section, + '', + ]) + table_titles = ["Constant name", "Constant help", "Constant value"] + sorted_names = sorted(go_cfg_constants[section].keys()) + table_values = [ + ['``' + x + '``' for x in sorted_names], + [go_cfg_constants[section][x][1] for x in sorted_names], + ['``' + go_cfg_constants[section][x][0] + '``' for x in sorted_names], + ] + doc.extend(mk_md_table(table_titles, table_values)) + + return '\n'.join(doc) + + def avail_easyconfig_constants(output_format=FORMAT_TXT): """Generate the easyconfig constant documentation""" return generate_doc('avail_easyconfig_constants_%s' % output_format, []) @@ -184,6 +213,28 @@ def avail_easyconfig_constants_rst(): return '\n'.join(doc) +def avail_easyconfig_constants_md(): + """Generate easyconfig constant documentation in MarkDown format""" + title = "Constants that can be used in easyconfigs" + + table_titles = [ + "Constant name", + "Constant value", + "Description", + ] + + sorted_keys = sorted(EASYCONFIG_CONSTANTS) + + table_values = [ + ["``%s``" % key for key in sorted_keys], + ["``%s``" % str(EASYCONFIG_CONSTANTS[key][0]) for key in sorted_keys], + [EASYCONFIG_CONSTANTS[key][1] for key in sorted_keys], + ] + + doc = md_title_and_table(title, table_titles, table_values) + return '\n'.join(doc) + + def avail_easyconfig_licenses(output_format=FORMAT_TXT): """Generate the easyconfig licenses documentation""" return generate_doc('avail_easyconfig_licenses_%s' % output_format, []) From c2a3e510d0d8efc875895e0e5fb48fa4b7efa7dd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 9 Dec 2022 11:18:12 +0100 Subject: [PATCH 09/23] add test for avail_cfgfile_constants docs function --- test/framework/docs.py | 47 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/test/framework/docs.py b/test/framework/docs.py index 7f799cbb0e..da73628ed2 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -32,9 +32,10 @@ from unittest import TextTestRunner from easybuild.tools.config import module_classes -from easybuild.tools.docs import avail_easyconfig_licenses, gen_easyblocks_overview_rst +from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_licenses, gen_easyblocks_overview_rst from easybuild.tools.docs import list_easyblocks, list_software, list_toolchains from easybuild.tools.docs import md_title_and_table, rst_title_and_table +from easybuild.tools.options import EasyBuildOptions from easybuild.tools.utilities import import_available_modules, mk_md_table, mk_rst_table from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config @@ -591,6 +592,50 @@ def test_list_software(self): expected_found = any(lines[i:i + len(expected)] == expected for i in range(len(lines))) self.assertTrue(expected_found, "%s found in: %s" % (expected, lines)) + def test_avail_cfgfile_constants(self): + """ + Test avail_cfgfile_constants to generate overview of constants that can be used in a configuration file. + """ + option_parser = EasyBuildOptions() + txt_patterns = [ + r"^Constants available \(only\) in configuration files:", + r"^syntax: %\(CONSTANT_NAME\)s", + r"^only in 'DEFAULT' section:", + r"^\* HOME: Current user's home directory, expanded '~' \[value: %s\]" % os.getenv('HOME'), + r"^\* USER: Current username, translated uid from password file \[value: %s\]" % os.getenv('USER'), + ] + txt = avail_cfgfile_constants(option_parser.go_cfg_constants) + for pattern in txt_patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt)) + + txt = avail_cfgfile_constants(option_parser.go_cfg_constants, output_format='txt') + for pattern in txt_patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt)) + + md_patterns = [ + r"^# Constants available \(only\) in configuration files", + r"^### Only in 'DEFAULT' section:", + r"^``HOME``\s*\|Current user's home directory, expanded '~'\s*\|``%s``$" % os.getenv('HOME'), + r"^``USER``\s*\|Current username, translated uid from password file\s*\|``%s``" % os.getenv('USER'), + ] + txt_md = avail_cfgfile_constants(option_parser.go_cfg_constants, output_format='md') + for pattern in md_patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt_md), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_md)) + + rst_patterns = [ + r"^Constants available \(only\) in configuration files\n-{49}\n", + r"^Only in 'DEFAULT' section:\n-{26}", + r"^``HOME``\s*Current user's home directory, expanded '~'\s*``%s``$" % os.getenv('HOME'), + r"^``USER``\s*Current username, translated uid from password file\s*``%s``" % os.getenv('USER'), + ] + txt_rst = avail_cfgfile_constants(option_parser.go_cfg_constants, output_format='rst') + for pattern in rst_patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + def test_mk_table(self): """ Tests for mk_*_table functions. From ab4c3345cbc2f51f43866717807d8feb7979a474 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 9 Dec 2022 11:30:36 +0100 Subject: [PATCH 10/23] run docs test suite before options test suite, to avoid included easyblocks triggering failing docs tests --- test/framework/suite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/suite.py b/test/framework/suite.py index d49d40bb6b..1f8a614fb3 100755 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -120,8 +120,8 @@ # call suite() for each module and then run them all # note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config -tests = [gen, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, lic, f_c, - tw, p, i, pkg, d, env, et, y, st, h, ct, lib, u, es, ou] +tests = [gen, d, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, lic, f_c, + tw, p, i, pkg, env, et, y, st, h, ct, lib, u, es, ou] SUITE = unittest.TestSuite([x.suite() for x in tests]) res = unittest.TextTestRunner().run(SUITE) From 30763d4380b5fca66318ee1f40a6a9af18f5a843 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 9 Dec 2022 11:37:55 +0100 Subject: [PATCH 11/23] add test for avail_easyconfig_constants function --- test/framework/docs.py | 47 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/test/framework/docs.py b/test/framework/docs.py index da73628ed2..ff1e3ed11c 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -32,8 +32,8 @@ from unittest import TextTestRunner from easybuild.tools.config import module_classes -from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_licenses, gen_easyblocks_overview_rst -from easybuild.tools.docs import list_easyblocks, list_software, list_toolchains +from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses +from easybuild.tools.docs import gen_easyblocks_overview_rst, list_easyblocks, list_software, list_toolchains from easybuild.tools.docs import md_title_and_table, rst_title_and_table from easybuild.tools.options import EasyBuildOptions from easybuild.tools.utilities import import_available_modules, mk_md_table, mk_rst_table @@ -636,6 +636,49 @@ def test_avail_cfgfile_constants(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + def test_avail_easyconfig_constants(self): + """ + Test avail_easyconfig_constants to generate overview of constants that can be used in easyconfig files. + """ + txt_patterns = [ + r"^Constants that can be used in easyconfigs", + r"^\s*ARCH: .* \(CPU architecture of current system \(aarch64, x86_64, ppc64le, ...\)\)", + r"^\s*OS_PKG_OPENSSL_DEV: \('openssl-devel', 'libssl-dev', 'libopenssl-devel'\) " + r"\(OS packages providing openSSL developement support\)", + ] + + txt = avail_easyconfig_constants() + for pattern in txt_patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt)) + + txt = avail_easyconfig_constants(output_format='txt') + for pattern in txt_patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt)) + + md_patterns = [ + r"^## Constants that can be used in easyconfigs", + r"^``ARCH``\s*\|``.*``\s*\|CPU architecture of current system \(aarch64, x86_64, ppc64le, ...\)$", + r"^``OS_PKG_OPENSSL_DEV``\s*\|``\('openssl-devel', 'libssl-dev', 'libopenssl-devel'\)``\s*\|" + r"OS packages providing openSSL developement support$", + ] + txt_md = avail_easyconfig_constants(output_format='md') + for pattern in md_patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt_md), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_md)) + + rst_patterns = [ + r"^Constants that can be used in easyconfigs\n-{41}", + r"^``ARCH``\s*``.*``\s*CPU architecture of current system \(aarch64, x86_64, ppc64le, ...\)$", + r"^``OS_PKG_OPENSSL_DEV``\s*``\('openssl-devel', 'libssl-dev', 'libopenssl-devel'\)``\s*" + r"OS packages providing openSSL developement support$", + ] + txt_rst = avail_easyconfig_constants(output_format='rst') + for pattern in rst_patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + def test_mk_table(self): """ Tests for mk_*_table functions. From d2387c57dcb7a62c04abfaab2db134ee2202aad6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 16 Dec 2022 11:38:28 +0100 Subject: [PATCH 12/23] implement support for '--avail-easyconfig-licenses --output-format md' --- easybuild/tools/docs.py | 21 +++++++++++++++++++++ test/framework/docs.py | 4 ++++ 2 files changed, 25 insertions(+) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 62aa541662..b210c9040b 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -274,6 +274,27 @@ def avail_easyconfig_licenses_rst(): return '\n'.join(doc) +def avail_easyconfig_licenses_md(): + """Generate easyconfig license documentation in MarkDown format""" + title = "License constants that can be used in easyconfigs" + + table_titles = [ + "License name", + "License description", + "Version", + ] + + lics = sorted(EASYCONFIG_LICENSES_DICT.items()) + table_values = [ + ["``%s``" % lic().name for _, lic in lics], + ["%s" % lic().description for _, lic in lics], + ["``%s``" % lic().version for _, lic in lics], + ] + + doc = md_title_and_table(title, table_titles, table_values) + return '\n'.join(doc) + + def avail_easyconfig_params_md(title, grouped_params): """ Compose overview of available easyconfig parameters, in MarkDown format. diff --git a/test/framework/docs.py b/test/framework/docs.py index ff1e3ed11c..c97c665a75 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -483,6 +483,10 @@ def test_license_docs(self): regex = re.compile(r"^``GPLv3``\s*The GNU General Public License", re.M) self.assertTrue(regex.search(lic_docs), "%s found in: %s" % (regex.pattern, lic_docs)) + lic_docs = avail_easyconfig_licenses(output_format='md') + regex = re.compile(r"^``GPLv3``\s*|The GNU General Public License", re.M) + self.assertTrue(regex.search(lic_docs), "%s found in: %s" % (regex.pattern, lic_docs)) + def test_list_easyblocks(self): """ Tests for list_easyblocks function From bd612f3bfbc50a9d7392e6d19273af08fb722def Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 2 Jan 2023 19:57:38 +0100 Subject: [PATCH 13/23] implement support for '--avail-easyconfig-templates --output-format md' --- easybuild/tools/docs.py | 72 +++++++++++++++++++++++++++++++++++++++++ test/framework/docs.py | 53 ++++++++++++++++++++++++++++-- 2 files changed, 123 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index b210c9040b..5ef2d0e7a6 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -433,35 +433,41 @@ def avail_easyconfig_templates_txt(): doc.append('Template names/values derived from easyconfig instance') for name in TEMPLATE_NAMES_EASYCONFIG: doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1])) + doc.append('') # step 2: add SOFTWARE_VERSIONS doc.append('Template names/values for (short) software versions') for name, pref in TEMPLATE_SOFTWARE_VERSIONS: doc.append("%s%%(%sshortver)s: short version for %s (.)" % (INDENT_4SPACES, pref, name)) doc.append("%s%%(%sver)s: full version for %s" % (INDENT_4SPACES, pref, name)) + doc.append('') # step 3: add remaining config doc.append('Template names/values as set in easyconfig') for name in TEMPLATE_NAMES_CONFIG: doc.append("%s%%(%s)s" % (INDENT_4SPACES, name)) + doc.append('') # step 4: make lower variants doc.append('Lowercase values of template values') for name in TEMPLATE_NAMES_LOWER: template_name = TEMPLATE_NAMES_LOWER_TEMPLATE % {'name': name} doc.append("%s%%(%s)s: lower case of value of %s" % (INDENT_4SPACES, template_name, name)) + doc.append('') # step 5: template_values can/should be updated from outside easyconfig # (eg the run_step code in EasyBlock) doc.append('Template values set outside EasyBlock runstep') for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP: doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1])) + doc.append('') # some template values are only defined dynamically, # see template_constant_dict function in easybuild.framework.easyconfigs.templates doc.append('Template values which are defined dynamically') for name in TEMPLATE_NAMES_DYNAMIC: doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1])) + doc.append('') doc.append('Template constants that can be used in easyconfigs') for cst in TEMPLATE_CONSTANTS: @@ -533,6 +539,72 @@ def avail_easyconfig_templates_rst(): return '\n'.join(doc) +def avail_easyconfig_templates_md(): + """Returns template documentation in MarkDown format.""" + table_titles = ['Template name', 'Template value'] + + title = 'Template names/values derived from easyconfig instance' + table_values = [ + ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYCONFIG], + [name[1] for name in TEMPLATE_NAMES_EASYCONFIG], + ] + doc = md_title_and_table(title, table_titles, table_values) + doc.append('') + + title = 'Template names/values for (short) software versions' + ver = [] + ver_desc = [] + for name, pref in TEMPLATE_SOFTWARE_VERSIONS: + ver.append('``%%(%sshortver)s``' % pref) + ver.append('``%%(%sver)s``' % pref) + ver_desc.append('short version for %s (.)' % name) + ver_desc.append('full version for %s' % name) + table_values = [ver, ver_desc] + doc.extend(md_title_and_table(title, table_titles, table_values)) + doc.append('') + + title = 'Template names/values as set in easyconfig' + doc.extend([title, '-' * len(title), '']) + for name in TEMPLATE_NAMES_CONFIG: + doc.append('* ``%%(%s)s``' % name) + doc.append('') + + title = 'Lowercase values of template values' + table_values = [ + ['``%%(%s)s``' % (TEMPLATE_NAMES_LOWER_TEMPLATE % {'name': name}) for name in TEMPLATE_NAMES_LOWER], + ['lower case of value of %s' % name for name in TEMPLATE_NAMES_LOWER], + ] + doc.extend(md_title_and_table(title, table_titles, table_values)) + doc.append('') + + title = 'Template values set outside EasyBlock runstep' + table_values = [ + ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP], + [name[1] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP], + ] + doc.extend(md_title_and_table(title, table_titles, table_values)) + doc.append('') + + title = 'Template values which are defined dynamically' + table_values = [ + ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_DYNAMIC], + [name[1] for name in TEMPLATE_NAMES_DYNAMIC], + ] + doc.extend(md_title_and_table(title, table_titles, table_values)) + doc.append('') + + title = 'Template constants that can be used in easyconfigs' + titles = ['Constant', 'Template value', 'Template name'] + table_values = [ + ['``%s``' % cst[0] for cst in TEMPLATE_CONSTANTS], + [cst[2] for cst in TEMPLATE_CONSTANTS], + ['``%s``' % cst[1] for cst in TEMPLATE_CONSTANTS], + ] + doc.extend(md_title_and_table(title, titles, table_values)) + + return '\n'.join(doc) + + def avail_classes_tree(classes, class_names, locations, detailed, format_strings, depth=0): """Print list of classes as a tree.""" txt = [] diff --git a/test/framework/docs.py b/test/framework/docs.py index c97c665a75..3384f4e959 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -33,8 +33,8 @@ from easybuild.tools.config import module_classes from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses -from easybuild.tools.docs import gen_easyblocks_overview_rst, list_easyblocks, list_software, list_toolchains -from easybuild.tools.docs import md_title_and_table, rst_title_and_table +from easybuild.tools.docs import avail_easyconfig_templates, gen_easyblocks_overview_rst, list_easyblocks +from easybuild.tools.docs import list_software, list_toolchains, md_title_and_table, rst_title_and_table from easybuild.tools.options import EasyBuildOptions from easybuild.tools.utilities import import_available_modules, mk_md_table, mk_rst_table from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config @@ -683,6 +683,55 @@ def test_avail_easyconfig_constants(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + def test_avail_easyconfig_templates(self): + """ + Test avail_easyconfig_templates to generate overview of templates that can be used in easyconfig files. + """ + txt_patterns = [ + r"^Template names/values derived from easyconfig instance", + r"^\s+%\(version_major\)s: Major version", + r"^Template names/values for \(short\) software versions", + r"^\s+%\(pyshortver\)s: short version for Python \(\.\)", + r"^Template constants that can be used in easyconfigs", + r"^\s+SOURCE_TAR_GZ: Source \.tar\.gz bundle \(%\(name\)s-%\(version\)s.tar.gz\)", + ] + + txt = avail_easyconfig_templates() + for pattern in txt_patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt)) + + txt = avail_easyconfig_templates(output_format='txt') + for pattern in txt_patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt)) + + md_patterns = [ + r"^## Template names/values derived from easyconfig instance", + r"^``%\(version_major\)s``\s+|Major version", + r"^## Template names/values for \(short\) software versions", + r"^``%\(pyshortver\)s``\s+|short version for Python \(\.\)", + r"^## Template constants that can be used in easyconfigs", + r"^``SOURCE_TAR_GZ``\s+|Source \.tar\.gz bundle \(%\(name\)s-%\(version\)s.tar.gz\)", + ] + txt_md = avail_easyconfig_templates(output_format='md') + for pattern in md_patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt_md), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_md)) + + rst_patterns = [ + r"^Template names/values derived from easyconfig instance\n\-+", + r"^``%\(version_major\)s``\s+|Major version", + r"^Template names/values for \(short\) software versions\n-+", + r"^``%\(pyshortver\)s``\s+|short version for Python \(\.\)", + r"^Template constants that can be used in easyconfigs\n\-+", + r"^``SOURCE_TAR_GZ``\s+|Source \.tar\.gz bundle \(%\(name\)s-%\(version\)s.tar.gz\)", + ] + txt_rst = avail_easyconfig_templates(output_format='rst') + for pattern in rst_patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + def test_mk_table(self): """ Tests for mk_*_table functions. From 47e745d339092c1f382ba5ef9fd33997f0038665 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 2 Jan 2023 20:00:15 +0100 Subject: [PATCH 14/23] stop using sort_looseversions in list_software_md --- easybuild/tools/docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index a347e86e01..29f6eb24de 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -841,7 +841,7 @@ def list_software_md(software, detailed=True): # sort pairs by version (and then by versionsuffix); # we sort by LooseVersion to obtain chronological version ordering, # but we also need to retain original string version for filtering-by-version done below - sorted_pairs = sort_looseversions((LooseVersion(v), vs, v) for v, vs in pairs) + sorted_pairs = sorted((LooseVersion(v), vs, v) for v, vs in pairs) for _, vsuff, ver in sorted_pairs: table_values[0].append('``%s``' % ver) From 58300f567d3b83faf5483fe1fb6eb386e2b9e447 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 2 Jan 2023 20:18:25 +0100 Subject: [PATCH 15/23] implement support for '--list-toolchains --output-format md' + provide control over title level in md_title_and_table --- easybuild/tools/docs.py | 78 ++++++++++++++++++++++++++++++++++++----- test/framework/docs.py | 4 +-- 2 files changed, 71 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 29f6eb24de..c8bf72452c 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -81,12 +81,12 @@ def generate_doc(name, params): return func(*params) -def md_title_and_table(title, table_titles, table_values): +def md_title_and_table(title, table_titles, table_values, title_level=1): """Generate table in section with title in MarkDown (.md) format.""" doc = [] if title is not None: doc.extend([ - '## ' + title, + '#' * title_level + ' ' + title, '', ]) doc.extend(mk_md_table(table_titles, table_values)) @@ -317,7 +317,7 @@ def avail_easyconfig_params_md(title, grouped_params): ['`' + str(quote_str(x[1])) + '`' for x in values] # default value ] - doc.extend(md_title_and_table(title, table_titles, table_values)) + doc.extend(md_title_and_table(title, table_titles, table_values, title_level=2)) doc.append('') return '\n'.join(doc) @@ -548,7 +548,7 @@ def avail_easyconfig_templates_md(): ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYCONFIG], [name[1] for name in TEMPLATE_NAMES_EASYCONFIG], ] - doc = md_title_and_table(title, table_titles, table_values) + doc = md_title_and_table(title, table_titles, table_values, title_level=2) doc.append('') title = 'Template names/values for (short) software versions' @@ -560,7 +560,7 @@ def avail_easyconfig_templates_md(): ver_desc.append('short version for %s (.)' % name) ver_desc.append('full version for %s' % name) table_values = [ver, ver_desc] - doc.extend(md_title_and_table(title, table_titles, table_values)) + doc.extend(md_title_and_table(title, table_titles, table_values, title_level=2)) doc.append('') title = 'Template names/values as set in easyconfig' @@ -574,7 +574,7 @@ def avail_easyconfig_templates_md(): ['``%%(%s)s``' % (TEMPLATE_NAMES_LOWER_TEMPLATE % {'name': name}) for name in TEMPLATE_NAMES_LOWER], ['lower case of value of %s' % name for name in TEMPLATE_NAMES_LOWER], ] - doc.extend(md_title_and_table(title, table_titles, table_values)) + doc.extend(md_title_and_table(title, table_titles, table_values, title_level=2)) doc.append('') title = 'Template values set outside EasyBlock runstep' @@ -582,7 +582,7 @@ def avail_easyconfig_templates_md(): ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP], [name[1] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP], ] - doc.extend(md_title_and_table(title, table_titles, table_values)) + doc.extend(md_title_and_table(title, table_titles, table_values, title_level=2)) doc.append('') title = 'Template values which are defined dynamically' @@ -590,7 +590,7 @@ def avail_easyconfig_templates_md(): ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_DYNAMIC], [name[1] for name in TEMPLATE_NAMES_DYNAMIC], ] - doc.extend(md_title_and_table(title, table_titles, table_values)) + doc.extend(md_title_and_table(title, table_titles, table_values, title_level=2)) doc.append('') title = 'Template constants that can be used in easyconfigs' @@ -600,7 +600,7 @@ def avail_easyconfig_templates_md(): [cst[2] for cst in TEMPLATE_CONSTANTS], ['``%s``' % cst[1] for cst in TEMPLATE_CONSTANTS], ] - doc.extend(md_title_and_table(title, titles, table_values)) + doc.extend(md_title_and_table(title, titles, table_values, title_level=2)) return '\n'.join(doc) @@ -1038,6 +1038,66 @@ def list_toolchains(output_format=FORMAT_TXT): return generate_doc('list_toolchains_%s' % output_format, [tcs]) +def list_toolchains_md(tcs): + """Returns overview of all toolchains in MarkDown format""" + title = "List of known toolchains" + + # Specify the column names for the table + table_titles = ['NAME', 'COMPILER', 'MPI', 'LINALG', 'FFT'] + + # Set up column name : display name pairs + col_names = { + 'NAME': 'Name', + 'COMPILER': 'Compiler(s)', + 'LINALG': "Linear algebra", + } + + # Create sorted list of toolchain names + sorted_tc_names = sorted(tcs.keys(), key=str.lower) + + # Create text placeholder to use for missing entries + none_txt = '*(none)*' + + # Initialize an empty list of lists for the table data + table_values = [[] for i in range(len(table_titles))] + + for col_id, col_name in enumerate(table_titles): + if col_name == 'NAME': + # toolchain names column gets bold face entry + table_values[col_id] = ['**%s**' % tcname for tcname in sorted_tc_names] + else: + for tc_name in sorted_tc_names: + tc = tcs[tc_name] + if 'cray' in tc_name.lower(): + if col_name == 'COMPILER': + entry = ', '.join(tc[col_name.upper()]) + elif col_name == 'MPI': + entry = 'cray-mpich' + elif col_name == 'LINALG': + entry = 'cray-libsci' + # Combine the linear algebra libraries into a single column + elif col_name == 'LINALG': + linalg = [] + for col in ['BLAS', 'LAPACK', 'SCALAPACK']: + linalg.extend(tc.get(col, [])) + entry = ', '.join(nub(linalg)) or none_txt + else: + # for other columns, we can grab the values via 'tc' + # key = col_name + entry = ', '.join(tc.get(col_name, [])) or none_txt + table_values[col_id].append(entry) + + # Set the table titles to the pretty ones + table_titles = [col_names.get(col, col) for col in table_titles] + + # Pass the data to the rst formatter, wich is returned as a list, each element + # is an rst formatted text row. + doc = md_title_and_table(title, table_titles, table_values) + + # Make a string with line endings suitable to write to document file + return '\n'.join(doc) + + def list_toolchains_rst(tcs): """ Returns overview of all toolchains in rst format """ title = "List of known toolchains" diff --git a/test/framework/docs.py b/test/framework/docs.py index 3384f4e959..3dd419860a 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -662,7 +662,7 @@ def test_avail_easyconfig_constants(self): self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt)) md_patterns = [ - r"^## Constants that can be used in easyconfigs", + r"^# Constants that can be used in easyconfigs", r"^``ARCH``\s*\|``.*``\s*\|CPU architecture of current system \(aarch64, x86_64, ppc64le, ...\)$", r"^``OS_PKG_OPENSSL_DEV``\s*\|``\('openssl-devel', 'libssl-dev', 'libopenssl-devel'\)``\s*\|" r"OS packages providing openSSL developement support$", @@ -798,7 +798,7 @@ def test_title_and_table(self): '====== ====== ==================', '', ] - res = md_title_and_table('test title', table[0], table[1:]) + res = md_title_and_table('test title', table[0], table[1:], title_level=2) self.assertEqual(res, expected_md) res = rst_title_and_table('test title', table[0], table[1:]) From 7c2bec6c1de4e51b8504f9a8c4eca9001a84c766 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 2 Jan 2023 20:30:32 +0100 Subject: [PATCH 16/23] add test for list_toolchains --- easybuild/tools/docs.py | 2 +- test/framework/docs.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index c8bf72452c..b871071dcc 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1160,7 +1160,7 @@ def list_toolchains_rst(tcs): def list_toolchains_txt(tcs): """ Returns overview of all toolchains in txt format """ - doc = ["List of known toolchains (toolchainname: module[,module...]):"] + doc = ["List of known toolchains (toolchain name: module[, module, ...]):"] for name in sorted(tcs): tc_elems = nub(sorted([e for es in tcs[name].values() for e in es])) doc.append("\t%s: %s" % (name, ', '.join(tc_elems))) diff --git a/test/framework/docs.py b/test/framework/docs.py index 3dd419860a..2567747a1a 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -596,6 +596,46 @@ def test_list_software(self): expected_found = any(lines[i:i + len(expected)] == expected for i in range(len(lines))) self.assertTrue(expected_found, "%s found in: %s" % (expected, lines)) + def test_list_toolchains(self): + """Test list_toolchains* functions.""" + + txt_patterns = [ + r"^List of known toolchains \(toolchain name: module\[, module, ...\]\):", + r"^\s+GCC: GCC", + r"^\s+foss: BLACS, FFTW, GCC, OpenBLAS, OpenMPI, ScaLAPACK", + r"^\s+intel: icc, ifort, imkl, impi", + r"^\s+system:\s*$", + ] + + for txt in (list_toolchains(), list_toolchains(output_format='txt')): + for pattern in txt_patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt)) + + md_patterns = [ + r"^# List of known toolchains", + r"^\*\*GCC\*\*\s+\|GCC\s+\|\*\(none\)\*\s+\|\*\(none\)\*\s+\|\*\(none\)\*$", + r"^\*\*foss\*\*\s+\|GCC\s+\|OpenMPI\s+\|OpenBLAS, ScaLAPACK\s+\|FFTW$", + r"^\*\*intel\*\*\s+\|icc, ifort\s+\|impi\s+\|imkl\s+\|imkl", + r"^\*\*system\*\*\s+\|\*\(none\)\*\s+\|\*\(none\)\*\s+\|\*\(none\)\*\s+\|\*\(none\)\*$", + ] + txt_md = list_toolchains(output_format='md') + for pattern in md_patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt_md), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_md)) + + rst_patterns = [ + r"^List of known toolchains\n\-{24}", + r"^\*\*GCC\*\*\s+GCC\s+\*\(none\)\*\s+\*\(none\)\*\s+\*\(none\)\*$", + r"^\*\*foss\*\*\s+GCC\s+OpenMPI\s+OpenBLAS, ScaLAPACK\s+FFTW$", + r"^\*\*intel\*\*\s+icc, ifort\s+impi\s+imkl\s+imkl", + r"^\*\*system\*\*\s+\*\(none\)\*\s+\*\(none\)\*\s+\*\(none\)\*\s+\*\(none\)\*$", + ] + txt_rst = list_toolchains(output_format='rst') + for pattern in rst_patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + def test_avail_cfgfile_constants(self): """ Test avail_cfgfile_constants to generate overview of constants that can be used in a configuration file. From 60f3aa806b52533a18a133f32ca92d71b11aa6e4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 3 Jan 2023 16:15:24 +0100 Subject: [PATCH 17/23] implement support for '--avail-toolchain-opts ... --output-format md' --- easybuild/tools/docs.py | 18 +++++++++ test/framework/docs.py | 89 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 105 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index b871071dcc..2be23e93f1 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1185,6 +1185,24 @@ def avail_toolchain_opts(name, output_format=FORMAT_TXT): return generate_doc('avail_toolchain_opts_%s' % output_format, [name, tc_dict]) +def avail_toolchain_opts_md(name, tc_dict): + """ Returns overview of toolchain options in MarkDown format """ + title = "Available options for %s toolchain" % name + + table_titles = ['option', 'description', 'default'] + + tc_items = sorted(tc_dict.items()) + table_values = [ + ['``%s``' % val[0] for val in tc_items], + ['%s' % val[1][1] for val in tc_items], + ['``%s``' % val[1][0] for val in tc_items], + ] + + doc = md_title_and_table(title, table_titles, table_values, title_level=2) + + return '\n'.join(doc) + + def avail_toolchain_opts_rst(name, tc_dict): """ Returns overview of toolchain options in rst format """ title = "Available options for %s toolchain" % name diff --git a/test/framework/docs.py b/test/framework/docs.py index 2567747a1a..2e86bac9da 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -33,8 +33,9 @@ from easybuild.tools.config import module_classes from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses -from easybuild.tools.docs import avail_easyconfig_templates, gen_easyblocks_overview_rst, list_easyblocks -from easybuild.tools.docs import list_software, list_toolchains, md_title_and_table, rst_title_and_table +from easybuild.tools.docs import avail_easyconfig_templates, avail_toolchain_opts, gen_easyblocks_overview_rst +from easybuild.tools.docs import list_easyblocks, list_software, list_toolchains, md_title_and_table, rst_title_and_table +from easybuild.tools.docs import rst_title_and_table from easybuild.tools.options import EasyBuildOptions from easybuild.tools.utilities import import_available_modules, mk_md_table, mk_rst_table from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config @@ -772,6 +773,90 @@ def test_avail_easyconfig_templates(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + def test_avail_toolchain_opts(self): + """ + Test avail_toolchain_opts to generate overview of supported toolchain options. + """ + txt_patterns_foss = [ + r"^Available options for foss toolchain:", + r"^\s+extra_cxxflags: Specify extra CXXFLAGS options. \(default: None\)", + r"^\s+optarch: Enable architecture optimizations \(default: True\)", + r"^\s+precise: High precision \(default: False\)", + ] + oneapi_txt = r"^\s+oneapi: Use oneAPI compilers icx/icpx/ifx instead of classic compilers \(default: None\)" + + for txt in (avail_toolchain_opts('foss'), avail_toolchain_opts('foss', output_format='txt')): + for pattern in txt_patterns_foss: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt)) + + regex = re.compile(oneapi_txt, re.M) + self.assertFalse(regex.search(txt), "Pattern '%s' should not be found in: %s" % (regex.pattern, txt)) + + txt_patterns_intel = [ + r"^Available options for intel toolchain:", + oneapi_txt, + ] + txt_patterns_foss[1:] + + for txt in (avail_toolchain_opts('intel'), avail_toolchain_opts('intel', output_format='txt')): + for pattern in txt_patterns_intel: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt)) + + # MarkDown output format + md_patterns_foss = [ + r"^## Available options for foss toolchain", + r"^``extra_cxxflags``\s+\|Specify extra CXXFLAGS options.\s+\|``None``", + r"^``optarch``\s+\|Enable architecture optimizations\s+\|``True``", + r"^``precise``\s+\|High precision\s+\|``False``", + ] + + txt_md = avail_toolchain_opts('foss', output_format='md') + for pattern in md_patterns_foss: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt_md), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_md)) + + oneapi_md = r"^``oneapi``\s+\|Use oneAPI compilers icx/icpx/ifx instead of classic compilers\s+\|``None``" + regex = re.compile(oneapi_md, re.M) + self.assertFalse(regex.search(txt_md), "Pattern '%s' should not be found in: %s" % (regex.pattern, txt_md)) + + md_patterns_intel = [ + r"^## Available options for intel toolchain", + oneapi_md, + ] + md_patterns_foss[1:] + + txt_md = avail_toolchain_opts('intel', output_format='md') + for pattern in md_patterns_intel: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt_md), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_md)) + + # rst output format + rst_patterns_foss = [ + r"^Available options for foss toolchain\n-{36}", + r"^``extra_cxxflags``\s+Specify extra CXXFLAGS options.\s+``None``", + r"^``optarch``\s+Enable architecture optimizations\s+``True``", + r"^``precise``\s+High precision\s+``False``", + ] + + txt_rst = avail_toolchain_opts('foss', output_format='rst') + for pattern in rst_patterns_foss: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + + oneapi_rst = r"^``oneapi``\s+Use oneAPI compilers icx/icpx/ifx instead of classic compilers\s+``None``" + regex = re.compile(oneapi_rst, re.M) + self.assertFalse(regex.search(txt_rst), "Pattern '%s' should not be found in: %s" % (regex.pattern, txt_rst)) + + rst_patterns_intel = [ + r"^Available options for intel toolchain\n-{37}", + oneapi_rst, + ] + rst_patterns_foss[1:] + + txt_rst = avail_toolchain_opts('intel', output_format='rst') + for pattern in rst_patterns_intel: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + def test_mk_table(self): """ Tests for mk_*_table functions. From 63df90a7b16a0abfa8ab5b55e2af0509bc2d3429 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 3 Jan 2023 18:25:00 +0100 Subject: [PATCH 18/23] add support for 'eb --help=md' + dedicated test --- easybuild/base/generaloption.py | 43 ++++++++++++++++- test/framework/docs.py | 84 ++++++++++++++++++++++++++++++++- 2 files changed, 123 insertions(+), 4 deletions(-) diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index 4895ed90e7..61c2b8b9d4 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -44,7 +44,7 @@ from easybuild.base.fancylogger import getLogger, setroot, setLogLevel, getDetailsLogLevels from easybuild.base.optcomplete import autocomplete, CompleterOption from easybuild.tools.py2vs3 import StringIO, configparser, string_type, subprocess_popen_text -from easybuild.tools.utilities import mk_rst_table, nub, shell_quote +from easybuild.tools.utilities import mk_md_table, mk_rst_table, nub, shell_quote try: import gettext @@ -65,7 +65,7 @@ def _gettext(message): return message -HELP_OUTPUT_FORMATS = ['', 'rst', 'short', 'config'] +HELP_OUTPUT_FORMATS = ['', 'md', 'rst', 'short', 'config'] def set_columns(cols=None): @@ -638,6 +638,45 @@ def print_help(self, fh=None): fh = self.check_help(fh) OptionParser.print_help(self, fh) + def print_mdhelp(self, fh=None): + """Print help in MarkDown format""" + fh = self.check_help(fh) + result = [] + if self.usage: + result.extend(["## Usage", '', '``%s``' % self.get_usage().replace("Usage: ", '').strip(), '']) + if self.description: + result.extend(["## Description", '', self.description, '']) + + result.append(self.format_option_mdhelp()) + + mdhelptxt = '\n'.join(result) + if fh is None: + fh = sys.stdout + fh.write(mdhelptxt) + + def format_option_mdhelp(self, formatter=None): + """ Formatting for help in rst format """ + if not formatter: + formatter = self.formatter + formatter.store_option_strings(self) + + res = [] + titles = ["Option flag", "Option description"] + + all_opts = [("Help options", self.option_list)] + \ + [(group.title, group.option_list) for group in self.option_groups] + for title, opts in all_opts: + values = [] + res.extend(['## ' + title, '']) + for opt in opts: + if opt.help is not nohelp: + values.append(['``%s``' % formatter.option_strings[opt], formatter.expand_default(opt)]) + + res.extend(mk_md_table(titles, map(list, zip(*values)))) + res.append('') + + return '\n'.join(res) + def print_rsthelp(self, fh=None): """ Print help in rst format """ fh = self.check_help(fh) diff --git a/test/framework/docs.py b/test/framework/docs.py index 2e86bac9da..bb13314cfe 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -34,8 +34,8 @@ from easybuild.tools.config import module_classes from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses from easybuild.tools.docs import avail_easyconfig_templates, avail_toolchain_opts, gen_easyblocks_overview_rst -from easybuild.tools.docs import list_easyblocks, list_software, list_toolchains, md_title_and_table, rst_title_and_table -from easybuild.tools.docs import rst_title_and_table +from easybuild.tools.docs import list_easyblocks, list_software, list_toolchains +from easybuild.tools.docs import md_title_and_table, rst_title_and_table from easybuild.tools.options import EasyBuildOptions from easybuild.tools.utilities import import_available_modules, mk_md_table, mk_rst_table from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config @@ -933,6 +933,86 @@ def test_title_and_table(self): self.assertErrorRegex(ValueError, error_pattern, md_title_and_table, '', titles, []) self.assertErrorRegex(ValueError, error_pattern, rst_title_and_table, '', titles, [('val 11', 'val 12')]) + def test_help(self): + """ + Test output produced by --help, with various output formats + """ + def get_eb_help_output(arg=''): + self.mock_stderr(True) + self.mock_stdout(True) + self.eb_main(['--help', arg]) + stderr = self.get_stderr() + stdout = self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertFalse(stderr) + return stdout + + txt_patterns = [ + r"^Usage: eb \[options\] easyconfig \[...\]", + r"^Options:\n\s+--version", + r"^\s+Basic options:\n\s+Basic runtime options for EasyBuild", + r"^\s+-f, --force\s+Force to rebuild software", + r"^\s+--module-only\s+Only generate module file\(s\)", + r"^\s+Software search and build options:", + r"^\s+--try-toolchain=NAME,VERSION", + r"^Boolean options support disable prefix", + r"^All long option names can be passed as environment variables", + ] + txt = get_eb_help_output() + for pattern in txt_patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt)) + + short_patterns = [ + r"^Usage: eb \[options\] easyconfig \[...\]", + r"^Options:\n\s+-h", + r"^\s+Basic options:\n\s+Basic runtime options for EasyBuild", + r"^\s+-f\s+Force to rebuild software", + r"^\s+Override options:\n\s+Override default EasyBuild behavior", + r"^\s+-e CLASS\s+easyblock to use", + r"^Boolean options support disable prefix", + r"^All long option names can be passed as environment variables", + ] + txt_short = get_eb_help_output('short') + for pattern in short_patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt_short), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_short)) + + config_patterns = [ + r"^\[MAIN\]\n# Enable debug log mode \(default: False\)\n#debug=", + r"^\[override\](\n.*)+#filter-deps=", + ] + txt_cfg = get_eb_help_output('config') + for pattern in config_patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt_cfg), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_cfg)) + + md_patterns = [ + r"^## Usage\n\n``eb \[options\] easyconfig \[...\]``", + r"^## Basic options", + r"^``-f, --force``\s+\|Force to rebuild software", + r"^## Override options", + r"^``-e CLASS, --easyblock=CLASS``\s+\|easyblock to use", + ] + txt_md = get_eb_help_output('md') + for pattern in md_patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt_md), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_md)) + + rst_patterns = [ + r"^Usage\n-{5}\n\n``eb \[options\] easyconfig \[...\]``", + r"^Basic options\n-{13}", + r"^``-f, --force``\s+Force to rebuild software", + r"^Override options\n-{16}", + r"^``-e CLASS, --easyblock=CLASS``\s+easyblock to use", + ] + txt_rst = get_eb_help_output('rst') + for pattern in rst_patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + def suite(): """ returns all test cases in this module """ From ff639936949701b9fb57ec09439adc92efe44114 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 3 Jan 2023 20:38:25 +0100 Subject: [PATCH 19/23] fix test_templating_doc --- test/framework/easyconfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 331fc9b7e4..b2a96788b4 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1261,7 +1261,7 @@ def test_python_whl_templating(self): def test_templating_doc(self): """test templating documentation""" doc = avail_easyconfig_templates() - # expected length: 1 per constant and 1 extra per constantgroup + # expected length: 1 per constant and 2 extra per constantgroup (title + empty line in between) temps = [ easyconfig.templates.TEMPLATE_NAMES_EASYCONFIG, easyconfig.templates.TEMPLATE_SOFTWARE_VERSIONS * 2, @@ -1272,7 +1272,7 @@ def test_templating_doc(self): easyconfig.templates.TEMPLATE_CONSTANTS, ] - self.assertEqual(len(doc.split('\n')), sum([len(temps)] + [len(x) for x in temps])) + self.assertEqual(len(doc.split('\n')), sum([2 * len(temps) - 1] + [len(x) for x in temps])) def test_constant_doc(self): """test constant documentation""" From f136c93901d4ae10da6c5378fe60c13b32112dcf Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 3 Jan 2023 20:59:55 +0100 Subject: [PATCH 20/23] add support for generating overview of easyblocks in MarkDown format with gen_easyblocks_overview_md --- easybuild/tools/docs.py | 145 +++++++++++++++++++++++++++++++++++++--- test/framework/docs.py | 65 +++++++++++++++++- 2 files changed, 197 insertions(+), 13 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 2be23e93f1..4848935b0b 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1230,24 +1230,57 @@ def avail_toolchain_opts_txt(name, tc_dict): return '\n'.join(doc) -def gen_easyblocks_overview_rst(package_name, path_to_examples, common_params={}, doc_functions=[]): +def get_easyblock_classes(package_name): """ - Compose overview of all easyblocks in the given package in rst format + Get list of all easyblock classes in specified easyblocks.* package """ + easyblocks = [] modules = import_available_modules(package_name) - doc = [] - all_blocks = [] - # get all blocks for mod in modules: for name, obj in inspect.getmembers(mod, inspect.isclass): eb_class = getattr(mod, name) # skip imported classes that are not easyblocks - if eb_class.__module__.startswith(package_name) and eb_class not in all_blocks: - all_blocks.append(eb_class) + if eb_class.__module__.startswith(package_name) and eb_class not in easyblocks: + easyblocks.append(eb_class) + + return easyblocks + + +def gen_easyblocks_overview_md(package_name, path_to_examples, common_params={}, doc_functions=[]): + """ + Compose overview of all easyblocks in the given package in MarkDown format + """ + eb_classes = get_easyblock_classes(package_name) + + eb_links = [] + for eb_class in sorted(eb_classes, key=lambda c: c.__name__): + eb_name = eb_class.__name__ + eb_links.append("" + eb_name + "") + + heading = [ + "# Overview of generic easyblocks", + '', + ' - '.join(eb_links), + '', + ] + + doc = [] + for eb_class in sorted(eb_classes, key=lambda c: c.__name__): + doc.extend(gen_easyblock_doc_section_md(eb_class, path_to_examples, common_params, doc_functions, eb_classes)) + + return heading + doc + + +def gen_easyblocks_overview_rst(package_name, path_to_examples, common_params={}, doc_functions=[]): + """ + Compose overview of all easyblocks in the given package in rst format + """ + eb_classes = get_easyblock_classes(package_name) - for eb_class in sorted(all_blocks, key=lambda c: c.__name__): - doc.extend(gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params, doc_functions, all_blocks)) + doc = [] + for eb_class in sorted(eb_classes, key=lambda c: c.__name__): + doc.extend(gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params, doc_functions, eb_classes)) title = 'Overview of generic easyblocks' @@ -1260,7 +1293,7 @@ def gen_easyblocks_overview_rst(package_name, path_to_examples, common_params={} '', ] - contents = [":ref:`" + b.__name__ + "`" for b in sorted(all_blocks, key=lambda b: b.__name__)] + contents = [":ref:`" + b.__name__ + "`" for b in sorted(eb_classes, key=lambda b: b.__name__)] toc = ' - '.join(contents) heading.append(toc) heading.append('') @@ -1268,6 +1301,98 @@ def gen_easyblocks_overview_rst(package_name, path_to_examples, common_params={} return heading + doc +def gen_easyblock_doc_section_md(eb_class, path_to_examples, common_params, doc_functions, all_eb_classes): + """ + Compose overview of one easyblock given class object of the easyblock in MarkDown format + """ + classname = eb_class.__name__ + + doc = [ + "", + '', + '## ``' + classname + '``', + '', + ] + + bases = [] + for base in eb_class.__bases__: + bname = base.__name__ + if base in all_eb_classes: + bases.append("``" + bname + "``") + else: + bases.append('``' + bname + '``') + + derived = '(derives from ' + ', '.join(bases) + ')' + doc.extend([derived, '']) + + # Description (docstring) + eb_docstring = eb_class.__doc__ + if eb_docstring is not None: + doc.extend(x.lstrip() for x in eb_docstring.splitlines()) + doc.append('') + + # Add extra options, if any + if eb_class.extra_options(): + title = "Extra easyconfig parameters specific to ``%s`` easyblock" % classname + ex_opt = eb_class.extra_options() + keys = sorted(ex_opt.keys()) + values = [ex_opt[k] for k in keys] + + table_titles = ['easyconfig parameter', 'description', 'default value'] + table_values = [ + ['``' + key + '``' for key in keys], # parameter name + [val[1] for val in values], # description + ['``' + str(quote_str(val[0])) + '``' for val in values] # default value + ] + + doc.extend(md_title_and_table(title, table_titles, table_values, title_level=3)) + doc.append('') + + # Add commonly used parameters + if classname in common_params: + title = "Commonly used easyconfig parameters with ``%s`` easyblock" % classname + + table_titles = ['easyconfig parameter', 'description'] + table_values = [ + [opt for opt in common_params[classname]], + [DEFAULT_CONFIG[opt][1] for opt in common_params[classname]], + ] + + doc.extend(md_title_and_table(title, table_titles, table_values, title_level=3)) + doc.append('') + + # Add docstring for custom steps + custom = [] + inh = '' + f = None + for func in doc_functions: + if func in eb_class.__dict__: + f = eb_class.__dict__[func] + + if f.__doc__: + custom.append('* ``' + func + '`` - ' + f.__doc__.strip() + inh) + custom.append('') + + if custom: + doc.append("### Customised steps in ``" + classname + "`` easyblock") + doc.extend(custom) + doc.append('') + + # Add example if available + example_ec = os.path.join(path_to_examples, '%s.eb' % classname) + if os.path.exists(example_ec): + doc.extend([ + "### Example easyconfig for ``" + classname + "`` easyblock", + '', + '```python', + read_file(example_ec), + '```', + '', + ]) + + return doc + + def gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params, doc_functions, all_blocks): """ Compose overview of one easyblock given class object of the easyblock in rst format diff --git a/test/framework/docs.py b/test/framework/docs.py index bb13314cfe..563074369f 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -33,7 +33,8 @@ from easybuild.tools.config import module_classes from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses -from easybuild.tools.docs import avail_easyconfig_templates, avail_toolchain_opts, gen_easyblocks_overview_rst +from easybuild.tools.docs import avail_easyconfig_templates, avail_toolchain_opts +from easybuild.tools.docs import get_easyblock_classes, gen_easyblocks_overview_md, gen_easyblocks_overview_rst from easybuild.tools.docs import list_easyblocks, list_software, list_toolchains from easybuild.tools.docs import md_title_and_table, rst_title_and_table from easybuild.tools.options import EasyBuildOptions @@ -411,8 +412,19 @@ class DocsTest(EnhancedTestCase): - def test_gen_easyblocks(self): - """ Test gen_easyblocks_overview_rst function """ + def test_get_easyblock_classes(self): + """ + Test for get_easyblock_classes function. + """ + # result should correspond with test easyblocks in test/framework/sandbox/easybuild/easyblocks/generic + eb_classes = get_easyblock_classes('easybuild.easyblocks.generic') + eb_names = [x.__name__ for x in eb_classes] + expected = ['ConfigureMake', 'DummyExtension', 'MakeCp', 'ModuleRC', + 'PythonBundle', 'Toolchain', 'Toy_Extension', 'bar'] + self.assertEqual(sorted(eb_names), expected) + + def test_gen_easyblocks_overview(self): + """ Test gen_easyblocks_overview_* functions """ gen_easyblocks_pkg = 'easybuild.easyblocks.generic' modules = import_available_modules(gen_easyblocks_pkg) common_params = { @@ -474,6 +486,53 @@ def test_gen_easyblocks(self): regex = re.compile(pattern) self.assertTrue(re.search(regex, ebdoc), "Pattern %s found in %s" % (regex.pattern, ebdoc)) + # MarkDown format + eb_overview = gen_easyblocks_overview_md(gen_easyblocks_pkg, 'easyconfigs', common_params, doc_functions) + ebdoc = '\n'.join(eb_overview) + + # extensive check for ConfigureMake easyblock + check_configuremake = '\n'.join([ + "", + '', + "## ``ConfigureMake``", + '', + "(derives from ``EasyBlock``)", + '', + "Dummy support for building and installing applications with configure/make/make install.", + '', + "### Extra easyconfig parameters specific to ``ConfigureMake`` easyblock", + '', + "easyconfig parameter|description |default value", + "--------------------|------------|-------------", + '``test_123`` |Test 1, 2, 3|``""``', + "``test_bool`` |Just a test |``False``", + "``test_none`` |Another test|``None``", + '', + "### Commonly used easyconfig parameters with ``ConfigureMake`` easyblock", + '', + "easyconfig parameter|description", + "--------------------|----------------------------------------------------------------", + "configopts |Extra options passed to configure (default already has --prefix)", + "buildopts |Extra options passed to make step (default already has -j X)", + "installopts |Extra options for installation", + ]) + + self.assertTrue(check_configuremake in ebdoc, "Found '%s' in: %s" % (check_configuremake, ebdoc)) + names = [] + + for mod in modules: + for name, obj in inspect.getmembers(mod, inspect.isclass): + eb_class = getattr(mod, name) + # skip imported classes that are not easyblocks + if eb_class.__module__.startswith(gen_easyblocks_pkg): + self.assertTrue(name in ebdoc) + names.append(name) + + toc = ["" + n + "" for n in sorted(set(names))] + pattern = " - ".join(toc) + regex = re.compile(pattern) + self.assertTrue(re.search(regex, ebdoc), "Pattern %s found in %s" % (regex.pattern, ebdoc)) + def test_license_docs(self): """Test license_documentation function.""" lic_docs = avail_easyconfig_licenses(output_format='txt') From 4ba33e48150ccb35fe3463590b31ce2b82f7084e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 3 Jan 2023 21:03:53 +0100 Subject: [PATCH 21/23] do not use mutable data structures for argument defaults in gen_easyblocks_overview_* functions --- easybuild/tools/docs.py | 18 ++++++++++++++---- test/framework/docs.py | 4 ++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 4848935b0b..960b99ddd6 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1238,7 +1238,7 @@ def get_easyblock_classes(package_name): modules = import_available_modules(package_name) for mod in modules: - for name, obj in inspect.getmembers(mod, inspect.isclass): + for name, _ in inspect.getmembers(mod, inspect.isclass): eb_class = getattr(mod, name) # skip imported classes that are not easyblocks if eb_class.__module__.startswith(package_name) and eb_class not in easyblocks: @@ -1247,10 +1247,15 @@ def get_easyblock_classes(package_name): return easyblocks -def gen_easyblocks_overview_md(package_name, path_to_examples, common_params={}, doc_functions=[]): +def gen_easyblocks_overview_md(package_name, path_to_examples, common_params=None, doc_functions=None): """ Compose overview of all easyblocks in the given package in MarkDown format """ + if common_params is None: + common_params = {} + if doc_functions is None: + doc_functions = [] + eb_classes = get_easyblock_classes(package_name) eb_links = [] @@ -1272,10 +1277,15 @@ def gen_easyblocks_overview_md(package_name, path_to_examples, common_params={}, return heading + doc -def gen_easyblocks_overview_rst(package_name, path_to_examples, common_params={}, doc_functions=[]): +def gen_easyblocks_overview_rst(package_name, path_to_examples, common_params=None, doc_functions=None): """ Compose overview of all easyblocks in the given package in rst format """ + if common_params is None: + common_params = {} + if doc_functions is None: + doc_functions = [] + eb_classes = get_easyblock_classes(package_name) doc = [] @@ -1382,7 +1392,7 @@ def gen_easyblock_doc_section_md(eb_class, path_to_examples, common_params, doc_ example_ec = os.path.join(path_to_examples, '%s.eb' % classname) if os.path.exists(example_ec): doc.extend([ - "### Example easyconfig for ``" + classname + "`` easyblock", + "### Example easyconfig for ``" + classname + "`` easyblock", '', '```python', read_file(example_ec), diff --git a/test/framework/docs.py b/test/framework/docs.py index 563074369f..43e8866ccb 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -473,7 +473,7 @@ def test_gen_easyblocks_overview(self): names = [] for mod in modules: - for name, obj in inspect.getmembers(mod, inspect.isclass): + for name, _ in inspect.getmembers(mod, inspect.isclass): eb_class = getattr(mod, name) # skip imported classes that are not easyblocks if eb_class.__module__.startswith(gen_easyblocks_pkg): @@ -521,7 +521,7 @@ def test_gen_easyblocks_overview(self): names = [] for mod in modules: - for name, obj in inspect.getmembers(mod, inspect.isclass): + for name, _ in inspect.getmembers(mod, inspect.isclass): eb_class = getattr(mod, name) # skip imported classes that are not easyblocks if eb_class.__module__.startswith(gen_easyblocks_pkg): From ba20c2b8cac16588d6afaa5dad9d0f01be95f9fd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 3 Jan 2023 21:32:43 +0100 Subject: [PATCH 22/23] fix test__list_toolchains --- test/framework/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 3578106483..4b007a2704 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -673,9 +673,9 @@ def test__list_toolchains(self): ] self.eb_main(args, logfile=dummylogfn, raise_error=True) - info_msg = r"INFO List of known toolchains \(toolchainname: module\[,module\.\.\.\]\):" + regex = re.compile(r"INFO List of known toolchains \(toolchain name: module\[, module, \.\.\.\]\):") logtxt = read_file(self.logfile) - self.assertTrue(re.search(info_msg, logtxt), "Info message with list of known toolchains found in: %s" % logtxt) + self.assertTrue(regex.search(logtxt), "Pattern '%s' should be found in: %s" % (regex.pattern, logtxt)) # toolchain elements should be in alphabetical order tcs = { 'system': [], From dd50ce63c569b0489bbab3d1d581e1f25addf14f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 3 Jan 2023 22:11:40 +0100 Subject: [PATCH 23/23] drop useless anchor in MarkDown version of easyblocks overview --- easybuild/tools/docs.py | 2 -- test/framework/docs.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 960b99ddd6..17f391aea3 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1318,8 +1318,6 @@ def gen_easyblock_doc_section_md(eb_class, path_to_examples, common_params, doc_ classname = eb_class.__name__ doc = [ - "", - '', '## ``' + classname + '``', '', ] diff --git a/test/framework/docs.py b/test/framework/docs.py index 43e8866ccb..da5f7bd882 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -492,8 +492,6 @@ def test_gen_easyblocks_overview(self): # extensive check for ConfigureMake easyblock check_configuremake = '\n'.join([ - "", - '', "## ``ConfigureMake``", '', "(derives from ``EasyBlock``)",