Skip to content

Commit

Permalink
add support for generating overview of easyblocks in MarkDown format …
Browse files Browse the repository at this point in the history 
…with gen_easyblocks_overview_md
  • Loading branch information
@boegel
boegel committed Jan 3, 2023
1 parent ff63993 commit f136c93
Show file tree
Hide file tree
Showing 2 changed files with 197 additions and 13 deletions.
145 changes: 135 additions & 10 deletions easybuild/tools/docs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1230,24 +1230,57 @@ def avail_toolchain_opts_txt(name, tc_dict):
return '\n'.join(doc)


def gen_easyblocks_overview_rst(package_name, path_to_examples, common_params={}, doc_functions=[]):
def get_easyblock_classes(package_name):
"""
Compose overview of all easyblocks in the given package in rst format
Get list of all easyblock classes in specified easyblocks.* package
"""
easyblocks = []
modules = import_available_modules(package_name)
doc = []
all_blocks = []

# get all blocks
for mod in modules:
for name, obj in inspect.getmembers(mod, inspect.isclass):
eb_class = getattr(mod, name)
# skip imported classes that are not easyblocks
if eb_class.__module__.startswith(package_name) and eb_class not in all_blocks:
all_blocks.append(eb_class)
if eb_class.__module__.startswith(package_name) and eb_class not in easyblocks:
easyblocks.append(eb_class)

return easyblocks


def gen_easyblocks_overview_md(package_name, path_to_examples, common_params={}, doc_functions=[]):
"""
Compose overview of all easyblocks in the given package in MarkDown format
"""
eb_classes = get_easyblock_classes(package_name)

eb_links = []
for eb_class in sorted(eb_classes, key=lambda c: c.__name__):
eb_name = eb_class.__name__
eb_links.append("<a href='#" + eb_name.lower() + "'>" + eb_name + "</a>")

heading = [
"# Overview of generic easyblocks",
'',
' - '.join(eb_links),
'',
]

doc = []
for eb_class in sorted(eb_classes, key=lambda c: c.__name__):
doc.extend(gen_easyblock_doc_section_md(eb_class, path_to_examples, common_params, doc_functions, eb_classes))

return heading + doc


def gen_easyblocks_overview_rst(package_name, path_to_examples, common_params={}, doc_functions=[]):
"""
Compose overview of all easyblocks in the given package in rst format
"""
eb_classes = get_easyblock_classes(package_name)

for eb_class in sorted(all_blocks, key=lambda c: c.__name__):
doc.extend(gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params, doc_functions, all_blocks))
doc = []
for eb_class in sorted(eb_classes, key=lambda c: c.__name__):
doc.extend(gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params, doc_functions, eb_classes))

title = 'Overview of generic easyblocks'

Expand All @@ -1260,14 +1293,106 @@ def gen_easyblocks_overview_rst(package_name, path_to_examples, common_params={}
'',
]

contents = [":ref:`" + b.__name__ + "`" for b in sorted(all_blocks, key=lambda b: b.__name__)]
contents = [":ref:`" + b.__name__ + "`" for b in sorted(eb_classes, key=lambda b: b.__name__)]
toc = ' - '.join(contents)
heading.append(toc)
heading.append('')

return heading + doc


def gen_easyblock_doc_section_md(eb_class, path_to_examples, common_params, doc_functions, all_eb_classes):
"""
Compose overview of one easyblock given class object of the easyblock in MarkDown format
"""
classname = eb_class.__name__

doc = [
"<a anchor='#" + classname.lower() + "'/>",
'',
'## ``' + classname + '``',
'',
]

bases = []
for base in eb_class.__bases__:
bname = base.__name__
if base in all_eb_classes:
bases.append("<a href='#" + bname.lower() + "'>``" + bname + "``</a>")
else:
bases.append('``' + bname + '``')

derived = '(derives from ' + ', '.join(bases) + ')'
doc.extend([derived, ''])

# Description (docstring)
eb_docstring = eb_class.__doc__
if eb_docstring is not None:
doc.extend(x.lstrip() for x in eb_docstring.splitlines())
doc.append('')

# Add extra options, if any
if eb_class.extra_options():
title = "Extra easyconfig parameters specific to ``%s`` easyblock" % classname
ex_opt = eb_class.extra_options()
keys = sorted(ex_opt.keys())
values = [ex_opt[k] for k in keys]

table_titles = ['easyconfig parameter', 'description', 'default value']
table_values = [
['``' + key + '``' for key in keys], # parameter name
[val[1] for val in values], # description
['``' + str(quote_str(val[0])) + '``' for val in values] # default value
]

doc.extend(md_title_and_table(title, table_titles, table_values, title_level=3))
doc.append('')

# Add commonly used parameters
if classname in common_params:
title = "Commonly used easyconfig parameters with ``%s`` easyblock" % classname

table_titles = ['easyconfig parameter', 'description']
table_values = [
[opt for opt in common_params[classname]],
[DEFAULT_CONFIG[opt][1] for opt in common_params[classname]],
]

doc.extend(md_title_and_table(title, table_titles, table_values, title_level=3))
doc.append('')

# Add docstring for custom steps
custom = []
inh = ''
f = None
for func in doc_functions:
if func in eb_class.__dict__:
f = eb_class.__dict__[func]

if f.__doc__:
custom.append('* ``' + func + '`` - ' + f.__doc__.strip() + inh)
custom.append('')

if custom:
doc.append("### Customised steps in ``" + classname + "`` easyblock")
doc.extend(custom)
doc.append('')

# Add example if available
example_ec = os.path.join(path_to_examples, '%s.eb' % classname)
if os.path.exists(example_ec):
doc.extend([
"### Example easyconfig for ``" + classname + "`` easyblock",
'',
'```python',
read_file(example_ec),
'```',
'',
])

return doc


def gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params, doc_functions, all_blocks):
"""
Compose overview of one easyblock given class object of the easyblock in rst format
Expand Down
65 changes: 62 additions & 3 deletions test/framework/docs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@

from easybuild.tools.config import module_classes
from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses
from easybuild.tools.docs import avail_easyconfig_templates, avail_toolchain_opts, gen_easyblocks_overview_rst
from easybuild.tools.docs import avail_easyconfig_templates, avail_toolchain_opts
from easybuild.tools.docs import get_easyblock_classes, gen_easyblocks_overview_md, gen_easyblocks_overview_rst
from easybuild.tools.docs import list_easyblocks, list_software, list_toolchains
from easybuild.tools.docs import md_title_and_table, rst_title_and_table
from easybuild.tools.options import EasyBuildOptions
Expand Down Expand Up @@ -411,8 +412,19 @@

class DocsTest(EnhancedTestCase):

def test_gen_easyblocks(self):
""" Test gen_easyblocks_overview_rst function """
def test_get_easyblock_classes(self):
"""
Test for get_easyblock_classes function.
"""
# result should correspond with test easyblocks in test/framework/sandbox/easybuild/easyblocks/generic
eb_classes = get_easyblock_classes('easybuild.easyblocks.generic')
eb_names = [x.__name__ for x in eb_classes]
expected = ['ConfigureMake', 'DummyExtension', 'MakeCp', 'ModuleRC',
'PythonBundle', 'Toolchain', 'Toy_Extension', 'bar']
self.assertEqual(sorted(eb_names), expected)

def test_gen_easyblocks_overview(self):
""" Test gen_easyblocks_overview_* functions """
gen_easyblocks_pkg = 'easybuild.easyblocks.generic'
modules = import_available_modules(gen_easyblocks_pkg)
common_params = {
Expand Down Expand Up @@ -474,6 +486,53 @@ def test_gen_easyblocks(self):
regex = re.compile(pattern)
self.assertTrue(re.search(regex, ebdoc), "Pattern %s found in %s" % (regex.pattern, ebdoc))

# MarkDown format
eb_overview = gen_easyblocks_overview_md(gen_easyblocks_pkg, 'easyconfigs', common_params, doc_functions)
ebdoc = '\n'.join(eb_overview)

# extensive check for ConfigureMake easyblock
check_configuremake = '\n'.join([
"<a anchor='#configuremake'/>",
'',
"## ``ConfigureMake``",
'',
"(derives from ``EasyBlock``)",
'',
"Dummy support for building and installing applications with configure/make/make install.",
'',
"### Extra easyconfig parameters specific to ``ConfigureMake`` easyblock",
'',
"easyconfig parameter|description |default value",
"--------------------|------------|-------------",
'``test_123`` |Test 1, 2, 3|``""``',
"``test_bool`` |Just a test |``False``",
"``test_none`` |Another test|``None``",
'',
"### Commonly used easyconfig parameters with ``ConfigureMake`` easyblock",
'',
"easyconfig parameter|description",
"--------------------|----------------------------------------------------------------",
"configopts |Extra options passed to configure (default already has --prefix)",
"buildopts |Extra options passed to make step (default already has -j X)",
"installopts |Extra options for installation",
])

self.assertTrue(check_configuremake in ebdoc, "Found '%s' in: %s" % (check_configuremake, ebdoc))
names = []

for mod in modules:
for name, obj in inspect.getmembers(mod, inspect.isclass):
eb_class = getattr(mod, name)
# skip imported classes that are not easyblocks
if eb_class.__module__.startswith(gen_easyblocks_pkg):
self.assertTrue(name in ebdoc)
names.append(name)

toc = ["<a href='#" + n.lower() + "'>" + n + "</a>" for n in sorted(set(names))]
pattern = " - ".join(toc)
regex = re.compile(pattern)
self.assertTrue(re.search(regex, ebdoc), "Pattern %s found in %s" % (regex.pattern, ebdoc))

def test_license_docs(self):
"""Test license_documentation function."""
lic_docs = avail_easyconfig_licenses(output_format='txt')
Expand Down

0 comments on commit f136c93

Please sign in to comment.