zero config

envs/docker/Dockerfile

@@ -1,10 +1,17 @@
+# Usage:
+# docker build -t fab .
+# docker run --env PYTHONPATH=/fab/source -v /home/byron/git/fab:/fab -v /home/byron:/home/byron -it fab bash
+
 FROM ubuntu:20.04
 
-RUN apt update && apt install -y gcc gfortran libclang-dev python-clang python3-pip rsync
+RUN apt update && apt install -y gcc gfortran libclang-dev python-clang python3-pip rsync git
 
 RUN mkdir -p ~/.local/lib/python3.8/site-packages
 RUN cp -vr /usr/lib/python3/dist-packages/clang ~/.local/lib/python3.8/site-packages/
 
-RUN pip install flake8 fparser matplotlib mypy pytest sphinx sphinx_rtd_theme
+RUN pip install pytest pytest-cov pytest-mock flake8 mypy
+RUN pip install sphinx sphinx_rtd_theme sphinx-autodoc-typehints
+RUN pip install svn GitPython matplotlib
+RUN pip install fparser
 
 CMD [ "python3", "--version" ]

setup.py

@@ -54,5 +54,10 @@
         'tests': tests,
         'docs': docs,
         'dev': [*tests, *docs, *features],
-    }
+    },
+    entry_points={
+        'console_scripts': [
+            'fab=fab.cli:cli_fab'
+        ]
+    },
 )

source/fab/cli.py

@@ -0,0 +1,83 @@
+# ##############################################################################
+#  (c) Crown copyright Met Office. All rights reserved.
+#  For further details please refer to the file COPYRIGHT
+#  which you should have received as part of this distribution
+# ##############################################################################
+from argparse import ArgumentParser
+from pathlib import Path
+from typing import Dict, Optional
+
+import fab
+from fab.artefacts import CollectionGetter
+from fab.build_config import BuildConfig
+from fab.constants import PRAGMAD_C
+from fab.steps.analyse import Analyse
+from fab.steps.c_pragma_injector import CPragmaInjector
+from fab.steps.compile_c import CompileC
+from fab.steps.compile_fortran import CompileFortran, get_fortran_compiler, get_fortran_preprocessor
+from fab.steps.find_source_files import FindSourceFiles
+from fab.steps.grab.folder import GrabFolder
+from fab.steps.link import LinkExe
+from fab.steps.preprocess import c_preprocessor, fortran_preprocessor
+from fab.steps.root_inc_files import RootIncFiles
+
+
+def _generic_build_config(folder: Path, kwargs: Optional[Dict] = None) -> BuildConfig:
+    folder = folder.resolve()
+    kwargs = kwargs or {}
+
+    # Within the fab workspace, we'll create a project workspace.
+    # Ideally we'd just use folder.name, but to avoid clashes, we'll use the full absolute path.
+    label = '/'.join(folder.parts[1:])
+
+    fpp, fpp_flags = get_fortran_preprocessor()
+    fc, fc_flags = get_fortran_compiler()
+
+    if fc == 'gfortran':
+        link_step = LinkExe(linker='gcc', flags=['-lgfortran'])
+    else:
+        raise NotImplementedError(f"Fab's zero config not yet configured for compiler: '{fc}'")
+
+    config = BuildConfig(
+        project_label=label,
+        steps=[
+            GrabFolder(folder),
+            FindSourceFiles(),
+
+            RootIncFiles(),  # JULES helper, get rid of this eventually
+
+            fortran_preprocessor(preprocessor=fpp, common_flags=fpp_flags),
+
+            CPragmaInjector(),
+            c_preprocessor(source=CollectionGetter(PRAGMAD_C)),
+
+            Analyse(find_programs=True),
+
+            CompileFortran(compiler=fc, common_flags=fc_flags),
+            CompileC(),
+
+            link_step,
+        ],
+        **kwargs,
+    )
+
+    return config
+
+
+def cli_fab():
+    """
+    Running Fab from the command line will attempt to build the project in the current or given folder.
+
+    """
+    arg_parser = ArgumentParser()
+    arg_parser.add_argument('folder', nargs='?', default='.', type=Path)
+    arg_parser.add_argument('-v', '--version', action='store_true')
+    args = arg_parser.parse_args()
+
+    if args.version:
+        print('Fab', fab.__version__)
+        exit(0)
+
+    config = _generic_build_config(args.folder)
+
+    config.run()

source/fab/parse/fortran.py

@@ -37,6 +37,7 @@ class AnalysedFortran(AnalysedDependent):
 
     """
     def __init__(self, fpath: Union[str, Path], file_hash: Optional[int] = None,
+                 program_defs: Optional[Iterable[str]] = None,
                  module_defs: Optional[Iterable[str]] = None, symbol_defs: Optional[Iterable[str]] = None,
                  module_deps: Optional[Iterable[str]] = None, symbol_deps: Optional[Iterable[str]] = None,
                  mo_commented_file_deps: Optional[Iterable[str]] = None, file_deps: Optional[Iterable[Path]] = None,
@@ -46,6 +47,8 @@ def __init__(self, fpath: Union[str, Path], file_hash: Optional[int] = None,
             The source file that was analysed.
         :param file_hash:
             The hash of the source. If omitted, Fab will evaluate lazily.
+        :param program_defs:
+            Set of program names defined by this source file.
         :param module_defs:
             Set of module names defined by this source file.
             A subset of symbol_defs
@@ -69,6 +72,7 @@ def __init__(self, fpath: Union[str, Path], file_hash: Optional[int] = None,
         super().__init__(fpath=fpath, file_hash=file_hash,
                          symbol_defs=symbol_defs, symbol_deps=symbol_deps, file_deps=file_deps)
 
+        self.program_defs: Set[str] = set(program_defs or [])
         self.module_defs: Set[str] = set(module_defs or [])
         self.module_deps: Set[str] = set(module_deps or [])
         self.mo_commented_file_deps: Set[str] = set(mo_commented_file_deps or [])
@@ -79,6 +83,10 @@ def __init__(self, fpath: Union[str, Path], file_hash: Optional[int] = None,
 
         self.validate()
 
+    def add_program_def(self, name):
+        self.program_defs.add(name.lower())
+        self.add_symbol_def(name)
+
     def add_module_def(self, name):
         self.module_defs.add(name.lower())
         self.add_symbol_def(name)
@@ -97,6 +105,7 @@ def field_names(cls):
         # we're not using the super class because we want to insert, not append the order of our attributes
         return [
             'fpath', 'file_hash',
+            'program_defs',
             'module_defs', 'symbol_defs',
             'module_deps', 'symbol_deps',
             'mo_commented_file_deps',
@@ -109,6 +118,7 @@ def to_dict(self) -> Dict[str, Any]:
         # We sort the lists for reproducibility in testing.
         result = super().to_dict()
         result.update({
+            "program_defs": list(sorted(self.program_defs)),
             "module_defs": list(sorted(self.module_defs)),
             "module_deps": list(sorted(self.module_deps)),
             "mo_commented_file_deps": list(sorted(self.mo_commented_file_deps)),
@@ -122,6 +132,7 @@ def from_dict(cls, d):
         result = cls(
             fpath=Path(d["fpath"]),
             file_hash=d["file_hash"],
+            program_defs=set(d["program_defs"]),
             module_defs=set(d["module_defs"]),
             symbol_defs=set(d["symbol_defs"]),
             module_deps=set(d["module_deps"]),
@@ -137,12 +148,14 @@ def from_dict(cls, d):
     def validate(self):
         assert self.file_hash is not None
 
+        assert all([d and len(d) for d in self.program_defs]), "bad program definitions"
         assert all([d and len(d) for d in self.module_defs]), "bad module definitions"
         assert all([d and len(d) for d in self.symbol_defs]), "bad symbol definitions"
         assert all([d and len(d) for d in self.module_deps]), "bad module dependencies"
         assert all([d and len(d) for d in self.symbol_deps]), "bad symbol dependencies"
 
         # todo: this feels a little clanky.
+        assert self.program_defs <= self.symbol_defs, "programs definitions must also be symbol definitions"
         assert self.module_defs <= self.symbol_defs, "modules definitions must also be symbol definitions"
         assert self.module_deps <= self.symbol_deps, "modules dependencies must also be symbol dependencies"
 
@@ -185,7 +198,7 @@ def walk_nodes(self, fpath, file_hash, node_tree) -> AnalysedFortran:
                         analysed_fortran.add_symbol_dep(called_name.string)
 
                 elif obj_type == Program_Stmt:
-                    analysed_fortran.add_symbol_def(str(obj.get_name()))
+                    analysed_fortran.add_program_def(str(obj.get_name()))
 
                 elif obj_type == Module_Stmt:
                     analysed_fortran.add_module_def(str(obj.get_name()))

source/fab/steps/analyse.py

@@ -33,16 +33,18 @@
 You'll have to manually read the file to determine which symbol definitions and dependencies it contains.
 
 """
-
+from itertools import chain
 import logging
 import sys
 import warnings
 from pathlib import Path
 from typing import Dict, List, Iterable, Set, Optional, Union
 
-from fab.parse.c import CAnalyser
+from fab import FabException
+
+from fab.parse.c import AnalysedC, CAnalyser
 
-from fab.parse.fortran import FortranParserWorkaround, FortranAnalyser
+from fab.parse.fortran import AnalysedFortran, FortranParserWorkaround, FortranAnalyser
 
 from fab.constants import BUILD_TREES, CURRENT_PREBUILDS
 from fab.dep_tree import extract_sub_tree, validate_dependencies, AnalysedDependent
@@ -79,7 +81,8 @@ class Analyse(Step):
     # todo: allow the user to specify a different output artefact collection name?
     def __init__(self,
                  source: Optional[ArtefactsGetter] = None,
-                 root_symbol: Optional[Union[str, List[str]]] = None,  # todo: iterable is more correct
+                 root_symbol: Optional[Union[str, List[str]]] = None,
+                 find_programs: bool = False,
                  std: str = "f2008",
                  special_measure_analysis_results: Optional[Iterable[FortranParserWorkaround]] = None,
                  unreferenced_deps: Optional[Iterable[str]] = None,
@@ -96,6 +99,9 @@ def __init__(self,
 
         :param source:
             An :class:`~fab.util.ArtefactsGetter` to get the source files.
+        :param find_programs:
+            Instructs the analyser to automatically identify program definitions in the source.
+            Alternatively, the required programs can be specified with the root_symbol argument.
         :param root_symbol:
             When building an executable, provide the Fortran Program name(s), or 'main' for C.
             If None, build tree extraction will not be performed and the entire source will be used
@@ -122,8 +128,12 @@ def __init__(self,
         # because the files they refer to probably don't exist yet,
         # because we're just creating steps at this point, so there's been no grab...
 
+        if find_programs and root_symbol:
+            raise ValueError("find_programs and root_symbol can't be used together")
+
         super().__init__(name)
         self.source_getter = source or DEFAULT_SOURCE_GETTER
+        self.find_programs = find_programs
         self.root_symbols: Optional[List[str]] = [root_symbol] if isinstance(root_symbol, str) else root_symbol
         self.special_measure_analysis_results: List[FortranParserWorkaround] = \
             list(special_measure_analysis_results or [])
@@ -167,6 +177,21 @@ def run(self, artefact_store: Dict, config):
         analysed_files = self._parse_files(files=files)
         self._add_manual_results(analysed_files)
 
+        # shall we search the results for fortran programs and a c function called main?
+        if self.find_programs:
+            # find fortran programs
+            sets_of_programs = [af.program_defs for af in by_type(analysed_files, AnalysedFortran)]
+            self.root_symbols = list(chain(*sets_of_programs))
+
+            # find c main()
+            c_with_main = list(filter(lambda c: 'main' in c.symbol_defs, by_type(analysed_files, AnalysedC)))
+            if c_with_main:
+                self.root_symbols.append('main')
+                if len(c_with_main) > 1:
+                    raise FabException("multiple c main() functions found")
+
+            logger.info(f'automatically found the following programs to build: {", ".join(self.root_symbols)}')
+
         # analyse
         project_source_tree, symbols = self._analyse_dependencies(analysed_files)
 

source/fab/steps/compile_fortran.py

@@ -346,6 +346,42 @@ def compile_file(self, analysed_file, flags, output_fpath):
             value={'time_taken': timer.taken, 'start': timer.start})
 
 
+def get_fortran_preprocessor():
+    fpp, fpp_flags = None, None
+
+    try:
+        fpp, fpp_flags = get_tool(os.getenv('FPP'))
+        logger.info(f'env defined fpp as "{fpp}"')
+    except ValueError:
+        pass
+
+    if not fpp:
+        try:
+            run_command(['which', 'fpp'])
+            fpp, fpp_flags = 'fpp', ['-P']
+            logger.info('detected fpp')
+        except RuntimeError:
+            # fpp not available
+            pass
+
+    if not fpp:
+        try:
+            run_command(['which', 'cpp'])
+            fpp, fpp_flags = 'cpp', ['-traditional-cpp', '-P']
+            logger.info('detected cpp')
+        except RuntimeError:
+            # fpp not available
+            pass
+
+    if not fpp:
+        raise RuntimeError('no fortran preprocessor specified or discovered')
+
+    if '-P' not in fpp_flags:
+        fpp_flags.append('-P')
+
+    return fpp, fpp_flags
+
+
 def get_fortran_compiler(compiler: Optional[str] = None):
     """
     Get the fortran compiler specified by the `$FC` environment variable,
@@ -356,8 +392,38 @@ def get_fortran_compiler(compiler: Optional[str] = None):
     :param compiler:
         Use this string instead of the $FC environment variable.
 
+    Returns the tool and a list of flags.
+
     """
-    return get_tool(compiler or os.getenv('FC', ''))  # type: ignore
+    fortran_compiler = None
+    try:
+        fortran_compiler = get_tool(compiler or os.getenv('FC', ''))  # type: ignore
+    except ValueError:
+        # tool not specified
+        pass
+
+    if not fortran_compiler:
+        try:
+            run_command(['gfortran', '--help'])
+            fortran_compiler = 'gfortran', []
+            logger.info('detected gfortran')
+        except RuntimeError:
+            # gfortran not available
+            pass
+
+    if not fortran_compiler:
+        try:
+            run_command(['ifort', '--help'])
+            fortran_compiler = 'ifort', []
+            logger.info('detected ifort')
+        except RuntimeError:
+            # gfortran not available
+            pass
+
+    if not fortran_compiler:
+        raise RuntimeError('no fortran compiler specified or discovered')
+
+    return fortran_compiler
 
 
 def get_mod_hashes(analysed_files: Set[AnalysedFortran], config) -> Dict[str, int]:

source/fab/tools.py

@@ -9,7 +9,7 @@
 """
 import subprocess
 import warnings
-from typing import Dict, List, Tuple
+from typing import Dict, List, Optional, Tuple
 
 from fab.util import logger, string_checksum
 
@@ -104,7 +104,7 @@ def run_command(command, env=None, cwd=None, capture_output=True):
         return res.stdout.decode()
 
 
-def get_tool(tool_str: str = '') -> Tuple[str, List[str]]:
+def get_tool(tool_str: Optional[str] = None) -> Tuple[str, List[str]]:
     """
     Get the compiler, preprocessor, etc, from the given string.
 
@@ -116,6 +116,8 @@ def get_tool(tool_str: str = '') -> Tuple[str, List[str]]:
         The environment variable from which to find the tool.
 
     """
+    tool_str = tool_str or ''
+
     tool_split = tool_str.split()
     if not tool_split:
         raise ValueError(f"Tool not specified in '{tool_str}'. Cannot continue.")