Initial implementation of zero configuration mode (#226)

setup.py

@@ -55,5 +55,10 @@
         'tests': tests,
         'docs': docs,
         'dev': [*tests, *docs, *features],
-    }
+    },
+    entry_points={
+        'console_scripts': [
+            'fab=fab.cli:cli_fab'
+        ]
+    },
 )

source/fab/cli.py

@@ -0,0 +1,84 @@
+# ##############################################################################
+#  (c) Crown copyright Met Office. All rights reserved.
+#  For further details please refer to the file COPYRIGHT
+#  which you should have received as part of this distribution
+# ##############################################################################
+from argparse import ArgumentParser
+from pathlib import Path
+from typing import Dict, Optional
+
+import fab
+from fab.artefacts import CollectionGetter
+from fab.build_config import BuildConfig
+from fab.constants import PRAGMAD_C
+from fab.steps.analyse import Analyse
+from fab.steps.c_pragma_injector import CPragmaInjector
+from fab.steps.compile_c import CompileC
+from fab.steps.compile_fortran import CompileFortran, get_fortran_compiler, get_fortran_preprocessor
+from fab.steps.find_source_files import FindSourceFiles
+from fab.steps.grab.folder import GrabFolder
+from fab.steps.link import LinkExe
+from fab.steps.preprocess import c_preprocessor, fortran_preprocessor
+from fab.steps.root_inc_files import RootIncFiles
+
+
+def _generic_build_config(folder: Path, kwargs: Optional[Dict] = None) -> BuildConfig:
+    folder = folder.resolve()
+    kwargs = kwargs or {}
+
+    # Within the fab workspace, we'll create a project workspace.
+    # Ideally we'd just use folder.name, but to avoid clashes, we'll use the full absolute path.
+    label = '/'.join(folder.parts[1:])
+
+    fpp, fpp_flags = get_fortran_preprocessor()
+    fc, fc_flags = get_fortran_compiler()
+
+    # linker and flags depend on compiler
+    linkers = {
+        'gfortran': ('gcc', ['-lgfortran']),
+        # 'ifort': (..., [...])
+    }
+    try:
+        linker, linker_flags = linkers[fc]
+    except KeyError:
+        raise NotImplementedError(f"Fab's zero configuration mode does not yet work with compiler '{fc}'")
+
+    config = BuildConfig(
+        project_label=label,
+        steps=[
+            GrabFolder(folder),
+            FindSourceFiles(),
+
+            RootIncFiles(),  # JULES helper, get rid of this eventually
+
+            fortran_preprocessor(preprocessor=fpp, common_flags=fpp_flags),
+
+            CPragmaInjector(),
+            c_preprocessor(source=CollectionGetter(PRAGMAD_C)),
+
+            Analyse(find_programs=True),
+
+            CompileFortran(compiler=fc, common_flags=fc_flags),
+            CompileC(),
+
+            LinkExe(linker=linker, flags=linker_flags),
+        ],
+        **kwargs,
+    )
+
+    return config
+
+
+def cli_fab():
+    """
+    Running Fab from the command line will attempt to build the project in the current or given folder.
+
+    """
+    arg_parser = ArgumentParser()
+    arg_parser.add_argument('folder', nargs='?', default='.', type=Path)
+    arg_parser.add_argument('--version', action='version', version=f'%(prog)s {fab.__version__}')
+    args = arg_parser.parse_args()
+
+    config = _generic_build_config(args.folder)
+
+    config.run()

source/fab/parse/fortran.py

@@ -11,7 +11,6 @@
 from pathlib import Path
 from typing import Union, Optional, Iterable, Dict, Any, Set
 
-from fab.dep_tree import AnalysedDependent
 from fparser.two.Fortran2003 import (  # type: ignore
     Use_Stmt, Module_Stmt, Program_Stmt, Subroutine_Stmt, Function_Stmt, Language_Binding_Spec,
     Char_Literal_Constant, Interface_Block, Name, Comment, Module, Call_Stmt, Derived_Type_Def, Derived_Type_Stmt,
@@ -21,6 +20,7 @@
 from fparser.two.Fortran2008 import (  # type: ignore
     Type_Declaration_Stmt, Attr_Spec_List, Entity_Decl_List)
 
+from fab.dep_tree import AnalysedDependent
 from fab.parse.fortran_common import iter_content, _has_ancestor_type, _typed_child, FortranAnalyserBase
 from fab.util import file_checksum, string_checksum
 
@@ -37,6 +37,7 @@ class AnalysedFortran(AnalysedDependent):
 
     """
     def __init__(self, fpath: Union[str, Path], file_hash: Optional[int] = None,
+                 program_defs: Optional[Iterable[str]] = None,
                  module_defs: Optional[Iterable[str]] = None, symbol_defs: Optional[Iterable[str]] = None,
                  module_deps: Optional[Iterable[str]] = None, symbol_deps: Optional[Iterable[str]] = None,
                  mo_commented_file_deps: Optional[Iterable[str]] = None, file_deps: Optional[Iterable[Path]] = None,
@@ -46,6 +47,8 @@ def __init__(self, fpath: Union[str, Path], file_hash: Optional[int] = None,
             The source file that was analysed.
         :param file_hash:
             The hash of the source. If omitted, Fab will evaluate lazily.
+        :param program_defs:
+            Set of program names defined by this source file.
         :param module_defs:
             Set of module names defined by this source file.
             A subset of symbol_defs
@@ -69,6 +72,7 @@ def __init__(self, fpath: Union[str, Path], file_hash: Optional[int] = None,
         super().__init__(fpath=fpath, file_hash=file_hash,
                          symbol_defs=symbol_defs, symbol_deps=symbol_deps, file_deps=file_deps)
 
+        self.program_defs: Set[str] = set(program_defs or [])
         self.module_defs: Set[str] = set(module_defs or [])
         self.module_deps: Set[str] = set(module_deps or [])
         self.mo_commented_file_deps: Set[str] = set(mo_commented_file_deps or [])
@@ -79,6 +83,10 @@ def __init__(self, fpath: Union[str, Path], file_hash: Optional[int] = None,
 
         self.validate()
 
+    def add_program_def(self, name):
+        self.program_defs.add(name.lower())
+        self.add_symbol_def(name)
+
     def add_module_def(self, name):
         self.module_defs.add(name.lower())
         self.add_symbol_def(name)
@@ -97,6 +105,7 @@ def field_names(cls):
         # we're not using the super class because we want to insert, not append the order of our attributes
         return [
             'fpath', 'file_hash',
+            'program_defs',
             'module_defs', 'symbol_defs',
             'module_deps', 'symbol_deps',
             'mo_commented_file_deps',
@@ -109,6 +118,7 @@ def to_dict(self) -> Dict[str, Any]:
         # We sort the lists for reproducibility in testing.
         result = super().to_dict()
         result.update({
+            "program_defs": list(sorted(self.program_defs)),
             "module_defs": list(sorted(self.module_defs)),
             "module_deps": list(sorted(self.module_deps)),
             "mo_commented_file_deps": list(sorted(self.mo_commented_file_deps)),
@@ -122,6 +132,7 @@ def from_dict(cls, d):
         result = cls(
             fpath=Path(d["fpath"]),
             file_hash=d["file_hash"],
+            program_defs=set(d["program_defs"]),
             module_defs=set(d["module_defs"]),
             symbol_defs=set(d["symbol_defs"]),
             module_deps=set(d["module_deps"]),
@@ -137,12 +148,14 @@ def from_dict(cls, d):
     def validate(self):
         assert self.file_hash is not None
 
+        assert all([d and len(d) for d in self.program_defs]), "bad program definitions"
         assert all([d and len(d) for d in self.module_defs]), "bad module definitions"
         assert all([d and len(d) for d in self.symbol_defs]), "bad symbol definitions"
         assert all([d and len(d) for d in self.module_deps]), "bad module dependencies"
         assert all([d and len(d) for d in self.symbol_deps]), "bad symbol dependencies"
 
         # todo: this feels a little clanky.
+        assert self.program_defs <= self.symbol_defs, "programs definitions must also be symbol definitions"
         assert self.module_defs <= self.symbol_defs, "modules definitions must also be symbol definitions"
         assert self.module_deps <= self.symbol_deps, "modules dependencies must also be symbol dependencies"
 
@@ -185,7 +198,7 @@ def walk_nodes(self, fpath, file_hash, node_tree) -> AnalysedFortran:
                         analysed_fortran.add_symbol_dep(called_name.string)
 
                 elif obj_type == Program_Stmt:
-                    analysed_fortran.add_symbol_def(str(obj.get_name()))
+                    analysed_fortran.add_program_def(str(obj.get_name()))
 
                 elif obj_type == Module_Stmt:
                     analysed_fortran.add_module_def(str(obj.get_name()))

source/fab/steps/analyse.py

@@ -33,20 +33,21 @@
 You'll have to manually read the file to determine which symbol definitions and dependencies it contains.
 
 """
-
+from itertools import chain
 import logging
 import sys
 import warnings
 from pathlib import Path
 from typing import Dict, List, Iterable, Set, Optional, Union
 
+from fab import FabException
 from fab.artefacts import ArtefactsGetter, CollectionConcat, SuffixFilter
 from fab.constants import BUILD_TREES
-from fab.dep_tree import AnalysedDependent, extract_sub_tree, validate_dependencies
+from fab.dep_tree import extract_sub_tree, validate_dependencies, AnalysedDependent
 from fab.mo import add_mo_commented_file_deps
 from fab.parse import AnalysedFile, EmptySourceFile
-from fab.parse.c import CAnalyser
-from fab.parse.fortran import FortranParserWorkaround, FortranAnalyser
+from fab.parse.c import AnalysedC, CAnalyser
+from fab.parse.fortran import AnalysedFortran, FortranParserWorkaround, FortranAnalyser
 from fab.steps import Step
 from fab.util import TimerLogger, by_type
 
@@ -78,7 +79,8 @@ class Analyse(Step):
     # todo: allow the user to specify a different output artefact collection name?
     def __init__(self,
                  source: Optional[ArtefactsGetter] = None,
-                 root_symbol: Optional[Union[str, List[str]]] = None,  # todo: iterable is more correct
+                 root_symbol: Optional[Union[str, List[str]]] = None,
+                 find_programs: bool = False,
                  std: str = "f2008",
                  special_measure_analysis_results: Optional[Iterable[FortranParserWorkaround]] = None,
                  unreferenced_deps: Optional[Iterable[str]] = None,
@@ -95,6 +97,9 @@ def __init__(self,
 
         :param source:
             An :class:`~fab.util.ArtefactsGetter` to get the source files.
+        :param find_programs:
+            Instructs the analyser to automatically identify program definitions in the source.
+            Alternatively, the required programs can be specified with the root_symbol argument.
         :param root_symbol:
             When building an executable, provide the Fortran Program name(s), or 'main' for C.
             If None, build tree extraction will not be performed and the entire source will be used
@@ -121,8 +126,12 @@ def __init__(self,
         # because the files they refer to probably don't exist yet,
         # because we're just creating steps at this point, so there's been no grab...
 
+        if find_programs and root_symbol:
+            raise ValueError("find_programs and root_symbol can't be used together")
+
         super().__init__(name)
         self.source_getter = source or DEFAULT_SOURCE_GETTER
+        self.find_programs = find_programs
         self.root_symbols: Optional[List[str]] = [root_symbol] if isinstance(root_symbol, str) else root_symbol
         self.special_measure_analysis_results: List[FortranParserWorkaround] = \
             list(special_measure_analysis_results or [])
@@ -166,6 +175,21 @@ def run(self, artefact_store: Dict, config):
         analysed_files = self._parse_files(files=files)
         self._add_manual_results(analysed_files)
 
+        # shall we search the results for fortran programs and a c function called main?
+        if self.find_programs:
+            # find fortran programs
+            sets_of_programs = [af.program_defs for af in by_type(analysed_files, AnalysedFortran)]
+            self.root_symbols = list(chain(*sets_of_programs))
+
+            # find c main()
+            c_with_main = list(filter(lambda c: 'main' in c.symbol_defs, by_type(analysed_files, AnalysedC)))
+            if c_with_main:
+                self.root_symbols.append('main')
+                if len(c_with_main) > 1:
+                    raise FabException("multiple c main() functions found")
+
+            logger.info(f'automatically found the following programs to build: {", ".join(self.root_symbols)}')
+
         # analyse
         project_source_tree, symbol_table = self._analyse_dependencies(analysed_files)
 

source/fab/steps/compile_fortran.py

@@ -344,6 +344,53 @@ def compile_file(self, analysed_file, flags, output_fpath):
             value={'time_taken': timer.taken, 'start': timer.start})
 
 
+def get_fortran_preprocessor():
+    """
+    Identify the fortran preprocessor and any flags from the environment.
+
+    Initially looks for the `FPP` environment variable, then tries to call the `fpp` and `cpp` command line tools.
+
+    Returns the executable and flags.
+
+    The returned flags will always include `-P` to suppress line numbers.
+    This fparser ticket requests line number handling https://github.com/stfc/fparser/issues/390 .
+
+    """
+    fpp, fpp_flags = None, None
+
+    try:
+        fpp, fpp_flags = get_tool(os.getenv('FPP'))
+        logger.info(f"The environment defined FPP as '{fpp}'")
+    except ValueError:
+        pass
+
+    if not fpp:
+        try:
+            run_command(['which', 'fpp'])
+            fpp, fpp_flags = 'fpp', ['-P']
+            logger.info('detected fpp')
+        except RuntimeError:
+            # fpp not available
+            pass
+
+    if not fpp:
+        try:
+            run_command(['which', 'cpp'])
+            fpp, fpp_flags = 'cpp', ['-traditional-cpp', '-P']
+            logger.info('detected cpp')
+        except RuntimeError:
+            # fpp not available
+            pass
+
+    if not fpp:
+        raise RuntimeError('no fortran preprocessor specified or discovered')
+
+    if '-P' not in fpp_flags:
+        fpp_flags.append('-P')
+
+    return fpp, fpp_flags
+
+
 def get_fortran_compiler(compiler: Optional[str] = None):
     """
     Get the fortran compiler specified by the `$FC` environment variable,
@@ -354,8 +401,38 @@ def get_fortran_compiler(compiler: Optional[str] = None):
     :param compiler:
         Use this string instead of the $FC environment variable.
 
+    Returns the tool and a list of flags.
+
     """
-    return get_tool(compiler or os.getenv('FC', ''))  # type: ignore
+    fortran_compiler = None
+    try:
+        fortran_compiler = get_tool(compiler or os.getenv('FC', ''))  # type: ignore
+    except ValueError:
+        # tool not specified
+        pass
+
+    if not fortran_compiler:
+        try:
+            run_command(['gfortran', '--help'])
+            fortran_compiler = 'gfortran', []
+            logger.info('detected gfortran')
+        except RuntimeError:
+            # gfortran not available
+            pass
+
+    if not fortran_compiler:
+        try:
+            run_command(['ifort', '--help'])
+            fortran_compiler = 'ifort', []
+            logger.info('detected ifort')
+        except RuntimeError:
+            # gfortran not available
+            pass
+
+    if not fortran_compiler:
+        raise RuntimeError('no fortran compiler specified or discovered')
+
+    return fortran_compiler
 
 
 def get_mod_hashes(analysed_files: Set[AnalysedFortran], config) -> Dict[str, int]: