Create artefact store

docs/source/advanced_config.rst

@@ -82,18 +82,20 @@ import your grab configuration to find out where it put the source.
 
     if __name__ == '__main__':
         with BuildConfig(project_label='<project_label>') as state:
-            grab_folder(state, src=grab_config.source_root),
+            grab_folder(state, src=my_grab_config.source_root),
 
 
 Housekeeping
 ============
 
-Fab will remove old files from the prebuilds folder. It will remove all prebuild files that are not part of the current build by default.
-
-If you add a :func:`~fab.steps.cleanup_prebuilds.cleanup_prebuilds` step, you
-can keep prebuild files for longer. This may be useful, for example, if you
-often switch between two versions of your code and want to keep the prebuild
-speed benefits when building both.
+You can add a :func:`~fab.steps.cleanup_prebuilds.cleanup_prebuilds`
+step, where you can explicitly control how long to keep prebuild files.
+This may be useful, for example, if you often switch between two versions
+of your code and want to keep the prebuild speed benefits when building
+both. If you do not add your own cleanup_prebuild step, Fab will
+automatically run a default step which will remove old files from the
+prebuilds folder. It will remove all prebuild files that are not part of
+the current build by default.
 
 
 Sharing Prebuilds
@@ -311,7 +313,7 @@ which most Fab steps accept. (See :ref:`Overriding default collections`)
 
     @step
     def custom_step(state):
-            state._artefact_store['custom_artefacts'] = do_something(state._artefact_store['step 1 artefacts'])
+        state.artefact_store['custom_artefacts'] = do_something(state.artefact_store['step 1 artefacts'])
 
 
     with BuildConfig(project_label='<project label>') as state:
@@ -328,7 +330,7 @@ Steps have access to multiprocessing methods through the
 
     @step
     def custom_step(state):
-        input_files = artefact_store['custom_artefacts']
+        input_files = state.artefact_store['custom_artefacts']
         results = run_mp(state, items=input_files, func=do_something)
 
 

docs/source/environment.rst

@@ -3,11 +3,11 @@
 Environment
 ***********
 
-Fab requires a suitible Python environment in which to run. This page outlines
+Fab requires a suitable Python environment in which to run. This page outlines
 some routes to achieving such an environment.
 
 This page contains general instructions, there are additional instructions for
-:ref:`Met Office<MetOffice>` users elsewhere.
+:ref:`Met Office<MetOfficeUsage>` users elsewhere.
 
 
 .. _Requirements:

docs/source/writing_config.rst

@@ -236,14 +236,14 @@ However preprocessing C currently requires a preceding step called the
 into the C code so Fab is able to deduce which inclusions are user code and
 which are system code. This allows system dependencies to be ignored.
 
-See also :ref:`Advanced C Code<Advanced C Code>`
+See also :ref:`Advanced C Code<C Pragma Injector>`
 
 
 Further Reading
 ===============
 
 More advanced configuration topics are discussed in
-:ref:`Advanced Configuration`.
+:ref:`Advanced Config`.
 
 You can see more complicated configurations in the
 `developer testing directory <https://github.com/metomi/fab/tree/master/run_configs>`_.

source/fab/artefacts.py

@@ -15,7 +15,7 @@
 from pathlib import Path
 from typing import Iterable, Union, Dict, List
 
-from fab.constants import BUILD_TREES
+from fab.constants import BUILD_TREES, CURRENT_PREBUILDS
 from fab.dep_tree import filter_source_tree, AnalysedDependent
 from fab.util import suffix_filter
 
@@ -32,7 +32,6 @@ def __call__(self, artefact_store):
             The artefact store from which to retrieve.
 
         """
-        pass
 
 
 class CollectionGetter(ArtefactsGetter):
@@ -158,3 +157,18 @@ def __call__(self, artefact_store):
             build_lists[root] = filter_source_tree(source_tree=tree, suffixes=self.suffixes)
 
         return build_lists
+
+
+class ArtefactStore(dict):
+    '''This object stores artefacts (which can be of any type). Each artefact
+    is index by a string.
+    '''
+    def __init__(self):
+        super().__init__()
+        self.reset()
+
+    def reset(self):
+        '''Clears the artefact store (but does not delete any files).
+        '''
+        self.clear()
+        self[CURRENT_PREBUILDS] = set()

source/fab/build_config.py

@@ -18,16 +18,18 @@
 from multiprocessing import cpu_count
 from pathlib import Path
 from string import Template
-from typing import List, Optional, Dict, Any, Iterable
+from typing import List, Optional, Iterable
 
+from fab.artefacts import ArtefactStore
 from fab.constants import BUILD_OUTPUT, SOURCE_ROOT, PREBUILD, CURRENT_PREBUILDS
 from fab.metrics import send_metric, init_metrics, stop_metrics, metrics_summary
+from fab.steps.cleanup_prebuilds import CLEANUP_COUNT, cleanup_prebuilds
 from fab.util import TimerLogger, by_type, get_fab_workspace
 
 logger = logging.getLogger(__name__)
 
 
-class BuildConfig(object):
+class BuildConfig():
     """
     Contains and runs a list of build steps.
 
@@ -105,9 +107,10 @@ def __init__(self, project_label: str, multiprocessing: bool = True, n_procs: Op
 
         # todo: should probably pull the artefact store out of the config
         # runtime
-        # todo: either make this public, add get/setters, or extract into a class.
-        self._artefact_store: Dict[str, Any] = {}
-        self.init_artefact_store()  # note: the artefact store is reset with every call to run()
+        self._artefact_store: ArtefactStore = ArtefactStore()
+
+        self._build_timer = None
+        self._start_time = None
 
     def __enter__(self):
 
@@ -130,8 +133,7 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_val, exc_tb):
 
         if not exc_type:  # None if there's no error.
-            from fab.steps.cleanup_prebuilds import CLEANUP_COUNT, cleanup_prebuilds
-            if CLEANUP_COUNT not in self._artefact_store:
+            if CLEANUP_COUNT not in self.artefact_store:
                 logger.info("no housekeeping step was run, using a default hard cleanup")
                 cleanup_prebuilds(config=self, all_unused=True)
 
@@ -142,19 +144,23 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         self._finalise_logging()
 
     @property
-    def build_output(self):
-        return self.project_workspace / BUILD_OUTPUT
+    def artefact_store(self) -> ArtefactStore:
+        ''':returns: the Artefact instance for this configuration.
+        '''
+        return self._artefact_store
 
-    def init_artefact_store(self):
-        # there's no point writing to this from a child process of Step.run_mp() because you'll be modifying a copy.
-        self._artefact_store = {CURRENT_PREBUILDS: set()}
+    @property
+    def build_output(self) -> Path:
+        ''':returns: the build output path.
+        '''
+        return self.project_workspace / BUILD_OUTPUT
 
     def add_current_prebuilds(self, artefacts: Iterable[Path]):
         """
         Mark the given file paths as being current prebuilds, not to be cleaned during housekeeping.
 
         """
-        self._artefact_store[CURRENT_PREBUILDS].update(artefacts)
+        self.artefact_store[CURRENT_PREBUILDS].update(artefacts)
 
     def _run_prep(self):
         self._init_logging()
@@ -168,7 +174,7 @@ def _run_prep(self):
         init_metrics(metrics_folder=self.metrics_folder)
 
         # note: initialising here gives a new set of artefacts each run
-        self.init_artefact_store()
+        self.artefact_store.reset()
 
     def _prep_folders(self):
         self.source_root.mkdir(parents=True, exist_ok=True)
@@ -210,7 +216,7 @@ def _finalise_metrics(self, start_time, steps_timer):
 
 
 # todo: better name? perhaps PathFlags?
-class AddFlags(object):
+class AddFlags():
     """
     Add command-line flags when our path filter matches.
     Generally used inside a :class:`~fab.build_config.FlagsConfig`.
@@ -265,14 +271,13 @@ def run(self, fpath: Path, input_flags: List[str], config):
             input_flags += add_flags
 
 
-class FlagsConfig(object):
+class FlagsConfig():
     """
     Return command-line flags for a given path.
 
     Simply allows appending flags but may evolve to also replace and remove flags.
 
     """
-
     def __init__(self, common_flags: Optional[List[str]] = None, path_flags: Optional[List[AddFlags]] = None):
         """
         :param common_flags:

source/fab/parse/fortran_common.py

@@ -134,8 +134,6 @@ def run(self, fpath: Path) \
 
         # find things in the node tree
         analysed_file = self.walk_nodes(fpath=fpath, file_hash=file_hash, node_tree=node_tree)
-
-        analysis_fpath = self._get_analysis_fpath(fpath, file_hash)
         analysed_file.save(analysis_fpath)
 
         return analysed_file, analysis_fpath

source/fab/steps/analyse.py

@@ -167,7 +167,7 @@ def analyse(
     c_analyser._config = config
 
     # parse
-    files: List[Path] = source_getter(config._artefact_store)
+    files: List[Path] = source_getter(config.artefact_store)
     analysed_files = _parse_files(config, files=files, fortran_analyser=fortran_analyser, c_analyser=c_analyser)
     _add_manual_results(special_measure_analysis_results, analysed_files)
 
@@ -206,7 +206,7 @@ def analyse(
         _add_unreferenced_deps(unreferenced_deps, symbol_table, project_source_tree, build_tree)
         validate_dependencies(build_tree)
 
-    config._artefact_store[BUILD_TREES] = build_trees
+    config.artefact_store[BUILD_TREES] = build_trees
 
 
 def _analyse_dependencies(analysed_files: Iterable[AnalysedDependent]):

source/fab/steps/archive_objects.py

@@ -95,14 +95,14 @@ def archive_objects(config: BuildConfig, source: Optional[ArtefactsGetter] = Non
     output_fpath = str(output_fpath) if output_fpath else None
     output_collection = output_collection
 
-    target_objects = source_getter(config._artefact_store)
+    target_objects = source_getter(config.artefact_store)
     assert target_objects.keys()
     if output_fpath and list(target_objects.keys()) != [None]:
         raise ValueError("You must not specify an output path (library) when there are root symbols (exes)")
     if not output_fpath and list(target_objects.keys()) == [None]:
         raise ValueError("You must specify an output path when building a library.")
 
-    output_archives = config._artefact_store.setdefault(output_collection, {})
+    output_archives = config.artefact_store.setdefault(output_collection, {})
     for root, objects in target_objects.items():
 
         if root:

source/fab/steps/c_pragma_injector.py

@@ -43,9 +43,9 @@ def c_pragma_injector(config, source: Optional[ArtefactsGetter] = None, output_n
     source_getter = source or DEFAULT_SOURCE_GETTER
     output_name = output_name or PRAGMAD_C
 
-    files = source_getter(config._artefact_store)
+    files = source_getter(config.artefact_store)
     results = run_mp(config, items=files, func=_process_artefact)
-    config._artefact_store[output_name] = list(results)
+    config.artefact_store[output_name] = list(results)
 
 
 def _process_artefact(fpath: Path):

source/fab/steps/cleanup_prebuilds.py

@@ -63,23 +63,23 @@ def cleanup_prebuilds(
 
     elif all_unused:
         num_removed = remove_all_unused(
-            found_files=prebuild_files, current_files=config._artefact_store[CURRENT_PREBUILDS])
+            found_files=prebuild_files, current_files=config.artefact_store[CURRENT_PREBUILDS])
 
     else:
         # get the file access time for every artefact
         prebuilds_ts = \
             dict(zip(prebuild_files, run_mp(config, prebuild_files, get_access_time)))  # type: ignore
 
         # work out what to delete
-        to_delete = by_age(older_than, prebuilds_ts, current_files=config._artefact_store[CURRENT_PREBUILDS])
-        to_delete |= by_version_age(n_versions, prebuilds_ts, current_files=config._artefact_store[CURRENT_PREBUILDS])
+        to_delete = by_age(older_than, prebuilds_ts, current_files=config.artefact_store[CURRENT_PREBUILDS])
+        to_delete |= by_version_age(n_versions, prebuilds_ts, current_files=config.artefact_store[CURRENT_PREBUILDS])
 
         # delete them all
         run_mp(config, to_delete, os.remove)
         num_removed = len(to_delete)
 
     logger.info(f'removed {num_removed} prebuild files')
-    config._artefact_store[CLEANUP_COUNT] = num_removed
+    config.artefact_store[CLEANUP_COUNT] = num_removed
 
 
 def by_age(older_than: Optional[timedelta],

source/fab/steps/compile_c.py

@@ -83,7 +83,7 @@ def compile_c(config, common_flags: Optional[List[str]] = None,
     source_getter = source or DEFAULT_SOURCE_GETTER
 
     # gather all the source to compile, for all build trees, into one big lump
-    build_lists: Dict = source_getter(config._artefact_store)
+    build_lists: Dict = source_getter(config.artefact_store)
     to_compile: list = sum(build_lists.values(), [])
     logger.info(f"compiling {len(to_compile)} c files")
 
@@ -101,7 +101,7 @@ def compile_c(config, common_flags: Optional[List[str]] = None,
     config.add_current_prebuilds(prebuild_files)
 
     # record the compilation results for the next step
-    store_artefacts(compiled_c, build_lists, config._artefact_store)
+    store_artefacts(compiled_c, build_lists, config.artefact_store)
 
 
 # todo: very similar code in fortran compiler

source/fab/steps/compile_fortran.py

@@ -67,7 +67,7 @@ def compile_fortran(config: BuildConfig, common_flags: Optional[List[str]] = Non
         A list of :class:`~fab.build_config.AddFlags`, defining flags to be included in the command line call
         for selected files.
     :param source:
-        An :class:`~fab.artefacts.ArtefactsGetter` which give us our c files to process.
+        An :class:`~fab.artefacts.ArtefactsGetter` which gives us our Fortran files to process.
 
     """
 
@@ -84,7 +84,7 @@ def compile_fortran(config: BuildConfig, common_flags: Optional[List[str]] = Non
     mod_hashes: Dict[str, int] = {}
 
     # get all the source to compile, for all build trees, into one big lump
-    build_lists: Dict[str, List] = source_getter(config._artefact_store)
+    build_lists: Dict[str, List] = source_getter(config.artefact_store)
 
     # build the arguments passed to the multiprocessing function
     mp_common_args = MpCommonArgs(
@@ -119,7 +119,7 @@ def compile_fortran(config: BuildConfig, common_flags: Optional[List[str]] = Non
         logger.info(f"stage 2 compiled {len(compiled_this_pass)} files")
 
     # record the compilation results for the next step
-    store_artefacts(compiled, build_lists, config._artefact_store)
+    store_artefacts(compiled, build_lists, config.artefact_store)
 
 
 def handle_compiler_args(common_flags=None, path_flags=None):

source/fab/steps/find_source_files.py

@@ -145,4 +145,4 @@ def find_source_files(config, source_root=None, output_collection="all_source",
     if not filtered_fpaths:
         raise RuntimeError("no source files found after filtering")
 
-    config._artefact_store[output_collection] = filtered_fpaths
+    config.artefact_store[output_collection] = filtered_fpaths

source/fab/steps/link.py

@@ -78,11 +78,11 @@ def link_exe(config, linker: Optional[str] = None, flags=None, source: Optional[
     flags = flags or []
     source_getter = source or DefaultLinkerSource()
 
-    target_objects = source_getter(config._artefact_store)
+    target_objects = source_getter(config.artefact_store)
     for root, objects in target_objects.items():
         exe_path = config.project_workspace / f'{root}'
         call_linker(linker=linker, flags=flags, filename=str(exe_path), objects=objects)
-        config._artefact_store.setdefault(EXECUTABLES, []).append(exe_path)
+        config.artefact_store.setdefault(EXECUTABLES, []).append(exe_path)
 
 
 # todo: the bit about Dict[None, object_files] seems too obscure - try to rethink this.
@@ -123,7 +123,7 @@ def link_shared_object(config, output_fpath: str, linker: Optional[str] = None,
             flags.append(f)
 
     # We expect a single build target containing the whole codebase, with no name (as it's not a root symbol).
-    target_objects = source_getter(config._artefact_store)
+    target_objects = source_getter(config.artefact_store)
     assert list(target_objects.keys()) == [None]
 
     objects = target_objects[None]