diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 1ed0a2d02..f5540a7d8 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -60,4 +60,5 @@ jobs: id: type-checks run: | touch .venv/lib/python${{ matrix.python-version }}/site-packages/ruamel/py.typed + touch .venv/lib/python${{ matrix.python-version }}/site-packages/networkx/py.typed poetry run mypy --namespace-packages -p hammer diff --git a/doc/Hammer-Use/Flowgraphs.rst b/doc/Hammer-Use/Flowgraphs.rst new file mode 100644 index 000000000..bfd0e3a32 --- /dev/null +++ b/doc/Hammer-Use/Flowgraphs.rst @@ -0,0 +1,157 @@ +.. _flowgraphs: + +Flowgraphs +========== + +Hammer has **experimental** support for flowgraph constructions, similar to tools like `mflowgen `_. +Their intention is to simplify the way flows are constructed and ran in Hammer. +They can be imported via the ``hammer.flowgraph`` module. + +Construction +------------ + +Flowgraphs are nothing more than a collection of ``Node`` instances linked together via a ``Graph`` instance. +Each ``Node`` "pulls" from a directory to feed in inputs and "pushes" output files to another directory to be used by other nodes. +``Node`` instances are roughly equivalent to a single call to the ``hammer-vlsi`` CLI, so they take in similar attributes: + +* The action being called +* The tool used to perform the action +* The pull and push directories +* Any *required* input/output files +* Any *optional* input/output files +* A driver to run the node with; this enables backwards compatibility with :ref:`hooks `. +* Options to specify steps within an action; this enables backwards compatibility with :ref:`flow control `. + + * ``start_before_step`` + * ``start_after_step`` + * ``stop_before_step`` + * ``stop_after_step`` + * ``only_step`` + +A minimal example of a ``Node``: + +.. code-block:: python + + from hammer.flowgraph import Node + + test = Node( + action="foo", + tool="nop", + pull_dir="foo_dir", + push_dir="/dev/null", + required_inputs=["foo.yml"], + required_outputs=["foo-out.json"], + ) + +Each ``Node`` has the ability to be "privileged", meaning that a flow *must* start with this node. +This only occurs when a flow is being controlled using any of the steps. + +Running a Flowgraph +------------------- + +``Node`` instances are linked together using an `adjacency list `_. +This list can be used to instantiate a ``Graph``: + +.. code-block:: python + + from hammer.flowgraph import Graph, Node + + root = Node( + "foo", "nop", "foo_dir", "", + ["foo.yml"], + ["foo-out.json"], + ) + child1 = Node( + "bar", "nop", "foo_dir", "bar_dir", + ["foo-out.json"], + ["bar-out.json"], + ) + graph = Graph({root: [child1]}) + +Using the Hammer CLI tool, separate actions are manually linked via an *auxiliary* action, such as ``syn-to-par``. +By using a flowgraph, ``Graph`` instances by default *automatically* insert auxiliary actions. +This means that actions no longer need to be specified in a flow; the necessary nodes are inserted by the flowgraph tool. +This feature can be disabled by setting ``auto_auxiliary=False``. + +A ``Graph`` can be run by calling the ``run`` method and passing in a starting node. +When running a flow, each ``Node`` keeps an internal status based on the status of the action being run: + +* ``NOT_RUN``: The action has yet to be run. +* ``RUNNING``: The action is currently running. +* ``COMPLETE``: The action has finished. +* ``INCOMPLETE``: The action ran into an error while being run. +* ``INVALID``: The action's outputs have been invalidated (e.g. inputs or attributes have changed). + +The interactions between the statuses are described in the diagram: + +.. mermaid:: + + stateDiagram-v2 + [*] --> NOT_RUN + NOT_RUN --> RUNNING + RUNNING --> INCOMPLETE + RUNNING --> COMPLETE + INCOMPLETE --> NOT_RUN + COMPLETE --> INVALID + INVALID --> NOT_RUN + +Regardless of whether a flow completes with or without errors, the graph at the time of completion or error is returned, allowing for a graph to be "resumed" once any errors have been fixed. + +Visualization +------------- + +A flowgraph can be visualized in Markdown files via the `Mermaid `_ tool. +Calling a ``Graph`` instance's ``to_mermaid`` method outputs a file named ``graph-viz.md``. +The file can be viewed in a site like `Mermaid's live editor `_ or using Github's native support. + +The flowgraph below would appear like this: + +.. code-block:: python + + from hammer.flowgraph import Graph, Node + + syn = Node( + "syn", "nop", + os.path.join(td, "syn_dir"), os.path.join(td, "s2p_dir"), + ["syn-in.yml"], + ["syn-out.json"], + ) + s2p = Node( + "syn-to-par", "nop", + os.path.join(td, "s2p_dir"), os.path.join(td, "par_dir"), + ["syn-out.json"], + ["s2p-out.json"], + ) + par = Node( + "par", "nop", + os.path.join(td, "par_dir"), os.path.join(td, "out_dir"), + ["s2p-out.json"], + ["par-out.json"], + ) + g = Graph({ + syn: [s2p], + s2p: [par], + par: [] + }) + + +Here are the contents of ``graph-viz.md`` after calling ``g.to_mermaid()``: + +.. code-block:: markdown + + ```mermaid + + stateDiagram-v2 + syn --> syn_to_par + syn_to_par --> par + ``` + +Which would render like this: + +.. mermaid:: + + stateDiagram-v2 + syn --> syn_to_par + syn_to_par --> par + +Note that the separators have been changed to comply with Mermaid syntax. diff --git a/doc/Hammer-Use/index.rst b/doc/Hammer-Use/index.rst index 2e29006e4..de3649b04 100644 --- a/doc/Hammer-Use/index.rst +++ b/doc/Hammer-Use/index.rst @@ -11,5 +11,6 @@ This documentation will walk through more advanced features of the Hammer infras Hammer-APIs Flow-Control Hooks + Flowgraphs Buildfile Hierarchical diff --git a/doc/conf.py b/doc/conf.py index 7c3ef9553..94d33e745 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -17,12 +17,12 @@ # -- Project information ----------------------------------------------------- -project = 'Hammer' -copyright = '2023, Berkeley Architecture Research' -author = 'Berkeley Architecture Research' +project = "Hammer" +copyright = "2023, Berkeley Architecture Research" +author = "Berkeley Architecture Research" # The full version, including alpha/beta/rc tags -release = '1.0.0' +release = "1.0.0" # -- General configuration --------------------------------------------------- @@ -33,30 +33,31 @@ extensions = [ "myst_parser", "sphinx-jsonschema", - "sphinx_rtd_size" + "sphinx_rtd_size", + "sphinxcontrib.mermaid", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] -source_suffix = '.rst' +source_suffix = ".rst" -master_doc = 'index' +master_doc = "index" # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" html_theme_options = { - 'collapse_navigation': False, + "collapse_navigation": False, } sphinx_rtd_size_width = "1200px" diff --git a/hammer/config/config_src.py b/hammer/config/config_src.py index f5340e950..80493940d 100644 --- a/hammer/config/config_src.py +++ b/hammer/config/config_src.py @@ -805,6 +805,7 @@ def get_config(self) -> dict: self.__config_cache_dirty = False return self.__config_cache + @property def get_config_types(self) -> dict: """ Get the types for the configuration of a database. @@ -906,9 +907,9 @@ def get_setting_type(self, key: str, nullvalue: Any = None) -> Any: :param nullvalue: Value to return for nulls. :return: Data type of key. """ - if key not in self.get_config_types(): + if key not in self.get_config_types: raise KeyError(f"Key type {key} is missing") - value = self.get_config_types()[key] + value = self.get_config_types[key] return nullvalue if value is None else value def set_setting_type(self, key: str, value: Any) -> None: @@ -928,7 +929,7 @@ def has_setting_type(self, key: str) -> bool: :param key: Desired key. :return: True if the given setting exists. """ - return key in self.get_config_types() + return key in self.get_config_types def check_setting(self, key: str, cfg: Optional[dict] = None) -> bool: """ @@ -940,12 +941,12 @@ def check_setting(self, key: str, cfg: Optional[dict] = None) -> bool: if cfg is None: cfg = self.get_config() - if key not in self.get_config_types(): + if key not in self.get_config_types: #TODO: compile this at the beginning instead of emitting every instance #self.logger.warning(f"Key {key} is not associated with a type") return True try: - exp_value_type = parse_setting_type(self.get_config_types()[key]) + exp_value_type = parse_setting_type(self.get_config_types[key]) except ValueError as ve: raise ValueError(f'Key {key} has an invalid outer type: perhaps you have "List" instead of "list" or "Dict" instead of "dict"?') from ve diff --git a/hammer/flowgraph/__init__.py b/hammer/flowgraph/__init__.py new file mode 100644 index 000000000..38b16ffff --- /dev/null +++ b/hammer/flowgraph/__init__.py @@ -0,0 +1,5 @@ +# Hammer logging code. +# +# See LICENSE for licence details. + +from .flowgraph import * diff --git a/hammer/flowgraph/flowgraph.py b/hammer/flowgraph/flowgraph.py new file mode 100644 index 000000000..000ec81bc --- /dev/null +++ b/hammer/flowgraph/flowgraph.py @@ -0,0 +1,362 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Represents a flowgraph in HAMMER that can be run and verified. +# See README.config for more details. +# +# See LICENSE for licence details. + +# pylint: disable=invalid-name + +import json +import os +import uuid +from dataclasses import dataclass, field, asdict +from enum import Enum +from typing import Any, Union + +import networkx as nx +from networkx.readwrite import json_graph + +from hammer.logging import HammerVLSILogging +from hammer.vlsi.cli_driver import CLIDriver + + +class Status(Enum): + """Represents the status of a node in the flowgraph. + """ + NOT_RUN = "NOT_RUN" + RUNNING = "RUNNING" + INCOMPLETE = "INCOMPLETE" + INVALID = "INVALID" + COMPLETE = "COMPLETE" + + +@dataclass +class Node: + """Defines a node for an action in a flowgraph. + + Returns: + Node: Complete description of an action. + """ + action: str + tool: str + pull_dir: str + push_dir: str + required_inputs: list[str] + required_outputs: list[str] + status: Status = Status.NOT_RUN + # __uuid: uuid.UUID = field(default_factory=uuid.uuid4) + driver: CLIDriver = field(default_factory=CLIDriver) + optional_inputs: list[str] = field(default_factory=list) + optional_outputs: list[str] = field(default_factory=list) + step_controls: dict[str, str] = field(default_factory=lambda: { + "start_before_step": "", + "start_after_step": "", + "stop_before_step": "", + "stop_after_step": "", + "only_step": "", + }) + + def __key(self) -> tuple: + """Key value for hashing. + + Returns: + tuple: All fields concatenated. + """ + return ( + self.action, + self.tool, + self.push_dir, + self.pull_dir, + tuple(self.required_inputs), + tuple(self.required_outputs), + tuple(self.optional_inputs), + tuple(self.optional_outputs), + self.status, + tuple(self.step_controls.items()) + ) + + def __hash__(self) -> int: + """Dunder method for uniquely hashing a `Node` object. + + Returns: + int: Hash of a `Node`. + """ + return hash(self.__key) + + def __is_privileged(self) -> bool: + """Private method for checking if + step control is applied to a `Node`. + + Returns: + bool: If the node is allowed to be the starting point of a flow. + """ + return any(i != "" for i in self.step_controls.values()) + + @property + def privileged(self) -> bool: + """Property for determining node privilege. + + Returns: + bool: If the node is allowed to be the starting point of a flow. + """ + return self.__is_privileged() + +class NodeEncoder(json.JSONEncoder): + def default(self, o: Any) -> Any: + if isinstance(o, Node): + return asdict(o) + elif isinstance(o, Status): + return o.value + return super().default(o) + +def as_node(dct: dict) -> Union[Node, dict]: + if "action" in dct: + return Node( + dct["action"], + dct["tool"], + dct["pull_dir"], + dct["push_dir"], + dct["required_inputs"], + dct["required_outputs"], + Status(dct["status"]), + dct["optional_inputs"], + dct["optional_outputs"], + ) + return dct + +@dataclass +class Graph: + """Defines a flowgraph. + + Returns: + Graph: HAMMER flowgraph. + """ + edge_list: dict[Node, list[Node]] + auto_auxiliary: bool = True + + def __post_init__(self) -> None: + self.networkx = nx.DiGraph(Graph.insert_auxiliary_actions(self.edge_list) if self.auto_auxiliary else self.edge_list) + + def verify(self) -> bool: + """Checks if a graph is valid via its inputs and outputs. + + Returns: + bool: If graph is valid. + """ + return all(self.__process(v) for v in convert_to_acyclic(self).networkx) + + def __process(self, v: Node) -> bool: + """Process a specific vertex of a graph. + + Args: + v (Node): Node to check the validity of. + + Returns: + bool: If the particular node is valid. + """ + parent_outs = \ + set().union(*(set(p.required_outputs) for p in self.networkx.predecessors(v))) \ + | set().union(*(set(p.optional_outputs) for p in self.networkx.predecessors(v))) + inputs = set(v.required_inputs) | set(v.optional_inputs) + return self.networkx.in_degree(v) == 0 or parent_outs >= inputs + + @staticmethod + def insert_auxiliary_actions(edge_list: dict[Node, list[Node]]) -> dict[Node, list[Node]]: + """Inserts x-to-y actions between two semantically related actions (e.g. if syn and par are connected, then we have to insert a syn-to-par node here). + + Args: + edge_list (dict[Node, list[Node]]): Edge list without auxiliary actions. + + Returns: + dict[Node, list[Node]]: Transformed edge list with auxiliary actions. + """ + valid_auxiliary_actions = [ + ("synthesis", "par"), + ("syn", "par"), + ("heir-par", "syn"), + ("heir_par", "syn"), + ("par", "drc"), + ("par", "lvs"), + ("synthesis", "sim"), + ("syn", "sim"), + ("par", "sim"), + ("syn", "power"), + ("par", "power"), + ("sim", "power"), + ("synthesis", "formal"), + ("syn", "formal"), + ("par", "formal"), + ("synthesis", "timing"), + ("syn", "timing"), + ("par", "timing"), + ] + + changes = [] + for parent_idx, (parent, children) in enumerate(edge_list.items()): + for child_idx, child in enumerate(children): + if (parent.action, child.action) in valid_auxiliary_actions: + aux_action = f"{parent.action}-to-{child.action}" + aux_node = Node( + aux_action, + parent.tool, + os.path.join(os.path.dirname(parent.pull_dir), f"{aux_action}-dir"), + child.pull_dir, + parent.required_outputs, + [f"{aux_action}-out.json"], + ) + changes.append((parent_idx, child_idx, aux_node)) + + edge_list_copy = edge_list.copy() + for parent_idx, child_idx, aux_node in changes: + parent, children = list(edge_list_copy.items())[parent_idx] + parent.push_dir = os.path.join( + os.path.dirname(parent.pull_dir), + f"{aux_node.action}-dir") + + child = children[child_idx] + child.required_inputs = aux_node.required_outputs + + children[child_idx] = aux_node + if aux_node not in edge_list_copy: + edge_list_copy[aux_node] = [] + edge_list_copy[aux_node].append(child) + return edge_list_copy + + + def run(self, start: Node) -> Any: + """Runs a flowgraph. + + Args: + start (Node): Node to start the run on. + + Raises: + RuntimeError: If the flowgraph is invalid. + RuntimeError: If the starting node is not in the flowgraph. + """ + if not self.verify(): + raise RuntimeError("Flowgraph is invalid. Please check your flow's inputs and outputs.") + if start not in self.networkx: + raise RuntimeError("Node not in flowgraph. Did you construct the graph correctly?") + if not start.privileged and any(i.privileged for i in self.networkx): + raise RuntimeError("Attempting to run non-privileged node in privileged flow. Please complete your stepped flow first.") + + start_code = Graph.__run_single(start) + if start_code != 0: + return self + else: + for _, c in nx.bfs_edges(self.networkx, start): + code = Graph.__run_single(c) + if code != 0: + break + return self + + @staticmethod + def __run_single(node: Node) -> int: + """Helper function to run a HAMMER node. + + Args: + node (Node): Node to run action on. + + Returns: + int: Status code. + """ + driver = node.driver + + arg_list = { + "action": node.action, + 'environment_config': None, + 'configs': [os.path.join(node.pull_dir, i) for i in node.required_inputs], + 'log': None, + 'obj_dir': node.push_dir, + 'syn_rundir': '', + 'par_rundir': '', + 'drc_rundir': '', + 'lvs_rundir': '', + 'sim_rundir': '', + 'power_rundir': '', + 'formal_rundir': '', + 'timing_rundir': '', + "from_step": node.step_controls["start_before_step"], + "after_step": node.step_controls["start_after_step"], + "to_step": node.step_controls["stop_before_step"], + "until_step": node.step_controls["stop_after_step"], + 'only_step': node.step_controls["only_step"], + 'output': os.path.join(node.push_dir, node.required_outputs[0]), # TODO: fix this + 'verilog': None, + 'firrtl': None, + 'top': None, + 'cad_files': None, + 'dump_history': False + } + + node.status = Status.RUNNING + ctxt = HammerVLSILogging.context(node.action) + ctxt.info(f"Running graph step {node.action}") + code = driver.run_main_parsed(arg_list) + if code == 0: + node.status = Status.COMPLETE + else: + node.status = Status.INCOMPLETE + ctxt.fatal(f"Step {node.action} failed") + return code + + def to_json(self) -> dict: + """Encodes a graph as a JSON string. + + Returns: + str: JSON dump of a flowgraph. + """ + return json_graph.node_link_data(self.networkx) + + def to_mermaid(self) -> str: + """Converts the flowgraph into Mermaid format for visualization. + + Args: + fname (str): Output file name. + + Returns: + str: Path to Mermaid Markdown file. + """ + folder = os.path.dirname(list(self.networkx.nodes)[0].pull_dir) + fname = os.path.join(folder, "graph-viz.md") + with open(fname, 'w', encoding="utf-8") as f: + f.write("```mermaid\nstateDiagram-v2\n") + for start in self.networkx: + f.writelines( + f" {start.action.replace('-', '_')} --> {child.action.replace('-', '_')}\n" + for child in nx.neighbors(self.networkx, start) + ) + f.write("```\n") + return fname + +def convert_to_acyclic(g: Graph) -> Graph: + """Eliminates cycles in a flowgraph for analysis. + + Args: + g (Graph): (presumably) cyclic graph to transform. + + Returns: + Graph: Graph with cloned nodes. + """ + cycles = nx.simple_cycles(g.networkx) + new_edge_list = g.edge_list.copy() + for cycle in cycles: + cut_start, cut_end = cycle[0], cycle[1] + cut_end_copy = Node( + cut_end.action, + cut_end.tool, + cut_end.pull_dir, cut_end.push_dir, + cut_end.required_inputs, cut_end.required_outputs, + cut_end.status, + cut_end.optional_inputs, cut_end.optional_outputs + ) + cut_start_children = new_edge_list[cut_start] + new_edge_list[cut_start] = [] + new_edge_list[cut_end_copy] = cut_start_children + return Graph(new_edge_list) + + +# TODO: serialization format +# TODO: cycles are conditional on user input diff --git a/hammer/vlsi/cli_driver.py b/hammer/vlsi/cli_driver.py index 6928d51a7..2470e8c04 100644 --- a/hammer/vlsi/cli_driver.py +++ b/hammer/vlsi/cli_driver.py @@ -322,8 +322,8 @@ def info_action(driver: HammerDriver, append_error_func: Callable[[str], None]) if "_meta" not in k: print(k) while True: + key = input("Select from the current level of keys: ") try: - key = input("Select from the current level of keys: ") next_level = curr_level[key] break except KeyError: diff --git a/poetry.lock b/poetry.lock index abbff4b65..d7e26da2a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -616,6 +616,25 @@ linkify = ["linkify-it-py (>=1.0,<2.0)"] rtd = ["ipython", "sphinx-book-theme", "sphinx-design", "sphinxcontrib.mermaid (>=0.7.1,<0.8.0)", "sphinxext-opengraph (>=0.6.3,<0.7.0)", "sphinxext-rediraffe (>=0.2.7,<0.3.0)"] testing = ["beautifulsoup4", "coverage[toml]", "pytest (>=6,<7)", "pytest-cov", "pytest-param-files (>=0.3.4,<0.4.0)", "pytest-regressions", "sphinx (<5.2)", "sphinx-pytest"] +[[package]] +name = "networkx" +version = "3.1" +description = "Python package for creating and manipulating graphs and networks" +category = "main" +optional = false +python-versions = ">=3.8" +files = [ + {file = "networkx-3.1-py3-none-any.whl", hash = "sha256:4f33f68cb2afcf86f28a45f43efc27a9386b535d567d2127f8f61d51dec58d36"}, + {file = "networkx-3.1.tar.gz", hash = "sha256:de346335408f84de0eada6ff9fafafff9bcda11f0a0dfaa931133debb146ab61"}, +] + +[package.extras] +default = ["matplotlib (>=3.4)", "numpy (>=1.20)", "pandas (>=1.3)", "scipy (>=1.8)"] +developer = ["mypy (>=1.1)", "pre-commit (>=3.2)"] +doc = ["nb2plots (>=0.6)", "numpydoc (>=1.5)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.13)", "sphinx (>=6.1)", "sphinx-gallery (>=0.12)", "texext (>=0.6.7)"] +extra = ["lxml (>=4.6)", "pydot (>=1.4.2)", "pygraphviz (>=1.10)", "sympy (>=1.10)"] +test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] + [[package]] name = "numpy" version = "1.24.3" @@ -1134,6 +1153,18 @@ files = [ [package.extras] test = ["flake8", "mypy", "pytest"] +[[package]] +name = "sphinxcontrib-mermaid" +version = "0.8.1" +description = "Mermaid diagrams in yours Sphinx powered docs" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sphinxcontrib-mermaid-0.8.1.tar.gz", hash = "sha256:fa3e5325d4ba395336e6137d113f55026b1a03ccd115dc54113d1d871a580466"}, + {file = "sphinxcontrib_mermaid-0.8.1-py3-none-any.whl", hash = "sha256:15491c24ec78cf1626b1e79e797a9ce87cb7959cf38f955eb72dd5512aeb6ce9"}, +] + [[package]] name = "sphinxcontrib-qthelp" version = "1.0.3" diff --git a/pyproject.toml b/pyproject.toml index 928078102..12f5de155 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,9 @@ python = "^3.9" pydantic = "^1.9.2" PyYAML = "^6.0" "ruamel.yaml" = "^0.17.21" +networkx = "^3.0" numpy = "^1.23.0" +sphinxcontrib-mermaid = "^0.8.1" gdstk = { version = "^0.9.0", optional = true } gdspy = { version = "1.4", optional = true } diff --git a/tests/test_flowgraph.py b/tests/test_flowgraph.py new file mode 100644 index 000000000..a39f0361f --- /dev/null +++ b/tests/test_flowgraph.py @@ -0,0 +1,526 @@ +import json +import os +import tempfile +import unittest +from textwrap import dedent + +import networkx as nx +import pytest + +from hammer.vlsi.cli_driver import ( + CLIDriver, + HammerTool, + HammerToolHookAction +) + +from hammer import flowgraph +from hammer.flowgraph import convert_to_acyclic, Graph, Node, Status +from hammer.logging import HammerVLSILogging + + +MOCK_CFG = dedent(""" +synthesis.inputs: + input_files: ["LICENSE", "README.md"] + top_module: "z1top.xdc" + +par.inputs: + input_files: ["LICENSE", "README.md"] + top_module: "z1top.xdc" + +drc.inputs: + input_files: ["LICENSE", "README.md"] + top_module: "z1top.xdc" + +lvs.inputs: + input_files: ["LICENSE", "README.md"] + schematic_files: ["LICENSE", "README.md"] + top_module: "z1top.xdc" + hcells_list: [] + +pcb.inputs: + top_module: "z1top.xdc" + +formal.inputs: + input_files: ["LICENSE", "README.md"] + top_module: "z1top.xdc" + +sim.inputs: + input_files: ["LICENSE", "README.md"] + top_module: "z1top.xdc" + +vlsi: + core: + technology: "hammer.technology.nop" + + synthesis_tool: "hammer.synthesis.nop" + par_tool: "hammer.par.nop" + drc_tool: "hammer.drc.nop" + lvs_tool: "hammer.lvs.nop" + power_tool: "hammer.power.nop" + sim_tool: "mocksim" +""") + +class TestFlowgraph(unittest.TestCase): + + def test_initialize_node(self) -> None: + """Test that we can instantiate a node.""" + test = Node( + "syn", "nop", "syn_dir", "", + ["foo.yml"], + ["syn-out.json"], + ) + self.assertEqual(test.action, "syn") + + def test_complex_graph(self) -> None: + """Creating a more complex graph.""" + root = Node( + "foo", "nop", "foo_dir", "", + ["foo.yml"], + ["foo-out.json"], + ) + child1 = Node( + "bar", "nop", "foo_dir", "bar_dir", + ["foo-out.json"], + ["bar-out.json"], + ) + graph = Graph({root: [child1]}) + self.assertEqual(len(graph.networkx), 2) + self.assertEqual(graph.networkx.number_of_edges(), 1) + self.assertTrue(graph.verify()) + + def test_invalid_graph(self) -> None: + """Test that invalid flowgraphs are detected.""" + root = Node( + "foo", "nop", "foo_dir", "", + ["foo.yml"], + ["foo-out.json"], + ) + child1 = Node( + "bar", "nop", "foo_dir", "bar_dir", + ["foo-in.json"], + ["bar-out.json"], + ) + graph = Graph({root: [child1]}) + self.assertFalse(graph.verify()) + + def test_cycle_reduction(self) -> None: + """Test that cycles are reduced to an acyclic graph.""" + v0 = Node( + "v0", "nop", "v0_dir", "v1_dir", + ["v0-in.json"], + ["v0-out.json"], + ) + v1 = Node( + "v1", "nop", "v1_dir", "v2_dir", + ["v0-out.json"], + ["v1-out.json"], + ) + v2 = Node( + "v2", "nop", "v2_dir", "v0_dir", + ["v1-out.json"], + ["v2-out.json"], + ) + g = Graph({ + v0: [v1], + v1: [v2], + v2: [v0], + }) + self.assertTrue(len(nx.find_cycle(g.networkx)) > 0) + g_acyclic = convert_to_acyclic(g) + with self.assertRaises(nx.NetworkXNoCycle): + nx.find_cycle(g_acyclic.networkx) + self.assertEqual(len(g_acyclic.networkx), 4) + self.assertEqual(g.networkx.number_of_edges(), 3) + + def test_big_cycle_reduction(self) -> None: + """Test that larger cycles are reduced to an acyclic graph.""" + v0 = Node( + "v0", "nop", "v0_dir", "v1_dir", + ["v0-in.json"], + ["v0-out.json"], + ) + v1 = Node( + "v1", "nop", "v1_dir", "v2_dir", + ["v0-out.json"], + ["v1-out.json"], + ) + v2 = Node( + "v2", "nop", "v2_dir", "v3_dir", + ["v1-out.json"], + ["v2-out.json"], + ) + v3 = Node( + "v3", "nop", "v3_dir", "v0_dir", + ["v2-out.json"], + ["v3-out.json"], + ) + g = Graph({ + v0: [v1], + v1: [v2], + v2: [v3], + v3: [v0] + }) + self.assertTrue(len(nx.find_cycle(g.networkx)) > 0) + g_acyclic = convert_to_acyclic(g) + with self.assertRaises(nx.NetworkXNoCycle): + nx.find_cycle(g_acyclic.networkx) + self.assertEqual(len(g_acyclic.networkx), 5) + self.assertEqual(g.networkx.number_of_edges(), 4) + + def test_multi_cycle_reduction(self) -> None: + """Test that multiple cycles are reduced to an acyclic graph.""" + v0 = Node( + "v0", "nop", "v0_dir", "v1_dir", + ["v0-in.json"], + ["v0-out.json"], + ) + v1 = Node( + "v1", "nop", "v1_dir", "v2_dir", + ["v0-out.json"], + ["v1-out.json"], + ) + v2 = Node( + "v2", "nop", "v2_dir", "v0_dir", + ["v1-out.json"], + ["v2-out.json"], + ) + v3 = Node( + "v3", "nop", "v3_dir", "v4_dir", + ["v2-out.json"], + ["v3-out.json"], + ) + v4 = Node( + "v4", "nop", "v4_dir", "v5_dir", + ["v3-out.json"], + ["v4-out.json"], + ) + v5 = Node( + "v5", "nop", "v5_dir", "v3_dir", + ["v4-out.json"], + ["v3-out.json"], + ) + g = Graph({ + v0: [v1], + v1: [v2], + v2: [v0, v3], + v3: [v4, v2], + v4: [v5], + v5: [v3] + }) + self.assertTrue(len(nx.find_cycle(g.networkx)) > 0) + g_acyclic = convert_to_acyclic(g) + with self.assertRaises(nx.NetworkXNoCycle): + nx.find_cycle(g_acyclic.networkx) + + def test_run_basic(self) -> None: + """Test a basic syn -> par flow, all with nop tools.""" + HammerVLSILogging.clear_callbacks() + HammerVLSILogging.add_callback(HammerVLSILogging.callback_buffering) + + with tempfile.TemporaryDirectory() as td: + os.mkdir(os.path.join(td, "syn_dir")) + os.mkdir(os.path.join(td, "par_dir")) + + with open(os.path.join(td, "syn_dir", "syn-in.yml"), 'w', encoding="utf-8") as tf1: + tf1.write(MOCK_CFG) + + syn = Node( + "syn", "nop", + os.path.join(td, "syn_dir"), os.path.join(td, "s2p_dir"), + ["syn-in.yml"], + ["syn-out.json"], + ) + s2p = Node( + "syn-to-par", "nop", + os.path.join(td, "s2p_dir"), os.path.join(td, "par_dir"), + ["syn-out.json"], + ["s2p-out.json"], + ) + par = Node( + "par", "nop", + os.path.join(td, "par_dir"), os.path.join(td, "out_dir"), + ["s2p-out.json"], + ["par-out.json"], + ) + g = Graph({ + syn: [s2p], + s2p: [par], + par: [] + }, auto_auxiliary=False) + g.run(syn) + + for n in g.networkx: + self.assertEqual(n.status, Status.COMPLETE) + + def test_invalid_run(self) -> None: + """ + Test that a failing run should not run + other steps and exit gracefully. + """ + HammerVLSILogging.clear_callbacks() + HammerVLSILogging.add_callback(HammerVLSILogging.callback_buffering) + + with tempfile.TemporaryDirectory() as td: + os.mkdir(os.path.join(td, "syn_dir")) + os.mkdir(os.path.join(td, "par_dir")) + + with open(os.path.join(td, "syn_dir", "syn-in.yml"), 'w', encoding="utf-8") as tf1: + tf1.write(MOCK_CFG) + + syn = Node( + "syn", "nop", + os.path.join(td, "syn_dir"), os.path.join(td, "s2p_dir"), + ["syn-in.yml"], + ["syn-out.json"], + ) + s2p_bad = Node( + "blah", "nop", + os.path.join(td, "s2p_dir"), os.path.join(td, "par_dir"), + ["syn-out.json"], + ["s2p-out.json"], + ) + par = Node( + "par", "nop", + os.path.join(td, "par_dir"), os.path.join(td, "out_dir"), + ["s2p-out.json"], + ["par-out.json"], + ) + g = Graph({ + syn: [s2p_bad], + s2p_bad: [par], + par: [] + }, auto_auxiliary=False) + g.run(syn) + + self.assertEqual(syn.status, Status.COMPLETE) + self.assertEqual(s2p_bad.status, Status.INCOMPLETE) + self.assertEqual(par.status, Status.NOT_RUN) + + def test_resume_graph(self) -> None: + """ + Test that a user can stop and start a flow if needed. + """ + HammerVLSILogging.clear_callbacks() + HammerVLSILogging.add_callback(HammerVLSILogging.callback_buffering) + + with tempfile.TemporaryDirectory() as td: + os.mkdir(os.path.join(td, "syn_dir")) + os.mkdir(os.path.join(td, "par_dir")) + + with open(os.path.join(td, "syn_dir", "syn-in.yml"), 'w', encoding="utf-8") as tf1: + tf1.write(MOCK_CFG) + + syn = Node( + "syn", "nop", + os.path.join(td, "syn_dir"), os.path.join(td, "s2p_dir"), + ["syn-in.yml"], + ["syn-out.json"], + ) + s2p_bad = Node( + "blah", "nop", + os.path.join(td, "s2p_dir"), os.path.join(td, "par_dir"), + ["syn-out.json"], + ["s2p-out.json"], + ) + par = Node( + "par", "nop", + os.path.join(td, "par_dir"), os.path.join(td, "out_dir"), + ["s2p-out.json"], + ["par-out.json"], + ) + g = Graph({ + syn: [s2p_bad], + s2p_bad: [par], + par: [] + }, auto_auxiliary=False) + g_failed_run = g.run(syn) + + self.assertEqual(syn.status, Status.COMPLETE) + self.assertEqual(s2p_bad.status, Status.INCOMPLETE) + self.assertEqual(par.status, Status.NOT_RUN) + + s2p_bad.action = "syn-to-par" + g_good = g_failed_run.run(s2p_bad) + for n in g_good.networkx: + self.assertEqual(n.status, Status.COMPLETE) + + def test_mermaid(self) -> None: + """ + Test that Mermaid visualization works. + """ + with tempfile.TemporaryDirectory() as td: + os.mkdir(os.path.join(td, "syn_dir")) + os.mkdir(os.path.join(td, "par_dir")) + + syn = Node( + "syn", "nop", + os.path.join(td, "syn_dir"), os.path.join(td, "s2p_dir"), + ["syn-in.yml"], + ["syn-out.json"], + ) + s2p = Node( + "syn-to-par", "nop", + os.path.join(td, "s2p_dir"), os.path.join(td, "par_dir"), + ["syn-out.json"], + ["s2p-out.json"], + ) + par = Node( + "par", "nop", + os.path.join(td, "par_dir"), os.path.join(td, "out_dir"), + ["s2p-out.json"], + ["par-out.json"], + ) + g = Graph({ + syn: [s2p], + s2p: [par], + par: [] + }, auto_auxiliary=False) + + fname = g.to_mermaid() + with open(fname, 'r', encoding="utf-8") as f: + s = f.readlines() + self.assertListEqual(s, + ["```mermaid\n", + "stateDiagram-v2\n", + " syn --> syn_to_par\n", + " syn_to_par --> par\n", + "```\n"]) + + def test_auto_auxiliary(self) -> None: + """ + Test that auxiliary actions are automatically inserted. + """ + HammerVLSILogging.clear_callbacks() + HammerVLSILogging.add_callback(HammerVLSILogging.callback_buffering) + + + with tempfile.TemporaryDirectory() as td: + os.mkdir(os.path.join(td, "syn_dir")) + os.mkdir(os.path.join(td, "par_dir")) + + with open(os.path.join(td, "syn_dir", "syn-in.yml"), 'w', encoding="utf-8") as tf1: + tf1.write(MOCK_CFG) + + syn = Node( + "syn", "nop", + os.path.join(td, "syn_dir"), os.path.join(td, "par_dir"), + ["syn-in.yml"], + ["syn-out.json"], + ) + par = Node( + "par", "nop", + os.path.join(td, "par_dir"), os.path.join(td, "out_dir"), + ["syn-out.json"], + ["par-out.json"], + ) + g = Graph({ + syn: [par], + par: [] + }) + self.assertEqual(g.networkx.number_of_nodes(), 3) # check that there are three nodes + self.assertEqual(g.networkx.number_of_edges(), 2) # check that there are two edge connections + g.run(syn) + + for n in g.networkx: + self.assertEqual(n.status, Status.COMPLETE) + + def test_flowgraph_hooks(self) -> None: + """ + Test that hooks can be used in flowgraphs. + """ + with tempfile.TemporaryDirectory() as td: + syn_dir = os.path.join(td, "syn_dir") + out_dir = os.path.join(td, "out_dir") + os.mkdir(syn_dir) + os.mkdir(out_dir) + + cfg = dedent(f""" + vlsi.core: + synthesis_tool: hammer.synthesis.mocksynth + technology: hammer.technology.nop + + synthesis.inputs: + top_module: dummy + input_files: ["/dev/null"] + + synthesis.mocksynth.temp_folder: {out_dir} + """) + + with open(os.path.join(syn_dir, "syn-in.yml"), 'w', encoding="utf-8") as tf1: + tf1.write(cfg) + + syn = Node( + "syn", "mocksynth", + syn_dir, out_dir, + ["syn-in.yml"], + ["syn-out.json"], + driver=NodeDummyDriver() + ) + + g = Graph({ + syn: [] + }) + g.run(syn) + + for i in range(1, 5): + file = os.path.join(out_dir, f"step{i}.txt") + if i in (2, 4): + self.assertFalse(os.path.exists(file)) + else: + self.assertTrue(os.path.exists(file)) + + @pytest.mark.skip + def test_encode_decode(self) -> None: + """ + Test that a flowgraph can be encoded and decoded. + """ + HammerVLSILogging.clear_callbacks() + HammerVLSILogging.add_callback(HammerVLSILogging.callback_buffering) + + with tempfile.TemporaryDirectory() as td: + os.mkdir(os.path.join(td, "syn_dir")) + os.mkdir(os.path.join(td, "par_dir")) + + with open(os.path.join(td, "syn_dir", "syn-in.yml"), 'w', encoding="utf-8") as tf1: + tf1.write(MOCK_CFG) + + syn = Node( + "syn", "nop", + os.path.join(td, "syn_dir"), os.path.join(td, "s2p_dir"), + ["syn-in.yml"], + ["syn-out.json"], + ) + s2p = Node( + "syn-to-par", "nop", + os.path.join(td, "s2p_dir"), os.path.join(td, "par_dir"), + ["syn-out.json"], + ["s2p-out.json"], + ) + par = Node( + "par", "nop", + os.path.join(td, "par_dir"), os.path.join(td, "out_dir"), + ["s2p-out.json"], + ["par-out.json"], + ) + g = Graph({ + syn: [s2p], + s2p: [par], + par: [] + }) + + out = json.dumps(g.to_json(), cls=flowgraph.NodeEncoder) + g_dec = json.loads(out, object_hook=flowgraph.as_node) + # print(g.to_json()) + # print(json_graph.node_link_graph(g_dec).nodes) + +class NodeDummyDriver(CLIDriver): + def get_extra_synthesis_hooks(self) -> list[HammerToolHookAction]: + extra_hooks = [ + HammerTool.make_removal_hook("step2"), + HammerTool.make_removal_hook("step4"), + ] + return extra_hooks + + +if __name__ == "__main__": + unittest.main()