Merge pull request #33 from entangled/feature-loom

Feature loom

.gitignore

@@ -5,4 +5,3 @@ coverage.*
 dist
 poetry.lock
 site
-

Makefile

@@ -4,6 +4,7 @@ test:
 	poetry run coverage run --source=entangled -m pytest
 	poetry run coverage xml
 	poetry run coverage report
+	poetry run mypy
 
 docs:
 	poetry run mkdocs build

README.md

@@ -224,9 +224,64 @@ In principle, you could do a lot of things with the `build` hook, supposing that
 That being said, candidates for hooks could be: 
 
 - *Code metadata*. Some code could use more meta data than just a name and language. One way to include metadata is by having a header that is separated with a three hyphen line `---` from the actual code content. A hook could change the way the code is tangled, possibly injecting the metadata as a docstring, or leaving it out of the tangled code and have the document generator use it for other purposes.
-- *She-bang lines*. When you're coding scripts, it may be desirable to have `#!/bin/bash` equivalent line at the top. This is currently not supported in the Python port of Entangled.
 - *Integration with package managers* like `cargo`, `cabal`, `poetry` etc. These are usually configured in a separate file. A hook could be used to specify dependencies. That way you could also document why a certain dependency is needed, or why it needs to have the version you specify.
 
+## Loom
+Entangled has a small build engine (similar to GNU Make) embedded, called Loom. You may give it a list of tasks (specified in TOML) that may depend on one another. Loom will run these when dependencies are newer than the target. Execution is lazy and in parallel. Loom supports:
+
+- Running tasks by passing a script to any configured interpreter, e.g. Bash or Python.
+- Redirecting `stdout` or `stdin` to or from files.
+- Defining so called "phony" targets.
+- Define `pattern` for programmable reuse.
+- `include` other Loom files, even ones that need to be generated by another `task`.
+- `stdin` and `stdout` are automatic dependency and target.
+
+### Examples
+To write out "Hello, World!" to a file `msg.txt`, we may do the following,
+
+```toml
+[[task]]
+stdout = "secret.txt"
+language = "Python"
+script = """
+print("Uryyb, Jbeyq!")
+"""
+```
+
+To have this message decoded define a pattern,
+
+```toml
+[pattern.rot13]
+stdout = "{stdout}"
+stdin = "{stdin}"
+language = "Bash"
+script = """
+tr a-zA-Z n-za-mN-ZA-M
+"""
+
+[[call]]
+pattern = "rot13"
+  [call.args]
+  stdin = "secret.txt"
+  stdout = "msg.txt"
+```
+
+To define a phony target "all",
+
+```toml
+[[task]]
+targets = ["phony(all)"]
+dependencies = ["msg.txt"]
+```
+
+Features on the roadmap:
+- Defining "tmpfile" targets.
+- Enable Jinja in patterns.
+- Specify that certain tasks should not run in parallel by having a named set of semaphores.
+- Enable versioned output directory (maybe Jinja solves this)
+
+We may yet decide to put Loom into an external Python package.
+
 ## Support for Document Generators
 Entangled has been used successfully with the following document generators. Note that some of these examples were built using older versions of Entangled, but they should work just the same.
 

entangled/commands/__init__.py

@@ -4,11 +4,13 @@
 from .sync import sync
 from .tangle import tangle
 from .watch import watch
+from .loom import loom
 
 
 __all__ = [
         "new",
-        "status"
+        "loom",
+        "status",
         "stitch",
         "sync",
         "tangle",

entangled/commands/loom.py

@@ -0,0 +1,27 @@
+from typing import Optional
+import argh  # type: ignore
+import asyncio
+
+from ..config import config
+from ..loom import resolve_tasks, Target
+from ..logging import logger
+
+log = logger()
+
+async def main(target_strs: list[str], force_run: bool, throttle: Optional[int]):
+    db = await resolve_tasks(config.loom)
+    for t in db.tasks:
+        log.debug(str(t))
+    if throttle:
+        db.throttle = asyncio.Semaphore(throttle)
+    db.force_run = force_run
+    jobs = [db.run(Target.from_str(t)) for t in target_strs]
+    await asyncio.gather(*jobs)
+
+
+@argh.arg("targets", nargs="+", help="name of target to run")
+@argh.arg("-B", "--force-run", help="rebuild all dependencies")
+@argh.arg("-j", "--throttle", help="limit number of concurrent jobs")
+def loom(targets: list[str], force_run: bool = False, throttle: Optional[int] = None):
+    """Build one of the configured targets."""
+    asyncio.run(main(targets, force_run, throttle))

entangled/config/__init__.py

@@ -4,7 +4,6 @@
 
 from __future__ import annotations
 
-import logging
 import threading
 from contextlib import contextmanager
 from copy import copy
@@ -15,9 +14,16 @@
 
 import tomlkit
 
+from entangled.errors.user import UserError
+
+from ..loom import Program
 from ..construct import construct
 from .language import Language, languages
 from .version import Version
+from ..logging import logger
+
+
+log = logger()
 
 
 class AnnotationMethod(Enum):
@@ -42,22 +48,35 @@ class Markers:
 
     open: str
     close: str
-    begin_ignore: str
-    end_ignore: str
+    begin_ignore: str = r"^\s*\~\~\~markdown\s*$"
+    end_ignore: str = r"^\s*\~\~\~\s*$"
 
 
 markers = Markers(
-    r"^(?P<indent>\s*)```\s*{(?P<properties>[^{}]*)}\s*$",
-    r"^(?P<indent>\s*)```\s*$",
-    r"^\s*\~\~\~markdown\s*$",
-    r"^\s*\~\~\~\s*$",
+    r"^(?P<indent>\s*)```\s*{(?P<properties>[^{}]*)}\s*$", r"^(?P<indent>\s*)```\s*$"
 )
 
 
 @dataclass
 class Config(threading.local):
-    """Main config class. This class is made thread-local to make
-    it possible to test in parallel."""
+    """Main config class.
+
+    Attributes:
+        version: Version of Entangled for which this config was created.
+            Entangled should read all versions lower than its own.
+        languages: List of programming languages and their comment styles.
+        markers: Regexes for detecting open and close of code blocks.
+        watch_list: List of glob-expressions indicating files to include
+            for tangling.
+        annotation: Style of annotation.
+        annotation_format: Extra annotation.
+        use_line_directives: Wether to print pragmas in source code for
+            indicating markdown source locations.
+        hooks: List of enabled hooks.
+        hook: Sub-config of hooks.
+        loom: Sub-config of loom.
+
+    This class is made thread-local to make it possible to test in parallel."""
 
     version: Version
     languages: list[Language] = field(default_factory=list)
@@ -69,6 +88,7 @@ class Config(threading.local):
     use_line_directives: bool = False
     hooks: list[str] = field(default_factory=list)
     hook: dict = field(default_factory=dict)
+    loom: Program = field(default_factory=Program)
 
     def __post_init__(self):
         self.languages = languages + self.languages
@@ -109,11 +129,11 @@ def read_config_from_toml(
                     json = json[s]
             return construct(Config, json)
     except ValueError as e:
-        logging.error("Could not read config: %s", e)
+        log.error("Could not read config: %s", e)
         return None
     except KeyError as e:
-        logging.debug("%s", e)
-        logging.debug("The config file %s should contain a section %s", path, section)
+        log.debug("%s", str(e))
+        log.debug("The config file %s should contain a section %s", path, section)
         return None
 
 
@@ -147,6 +167,7 @@ def __call__(self, **kwargs):
             setattr(self.config, k, backup[k])
 
     def get_language(self, lang_name: str) -> Optional[Language]:
+        assert self.config
         return self.config.language_index.get(lang_name, None)
 
 

entangled/construct.py

@@ -1,18 +1,28 @@
-from typing import Union
+from pathlib import Path
+from typing import Any, Union
 from dataclasses import is_dataclass
 from enum import Enum
 
 import typing
 import types
 
+from entangled.errors.user import ConfigError
+
 from .parsing import Parser
 
 
 def isgeneric(annot):
     return hasattr(annot, "__origin__") and hasattr(annot, "__args__")
 
 
-def construct(annot, json):
+def construct(annot: Any, json: Any) -> Any:
+    try:
+        return _construct(annot, json)
+    except (AssertionError, ValueError):
+        raise ConfigError(annot, json)
+
+
+def _construct(annot: Any, json: Any) -> Any:
     """Construct an object from a given type from a JSON stream.
 
     The `annot` type should be one of: str, int, list[T], Optional[T],
@@ -28,9 +38,20 @@ def construct(annot, json):
     if isinstance(json, str) and isinstance(annot, Parser):
         result, _ = annot.read(json)
         return result
+    if (
+        isgeneric(annot)
+        and typing.get_origin(annot) is dict
+        and typing.get_args(annot)[0] is str
+    ):
+        assert isinstance(json, dict)
+        return {k: construct(typing.get_args(annot)[1], v) for k, v in json.items()}
+    if annot is Any:
+        return json
     if annot is dict or isgeneric(annot) and typing.get_origin(annot) is dict:
         assert isinstance(json, dict)
         return json
+    if annot is Path and isinstance(json, str):
+        return Path(json)
     if isgeneric(annot) and typing.get_origin(annot) is list:
         assert isinstance(json, list)
         return [construct(typing.get_args(annot)[0], item) for item in json]
@@ -49,7 +70,7 @@ def construct(annot, json):
         # assert all(k in json for k in arg_annot)
         args = {k: construct(arg_annot[k], json[k]) for k in json}
         return annot(**args)
-    if isinstance(json, str) and issubclass(annot, Enum):
+    if isinstance(json, str) and isinstance(annot, type) and issubclass(annot, Enum):
         options = {opt.name.lower(): opt for opt in annot}
         assert json.lower() in options
         return options[json.lower()]

entangled/document.py

@@ -7,6 +7,7 @@
 from .config import Language, AnnotationMethod, config
 from .properties import Property, get_attribute
 from .errors.internal import InternalError
+from .text_location import TextLocation
 
 
 def length(iter: Iterable[Any]) -> int:
@@ -31,15 +32,6 @@ class PlainText:
 Content = Union[PlainText, ReferenceId]
 
 
-@dataclass
-class TextLocation:
-    filename: str
-    line_number: int = 0
-
-    def __str__(self):
-        return f"{self.filename}:{self.line_number}"
-
-
 @dataclass
 class CodeBlock:
     language: Language

entangled/errors/internal.py

@@ -13,7 +13,7 @@ def __str__(self):
         return f"Internal error: {self.msg}"
 
 
-def bug_contact():
+def bug_contact(e: Exception):
     logging.error(
         "This error is due to an internal bug in Entangled. Please file an "
         "issue including the above stack trace "

entangled/errors/user.py

@@ -1,15 +1,23 @@
 from dataclasses import dataclass
-from textwrap import wrap
-from typing import Callable
+from typing import Any, Callable
 
-from ..document import TextLocation
+from ..text_location import TextLocation
 
 
 class UserError(Exception):
     def __str__(self):
         return "Unknown user error."
 
 
+@dataclass
+class ConfigError(UserError):
+    expected: str
+    got: Any
+
+    def __str__(self):
+        return f"Expected {self.expected}, got: {self.got}"
+
+
 @dataclass
 class HelpfulUserError(UserError):
     """Raise a user error and supply an optional function `func` for context.

entangled/logging.py

@@ -0,0 +1,42 @@
+import logging
+from rich.logging import RichHandler
+from rich.highlighter import RegexHighlighter
+
+from .version import __version__
+
+LOGGING_SETUP = False
+
+
+class BackTickHighlighter(RegexHighlighter):
+    highlights = [r"`(?P<bold>[^`]*)`"]
+
+
+def logger():
+    return logging.getLogger("entangled")
+
+
+def configure(debug=False):
+    global LOGGING_SETUP
+    if LOGGING_SETUP:
+        return
+
+    if debug:
+        level = logging.DEBUG
+    else:
+        level = logging.INFO
+
+    FORMAT = "%(message)s"
+    logging.basicConfig(
+        level=level,
+        format=FORMAT,
+        datefmt="[%X]",
+        handlers=
+        [RichHandler(show_path=debug, highlighter=BackTickHighlighter())],
+    )
+    log = logging.getLogger("entangled")
+    log.setLevel(level)
+    # log.addHandler(RichHandler(show_path=debug, highlighter=BackTickHighlighter()))
+    # log.propagate = False
+    log.info(f"Entangled {__version__} (https://entangled.github.io/)")
+
+    LOGGING_SETUP = True

entangled/loom/__init__.py

@@ -0,0 +1,4 @@
+from .program import Program, resolve_tasks
+from .task import Task, TaskDB, Target
+
+__all__ = ["Program", "resolve_tasks", "Task", "TaskDB", "Target"]