documentation in readme; test of rot13; black

Makefile

@@ -4,6 +4,7 @@ test:
 	poetry run coverage run --source=entangled -m pytest
 	poetry run coverage xml
 	poetry run coverage report
+	poetry run mypy
 
 docs:
 	poetry run mkdocs build

README.md

@@ -224,9 +224,67 @@ In principle, you could do a lot of things with the `build` hook, supposing that
 That being said, candidates for hooks could be: 
 
 - *Code metadata*. Some code could use more meta data than just a name and language. One way to include metadata is by having a header that is separated with a three hyphen line `---` from the actual code content. A hook could change the way the code is tangled, possibly injecting the metadata as a docstring, or leaving it out of the tangled code and have the document generator use it for other purposes.
-- *She-bang lines*. When you're coding scripts, it may be desirable to have `#!/bin/bash` equivalent line at the top. This is currently not supported in the Python port of Entangled.
 - *Integration with package managers* like `cargo`, `cabal`, `poetry` etc. These are usually configured in a separate file. A hook could be used to specify dependencies. That way you could also document why a certain dependency is needed, or why it needs to have the version you specify.
 
+## Loom
+Entangled has a small build engine (similar to GNU Make) embedded, called Loom. You may give it a list of tasks (specified in TOML) that may depend on one another. Loom will run these when dependencies are newer than the target. Execution is lazy and in parallel. Loom supports:
+
+- Running tasks by passing a script to any configured interpreter, e.g. Bash or Python.
+- Redirecting `stdout` or `stdin` to or from files.
+- Defining so called "phony" targets.
+- Define `pattern` for programmable reuse.
+- `include` other Loom files, even ones that need to be generated by another `task`.
+
+### Examples
+To write out "Hello, World!" to a file `msg.txt`, we may do the following,
+
+```toml
+[[task]]
+targets = ["secret.txt"]
+stdout = "secret.txt"
+language = "Python"
+script = """
+print("Uryyb, Jbeyq!")
+"""
+```
+
+To have this message encoded define a pattern,
+
+```toml
+[pattern.rot13]
+targets = ["{stdout}"]
+dependencies = ["{stdin}"]
+stdout = "{stdout}"
+stdin = "{stdin}"
+language = "Bash"
+script = """
+tr a-zA-Z n-za-mN-ZA-M
+"""
+
+[[call]]
+pattern = "rot13"
+  [call.args]
+  stdin = "secret.txt"
+  stdout = "msg.txt"
+```
+
+To define a phony target "all",
+
+```toml
+[[task]]
+targets = [["phony(all)"]]
+dependencies = ["msg.txt"]
+```
+
+Features on the roadmap:
+- Defining "tmpfile" targets.
+- Enable Jinja in patterns.
+- Specify that certain tasks should not run in parallel by having a named set of semaphores.
+- Enable versioned output directory (maybe Jinja solves this)
+- Make tasks less verbose, having `stdin` automatic dependency and `stdout` automatic target.
+
+We may yet decide to put Loom into an external Python package.
+
 ## Support for Document Generators
 Entangled has been used successfully with the following document generators. Note that some of these examples were built using older versions of Entangled, but they should work just the same.
 

entangled/commands/loom.py

@@ -15,17 +15,9 @@ async def main(target_strs: list[str], force_run: bool, throttle: Optional[int])
     await asyncio.gather(*jobs)
 
 
-@argh.arg(
-    "targets", nargs="*", default=["phony(all)"],
-    help="name of target to run"
-)
-@argh.arg(
-    "-B", "--force-run", help="rebuild all dependencies"
-)
-@argh.arg(
-    "-j", "--throttle", help="limit number of concurrent jobs"
-)
+@argh.arg("targets", nargs="*", default=["phony(all)"], help="name of target to run")
+@argh.arg("-B", "--force-run", help="rebuild all dependencies")
+@argh.arg("-j", "--throttle", help="limit number of concurrent jobs")
 def loom(targets: list[str], force_run: bool = False, throttle: Optional[int] = None):
     """Build one of the configured targets."""
     asyncio.run(main(targets, force_run, throttle))
-

entangled/config/__init__.py

@@ -48,8 +48,7 @@ class Markers:
 
 
 markers = Markers(
-    r"^(?P<indent>\s*)```\s*{(?P<properties>[^{}]*)}\s*$",
-    r"^(?P<indent>\s*)```\s*$"
+    r"^(?P<indent>\s*)```\s*{(?P<properties>[^{}]*)}\s*$", r"^(?P<indent>\s*)```\s*$"
 )
 
 

entangled/loom/__init__.py

@@ -2,4 +2,3 @@
 from .file_task import LoomTask, LoomTaskDB, Target
 
 __all__ = ["LoomProgram", "resolve_tasks", "LoomTask", "LoomTaskDB", "Target"]
-

entangled/loom/file_task.py

@@ -145,7 +145,9 @@ async def run(self, cfg):
         tgt_str = "(" + " ".join(str(t) for t in self.targets) + ")"
         logging.info(f"{tgt_str} -> {runner.command} " + " ".join(args))
         async with cfg.throttle or nullcontext():
-            proc = await create_subprocess_exec(runner.command, *args, stdin=stdin, stdout=stdout)
+            proc = await create_subprocess_exec(
+                runner.command, *args, stdin=stdin, stdout=stdout
+            )
         await proc.communicate()
 
         if tmpfile is not None:

entangled/text_location.py

@@ -8,4 +8,3 @@ class TextLocation:
 
     def __str__(self):
         return f"{self.filename}:{self.line_number}"
-

test/loom/test_loom.py

@@ -27,9 +27,13 @@ async def test_hello(tmp_path: Path):
     with chdir(tmp_path):
         db = LoomTaskDB()
         tgt = Path("hello.txt")
-        db.target(tgt, [], language="Python", script=\
-            f"with open('{tgt}', 'w') as f:\n" \
-            f"   print(\"Hello, World!\", file=f)\n")
+        db.target(
+            tgt,
+            [],
+            language="Python",
+            script=f"with open('{tgt}', 'w') as f:\n"
+            f'   print("Hello, World!", file=f)\n',
+        )
         db.phony("all", [Target(tgt)])
 
         await db.run(Target(Phony("all")))
@@ -43,8 +47,9 @@ async def test_hello_stdout(tmp_path: Path):
     with chdir(tmp_path):
         db = LoomTaskDB()
         tgt = Path("hello.txt")
-        db.target(tgt, [], language="Python", stdout=tgt, script=\
-            "print(\"Hello, World!\")\n")
+        db.target(
+            tgt, [], language="Python", stdout=tgt, script='print("Hello, World!")\n'
+        )
         db.phony("all", [Target(tgt)])
 
         await db.run(Target(Phony("all")))
@@ -58,9 +63,7 @@ async def test_runtime(tmp_path: Path):
     with chdir(tmp_path):
         db = LoomTaskDB()
         for a in range(4):
-            db.phony(
-                f"sleep{a}", [], language="Bash",
-                script=f"sleep 0.2\n")
+            db.phony(f"sleep{a}", [], language="Bash", script=f"sleep 0.2\n")
         db.phony("all", [Target(Phony(f"sleep{a}")) for a in range(4)])
         async with timer() as t:
             await db.run(Target(Phony("all")))
@@ -82,14 +85,29 @@ async def test_rebuild(tmp_path: Path):
         # Make tasks
         a, b, c = (Path(x) for x in "abc")
         # a = i1 + 1
-        db.target(a, [Target(i1)], language="Python", stdout=a,
-                  script="print(int(open('i1','r').read()) + 1)")
+        db.target(
+            a,
+            [Target(i1)],
+            language="Python",
+            stdout=a,
+            script="print(int(open('i1','r').read()) + 1)",
+        )
         # b = a * i2
-        db.target(b, [Target(a), Target(i2)], language="Python", stdout=b,
-                  script="print(int(open('a','r').read()) * int(open('i2','r').read()))")
+        db.target(
+            b,
+            [Target(a), Target(i2)],
+            language="Python",
+            stdout=b,
+            script="print(int(open('a','r').read()) * int(open('i2','r').read()))",
+        )
         # c = a + b
-        db.target(c, [Target(a), Target(b)], language="Python", stdout=c,
-                  script="print(int(open('b','r').read()) * int(open('a','r').read()))")
+        db.target(
+            c,
+            [Target(a), Target(b)],
+            language="Python",
+            stdout=c,
+            script="print(int(open('b','r').read()) * int(open('a','r').read()))",
+        )
         await db.run(Target(c))
         assert all(x.exists() for x in (a, b, c))
         assert c.read_text() == "12\n"
@@ -108,4 +126,3 @@ async def test_rebuild(tmp_path: Path):
         assert a.read_text() == "2\n"
         assert b.read_text() == "8\n"
         assert c.read_text() == "16\n"
-

test/loom/test_phony.py

@@ -1,9 +1,9 @@
 from entangled.loom.file_task import Phony
-# from entangled.parsing import 
+
+# from entangled.parsing import
 
 
 def test_phony_parsing():
     x, _ = Phony.read("phony(all)")
     assert x == Phony("all")
     assert str(x) == "phony(all)"
-

test/loom/test_program.py

@@ -1,5 +1,6 @@
 from contextlib import chdir
 from pathlib import Path
+import sys
 
 import pytest
 from entangled.loom.file_task import Phony, Target
@@ -57,6 +58,7 @@ async def test_loom(tmp_path):
 dependencies = ["hello.txt"]
 """
 
+
 @pytest.mark.asyncio
 async def test_include(tmp_path):
     with chdir(tmp_path):
@@ -89,6 +91,7 @@ async def test_include(tmp_path):
 args = { stdout = "hello.txt", text = "Hello, World" }
 """
 
+
 @pytest.mark.asyncio
 async def test_pattern(tmp_path):
     with chdir(tmp_path):
@@ -101,3 +104,49 @@ async def test_pattern(tmp_path):
         await db.run(Target(Phony("all")))
         assert tgt.exists()
         assert tgt.read_text() == "Hello, World\n"
+
+
+rot_13_loom = """
+[[task]]
+targets = ["secret.txt"]
+stdout = "secret.txt"
+language = "Python"
+script = \"\"\"
+print("Uryyb, Jbeyq!")
+\"\"\"
+
+[pattern.rot13]
+targets = ["{stdout}"]
+dependencies = ["{stdin}"]
+stdout = "{stdout}"
+stdin = "{stdin}"
+language = "Bash"
+script = \"\"\"
+tr a-zA-Z n-za-mN-ZA-M
+\"\"\"
+
+[[call]]
+pattern = "rot13"
+  [call.args]
+  stdin = "secret.txt"
+  stdout = "hello.txt"
+
+[[task]]
+targets = ["phony(all)"]
+dependencies = ["hello.txt"]
+"""
+
+
+@pytest.mark.skipif(sys.platform == "win32", reason="no `tr` on windows")
+@pytest.mark.asyncio
+async def test_rot13(tmp_path):
+    with chdir(tmp_path):
+        src = Path("hello.toml")
+        tgt = Path("hello.txt")
+        src.write_text(rot_13_loom)
+        prg = LoomProgram.read(src)
+        db = await resolve_tasks(prg)
+        assert db.index[Target(tgt)].stdout == tgt
+        await db.run(Target(Phony("all")))
+        assert tgt.exists()
+        assert tgt.read_text() == "Hello, World!\n"

test/loom/test_task.py

@@ -5,6 +5,7 @@
 from entangled.loom.task import Task, TaskDB
 import uuid
 
+
 @dataclass
 class PyFunc(Task[str, Any]):
     foo: Any
@@ -17,13 +18,15 @@ async def run(self):
     async def eval(self):
         return await self.db.run(self.targets[0])
 
+
 @dataclass
 class PyLiteral(Task[str, Any]):
     value: Any
 
     async def run(self):
         return self.value
 
+
 class PyTaskDB(TaskDB[str, Any]):
     def lazy(self, f):
         def delayed(*args):
@@ -40,8 +43,10 @@ def delayed(*args):
             task = PyFunc([target], deps, f, self)
             self.add(task)
             return task
+
         return delayed
 
+
 @pytest.mark.asyncio
 async def test_noodles():
     db = PyTaskDB()
@@ -61,6 +66,7 @@ def pure(v):
     db.clean()
 
     exec_order = []
+
     @db.lazy
     def add2(label, x, y):
         exec_order.append(label)
@@ -75,4 +81,3 @@ def add2(label, x, y):
     assert w_result.value == 13
     assert exec_order[-1] == "w"
     assert exec_order[0] == "x"
-

test/test_daemon.py

@@ -9,27 +9,30 @@
 
 from contextlib import chdir
 
+
 def wait_for_file(filename, timeout=5):
     start_time = time.time()
 
     while time.time() - start_time < timeout:
         if os.path.exists(filename):
             return True
-        time.sleep(0.1)  
+        time.sleep(0.1)
 
     return False
 
+
 def wait_for_stat_diff(md_stat, filename, timeout=5):
     start_time = time.time()
 
     while time.time() - start_time < timeout:
         md_stat2 = stat(Path(filename))
-        if(md_stat != md_stat2):
+        if md_stat != md_stat2:
             return True
-        time.sleep(0.1)  
+        time.sleep(0.1)
 
     return False
 
+
 def test_daemon(tmp_path: Path):
     config.read()
     with chdir(tmp_path):

test/test_filedb.py

@@ -4,6 +4,7 @@
 import pytest
 from contextlib import chdir
 
+
 @pytest.fixture(scope="session")
 def example_files(tmp_path_factory: pytest.TempPathFactory):
     tmp_path = tmp_path_factory.mktemp("test-filedb")