From b81c4b50d8cb20ca9beadc18e9132aea92ce0019 Mon Sep 17 00:00:00 2001 From: Johan Hidding Date: Wed, 18 Oct 2023 23:47:23 +0200 Subject: [PATCH] documentation in readme; test of rot13; black --- Makefile | 1 + README.md | 60 +++++++++++++++++++++++++++++++++++- entangled/commands/loom.py | 14 ++------- entangled/config/__init__.py | 3 +- entangled/loom/__init__.py | 1 - entangled/loom/file_task.py | 4 ++- entangled/text_location.py | 1 - test/loom/test_loom.py | 47 +++++++++++++++++++--------- test/loom/test_phony.py | 4 +-- test/loom/test_program.py | 49 +++++++++++++++++++++++++++++ test/loom/test_task.py | 7 ++++- test/test_daemon.py | 9 ++++-- test/test_filedb.py | 1 + test/test_shebang.py | 7 ++--- test/test_watch_dir.py | 6 ++-- 15 files changed, 169 insertions(+), 45 deletions(-) diff --git a/Makefile b/Makefile index dc3d059..5e6b1cb 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ test: poetry run coverage run --source=entangled -m pytest poetry run coverage xml poetry run coverage report + poetry run mypy docs: poetry run mkdocs build diff --git a/README.md b/README.md index 762cb22..895231b 100644 --- a/README.md +++ b/README.md @@ -224,9 +224,67 @@ In principle, you could do a lot of things with the `build` hook, supposing that That being said, candidates for hooks could be: - *Code metadata*. Some code could use more meta data than just a name and language. One way to include metadata is by having a header that is separated with a three hyphen line `---` from the actual code content. A hook could change the way the code is tangled, possibly injecting the metadata as a docstring, or leaving it out of the tangled code and have the document generator use it for other purposes. -- *She-bang lines*. When you're coding scripts, it may be desirable to have `#!/bin/bash` equivalent line at the top. This is currently not supported in the Python port of Entangled. - *Integration with package managers* like `cargo`, `cabal`, `poetry` etc. These are usually configured in a separate file. A hook could be used to specify dependencies. That way you could also document why a certain dependency is needed, or why it needs to have the version you specify. +## Loom +Entangled has a small build engine (similar to GNU Make) embedded, called Loom. You may give it a list of tasks (specified in TOML) that may depend on one another. Loom will run these when dependencies are newer than the target. Execution is lazy and in parallel. Loom supports: + +- Running tasks by passing a script to any configured interpreter, e.g. Bash or Python. +- Redirecting `stdout` or `stdin` to or from files. +- Defining so called "phony" targets. +- Define `pattern` for programmable reuse. +- `include` other Loom files, even ones that need to be generated by another `task`. + +### Examples +To write out "Hello, World!" to a file `msg.txt`, we may do the following, + +```toml +[[task]] +targets = ["secret.txt"] +stdout = "secret.txt" +language = "Python" +script = """ +print("Uryyb, Jbeyq!") +""" +``` + +To have this message encoded define a pattern, + +```toml +[pattern.rot13] +targets = ["{stdout}"] +dependencies = ["{stdin}"] +stdout = "{stdout}" +stdin = "{stdin}" +language = "Bash" +script = """ +tr a-zA-Z n-za-mN-ZA-M +""" + +[[call]] +pattern = "rot13" + [call.args] + stdin = "secret.txt" + stdout = "msg.txt" +``` + +To define a phony target "all", + +```toml +[[task]] +targets = [["phony(all)"]] +dependencies = ["msg.txt"] +``` + +Features on the roadmap: +- Defining "tmpfile" targets. +- Enable Jinja in patterns. +- Specify that certain tasks should not run in parallel by having a named set of semaphores. +- Enable versioned output directory (maybe Jinja solves this) +- Make tasks less verbose, having `stdin` automatic dependency and `stdout` automatic target. + +We may yet decide to put Loom into an external Python package. + ## Support for Document Generators Entangled has been used successfully with the following document generators. Note that some of these examples were built using older versions of Entangled, but they should work just the same. diff --git a/entangled/commands/loom.py b/entangled/commands/loom.py index c82b561..30a724c 100644 --- a/entangled/commands/loom.py +++ b/entangled/commands/loom.py @@ -15,17 +15,9 @@ async def main(target_strs: list[str], force_run: bool, throttle: Optional[int]) await asyncio.gather(*jobs) -@argh.arg( - "targets", nargs="*", default=["phony(all)"], - help="name of target to run" -) -@argh.arg( - "-B", "--force-run", help="rebuild all dependencies" -) -@argh.arg( - "-j", "--throttle", help="limit number of concurrent jobs" -) +@argh.arg("targets", nargs="*", default=["phony(all)"], help="name of target to run") +@argh.arg("-B", "--force-run", help="rebuild all dependencies") +@argh.arg("-j", "--throttle", help="limit number of concurrent jobs") def loom(targets: list[str], force_run: bool = False, throttle: Optional[int] = None): """Build one of the configured targets.""" asyncio.run(main(targets, force_run, throttle)) - diff --git a/entangled/config/__init__.py b/entangled/config/__init__.py index c94557f..30534da 100644 --- a/entangled/config/__init__.py +++ b/entangled/config/__init__.py @@ -48,8 +48,7 @@ class Markers: markers = Markers( - r"^(?P\s*)```\s*{(?P[^{}]*)}\s*$", - r"^(?P\s*)```\s*$" + r"^(?P\s*)```\s*{(?P[^{}]*)}\s*$", r"^(?P\s*)```\s*$" ) diff --git a/entangled/loom/__init__.py b/entangled/loom/__init__.py index fa9313a..efe9e66 100644 --- a/entangled/loom/__init__.py +++ b/entangled/loom/__init__.py @@ -2,4 +2,3 @@ from .file_task import LoomTask, LoomTaskDB, Target __all__ = ["LoomProgram", "resolve_tasks", "LoomTask", "LoomTaskDB", "Target"] - diff --git a/entangled/loom/file_task.py b/entangled/loom/file_task.py index f523802..dc4a219 100644 --- a/entangled/loom/file_task.py +++ b/entangled/loom/file_task.py @@ -145,7 +145,9 @@ async def run(self, cfg): tgt_str = "(" + " ".join(str(t) for t in self.targets) + ")" logging.info(f"{tgt_str} -> {runner.command} " + " ".join(args)) async with cfg.throttle or nullcontext(): - proc = await create_subprocess_exec(runner.command, *args, stdin=stdin, stdout=stdout) + proc = await create_subprocess_exec( + runner.command, *args, stdin=stdin, stdout=stdout + ) await proc.communicate() if tmpfile is not None: diff --git a/entangled/text_location.py b/entangled/text_location.py index c1a7607..21b597c 100644 --- a/entangled/text_location.py +++ b/entangled/text_location.py @@ -8,4 +8,3 @@ class TextLocation: def __str__(self): return f"{self.filename}:{self.line_number}" - diff --git a/test/loom/test_loom.py b/test/loom/test_loom.py index 4e97f5b..e04864e 100644 --- a/test/loom/test_loom.py +++ b/test/loom/test_loom.py @@ -27,9 +27,13 @@ async def test_hello(tmp_path: Path): with chdir(tmp_path): db = LoomTaskDB() tgt = Path("hello.txt") - db.target(tgt, [], language="Python", script=\ - f"with open('{tgt}', 'w') as f:\n" \ - f" print(\"Hello, World!\", file=f)\n") + db.target( + tgt, + [], + language="Python", + script=f"with open('{tgt}', 'w') as f:\n" + f' print("Hello, World!", file=f)\n', + ) db.phony("all", [Target(tgt)]) await db.run(Target(Phony("all"))) @@ -43,8 +47,9 @@ async def test_hello_stdout(tmp_path: Path): with chdir(tmp_path): db = LoomTaskDB() tgt = Path("hello.txt") - db.target(tgt, [], language="Python", stdout=tgt, script=\ - "print(\"Hello, World!\")\n") + db.target( + tgt, [], language="Python", stdout=tgt, script='print("Hello, World!")\n' + ) db.phony("all", [Target(tgt)]) await db.run(Target(Phony("all"))) @@ -58,9 +63,7 @@ async def test_runtime(tmp_path: Path): with chdir(tmp_path): db = LoomTaskDB() for a in range(4): - db.phony( - f"sleep{a}", [], language="Bash", - script=f"sleep 0.2\n") + db.phony(f"sleep{a}", [], language="Bash", script=f"sleep 0.2\n") db.phony("all", [Target(Phony(f"sleep{a}")) for a in range(4)]) async with timer() as t: await db.run(Target(Phony("all"))) @@ -82,14 +85,29 @@ async def test_rebuild(tmp_path: Path): # Make tasks a, b, c = (Path(x) for x in "abc") # a = i1 + 1 - db.target(a, [Target(i1)], language="Python", stdout=a, - script="print(int(open('i1','r').read()) + 1)") + db.target( + a, + [Target(i1)], + language="Python", + stdout=a, + script="print(int(open('i1','r').read()) + 1)", + ) # b = a * i2 - db.target(b, [Target(a), Target(i2)], language="Python", stdout=b, - script="print(int(open('a','r').read()) * int(open('i2','r').read()))") + db.target( + b, + [Target(a), Target(i2)], + language="Python", + stdout=b, + script="print(int(open('a','r').read()) * int(open('i2','r').read()))", + ) # c = a + b - db.target(c, [Target(a), Target(b)], language="Python", stdout=c, - script="print(int(open('b','r').read()) * int(open('a','r').read()))") + db.target( + c, + [Target(a), Target(b)], + language="Python", + stdout=c, + script="print(int(open('b','r').read()) * int(open('a','r').read()))", + ) await db.run(Target(c)) assert all(x.exists() for x in (a, b, c)) assert c.read_text() == "12\n" @@ -108,4 +126,3 @@ async def test_rebuild(tmp_path: Path): assert a.read_text() == "2\n" assert b.read_text() == "8\n" assert c.read_text() == "16\n" - diff --git a/test/loom/test_phony.py b/test/loom/test_phony.py index e90dfb8..662c66e 100644 --- a/test/loom/test_phony.py +++ b/test/loom/test_phony.py @@ -1,9 +1,9 @@ from entangled.loom.file_task import Phony -# from entangled.parsing import + +# from entangled.parsing import def test_phony_parsing(): x, _ = Phony.read("phony(all)") assert x == Phony("all") assert str(x) == "phony(all)" - diff --git a/test/loom/test_program.py b/test/loom/test_program.py index d95b836..d7e84f5 100644 --- a/test/loom/test_program.py +++ b/test/loom/test_program.py @@ -1,5 +1,6 @@ from contextlib import chdir from pathlib import Path +import sys import pytest from entangled.loom.file_task import Phony, Target @@ -57,6 +58,7 @@ async def test_loom(tmp_path): dependencies = ["hello.txt"] """ + @pytest.mark.asyncio async def test_include(tmp_path): with chdir(tmp_path): @@ -89,6 +91,7 @@ async def test_include(tmp_path): args = { stdout = "hello.txt", text = "Hello, World" } """ + @pytest.mark.asyncio async def test_pattern(tmp_path): with chdir(tmp_path): @@ -101,3 +104,49 @@ async def test_pattern(tmp_path): await db.run(Target(Phony("all"))) assert tgt.exists() assert tgt.read_text() == "Hello, World\n" + + +rot_13_loom = """ +[[task]] +targets = ["secret.txt"] +stdout = "secret.txt" +language = "Python" +script = \"\"\" +print("Uryyb, Jbeyq!") +\"\"\" + +[pattern.rot13] +targets = ["{stdout}"] +dependencies = ["{stdin}"] +stdout = "{stdout}" +stdin = "{stdin}" +language = "Bash" +script = \"\"\" +tr a-zA-Z n-za-mN-ZA-M +\"\"\" + +[[call]] +pattern = "rot13" + [call.args] + stdin = "secret.txt" + stdout = "hello.txt" + +[[task]] +targets = ["phony(all)"] +dependencies = ["hello.txt"] +""" + + +@pytest.mark.skipif(sys.platform == "win32", reason="no `tr` on windows") +@pytest.mark.asyncio +async def test_rot13(tmp_path): + with chdir(tmp_path): + src = Path("hello.toml") + tgt = Path("hello.txt") + src.write_text(rot_13_loom) + prg = LoomProgram.read(src) + db = await resolve_tasks(prg) + assert db.index[Target(tgt)].stdout == tgt + await db.run(Target(Phony("all"))) + assert tgt.exists() + assert tgt.read_text() == "Hello, World!\n" diff --git a/test/loom/test_task.py b/test/loom/test_task.py index 2efa830..18a21bf 100644 --- a/test/loom/test_task.py +++ b/test/loom/test_task.py @@ -5,6 +5,7 @@ from entangled.loom.task import Task, TaskDB import uuid + @dataclass class PyFunc(Task[str, Any]): foo: Any @@ -17,6 +18,7 @@ async def run(self): async def eval(self): return await self.db.run(self.targets[0]) + @dataclass class PyLiteral(Task[str, Any]): value: Any @@ -24,6 +26,7 @@ class PyLiteral(Task[str, Any]): async def run(self): return self.value + class PyTaskDB(TaskDB[str, Any]): def lazy(self, f): def delayed(*args): @@ -40,8 +43,10 @@ def delayed(*args): task = PyFunc([target], deps, f, self) self.add(task) return task + return delayed + @pytest.mark.asyncio async def test_noodles(): db = PyTaskDB() @@ -61,6 +66,7 @@ def pure(v): db.clean() exec_order = [] + @db.lazy def add2(label, x, y): exec_order.append(label) @@ -75,4 +81,3 @@ def add2(label, x, y): assert w_result.value == 13 assert exec_order[-1] == "w" assert exec_order[0] == "x" - diff --git a/test/test_daemon.py b/test/test_daemon.py index 9e9fcaa..c542711 100644 --- a/test/test_daemon.py +++ b/test/test_daemon.py @@ -9,27 +9,30 @@ from contextlib import chdir + def wait_for_file(filename, timeout=5): start_time = time.time() while time.time() - start_time < timeout: if os.path.exists(filename): return True - time.sleep(0.1) + time.sleep(0.1) return False + def wait_for_stat_diff(md_stat, filename, timeout=5): start_time = time.time() while time.time() - start_time < timeout: md_stat2 = stat(Path(filename)) - if(md_stat != md_stat2): + if md_stat != md_stat2: return True - time.sleep(0.1) + time.sleep(0.1) return False + def test_daemon(tmp_path: Path): config.read() with chdir(tmp_path): diff --git a/test/test_filedb.py b/test/test_filedb.py index 95534d7..091518b 100644 --- a/test/test_filedb.py +++ b/test/test_filedb.py @@ -4,6 +4,7 @@ import pytest from contextlib import chdir + @pytest.fixture(scope="session") def example_files(tmp_path_factory: pytest.TempPathFactory): tmp_path = tmp_path_factory.mktemp("test-filedb") diff --git a/test/test_shebang.py b/test/test_shebang.py index 2ecc75d..7d0c347 100644 --- a/test/test_shebang.py +++ b/test/test_shebang.py @@ -37,6 +37,7 @@ ``` """ + def test_shebang(): md = MarkdownReader("-") md.run(input_md) @@ -44,12 +45,8 @@ def test_shebang(): assert next(md.reference_map["test.sh"]).header == "#!/bin/bash" content, _ = tangle_ref(md.reference_map, "test.sh", AnnotationMethod.STANDARD) assert content.strip() == output_test_sh.strip() - + cr = CodeReader("test.sh", md.reference_map) cr.run(output_test_sh_modified) md_content = stitch_markdown(md.reference_map, md.content) assert md_content.strip() == input_md_modified.strip() - - - - diff --git a/test/test_watch_dir.py b/test/test_watch_dir.py index f32ef4d..ee6a415 100644 --- a/test/test_watch_dir.py +++ b/test/test_watch_dir.py @@ -29,6 +29,7 @@ Don't tangle me! """ + def test_watch_dirs(tmp_path): with chdir(tmp_path): Path("./docs").mkdir() @@ -41,7 +42,8 @@ def test_watch_dirs(tmp_path): assert set(find_watch_dirs()) == set([Path("./docs"), Path("./src")]) Path("./docs/index.md").write_text(index_md_1 + "\n" + index_md_2) tangle() - assert set(find_watch_dirs()) == set([Path("."), Path("./docs"), Path("./src")]) + assert set(find_watch_dirs()) == set( + [Path("."), Path("./docs"), Path("./src")] + ) assert sorted(list_input_files()) == [Path("./docs/index.md")] -