Path fixes

.gitignore

@@ -4,3 +4,5 @@ gen
 build
 dist
 __pycache__
+scratch
+

base_rules.hancho

@@ -46,9 +46,11 @@ compile_cpp = hancho.Config(
     warnings = None,
     defines = None,
     includes = None,
-    in_depfile="{swap_ext(in_src, '.d')}",
     out_obj="{swap_ext(in_src, '.o')}",
 
+    # Note - this swaps out_obj and not in_src in case the user redirects out_obj
+    in_depfile="{swap_ext(out_obj, '.d')}",
+
     joined_warnings="{join_prefix('-W', warnings)}",
     joined_defines="{join_prefix('-D', defines)}",
     joined_includes="{join_prefix('-I', includes)}",

build.hancho

@@ -1,7 +1,10 @@
 hancho.config.force = True
 
-if 'base_rules' not in hancho:
-    hancho.base_rules = hancho.load("{hancho_dir}/base_rules.hancho")
+# Check that all our rule files load
+hancho.base_rules  = hancho.load("{hancho_dir}/base_rules.hancho")
+hancho.fpga_rules  = hancho.load("{hancho_dir}/fpga_rules.hancho")
+hancho.riscv_rules = hancho.load("{hancho_dir}/riscv_rules.hancho")
+hancho.wasm_rules  = hancho.load("{hancho_dir}/wasm_rules.hancho")
 
 #hancho.load("tutorial/tutorial.hancho")
 hancho.load("examples/examples.hancho")

examples/bazel-cpp-tutorial/stage1/stage1_rules.hancho

@@ -38,7 +38,7 @@ _compile_cpp = hancho.Config(
     desc       = "Compiling C++ {in_src} -> {out_obj}",
     command    = "g++ {gcc_flags} -c {in_src} -o {out_obj}",
     out_obj    = "{swap_ext(in_src, '.o')}",
-    in_depfile = "{swap_ext(in_src, '.d')}",
+    in_depfile = "{swap_ext(out_obj, '.d')}",
     gcc_flags  = _gcc_flags,
 )
 

examples/bazel-cpp-tutorial/stage2/stage2_rules.hancho

@@ -40,7 +40,7 @@ compile_cpp = hancho.Config(
     gcc_flags  = _gcc_flags,
     includes   = None,
     out_obj    = "{swap_ext(in_src, '.o')}",
-    in_depfile = "{swap_ext(in_src, '.d')}",
+    in_depfile = "{swap_ext(out_obj, '.d')}",
 )
 
 link_cpp_bin = hancho.Config(

examples/bazel-cpp-tutorial/stage3/stage3_rules.hancho

@@ -40,7 +40,7 @@ compile_cpp = hancho.Config(
     gcc_flags  = _gcc_flags,
     includes   = None,
     out_obj    = "{swap_ext(in_src, '.o')}",
-    in_depfile = "{swap_ext(in_src, '.d')}",
+    in_depfile = "{swap_ext(out_obj, '.d')}",
 )
 
 link_cpp_bin = hancho.Config(

examples/hello_world/build.hancho

@@ -4,7 +4,7 @@ compile_cpp = hancho.Config(
     desc       = "Compiling C++ {in_src} -> {out_obj}",
     command    = "g++ -MMD -c {in_src} -o {out_obj}",
     out_obj    = "{swap_ext(in_src, '.o')}",
-    in_depfile = "{swap_ext(in_src, '.d')}",
+    in_depfile = "{swap_ext(out_obj, '.d')}",
 )
 
 main_o = hancho(compile_cpp, in_src = "main.cpp")

examples/windows/build.hancho

@@ -6,7 +6,7 @@ compile_cpp = hancho.Config(
     desc       = "Compile {in_src} -> {out_obj}",
     command    = "cl.exe /nologo /c {in_src} /sourceDependencies {in_depfile} /Fo:{out_obj}",
     out_obj    = "{swap_ext(in_src, '.o')}",
-    in_depfile = "{swap_ext(in_src, '.d')}",
+    in_depfile = "{swap_ext(out_obj, '.d')}",
 )
 
 link_cpp = hancho.Config(

hancho.py

@@ -100,7 +100,7 @@ def abs_path(raw_path, strict=False) -> str | list[str]:
     if listlike(raw_path):
         return [abs_path(p, strict) for p in raw_path]
 
-    result = path.realpath(raw_path)
+    result = path.abspath(raw_path)
     if strict and not path.exists(result):
         raise FileNotFoundError(raw_path)
     return result
@@ -142,7 +142,7 @@ def normalize_path(file_path):
     assert isinstance(file_path, str)
     assert not macro_regex.search(file_path)
 
-    file_path = path.realpath(path.join(os.getcwd(), file_path))
+    file_path = path.abspath(path.join(os.getcwd(), file_path))
     assert path.isabs(file_path)
     if not path.isfile(file_path):
         print(f"Could not find file {file_path}")
@@ -243,6 +243,16 @@ def merge_variant(lhs, rhs):
     return copy.deepcopy(rhs)
 
 
+def apply_variant(key, val, apply):
+    apply(key, val)
+    if dictlike(val):
+        for key2, val2 in val.items():
+            apply_variant(key2, val2, apply)
+    elif listlike(val):
+        for key2, val2 in enumerate(val):
+            apply_variant(key2, val2, apply)
+    return val
+
 def map_variant(key, val, apply):
     val = apply(key, val)
     if dictlike(val):
@@ -348,7 +358,7 @@ class Utils:
     color       = staticmethod(color)
     flatten     = staticmethod(flatten)
     glob        = staticmethod(glob.glob)
-    hancho_dir  = abs_path(path.dirname(path.realpath(__file__)))
+    hancho_dir  = path.dirname(path.realpath(__file__))
     join_path   = staticmethod(join_path)
     join_prefix = staticmethod(join_prefix)
     join_suffix = staticmethod(join_suffix)
@@ -823,47 +833,83 @@ def task_init(self):
         debug = self.config.get("debug", app.flags.debug)
 
         if debug:
-            log(f"\nTask before expand: {self.config}")
+            log(f"\nTask before expand: {self}")
 
         # ----------------------------------------
         # Expand task_dir and build_dir
 
         # pylint: disable=attribute-defined-outside-init
-        self.config.task_dir = abs_path(self.config.expand(self.config.task_dir))
-        self.config.build_dir = abs_path(self.config.expand(self.config.build_dir))
+
+        self.config.task_dir   = abs_path(self.config.expand(self.config.task_dir))
+        self.config.build_dir  = abs_path(self.config.expand(self.config.build_dir))
+
+        if root_dir := self.config.get("root_dir", None):
+            if not self.config.build_dir.startswith(root_dir):
+                raise ValueError(
+                    f"Path error, build_dir {self.config.build_dir} is not under root_dir {root_dir}"
+                )
 
         # ----------------------------------------
-        # Expand all in_ and out_ filenames.
+        # Expand all in_ and out_ filenames
         # We _must_ expand these first before joining paths or the paths will be incorrect:
         # prefix + swap(abs_path) != abs(prefix + swap(path))
 
         for key, val in self.config.items():
             if key.startswith("in_") or key.startswith("out_"):
-                self.config[key] = self.config.expand(val)
+                def expand_path(_, val):
+                    if not isinstance(val, str):
+                        return val
+                    val = self.config.expand(val)
+                    val = path.normpath(val)
+                    return val
+                self.config[key] = map_variant(key, val, expand_path)
 
-        # ----------------------------------------
-        # Convert all in_, out_, and deps filenames to absolute paths.
-
-        def join_dir(key, val):
-            if isinstance(key, str) and val is not None:
-                if key == "in_depfile":
-                    val = join_path(self.config.build_dir, val)
-                    val = abs_path(val)
-                    # Note - we only add the depfile to in_files _if_it_exists_, otherwise we will
-                    # fail a check that all our inputs are present.
-                    if path.isfile(val):
-                        self.in_files.append(val)
-                elif key.startswith("out_"):
-                    val = join_path(self.config.build_dir, val)
-                    val = abs_path(val)
-                    self.out_files.extend(flatten(val))
-                elif key.startswith("in_"):
-                    val = join_path(self.config.task_dir, val)
-                    val = abs_path(val)
-                    self.in_files.extend(flatten(val))
-            return val
-
-        map_variant(None, self.config, join_dir)
+        # Make all in_ and out_ file paths absolute
+
+        # FIXME feeling like in_depfile should really be io_depfile...
+
+        for key, val in self.config.items():
+            if key.startswith("out_") or key == "in_depfile":
+                def move_to_builddir(_, val):
+                    if not isinstance(val, str):
+                        return val
+                    # Note this conditional needs to be first, as build_dir can itself be under
+                    # task_dir
+                    if val.startswith(self.config.build_dir):
+                        # Absolute path under build_dir, do nothing.
+                        pass
+                    elif val.startswith(self.config.task_dir):
+                        # Absolute path under task_dir, move to build_dir
+                        val = rel_path(val, self.config.task_dir)
+                        val = join_path(self.config.build_dir, val)
+                    elif val.startswith("/"):
+                        raise ValueError(f"Output file has absolute path that is not under task_dir or build_dir : {val}")
+                    else:
+                        # Relative path, add build_dir
+                        val = join_path(self.config.build_dir, val)
+                    return val
+                self.config[key] = map_variant(key, val, move_to_builddir)
+            elif key.startswith("in_"):
+                def move_to_taskdir(key, val):
+                    if not isinstance(val, str):
+                        return val
+                    if not val.startswith("/"):
+                        val = join_path(self.config.task_dir, val)
+                    return val
+                self.config[key] = map_variant(key, val, move_to_taskdir)
+
+        # Gather all inputs to task.in_files and outputs to task.out_files
+
+        for key, val in self.config.items():
+            # Note - we only add the depfile to in_files _if_it_exists_, otherwise we will fail a check
+            # that all our inputs are present.
+            if key == "in_depfile":
+                if path.isfile(val):
+                    self.in_files.append(val)
+            elif key.startswith("out_"):
+                self.out_files.extend(flatten(val))
+            elif key.startswith("in_"):
+                self.in_files.extend(flatten(val))
 
         # ----------------------------------------
         # And now we can expand the command.
@@ -872,7 +918,7 @@ def join_dir(key, val):
         self.config.command = self.config.expand(self.config.command)
 
         if debug:
-            log(f"\nTask after expand: {self.config}")
+            log(f"\nTask after expand: {self}")
 
         # ----------------------------------------
         # Sanity checks
@@ -891,12 +937,10 @@ def join_dir(key, val):
         for file in self.out_files:
             if file is None:
                 raise ValueError("out_files contained a None")
-            # Raw tasks may not have a root_dir
-            if root_dir := self.config.get("root_dir", None):
-                if not file.startswith(root_dir):
-                    raise ValueError(
-                        f"Path error, output file {file} is not under root_dir {root_dir}"
-                    )
+            if not file.startswith(self.config.build_dir):
+                raise ValueError(
+                    f"Path error, output file {file} is not under build_dir {self.config.build_dir}"
+                )
 
         # Check for duplicate task outputs
         if self.config.command:
@@ -1310,7 +1354,7 @@ def parse_flags(self, argv):
     def create_root_context(self):
 
         root_file = self.flags.root_file
-        root_dir = path.realpath(self.flags.root_dir)  # Root path must be absolute.
+        root_dir = path.abspath(self.flags.root_dir)  # Root path must be absolute.
         root_path = path.join(root_dir, root_file)
 
         root_config = Config(

riscv_rules.hancho

@@ -31,7 +31,7 @@ rv_compile = hancho.Config(
     rv_config,
     command    = "{rv_toolchain}-g++ {rv_flags_c} {joined_warnings} {joined_defines} {joined_includes} -c {in_src} -o {out_obj}",
     out_obj    = "{swap_ext(in_src, '.o')}",
-    in_depfile = "{swap_ext(in_src, '.d')}",
+    in_depfile = "{swap_ext(out_obj, '.d')}",
 )
 
 rv_link = hancho.Config(

tests/run_tests.py

@@ -90,6 +90,7 @@ def setUp(self):
         #hancho_py.app.parse_flags(["--quiet"])
         hancho_py.app.parse_flags([])
         #hancho_py.app.parse_flags(["-v"])
+        #hancho_py.app.parse_flags(["-d"])
         self.hancho = hancho_py.app.create_root_context()
 
     ########################################
@@ -199,16 +200,15 @@ def test_bad_build_path(self):
         self.assertNotEqual(0, hancho_py.app.build_all())
         self.assertEqual(bad_task._state, hancho_py.TaskState.BROKEN)
         self.assertFalse(Path("build/foo.o").exists())
-        self.assertTrue("Path error" in hancho_py.app.log)
 
     ########################################
 
     def test_raw_task(self):
         self.hancho.Task(
-            command   = "touch {rel(out_obj)}",
-            in_src    = "src/foo.c",
-            out_obj   = "foo.o",
-            task_dir  = os.getcwd(),
+            command    = "touch {rel(out_obj)}",
+            in_src     = "src/foo.c",
+            out_obj    = "foo.o",
+            task_dir   = os.getcwd(),
             build_dir = "build"
         )
         #print(task)
@@ -231,6 +231,24 @@ def test_missing_input(self):
 
     ########################################
 
+    def test_absolute_inputs(self):
+        """
+        If input filenames are absolute paths, we should still end up with build files under
+        build_root.
+        """
+
+        self.hancho(
+            command = "cp {in_src} {out_obj}",
+            in_src  = path.abspath("src/foo.c"),
+            out_obj = "{swap_ext(in_src, '.o')}",
+        )
+
+        self.assertEqual(0, hancho_py.app.build_all())
+        self.assertTrue(Path("build/src/foo.o").exists())
+
+
+    ########################################
+
     def test_missing_dep(self):
         """Missing dep should fail"""
         bad_task = self.hancho(
@@ -371,7 +389,7 @@ def run():
             compile = self.hancho.Config(
                 command    = "gcc -MMD -c {rel(in_src)} -o {rel(out_obj)}",
                 out_obj    = "{swap_ext(in_src, '.o')}",
-                in_depfile = "{swap_ext(in_src, '.d')}",
+                in_depfile = "{swap_ext(out_obj, '.d')}",
             )
             self.hancho(compile, in_src = "src/test.cpp")
             self.assertEqual(0, hancho_py.app.build_all())
@@ -396,7 +414,7 @@ def run():
             compile = self.hancho.Config(
                 command    = "gcc -MMD -c {rel(in_src)} -o {rel(out_obj)}",
                 out_obj    = "{swap_ext(in_src, '.o')}",
-                in_depfile = "{swap_ext(in_src, '.d')}",
+                in_depfile = "{swap_ext(out_obj, '.d')}",
             )
             self.hancho(compile, in_src = "src/test.cpp")
             self.assertEqual(0, hancho_py.app.build_all())

tutorial/tut00.hancho

@@ -61,7 +61,7 @@ compile_cpp = hancho.Config(
     desc    = "Compile {in_src} -> {out_obj}",
     command = "g++ -MMD -c {in_src} -o {out_obj}",
     out_obj = "{swap_ext(in_src, '.o')}",
-    in_depfile  = "{swap_ext(in_src, '.d')}",
+    in_depfile  = "{swap_ext(out_obj, '.d')}",
 )
 
 link_cpp = hancho.Config(
@@ -87,7 +87,7 @@ compile_cpp = hancho.Config(
     desc    = "Compile {in_src} -> {out_obj}",
     command = "g++ -MMD -c {in_src} -o {out_obj}",
     out_obj = "{swap_ext(in_src, '.o')}",
-    in_depfile  = "{swap_ext(in_src, '.d')}",
+    in_depfile  = "{swap_ext(out_obj, '.d')}",
 )
 
 link_cpp = hancho.Config(
@@ -109,7 +109,7 @@ compile_cpp = hancho.Config(
     desc    = "Compile {rel(in_src)} -> {rel(out_obj)}",
     command = "g++ -MMD -c {rel(in_src)} -o {rel(out_obj)}",
     out_obj = "{swap_ext(in_src, '.o')}",
-    in_depfile  = "{swap_ext(in_src, '.d')}",
+    in_depfile  = "{swap_ext(out_obj, '.d')}",
 )
 
 link_cpp = hancho.Config(