Improve include file filter

include-files/Makefile

@@ -1,15 +1,20 @@
 DIFF ?= diff --strip-trailing-cr -u
 
+.EXPORT_ALL_VARIABLES:
+
+SUBDIR_NAME=subdir
+
+
 test: sample.md file-a.md file-b.md file-c.md include-files.lua
-	@pandoc --lua-filter=include-files.lua --to=native $< \
+	@pandoc --lua-filter=include-files.lua -M subdir-name="subdir" --to=native $< \
 	    | $(DIFF) expected.native -
-	@pandoc --lua-filter=include-files.lua -M include-auto --to=native $< \
+	@pandoc --lua-filter=include-files.lua -M subdir-name="subdir" -M include-auto --to=native $< \
 	    | $(DIFF) expected-auto.native -
 
 expected.native: sample.md file-a.md file-b.md file-c.md include-files.lua
-	pandoc --lua-filter=include-files.lua --output $@ $<
+	pandoc --lua-filter=include-files.lua -M subdir-name="subdir" --output $@ $<
 
 expected-auto.native: sample.md file-a.md file-b.md file-c.md include-files.lua
-	pandoc --lua-filter=include-files.lua -M include-auto --output $@ $<
+	pandoc --lua-filter=include-files.lua -M subdir-name="subdir" -M include-auto --output $@ $<
 
 .PHONY: test

include-files/README.md

@@ -18,12 +18,12 @@ be convenient to modify the level of headers; a top-level header
 in an included file should be a second or third-level header in
 the final document.
 
-#### Manual shifting
+#### Manual Shifting
 
 Use the `shift-heading-level-by` attribute to control header
 shifting.
 
-#### Automatic shifting
+#### Automatic Shifting
 
 1. Add metadata `-M include-auto` to enable automatic shifting.
 2. Do not specify `shift-heading-level-by`
@@ -52,24 +52,81 @@ file-a.md
 Comment lines can be added in the include block by beginning a
 line with two `//` characters.
 
-### Different formats
+### Different Formats
 
 Files are assumed to be written in Markdown, but sometimes one
 will want to include files written in a different format. An
 alternative format can be specified via the `format` attribute.
-Only plain-text formats are accepted.
+Only plain-text formats are accepted. The default format for all includes
+can be set by the meta data variable `include-format`, which is useful
+if you want all your files to be parsed in the same way as your main document,
+e.g. with extensions.
 
-### Recursive transclusion
+### Filter by Formats
+
+Files can be included and excluded on different formats by
+using attribute `include-if-format=formatA;formatB;...`
+and `exclude-if-format=formatA;formatB;...`, e.g.
+
+````md
+```{.include include-if-format=native;html;latex}
+subdir/file-h.md
+```
+
+```{.include exclude-if-format=native;commonmark}
+subdir/file-i.md
+```
+````
+
+### Variable Substitution in Paths
+
+The patterns `${meta:<meta-var>}` or `${env:<env-var>}` inside include paths
+will be replaced by the corresponding meta data variable `<meta-var>` or the
+environment variable `<env-var>`, e.g.
+
+````md
+```{.include}
+${meta:subdir-name}/file-h.md
+${env:SUBDIR_NAME}/file-h.md
+```
+````
+
+### Raw Includes
+
+You can also include the files in a raw block by using `raw=true`, e.g.
+
+````md
+```{.include raw=true format=latex include-if-format=latex}
+subdir/file-h-latex.md
+```
+````
+
+### Recursive Transclusion
 
 Included files can in turn include other files. Note that all
 filenames must be relative to the directory from which they are
-included. I.e., if a file `a/b.md` is included in the main
+included.
+I.e., if a file `a/b.md` is included in the main
 document, and another file `a/b/c.md` should be included from
 `a/b.md`, then the relative path from `a/b.md` must be used, in
 this case `b/c.md`. The full relative path will be automatically
 generated in the final document. The same goes for image paths and
 codeblock file paths using the `include-code-files` filter.
 
+If you set the meta variable `include-paths-relative-to-cwd` to `true`,
+all include paths are treated relative to pandoc's working directory.
+You can selectively set the attribute `relative-to-current` on all pandoc
+elements modiefied by this filter (e.g. image, codeblock file paths,
+codeblock transclusion) which will treat the
+paths relative to the directory from which they are included.
+
+### Missing Includes
+
+You can set the meta data variable `include-fail-if-read-error` to `true`
+such that any not found include or read failure will fail
+the convertion immediateley and error out.
+Otherwise the missing includes are skipped by default.
+
 ## Example
 
 Let's assume we are writing a longer document, like a thesis.
@@ -79,6 +136,11 @@ some additional information in the main file `main.md`:
     ---
     author: me
     title: Thesis
+
+    include-format: markdown+markdown_in_html_blocks+link_attributes+tex_math_dollars+fenced_divs+bracketed_spans
+    include-auto: true
+    include-fail-if-read-error: true
+    include-paths-relative-to-cwd: false
     ---
 
     # Frontmatter

include-files/expected-auto.native

@@ -23,6 +23,15 @@
 ,Header 2 ("source-include",[],[]) [Str "Source",Space,Str "include"]
 ,Para [Str "File",Space,Str "inclusion",Space,Str "codeblocks",Space,Str "for",Space,Str "use",Space,Str "with",Space,Str "include-code-files",Space,Str "will",Space,Str "be",SoftBreak,Str "updated",Space,Str "too."]
 ,CodeBlock ("",["c"],[("include","subdir/somecode.c")]) ""
+,Header 1 ("includeexclude-if-format",[],[]) [Str "Include/exclude",Space,Str "if",Space,Str "format"]
+,Para [Str "The",Space,Str "next",Space,Str "document",Space,Str "should",Space,Str "be",Space,Str "included",Space,Str "in",Space,Str "formats",Space,Code ("",[],[]) "native, html, latex",Str "."]
+,Header 2 ("some-other-stuff",[],[]) [Str "Some",Space,Str "other",Space,Str "stuff"]
+,Para [Str "The",Space,Str "next",Space,Str "document",Space,Str "should",Space,Str "not",Space,Str "be",Space,Str "included",Space,Str "in",Space,Str "formats",Space,Code ("",[],[]) "native, commonmark",Str "."]
+,Header 1 ("meta-and-env-var-replacement",[],[]) [Str "Meta",Space,Str "and",Space,Str "env",Space,Str "var",Space,Str "replacement"]
+,Header 2 ("some-other-stuff",[],[]) [Str "Some",Space,Str "other",Space,Str "stuff"]
+,Header 2 ("some-other-stuff",[],[]) [Str "Some",Space,Str "other",Space,Str "stuff"]
+,Header 1 ("raw-include",[],[]) [Str "Raw",Space,Str "include"]
+,RawBlock (Format "latex") "\\section{Some other stuff}\n"
 ,Header 1 ("appendix",[],[]) [Str "Appendix"]
 ,Para [Str "More",Space,Str "info",Space,Str "goes",Space,Str "here."]
 ,Header 2 ("questionaire",[],[]) [Str "Questionaire"]

include-files/expected.native

@@ -23,6 +23,15 @@
 ,Header 1 ("source-include",[],[]) [Str "Source",Space,Str "include"]
 ,Para [Str "File",Space,Str "inclusion",Space,Str "codeblocks",Space,Str "for",Space,Str "use",Space,Str "with",Space,Str "include-code-files",Space,Str "will",Space,Str "be",SoftBreak,Str "updated",Space,Str "too."]
 ,CodeBlock ("",["c"],[("include","subdir/somecode.c")]) ""
+,Header 1 ("includeexclude-if-format",[],[]) [Str "Include/exclude",Space,Str "if",Space,Str "format"]
+,Para [Str "The",Space,Str "next",Space,Str "document",Space,Str "should",Space,Str "be",Space,Str "included",Space,Str "in",Space,Str "formats",Space,Code ("",[],[]) "native, html, latex",Str "."]
+,Header 1 ("some-other-stuff",[],[]) [Str "Some",Space,Str "other",Space,Str "stuff"]
+,Para [Str "The",Space,Str "next",Space,Str "document",Space,Str "should",Space,Str "not",Space,Str "be",Space,Str "included",Space,Str "in",Space,Str "formats",Space,Code ("",[],[]) "native, commonmark",Str "."]
+,Header 1 ("meta-and-env-var-replacement",[],[]) [Str "Meta",Space,Str "and",Space,Str "env",Space,Str "var",Space,Str "replacement"]
+,Header 1 ("some-other-stuff",[],[]) [Str "Some",Space,Str "other",Space,Str "stuff"]
+,Header 1 ("some-other-stuff",[],[]) [Str "Some",Space,Str "other",Space,Str "stuff"]
+,Header 1 ("raw-include",[],[]) [Str "Raw",Space,Str "include"]
+,RawBlock (Format "latex") "\\section{Some other stuff}\n"
 ,Header 1 ("appendix",[],[]) [Str "Appendix"]
 ,Para [Str "More",Space,Str "info",Space,Str "goes",Space,Str "here."]
 ,Header 2 ("questionaire",[],[]) [Str "Questionaire"]