diff --git a/include-files/Makefile b/include-files/Makefile index 0e060f86..bd9f8315 100644 --- a/include-files/Makefile +++ b/include-files/Makefile @@ -1,15 +1,20 @@ DIFF ?= diff --strip-trailing-cr -u +.EXPORT_ALL_VARIABLES: + +SUBDIR_NAME=subdir + + test: sample.md file-a.md file-b.md file-c.md include-files.lua - @pandoc --lua-filter=include-files.lua --to=native $< \ + @pandoc --lua-filter=include-files.lua -M subdir-name="subdir" --to=native $< \ | $(DIFF) expected.native - - @pandoc --lua-filter=include-files.lua -M include-auto --to=native $< \ + @pandoc --lua-filter=include-files.lua -M subdir-name="subdir" -M include-auto --to=native $< \ | $(DIFF) expected-auto.native - expected.native: sample.md file-a.md file-b.md file-c.md include-files.lua - pandoc --lua-filter=include-files.lua --output $@ $< + pandoc --lua-filter=include-files.lua -M subdir-name="subdir" --output $@ $< expected-auto.native: sample.md file-a.md file-b.md file-c.md include-files.lua - pandoc --lua-filter=include-files.lua -M include-auto --output $@ $< + pandoc --lua-filter=include-files.lua -M subdir-name="subdir" -M include-auto --output $@ $< .PHONY: test diff --git a/include-files/README.md b/include-files/README.md index 38664b8a..47696beb 100644 --- a/include-files/README.md +++ b/include-files/README.md @@ -18,12 +18,12 @@ be convenient to modify the level of headers; a top-level header in an included file should be a second or third-level header in the final document. -#### Manual shifting +#### Manual Shifting Use the `shift-heading-level-by` attribute to control header shifting. -#### Automatic shifting +#### Automatic Shifting 1. Add metadata `-M include-auto` to enable automatic shifting. 2. Do not specify `shift-heading-level-by` @@ -52,14 +52,56 @@ file-a.md Comment lines can be added in the include block by beginning a line with two `//` characters. -### Different formats +### Different Formats Files are assumed to be written in Markdown, but sometimes one will want to include files written in a different format. An alternative format can be specified via the `format` attribute. -Only plain-text formats are accepted. +Only plain-text formats are accepted. The default format for all includes +can be set by the meta data variable `include-format`, which is useful +if you want all your files to be parsed in the same way as your main document, +e.g. with extensions. -### Recursive transclusion +### Filter by Formats + +Files can be included and excluded on different formats by +using attribute `include-if-format=formatA;formatB;...` +and `exclude-if-format=formatA;formatB;...`, e.g. + +````md +```{.include include-if-format=native;html;latex} +subdir/file-h.md +``` + +```{.include exclude-if-format=native;commonmark} +subdir/file-i.md +``` +```` + +### Variable Substitution in Paths + +If attribute `var-replace` is used, the patterns `${meta:}` or `${env:}` +will be replaced by the corresponding meta data variable `` in the document or the +environment variable ``, e.g. + +````md +```{.include .var-replace} +${meta:subdir-name}/file-h.md +${env:SUBDIR_NAME}/file-h.md +``` +```` + +### Raw Includes + +You can also include the files in a raw block by using `raw=true`, e.g. + +````md +```{.include raw=true format=latex include-if-format=latex} +subdir/file-h-latex.md +``` +```` + +### Recursive Transclusion Included files can in turn include other files. Note that all filenames must be relative to the directory from which they are @@ -70,6 +112,12 @@ this case `b/c.md`. The full relative path will be automatically generated in the final document. The same goes for image paths and codeblock file paths using the `include-code-files` filter. +### Missing Includes + +You can set the meta data variable `include-fail-if-read-error` to `true` +such that any not found include will fail the convertion and error out +immediateley. + ## Example Let's assume we are writing a longer document, like a thesis. diff --git a/include-files/expected-auto.native b/include-files/expected-auto.native index 4decb2f7..a61761a1 100644 --- a/include-files/expected-auto.native +++ b/include-files/expected-auto.native @@ -23,6 +23,15 @@ ,Header 2 ("source-include",[],[]) [Str "Source",Space,Str "include"] ,Para [Str "File",Space,Str "inclusion",Space,Str "codeblocks",Space,Str "for",Space,Str "use",Space,Str "with",Space,Str "include-code-files",Space,Str "will",Space,Str "be",SoftBreak,Str "updated",Space,Str "too."] ,CodeBlock ("",["c"],[("include","subdir/somecode.c")]) "" +,Header 1 ("includeexclude-if-format",[],[]) [Str "Include/exclude",Space,Str "if",Space,Str "format"] +,Para [Str "The",Space,Str "next",Space,Str "document",Space,Str "should",Space,Str "be",Space,Str "included",Space,Str "in",Space,Str "formats",Space,Code ("",[],[]) "native, html, latex",Str "."] +,Header 2 ("some-other-stuff",[],[]) [Str "Some",Space,Str "other",Space,Str "stuff"] +,Para [Str "The",Space,Str "next",Space,Str "document",Space,Str "should",Space,Str "not",Space,Str "be",Space,Str "included",Space,Str "in",Space,Str "formats",Space,Code ("",[],[]) "native, commonmark",Str "."] +,Header 1 ("meta-and-env-var-replacement",[],[]) [Str "Meta",Space,Str "and",Space,Str "env",Space,Str "var",Space,Str "replacement"] +,Header 2 ("some-other-stuff",[],[]) [Str "Some",Space,Str "other",Space,Str "stuff"] +,Header 2 ("some-other-stuff",[],[]) [Str "Some",Space,Str "other",Space,Str "stuff"] +,Header 1 ("raw-include",[],[]) [Str "Raw",Space,Str "include"] +,RawBlock (Format "latex") "\\section{Some other stuff}\n" ,Header 1 ("appendix",[],[]) [Str "Appendix"] ,Para [Str "More",Space,Str "info",Space,Str "goes",Space,Str "here."] ,Header 2 ("questionaire",[],[]) [Str "Questionaire"] diff --git a/include-files/expected.native b/include-files/expected.native index 46a02883..94cd164b 100644 --- a/include-files/expected.native +++ b/include-files/expected.native @@ -23,6 +23,15 @@ ,Header 1 ("source-include",[],[]) [Str "Source",Space,Str "include"] ,Para [Str "File",Space,Str "inclusion",Space,Str "codeblocks",Space,Str "for",Space,Str "use",Space,Str "with",Space,Str "include-code-files",Space,Str "will",Space,Str "be",SoftBreak,Str "updated",Space,Str "too."] ,CodeBlock ("",["c"],[("include","subdir/somecode.c")]) "" +,Header 1 ("includeexclude-if-format",[],[]) [Str "Include/exclude",Space,Str "if",Space,Str "format"] +,Para [Str "The",Space,Str "next",Space,Str "document",Space,Str "should",Space,Str "be",Space,Str "included",Space,Str "in",Space,Str "formats",Space,Code ("",[],[]) "native, html, latex",Str "."] +,Header 1 ("some-other-stuff",[],[]) [Str "Some",Space,Str "other",Space,Str "stuff"] +,Para [Str "The",Space,Str "next",Space,Str "document",Space,Str "should",Space,Str "not",Space,Str "be",Space,Str "included",Space,Str "in",Space,Str "formats",Space,Code ("",[],[]) "native, commonmark",Str "."] +,Header 1 ("meta-and-env-var-replacement",[],[]) [Str "Meta",Space,Str "and",Space,Str "env",Space,Str "var",Space,Str "replacement"] +,Header 1 ("some-other-stuff",[],[]) [Str "Some",Space,Str "other",Space,Str "stuff"] +,Header 1 ("some-other-stuff",[],[]) [Str "Some",Space,Str "other",Space,Str "stuff"] +,Header 1 ("raw-include",[],[]) [Str "Raw",Space,Str "include"] +,RawBlock (Format "latex") "\\section{Some other stuff}\n" ,Header 1 ("appendix",[],[]) [Str "Appendix"] ,Para [Str "More",Space,Str "info",Space,Str "goes",Space,Str "here."] ,Header 2 ("questionaire",[],[]) [Str "Questionaire"] diff --git a/include-files/include-files.lua b/include-files/include-files.lua index b9338369..f4074417 100644 --- a/include-files/include-files.lua +++ b/include-files/include-files.lua @@ -9,15 +9,99 @@ PANDOC_VERSION:must_be_at_least '2.12' local List = require 'pandoc.List' local path = require 'pandoc.path' local system = require 'pandoc.system' +local cs = PANDOC_STATE ---- Get include auto mode +-- This is the codeblock-var-replace +-- filter directly copied, since we +-- cannot run Lua filters inside this filter +-- https://github.com/jgm/pandoc/issues/6830 +-- We replace variables in include blocks. + +local sys = require 'pandoc.system' +local utils = require 'pandoc.utils' +-- local ut = require "module-lua.utils" + +-- Save env. variables +local env = sys.environment() + +-- Save meta table and metadata +local meta +function save_meta (m) + meta = m +end + +--- Replace variables in code blocks +local metaMap +local function var_replace_codeblocks (cb) + --- Replace variable with values from environment + --- and meta data (stringifing). + local function replace(what, var) + local repl = nil + if what == "env" then + repl = env[var] + elseif what == "meta" then + local v = metaMap[var] + if v then + repl = utils.stringify(v) + end + end + + if repl == nil then + io.stderr:write("Could not replace variable in codeblock: '".. var .."'\n") + end + + return repl + end + + -- ignore code blocks which are not of class "var-replace". + if not cb.classes:includes 'var-replace' then + return + end + + cb.text = cb.text:gsub("%${(%l+):([^}]+)}", replace) +end + +--- Include/exclude by attribute +--- `exclude-if-format='formatA;formatB;...' +--- `include-if-format='formatA;formatB;...` +--- Default: true +local function is_included(cb) + local include = true + local exclude = false + + if cb.attributes['include-if-format'] then + include = cb.attributes['include-if-format']:match(FORMAT) ~= nil + end + + if cb.attributes['exclude-if-format'] then + exclude = cb.attributes['exclude-if-format']:match(FORMAT) ~= nil + end + + return include == true and exclude == false +end + +--- Get default settings local include_auto = false +local default_format = nil +local include_fail_if_read_error = false + function get_vars (meta) if meta['include-auto'] then include_auto = true end + + if meta['include-fail-if-read-error'] then + include_fail_if_read_error = true + end + + -- If this is nil, markdown is used as a default format. + default_format = meta['include-format'] + + -- Save meta table for var_replace + metaMap = meta end + --- Keep last heading level found local last_heading_level = 0 function update_last_level(header) @@ -62,8 +146,23 @@ function transclude (cb) return end - -- Markdown is used if this is nil. + -- Filter by includes and excludes + if not is_included(cb) then + return List{} -- remove block + end + + -- Variable substitution + var_replace_codeblocks(cb) + local format = cb.attributes['format'] + if not format then + -- Markdown is used if this is nil. + format = default_format + end + + -- Check if we include the file as raw inline + local raw = cb.attributes['raw'] + raw = raw == "true" -- Attributes shift headings local shift_heading_level_by = 0 @@ -77,35 +176,63 @@ function transclude (cb) end end - --- keep track of level before recusion + --- Keep track of level before recursion local buffer_last_heading_level = last_heading_level local blocks = List:new() for line in cb.text:gmatch('[^\n]+') do - if line:sub(1,2) ~= '//' then - local fh = io.open(line) - if not fh then - io.stderr:write("Cannot open file " .. line .. " | Skipping includes\n") + if line:sub(1,2) == '//' then + goto skip_to_next + end + + if cs.verbosity == "INFO" then + io.stderr:write(string.format("Including: [format: %s, raw: %s]\n - '%s'\n", + format, + tostring(raw), line)) + end + + local fh = io.open(line) + if not fh then + local cwd = system.get_working_directory() + local msg = "Cannot find include file: '" .. line .. "' in working dir: '" .. cwd .. "'" + if include_fail_if_read_error then + io.stderr:write(msg .. " | error\n") + error("Abort due to include failure") else - local contents = pandoc.read(fh:read '*a', format).blocks - last_heading_level = 0 - -- recursive transclusion - contents = system.with_working_directory( - path.directory(line), - function () - return pandoc.walk_block( - pandoc.Div(contents), - { Header = update_last_level, CodeBlock = transclude } - ) - end).content - --- reset to level before recursion - last_heading_level = buffer_last_heading_level - blocks:extend(update_contents(contents, shift_heading_level_by, - path.directory(line))) - fh:close() + io.stderr:write(msg .. " | skipping include\n") + goto skip_to_next end end + + -- Read the file + local text = fh:read('*a') + fh:close() + + if raw then + -- Include as raw inline element + blocks:extend({pandoc.RawBlock(format, text)}) + else + -- Inlcude as parsed AST + local contents = pandoc.read(text, format).blocks + last_heading_level = 0 + -- Recursive transclusion + contents = system.with_working_directory( + path.directory(line), + function () + return pandoc.walk_block( + pandoc.Div(contents), + { Header = update_last_level, CodeBlock = transclude } + ) + end).content + --- Reset to level before recursion + last_heading_level = buffer_last_heading_level + blocks:extend(update_contents(contents, shift_heading_level_by, + path.directory(line))) + end + + ::skip_to_next:: end + return blocks end diff --git a/include-files/sample.md b/include-files/sample.md index c2831709..32550747 100644 --- a/include-files/sample.md +++ b/include-files/sample.md @@ -37,6 +37,38 @@ file-f.md subdir/file-g.md ``` +# Include/exclude if format + +The next document should be included in formats `native, html, latex`. + +```{.include include-if-format=native;html;latex} +subdir/file-h.md +``` + +The next document should not be included in formats `native, commonmark`. + +```{.include exclude-if-format=native;commonmark} +subdir/file-i.md +``` + +# Meta and env var replacement + +```{.include .var-replace} +// Replace meta data variable in path +${meta:subdir-name}/file-h.md +``` + +```{.include .var-replace} +// Replace envrionment variable in path +${env:SUBDIR_NAME}/file-h.md +``` + +# Raw include + +```{.include raw=true format=latex} +subdir/file-h-latex.md +``` + # Appendix More info goes here. diff --git a/include-files/subdir/file-h-latex.md b/include-files/subdir/file-h-latex.md new file mode 100644 index 00000000..390af167 --- /dev/null +++ b/include-files/subdir/file-h-latex.md @@ -0,0 +1 @@ +\section{Some other stuff} diff --git a/include-files/subdir/file-h.md b/include-files/subdir/file-h.md new file mode 100644 index 00000000..470e03d5 --- /dev/null +++ b/include-files/subdir/file-h.md @@ -0,0 +1 @@ +# Some other stuff