perf: Use non-lazy AST traversal for filters when possible

.github/workflows/test-smokes.yml

@@ -53,6 +53,8 @@ jobs:
           - os: windows-latest
             time-test: true
     runs-on: ${{ matrix.os }}
+    env:
+      QUARTO_JOG_CHECK: true
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v4

src/resources/filters/ast/customnodes.lua

@@ -41,7 +41,7 @@ function is_regular_node(node, name)
   return node
 end
 
-function run_emulated_filter(doc, filter)
+function run_emulated_filter(doc, filter, force_use_walk)
   if doc == nil then
     return nil
   end
@@ -73,7 +73,10 @@ function run_emulated_filter(doc, filter)
         -- luacov: enable
       end
     end
-    return node:walk(filter_param)
+    _QUARTO_USE_WALK = force_use_walk
+    local result = _quarto.modules.jog(node, filter_param)
+    _QUARTO_USE_WALK = false
+    return result
   end
 
   -- performance: if filter is empty, do nothing
@@ -240,6 +243,11 @@ function create_custom_node_scaffold(t, context)
   return result
 end
 
+
+local ptype  = pandoc.utils.type
+local InlinesMT = getmetatable(pandoc.Inlines{})
+local BlocksMT = getmetatable(pandoc.Blocks{})
+
 function create_emulated_node(t, tbl, context, forwarder)
   local result = create_custom_node_scaffold(t, context)
   tbl.t = t -- set t always to custom ast type
@@ -337,21 +345,27 @@ _quarto.ast = {
         end
         local node = node_accessor(table)
         local t = pandoc.utils.type(value)
-        -- FIXME this is broken; that can only be "Block", "Inline", etc
-        if t == "Div" or t == "Span" then
-          local custom_data, t, kind = _quarto.ast.resolve_custom_data(value)
-          if custom_data ~= nil then
-            value = custom_data
-          end
-        end
+        quarto_assert(t ~= 'Div' and t ~= 'Span', "")
         if index > #node.content then
           _quarto.ast.grow_scaffold(node, index)
         end
-        local pt = pandoc.utils.type(value)
-        if pt == "Block" or pt == "Inline" then
-          node.content[index].content = {value}
+        local inner_node = node.content[index]
+        quarto_assert(inner_node ~= nil)
+        if t == "Block" then
+          inner_node.content = quarto.utils.as_blocks(value)
+        elseif t == "Inline" then
+          inner_node.content = quarto.utils.as_inlines(value)
         else
-          node.content[index].content = value
+          local innertype = pandoc.utils.type(inner_node)
+          if innertype == 'Block' then
+            inner_node.content = quarto.utils.as_blocks(value)
+          elseif innertype == 'Inline' then
+            inner_node.content = quarto.utils.as_inlines(value)
+          else
+            warn('Cannot find the right content type for value ' .. t)
+            warn(debug.traceback())
+            inner_node.content = value
+          end
         end
       end
     }
@@ -422,13 +436,15 @@ _quarto.ast = {
       -- luacov: enable
     end
 
-    local forwarder = { }
+    local forwarder
     if tisarray(handler.slots) then
+      forwarder = pandoc.List{}
       for i, slot in ipairs(handler.slots) do
         forwarder[slot] = i
       end
-    else
-      forwarder = handler.slots
+    elseif handler.slots ~= nil then
+      warn('Expected `slots` to be either an array or nil, got ' ..
+           tostring(handler.slots))
     end
 
     quarto[handler.ast_name] = function(params)

src/resources/filters/ast/emulatedfilter.lua

@@ -68,6 +68,9 @@ inject_user_filters_at_entry_points = function(filter_list)
     end
     local filter = {
       name = entry_point .. "-user-" .. tostring(entry_point_counts[entry_point]),
+      -- The filter might not work as expected when doing a non-lazy jog, so
+      -- make sure it is processed with the default 'walk' function.
+      force_pandoc_walk = true,
     }
     if is_many_filters then
       filter.filters = wrapped

src/resources/filters/ast/runemulation.lua

@@ -47,8 +47,97 @@ local function remove_vault(doc)
   end
 end
 
+--- Create a deep copy of a table.
+local function copy_table (tbl, depth, seen)
+  local tp = type(tbl)
+  if tp == 'table' then
+    local copy = {}
+    -- Iterate 'raw' pairs, i.e., without using metamethods
+    for key, value in next, tbl, nil do
+      if depth == 'shallow' then
+        copy[key] = value
+      else
+        copy[copy_table(key)] = copy_table(value)
+      end
+    end
+    return setmetatable(copy, getmetatable(tbl))
+  elseif tp == 'userdata' then
+    return tbl:clone()
+  else -- number, string, boolean, etc
+    return tbl
+  end
+end
+
+--- Checks if two tables are equal
+function equals(o1, o2)
+  if o1 == o2 then
+    return true
+  end
+  local o1type = type(o1)
+  local o2type = type(o2)
+  if o1type ~= o2type or o1type ~= 'table' then
+    return false
+  end
+
+  local keys = {}
+
+  for key1, value1 in pairs(o1) do
+    local value2 = o2[key1]
+    if value2 == nil or equals(value1, value2) == false then
+      return false
+    end
+    keys[key1] = true
+  end
+
+  for key2 in pairs(o2) do
+    if not keys[key2] then return false end
+  end
+  return true
+end
+
+--- Checks if a filter follows the "nondestructive" property.
+-- The nondestructive property is fulfilled if filter functions returns
+-- an explicit object, or if it returns `nil` while leaving the passed
+-- in object unmodified.
+--
+-- An error is raised if the property is violated.
+--
+-- Only filters with this property can use jog safely, without
+-- unintended consequences.
+local function check_nondestructive_property (namedfilter)
+  for name, fn in pairs(namedfilter.filter) do
+    if type(fn) == 'function' then
+      local copy = function (x)
+        local tp = type(x)
+        return tp ~= 'table' and x:clone() or
+          (pandoc.utils.type(x) == 'Meta' and pandoc.Meta(x) or copy_table(x))
+      end
+      namedfilter.filter[name] = function (obj, context)
+        local orig = copy(obj)
+        local result, descend = fn(obj, context)
+        if result == nil then
+          if type(obj) ~= 'table' and not equals(obj, orig) then
+            warn(
+              "\nFunction '" .. name .. "' in filter '" .. namedfilter.name ..
+              "' returned `nil`, but modified the input."
+            )
+          end
+        -- elseif result.t == obj.t and not rawequal(result, obj) then
+        --   warn(
+        --     "\nFunction '" .. name .. "' in filter '" .. namedfilter.name ..
+        --     "' returned a new object instead of passing the original one through."
+        --   )
+        end
+        return result, descend
+      end
+    end
+  end
+  return namedfilter
+end
+
 local function run_emulated_filter_chain(doc, filters, afterFilterPass, profiling)
   init_trace(doc)
+  local compare_jog_and_walk = os.getenv 'QUARTO_JOG_CHECK'
   for i, v in ipairs(filters) do
     local function callback()
       if v.flags then
@@ -79,7 +168,18 @@ local function run_emulated_filter_chain(doc, filters, afterFilterPass, profilin
         print(pandoc.write(doc, "native"))
       else
         _quarto.ast._current_doc = doc
-        doc = run_emulated_filter(doc, v.filter)
+
+        if compare_jog_and_walk and not v.force_pandoc_walk then
+          v = check_nondestructive_property(v)
+          io.stderr:write('Running: ' .. v.name .. '\n')
+        end
+        doc = run_emulated_filter(doc, v.filter, v.force_pandoc_walk)
+
+        if compare_jog_and_walk and not v.force_pandoc_walk then
+          -- Types of meta values are only check on assignment.
+          doc.meta = doc.meta
+        end
+
         ensure_vault(doc)
 
         add_trace(doc, v.name)
@@ -204,4 +304,4 @@ function run_as_extended_ast(specTable)
   end
 
   return pandocFilterList
-end
+end

src/resources/filters/common/error.lua

@@ -15,8 +15,15 @@ function fail(message, level)
   end
 end
 
-function internal_error()
-  fail("This is an internal error. Please file a bug report at https://github.com/quarto-dev/quarto-cli/", 5)
+function internal_error(msg, level)
+  fail((msg and (msg .. '\n') or '') ..
+    "This is an internal error. Please file a bug report at https://github.com/quarto-dev/quarto-cli/", level or 5)
+end
+
+function quarto_assert (test, msg, level)
+  if not test then
+    internal_error(msg, level or 6)
+  end
 end
 
 function currentFile() 

src/resources/filters/common/layout.lua

@@ -56,7 +56,7 @@ end
 -- we often wrap a table in a div, unwrap it
 function tableFromLayoutCell(cell)
   local tbl
-  cell:walk({
+  _quarto.modules.jog(cell, {
     Table = function(t)
       tbl = t
     end

src/resources/filters/common/log.lua

@@ -20,13 +20,14 @@ function warn(message, offset)
   io.stderr:write(lunacolors.yellow("WARNING (" .. caller_info(offset) .. ") " .. message .. "\n"))
 end
 
+local orig_error = error
 function error(message, offset)
   io.stderr:write(lunacolors.red("ERROR (" .. caller_info(offset) .. ") " .. message .. "\n"))
 end
 
 function fatal(message, offset)
   io.stderr:write(lunacolors.red("FATAL (" .. caller_info(offset) .. ") " ..message .. "\n"))
   -- TODO write stack trace into log, and then exit.
-  crash_with_stack_trace()  
+  orig_error('FATAL QUARTO ERROR', offset)
 end
--- luacov: enable
+-- luacov: enable

src/resources/filters/common/pandoc.lua

@@ -217,14 +217,14 @@ function string_to_quarto_ast_blocks(text, opts)
 
   -- run the whole normalization pipeline here to get extended AST nodes, etc.
   for _, filter in ipairs(quarto_ast_pipeline()) do
-    doc = doc:walk(filter.filter)
+    doc = _quarto.modules.jog(doc, filter.filter)
   end
 
   -- compute flags so we don't skip filters that depend on them
-  doc:walk(compute_flags())
+  _quarto.modules.jog(doc, compute_flags())
   return doc.blocks
 end
 
 function string_to_quarto_ast_inlines(text, sep)
   return pandoc.utils.blocks_to_inlines(string_to_quarto_ast_blocks(text), sep)
-end
+end

src/resources/filters/common/wrapped-filter.lua

@@ -97,7 +97,7 @@ function makeWrappedJsonFilter(scriptFile, filterHandler)
         path = quarto.utils.resolve_path_relative_to_document(scriptFile)
         local custom_node_map = {}
         local has_custom_nodes = false
-        doc = doc:walk({
+        doc = _quarto.modules.jog(doc, {
           -- FIXME: This is broken with new AST. Needs to go through Custom node instead.
           RawInline = function(raw)
             local custom_node, t, kind = _quarto.ast.resolve_custom_data(raw)
@@ -130,7 +130,7 @@ function makeWrappedJsonFilter(scriptFile, filterHandler)
           return nil
         end
         if has_custom_nodes then
-          doc:walk({
+          _quarto.modules.jog(doc, {
             Meta = function(meta)
               _quarto.ast.reset_custom_tbl(meta["quarto-custom-nodes"])
             end

src/resources/filters/crossref/equations.lua

@@ -20,7 +20,7 @@ function process_equations(blockEl)
   end
 
   local mathInlines = nil
-  local targetInlines = pandoc.List()
+  local targetInlines = pandoc.Inlines{}
 
   for i, el in ipairs(inlines) do
 

src/resources/filters/crossref/index.lua

@@ -65,7 +65,7 @@ end
 -- add an entry to the index
 function indexAddEntry(label, parent, order, caption, appendix)
   if caption ~= nil then
-    caption = pandoc.List(caption)
+    caption = pandoc.List{caption}
   else
     caption = pandoc.List({})
   end

src/resources/filters/crossref/preprocess.lua

@@ -7,7 +7,7 @@ function crossref_mark_subfloats()
   return {
     traverse = "topdown",
     FloatRefTarget = function(float)
-      float.content = _quarto.ast.walk(float.content, {
+      float.content = _quarto.ast.walk(float.content or pandoc.Blocks{}, {
         FloatRefTarget = function(subfloat)
           float.has_subfloats = true
           crossref.subfloats[subfloat.identifier] = {

src/resources/filters/customnodes/callout.lua

@@ -260,11 +260,18 @@ function _callout_main()
         icon_color = "brand-color." .. callout_theme_color_map[callout.type]
       end
     end
+    local function is_nonempty_node (node)
+      return not not
+        (node or
+         (node.content and next(node.content)) or
+         next(node))
+    end
     if callout.attr.identifier == "" then
       return _quarto.format.typst.function_call("callout", { 
         { "body", _quarto.format.typst.as_typst_content(callout.content) },
         { "title", _quarto.format.typst.as_typst_content(
-          callout.title or pandoc.Plain(_quarto.modules.callouts.displayName(callout.type))
+          (not is_nonempty_node(callout.title) and callout.title) or
+          pandoc.Plain(_quarto.modules.callouts.displayName(callout.type))
         )},
         { "background_color", pandoc.RawInline("typst", background_color) },
         { "icon_color", pandoc.RawInline("typst", icon_color) },

src/resources/filters/customnodes/content-hidden.lua

@@ -105,7 +105,8 @@ local _content_hidden_meta = nil
 function content_hidden_meta(meta)
   -- return {
   --   Meta = function(meta)
-  _content_hidden_meta = meta
+  -- The call to `pandoc.Meta` ensures that we hold a copy.
+  _content_hidden_meta = pandoc.Meta(meta)
   --   end
   -- }
 end

src/resources/filters/customnodes/floatreftarget.lua

@@ -767,7 +767,7 @@ function float_reftarget_render_html_figure(float)
   local float_content = pandoc.Div(_quarto.ast.walk(float.content, {
     -- strip image captions
     Image = function(image)
-      image.caption = {}
+      image.caption = pandoc.Inlines{}
       return image
     end
   }) or pandoc.Div({})) -- this should never happen but the lua analyzer doesn't know it
@@ -1098,4 +1098,4 @@ end, function(float)
   return pandoc.Para({im})
 end)
 
-global_table_guid_id = 0
+global_table_guid_id = 0