Add watch and update capability for hot reloading to Builder

lib/opal/builder.rb

@@ -7,7 +7,9 @@
 require 'opal/builder/scheduler'
 require 'opal/project'
 require 'opal/builder/directory'
+require 'opal/builder/watcher'
 require 'set'
+# opal/builder/processor required at the bottom
 
 module Opal
   class Builder
@@ -64,6 +66,10 @@ class MissingRequire < LoadError
     class ProcessorNotFound < LoadError
     end
 
+    include Project::Collection
+    include Builder::Directory
+    include Builder::Watcher
+
     def initialize(options = nil)
       (options || {}).each_pair do |k, v|
         public_send("#{k}=", v)
@@ -101,6 +107,7 @@ def source_for(path)
 
     def build_str(source, rel_path, options = {})
       return if source.nil?
+      @build_time = Time.now
       abs_path = expand_path(rel_path)
       setup_project(abs_path)
       rel_path = expand_ext(rel_path)
@@ -176,13 +183,10 @@ def already_processed
       @already_processed ||= Set.new
     end
 
-    include Project::Collection
-    include Builder::Directory
-
     attr_reader :processed
 
     attr_accessor :processors, :path_reader, :stubs, :prerequired, :preload,
-      :compiler_options, :missing_require_severity, :cache, :scheduler
+      :compiler_options, :missing_require_severity, :cache, :scheduler, :build_time
 
     def esm?
       @compiler_options[:esm]
@@ -191,11 +195,7 @@ def esm?
     # Output extension, to be used by runners. At least Node.JS switches
     # to ESM mode only if the extension is "mjs"
     def output_extension
-      if esm?
-        'mjs'
-      else
-        'js'
-      end
+      esm? ? 'mjs' : 'js'
     end
 
     # Return a list of dependent files, for watching purposes
@@ -240,8 +240,14 @@ def tree_requires(asset, asset_path)
         if abs_base_path
           abs_base_path = Pathname(abs_base_path)
           entries_glob  = Pathname(abs_tree_path).join('**', "*{.js,}.{#{extensions.join ','}}")
-
-          Pathname.glob(entries_glob).map { |file| file.relative_path_from(abs_base_path).to_s }
+          Pathname.glob(entries_glob).map do |file|
+            if file.extname == '.rb'
+              # remove .rb so file can be found in already_processed
+              file.relative_path_from(abs_base_path).to_s.delete_suffix('.rb')
+            else
+              file.relative_path_from(abs_base_path).to_s
+            end
+          end
         else
           [] # the tree is not part of any known base path
         end
@@ -291,7 +297,8 @@ def read(path, autoload)
 
     def expand_path(path)
       return if stub?(path)
-      (path_reader.expand(path) || File.expand_path(path)).to_s
+      path = (path_reader.expand(path) || File.expand_path(path)).to_s
+      path if File.exist?(path)
     end
 
     def stub?(path)
@@ -303,3 +310,5 @@ def extensions
     end
   end
 end
+
+require 'opal/builder/processor'

lib/opal/builder/processor.rb

@@ -7,20 +7,45 @@
 module Opal
   class Builder
     class Processor
-      def initialize(source, filename, abs_path = nil, options = {})
-        options = abs_path if abs_path.is_a? Hash
-
-        source += "\n" unless source.end_with?("\n")
-        @source, @filename, @abs_path, @options = source, filename, abs_path, options.dup
+      def initialize(orig_source, filename, abs_path = nil, options = {})
+        if abs_path.is_a? Hash
+          options = abs_path
+          abs_path = nil
+        end
+        self.source = orig_source
+        @filename, @abs_path, @options = filename, abs_path, options.dup
         @cache = @options.delete(:cache) { Opal.cache }
         @requires = []
         @required_trees = []
         @autoloads = []
+        @mtime = file_mtime
       end
       attr_reader :source, :filename, :options, :requires, :required_trees, :autoloads, :abs_path
 
       alias original_source source
 
+      def source=(src)
+        src += "\n" unless src.end_with?("\n")
+        @source = src
+      end
+
+      def update(new_source)
+        @mtime = file_mtime
+        self.source = new_source
+      end
+
+      def file_mtime
+        File::Stat.new(@abs_path).mtime if @abs_path
+      end
+
+      def changed?
+        if @abs_path
+          return :removed unless File.exist?(@abs_path)
+          return :modified unless file_mtime == @mtime
+        end
+        :no
+      end
+
       def to_s
         source.to_s
       end
@@ -71,6 +96,11 @@ def source_map
         def source
           @source.to_s + mark_as_required(@filename)
         end
+
+        def update(new_source)
+          super
+          @source_map = nil
+        end
       end
 
       class RubyProcessor < Processor
@@ -90,10 +120,12 @@ def compiled
             compiler.compile
             compiler
           end
+          @required_trees_mtime ||= rts_mtime(@compiled)
+          @compiled
         end
 
         def cache_key
-          [self.class, @filename, @source, @options]
+          [self.class, @filename, @source, @options, @mtime]
         end
 
         def compiler_for(source, options = {})
@@ -112,10 +144,32 @@ def autoloads
           compiled.autoloads
         end
 
+        def update(new_source)
+          super
+          @compiled = nil
+        end
+
         # Also catch a files with missing extensions and nil.
         def self.match?(other)
           super || File.extname(other.to_s) == ''
         end
+
+        def rts_mtime(compiler)
+          if compiler&.required_trees&.any? && abs_path
+            dir = File.dirname(abs_path)
+            rt_mtimes = compiler.required_trees.map do |path|
+              rt_path = File.expand_path("#{dir}/#{path}")
+              Dir.each_child(rt_path).map { |file| File::Stat.new("#{dir}/#{path}/#{file}").mtime }.sort.last
+            end
+            rt_mtimes.sort.last
+          end
+        end
+
+        def changed?
+          res = super
+          return :modified if res == :no && @compiled&.required_trees&.any? && rts_mtime(@compiled) != @required_trees_mtime
+          res
+        end
       end
 
       # This handler is for files named ".opalerb", which ought to
@@ -136,6 +190,12 @@ def requires
           ['erb'] + super
         end
 
+        def update(new_source)
+          super
+          @source = prepare(@source, @filename)
+          @compiled = nil
+        end
+
         private
 
         def prepare(source, path)

lib/opal/builder/watcher.rb

@@ -0,0 +1,102 @@
+module Opal
+  class Builder
+    module Watcher
+      # Builds and watches all paths for changes and then rebuilds, does not return
+      def watch
+        loop do
+          changes = updates
+          if changes[:modified].any? || changes[:added].any? || changes[:removed].any?
+            yield self, changes
+          end
+          sleep 0.5
+        end
+      end
+
+      # Return the updates in the paths since last build
+      def updates
+        current_build_time = Time.now
+        changes = { added: [], modified: [], removed: [] }
+        directories = {}
+        pwd = Dir.pwd
+        last_build_time = build_time
+
+        # check processed files
+        processed.dup.each do |asset|
+          case asset.changed?
+          when :removed
+            changes[:removed] << asset
+            processed.delete(asset)
+          when :modified
+            asset_count = processed.length
+            changes[:modified] << update(asset)
+            if processed.size != asset_count
+              added_assets = processed.pop(processed.size - asset_count)
+              # If assets have been added by require_tree
+              # ensure they are inserted before current asset.
+              processed.insert(processed.index(asset), *added_assets)
+              changes[:added].concat(added_assets)
+            end
+            add_to_directories(directories, asset)
+          else
+            add_to_directories(directories, asset)
+          end
+        end
+
+        # check for added files
+        directories.each do |dir, processed_entries|
+          if dir.start_with?(pwd) # only check project directories
+            current_entries = []
+            Dir.each_child(dir) do |entry|
+              current_entries << entry unless Dir.exist?(File.join(dir, entry))
+            end
+            (current_entries - processed_entries).each do |path|
+              abs_path = File.join(dir, path)
+              if File::Stat.new(abs_path).mtime > last_build_time # ignore files that existed before the last build
+                # check if asset has already been added by require_tree above
+                unless changes[:added].index { |ast| ast.abs_path == abs_path }
+                  asset_count = processed.length
+                  changes[:added] << build_str(source_for(abs_path), path)
+                  if processed.length > (asset_count + 1)
+                    changes[:added].concat(processed[asset_count..-1])
+                  end
+                end
+              end
+            end
+          end
+        end
+
+        # #build is called multiple times above, with each call setting the build_time.
+        # If files are added in between to a directory, that has already been processed,
+        # they may get skipped, because their mtime may be earlier than the last build_time.
+        # Ensure build_time is set to the time #update was called first, so that the files
+        # that may have been added in between #build calls above, are processed with
+        # the next call to #update.
+        build_time = current_build_time
+
+        changes
+      rescue StandardError, Opal::SyntaxError => e
+        $stderr.puts "Opal::Builder rebuilding failed: #{e.message}"
+        changes[:error] = e
+        changes
+      end
+
+      private
+
+      def update(asset)
+        asset.update(source_for(asset.abs_path))
+        requires = preload + asset.requires + tree_requires(asset, asset.abs_path)
+        # Don't automatically load modules required by the module
+        process_requires(asset.filename, requires, asset.autoloads, asset.options)
+        # TODO: the order corrector may be required here
+        asset
+      end
+
+      def add_to_directories(directories, asset)
+        return unless asset.abs_path
+        dirname = File.dirname(asset.abs_path)
+        directories[dirname] = [] unless directories.key?(dirname)
+        directories[dirname] << File.basename(asset.abs_path)
+      end
+    end
+  end
+end