Add watch and update capability for hot reloading to Builder

lib/opal/builder.rb

@@ -92,6 +92,7 @@ def self.build(*args, &block)
 
     def build(path, options = {})
       build_str(source_for(path), path, options)
+      self
     end
 
     # Retrieve the source for a given path the same way #build would do.
@@ -109,7 +110,7 @@ def build_str(source, rel_path, options = {})
       # Don't automatically load modules required by the module
       process_requires(rel_path, requires, asset.autoloads, options.merge(load: false))
       processed << asset
-      self
+      asset
     end
 
     def build_require(path, options = {})
@@ -182,7 +183,7 @@ def already_processed
     attr_reader :processed
 
     attr_accessor :processors, :path_reader, :stubs, :prerequired, :preload,
-      :compiler_options, :missing_require_severity, :cache, :scheduler
+      :compiler_options, :missing_require_severity, :cache, :scheduler, :watch_type
 
     def esm?
       @compiler_options[:esm]
@@ -191,11 +192,7 @@ def esm?
     # Output extension, to be used by runners. At least Node.JS switches
     # to ESM mode only if the extension is "mjs"
     def output_extension
-      if esm?
-        'mjs'
-      else
-        'js'
-      end
+      esm? ? 'mjs' : 'js'
     end
 
     # Return a list of dependent files, for watching purposes
@@ -223,8 +220,77 @@ def compiled_source(with_source_map: true)
       compiled_source
     end
 
+    # Builds and watches all paths for changes and then rebuilds, does not return
+    def watch
+      loop do
+        changes = updates
+        if changes[:modified].any? || changes[:added].any? || changes[:removed].any?
+          yield self, changes
+        end
+        sleep 0.5
+      end
+    end
+
+    # Return the updates in the paths since last build
+    def updates
+      changes = { added: [], modified: [], removed: [] }
+      directories = {}
+
+      # check processed files
+      processed.select! do |asset|
+        case asset.changed?
+        when :removed
+          changes[:removed] << asset
+          false
+        when :modified
+          changes[:modified] << update(asset)
+          add_to_directories(directories, asset)
+          true
+        else
+          add_to_directories(directories, asset)
+          true
+        end
+      end
+
+      # check for added files
+      directories.each do |dir, processed_entries|
+        current_entries = []
+        Dir.each_child(dir) do |entry|
+          current_entries << File.join(dir, entry)
+        end
+        (current_entries - processed_entries).each do |path|
+          asset_count = processed.length
+          changes[:added] << build(path)
+          if processed.length > (asset_count + 1)
+            changes[:added].concat(processed[asset_count..-1])
+          end
+        end
+      end
+
+      changes
+    rescue StandardError, Opal::SyntaxError => e
+      $stderr.puts "Opal::Builder rebuilding failed: #{e.message}"
+      changes[:error] = e
+      changes
+    end
+
     private
 
+    def update(asset)
+      asset.update(source_for(asset.abs_path))
+      requires = preload + asset.requires + tree_requires(asset, abs_path)
+      # Don't automatically load modules required by the module
+      process_requires(rel_path, requires, asset.autoloads, options.merge(load: false))
+      # TODO: the order corrector may be required here
+    end
+
+    def add_to_directories(asset, directories)
+      return unless asset.abs_path
+      dirname = File.dirname(asset.abs_path)
+      directories[dirname] = [] unless directories.key?(dirname)
+      directories[dirname] << File.basename(asset.abs_path)
+    end
+
     def process_requires(rel_path, requires, autoloads, options)
       @scheduler.process_requires(rel_path, requires, autoloads, options)
     end
@@ -291,7 +357,8 @@ def read(path, autoload)
 
     def expand_path(path)
       return if stub?(path)
-      (path_reader.expand(path) || File.expand_path(path)).to_s
+      path = (path_reader.expand(path) || File.expand_path(path)).to_s
+      path if File.exist?(path)
     end
 
     def stub?(path)

lib/opal/builder/processor.rb

@@ -7,20 +7,45 @@
 module Opal
   class Builder
     class Processor
-      def initialize(source, filename, abs_path = nil, options = {})
-        options = abs_path if abs_path.is_a? Hash
-
-        source += "\n" unless source.end_with?("\n")
-        @source, @filename, @abs_path, @options = source, filename, abs_path, options.dup
+      def initialize(orig_source, filename, abs_path = nil, options = {})
+        if abs_path.is_a? Hash
+          options = abs_path
+          abs_path = nil
+        end
+        self.source = orig_source
+        @filename, @abs_path, @options = filename, abs_path, options.dup
         @cache = @options.delete(:cache) { Opal.cache }
         @requires = []
         @required_trees = []
         @autoloads = []
+        @mtime = file_mtime
       end
       attr_reader :source, :filename, :options, :requires, :required_trees, :autoloads, :abs_path
 
       alias original_source source
 
+      def source=(src)
+        src += "\n" unless src.end_with?("\n")
+        @source = src
+      end
+
+      def update(new_source)
+        @mtime = file_mtime
+        self.source = new_source
+      end
+
+      def file_mtime
+        File::Stat.new(@abs_path).mtime if @abs_path
+      end
+
+      def changed?
+        if @abs_path
+          return :removed unless File.exist?(@abs_path)
+          return :modified unless file_mtime == @mtime
+        end
+        :no
+      end
+
       def to_s
         source.to_s
       end
@@ -71,6 +96,11 @@ def source_map
         def source
           @source.to_s + mark_as_required(@filename)
         end
+
+        def update(new_source)
+          super
+          @source_map = nil
+        end
       end
 
       class RubyProcessor < Processor
@@ -93,7 +123,7 @@ def compiled
         end
 
         def cache_key
-          [self.class, @filename, @source, @options]
+          [self.class, @filename, @source, @options, @mtime]
         end
 
         def compiler_for(source, options = {})
@@ -112,6 +142,11 @@ def autoloads
           compiled.autoloads
         end
 
+        def update(new_source)
+          super
+          @compiled = nil
+        end
+
         # Also catch a files with missing extensions and nil.
         def self.match?(other)
           super || File.extname(other.to_s) == ''
@@ -136,6 +171,12 @@ def requires
           ['erb'] + super
         end
 
+        def update(new_source)
+          super
+          @source = prepare(@source, @filename)
+          @compiled = nil
+        end
+
         private
 
         def prepare(source, path)

lib/opal/cli_runners/compiler.rb

@@ -17,8 +17,8 @@ def initialize(data)
     @directory       = @options[:directory]
   end
 
-  def compile
-    builder = @builder_factory.call
+  def compile(builder = nil)
+    builder ||= @builder_factory.call
 
     if @directory
       builder.compile_to_directory(@output, with_source_map: !@options[:no_source_map])
@@ -36,13 +36,6 @@ def compile
     builder
   end
 
-  def compile_noraise
-    compile
-  rescue StandardError, Opal::SyntaxError => e
-    $stderr.puts "* Compilation failed: #{e.message}"
-    nil
-  end
-
   def rewind_output
     if !@output.is_a?(File) || @output.tty?
       fail_unrewindable!
@@ -74,50 +67,41 @@ def fail_no_listen!
   end
 
   def watch_compile
-    begin
-      require 'listen'
-    rescue LoadError
-      fail_no_listen!
-    end
-
     @opal_deps = Opal.dependent_files
 
     builder = compile
     code_deps = builder.dependent_files
     @files = @opal_deps + code_deps
-    @code_listener = watch_files
-    @code_listener.start
 
     $stderr.puts "* Opal v#{Opal::VERSION} successfully compiled your program in --watch mode"
 
-    sleep
-  rescue Interrupt
-    $stderr.puts '* Stopping watcher...'
-    @code_listener.stop
+    builder.watch do |bldr, changes|
+      unless changes.key?(:error)
+        modified = changes[:added].map(&:abs_path) + changes[:modified].map(&:abs_path) + changes[:removed].map(&:abs_path)
+        on_code_change(bldr, modified)
+      end
+    end
   end
 
   def reexec
     Process.kill('USR2', Process.pid)
   end
 
-  def on_code_change(modified)
+  def on_code_change(builder, modified)
     if !(modified & @opal_deps).empty?
       $stderr.puts "* Modified core Opal files: #{modified.join(', ')}; reexecuting"
       reexec
     elsif !modified.all? { |file| @directories.any? { |dir| file.start_with?(dir + '/') } }
       $stderr.puts "* New unwatched files: #{modified.join(', ')}; reexecuting"
       reexec
     end
+    $stderr.puts '* Modified code rebuilding'
 
-    $stderr.puts "* Modified code: #{modified.join(', ')}; rebuilding"
-
-    builder = compile_noraise
+    compile(builder)
 
     # Ignore the bad compilation
-    if builder
-      code_deps = builder.dependent_files
-      @files = @opal_deps + code_deps
-    end
+    code_deps = builder.dependent_files
+    @files = @opal_deps + code_deps
   end
 
   def files_to_directories
@@ -136,15 +120,6 @@ def files_to_directories
     directories.compact
   end
 
-  def watch_files
-    @directories = files_to_directories
-
-    Listen.to(*@directories, ignore!: []) do |modified, added, removed|
-      our_modified = @files & (modified + added + removed)
-      on_code_change(our_modified) unless our_modified.empty?
-    end
-  end
-
   def start
     if @watch
       watch_compile

lib/opal/project.rb

@@ -170,6 +170,7 @@ def self.setup_project_for(collection, file)
     # @return [String, nil] The path to the root directory, or nil if not found or
     # the file does not exist.
     def self.locate_root_dir(file)
+      return nil unless file
       begin
         file = File.realpath(file)
       rescue Errno::ENOENT

spec/lib/cli_runners/server_spec.rb

@@ -15,7 +15,11 @@ def app
       expect(options[:Port]).to eq(1234)
     end
 
-    builder = -> { Opal::Builder.new.build_str("puts 123", "app.rb") }
+    builder = -> do
+      bldr = Opal::Builder.new
+      bldr.build_str("puts 123", "app.rb")
+      bldr
+    end
     described_class.call(builder: builder, options: {port: 1234})
 
     get '/assets/cli_runner.js'