Read an extra Rclone config file

Make Rclone in OOD support an extra configuration file
(OOD_RCLONE_EXTRA_CONFIG), e.g. for allowing both user and OOD managed
configs. Extra config file is read into env vars passed to Rclone.

apps/dashboard/config/configuration_singleton.rb

@@ -63,7 +63,8 @@ def string_configs
       :auto_groups_filter       => nil,
       :bc_clean_old_dirs_days   => '30',
       :google_analytics_tag_id  => nil,
-      :project_template_dir     => "#{config_root}/projects"
+      :project_template_dir     => "#{config_root}/projects",
+      :rclone_extra_config      => nil
     }.freeze
   end
 

apps/dashboard/lib/rclone_util.rb

@@ -237,19 +237,45 @@ def list_remotes
       end
     end
 
+    # Gets the config from OOD_RCLONE_EXTRA_CONFIG file as a Hash of env vars.
+    # @return [Hash] environment variables corresponding to config
+    def extra_config
+      return {} if Configuration.rclone_extra_config.blank?
+
+      # Rclone config dump outputs the configured remotes as JSON.
+      # TODO: Cache these results when generating favorite paths?
+      o, _, s = rclone('config', 'dump', env: { 'RCLONE_CONFIG' => Configuration.rclone_extra_config })
+      return {} unless s.success?
+
+      remotes = JSON.parse(o)
+
+      # Combine config into a single Hash where keys and values are e.g.
+      # RCLONE_CONFIG_MYREMOTE_TYPE: "s3"
+      remotes.map do |remote_name, remote_config|
+        remote_config.transform_keys do |option|
+          "RCLONE_CONFIG_#{remote_name}_#{option}".upcase
+        end
+      end.reduce(&:merge) || {} # reduce on empty array returns nil
+    rescue StandardError => e
+      Rails.logger.error("Could not read extra Rclone configuration: #{e.message}")
+      {}
+    end
+
     def rclone_cmd
       # TODO: Make this configurable
       "rclone"
     end
 
-    def rclone(*args)
-      Open3.capture3(rclone_cmd, *args)
+    def rclone(*args, env: extra_config, **kwargs)
+      # config/initalizers/open3_extensions.rb overrides Open3.capture3 to log calls.
+      # Get a reference to the original method to avoid logging the sensitive env vars.
+      Open3.singleton_method(:capture3).call(env, rclone_cmd, *args, **kwargs)
     end
 
-    def rclone_popen(*args, stdin_data: nil, &block)
+    def rclone_popen(*args, stdin_data: nil, env: extra_config, **kwargs, &block)
       # Use -q to suppress message about config file not existing
       # need it here as we check err.present?
-      Open3.popen3(rclone_cmd, "--quiet", *args) do |i, o, e, t|
+      Open3.popen3(env, rclone_cmd, "--quiet", *args) do |i, o, e, t|
         if stdin_data
           i.write(stdin_data)
         end

apps/dashboard/test/fixtures/config/ondemand.d/string.yml

@@ -5,3 +5,4 @@ auto_groups_filter: 'string from file'
 bc_clean_old_dirs_days: 'string from file'
 google_analytics_tag_id: 'string from file'
 project_template_dir: 'string from file'
+rclone_extra_config: 'string from file'

apps/dashboard/test/fixtures/config/rclone/extra_config.conf

@@ -0,0 +1,2 @@
+[extra_remote]
+type = local

apps/dashboard/test/rclone_helper.rb

@@ -18,4 +18,10 @@ def with_rclone_conf(root_dir, &block)
     conf = remote_files_conf(root_dir)
     with_modified_env(conf, &block)
   end
+
+  def with_extra_rclone_conf(root_dir, &block)
+    conf = remote_files_conf(root_dir)
+      .merge({OOD_RCLONE_EXTRA_CONFIG: Rails.root.join('test/fixtures/config/rclone/extra_config.conf').to_s})
+    with_modified_env(conf, &block)
+  end
 end

apps/dashboard/test/system/remote_files_test.rb

@@ -214,6 +214,49 @@ class RemoteFilesTest < ApplicationSystemTestCase
     end
   end
 
+  # Using files move test to specifically test that RcloneUtil.rclone_popen works
+  test 'moving files with remote configured in extra config' do
+    Dir.mktmpdir do |dir|
+      with_extra_rclone_conf(dir) do
+        # Test in a subdir of the temp directory
+        dir = File.join(dir, 'bucket')
+        Dir.mkdir(dir)
+
+        # copy to dest/app
+        dest = File.join(dir, 'dest')
+        FileUtils.mkpath dest
+
+        `cp -r #{Rails.root.join('app')} #{Rails.root.join('config')}  #{Rails.root.join('manifest.yml')} #{dir}`
+
+        # select dir to move
+        visit files_url('extra_remote', dir)
+        %w(app config manifest.yml).each do |f|
+          find('a', exact_text: f).ancestor('tr').click(:meta)
+        end
+        assert_selector '.selected', count: 3
+
+        find('#copy-move-btn').click
+
+        # move to new location
+        visit files_url('extra_remote', "#{dir}/dest")
+        find('#clipboard-move-to-dir').click
+        find('tbody a', exact_text: 'app', wait: MAX_WAIT)
+        find('tbody a', exact_text: 'config', wait: MAX_WAIT)
+        find('tbody a', exact_text: 'manifest.yml', wait: MAX_WAIT)
+
+        # verify contents moved
+        assert_equal '', `diff -rq #{File.join(dest, 'app')} #{Rails.root.join('app')}`.strip, 'failed to mv app and all contents'
+        assert_equal '', `diff -rq #{File.join(dest, 'config')} #{Rails.root.join('config')}`.strip, 'failed to recursively copy config dir'
+        assert_equal '', `diff -q #{File.join(dest, 'manifest.yml')} #{Rails.root.join('manifest.yml')}`.strip, 'failed to copy manifest.yml'
+
+        # verify original does not exist
+        refute File.directory?(File.join(dir, 'app'))
+        refute File.directory?(File.join(dir, 'config'))
+        refute File.directory?(File.join(dir, 'manifest.yml'))
+      end
+    end
+  end
+
   test 'removing files' do
     Dir.mktmpdir do |dir|
       with_rclone_conf(dir) do

apps/dashboard/test/unit/rclone_util_test.rb

@@ -14,4 +14,10 @@ class RcloneUtilTest < ActiveSupport::TestCase
         assert_equal [], RcloneUtil.list_remotes
     end
   end
+
+  test "list_remotes handles rclone.conf, env and extra rclone config" do
+    with_extra_rclone_conf("/") do
+        assert_equal ["alias_remote", "extra_remote", "local_remote", "s3"], RcloneUtil.list_remotes
+    end
+  end
 end