diff --git a/apps/dashboard/config/configuration_singleton.rb b/apps/dashboard/config/configuration_singleton.rb index 8a76c01a5e..07fe087278 100644 --- a/apps/dashboard/config/configuration_singleton.rb +++ b/apps/dashboard/config/configuration_singleton.rb @@ -64,7 +64,8 @@ def string_configs :auto_groups_filter => nil, :bc_clean_old_dirs_days => '30', :google_analytics_tag_id => nil, - :project_template_dir => "#{config_root}/projects" + :project_template_dir => "#{config_root}/projects", + :rclone_extra_config => nil }.freeze end diff --git a/apps/dashboard/lib/rclone_util.rb b/apps/dashboard/lib/rclone_util.rb index 41ed8c7c92..0c66a0dd5c 100644 --- a/apps/dashboard/lib/rclone_util.rb +++ b/apps/dashboard/lib/rclone_util.rb @@ -252,19 +252,45 @@ def valid?(remote) false end + # Gets the config from OOD_RCLONE_EXTRA_CONFIG file as a Hash of env vars. + # @return [Hash] environment variables corresponding to config + def extra_config + return {} if Configuration.rclone_extra_config.blank? + + # Rclone config dump outputs the configured remotes as JSON. + # TODO: Cache these results when generating favorite paths? + o, _, s = rclone('config', 'dump', env: { 'RCLONE_CONFIG' => Configuration.rclone_extra_config }) + return {} unless s.success? + + remotes = JSON.parse(o) + + # Combine config into a single Hash where keys and values are e.g. + # RCLONE_CONFIG_MYREMOTE_TYPE: "s3" + remotes.map do |remote_name, remote_config| + remote_config.transform_keys do |option| + "RCLONE_CONFIG_#{remote_name}_#{option}".upcase + end + end.reduce(&:merge) || {} # reduce on empty array returns nil + rescue StandardError => e + Rails.logger.error("Could not read extra Rclone configuration: #{e.message}") + {} + end + def rclone_cmd # TODO: Make this configurable "rclone" end - def rclone(*args) - Open3.capture3(rclone_cmd, *args) + def rclone(*args, env: extra_config, **kwargs) + # config/initalizers/open3_extensions.rb overrides Open3.capture3 to log calls. + # Get a reference to the original method to avoid logging the sensitive env vars. + Open3.singleton_method(:capture3).call(env, rclone_cmd, *args, **kwargs) end - def rclone_popen(*args, stdin_data: nil, &block) + def rclone_popen(*args, stdin_data: nil, env: extra_config, &block) # Use -q to suppress message about config file not existing # need it here as we check err.present? - Open3.popen3(rclone_cmd, "--quiet", *args) do |i, o, e, t| + Open3.popen3(env, rclone_cmd, "--quiet", *args) do |i, o, e, t| if stdin_data i.write(stdin_data) end diff --git a/apps/dashboard/test/fixtures/config/ondemand.d/string.yml b/apps/dashboard/test/fixtures/config/ondemand.d/string.yml index 0bf940e4d8..b2e4184582 100644 --- a/apps/dashboard/test/fixtures/config/ondemand.d/string.yml +++ b/apps/dashboard/test/fixtures/config/ondemand.d/string.yml @@ -5,3 +5,4 @@ auto_groups_filter: 'string from file' bc_clean_old_dirs_days: 'string from file' google_analytics_tag_id: 'string from file' project_template_dir: 'string from file' +rclone_extra_config: 'string from file' \ No newline at end of file diff --git a/apps/dashboard/test/fixtures/config/rclone/extra_config.conf b/apps/dashboard/test/fixtures/config/rclone/extra_config.conf new file mode 100644 index 0000000000..f64f6fb6df --- /dev/null +++ b/apps/dashboard/test/fixtures/config/rclone/extra_config.conf @@ -0,0 +1,2 @@ +[extra_remote] +type = local diff --git a/apps/dashboard/test/rclone_helper.rb b/apps/dashboard/test/rclone_helper.rb index 052cd27f5c..9cb8835e52 100644 --- a/apps/dashboard/test/rclone_helper.rb +++ b/apps/dashboard/test/rclone_helper.rb @@ -18,4 +18,10 @@ def with_rclone_conf(root_dir, &block) conf = remote_files_conf(root_dir) with_modified_env(conf, &block) end + + def with_extra_rclone_conf(root_dir, &block) + conf = remote_files_conf(root_dir) + .merge({OOD_RCLONE_EXTRA_CONFIG: Rails.root.join('test/fixtures/config/rclone/extra_config.conf').to_s}) + with_modified_env(conf, &block) + end end \ No newline at end of file diff --git a/apps/dashboard/test/system/remote_files_test.rb b/apps/dashboard/test/system/remote_files_test.rb index 965c5108e2..0f055e1e21 100644 --- a/apps/dashboard/test/system/remote_files_test.rb +++ b/apps/dashboard/test/system/remote_files_test.rb @@ -214,6 +214,49 @@ class RemoteFilesTest < ApplicationSystemTestCase end end + # Using files move test to specifically test that RcloneUtil.rclone_popen works + test 'moving files with remote configured in extra config' do + Dir.mktmpdir do |dir| + with_extra_rclone_conf(dir) do + # Test in a subdir of the temp directory + dir = File.join(dir, 'bucket') + Dir.mkdir(dir) + + # copy to dest/app + dest = File.join(dir, 'dest') + FileUtils.mkpath dest + + `cp -r #{Rails.root.join('app')} #{Rails.root.join('config')} #{Rails.root.join('manifest.yml')} #{dir}` + + # select dir to move + visit files_url('extra_remote', dir) + %w(app config manifest.yml).each do |f| + find('a', exact_text: f).ancestor('tr').click(:meta) + end + assert_selector '.selected', count: 3 + + find('#copy-move-btn').click + + # move to new location + visit files_url('extra_remote', "#{dir}/dest") + find('#clipboard-move-to-dir').click + find('tbody a', exact_text: 'app', wait: MAX_WAIT) + find('tbody a', exact_text: 'config', wait: MAX_WAIT) + find('tbody a', exact_text: 'manifest.yml', wait: MAX_WAIT) + + # verify contents moved + assert_equal '', `diff -rq #{File.join(dest, 'app')} #{Rails.root.join('app')}`.strip, 'failed to mv app and all contents' + assert_equal '', `diff -rq #{File.join(dest, 'config')} #{Rails.root.join('config')}`.strip, 'failed to recursively copy config dir' + assert_equal '', `diff -q #{File.join(dest, 'manifest.yml')} #{Rails.root.join('manifest.yml')}`.strip, 'failed to copy manifest.yml' + + # verify original does not exist + refute File.directory?(File.join(dir, 'app')) + refute File.directory?(File.join(dir, 'config')) + refute File.directory?(File.join(dir, 'manifest.yml')) + end + end + end + test 'removing files' do Dir.mktmpdir do |dir| with_rclone_conf(dir) do diff --git a/apps/dashboard/test/unit/rclone_util_test.rb b/apps/dashboard/test/unit/rclone_util_test.rb index 32b2e1aa1d..263e0b1960 100644 --- a/apps/dashboard/test/unit/rclone_util_test.rb +++ b/apps/dashboard/test/unit/rclone_util_test.rb @@ -22,4 +22,10 @@ class RcloneUtilTest < ActiveSupport::TestCase assert RcloneUtil.valid?('local_remote') end end + + test "list_remotes handles rclone.conf, env and extra rclone config" do + with_extra_rclone_conf("/") do + assert_equal ["alias_remote", "extra_remote", "local_remote", "missing_auth", "s3"], RcloneUtil.list_remotes + end + end end