Fixing #10, s3 directory listing works, generalized backup path grabb…

…ing for any new transports, using S3 ls rather than mounting s3 as a directory. (#14)

Gemfile

@@ -2,6 +2,7 @@ source 'https://rubygems.org'
 
 gem 'sequel',     '~> 4.30'
 gem 'docker-api', '~> 1.33.2'
+gem 'aws-sdk', '~> 2.2', '>= 2.2.12'
 
 group :tester do
   # core
@@ -16,7 +17,6 @@ group :tester do
   gem 'sqlite3',   '~> 1.3', '>= 1.3.11'
 
   # transports
-  gem 'aws-sdk', '~> 2.2', '>= 2.2.12'
 end
 
 group :ui do

config.ru

@@ -1,4 +1,4 @@
 $:.unshift(File.dirname(__FILE__))
 
 require 'ui'
-run Uphold::Ui
+run Uphold::Ui

lib/files.rb

@@ -6,16 +6,7 @@ class Files
 
     def self.backups(config)
       # Fuck me this assumes uncompressed...
-      path = config[:transport][:settings][:path]
-      if path.include? '{date'
-        index = path.index('{date') # 0 based index
-        general_path = '/mount' + path[0..index-1]
-        backup_paths = []
-        Find.find(general_path) do |path|
-          backup_paths << path.gsub!('/mount', '') if path.include? "#{config[:engine][:settings][:extension]}"
-        end
-        backup_paths
-      end
+      config[:transport][:klass].get_backup_paths(config)
     end
 
     def self.raw_test_logs
@@ -44,5 +35,38 @@ def self.extract_datetime_from_backup_path(config, path)
       dates.max
     end
 
+    def self.get_general_path(config, prefix)
+      path = config[:transport][:settings][:path]
+      general_path = path.clone
+      if general_path.include? '{date'
+        index = path.index('{date') # 0 based index
+        general_path = prefix + path[0..index-1]
+      end
+      general_path
+    end
+
+    def self.get_date_regexes_from_config(config)
+      regexes = []
+      config[:transport][:settings][:dates].each do |date|
+        # GOT IT: Need to stop removing {dateX} because there are outliers in the s3 bucket that don't conform. Need
+        # better verification. Instead of this crap, sub in the dates with a regex!!!! THATS WHAT ITS FOR OMG LOL
+        date_format = date[:date_format] || '%Y-%m-%d'
+        regexes << DateHelper.regex_from_posix(date_format).to_s
+      end
+      regexes
+    end
+
+    def self.get_paths_matching_regexes(paths, regexes, extension)
+      paths_local = paths.clone
+      regexes.each do |regex|
+        paths_selected = []
+        paths_local.each do |path|
+          paths_selected << path if path.match(regex) and path.include? extension
+        end
+        paths_local = paths_selected
+      end
+      paths_local
+    end
+
   end
 end

lib/transport.rb

@@ -48,5 +48,9 @@ def fetch
     def fetch_backup
       fail "Your transport must implement the 'fetch' method"
     end
+
+    def get_backup_paths
+      fail "Your transport must implement the 'get_backup_paths' method"
+    end
   end
 end

lib/transports/local.rb

@@ -1,6 +1,8 @@
 module Uphold
   module Transports
     class Local < Transport
+      include DateHelper
+
       def initialize(params)
         super(params)
       end
@@ -24,6 +26,31 @@ def fetch_backup
         end
       end
 
+      def self.get_backup_paths(config)
+        # Regexes from dates in config
+        regexes = Uphold::Files.get_date_regexes_from_config(config)
+
+        # Top level folder path (without dates)
+        general_path = Uphold::Files.get_general_path(config, "/mount-#{config[:transport][:type]}-#{config[:name]}")
+
+        # Objective of any transport is to get the paths of relevant backups given the general path directory
+        # and the regexes to match with. Here, we first get all paths in the general directory
+        paths = []
+        Find.find(general_path) do |path|
+          paths << path
+        end
+
+        # Now, we filter the paths we got above with the regexes from dates
+        paths = Uphold::Files.get_paths_matching_regexes(paths, regexes, config[:engine][:settings][:extension])
+
+        # Stripping out mount prefix
+        paths_stripped = []
+        paths.each do |path|
+          paths_stripped << path.gsub!("/mount-#{config[:transport][:type]}-#{config[:name]}", '')
+        end
+        paths_stripped
+      end
+
     end
   end
 end

lib/transports/s3.rb

@@ -1,6 +1,9 @@
 module Uphold
   module Transports
     class S3 < Transport
+      require 'aws-sdk'
+      include DateHelper
+
       def initialize(params)
         super(params)
         @region = params[:region]
@@ -28,6 +31,27 @@ def fetch_backup
           @tmpdir
         end
       end
+
+      def self.get_backup_paths(config)
+        region = config[:transport][:settings][:region]
+        access_key_id = config[:transport][:settings][:access_key_id]
+        secret_access_key = config[:transport][:settings][:secret_access_key]
+        bucket = config[:transport][:settings][:bucket]
+
+        # Regexes from dates in config
+        regexes = Uphold::Files.get_date_regexes_from_config(config)
+
+        # Top level folder path (without dates)
+        general_path = Uphold::Files.get_general_path(config, '') # Ensure to pass empty string for no prefix
+
+        # Objective of any transport is to get the paths of relevant backups given the general path directory
+        # and the regexes to match with. Here, we first get all paths in the general directory
+        s3 = Aws::S3::Client.new(region: region, access_key_id: access_key_id, secret_access_key: secret_access_key)
+        paths = s3.list_objects(bucket: bucket, prefix: general_path).contents.collect(&:key)
+
+        # Now, we filter the paths we got above with the regexes from dates
+        Uphold::Files.get_paths_matching_regexes(paths, regexes, config[:engine][:settings][:extension])
+      end
     end
   end
 end

ui.rb

@@ -37,7 +37,6 @@ def datetime_to_epoch(datetime)
 
     get '/' do
       @data = backups_with_logs
-      logger.debug @data
       erb :index
     end