From 8547f4ecdc813ca1b571d5843219407d02845b6d Mon Sep 17 00:00:00 2001
From: Varjitt Jeeva <varjitt.jeeva@plangrid.com>
Date: Mon, 9 Oct 2017 23:23:32 -0700
Subject: [PATCH] Fixing #10, s3 directory listing works, generalized backup
 path grabbing for any new transports, using S3 ls rather than mounting s3 as
 a directory. (#14)

---
 Gemfile                 |  2 +-
 config.ru               |  2 +-
 lib/files.rb            | 44 +++++++++++++++++++++++++++++++----------
 lib/transport.rb        |  4 ++++
 lib/transports/local.rb | 27 +++++++++++++++++++++++++
 lib/transports/s3.rb    | 24 ++++++++++++++++++++++
 ui.rb                   |  1 -
 7 files changed, 91 insertions(+), 13 deletions(-)

diff --git a/Gemfile b/Gemfile
index e2d53e9..f21daae 100644
--- a/Gemfile
+++ b/Gemfile
@@ -2,6 +2,7 @@ source 'https://rubygems.org'
 
 gem 'sequel',     '~> 4.30'
 gem 'docker-api', '~> 1.33.2'
+gem 'aws-sdk', '~> 2.2', '>= 2.2.12'
 
 group :tester do
   # core
@@ -16,7 +17,6 @@ group :tester do
   gem 'sqlite3',   '~> 1.3', '>= 1.3.11'
 
   # transports
-  gem 'aws-sdk', '~> 2.2', '>= 2.2.12'
 end
 
 group :ui do
diff --git a/config.ru b/config.ru
index 5c7899f..50ec1b2 100644
--- a/config.ru
+++ b/config.ru
@@ -1,4 +1,4 @@
 $:.unshift(File.dirname(__FILE__))
 
 require 'ui'
-run Uphold::Ui
+run Uphold::Ui
\ No newline at end of file
diff --git a/lib/files.rb b/lib/files.rb
index 709045e..c431832 100644
--- a/lib/files.rb
+++ b/lib/files.rb
@@ -6,16 +6,7 @@ class Files
 
     def self.backups(config)
       # Fuck me this assumes uncompressed...
-      path = config[:transport][:settings][:path]
-      if path.include? '{date'
-        index = path.index('{date') # 0 based index
-        general_path = '/mount' + path[0..index-1]
-        backup_paths = []
-        Find.find(general_path) do |path|
-          backup_paths << path.gsub!('/mount', '') if path.include? "#{config[:engine][:settings][:extension]}"
-        end
-        backup_paths
-      end
+      config[:transport][:klass].get_backup_paths(config)
     end
 
     def self.raw_test_logs
@@ -44,5 +35,38 @@ def self.extract_datetime_from_backup_path(config, path)
       dates.max
     end
 
+    def self.get_general_path(config, prefix)
+      path = config[:transport][:settings][:path]
+      general_path = path.clone
+      if general_path.include? '{date'
+        index = path.index('{date') # 0 based index
+        general_path = prefix + path[0..index-1]
+      end
+      general_path
+    end
+
+    def self.get_date_regexes_from_config(config)
+      regexes = []
+      config[:transport][:settings][:dates].each do |date|
+        # GOT IT: Need to stop removing {dateX} because there are outliers in the s3 bucket that don't conform. Need
+        # better verification. Instead of this crap, sub in the dates with a regex!!!! THATS WHAT ITS FOR OMG LOL
+        date_format = date[:date_format] || '%Y-%m-%d'
+        regexes << DateHelper.regex_from_posix(date_format).to_s
+      end
+      regexes
+    end
+
+    def self.get_paths_matching_regexes(paths, regexes, extension)
+      paths_local = paths.clone
+      regexes.each do |regex|
+        paths_selected = []
+        paths_local.each do |path|
+          paths_selected << path if path.match(regex) and path.include? extension
+        end
+        paths_local = paths_selected
+      end
+      paths_local
+    end
+
   end
 end
\ No newline at end of file
diff --git a/lib/transport.rb b/lib/transport.rb
index a579abb..f067c72 100644
--- a/lib/transport.rb
+++ b/lib/transport.rb
@@ -48,5 +48,9 @@ def fetch
     def fetch_backup
       fail "Your transport must implement the 'fetch' method"
     end
+
+    def get_backup_paths
+      fail "Your transport must implement the 'get_backup_paths' method"
+    end
   end
 end
diff --git a/lib/transports/local.rb b/lib/transports/local.rb
index 6f01de6..6e305b6 100644
--- a/lib/transports/local.rb
+++ b/lib/transports/local.rb
@@ -1,6 +1,8 @@
 module Uphold
   module Transports
     class Local < Transport
+      include DateHelper
+
       def initialize(params)
         super(params)
       end
@@ -24,6 +26,31 @@ def fetch_backup
         end
       end
 
+      def self.get_backup_paths(config)
+        # Regexes from dates in config
+        regexes = Uphold::Files.get_date_regexes_from_config(config)
+
+        # Top level folder path (without dates)
+        general_path = Uphold::Files.get_general_path(config, "/mount-#{config[:transport][:type]}-#{config[:name]}")
+
+        # Objective of any transport is to get the paths of relevant backups given the general path directory
+        # and the regexes to match with. Here, we first get all paths in the general directory
+        paths = []
+        Find.find(general_path) do |path|
+          paths << path
+        end
+
+        # Now, we filter the paths we got above with the regexes from dates
+        paths = Uphold::Files.get_paths_matching_regexes(paths, regexes, config[:engine][:settings][:extension])
+
+        # Stripping out mount prefix
+        paths_stripped = []
+        paths.each do |path|
+          paths_stripped << path.gsub!("/mount-#{config[:transport][:type]}-#{config[:name]}", '')
+        end
+        paths_stripped
+      end
+
     end
   end
 end
diff --git a/lib/transports/s3.rb b/lib/transports/s3.rb
index d7dc3aa..a8adb47 100644
--- a/lib/transports/s3.rb
+++ b/lib/transports/s3.rb
@@ -1,6 +1,9 @@
 module Uphold
   module Transports
     class S3 < Transport
+      require 'aws-sdk'
+      include DateHelper
+
       def initialize(params)
         super(params)
         @region = params[:region]
@@ -28,6 +31,27 @@ def fetch_backup
           @tmpdir
         end
       end
+
+      def self.get_backup_paths(config)
+        region = config[:transport][:settings][:region]
+        access_key_id = config[:transport][:settings][:access_key_id]
+        secret_access_key = config[:transport][:settings][:secret_access_key]
+        bucket = config[:transport][:settings][:bucket]
+
+        # Regexes from dates in config
+        regexes = Uphold::Files.get_date_regexes_from_config(config)
+
+        # Top level folder path (without dates)
+        general_path = Uphold::Files.get_general_path(config, '') # Ensure to pass empty string for no prefix
+
+        # Objective of any transport is to get the paths of relevant backups given the general path directory
+        # and the regexes to match with. Here, we first get all paths in the general directory
+        s3 = Aws::S3::Client.new(region: region, access_key_id: access_key_id, secret_access_key: secret_access_key)
+        paths = s3.list_objects(bucket: bucket, prefix: general_path).contents.collect(&:key)
+
+        # Now, we filter the paths we got above with the regexes from dates
+        Uphold::Files.get_paths_matching_regexes(paths, regexes, config[:engine][:settings][:extension])
+      end
     end
   end
 end
diff --git a/ui.rb b/ui.rb
index fc30f9e..361908f 100644
--- a/ui.rb
+++ b/ui.rb
@@ -37,7 +37,6 @@ def datetime_to_epoch(datetime)
 
     get '/' do
       @data = backups_with_logs
-      logger.debug @data
       erb :index
     end