DatabaseBackupReplicationJob : compare taille d'hier et d'aujourd'hui (…

…#4381)

apps/transport/lib/jobs/database_backup_replication_job.ex

@@ -8,7 +8,7 @@ defmodule Transport.Jobs.DatabaseBackupReplicationJob do
     - cannot list buckets
     - cannot list objects in destination bucket
     - cannot delete and object in destination bucket
-  - the latest dump has an appropriate size (between 90% and 110% of the previous dump)
+  - the latest dump has an appropriate size (between 90% and 110% compared to yesterday's dump)
   - the dump is recent enough
   - not too large
   """
@@ -57,13 +57,25 @@ defmodule Transport.Jobs.DatabaseBackupReplicationJob do
   end
 
   def check_appropriate_size! do
-    [yesterday_size, today_size] = latest_source_dumps(2) |> Enum.map(&dump_size/1)
+    today = Date.utc_today()
+    yesterday = Date.add(today, -1)
+    dumps = latest_source_dumps(20)
+    today_size = latest_dump_for_date(dumps, today) |> dump_size()
+    yesterday_size = latest_dump_for_date(dumps, yesterday) |> dump_size()
 
     unless 0.9 * yesterday_size <= today_size and today_size <= 1.1 * yesterday_size do
       raise "Latest backup size is unexpected. Yesterday: #{yesterday_size}, today: #{today_size}"
     end
   end
 
+  def latest_dump_for_date(latest_dumps, %Date{} = date) do
+    latest_dumps
+    |> Enum.filter(fn %{last_modified: last_modified} ->
+      Date.compare(last_modified |> datetime_to_date(), date) == :eq
+    end)
+    |> List.first()
+  end
+
   def latest_dump, do: List.first(latest_source_dumps(1))
 
   def latest_source_dumps(nb_to_keep) when is_integer(nb_to_keep) and nb_to_keep > 0 do
@@ -140,4 +152,9 @@ defmodule Transport.Jobs.DatabaseBackupReplicationJob do
       target
     )
   end
+
+  defp datetime_to_date(dt_string) when is_binary(dt_string) do
+    {:ok, dt, 0} = "#{String.trim_trailing(dt_string, "Z")}Z" |> DateTime.from_iso8601()
+    DateTime.to_date(dt)
+  end
 end

apps/transport/test/transport/jobs/database_backup_replication_job_test.exs

@@ -44,7 +44,7 @@ defmodule Transport.Test.Transport.Jobs.DatabaseBackupReplicationJobTest do
     end
   end
 
-  test "latest_dump and latest_source_dumps" do
+  test "latest_dump, latest_dump_for_date and latest_source_dumps" do
     # List objects in source bucket
     Transport.ExAWS.Mock
     |> expect(:request!, 2, fn operation, config ->
@@ -60,20 +60,26 @@ defmodule Transport.Test.Transport.Jobs.DatabaseBackupReplicationJobTest do
       assert config_is_source?(config)
 
       recent_datetime = NaiveDateTime.utc_now() |> NaiveDateTime.add(-60 * 60 * 2, :second)
-      older_datetime = recent_datetime |> NaiveDateTime.add(-60 * 60 * 10, :second)
+      yesterday_datetime = recent_datetime |> NaiveDateTime.add(-1, :day)
+      yesterday_older_datetime = yesterday_datetime |> NaiveDateTime.add(-10, :second)
 
       %{
         body: %{
           contents: [
-            %{last_modified: older_datetime |> NaiveDateTime.to_iso8601(), key: "second", size: "100"},
+            %{last_modified: yesterday_older_datetime |> NaiveDateTime.to_iso8601(), key: "third", size: "50"},
+            %{last_modified: yesterday_datetime |> NaiveDateTime.to_iso8601(), key: "second", size: "100"},
             %{last_modified: recent_datetime |> NaiveDateTime.to_iso8601(), key: "first", size: "200"}
           ]
         }
       }
     end)
 
-    assert [%{key: "first", size: "200"}, %{key: "second", size: "100"}] =
-             DatabaseBackupReplicationJob.latest_source_dumps(3)
+    dumps = DatabaseBackupReplicationJob.latest_source_dumps(3)
+
+    assert [%{key: "first", size: "200"}, %{key: "second", size: "100"}, %{key: "third", size: "50"}] = dumps
+
+    assert %{key: "second"} = DatabaseBackupReplicationJob.latest_dump_for_date(dumps, Date.utc_today() |> Date.add(-1))
+    assert %{key: "first"} = DatabaseBackupReplicationJob.latest_dump_for_date(dumps, Date.utc_today())
 
     assert %{key: "first", size: "200"} = DatabaseBackupReplicationJob.latest_dump()
   end
@@ -102,14 +108,14 @@ defmodule Transport.Test.Transport.Jobs.DatabaseBackupReplicationJobTest do
 
       assert config_is_source?(config)
 
-      recent_datetime = NaiveDateTime.utc_now() |> NaiveDateTime.add(-60 * 60 * 2, :second)
-      older_datetime = recent_datetime |> NaiveDateTime.add(-60 * 60 * 10, :second)
+      today_datetime = NaiveDateTime.utc_now() |> NaiveDateTime.add(-60 * 60 * 2, :second)
+      yesterday_datetime = today_datetime |> NaiveDateTime.add(-1, :day)
 
       %{
         body: %{
           contents: [
-            %{last_modified: older_datetime |> NaiveDateTime.to_iso8601(), key: Ecto.UUID.generate(), size: "195"},
-            %{last_modified: recent_datetime |> NaiveDateTime.to_iso8601(), key: latest_dump_filename, size: "200"}
+            %{last_modified: yesterday_datetime |> NaiveDateTime.to_iso8601(), key: Ecto.UUID.generate(), size: "195"},
+            %{last_modified: today_datetime |> NaiveDateTime.to_iso8601(), key: latest_dump_filename, size: "200"}
           ]
         }
       }
@@ -159,14 +165,20 @@ defmodule Transport.Test.Transport.Jobs.DatabaseBackupReplicationJobTest do
 
       assert config_is_source?(config)
 
-      recent_datetime = NaiveDateTime.utc_now() |> NaiveDateTime.add(-60 * 60 * 2, :second)
-      older_datetime = recent_datetime |> NaiveDateTime.add(-60 * 60 * 10, :second)
+      today_datetime = NaiveDateTime.utc_now() |> NaiveDateTime.add(-60 * 60 * 2, :second)
+      yesterday_datetime = today_datetime |> NaiveDateTime.add(-1, :day)
+      yesterday_older_datetime = yesterday_datetime |> NaiveDateTime.add(-10, :second)
 
       %{
         body: %{
           contents: [
-            %{last_modified: older_datetime |> NaiveDateTime.to_iso8601(), key: Ecto.UUID.generate(), size: "100"},
-            %{last_modified: recent_datetime |> NaiveDateTime.to_iso8601(), key: Ecto.UUID.generate(), size: "89"}
+            %{
+              last_modified: yesterday_older_datetime |> NaiveDateTime.to_iso8601(),
+              key: Ecto.UUID.generate(),
+              size: "88"
+            },
+            %{last_modified: yesterday_datetime |> NaiveDateTime.to_iso8601(), key: Ecto.UUID.generate(), size: "89"},
+            %{last_modified: today_datetime |> NaiveDateTime.to_iso8601(), key: Ecto.UUID.generate(), size: "100"}
           ]
         }
       }
@@ -199,19 +211,19 @@ defmodule Transport.Test.Transport.Jobs.DatabaseBackupReplicationJobTest do
 
         assert config_is_source?(config)
 
-        recent_datetime = NaiveDateTime.utc_now() |> NaiveDateTime.add(-60 * 60 * 2, :second)
-        older_datetime = recent_datetime |> NaiveDateTime.add(-60 * 60 * 10, :second)
+        today_datetime = NaiveDateTime.utc_now() |> NaiveDateTime.add(-60 * 60 * 2, :second)
+        yesterday_datetime = today_datetime |> NaiveDateTime.add(-1, :day)
 
         %{
           body: %{
             contents: [
               %{
-                last_modified: older_datetime |> NaiveDateTime.to_iso8601(),
+                last_modified: yesterday_datetime |> NaiveDateTime.to_iso8601(),
                 key: Ecto.UUID.generate(),
                 size: to_string(yesterday_size)
               },
               %{
-                last_modified: recent_datetime |> NaiveDateTime.to_iso8601(),
+                last_modified: today_datetime |> NaiveDateTime.to_iso8601(),
                 key: Ecto.UUID.generate(),
                 size: to_string(today_size)
               }