From 1b742c5d7ebfd5bab1b178fa2f590b1886c6ee60 Mon Sep 17 00:00:00 2001 From: luciajanikova <19lucia99@gmail.com> Date: Mon, 9 Dec 2024 20:21:59 +0100 Subject: [PATCH] Add setting to Govbox::Folder & allow to fetch only messages from folder page & message ID --- app/jobs/govbox/sync_folder_job.rb | 7 +- app/models/govbox/folder.rb | 1 + ...09184812_add_seetings_to_govbox_folders.rb | 5 + db/schema.rb | 3 +- test/jobs/govbox/sync_folder_job_test.rb | 157 +++++++++++++++++- 5 files changed, 166 insertions(+), 7 deletions(-) create mode 100644 db/migrate/20241209184812_add_seetings_to_govbox_folders.rb diff --git a/app/jobs/govbox/sync_folder_job.rb b/app/jobs/govbox/sync_folder_job.rb index 348b292e5..144005301 100644 --- a/app/jobs/govbox/sync_folder_job.rb +++ b/app/jobs/govbox/sync_folder_job.rb @@ -5,8 +5,10 @@ def perform(folder, upvs_client: UpvsEnvironment.upvs_client, batch_size: 1000) new_messages_ids = [] sync_since = Date.parse(folder.box.settings['sync_since']) if folder.box.settings['sync_since'].present? + sync_from_page = folder.settings['sync_from_page'] || 0 + sync_from_message_id = folder.settings['sync_from_message_id'] - 0.step do |k| + sync_from_page.step do |k| response_status, raw_messages = edesk_api.fetch_messages(folder.edesk_folder_id, page: k + 1, count: batch_size) raise "Unable to fetch folder messages" if response_status != 200 @@ -18,6 +20,9 @@ def perform(folder, upvs_client: UpvsEnvironment.upvs_client, batch_size: 1000) next if sync_since && (Date.parse(raw_message['delivered_at']) < sync_since) edesk_message_id = raw_message['id'] + + raise "MessageID out of order!" if sync_from_message_id && edesk_message_id < sync_from_message_id + old_folder_id = edesk_message_ids_to_folder_ids[edesk_message_id] if old_folder_id.nil? diff --git a/app/models/govbox/folder.rb b/app/models/govbox/folder.rb index c2e6c325a..0d6163cca 100644 --- a/app/models/govbox/folder.rb +++ b/app/models/govbox/folder.rb @@ -4,6 +4,7 @@ # # id :bigint not null, primary key # name :string not null +# settings :jsonb # system :boolean not null # created_at :datetime not null # updated_at :datetime not null diff --git a/db/migrate/20241209184812_add_seetings_to_govbox_folders.rb b/db/migrate/20241209184812_add_seetings_to_govbox_folders.rb new file mode 100644 index 000000000..a576bd7d9 --- /dev/null +++ b/db/migrate/20241209184812_add_seetings_to_govbox_folders.rb @@ -0,0 +1,5 @@ +class AddSeetingsToGovboxFolders < ActiveRecord::Migration[7.1] + def change + add_column :govbox_folders, :settings, :jsonb, default: {} + end +end diff --git a/db/schema.rb b/db/schema.rb index ba2de7a6b..b8f06be9a 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_11_06_152337) do +ActiveRecord::Schema[7.1].define(version: 2024_12_09_184812) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -323,6 +323,7 @@ t.bigint "box_id", null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false + t.jsonb "settings", default: {} t.index ["box_id"], name: "index_govbox_folders_on_box_id" t.index ["edesk_folder_id", "box_id"], name: "index_govbox_folders_on_edesk_folder_id_and_box_id", unique: true t.index ["parent_folder_id"], name: "index_govbox_folders_on_parent_folder_id" diff --git a/test/jobs/govbox/sync_folder_job_test.rb b/test/jobs/govbox/sync_folder_job_test.rb index 069793fb0..ef8b149c4 100644 --- a/test/jobs/govbox/sync_folder_job_test.rb +++ b/test/jobs/govbox/sync_folder_job_test.rb @@ -2,7 +2,97 @@ class Govbox::SyncFolderJobTest < ActiveJob::TestCase test "downloads all messages unless box sync_since value in box settings set" do + folder = govbox_folders(:ssd_one) + + edesk_api_mock = Minitest::Mock.new + edesk_api_mock.expect :fetch_messages, [200, [ + { + "id"=>4905707493, + "class"=>"ED_DELIVERY_NOTIFICATION", + "message_id"=>"7cb378ef-9c76-493b-b41d-51f1e40dd68e", + "correlation_id"=>"6baa28c3-96e5-493b-a76c-9837a6d637b3", + "subject"=>"Notifikácia o doručení k \"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia\"", + "delivered_at"=>"2023-07-10T09:51:36.533Z" + }, + { + "id"=>4906445420, + "class"=>"EGOV_DOCUMENT", + "message_id"=>"1fe26465-ff59-4fed-b948-630cc994507a", + "correlation_id"=>"5aeddd85-952e-4534-a9fd-9c73774049f3", + "subject"=>"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia", + "delivered_at"=>"2023-07-14T12:51:50.337Z" + }, + { + "id"=>4906376126, + "class"=>"EGOV_DOCUMENT", + "message_id"=>"1d5f449c-7b8e-40d6-9662-64ff0af527cd", + "correlation_id"=>"7f2956fd-d8d8-4062-9d34-f9b0418c9f0b", + "subject"=>"Všeobecná agenda - rozhodnutie do vlastných rúk", + "delivered_at"=>"2023-07-15T14:16:32.600Z" + }, + { + "id"=>4905707496, + "class"=>"ED_DELIVERY_NOTIFICATION", + "message_id"=>"14fe42a3-c3b7-419b-bb76-485f49543e53", + "correlation_id"=>"8c359d95-3d72-4e76-b0f2-ddeba11c3b5b", + "subject"=>"Notifikácia o doručení k \"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia\"", + "delivered_at"=>"2023-08-10T09:51:36.560Z" + }, + { + "id"=>4905707497, + "class"=>"ED_DELIVERY_NOTIFICATION", + "message_id"=>"14fe42a3-c3b7-419b-bb76-485f49543e53", + "correlation_id"=>"8c359d95-3d72-4e76-b0f2-ddeba11c3b5b", + "subject"=>"Notifikácia o doručení k \"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia\"", + "delivered_at"=>"2023-08-11T09:51:36.560Z" + }, + ]], [folder.edesk_folder_id], **{ page: 1, count: 5 } + edesk_api_mock.expect :fetch_messages, [200, [ + { + "id"=>4905707493, + "class"=>"ED_DELIVERY_NOTIFICATION", + "message_id"=>"7cb378ef-9c76-493b-b41d-51f1e40dd68e", + "correlation_id"=>"6baa28c3-96e5-493b-a76c-9837a6d637b3", + "subject"=>"Notifikácia o doručení k \"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia\"", + "delivered_at"=>"2023-08-12T09:51:36.533Z" + }, + { + "id"=>4906445420, + "class"=>"EGOV_DOCUMENT", + "message_id"=>"1fe26465-ff59-4fed-b948-630cc994507a", + "correlation_id"=>"5aeddd85-952e-4534-a9fd-9c73774049f3", + "subject"=>"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia", + "delivered_at"=>"2023-08-14T12:51:50.337Z" + }, + { + "id"=>4906376126, + "class"=>"EGOV_DOCUMENT", + "message_id"=>"1d5f449c-7b8e-40d6-9662-64ff0af527cd", + "correlation_id"=>"7f2956fd-d8d8-4062-9d34-f9b0418c9f0b", + "subject"=>"Všeobecná agenda - rozhodnutie do vlastných rúk", + "delivered_at"=>"2023-11-15T14:16:32.600Z" + }, + { + "id"=>4905707496, + "class"=>"ED_DELIVERY_NOTIFICATION", + "message_id"=>"14fe42a3-c3b7-419b-bb76-485f49543e53", + "correlation_id"=>"8c359d95-3d72-4e76-b0f2-ddeba11c3b5b", + "subject"=>"Notifikácia o doručení k \"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia\"", + "delivered_at"=>"2024-07-10T09:51:36.560Z" + } + ]], [folder.edesk_folder_id], **{ page: 2, count: 5 } + + ::Upvs::GovboxApi::Edesk.stub :new, edesk_api_mock do + Govbox::SyncFolderJob.new.perform(folder, batch_size: 5) + end + + assert_enqueued_jobs 9 + end + + test "does not download older messages than box sync_since value in box settings" do box = boxes(:ssd_main) + box.settings['sync_since'] = '2023-11-15' + box.save folder = govbox_folders(:ssd_one) @@ -46,15 +136,18 @@ class Govbox::SyncFolderJobTest < ActiveJob::TestCase Govbox::SyncFolderJob.new.perform(folder) end - assert_enqueued_jobs 4 + assert_enqueued_jobs 2 end - test "does not download older messages than box sync_since value in box settings" do + test "starts fetching messages from selected folder page and downloads only messages from selected ID (folder settings)" do box = boxes(:ssd_main) box.settings['sync_since'] = '2023-11-15' box.save folder = govbox_folders(:ssd_one) + folder.settings['sync_from_page'] = 9 + folder.settings['sync_from_message_id'] = 4905707495 + folder.save edesk_api_mock = Minitest::Mock.new edesk_api_mock.expect :fetch_messages, [200, [ @@ -67,7 +160,7 @@ class Govbox::SyncFolderJobTest < ActiveJob::TestCase "delivered_at"=>"2023-07-10T09:51:36.533Z" }, { - "id"=>4906445420, + "id"=>4905707494, "class"=>"EGOV_DOCUMENT", "message_id"=>"1fe26465-ff59-4fed-b948-630cc994507a", "correlation_id"=>"5aeddd85-952e-4534-a9fd-9c73774049f3", @@ -75,7 +168,7 @@ class Govbox::SyncFolderJobTest < ActiveJob::TestCase "delivered_at"=>"2023-11-14T12:51:50.337Z" }, { - "id"=>4906376126, + "id"=>4905707495, "class"=>"EGOV_DOCUMENT", "message_id"=>"1d5f449c-7b8e-40d6-9662-64ff0af527cd", "correlation_id"=>"7f2956fd-d8d8-4062-9d34-f9b0418c9f0b", @@ -90,7 +183,8 @@ class Govbox::SyncFolderJobTest < ActiveJob::TestCase "subject"=>"Notifikácia o doručení k \"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia\"", "delivered_at"=>"2024-07-10T09:51:36.560Z" }, - ]], [folder.edesk_folder_id], **{ page: 1, count: 1000 } + ]], [folder.edesk_folder_id], **{ page: 10, count: 1000 } + ::Upvs::GovboxApi::Edesk.stub :new, edesk_api_mock do Govbox::SyncFolderJob.new.perform(folder) @@ -98,4 +192,57 @@ class Govbox::SyncFolderJobTest < ActiveJob::TestCase assert_enqueued_jobs 2 end + + test "raises if downloading messages from selected ID and smaller ID is found (box settings)" do + box = boxes(:ssd_main) + box.settings['sync_since'] = '2023-11-15' + box.save + + folder = govbox_folders(:ssd_one) + folder.settings['sync_from_page'] = 9 + folder.settings['sync_from_message_id'] = 4906376126 + folder.save + + edesk_api_mock = Minitest::Mock.new + edesk_api_mock.expect :fetch_messages, [200, [ + { + "id"=>4905707493, + "class"=>"ED_DELIVERY_NOTIFICATION", + "message_id"=>"7cb378ef-9c76-493b-b41d-51f1e40dd68e", + "correlation_id"=>"6baa28c3-96e5-493b-a76c-9837a6d637b3", + "subject"=>"Notifikácia o doručení k \"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia\"", + "delivered_at"=>"2023-07-10T09:51:36.533Z" + }, + { + "id"=>4906445420, + "class"=>"EGOV_DOCUMENT", + "message_id"=>"1fe26465-ff59-4fed-b948-630cc994507a", + "correlation_id"=>"5aeddd85-952e-4534-a9fd-9c73774049f3", + "subject"=>"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia", + "delivered_at"=>"2023-11-14T12:51:50.337Z" + }, + { + "id"=>4906376126, + "class"=>"EGOV_DOCUMENT", + "message_id"=>"1d5f449c-7b8e-40d6-9662-64ff0af527cd", + "correlation_id"=>"7f2956fd-d8d8-4062-9d34-f9b0418c9f0b", + "subject"=>"Všeobecná agenda - rozhodnutie do vlastných rúk", + "delivered_at"=>"2023-11-15T14:16:32.600Z" + }, + { + "id"=>4905707496, + "class"=>"ED_DELIVERY_NOTIFICATION", + "message_id"=>"14fe42a3-c3b7-419b-bb76-485f49543e53", + "correlation_id"=>"8c359d95-3d72-4e76-b0f2-ddeba11c3b5b", + "subject"=>"Notifikácia o doručení k \"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia\"", + "delivered_at"=>"2024-07-10T09:51:36.560Z" + }, + ]], [folder.edesk_folder_id], **{ page: 10, count: 1000 } + + assert_raise('MessageID out of order!') do + ::Upvs::GovboxApi::Edesk.stub :new, edesk_api_mock do + Govbox::SyncFolderJob.new.perform(folder) + end + end + end end