From e73faae8795f287cd0ee72a414df6b6dd6a44bf0 Mon Sep 17 00:00:00 2001 From: Sergei Tsoganov Date: Thu, 14 Dec 2023 13:26:31 +0200 Subject: [PATCH 01/10] Refactored statistics caclulation --- app/controllers/repp/v1/stats_controller.rb | 71 ++++++++++++++----- test/fixtures/domains.yml | 4 +- test/fixtures/log_domains.yml | 48 ++++++++++++- .../repp/v1/stats/market_share_test.rb | 17 ++--- 4 files changed, 111 insertions(+), 29 deletions(-) diff --git a/app/controllers/repp/v1/stats_controller.rb b/app/controllers/repp/v1/stats_controller.rb index 052cdcbc45..9f7b60f1c3 100644 --- a/app/controllers/repp/v1/stats_controller.rb +++ b/app/controllers/repp/v1/stats_controller.rb @@ -36,6 +36,7 @@ def market_share_growth_rate data: { name: search_params[:end_date], domains: serialize_growth_rate_result(domains_by_rar), market_share: serialize_growth_rate_result(market_share_by_rar) } } + render_success(data: result) end # rubocop:enable Metrics/MethodLength @@ -49,9 +50,9 @@ def search_params def set_date_params @date_to = to_date(search_params[:end_date]).end_of_month - @date_from = to_date(search_params[:start_date] || '01.05') + @date_from = to_date(search_params[:start_date] || '01.00') @date_compare_to = to_date(search_params[:compare_to_end_date]).end_of_month - @date_compare_from = to_date(search_params[:compare_to_start_date] || '01.05') + @date_compare_from = to_date(search_params[:compare_to_start_date] || '01.00') end def to_date(date_param) @@ -78,15 +79,49 @@ def calculate_market_share(domains_by_rar) end end + # def domains_by_registrar(date_to, date_from) + # log_domains_del = log_domains(event: 'destroy', date_to: date_to, date_from: date_from) + # log_domains_trans = log_domains(event: 'update', date_to: date_to, date_from: date_from) + # logged_domains = log_domains_trans.map { |ld| ld.object['name'] } + + # log_domains_del.map { |ld| ld.object['name'] } + # domains_grouped = ::Domain.where('created_at <= ? AND created_at >= ?', date_to, date_from) + # .where.not(name: logged_domains.uniq) + # .group(:registrar_id).count.stringify_keys + + # summarize([group(log_domains_del), group(log_domains_trans), domains_grouped]) + # end + def domains_by_registrar(date_to, date_from) - log_domains_del = log_domains(event: 'destroy', date_to: date_to, date_from: date_from) - log_domains_trans = log_domains(event: 'update', date_to: date_to, date_from: date_from) - logged_domains = log_domains_trans.map { |ld| ld.object['name'] } + - log_domains_del.map { |ld| ld.object['name'] } - domains_grouped = ::Domain.where('created_at <= ? AND created_at >= ?', date_to, date_from) - .where.not(name: logged_domains.uniq) - .group(:registrar_id).count.stringify_keys - summarize([group(log_domains_del), group(log_domains_trans), domains_grouped]) + domain_versions = ::Version::DomainVersion.where('object_changes IS NOT NULL') + .where('created_at >= ? AND created_at <= ?', date_from, date_to) + registrar_counts = Hash.new(0) + processed_domains = [] + domain_versions.find_each(batch_size: 1000) do |v| + registrar_ids = v.object_changes['registrar_id'] + next if registrar_ids.nil? || registrar_ids.empty? + + case v.event + when 'create' + processed_domains << v.object_changes['name'][1] + registrar_counts[registrar_ids[1].to_s] += 1 + when 'update' + if processed_domains.include?(v.object['name']) + registrar_counts[registrar_ids[0].to_s] -= 1 + registrar_counts[registrar_ids[1].to_s] += 1 + else + registrar_counts[registrar_ids[1].to_s] += 1 + processed_domains << v.object['name'] + end + when 'destroy' + registrar_counts[registrar_ids[0].to_s] -= 1 + end + end + + current_domains_grouped = ::Domain.where('created_at <= ? AND created_at >= ?', date_to, date_from) + .where.not(name: processed_domains.uniq) + .group(:registrar_id).count.stringify_keys + + summarize([registrar_counts, current_domains_grouped]) end def summarize(arr) @@ -94,14 +129,14 @@ def summarize(arr) end def log_domains(event:, date_to:, date_from:) - domains = ::Version::DomainVersion.where(event: event) - domains.where!("object_changes ->> 'registrar_id' IS NOT NULL") if event == 'update' - domains.where('created_at > ?', date_to) - .where("object ->> 'created_at' <= ?", date_to) - .where("object ->> 'created_at' >= ?", date_from) - .select("DISTINCT ON (object ->> 'name') object, created_at") - .order(Arel.sql("object ->> 'name', created_at desc")) - end + domain_version = ::Version::DomainVersion.where(event: event) + domain_version.where!("object_changes ->> 'registrar_id' IS NOT NULL") if event == 'update' + domain_version.where('created_at > ?', date_to) + .where("object ->> 'created_at' <= ?", date_to) + .where("object ->> 'created_at' >= ?", date_from) + .select("DISTINCT ON (object ->> 'name') object, created_at") + .order(Arel.sql("object ->> 'name', created_at desc")) + end def group(domains) domains.group_by { |ld| ld.object['registrar_id'].to_s } diff --git a/test/fixtures/domains.yml b/test/fixtures/domains.yml index 01c43d644d..770559bdf0 100644 --- a/test/fixtures/domains.yml +++ b/test/fixtures/domains.yml @@ -12,7 +12,7 @@ shop: period: 1 period_unit: m uuid: 1b3ee442-e8fe-4922-9492-8fcb9dccc69c - created_at: <%= 2.days.ago.to_s :db %> + created_at: <%= 2.months.ago.to_s :db %> airport: name: airport.test @@ -51,7 +51,7 @@ metro: period: 1 period_unit: m uuid: ef97cb80-333b-4893-b9df-163f2b452798 - created_at: <%= 2.days.ago.to_s :db %> + created_at: <%= 3.months.ago.to_s :db %> hospital: name: hospital.test diff --git a/test/fixtures/log_domains.yml b/test/fixtures/log_domains.yml index f328f902a6..3937f5576c 100644 --- a/test/fixtures/log_domains.yml +++ b/test/fixtures/log_domains.yml @@ -5,4 +5,50 @@ one: object: registrant_id: <%= ActiveRecord::FixtureSet.identify(:john) %> updated_at: <%= Time.zone.parse('2010-07-05') %> - created_at: <%= Time.zone.parse('2010-07-05') %> \ No newline at end of file + created_at: <%= Time.zone.parse('2010-07-05') %> + +transfer_one: + item_id: <%= ActiveRecord::FixtureSet.identify(:shop) %> + item_type: Domain + event: update + object: + name: 'shop.test' + object_changes: + registrar_id: [<%= ActiveRecord::FixtureSet.identify(:goodnames) %>, <%= ActiveRecord::FixtureSet.identify(:bestnames) %>] + created_at: <%= 2.days.ago.to_s :db %> + +create_one: + item_id: 1111111 + item_type: Domain + event: create + object_changes: + name: [null, 'deleted.test'] + registrar_id: [null, <%= ActiveRecord::FixtureSet.identify(:goodnames) %>] + created_at: <%= 3.months.ago.to_s :db %> + +destroy_one: + item_id: 1111111 + item_type: Domain + event: destroy + object_changes: + name: ['deleted.test', null] + registrar_id: [<%= ActiveRecord::FixtureSet.identify(:goodnames) %>, null] + created_at: <%= 2.days.ago.to_s :db %> + +create_two: + item_id: <%= ActiveRecord::FixtureSet.identify(:library) %> + item_type: Domain + event: create + object_changes: + name: [null, 'library.test'] + registrar_id: [null, <%= ActiveRecord::FixtureSet.identify(:bestnames) %>] + created_at: <%= 2.days.ago.to_s :db %> + +create_three: + item_id: <%= ActiveRecord::FixtureSet.identify(:airport) %> + item_type: Domain + event: create + object_changes: + name: [null, 'airport.test'] + registrar_id: [null, <%= ActiveRecord::FixtureSet.identify(:bestnames) %>] + created_at: <%= 2.days.ago.to_s :db %> \ No newline at end of file diff --git a/test/integration/repp/v1/stats/market_share_test.rb b/test/integration/repp/v1/stats/market_share_test.rb index d3acba89b9..9de498116b 100644 --- a/test/integration/repp/v1/stats/market_share_test.rb +++ b/test/integration/repp/v1/stats/market_share_test.rb @@ -19,8 +19,8 @@ def test_shows_market_share_distribution_data assert_equal 1000, json[:code] assert_equal 'Command completed successfully', json[:message] - assert json[:data].is_a? Array - assert json[:data][0].is_a? Hash + assert_equal json[:data], [{ name: @user.registrar.name, y: 4, sliced: true, selected: true }, + { name: 'Good Names', y: 2 }] end def test_shows_market_share_growth_rate_data @@ -34,10 +34,11 @@ def test_shows_market_share_growth_rate_data assert_equal 1000, json[:code] assert_equal 'Command completed successfully', json[:message] - data = json[:data] - assert data[:data].is_a? Hash - assert data[:prev_data].is_a? Hash - assert_equal data[:data][:name], @today - assert data[:data][:domains].is_a? Array + assert_equal json[:data], prev_data: { name: prev_date, + domains: [['Best Names', 1], ['Good Names', 2]], + market_share: [['Best Names', 33.3], ['Good Names', 66.7]] }, + data: { name: @today, + domains: [['Best Names', 4], ['Good Names', 2]], + market_share: [['Best Names', 66.7], ['Good Names', 33.3]] } end -end \ No newline at end of file +end From 639da20cc60bf7fee9ee08c5cd5ed33d682fb3e9 Mon Sep 17 00:00:00 2001 From: Sergei Tsoganov Date: Thu, 14 Dec 2023 15:46:37 +0200 Subject: [PATCH 02/10] Updated fixtures --- test/fixtures/files/domain_versions.csv | 5 +++++ test/fixtures/log_domains.yml | 26 +++++++++++++++---------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/test/fixtures/files/domain_versions.csv b/test/fixtures/files/domain_versions.csv index 8fa7fa9fe8..876e021553 100644 --- a/test/fixtures/files/domain_versions.csv +++ b/test/fixtures/files/domain_versions.csv @@ -1,3 +1,8 @@ Name,Registrant,Registrar,Action,Created at +,John,,destroy,2023-12-04 22:00:00 +shop.test,John,Best Names,update,2023-12-04 22:00:00 +library.test,Acme Ltd,Best Names,create,2023-12-04 22:00:00 +metro.test,Jack,Good Names,create,2023-09-04 21:00:00 +cinema.test,John,Good Names,create,2023-09-04 21:00:00 ,test_code,Best Names,update,2018-04-23 15:50:48 ,John,,update,2010-07-04 21:00:00 diff --git a/test/fixtures/log_domains.yml b/test/fixtures/log_domains.yml index 3937f5576c..fd9e9c6f17 100644 --- a/test/fixtures/log_domains.yml +++ b/test/fixtures/log_domains.yml @@ -13,27 +13,31 @@ transfer_one: event: update object: name: 'shop.test' + registrant_id: <%= ActiveRecord::FixtureSet.identify(:john) %> object_changes: registrar_id: [<%= ActiveRecord::FixtureSet.identify(:goodnames) %>, <%= ActiveRecord::FixtureSet.identify(:bestnames) %>] - created_at: <%= 2.days.ago.to_s :db %> + created_at: <%= Time.zone.parse('2023-12-05') %> create_one: item_id: 1111111 item_type: Domain event: create object_changes: - name: [null, 'deleted.test'] + name: [null, 'cinema.test'] registrar_id: [null, <%= ActiveRecord::FixtureSet.identify(:goodnames) %>] - created_at: <%= 3.months.ago.to_s :db %> + registrant_id: [null, <%= ActiveRecord::FixtureSet.identify(:john) %>] + created_at: <%= Time.zone.parse('2023-09-05') %> destroy_one: item_id: 1111111 item_type: Domain event: destroy + object: + registrant_id: <%= ActiveRecord::FixtureSet.identify(:john) %> object_changes: - name: ['deleted.test', null] + name: ['cinema.test', null] registrar_id: [<%= ActiveRecord::FixtureSet.identify(:goodnames) %>, null] - created_at: <%= 2.days.ago.to_s :db %> + created_at: <%= Time.zone.parse('2023-12-05') %> create_two: item_id: <%= ActiveRecord::FixtureSet.identify(:library) %> @@ -42,13 +46,15 @@ create_two: object_changes: name: [null, 'library.test'] registrar_id: [null, <%= ActiveRecord::FixtureSet.identify(:bestnames) %>] - created_at: <%= 2.days.ago.to_s :db %> + registrant_id: [null, <%= ActiveRecord::FixtureSet.identify(:acme_ltd) %>] + created_at: <%= Time.zone.parse('2023-12-05') %> create_three: - item_id: <%= ActiveRecord::FixtureSet.identify(:airport) %> + item_id: <%= ActiveRecord::FixtureSet.identify(:metro) %> item_type: Domain event: create object_changes: - name: [null, 'airport.test'] - registrar_id: [null, <%= ActiveRecord::FixtureSet.identify(:bestnames) %>] - created_at: <%= 2.days.ago.to_s :db %> \ No newline at end of file + name: [null, 'metro.test'] + registrar_id: [null, <%= ActiveRecord::FixtureSet.identify(:goodnames) %>] + registrant_id: [null, <%= ActiveRecord::FixtureSet.identify(:jack) %>] + created_at: <%= Time.zone.parse('2023-09-05') %> \ No newline at end of file From ea48a1cca1dfd6ee08044a99a02323c496c1b2c7 Mon Sep 17 00:00:00 2001 From: Sergei Tsoganov Date: Thu, 14 Dec 2023 20:55:17 +0200 Subject: [PATCH 03/10] Removed batching of log_domain records --- app/controllers/repp/v1/stats_controller.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/controllers/repp/v1/stats_controller.rb b/app/controllers/repp/v1/stats_controller.rb index 9f7b60f1c3..ede70d335d 100644 --- a/app/controllers/repp/v1/stats_controller.rb +++ b/app/controllers/repp/v1/stats_controller.rb @@ -96,7 +96,8 @@ def domains_by_registrar(date_to, date_from) .where('created_at >= ? AND created_at <= ?', date_from, date_to) registrar_counts = Hash.new(0) processed_domains = [] - domain_versions.find_each(batch_size: 1000) do |v| + Rails.logger.info "Processing total #{domain_versions.size} log_domain records" + domain_versions.each do |v| registrar_ids = v.object_changes['registrar_id'] next if registrar_ids.nil? || registrar_ids.empty? From 533b10aad1391e84af13b808f31fd8d05f573ad5 Mon Sep 17 00:00:00 2001 From: Sergei Tsoganov Date: Thu, 14 Dec 2023 21:19:07 +0200 Subject: [PATCH 04/10] Temporarily changed date_from for stats --- app/controllers/repp/v1/stats_controller.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/controllers/repp/v1/stats_controller.rb b/app/controllers/repp/v1/stats_controller.rb index ede70d335d..f82ca792a0 100644 --- a/app/controllers/repp/v1/stats_controller.rb +++ b/app/controllers/repp/v1/stats_controller.rb @@ -50,9 +50,9 @@ def search_params def set_date_params @date_to = to_date(search_params[:end_date]).end_of_month - @date_from = to_date(search_params[:start_date] || '01.00') + @date_from = to_date(search_params[:start_date] || '01.22') @date_compare_to = to_date(search_params[:compare_to_end_date]).end_of_month - @date_compare_from = to_date(search_params[:compare_to_start_date] || '01.00') + @date_compare_from = to_date(search_params[:compare_to_start_date] || '01.22') end def to_date(date_param) @@ -94,6 +94,7 @@ def calculate_market_share(domains_by_rar) def domains_by_registrar(date_to, date_from) domain_versions = ::Version::DomainVersion.where('object_changes IS NOT NULL') .where('created_at >= ? AND created_at <= ?', date_from, date_to) + .select(:event, :object, :object_changes) registrar_counts = Hash.new(0) processed_domains = [] Rails.logger.info "Processing total #{domain_versions.size} log_domain records" From c1e48197cbf52b7cd90cf45bc14065d7be147a18 Mon Sep 17 00:00:00 2001 From: Sergei Tsoganov Date: Fri, 15 Dec 2023 15:49:29 +0200 Subject: [PATCH 05/10] Refactored sql query for stats calculation --- app/controllers/repp/v1/stats_controller.rb | 255 +++++++++++++++++--- 1 file changed, 222 insertions(+), 33 deletions(-) diff --git a/app/controllers/repp/v1/stats_controller.rb b/app/controllers/repp/v1/stats_controller.rb index f82ca792a0..ac840bb4b6 100644 --- a/app/controllers/repp/v1/stats_controller.rb +++ b/app/controllers/repp/v1/stats_controller.rb @@ -79,6 +79,195 @@ def calculate_market_share(domains_by_rar) end end + def domains_by_registrar(date_to, date_from) + sql = <<-SQL + SELECT + registrar_id, + SUM(domain_count) AS total_domain_count + FROM ( + -- Count domains from the domains table, excluding those in created_domains + SELECT + registrar_id::text AS registrar_id, + COUNT(*) AS domain_count + FROM + domains + WHERE + created_at >= :date_from + GROUP BY + registrar_id + + UNION ALL + + SELECT + (object_changes->'registrar_id'->>1)::text AS registrar_id, + COUNT(*) * -1 AS domain_count + FROM + log_domains + WHERE + event = 'create' + AND created_at > :date_to + GROUP BY + registrar_id + + UNION ALL + + -- Query for 'update' events and count domains transferred to a new registrar only for domains in created_domains + SELECT + (object_changes->'registrar_id'->>1)::text AS registrar_id, + COUNT(*) * -1 AS domain_count + FROM + log_domains + WHERE + event = 'update' + AND object_changes->'registrar_id' IS NOT NULL + AND created_at > :date_to + GROUP BY + registrar_id + + UNION ALL + + -- Query for 'update' events and count domains transferred from an old registrar only for domains in created_domains + SELECT + (object_changes->'registrar_id'->>0)::text AS registrar_id, + COUNT(*) AS domain_count + FROM + log_domains + WHERE + event = 'update' + AND object_changes->'registrar_id' IS NOT NULL + AND created_at > :date_to + GROUP BY + registrar_id + + UNION ALL + + -- Query for 'destroy' events and count the number of domains destroyed associated with each registrar only for domains in created_domains + SELECT + (object_changes->'registrar_id'->>0)::text AS registrar_id, + COUNT(*) AS domain_count + FROM + log_domains + WHERE + event = 'destroy' + AND created_at > :date_to + GROUP BY + registrar_id + + ) AS combined + GROUP BY + registrar_id; + SQL + + results = ActiveRecord::Base.connection.execute( + ActiveRecord::Base.send(:sanitize_sql_array, [sql, date_from: date_from, date_to: date_to]) + ).each_with_object({}) do |row, hash| + hash[row['registrar_id']] = row['total_domain_count'].to_i + end + + results + end + + # def domains_by_registrar(date_to, date_from) + # p 'Count the number of domains created by each registrar' + # sql = <<-SQL + # WITH created_domains AS ( + # SELECT + # (object_changes->'registrar_id'->>1)::text AS registrar_id, + # (object_changes->'name'->>1)::text AS domain_name + # FROM + # log_domains + # WHERE + # event = 'create' + # AND created_at BETWEEN :date_from AND :date_to + # ) + # SELECT + # registrar_id, + # SUM(domain_count) AS total_domain_count + # FROM ( + # -- Count domains from created_domains + # SELECT + # registrar_id, + # COUNT(*) AS domain_count + # FROM + # created_domains + # GROUP BY + # registrar_id + + # UNION ALL + + # -- Query for 'update' events and count domains transferred to a new registrar only for domains in created_domains + # SELECT + # (object_changes->'registrar_id'->>1)::text AS registrar_id, + # COUNT(*) AS domain_count + # FROM + # log_domains + # WHERE + # event = 'update' + # AND object_changes->'registrar_id' IS NOT NULL + # AND (object->'name')::text IN (SELECT domain_name FROM created_domains) + # AND created_at BETWEEN :date_from AND :date_to + # GROUP BY + # registrar_id + + # UNION ALL + + # -- Query for 'update' events and count domains transferred from an old registrar only for domains in created_domains + # SELECT + # (object_changes->'registrar_id'->>0)::text AS registrar_id, + # COUNT(*) * -1 AS domain_count + # FROM + # log_domains + # WHERE + # event = 'update' + # AND object_changes->'registrar_id' IS NOT NULL + # AND (object->'name')::text IN (SELECT domain_name FROM created_domains) + # AND created_at BETWEEN :date_from AND :date_to + # GROUP BY + # registrar_id + + # UNION ALL + + # -- Query for 'destroy' events and count the number of domains destroyed associated with each registrar only for domains in created_domains + # SELECT + # (object_changes->'registrar_id'->>0)::text AS registrar_id, + # COUNT(*) * -1 AS domain_count + # FROM + # log_domains + # WHERE + # event = 'destroy' + # AND (object_changes->'name'->>0)::text IN (SELECT domain_name FROM created_domains) + # AND created_at BETWEEN :date_from AND :date_to + # GROUP BY + # registrar_id + + # UNION ALL + + # -- Count domains from the domains table, excluding those in created_domains + # SELECT + # registrar_id::text AS registrar_id, + # COUNT(*) AS domain_count + # FROM + # domains + # WHERE + # name NOT IN (SELECT domain_name FROM created_domains) + # AND created_at BETWEEN :date_from AND :date_to + # GROUP BY + # registrar_id + + # ) AS combined + # GROUP BY + # registrar_id; + # SQL + + # results = ActiveRecord::Base.connection.execute( + # ActiveRecord::Base.send(:sanitize_sql_array, [sql, date_from: date_from, date_to: date_to]) + # ).each_with_object({}) do |row, hash| + # hash[row['registrar_id']] = row['total_domain_count'].to_i + # end + + # p results + # end + # def domains_by_registrar(date_to, date_from) # log_domains_del = log_domains(event: 'destroy', date_to: date_to, date_from: date_from) # log_domains_trans = log_domains(event: 'update', date_to: date_to, date_from: date_from) @@ -91,40 +280,40 @@ def calculate_market_share(domains_by_rar) # summarize([group(log_domains_del), group(log_domains_trans), domains_grouped]) # end - def domains_by_registrar(date_to, date_from) - domain_versions = ::Version::DomainVersion.where('object_changes IS NOT NULL') - .where('created_at >= ? AND created_at <= ?', date_from, date_to) - .select(:event, :object, :object_changes) - registrar_counts = Hash.new(0) - processed_domains = [] - Rails.logger.info "Processing total #{domain_versions.size} log_domain records" - domain_versions.each do |v| - registrar_ids = v.object_changes['registrar_id'] - next if registrar_ids.nil? || registrar_ids.empty? - - case v.event - when 'create' - processed_domains << v.object_changes['name'][1] - registrar_counts[registrar_ids[1].to_s] += 1 - when 'update' - if processed_domains.include?(v.object['name']) - registrar_counts[registrar_ids[0].to_s] -= 1 - registrar_counts[registrar_ids[1].to_s] += 1 - else - registrar_counts[registrar_ids[1].to_s] += 1 - processed_domains << v.object['name'] - end - when 'destroy' - registrar_counts[registrar_ids[0].to_s] -= 1 - end - end + # def domains_by_registrar(date_to, date_from) + # domain_versions = ::Version::DomainVersion.where('object_changes IS NOT NULL') + # .where('created_at >= ? AND created_at <= ?', date_from, date_to) + # .select(:event, :object, :object_changes) + # registrar_counts = Hash.new(0) + # processed_domains = [] + # Rails.logger.info "Processing total #{domain_versions.size} log_domain records" + # domain_versions.each do |v| + # registrar_ids = v.object_changes['registrar_id'] + # next if registrar_ids.nil? || registrar_ids.empty? - current_domains_grouped = ::Domain.where('created_at <= ? AND created_at >= ?', date_to, date_from) - .where.not(name: processed_domains.uniq) - .group(:registrar_id).count.stringify_keys + # case v.event + # when 'create' + # processed_domains << v.object_changes['name'][1] + # registrar_counts[registrar_ids[1].to_s] += 1 + # when 'update' + # if processed_domains.include?(v.object['name']) + # registrar_counts[registrar_ids[0].to_s] -= 1 + # registrar_counts[registrar_ids[1].to_s] += 1 + # else + # registrar_counts[registrar_ids[1].to_s] += 1 + # processed_domains << v.object['name'] + # end + # when 'destroy' + # registrar_counts[registrar_ids[0].to_s] -= 1 + # end + # end - summarize([registrar_counts, current_domains_grouped]) - end + # current_domains_grouped = ::Domain.where('created_at <= ? AND created_at >= ?', date_to, date_from) + # .where.not(name: processed_domains.uniq) + # .group(:registrar_id).count.stringify_keys + + # summarize([registrar_counts, current_domains_grouped]) + # end def summarize(arr) arr.inject { |memo, el| memo.merge(el) { |_, old_v, new_v| old_v + new_v } } @@ -138,7 +327,7 @@ def log_domains(event:, date_to:, date_from:) .where("object ->> 'created_at' >= ?", date_from) .select("DISTINCT ON (object ->> 'name') object, created_at") .order(Arel.sql("object ->> 'name', created_at desc")) - end + end def group(domains) domains.group_by { |ld| ld.object['registrar_id'].to_s } From 0ca07a50fbd8deea20abaa28ffd41fae2c3f7487 Mon Sep 17 00:00:00 2001 From: Sergei Tsoganov Date: Fri, 15 Dec 2023 16:19:52 +0200 Subject: [PATCH 06/10] Changed date_from default dates --- app/controllers/repp/v1/stats_controller.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/repp/v1/stats_controller.rb b/app/controllers/repp/v1/stats_controller.rb index ac840bb4b6..4b7644bd5b 100644 --- a/app/controllers/repp/v1/stats_controller.rb +++ b/app/controllers/repp/v1/stats_controller.rb @@ -50,9 +50,9 @@ def search_params def set_date_params @date_to = to_date(search_params[:end_date]).end_of_month - @date_from = to_date(search_params[:start_date] || '01.22') + @date_from = to_date(search_params[:start_date] || '01.00') @date_compare_to = to_date(search_params[:compare_to_end_date]).end_of_month - @date_compare_from = to_date(search_params[:compare_to_start_date] || '01.22') + @date_compare_from = to_date(search_params[:compare_to_start_date] || '01.00') end def to_date(date_param) From 7db8b5d970edc9fc724649db3281f29ea9a657b7 Mon Sep 17 00:00:00 2001 From: Sergei Tsoganov Date: Tue, 19 Dec 2023 09:38:00 +0200 Subject: [PATCH 07/10] Removed date_from attribute --- app/controllers/repp/v1/stats_controller.rb | 201 ++---------------- test/fixtures/files/domain_versions.csv | 8 +- test/fixtures/log_domains.yml | 46 +++- .../repp/v1/stats/market_share_test.rb | 12 +- test/learning/paper_trail_test.rb | 11 +- 5 files changed, 78 insertions(+), 200 deletions(-) diff --git a/app/controllers/repp/v1/stats_controller.rb b/app/controllers/repp/v1/stats_controller.rb index 4b7644bd5b..71e8aafb9b 100644 --- a/app/controllers/repp/v1/stats_controller.rb +++ b/app/controllers/repp/v1/stats_controller.rb @@ -9,7 +9,7 @@ class StatsController < BaseController # rubocop:disable Metrics/ClassLength param :end_date, String, required: true, desc: 'Period end date' end def market_share_distribution - domains_by_rar = domains_by_registrar(@date_to, @date_from) + domains_by_rar = domains_by_registrar(@date_to) result = serialize_distribution_result(domains_by_rar) render_success(data: result) end @@ -22,8 +22,8 @@ def market_share_distribution param :compare_to_end_date, String, required: true, desc: 'Comparison date' end def market_share_growth_rate - domains_by_rar = domains_by_registrar(@date_to, @date_from) - prev_domains_by_rar = domains_by_registrar(@date_compare_to, @date_compare_from) + domains_by_rar = domains_by_registrar(@date_to) + prev_domains_by_rar = domains_by_registrar(@date_compare_to) set_zero_values!(domains_by_rar, prev_domains_by_rar) @@ -50,9 +50,7 @@ def search_params def set_date_params @date_to = to_date(search_params[:end_date]).end_of_month - @date_from = to_date(search_params[:start_date] || '01.00') @date_compare_to = to_date(search_params[:compare_to_end_date]).end_of_month - @date_compare_from = to_date(search_params[:compare_to_start_date] || '01.00') end def to_date(date_param) @@ -79,25 +77,25 @@ def calculate_market_share(domains_by_rar) end end - def domains_by_registrar(date_to, date_from) + # rubocop:disable Metrics/MethodLength + def domains_by_registrar(date_to) sql = <<-SQL SELECT registrar_id, SUM(domain_count) AS total_domain_count FROM ( - -- Count domains from the domains table, excluding those in created_domains + -- Query for current domains for each registrar SELECT registrar_id::text AS registrar_id, COUNT(*) AS domain_count FROM domains - WHERE - created_at >= :date_from GROUP BY registrar_id UNION ALL + -- Query for 'create' events and count domains created by registrar SELECT (object_changes->'registrar_id'->>1)::text AS registrar_id, COUNT(*) * -1 AS domain_count @@ -111,7 +109,7 @@ def domains_by_registrar(date_to, date_from) UNION ALL - -- Query for 'update' events and count domains transferred to a new registrar only for domains in created_domains + -- Query for 'update' events and count domains transferred to a new registrar SELECT (object_changes->'registrar_id'->>1)::text AS registrar_id, COUNT(*) * -1 AS domain_count @@ -126,7 +124,7 @@ def domains_by_registrar(date_to, date_from) UNION ALL - -- Query for 'update' events and count domains transferred from an old registrar only for domains in created_domains + -- Query for 'update' events and count domains transferred from an old registrar SELECT (object_changes->'registrar_id'->>0)::text AS registrar_id, COUNT(*) AS domain_count @@ -141,7 +139,7 @@ def domains_by_registrar(date_to, date_from) UNION ALL - -- Query for 'destroy' events and count the number of domains destroyed associated with each registrar only for domains in created_domains + -- Query for 'destroy' events and count the number of domains destroyed associated with each registrar SELECT (object_changes->'registrar_id'->>0)::text AS registrar_id, COUNT(*) AS domain_count @@ -158,181 +156,13 @@ def domains_by_registrar(date_to, date_from) registrar_id; SQL - results = ActiveRecord::Base.connection.execute( - ActiveRecord::Base.send(:sanitize_sql_array, [sql, date_from: date_from, date_to: date_to]) + ActiveRecord::Base.connection.execute( + ActiveRecord::Base.send(:sanitize_sql_array, [sql, date_to: date_to]) ).each_with_object({}) do |row, hash| hash[row['registrar_id']] = row['total_domain_count'].to_i end - - results - end - - # def domains_by_registrar(date_to, date_from) - # p 'Count the number of domains created by each registrar' - # sql = <<-SQL - # WITH created_domains AS ( - # SELECT - # (object_changes->'registrar_id'->>1)::text AS registrar_id, - # (object_changes->'name'->>1)::text AS domain_name - # FROM - # log_domains - # WHERE - # event = 'create' - # AND created_at BETWEEN :date_from AND :date_to - # ) - # SELECT - # registrar_id, - # SUM(domain_count) AS total_domain_count - # FROM ( - # -- Count domains from created_domains - # SELECT - # registrar_id, - # COUNT(*) AS domain_count - # FROM - # created_domains - # GROUP BY - # registrar_id - - # UNION ALL - - # -- Query for 'update' events and count domains transferred to a new registrar only for domains in created_domains - # SELECT - # (object_changes->'registrar_id'->>1)::text AS registrar_id, - # COUNT(*) AS domain_count - # FROM - # log_domains - # WHERE - # event = 'update' - # AND object_changes->'registrar_id' IS NOT NULL - # AND (object->'name')::text IN (SELECT domain_name FROM created_domains) - # AND created_at BETWEEN :date_from AND :date_to - # GROUP BY - # registrar_id - - # UNION ALL - - # -- Query for 'update' events and count domains transferred from an old registrar only for domains in created_domains - # SELECT - # (object_changes->'registrar_id'->>0)::text AS registrar_id, - # COUNT(*) * -1 AS domain_count - # FROM - # log_domains - # WHERE - # event = 'update' - # AND object_changes->'registrar_id' IS NOT NULL - # AND (object->'name')::text IN (SELECT domain_name FROM created_domains) - # AND created_at BETWEEN :date_from AND :date_to - # GROUP BY - # registrar_id - - # UNION ALL - - # -- Query for 'destroy' events and count the number of domains destroyed associated with each registrar only for domains in created_domains - # SELECT - # (object_changes->'registrar_id'->>0)::text AS registrar_id, - # COUNT(*) * -1 AS domain_count - # FROM - # log_domains - # WHERE - # event = 'destroy' - # AND (object_changes->'name'->>0)::text IN (SELECT domain_name FROM created_domains) - # AND created_at BETWEEN :date_from AND :date_to - # GROUP BY - # registrar_id - - # UNION ALL - - # -- Count domains from the domains table, excluding those in created_domains - # SELECT - # registrar_id::text AS registrar_id, - # COUNT(*) AS domain_count - # FROM - # domains - # WHERE - # name NOT IN (SELECT domain_name FROM created_domains) - # AND created_at BETWEEN :date_from AND :date_to - # GROUP BY - # registrar_id - - # ) AS combined - # GROUP BY - # registrar_id; - # SQL - - # results = ActiveRecord::Base.connection.execute( - # ActiveRecord::Base.send(:sanitize_sql_array, [sql, date_from: date_from, date_to: date_to]) - # ).each_with_object({}) do |row, hash| - # hash[row['registrar_id']] = row['total_domain_count'].to_i - # end - - # p results - # end - - # def domains_by_registrar(date_to, date_from) - # log_domains_del = log_domains(event: 'destroy', date_to: date_to, date_from: date_from) - # log_domains_trans = log_domains(event: 'update', date_to: date_to, date_from: date_from) - # logged_domains = log_domains_trans.map { |ld| ld.object['name'] } + - # log_domains_del.map { |ld| ld.object['name'] } - # domains_grouped = ::Domain.where('created_at <= ? AND created_at >= ?', date_to, date_from) - # .where.not(name: logged_domains.uniq) - # .group(:registrar_id).count.stringify_keys - - # summarize([group(log_domains_del), group(log_domains_trans), domains_grouped]) - # end - - # def domains_by_registrar(date_to, date_from) - # domain_versions = ::Version::DomainVersion.where('object_changes IS NOT NULL') - # .where('created_at >= ? AND created_at <= ?', date_from, date_to) - # .select(:event, :object, :object_changes) - # registrar_counts = Hash.new(0) - # processed_domains = [] - # Rails.logger.info "Processing total #{domain_versions.size} log_domain records" - # domain_versions.each do |v| - # registrar_ids = v.object_changes['registrar_id'] - # next if registrar_ids.nil? || registrar_ids.empty? - - # case v.event - # when 'create' - # processed_domains << v.object_changes['name'][1] - # registrar_counts[registrar_ids[1].to_s] += 1 - # when 'update' - # if processed_domains.include?(v.object['name']) - # registrar_counts[registrar_ids[0].to_s] -= 1 - # registrar_counts[registrar_ids[1].to_s] += 1 - # else - # registrar_counts[registrar_ids[1].to_s] += 1 - # processed_domains << v.object['name'] - # end - # when 'destroy' - # registrar_counts[registrar_ids[0].to_s] -= 1 - # end - # end - - # current_domains_grouped = ::Domain.where('created_at <= ? AND created_at >= ?', date_to, date_from) - # .where.not(name: processed_domains.uniq) - # .group(:registrar_id).count.stringify_keys - - # summarize([registrar_counts, current_domains_grouped]) - # end - - def summarize(arr) - arr.inject { |memo, el| memo.merge(el) { |_, old_v, new_v| old_v + new_v } } - end - - def log_domains(event:, date_to:, date_from:) - domain_version = ::Version::DomainVersion.where(event: event) - domain_version.where!("object_changes ->> 'registrar_id' IS NOT NULL") if event == 'update' - domain_version.where('created_at > ?', date_to) - .where("object ->> 'created_at' <= ?", date_to) - .where("object ->> 'created_at' >= ?", date_from) - .select("DISTINCT ON (object ->> 'name') object, created_at") - .order(Arel.sql("object ->> 'name', created_at desc")) - end - - def group(domains) - domains.group_by { |ld| ld.object['registrar_id'].to_s } - .transform_values(&:count) end + # rubocop:enable Metrics/MethodLength def registrar_names @registrar_names ||= ::Registrar.where(test_registrar: false) @@ -346,7 +176,10 @@ def serialize_distribution_result(result) name = registrar_names[key] hash = { name: name, y: value } - hash.merge!({ sliced: true, selected: true }) if current_user.registrar.name == name + if current_user.registrar.name == name + hash[:sliced] = true + hash[:selected] = true + end hash end.compact end diff --git a/test/fixtures/files/domain_versions.csv b/test/fixtures/files/domain_versions.csv index 876e021553..52492e1539 100644 --- a/test/fixtures/files/domain_versions.csv +++ b/test/fixtures/files/domain_versions.csv @@ -1,8 +1,12 @@ Name,Registrant,Registrar,Action,Created at ,John,,destroy,2023-12-04 22:00:00 -shop.test,John,Best Names,update,2023-12-04 22:00:00 +hospital.test,John,Good Names,create,2023-12-04 22:00:00 library.test,Acme Ltd,Best Names,create,2023-12-04 22:00:00 -metro.test,Jack,Good Names,create,2023-09-04 21:00:00 +shop.test,John,Best Names,update,2023-12-04 22:00:00 +invalid.test,any,Best Names,create,2023-12-04 22:00:00 +airport.test,John,Best Names,create,2023-12-04 22:00:00 +shop.test,John,Good Names,create,2023-10-04 21:00:00 cinema.test,John,Good Names,create,2023-09-04 21:00:00 +metro.test,Jack,Good Names,create,2023-09-04 21:00:00 ,test_code,Best Names,update,2018-04-23 15:50:48 ,John,,update,2010-07-04 21:00:00 diff --git a/test/fixtures/log_domains.yml b/test/fixtures/log_domains.yml index fd9e9c6f17..bbd161c64e 100644 --- a/test/fixtures/log_domains.yml +++ b/test/fixtures/log_domains.yml @@ -7,6 +7,46 @@ one: updated_at: <%= Time.zone.parse('2010-07-05') %> created_at: <%= Time.zone.parse('2010-07-05') %> +create_one: + item_id: <%= ActiveRecord::FixtureSet.identify(:shop) %> + item_type: Domain + event: create + object_changes: + name: [null, 'shop.test'] + registrar_id: [null, <%= ActiveRecord::FixtureSet.identify(:goodnames) %>] + registrant_id: [null, <%= ActiveRecord::FixtureSet.identify(:john) %>] + created_at: <%= Time.zone.parse('2023-10-05') %> + +create_two: + item_id: <%= ActiveRecord::FixtureSet.identify(:airport) %> + item_type: Domain + event: create + object_changes: + name: [null, 'airport.test'] + registrar_id: [null, <%= ActiveRecord::FixtureSet.identify(:bestnames) %>] + registrant_id: [null, <%= ActiveRecord::FixtureSet.identify(:john) %>] + created_at: <%= Time.zone.parse('2023-12-05') %> + +create_three: + item_id: <%= ActiveRecord::FixtureSet.identify(:hospital) %> + item_type: Domain + event: create + object_changes: + name: [null, 'hospital.test'] + registrar_id: [null, <%= ActiveRecord::FixtureSet.identify(:goodnames) %>] + registrant_id: [null, <%= ActiveRecord::FixtureSet.identify(:john) %>] + created_at: <%= Time.zone.parse('2023-12-05') %> + +create_four: + item_id: <%= ActiveRecord::FixtureSet.identify(:invalid) %> + item_type: Domain + event: create + object_changes: + name: [null, 'invalid.test'] + registrar_id: [null, <%= ActiveRecord::FixtureSet.identify(:bestnames) %>] + registrant_id: [null, <%= ActiveRecord::FixtureSet.identify(:invalid) %>] + created_at: <%= Time.zone.parse('2023-12-05') %> + transfer_one: item_id: <%= ActiveRecord::FixtureSet.identify(:shop) %> item_type: Domain @@ -18,7 +58,7 @@ transfer_one: registrar_id: [<%= ActiveRecord::FixtureSet.identify(:goodnames) %>, <%= ActiveRecord::FixtureSet.identify(:bestnames) %>] created_at: <%= Time.zone.parse('2023-12-05') %> -create_one: +create_five: item_id: 1111111 item_type: Domain event: create @@ -39,7 +79,7 @@ destroy_one: registrar_id: [<%= ActiveRecord::FixtureSet.identify(:goodnames) %>, null] created_at: <%= Time.zone.parse('2023-12-05') %> -create_two: +create_six: item_id: <%= ActiveRecord::FixtureSet.identify(:library) %> item_type: Domain event: create @@ -49,7 +89,7 @@ create_two: registrant_id: [null, <%= ActiveRecord::FixtureSet.identify(:acme_ltd) %>] created_at: <%= Time.zone.parse('2023-12-05') %> -create_three: +create_seven: item_id: <%= ActiveRecord::FixtureSet.identify(:metro) %> item_type: Domain event: create diff --git a/test/integration/repp/v1/stats/market_share_test.rb b/test/integration/repp/v1/stats/market_share_test.rb index 9de498116b..2f129d8439 100644 --- a/test/integration/repp/v1/stats/market_share_test.rb +++ b/test/integration/repp/v1/stats/market_share_test.rb @@ -19,8 +19,8 @@ def test_shows_market_share_distribution_data assert_equal 1000, json[:code] assert_equal 'Command completed successfully', json[:message] - assert_equal json[:data], [{ name: @user.registrar.name, y: 4, sliced: true, selected: true }, - { name: 'Good Names', y: 2 }] + assert_equal json[:data], [{ name: 'Good Names', y: 2 }, + { name: @user.registrar.name, y: 4, sliced: true, selected: true }] end def test_shows_market_share_growth_rate_data @@ -35,10 +35,10 @@ def test_shows_market_share_growth_rate_data assert_equal 'Command completed successfully', json[:message] assert_equal json[:data], prev_data: { name: prev_date, - domains: [['Best Names', 1], ['Good Names', 2]], - market_share: [['Best Names', 33.3], ['Good Names', 66.7]] }, + domains: [['Good Names', 3], ['Best Names', 0]], + market_share: [['Good Names', 100.0], ['Best Names', 0.0]] }, data: { name: @today, - domains: [['Best Names', 4], ['Good Names', 2]], - market_share: [['Best Names', 66.7], ['Good Names', 33.3]] } + domains: [['Good Names', 2], ['Best Names', 4]], + market_share: [['Good Names', 33.3], ['Best Names', 66.7]] } end end diff --git a/test/learning/paper_trail_test.rb b/test/learning/paper_trail_test.rb index 9e116bf639..bd6c7993e0 100644 --- a/test/learning/paper_trail_test.rb +++ b/test/learning/paper_trail_test.rb @@ -24,13 +24,14 @@ def test_returns_version_list def test_returns_version_count_on_domains @domain = domains(:airport) - @domain.save - - assert_equal 1, @domain.versions.count + assert_difference -> { @domain.versions.count }, 1 do + @domain.save + end @domain.name = 'domain.test' - @domain.save! - assert_equal 2, @domain.versions.count + assert_difference -> { @domain.versions.count }, 1 do + @domain.save! + end end def test_returns_version_count_on_users From e68039902dd61e36d577776da6fb6565c863fe08 Mon Sep 17 00:00:00 2001 From: Sergei Tsoganov Date: Tue, 19 Dec 2023 11:09:17 +0200 Subject: [PATCH 08/10] Foxed codeclimate issue --- app/controllers/repp/v1/stats_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/repp/v1/stats_controller.rb b/app/controllers/repp/v1/stats_controller.rb index 71e8aafb9b..100ea7ab7b 100644 --- a/app/controllers/repp/v1/stats_controller.rb +++ b/app/controllers/repp/v1/stats_controller.rb @@ -157,7 +157,7 @@ def domains_by_registrar(date_to) SQL ActiveRecord::Base.connection.execute( - ActiveRecord::Base.send(:sanitize_sql_array, [sql, date_to: date_to]) + ActiveRecord::Base.send(:sanitize_sql_array, [sql, { date_to: date_to }]) ).each_with_object({}) do |row, hash| hash[row['registrar_id']] = row['total_domain_count'].to_i end From 6e271a9242994d6e8ae6eea39229257dacfdf6da Mon Sep 17 00:00:00 2001 From: Sergei Tsoganov Date: Tue, 19 Dec 2023 13:09:11 +0200 Subject: [PATCH 09/10] Updated .codeclimate.yml --- .codeclimate.yml | 2 +- app/controllers/repp/v1/stats_controller.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.codeclimate.yml b/.codeclimate.yml index ffcd2856cf..3869f2a950 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -31,7 +31,7 @@ plugins: enabled: false rubocop: enabled: true - channel: rubocop-1-18-3 + channel: rubocop-1-45 checks: Rubocop/Style/ClassAndModuleChildren: enabled: false diff --git a/app/controllers/repp/v1/stats_controller.rb b/app/controllers/repp/v1/stats_controller.rb index 100ea7ab7b..bca81ba78d 100644 --- a/app/controllers/repp/v1/stats_controller.rb +++ b/app/controllers/repp/v1/stats_controller.rb @@ -44,7 +44,7 @@ def market_share_growth_rate private def search_params - params.permit(:q, q: %i[start_date end_date compare_to_end_date compare_to_start_date]) + params.permit(:q, q: %i[end_date compare_to_end_date]) .fetch(:q, {}) || {} end From 7736e1c828a54a5e5d1237f6e60a86287d28f55d Mon Sep 17 00:00:00 2001 From: Sergei Tsoganov Date: Thu, 21 Dec 2023 12:00:55 +0200 Subject: [PATCH 10/10] Added extra query for log domains destroy events --- app/controllers/repp/v1/stats_controller.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/app/controllers/repp/v1/stats_controller.rb b/app/controllers/repp/v1/stats_controller.rb index bca81ba78d..b8c3b053fc 100644 --- a/app/controllers/repp/v1/stats_controller.rb +++ b/app/controllers/repp/v1/stats_controller.rb @@ -147,6 +147,22 @@ def domains_by_registrar(date_to) log_domains WHERE event = 'destroy' + AND object_changes->'registrar_id' IS NOT NULL + AND created_at > :date_to + GROUP BY + registrar_id + + UNION ALL + + -- Query for 'destroy' events and count the number of domains destroyed associated with each registrar + SELECT + (object->'registrar_id')::text AS registrar_id, + COUNT(*) AS domain_count + FROM + log_domains + WHERE + event = 'destroy' + AND object_changes IS NULL AND created_at > :date_to GROUP BY registrar_id