From 4273cbb312b7998e0f272dac4db41f5f85938c32 Mon Sep 17 00:00:00 2001 From: Sergei Tsoganov Date: Wed, 30 Nov 2022 12:10:26 +0200 Subject: [PATCH 01/11] Refactored market share distribution request --- app/controllers/repp/v1/stats_controller.rb | 44 ++++++++++++++++++--- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/app/controllers/repp/v1/stats_controller.rb b/app/controllers/repp/v1/stats_controller.rb index ae8489c34c..e0b56b6318 100644 --- a/app/controllers/repp/v1/stats_controller.rb +++ b/app/controllers/repp/v1/stats_controller.rb @@ -7,15 +7,43 @@ class StatsController < BaseController param :end_date, String, required: true, desc: 'Period end date' end def market_share_distribution - registrars = ::Registrar.where(test_registrar: false).joins(:domains) - .where(from_condition).where(to_condition) - grouped = registrars.group(:name).count + date_to = to_date(search_params[:end_date]).end_of_month + date_from = to_date(search_params[:start_date] || '01.1991') + log_domains_del = ::Version::DomainVersion.where('event = ? AND created_at > ?' \ + "AND object ->> 'created_at' <= ?" \ + "AND object ->> 'created_at' >= ?", + 'destroy', date_to, date_to, date_from) + .group("object ->> 'registrar_id'").count + + log_domains_trans = ::Version::DomainVersion.where('event = ? AND created_at > ?' \ + "AND object ->> 'created_at' <= ?" \ + "AND object ->> 'created_at' >= ?" \ + "AND object_changes ->> 'registrar_id' IS NOT NULL", + 'update', date_to, date_to, date_from) + + log_domains_trans_grouped = log_domains_trans.group("object ->> 'registrar_id'") + .count + + domains = ::Domain.where(from_condition) + .where(to_condition) + .where.not(name: log_domains_trans.map { |ld| ld.object['name'] }) + .group(:registrar_id).count.stringify_keys + + grouped = summarize([log_domains_del, log_domains_trans_grouped, domains]) + + registrar_names = ::Registrar.where(test_registrar: false) + .map { |r| { "#{r.id}": r.name }.with_indifferent_access } + .inject(:merge) result = grouped.map do |key, value| - hash = { name: key.strip, y: value } - hash.merge!({ sliced: true, selected: true }) if current_user.registrar.name == key + next unless registrar_names.key?(key) + + name = registrar_names[key] + hash = { name: registrar_names[key], y: value } + hash.merge!({ sliced: true, selected: true }) if current_user.registrar.name == name hash end + render_success(data: result) end @@ -73,6 +101,8 @@ def compare_to_condition end def to_date(date_param) + return if date_param.blank? + Date.strptime(date_param, '%m.%y') end @@ -97,6 +127,10 @@ def calculate_market_share(domains_by_rar) value < 0.1 ? value.round(3) : value.round(1) end end + + def summarize(arr) + arr.inject { |memo, el| memo.merge(el) { |_, old_v, new_v| old_v + new_v } } + end end end end From 003596e9887d1074fd4c28c1ee2123f9c73eb398 Mon Sep 17 00:00:00 2001 From: Sergei Tsoganov Date: Thu, 1 Dec 2022 17:23:19 +0200 Subject: [PATCH 02/11] Corrected stats controller log_domains query --- app/controllers/repp/v1/stats_controller.rb | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/app/controllers/repp/v1/stats_controller.rb b/app/controllers/repp/v1/stats_controller.rb index e0b56b6318..8205a490b2 100644 --- a/app/controllers/repp/v1/stats_controller.rb +++ b/app/controllers/repp/v1/stats_controller.rb @@ -13,23 +13,31 @@ def market_share_distribution "AND object ->> 'created_at' <= ?" \ "AND object ->> 'created_at' >= ?", 'destroy', date_to, date_to, date_from) - .group("object ->> 'registrar_id'").count + + log_domains_del_grouped = log_domains_del.group("object ->> 'registrar_id'").count + # p "log_domains_del" + # p log_domains_del.map{|d| d.object['name']} log_domains_trans = ::Version::DomainVersion.where('event = ? AND created_at > ?' \ "AND object ->> 'created_at' <= ?" \ "AND object ->> 'created_at' >= ?" \ "AND object_changes ->> 'registrar_id' IS NOT NULL", 'update', date_to, date_to, date_from) - + # p "log_domains_trans" + # p log_domains_trans.map{|d| d.object['name']} log_domains_trans_grouped = log_domains_trans.group("object ->> 'registrar_id'") .count + updated = log_domains_trans.map { |ld| ld.object['name'] } | log_domains_del.map { |ld| ld.object['name'] } + domains = ::Domain.where(from_condition) .where(to_condition) - .where.not(name: log_domains_trans.map { |ld| ld.object['name'] }) - .group(:registrar_id).count.stringify_keys + .where.not(name: updated) + domains_grouped = domains.group(:registrar_id).count.stringify_keys + # p "domains" + # p domains.pluck(:name) - grouped = summarize([log_domains_del, log_domains_trans_grouped, domains]) + grouped = summarize([log_domains_del_grouped, log_domains_trans_grouped, domains_grouped]) registrar_names = ::Registrar.where(test_registrar: false) .map { |r| { "#{r.id}": r.name }.with_indifferent_access } @@ -43,6 +51,7 @@ def market_share_distribution hash.merge!({ sliced: true, selected: true }) if current_user.registrar.name == name hash end + # p result render_success(data: result) end From cb07aff62d71b598d2a740308822643c735e0ffb Mon Sep 17 00:00:00 2001 From: Sergei Tsoganov Date: Mon, 5 Dec 2022 12:58:43 +0200 Subject: [PATCH 03/11] Modified stats controller, query unique domain names --- app/controllers/repp/v1/stats_controller.rb | 42 +++++++++++++-------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/app/controllers/repp/v1/stats_controller.rb b/app/controllers/repp/v1/stats_controller.rb index 8205a490b2..0eaff53ff8 100644 --- a/app/controllers/repp/v1/stats_controller.rb +++ b/app/controllers/repp/v1/stats_controller.rb @@ -9,24 +9,36 @@ class StatsController < BaseController def market_share_distribution date_to = to_date(search_params[:end_date]).end_of_month date_from = to_date(search_params[:start_date] || '01.1991') - log_domains_del = ::Version::DomainVersion.where('event = ? AND created_at > ?' \ - "AND object ->> 'created_at' <= ?" \ - "AND object ->> 'created_at' >= ?", - 'destroy', date_to, date_to, date_from) + log_domain = ::Version::DomainVersion + log_domains_del = log_domain.where('event = ? AND created_at > ?' \ + "AND object ->> 'created_at' <= ?" \ + "AND object ->> 'created_at' >= ?", + 'destroy', date_to, date_to, date_from) + .select("DISTINCT ON (object ->> 'name') object, created_at") + .order(Arel.sql("object ->> 'name', created_at desc")) - log_domains_del_grouped = log_domains_del.group("object ->> 'registrar_id'").count # p "log_domains_del" - # p log_domains_del.map{|d| d.object['name']} - - log_domains_trans = ::Version::DomainVersion.where('event = ? AND created_at > ?' \ - "AND object ->> 'created_at' <= ?" \ - "AND object ->> 'created_at' >= ?" \ - "AND object_changes ->> 'registrar_id' IS NOT NULL", - 'update', date_to, date_to, date_from) + # log_domains_del.each do |d| + # p d.attributes + # end + # log_domains_del_grouped = log_domains_del.group("object ->> 'registrar_id'").count + log_domains_del_grouped = log_domains_del.group_by { |ld| ld.object['registrar_id'].to_s } + .transform_values(&:count) + # p log_domains_del_grouped + + + log_domains_trans = log_domain.where('event = ? AND created_at > ?' \ + "AND object ->> 'created_at' <= ?" \ + "AND object ->> 'created_at' >= ?" \ + "AND object_changes ->> 'registrar_id' IS NOT NULL", + 'update', date_to, date_to, date_from) + .select("DISTINCT ON (object ->> 'name') object, created_at") + .order(Arel.sql("object ->> 'name', created_at desc")) # p "log_domains_trans" - # p log_domains_trans.map{|d| d.object['name']} - log_domains_trans_grouped = log_domains_trans.group("object ->> 'registrar_id'") - .count + # p log_domains_trans.map{|d| [d.object['name'], d.created_at] } + log_domains_trans_grouped = log_domains_trans.group_by { |ld| ld.object['registrar_id'].to_s } + .transform_values(&:count) + # p log_domains_trans_grouped updated = log_domains_trans.map { |ld| ld.object['name'] } | log_domains_del.map { |ld| ld.object['name'] } From 925353e4ba0cc4169657758e931355bf61293790 Mon Sep 17 00:00:00 2001 From: Sergei Tsoganov Date: Mon, 5 Dec 2022 13:37:56 +0200 Subject: [PATCH 04/11] Modified stats controller, query unique domain names --- app/controllers/repp/v1/stats_controller.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/controllers/repp/v1/stats_controller.rb b/app/controllers/repp/v1/stats_controller.rb index 0eaff53ff8..0368b0358f 100644 --- a/app/controllers/repp/v1/stats_controller.rb +++ b/app/controllers/repp/v1/stats_controller.rb @@ -42,12 +42,11 @@ def market_share_distribution updated = log_domains_trans.map { |ld| ld.object['name'] } | log_domains_del.map { |ld| ld.object['name'] } - domains = ::Domain.where(from_condition) - .where(to_condition) + domains = ::Domain.where('created_at <= ? AND created_at >= ?', date_to, date_from) .where.not(name: updated) domains_grouped = domains.group(:registrar_id).count.stringify_keys # p "domains" - # p domains.pluck(:name) + # p domains_grouped grouped = summarize([log_domains_del_grouped, log_domains_trans_grouped, domains_grouped]) From b0193144065f6fe171a6d6218bfed15ffb58247b Mon Sep 17 00:00:00 2001 From: Sergei Tsoganov Date: Mon, 5 Dec 2022 14:01:45 +0200 Subject: [PATCH 05/11] Repaired date_from param --- app/controllers/repp/v1/stats_controller.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/controllers/repp/v1/stats_controller.rb b/app/controllers/repp/v1/stats_controller.rb index 0368b0358f..a2c1ee1616 100644 --- a/app/controllers/repp/v1/stats_controller.rb +++ b/app/controllers/repp/v1/stats_controller.rb @@ -8,7 +8,7 @@ class StatsController < BaseController end def market_share_distribution date_to = to_date(search_params[:end_date]).end_of_month - date_from = to_date(search_params[:start_date] || '01.1991') + date_from = to_date(search_params[:start_date] || '01.91') log_domain = ::Version::DomainVersion log_domains_del = log_domain.where('event = ? AND created_at > ?' \ "AND object ->> 'created_at' <= ?" \ @@ -54,6 +54,7 @@ def market_share_distribution .map { |r| { "#{r.id}": r.name }.with_indifferent_access } .inject(:merge) + result = grouped.map do |key, value| next unless registrar_names.key?(key) From 10fb39980bbbba4b50a3db327ab292b12178209f Mon Sep 17 00:00:00 2001 From: Sergei Tsoganov Date: Mon, 5 Dec 2022 16:34:24 +0200 Subject: [PATCH 06/11] Refactored --- app/controllers/repp/v1/stats_controller.rb | 34 ++++++++++----------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/app/controllers/repp/v1/stats_controller.rb b/app/controllers/repp/v1/stats_controller.rb index a2c1ee1616..2742cce135 100644 --- a/app/controllers/repp/v1/stats_controller.rb +++ b/app/controllers/repp/v1/stats_controller.rb @@ -8,14 +8,13 @@ class StatsController < BaseController end def market_share_distribution date_to = to_date(search_params[:end_date]).end_of_month - date_from = to_date(search_params[:start_date] || '01.91') - log_domain = ::Version::DomainVersion - log_domains_del = log_domain.where('event = ? AND created_at > ?' \ - "AND object ->> 'created_at' <= ?" \ - "AND object ->> 'created_at' >= ?", - 'destroy', date_to, date_to, date_from) - .select("DISTINCT ON (object ->> 'name') object, created_at") - .order(Arel.sql("object ->> 'name', created_at desc")) + date_from = to_date(search_params[:start_date] || '01.05') + log_domains_del = ::Version::DomainVersion.where(event: 'destroy') + .where('created_at > ?', date_to) + .where("object ->> 'created_at' <= ?", date_to) + .where("object ->> 'created_at' >= ?", date_from) + .select("DISTINCT ON (object ->> 'name') object, created_at") + .order(Arel.sql("object ->> 'name', created_at desc")) # p "log_domains_del" # log_domains_del.each do |d| @@ -26,24 +25,23 @@ def market_share_distribution .transform_values(&:count) # p log_domains_del_grouped - - log_domains_trans = log_domain.where('event = ? AND created_at > ?' \ - "AND object ->> 'created_at' <= ?" \ - "AND object ->> 'created_at' >= ?" \ - "AND object_changes ->> 'registrar_id' IS NOT NULL", - 'update', date_to, date_to, date_from) - .select("DISTINCT ON (object ->> 'name') object, created_at") - .order(Arel.sql("object ->> 'name', created_at desc")) + log_domains_trans = ::Version::DomainVersion.where(event: 'update') + .where('created_at > ?', date_to) + .where("object ->> 'created_at' <= ?", date_to) + .where("object ->> 'created_at' >= ?", date_from) + .where("object_changes ->> 'registrar_id' IS NOT NULL") + .select("DISTINCT ON (object ->> 'name') object, created_at") + .order(Arel.sql("object ->> 'name', created_at desc")) # p "log_domains_trans" # p log_domains_trans.map{|d| [d.object['name'], d.created_at] } log_domains_trans_grouped = log_domains_trans.group_by { |ld| ld.object['registrar_id'].to_s } .transform_values(&:count) # p log_domains_trans_grouped - updated = log_domains_trans.map { |ld| ld.object['name'] } | log_domains_del.map { |ld| ld.object['name'] } + updated = log_domains_trans.map { |ld| ld.object['name'] } + log_domains_del.map { |ld| ld.object['name'] } domains = ::Domain.where('created_at <= ? AND created_at >= ?', date_to, date_from) - .where.not(name: updated) + .where.not(name: updated.uniq) domains_grouped = domains.group(:registrar_id).count.stringify_keys # p "domains" # p domains_grouped From a682681625aead99da3ec29e854ceab9eb89411a Mon Sep 17 00:00:00 2001 From: Sergei Tsoganov Date: Wed, 7 Dec 2022 11:21:30 +0200 Subject: [PATCH 07/11] Added indexes to log_domains table --- Gemfile | 2 + Gemfile.lock | 5 +- app/controllers/repp/v1/stats_controller.rb | 159 +++++------- config/initializers/strong_migrations.rb | 26 ++ ...fy_log_domains_object_changes_data_type.rb | 21 ++ ...221206091556_add_indexes_to_log_domains.rb | 15 ++ db/structure.sql | 243 ++++++++++++++++-- 7 files changed, 364 insertions(+), 107 deletions(-) create mode 100644 config/initializers/strong_migrations.rb create mode 100644 db/migrate/20221206090120_modify_log_domains_object_changes_data_type.rb create mode 100644 db/migrate/20221206091556_add_indexes_to_log_domains.rb diff --git a/Gemfile b/Gemfile index df067a040a..162dd32642 100644 --- a/Gemfile +++ b/Gemfile @@ -78,6 +78,8 @@ gem 'wkhtmltopdf-binary', '~> 0.12.5.1' gem 'directo', github: 'internetee/directo', branch: 'master' +gem 'strong_migrations' + group :development, :test do gem 'pry', '0.14.1' end diff --git a/Gemfile.lock b/Gemfile.lock index 9b8646c7d3..5836b1e201 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -474,6 +474,8 @@ GEM activesupport (>= 4.0) sprockets (>= 3.0.0) spy (1.0.1) + strong_migrations (1.4.0) + activerecord (>= 5.2) swd (1.3.0) activesupport (>= 3) attr_required (>= 0.0.5) @@ -588,6 +590,7 @@ DEPENDENCIES simplecov (= 0.17.1) simpleidn (= 0.2.1) spy + strong_migrations truemail (~> 2.4) uglifier validates_email_format_of (= 1.7.2) @@ -597,4 +600,4 @@ DEPENDENCIES wkhtmltopdf-binary (~> 0.12.5.1) BUNDLED WITH - 2.3.25 + 2.3.26 diff --git a/app/controllers/repp/v1/stats_controller.rb b/app/controllers/repp/v1/stats_controller.rb index 2742cce135..c8085f6bc9 100644 --- a/app/controllers/repp/v1/stats_controller.rb +++ b/app/controllers/repp/v1/stats_controller.rb @@ -1,68 +1,16 @@ module Repp module V1 - class StatsController < BaseController + class StatsController < BaseController # rubocop:disable Metrics/ClassLength + before_action :set_date_params + api :get, '/repp/v1/stats/market_share_distribution' desc 'Get market share and distribution of registrars' param :q, Hash, required: true, desc: 'Period parameters for data' do param :end_date, String, required: true, desc: 'Period end date' end def market_share_distribution - date_to = to_date(search_params[:end_date]).end_of_month - date_from = to_date(search_params[:start_date] || '01.05') - log_domains_del = ::Version::DomainVersion.where(event: 'destroy') - .where('created_at > ?', date_to) - .where("object ->> 'created_at' <= ?", date_to) - .where("object ->> 'created_at' >= ?", date_from) - .select("DISTINCT ON (object ->> 'name') object, created_at") - .order(Arel.sql("object ->> 'name', created_at desc")) - - # p "log_domains_del" - # log_domains_del.each do |d| - # p d.attributes - # end - # log_domains_del_grouped = log_domains_del.group("object ->> 'registrar_id'").count - log_domains_del_grouped = log_domains_del.group_by { |ld| ld.object['registrar_id'].to_s } - .transform_values(&:count) - # p log_domains_del_grouped - - log_domains_trans = ::Version::DomainVersion.where(event: 'update') - .where('created_at > ?', date_to) - .where("object ->> 'created_at' <= ?", date_to) - .where("object ->> 'created_at' >= ?", date_from) - .where("object_changes ->> 'registrar_id' IS NOT NULL") - .select("DISTINCT ON (object ->> 'name') object, created_at") - .order(Arel.sql("object ->> 'name', created_at desc")) - # p "log_domains_trans" - # p log_domains_trans.map{|d| [d.object['name'], d.created_at] } - log_domains_trans_grouped = log_domains_trans.group_by { |ld| ld.object['registrar_id'].to_s } - .transform_values(&:count) - # p log_domains_trans_grouped - - updated = log_domains_trans.map { |ld| ld.object['name'] } + log_domains_del.map { |ld| ld.object['name'] } - - domains = ::Domain.where('created_at <= ? AND created_at >= ?', date_to, date_from) - .where.not(name: updated.uniq) - domains_grouped = domains.group(:registrar_id).count.stringify_keys - # p "domains" - # p domains_grouped - - grouped = summarize([log_domains_del_grouped, log_domains_trans_grouped, domains_grouped]) - - registrar_names = ::Registrar.where(test_registrar: false) - .map { |r| { "#{r.id}": r.name }.with_indifferent_access } - .inject(:merge) - - - result = grouped.map do |key, value| - next unless registrar_names.key?(key) - - name = registrar_names[key] - hash = { name: registrar_names[key], y: value } - hash.merge!({ sliced: true, selected: true }) if current_user.registrar.name == name - hash - end - # p result - + domains_by_rar = domains_by_registrar(@date_to, @date_from) + result = serialize_distribution_result(domains_by_rar) render_success(data: result) end @@ -71,25 +19,23 @@ def market_share_distribution desc 'Get market share and growth rate of registrars' param :q, Hash, required: true, desc: 'Period parameters for data' do param :end_date, String, required: true, desc: 'Period end date' - param :compare_to_date, String, required: true, desc: 'Comparison date' + param :compare_to_end_date, String, required: true, desc: 'Comparison date' end def market_share_growth_rate - registrars = ::Registrar.where(test_registrar: false).joins(:domains) - - domains_by_rar = registrars.where(from_condition).where(to_condition).group(:name).count - prev_domains_by_rar = registrars.where(compare_to_condition).group(:name).count + domains_by_rar = domains_by_registrar(@date_to, @date_from) + prev_domains_by_rar = domains_by_registrar(@date_compare_to, @date_compare_from) set_zero_values!(domains_by_rar, prev_domains_by_rar) market_share_by_rar = calculate_market_share(domains_by_rar) prev_market_share_by_rar = calculate_market_share(prev_domains_by_rar) - result = { prev_data: { name: search_params[:compare_to_date], - domains: serialize(prev_domains_by_rar), - market_share: serialize(prev_market_share_by_rar) }, + result = { prev_data: { name: search_params[:compare_to_end_date], + domains: serialize_growth_rate_result(prev_domains_by_rar), + market_share: serialize_growth_rate_result(prev_market_share_by_rar) }, data: { name: search_params[:end_date], - domains: serialize(domains_by_rar), - market_share: serialize(market_share_by_rar) } } + domains: serialize_growth_rate_result(domains_by_rar), + market_share: serialize_growth_rate_result(market_share_by_rar) } } render_success(data: result) end # rubocop:enable Metrics/MethodLength @@ -97,26 +43,15 @@ def market_share_growth_rate private def search_params - params.permit(:q, q: %i[start_date end_date compare_to_date]) + params.permit(:q, q: %i[start_date end_date compare_to_end_date compare_to_start_date]) .fetch(:q, {}) || {} end - def from_condition - return unless search_params[:start_date] - - "domains.created_at >= '#{to_date(search_params[:start_date])}'" - end - - def to_condition - return unless search_params[:end_date] - - "domains.created_at <= '#{to_date(search_params[:end_date]).end_of_month}'" - end - - def compare_to_condition - return unless search_params[:compare_to_date] - - "domains.created_at <= '#{to_date(search_params[:compare_to_date]).end_of_month}'" + def set_date_params + @date_to = to_date(search_params[:end_date]).end_of_month + @date_from = to_date(search_params[:start_date] || '01.05') + @date_compare_to = to_date(search_params[:compare_to_end_date]).end_of_month + @date_compare_from = to_date(search_params[:compare_to_start_date] || '01.05') end def to_date(date_param) @@ -125,10 +60,6 @@ def to_date(date_param) Date.strptime(date_param, '%m.%y') end - def serialize(rars) - rars.map { |key, value| [key, value] } - end - def set_zero_values!(cur, prev) cur_dup = cur.dup cur_dup.each_key do |k| @@ -147,9 +78,61 @@ def calculate_market_share(domains_by_rar) end end + def domains_by_registrar(date_to, date_from) + log_domains_del = log_domains(event: 'destroy', date_to: date_to, date_from: date_from) + log_domains_trans = log_domains(event: 'update', date_to: date_to, date_from: date_from) + logged_domains = log_domains_trans.map { |ld| ld.object['name'] } + + log_domains_del.map { |ld| ld.object['name'] } + domains_grouped = ::Domain.where('created_at <= ? AND created_at >= ?', date_to, date_from) + .where.not(name: logged_domains.uniq) + .group(:registrar_id).count.stringify_keys + summarize([group(log_domains_del), group(log_domains_trans), domains_grouped]) + end + def summarize(arr) arr.inject { |memo, el| memo.merge(el) { |_, old_v, new_v| old_v + new_v } } end + + def log_domains(event:, date_to:, date_from:) + domains = ::Version::DomainVersion.where(event: event) + domains.where!("object_changes ->> 'registrar_id' IS NOT NULL") if event == 'update' + domains.where('created_at > ?', date_to) + .where("object ->> 'created_at' <= ?", date_to) + .where("object ->> 'created_at' >= ?", date_from) + .select("DISTINCT ON (object ->> 'name') object, created_at") + .order(Arel.sql("object ->> 'name', created_at desc")) + end + + def group(domains) + domains.group_by { |ld| ld.object['registrar_id'].to_s } + .transform_values(&:count) + end + + def registrar_names + @registrar_names ||= ::Registrar.where(test_registrar: false) + .map { |r| { "#{r.id}": r.name }.with_indifferent_access } + .inject(:merge) + end + + def serialize_distribution_result(grouped) + grouped.map do |key, value| + next unless registrar_names.key?(key) + + name = registrar_names[key] + hash = { name: registrar_names[key], y: value } + hash.merge!({ sliced: true, selected: true }) if current_user.registrar.name == name + hash + end + end + + def serialize_growth_rate_result(grouped) + grouped.map do |key, value| + next unless registrar_names.key?(key) + + name = registrar_names[key] + [name, value] + end + end end end end diff --git a/config/initializers/strong_migrations.rb b/config/initializers/strong_migrations.rb new file mode 100644 index 0000000000..11ce430939 --- /dev/null +++ b/config/initializers/strong_migrations.rb @@ -0,0 +1,26 @@ +# Mark existing migrations as safe +StrongMigrations.start_after = 20221206081743 + +# Set timeouts for migrations +# If you use PgBouncer in transaction mode, delete these lines and set timeouts on the database user +StrongMigrations.lock_timeout = 10.seconds +StrongMigrations.statement_timeout = 1.hour + +# Analyze tables after indexes are added +# Outdated statistics can sometimes hurt performance +StrongMigrations.auto_analyze = true + +# Set the version of the production database +# so the right checks are run in development +# StrongMigrations.target_version = 10 + +# Add custom checks +# StrongMigrations.add_check do |method, args| +# if method == :add_index && args[0].to_s == "users" +# stop! "No more indexes on the users table" +# end +# end + +# Make some operations safe by default +# See https://github.com/ankane/strong_migrations#safe-by-default +# StrongMigrations.safe_by_default = true diff --git a/db/migrate/20221206090120_modify_log_domains_object_changes_data_type.rb b/db/migrate/20221206090120_modify_log_domains_object_changes_data_type.rb new file mode 100644 index 0000000000..e7308ae594 --- /dev/null +++ b/db/migrate/20221206090120_modify_log_domains_object_changes_data_type.rb @@ -0,0 +1,21 @@ +class ModifyLogDomainsObjectChangesDataType < ActiveRecord::Migration[6.1] + def up + add_column :log_domains, :object_changes_jsonb, :jsonb, default: '{}' + + # Copy data from old column to the new one + Version::DomainVersion.update_all('object_changes_jsonb = object_changes::jsonb') + + # Rename columns instead of modify their type, it's way faster + safety_assured do + rename_column :log_domains, :object_changes, :object_changes_json + rename_column :log_domains, :object_changes_jsonb, :object_changes + end + end + + def down + safety_assured do + rename_column :log_domains, :object_changes, :object_changes_jsonb + rename_column :log_domains, :object_changes_json, :object_changes + end + end +end \ No newline at end of file diff --git a/db/migrate/20221206091556_add_indexes_to_log_domains.rb b/db/migrate/20221206091556_add_indexes_to_log_domains.rb new file mode 100644 index 0000000000..0dbbf944f3 --- /dev/null +++ b/db/migrate/20221206091556_add_indexes_to_log_domains.rb @@ -0,0 +1,15 @@ +class AddIndexesToLogDomains < ActiveRecord::Migration[6.1] + disable_ddl_transaction! + def up + enable_extension 'btree_gin' + add_index :log_domains, :event, algorithm: :concurrently + add_index :log_domains, :object, using: :gin, algorithm: :concurrently + add_index :log_domains, :object_changes, using: :gin, algorithm: :concurrently + end + + def down + remove_index :log_domains, :event + remove_index :log_domains, :object + remove_index :log_domains, :object_changes + end +end diff --git a/db/structure.sql b/db/structure.sql index 661b4e66b7..6b1a3358ff 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -9,6 +9,27 @@ SET xmloption = content; SET client_min_messages = warning; SET row_security = off; +-- +-- Name: public; Type: SCHEMA; Schema: -; Owner: - +-- + +-- *not* creating schema, since initdb creates it + + +-- +-- Name: btree_gin; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS btree_gin WITH SCHEMA public; + + +-- +-- Name: EXTENSION btree_gin; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION btree_gin IS 'support for indexing common datatypes in GIN'; + + -- -- Name: btree_gist; Type: EXTENSION; Schema: -; Owner: - -- @@ -956,15 +977,14 @@ CREATE TABLE public.domains ( pending_json jsonb, force_delete_date date, statuses character varying[], - statuses_before_force_delete character varying[] DEFAULT '{}'::character varying[], + status_notes public.hstore, upid integer, up_date timestamp without time zone, uuid uuid DEFAULT public.gen_random_uuid() NOT NULL, locked_by_registrant_at timestamp without time zone, force_delete_start timestamp without time zone, force_delete_data public.hstore, - json_statuses_history jsonb, - status_notes public.hstore + json_statuses_history jsonb ); @@ -1651,11 +1671,12 @@ CREATE TABLE public.log_domains ( event character varying NOT NULL, whodunnit character varying, object jsonb, - object_changes json, + object_changes_json json, created_at timestamp without time zone, session character varying, children jsonb, - uuid character varying + uuid character varying, + object_changes jsonb DEFAULT '"{}"'::jsonb ); @@ -2287,6 +2308,74 @@ CREATE SEQUENCE public.payment_orders_id_seq ALTER SEQUENCE public.payment_orders_id_seq OWNED BY public.payment_orders.id; +-- +-- Name: pghero_query_stats; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.pghero_query_stats ( + id bigint NOT NULL, + database text, + "user" text, + query text, + query_hash bigint, + total_time double precision, + calls bigint, + captured_at timestamp without time zone +); + + +-- +-- Name: pghero_query_stats_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.pghero_query_stats_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: pghero_query_stats_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.pghero_query_stats_id_seq OWNED BY public.pghero_query_stats.id; + + +-- +-- Name: pghero_space_stats; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.pghero_space_stats ( + id bigint NOT NULL, + database text, + schema text, + relation text, + size bigint, + captured_at timestamp without time zone +); + + +-- +-- Name: pghero_space_stats_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.pghero_space_stats_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: pghero_space_stats_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.pghero_space_stats_id_seq OWNED BY public.pghero_space_stats.id; + + -- -- Name: prices; Type: TABLE; Schema: public; Owner: - -- @@ -2325,6 +2414,48 @@ CREATE SEQUENCE public.prices_id_seq ALTER SEQUENCE public.prices_id_seq OWNED BY public.prices.id; +-- +-- Name: que_jobs; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.que_jobs ( + priority smallint DEFAULT 100 NOT NULL, + run_at timestamp with time zone DEFAULT now() NOT NULL, + job_id bigint NOT NULL, + job_class text NOT NULL, + args json DEFAULT '[]'::json NOT NULL, + error_count integer DEFAULT 0 NOT NULL, + last_error text, + queue text DEFAULT ''::text NOT NULL +); + + +-- +-- Name: TABLE que_jobs; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON TABLE public.que_jobs IS '3'; + + +-- +-- Name: que_jobs_job_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.que_jobs_job_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: que_jobs_job_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.que_jobs_job_id_seq OWNED BY public.que_jobs.job_id; + + -- -- Name: registrant_verifications; Type: TABLE; Schema: public; Owner: - -- @@ -2605,8 +2736,7 @@ CREATE TABLE public.validation_events ( validation_eventable_type character varying, validation_eventable_id bigint, created_at timestamp(6) without time zone NOT NULL, - updated_at timestamp(6) without time zone NOT NULL, - event_type public.validation_type + updated_at timestamp(6) without time zone NOT NULL ); @@ -3148,6 +3278,20 @@ ALTER TABLE ONLY public.notifications ALTER COLUMN id SET DEFAULT nextval('publi ALTER TABLE ONLY public.payment_orders ALTER COLUMN id SET DEFAULT nextval('public.payment_orders_id_seq'::regclass); +-- +-- Name: pghero_query_stats id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.pghero_query_stats ALTER COLUMN id SET DEFAULT nextval('public.pghero_query_stats_id_seq'::regclass); + + +-- +-- Name: pghero_space_stats id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.pghero_space_stats ALTER COLUMN id SET DEFAULT nextval('public.pghero_space_stats_id_seq'::regclass); + + -- -- Name: prices id; Type: DEFAULT; Schema: public; Owner: - -- @@ -3155,6 +3299,13 @@ ALTER TABLE ONLY public.payment_orders ALTER COLUMN id SET DEFAULT nextval('publ ALTER TABLE ONLY public.prices ALTER COLUMN id SET DEFAULT nextval('public.prices_id_seq'::regclass); +-- +-- Name: que_jobs job_id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.que_jobs ALTER COLUMN job_id SET DEFAULT nextval('public.que_jobs_job_id_seq'::regclass); + + -- -- Name: registrant_verifications id; Type: DEFAULT; Schema: public; Owner: - -- @@ -3664,6 +3815,22 @@ ALTER TABLE ONLY public.payment_orders ADD CONSTRAINT payment_orders_pkey PRIMARY KEY (id); +-- +-- Name: pghero_query_stats pghero_query_stats_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.pghero_query_stats + ADD CONSTRAINT pghero_query_stats_pkey PRIMARY KEY (id); + + +-- +-- Name: pghero_space_stats pghero_space_stats_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.pghero_space_stats + ADD CONSTRAINT pghero_space_stats_pkey PRIMARY KEY (id); + + -- -- Name: prices prices_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -3672,6 +3839,14 @@ ALTER TABLE ONLY public.prices ADD CONSTRAINT prices_pkey PRIMARY KEY (id); +-- +-- Name: que_jobs que_jobs_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.que_jobs + ADD CONSTRAINT que_jobs_pkey PRIMARY KEY (queue, priority, run_at, job_id); + + -- -- Name: registrant_verifications registrant_verifications_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -4251,6 +4426,13 @@ CREATE INDEX index_log_domain_contacts_on_item_type_and_item_id ON public.log_do CREATE INDEX index_log_domain_contacts_on_whodunnit ON public.log_domain_contacts USING btree (whodunnit); +-- +-- Name: index_log_domains_on_event; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_log_domains_on_event ON public.log_domains USING btree (event); + + -- -- Name: index_log_domains_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- @@ -4258,6 +4440,20 @@ CREATE INDEX index_log_domain_contacts_on_whodunnit ON public.log_domain_contact CREATE INDEX index_log_domains_on_item_type_and_item_id ON public.log_domains USING btree (item_type, item_id); +-- +-- Name: index_log_domains_on_object; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_log_domains_on_object ON public.log_domains USING gin (object); + + +-- +-- Name: index_log_domains_on_object_changes; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_log_domains_on_object_changes ON public.log_domains USING gin (object_changes); + + -- -- Name: index_log_domains_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- @@ -4426,6 +4622,20 @@ CREATE INDEX index_notifications_on_registrar_id ON public.notifications USING b CREATE INDEX index_payment_orders_on_invoice_id ON public.payment_orders USING btree (invoice_id); +-- +-- Name: index_pghero_query_stats_on_database_and_captured_at; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_pghero_query_stats_on_database_and_captured_at ON public.pghero_query_stats USING btree (database, captured_at); + + +-- +-- Name: index_pghero_space_stats_on_database_and_captured_at; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_pghero_space_stats_on_database_and_captured_at ON public.pghero_space_stats USING btree (database, captured_at); + + -- -- Name: index_prices_on_zone_id; Type: INDEX; Schema: public; Owner: - -- @@ -4482,13 +4692,6 @@ CREATE INDEX index_users_on_registrar_id ON public.users USING btree (registrar_ CREATE INDEX index_validation_events_on_event_data ON public.validation_events USING gin (event_data); --- --- Name: index_validation_events_on_event_type; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX index_validation_events_on_event_type ON public.validation_events USING btree (event_type); - - -- -- Name: index_validation_events_on_validation_eventable; Type: INDEX; Schema: public; Owner: - -- @@ -5234,9 +5437,11 @@ INSERT INTO "schema_migrations" (version) VALUES ('20210708131814'), ('20210729131100'), ('20210729134625'), -('20210827185249'), -('20211029073644'), +('20211028122103'), +('20211028125245'), +('20211029082225'), ('20211124071418'), +('20211124084308'), ('20211125181033'), ('20211125184334'), ('20211126085139'), @@ -5245,7 +5450,6 @@ INSERT INTO "schema_migrations" (version) VALUES ('20220113201642'), ('20220113220809'), ('20220124105717'), -('20220216113112'), ('20220228093211'), ('20220316140727'), ('20220406085500'), @@ -5258,6 +5462,9 @@ INSERT INTO "schema_migrations" (version) VALUES ('20220701113409'), ('20220715145808'), ('20220818075833'), -('20221011061840'); +('20221011061840'), +('20221206075912'), +('20221206090120'), +('20221206091556'); From 045c74b7e2205e785998bcc136865ab35e791f9b Mon Sep 17 00:00:00 2001 From: Sergei Tsoganov Date: Wed, 7 Dec 2022 11:45:31 +0200 Subject: [PATCH 08/11] Fixed test --- app/controllers/repp/v1/stats_controller.rb | 12 ++++++------ test/integration/repp/v1/stats/market_share_test.rb | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/controllers/repp/v1/stats_controller.rb b/app/controllers/repp/v1/stats_controller.rb index c8085f6bc9..61bdc1e2f1 100644 --- a/app/controllers/repp/v1/stats_controller.rb +++ b/app/controllers/repp/v1/stats_controller.rb @@ -55,7 +55,7 @@ def set_date_params end def to_date(date_param) - return if date_param.blank? + return Date.today if date_param.blank? Date.strptime(date_param, '%m.%y') end @@ -114,19 +114,19 @@ def registrar_names .inject(:merge) end - def serialize_distribution_result(grouped) - grouped.map do |key, value| + def serialize_distribution_result(result) + result.map do |key, value| next unless registrar_names.key?(key) name = registrar_names[key] - hash = { name: registrar_names[key], y: value } + hash = { name: name, y: value } hash.merge!({ sliced: true, selected: true }) if current_user.registrar.name == name hash end end - def serialize_growth_rate_result(grouped) - grouped.map do |key, value| + def serialize_growth_rate_result(result) + result.map do |key, value| next unless registrar_names.key?(key) name = registrar_names[key] diff --git a/test/integration/repp/v1/stats/market_share_test.rb b/test/integration/repp/v1/stats/market_share_test.rb index 614b5868dc..03721cebdf 100644 --- a/test/integration/repp/v1/stats/market_share_test.rb +++ b/test/integration/repp/v1/stats/market_share_test.rb @@ -21,15 +21,15 @@ def test_shows_market_share_distribution_data assert json[:data].is_a? Array assert json[:data][0].is_a? Hash - assert_equal json[:data][0][:name], 'Best Names' - assert json[:data][0][:selected] + assert_equal json[:data][0][:name], 'Good Names' + assert_nil json[:data][0][:selected] end def test_shows_market_share_growth_rate_data prev_date = Time.zone.today.last_month.strftime('%m.%y') get '/repp/v1/stats/market_share_growth_rate', headers: @auth_headers, params: { q: { end_date: @today, - compare_to_date: prev_date } } + compare_to_end_date: prev_date } } json = JSON.parse(response.body, symbolize_names: true) assert_response :ok From 1b1f7eadf50c4ba0575b7dd2fc4ec7b17e0d0ec8 Mon Sep 17 00:00:00 2001 From: Sergei Tsoganov Date: Wed, 7 Dec 2022 12:18:38 +0200 Subject: [PATCH 09/11] Fixed codeclimate issue --- app/controllers/repp/v1/stats_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/repp/v1/stats_controller.rb b/app/controllers/repp/v1/stats_controller.rb index 61bdc1e2f1..86448e2376 100644 --- a/app/controllers/repp/v1/stats_controller.rb +++ b/app/controllers/repp/v1/stats_controller.rb @@ -55,7 +55,7 @@ def set_date_params end def to_date(date_param) - return Date.today if date_param.blank? + return Time.zone.today if date_param.blank? Date.strptime(date_param, '%m.%y') end From e70be801c2ba320094e6c1047cb934685c47be9b Mon Sep 17 00:00:00 2001 From: Sergei Tsoganov Date: Wed, 7 Dec 2022 12:31:41 +0200 Subject: [PATCH 10/11] Changed log_domains object_changes default value --- ...102831_change_log_domains_object_changes_default_value.rb | 5 +++++ db/structure.sql | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 db/migrate/20221207102831_change_log_domains_object_changes_default_value.rb diff --git a/db/migrate/20221207102831_change_log_domains_object_changes_default_value.rb b/db/migrate/20221207102831_change_log_domains_object_changes_default_value.rb new file mode 100644 index 0000000000..84a3c32cf8 --- /dev/null +++ b/db/migrate/20221207102831_change_log_domains_object_changes_default_value.rb @@ -0,0 +1,5 @@ +class ChangeLogDomainsObjectChangesDefaultValue < ActiveRecord::Migration[6.1] + def change + change_column_default :log_domains, :object_changes, nil + end +end diff --git a/db/structure.sql b/db/structure.sql index 6b1a3358ff..fd26dd07a2 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1676,7 +1676,7 @@ CREATE TABLE public.log_domains ( session character varying, children jsonb, uuid character varying, - object_changes jsonb DEFAULT '"{}"'::jsonb + object_changes jsonb ); @@ -5465,6 +5465,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20221011061840'), ('20221206075912'), ('20221206090120'), -('20221206091556'); +('20221206091556'), +('20221207102831'); From 7c84949fcbf4b6e2ef08b00a3672cc979560c7b4 Mon Sep 17 00:00:00 2001 From: Sergei Tsoganov Date: Wed, 7 Dec 2022 16:27:59 +0200 Subject: [PATCH 11/11] Fixed chart data serialization --- app/controllers/repp/v1/stats_controller.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/repp/v1/stats_controller.rb b/app/controllers/repp/v1/stats_controller.rb index 86448e2376..052cdcbc45 100644 --- a/app/controllers/repp/v1/stats_controller.rb +++ b/app/controllers/repp/v1/stats_controller.rb @@ -111,7 +111,7 @@ def group(domains) def registrar_names @registrar_names ||= ::Registrar.where(test_registrar: false) .map { |r| { "#{r.id}": r.name }.with_indifferent_access } - .inject(:merge) + .reduce({}, :merge) end def serialize_distribution_result(result) @@ -122,7 +122,7 @@ def serialize_distribution_result(result) hash = { name: name, y: value } hash.merge!({ sliced: true, selected: true }) if current_user.registrar.name == name hash - end + end.compact end def serialize_growth_rate_result(result) @@ -131,7 +131,7 @@ def serialize_growth_rate_result(result) name = registrar_names[key] [name, value] - end + end.compact end end end