From ab5c002dc0313c5b3aa6eebd281538ac6155ea09 Mon Sep 17 00:00:00 2001 From: Rabbit Date: Wed, 15 May 2024 19:53:44 +0800 Subject: [PATCH 1/9] refactor: export xudt snapshot --- .../csv_exportable/export_udt_snapshot_job.rb | 38 ++++++++++++------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/app/jobs/csv_exportable/export_udt_snapshot_job.rb b/app/jobs/csv_exportable/export_udt_snapshot_job.rb index e2720432e..a1697b20d 100644 --- a/app/jobs/csv_exportable/export_udt_snapshot_job.rb +++ b/app/jobs/csv_exportable/export_udt_snapshot_job.rb @@ -12,12 +12,26 @@ def perform(args) block_timestamp <= #{@block.timestamp} AND (consumed_block_timestamp > #{@block.timestamp} OR consumed_block_timestamp IS NULL) SQL - snapshot = CellOutput.where(condition).group(:address_id).sum(:udt_amount) + cell_outputs = CellOutput.where(condition).group(:address_id).sum(:udt_amount) + cell_outputs = cell_outputs.reject { |_, v| v.to_f.zero? } + + data = [] + cell_outputs.keys.each_slice(1000) do |address_ids| + addresses = Address.includes(bitcoin_address_mapping: [:bitcoin_address]). + where(id: address_ids).pluck("addresses.id", "addresses.address_hash", "bitcoin_addresses.address_hash") + + addresses.each do |address| + data << { + address_hash: address[1], + bitcoin_address_hash: address[2], + udt_amount: cell_outputs[address[0]], + } + end + end rows = [] - snapshot = snapshot.reject { |_, v| v.to_f.zero? } - snapshot.sort_by { |_k, v| -v }.each do |address_id, udt_amount| - row = generate_row(address_id, udt_amount) + data.sort_by { |item| -item[:udt_amount] }.each do |item| + row = generate_row(item) next if row.blank? rows << row @@ -27,11 +41,7 @@ def perform(args) generate_csv(header, rows) end - def generate_row(address_id, udt_amount) - address = Address.find_by(id: address_id) - return unless address - - address_hash = address.bitcoin_address&.address_hash || address.address_hash + def generate_row(item) datetime = datetime_utc(@block.timestamp) if (decimal = @udt.decimal) @@ -40,8 +50,9 @@ def generate_row(address_id, udt_amount) @block.number, @block.timestamp, datetime, - address_hash, - parse_udt_amount(udt_amount.to_d, decimal), + item[:bitcoin_address_hash] || item[:address_hash], + item[:address_hash], + parse_udt_amount(item[:udt_amount].to_d, decimal), ] else [ @@ -49,8 +60,9 @@ def generate_row(address_id, udt_amount) @block.number, @block.timestamp, datetime, - address_hash, - "#{udt_amount} (raw)", + item[:bitcoin_address_hash] || item[:address_hash], + item[:address_hash], + "#{item[:udt_amount]} (raw)", ] end end From 07a8708c3774823c7df5b1c082685eacbdb58538 Mon Sep 17 00:00:00 2001 From: Miles Zhang Date: Tue, 27 Aug 2024 11:06:35 +0800 Subject: [PATCH 2/9] hotfix: commit rack cache Signed-off-by: Miles Zhang --- config/environments/production.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index 340573921..dc1864b9e 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -103,11 +103,11 @@ # Do not dump schema after migrations. config.active_record.dump_schema_after_migration = false - client = Dalli::Client.new(ENV["MEMCACHED_URL"], - value_max_bytes: 10485760) - config.action_dispatch.rack_cache = { - verbose: true, - metastore: client, - entitystore: client, - } + # client = Dalli::Client.new(ENV["MEMCACHED_URL"], + # value_max_bytes: 10485760) + # config.action_dispatch.rack_cache = { + # verbose: true, + # metastore: client, + # entitystore: client, + # } end From 411430d50520a0426fc2e7888050606671d634a0 Mon Sep 17 00:00:00 2001 From: Miles Zhang Date: Tue, 27 Aug 2024 11:11:48 +0800 Subject: [PATCH 3/9] hotfix: commit script txs cache Signed-off-by: Miles Zhang --- app/controllers/api/v2/scripts_controller.rb | 2 +- config/environments/production.rb | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/v2/scripts_controller.rb b/app/controllers/api/v2/scripts_controller.rb index 4b83752eb..736109eb7 100644 --- a/app/controllers/api/v2/scripts_controller.rb +++ b/app/controllers/api/v2/scripts_controller.rb @@ -16,7 +16,7 @@ def general_info def ckb_transactions head :not_found and return if @contract.blank? - expires_in 15.seconds, public: true, must_revalidate: true, stale_while_revalidate: 5.seconds + # expires_in 15.seconds, public: true, must_revalidate: true, stale_while_revalidate: 5.seconds @ckb_transactions = if @contract.ckb_transactions_count.zero? CkbTransaction.none diff --git a/config/environments/production.rb b/config/environments/production.rb index dc1864b9e..6f7d414fb 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -103,11 +103,9 @@ # Do not dump schema after migrations. config.active_record.dump_schema_after_migration = false - # client = Dalli::Client.new(ENV["MEMCACHED_URL"], - # value_max_bytes: 10485760) - # config.action_dispatch.rack_cache = { - # verbose: true, - # metastore: client, - # entitystore: client, - # } + config.action_dispatch.rack_cache = { + verbose: true, + metastore: File.join(ENV["MEMCACHED_URL"], "meta"), + entitystore: File.join(ENV["MEMCACHED_URL"], "body"), + } end From 8e37c1a6061464acb936b6dc6bf750634a5114ba Mon Sep 17 00:00:00 2001 From: Miles Zhang Date: Tue, 27 Aug 2024 11:20:12 +0800 Subject: [PATCH 4/9] hotfix: revert market data total_supply Signed-off-by: Miles Zhang --- app/models/market_data.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/models/market_data.rb b/app/models/market_data.rb index 3e4e5cfb2..cc444a20b 100644 --- a/app/models/market_data.rb +++ b/app/models/market_data.rb @@ -87,7 +87,7 @@ def parsed_dao def total_supply result = if current_timestamp > first_released_timestamp_may - parsed_dao.c_i - BURN_QUOTA - (parsed_dao.s_i - unmade_dao_interests) + parsed_dao.c_i - BURN_QUOTA - yesterday_treasury_amount.to_i else parsed_dao.c_i - BURN_QUOTA end @@ -167,4 +167,9 @@ def third_released_timestamp_other lock_address.present? ? lock_address.lock_script.lock_info[:estimated_unlock_time].to_i : CkbUtils.time_in_milliseconds(Time.find_zone("UTC").parse("2022-12-31")) end end + + def yesterday_treasury_amount + treasury_amounts = DailyStatistic.order(created_at_unixtimestamp: :desc).first(2).pluck(:treasury_amount) + treasury_amounts[0] == "0" ? treasury_amounts[1] : treasury_amounts[0] + end end From fe043d030b00604151f5c2d010520e2b22038d57 Mon Sep 17 00:00:00 2001 From: Miles Zhang Date: Tue, 27 Aug 2024 12:24:58 +0800 Subject: [PATCH 5/9] chore: use custom dalli gem Signed-off-by: Miles Zhang --- Gemfile | 2 +- Gemfile.lock | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 434a09bed..14a3aa090 100644 --- a/Gemfile +++ b/Gemfile @@ -111,7 +111,7 @@ end # Windows does not include zoneinfo files, so bundle the tzinfo-data gem gem "after_commit_everywhere" -gem "dalli" +gem "dalli", git: "https://github.com/rabbitz/dalli.git", branch: "value_max_bytes" gem "http" gem "kredis" gem "pagy" diff --git a/Gemfile.lock b/Gemfile.lock index 7cad15742..d693f54f6 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -7,6 +7,13 @@ GIT rbnacl (~> 7.1.1) rbsecp256k1 (~> 5.1.1) +GIT + remote: https://github.com/rabbitz/dalli.git + revision: c5ef4323ddff0b7cf4a7bc9e00ceb6055b71a425 + branch: value_max_bytes + specs: + dalli (3.2.3) + GEM remote: https://rubygems.org/ specs: @@ -149,7 +156,6 @@ GEM rexml crass (1.0.6) daemon-spawn (0.4.2) - dalli (3.2.3) database_cleaner (2.0.1) database_cleaner-active_record (~> 2.0.0) database_cleaner-active_record (2.0.1) @@ -499,7 +505,7 @@ DEPENDENCIES bootsnap ckb-sdk-ruby! config - dalli + dalli! database_cleaner database_cleaner-active_record digest-crc From a5e7e479938dc774301c6cd111d0fa711dd3be73 Mon Sep 17 00:00:00 2001 From: Miles Zhang Date: Tue, 27 Aug 2024 14:38:30 +0800 Subject: [PATCH 6/9] Revert "chore: use custom dalli gem" This reverts commit fe043d030b00604151f5c2d010520e2b22038d57. --- Gemfile | 2 +- Gemfile.lock | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/Gemfile b/Gemfile index 14a3aa090..434a09bed 100644 --- a/Gemfile +++ b/Gemfile @@ -111,7 +111,7 @@ end # Windows does not include zoneinfo files, so bundle the tzinfo-data gem gem "after_commit_everywhere" -gem "dalli", git: "https://github.com/rabbitz/dalli.git", branch: "value_max_bytes" +gem "dalli" gem "http" gem "kredis" gem "pagy" diff --git a/Gemfile.lock b/Gemfile.lock index d693f54f6..7cad15742 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -7,13 +7,6 @@ GIT rbnacl (~> 7.1.1) rbsecp256k1 (~> 5.1.1) -GIT - remote: https://github.com/rabbitz/dalli.git - revision: c5ef4323ddff0b7cf4a7bc9e00ceb6055b71a425 - branch: value_max_bytes - specs: - dalli (3.2.3) - GEM remote: https://rubygems.org/ specs: @@ -156,6 +149,7 @@ GEM rexml crass (1.0.6) daemon-spawn (0.4.2) + dalli (3.2.3) database_cleaner (2.0.1) database_cleaner-active_record (~> 2.0.0) database_cleaner-active_record (2.0.1) @@ -505,7 +499,7 @@ DEPENDENCIES bootsnap ckb-sdk-ruby! config - dalli! + dalli database_cleaner database_cleaner-active_record digest-crc From e69ae96d66e9f55ef1d534346b9d5c5f85a33ad6 Mon Sep 17 00:00:00 2001 From: Miles Zhang Date: Tue, 27 Aug 2024 14:38:45 +0800 Subject: [PATCH 7/9] Revert "hotfix: revert market data total_supply" This reverts commit 8e37c1a6061464acb936b6dc6bf750634a5114ba. --- app/models/market_data.rb | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/app/models/market_data.rb b/app/models/market_data.rb index cc444a20b..3e4e5cfb2 100644 --- a/app/models/market_data.rb +++ b/app/models/market_data.rb @@ -87,7 +87,7 @@ def parsed_dao def total_supply result = if current_timestamp > first_released_timestamp_may - parsed_dao.c_i - BURN_QUOTA - yesterday_treasury_amount.to_i + parsed_dao.c_i - BURN_QUOTA - (parsed_dao.s_i - unmade_dao_interests) else parsed_dao.c_i - BURN_QUOTA end @@ -167,9 +167,4 @@ def third_released_timestamp_other lock_address.present? ? lock_address.lock_script.lock_info[:estimated_unlock_time].to_i : CkbUtils.time_in_milliseconds(Time.find_zone("UTC").parse("2022-12-31")) end end - - def yesterday_treasury_amount - treasury_amounts = DailyStatistic.order(created_at_unixtimestamp: :desc).first(2).pluck(:treasury_amount) - treasury_amounts[0] == "0" ? treasury_amounts[1] : treasury_amounts[0] - end end From 7753b681675abca56365175904767fac1a03d7a8 Mon Sep 17 00:00:00 2001 From: Miles Zhang Date: Tue, 27 Aug 2024 14:38:56 +0800 Subject: [PATCH 8/9] Revert "hotfix: commit script txs cache" This reverts commit 411430d50520a0426fc2e7888050606671d634a0. --- app/controllers/api/v2/scripts_controller.rb | 2 +- config/environments/production.rb | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/v2/scripts_controller.rb b/app/controllers/api/v2/scripts_controller.rb index 736109eb7..4b83752eb 100644 --- a/app/controllers/api/v2/scripts_controller.rb +++ b/app/controllers/api/v2/scripts_controller.rb @@ -16,7 +16,7 @@ def general_info def ckb_transactions head :not_found and return if @contract.blank? - # expires_in 15.seconds, public: true, must_revalidate: true, stale_while_revalidate: 5.seconds + expires_in 15.seconds, public: true, must_revalidate: true, stale_while_revalidate: 5.seconds @ckb_transactions = if @contract.ckb_transactions_count.zero? CkbTransaction.none diff --git a/config/environments/production.rb b/config/environments/production.rb index 6f7d414fb..dc1864b9e 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -103,9 +103,11 @@ # Do not dump schema after migrations. config.active_record.dump_schema_after_migration = false - config.action_dispatch.rack_cache = { - verbose: true, - metastore: File.join(ENV["MEMCACHED_URL"], "meta"), - entitystore: File.join(ENV["MEMCACHED_URL"], "body"), - } + # client = Dalli::Client.new(ENV["MEMCACHED_URL"], + # value_max_bytes: 10485760) + # config.action_dispatch.rack_cache = { + # verbose: true, + # metastore: client, + # entitystore: client, + # } end From 3101cd36524958ab5a877a1de9b11e046ec76f51 Mon Sep 17 00:00:00 2001 From: Miles Zhang Date: Tue, 27 Aug 2024 14:39:15 +0800 Subject: [PATCH 9/9] Revert "hotfix: commit rack cache" This reverts commit 07a8708c3774823c7df5b1c082685eacbdb58538. --- config/environments/production.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index dc1864b9e..340573921 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -103,11 +103,11 @@ # Do not dump schema after migrations. config.active_record.dump_schema_after_migration = false - # client = Dalli::Client.new(ENV["MEMCACHED_URL"], - # value_max_bytes: 10485760) - # config.action_dispatch.rack_cache = { - # verbose: true, - # metastore: client, - # entitystore: client, - # } + client = Dalli::Client.new(ENV["MEMCACHED_URL"], + value_max_bytes: 10485760) + config.action_dispatch.rack_cache = { + verbose: true, + metastore: client, + entitystore: client, + } end