From db2107e23962d4775644b357520f6510c09f045a Mon Sep 17 00:00:00 2001 From: sneakers-the-rat Date: Wed, 18 Sep 2024 20:35:36 -0700 Subject: [PATCH] don't do it for every create, only do recursive reply expansion when requested from context endpoint, but async --- app/controllers/api/v1/statuses_controller.rb | 2 +- app/lib/activitypub/activity/create.rb | 2 +- app/models/status.rb | 2 +- .../activitypub/fetch_replies_service.rb | 2 +- app/workers/fetch_reply_worker.rb | 23 ++++++++++++++-- db/schema.rb | 26 +++++++++---------- 6 files changed, 38 insertions(+), 19 deletions(-) diff --git a/app/controllers/api/v1/statuses_controller.rb b/app/controllers/api/v1/statuses_controller.rb index 462802f3cb362b..c7569f4285dfea 100644 --- a/app/controllers/api/v1/statuses_controller.rb +++ b/app/controllers/api/v1/statuses_controller.rb @@ -50,7 +50,7 @@ def context descendants_depth_limit = DESCENDANTS_DEPTH_LIMIT else unless @status.local? && !@status.should_fetch_replies? - json_status = fetch_resource(@status.uri, true, @current_account) + json_status = fetch_resource(@status.uri, true, current_account) # rescue this whole block on failure, don't want to fail the whole context request if we can't do this collection = json_status['replies'] diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 1aa8576345bf73..f69b68392a5a01 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -338,7 +338,7 @@ def fetch_replies(status) collection = @object['replies'] return if collection.blank? - replies = ActivityPub::FetchRepliesService.new.call(status, collection, allow_synchronous_requests: false, request_id: @options[:request_id], all_replies: status.should_fetch_replies?) + replies = ActivityPub::FetchRepliesService.new.call(status, collection, allow_synchronous_requests: false, request_id: @options[:request_id]) return unless replies.nil? uri = value_or_id(collection) diff --git a/app/models/status.rb b/app/models/status.rb index 41505279963613..285e48ee8c47a1 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -445,7 +445,7 @@ def unlink_from_conversations! def should_fetch_replies? # we aren't brand new, and we haven't fetched replies since the debounce window - created_at <= 10.minutes.ago && ( + !local? && created_at <= 10.minutes.ago && ( fetched_replies_at.nil? || fetched_replies_at <= FETCH_REPLIES_DEBOUNCE.ago ) end diff --git a/app/services/activitypub/fetch_replies_service.rb b/app/services/activitypub/fetch_replies_service.rb index 3ab7e7e0460d11..c846b60a331379 100644 --- a/app/services/activitypub/fetch_replies_service.rb +++ b/app/services/activitypub/fetch_replies_service.rb @@ -23,7 +23,7 @@ def call(parent_status, collection_or_uri, allow_synchronous_requests: true, req @items = collection_items(collection_or_uri) return if @items.nil? - FetchReplyWorker.push_bulk(filtered_replies) { |reply_uri| [reply_uri, { 'request_id' => request_id }] } + FetchReplyWorker.push_bulk(filtered_replies) { |reply_uri| [reply_uri, { 'request_id' => request_id }, @all_replies] } # Store last fetched all to debounce @status.update(fetched_replies_at: Time.now.utc) if fetch_all_replies? diff --git a/app/workers/fetch_reply_worker.rb b/app/workers/fetch_reply_worker.rb index 68a7414bebeaa0..af33e8d6813d55 100644 --- a/app/workers/fetch_reply_worker.rb +++ b/app/workers/fetch_reply_worker.rb @@ -6,7 +6,26 @@ class FetchReplyWorker sidekiq_options queue: 'pull', retry: 3 - def perform(child_url, options = {}) - FetchRemoteStatusService.new.call(child_url, **options.deep_symbolize_keys) + def perform(child_url, options = {}, all_replies: false) + status = FetchRemoteStatusService.new.call(child_url, **options.deep_symbolize_keys) + + # asked to fetch replies recursively - do the second-level calls async + if all_replies && status.should_fetch_replies? + json_status = fetch_resource(status.uri, true) + + collection = json_status['replies'] + unless collection.nil? + # if expanding replies recursively, spread out the recursive calls + ActivityPub::FetchRepliesWorker.perform_in( + rand(1..30).seconds, + status.id, + collection, + { + allow_synchronous_requests: true, + all_replies: true, + } + ) + end + end end end diff --git a/db/schema.rb b/db/schema.rb index e550fa618d250c..a8e3ef450f071b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -194,8 +194,8 @@ t.integer "avatar_storage_schema_version" t.integer "header_storage_schema_version" t.string "devices_url" - t.datetime "sensitized_at", precision: nil t.integer "suspension_origin" + t.datetime "sensitized_at", precision: nil t.boolean "trendable" t.datetime "reviewed_at", precision: nil t.datetime "requested_review_at", precision: nil @@ -579,12 +579,12 @@ end create_table "ip_blocks", force: :cascade do |t| + t.datetime "created_at", precision: nil, null: false + t.datetime "updated_at", precision: nil, null: false + t.datetime "expires_at", precision: nil t.inet "ip", default: "0.0.0.0", null: false t.integer "severity", default: 0, null: false - t.datetime "expires_at", precision: nil t.text "comment", default: "", null: false - t.datetime "created_at", precision: nil, null: false - t.datetime "updated_at", precision: nil, null: false t.index ["ip"], name: "index_ip_blocks_on_ip", unique: true end @@ -1429,9 +1429,9 @@ add_index "instances", ["domain"], name: "index_instances_on_domain", unique: true create_view "user_ips", sql_definition: <<-SQL - SELECT user_id, - ip, - max(used_at) AS used_at + SELECT t0.user_id, + t0.ip, + max(t0.used_at) AS used_at FROM ( SELECT users.id AS user_id, users.sign_up_ip AS ip, users.created_at AS used_at @@ -1448,7 +1448,7 @@ login_activities.created_at FROM login_activities WHERE (login_activities.success = true)) t0 - GROUP BY user_id, ip; + GROUP BY t0.user_id, t0.ip; SQL create_view "account_summaries", materialized: true, sql_definition: <<-SQL SELECT accounts.id AS account_id, @@ -1469,9 +1469,9 @@ add_index "account_summaries", ["account_id"], name: "index_account_summaries_on_account_id", unique: true create_view "global_follow_recommendations", materialized: true, sql_definition: <<-SQL - SELECT account_id, - sum(rank) AS rank, - array_agg(reason) AS reason + SELECT t0.account_id, + sum(t0.rank) AS rank, + array_agg(t0.reason) AS reason FROM ( SELECT account_summaries.account_id, ((count(follows.id))::numeric / (1.0 + (count(follows.id))::numeric)) AS rank, 'most_followed'::text AS reason @@ -1495,8 +1495,8 @@ WHERE (follow_recommendation_suppressions.account_id = statuses.account_id))))) GROUP BY account_summaries.account_id HAVING (sum((status_stats.reblogs_count + status_stats.favourites_count)) >= (5)::numeric)) t0 - GROUP BY account_id - ORDER BY (sum(rank)) DESC; + GROUP BY t0.account_id + ORDER BY (sum(t0.rank)) DESC; SQL add_index "global_follow_recommendations", ["account_id"], name: "index_global_follow_recommendations_on_account_id", unique: true