Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Working version of fetch all replies service with global maximum on f…
Browse files Browse the repository at this point in the history
…etching
sneakers-the-rat committed Oct 13, 2024
1 parent 58ddff3 commit 793ccd0
Showing 3 changed files with 28 additions and 13 deletions.
2 changes: 1 addition & 1 deletion app/controllers/api/v1/statuses_controller.rb
Original file line number Diff line number Diff line change
@@ -62,9 +62,9 @@ def context
if !current_account.nil? && @status.should_fetch_replies?
ActivityPub::FetchAllRepliesWorker.perform_async(
@status.id,
current_account.id,
{
allow_synchronous_requests: true,
current_account_id: current_account.id,
}
)
end
2 changes: 1 addition & 1 deletion app/models/concerns/status/fetch_replies_concern.rb
Original file line number Diff line number Diff line change
@@ -20,7 +20,7 @@ module Status::FetchRepliesConcern

def should_fetch_replies?
# we aren't brand new, and we haven't fetched replies since the debounce window
!local? && created_at <= 10.minutes.ago && (
!local? && created_at <= CREATED_RECENTLY_DEBOUNCE.ago && (
fetched_replies_at.nil? || fetched_replies_at <= FETCH_REPLIES_DEBOUNCE.ago
)
end
37 changes: 26 additions & 11 deletions app/workers/activitypub/fetch_all_replies_worker.rb
Original file line number Diff line number Diff line change
@@ -7,39 +7,54 @@
class ActivityPub::FetchAllRepliesWorker
include Sidekiq::Worker
include ExponentialBackoff
include JsonLdHelper

sidekiq_options queue: 'pull', retry: 3

# Global max replies to fetch per request
MAX_REPLIES = 1000

def perform(parent_status_id, options = {})
def perform(parent_status_id, current_account_id = nil, options = {})
@parent_status = Status.find(parent_status_id)
@current_account_id = options.fetch(:current_account_id, nil)
@current_account = @current_account_id.nil? ? nil : Account.find(id: @current_account_id)
@current_account_id = current_account_id
@current_account = @current_account_id.nil? ? nil : Account.find(@current_account_id)
Rails.logger.debug { "FetchAllRepliesWorker - #{parent_status_id}: Fetching all replies for status: #{@parent_status}" }

all_replies = get_replies(@parent_status.uri)
all_replies = get_replies(@parent_status.uri, options)
got_replies = all_replies.length
until all_replies.empty? || got_replies >= MAX_REPLIES
new_replies = get_replies(all_replies.pop)
next_reply = all_replies.pop
next if next_reply.nil?

new_replies = get_replies(next_reply, options)
next if new_replies.nil?

got_replies += new_replies.length
all_replies << new_replies
all_replies.concat(new_replies)
end

Rails.logger.debug { "FetchAllRepliesWorker - #{parent_status_id}: fetched #{got_replies} replies" }
got_replies
end

private

def get_replies(status_uri)
def get_replies(status_uri, options = {})
replies_uri = get_replies_uri(status_uri)
return if replies_uri.nil?

ActivityPub::FetchAllRepliesService.new.call(replies_uri, **options.deep_symbolize_keys)
end

def get_replies_uri(parent_status_uri)
json_status = fetch_resource(parent_status_uri, true, @current_account)
replies_uri = json_status['replies']
Rails.logger.debug { "Could not find replies uri for status URI: #{parent_status_uri}" } if replies_uri.nil?
replies_uri
begin
json_status = fetch_resource(parent_status_uri, true, @current_account)
replies_uri = json_status['replies']
Rails.logger.debug { "FetchAllRepliesWorker - #{@parent_status_id}: replies URI was nil" } if replies_uri.nil?
replies_uri
rescue => e
Rails.logger.error { "FetchAllRepliesWorker - #{@parent_status_id}: Got exception while resolving replies URI: #{e} - #{e.message}" }
nil
end
end
end

0 comments on commit 793ccd0

Please sign in to comment.