* Fix #2473 - Use sidekiq scheduler to refresh PuSH subscriptions instead of cron Fix an issue where / in domain would raise exception in TagManager#normalize_domain PuSH subscriptions refresh done in a round-robin way to avoid hammering a single server's hub in sequence. Correct handling of failures/retries through Sidekiq (see also #2613). Optimize Account#with_followers scope. Also, since subscriptions are now delegated to Sidekiq jobs, an uncaught exception will not stop the entire refreshing operation halfway through Fix #2702 - Correct user agent header on outgoing http requests * Add test for SubscribeService * Extract #expiring_accounts into method * Make mastodon:push:refresh no-op * Queues are now defined in sidekiq.yml * Queues are now in sidekiq.ymlclosed-social-glitch-2
@ -1,2 +1,2 @@ | |||||
web: bundle exec puma -C config/puma.rb | web: bundle exec puma -C config/puma.rb | ||||
worker: bundle exec sidekiq -q default -q push -q pull -q mailers | |||||
worker: bundle exec sidekiq |
@ -1,3 +1,4 @@ | |||||
web: PORT=3000 bundle exec puma -C config/puma.rb | web: PORT=3000 bundle exec puma -C config/puma.rb | ||||
sidekiq: bundle exec sidekiq | |||||
stream: PORT=4000 yarn run start | stream: PORT=4000 yarn run start | ||||
webpack: ./bin/webpack-dev-server --host 0.0.0.0 | webpack: ./bin/webpack-dev-server --host 0.0.0.0 |
@ -1,13 +1,17 @@ | |||||
# frozen_string_literal: true | # frozen_string_literal: true | ||||
module HttpHelper | module HttpHelper | ||||
USER_AGENT = "#{HTTP::Request::USER_AGENT} (Mastodon/#{Mastodon::Version}; +http://#{Rails.configuration.x.local_domain}/)" | |||||
def http_client(options = {}) | def http_client(options = {}) | ||||
timeout = { write: 10, connect: 10, read: 10 }.merge(options) | timeout = { write: 10, connect: 10, read: 10 }.merge(options) | ||||
HTTP.headers(user_agent: USER_AGENT) | |||||
HTTP.headers(user_agent: user_agent) | |||||
.timeout(:per_operation, timeout) | .timeout(:per_operation, timeout) | ||||
.follow | .follow | ||||
end | end | ||||
private | |||||
def user_agent | |||||
@user_agent ||= "#{HTTP::Request::USER_AGENT} (Mastodon/#{Mastodon::Version}; +http://#{Rails.configuration.x.local_domain}/)" | |||||
end | |||||
end | end |
@ -0,0 +1,13 @@ | |||||
# frozen_string_literal: true | |||||
class Pubsubhubbub::SubscribeWorker | |||||
include Sidekiq::Worker | |||||
sidekiq_options queue: 'push' | |||||
def perform(account_id) | |||||
account = Account.find(account_id) | |||||
Rails.logger.debug "PuSH re-subscribing to #{account.acct}" | |||||
::SubscribeService.new.call(account) | |||||
end | |||||
end |
@ -0,0 +1,20 @@ | |||||
# frozen_string_literal: true | |||||
require 'sidekiq-scheduler' | |||||
class Scheduler::SubscriptionsScheduler | |||||
include Sidekiq::Worker | |||||
def perform | |||||
Rails.logger.debug 'Queueing PuSH re-subscriptions' | |||||
expiring_accounts.pluck(:id) do |id| | |||||
Pubsubhubbub::SubscribeWorker.perform_async(id) | |||||
end | |||||
end | |||||
private | |||||
def expiring_accounts | |||||
Account.expiring(1.day.from_now).partitioned | |||||
end | |||||
end |
@ -1,2 +1,11 @@ | |||||
--- | --- | ||||
:concurrency: 5 | :concurrency: 5 | ||||
:queues: | |||||
- default | |||||
- push | |||||
- pull | |||||
- mailers | |||||
:schedule: | |||||
subscriptions_scheduler: | |||||
cron: '0 5 * * *' | |||||
class: Scheduler::SubscriptionsScheduler |
@ -0,0 +1,38 @@ | |||||
require 'rails_helper' | |||||
RSpec.describe SubscribeService do | |||||
let(:account) { Fabricate(:account, username: 'bob', domain: 'example.com', hub_url: 'http://hub.example.com') } | |||||
subject { SubscribeService.new } | |||||
it 'sends subscription request to PuSH hub' do | |||||
stub_request(:post, 'http://hub.example.com/').to_return(status: 202) | |||||
subject.call(account) | |||||
expect(a_request(:post, 'http://hub.example.com/')).to have_been_made.once | |||||
end | |||||
it 'generates and keeps PuSH secret on successful call' do | |||||
stub_request(:post, 'http://hub.example.com/').to_return(status: 202) | |||||
subject.call(account) | |||||
expect(account.secret).to_not be_blank | |||||
end | |||||
it 'fails silently if PuSH hub forbids subscription' do | |||||
stub_request(:post, 'http://hub.example.com/').to_return(status: 403) | |||||
subject.call(account) | |||||
end | |||||
it 'fails silently if PuSH hub is not found' do | |||||
stub_request(:post, 'http://hub.example.com/').to_return(status: 404) | |||||
subject.call(account) | |||||
end | |||||
it 'fails loudly if there is a network error' do | |||||
stub_request(:post, 'http://hub.example.com/').to_raise(HTTP::Error) | |||||
expect { subject.call(account) }.to raise_error HTTP::Error | |||||
end | |||||
it 'fails loudly if PuSH hub is unavailable' do | |||||
stub_request(:post, 'http://hub.example.com/').to_return(status: 503) | |||||
expect { subject.call(account) }.to raise_error | |||||
end | |||||
end |