Browse Source

Speed up some rake tasks by moving execution to Sidekiq (#7678)

* Speed up some rake tasks by moving execution to Sidekiq

mastodon:media:remove_silenced
mastodon:media:remove_remote
mastodon:media:redownload_avatars
mastodon:feeds:build

* Fix code style issue
pull/4/head
Eugen Rochko 5 years ago
committed by GitHub
parent
commit
ad40bf5e0c
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 57 additions and 16 deletions
  1. +14
    -0
      app/workers/maintenance/destroy_media_worker.rb
  2. +16
    -0
      app/workers/maintenance/redownload_account_media_worker.rb
  3. +18
    -0
      app/workers/maintenance/uncache_media_worker.rb
  4. +9
    -16
      lib/tasks/mastodon.rake

+ 14
- 0
app/workers/maintenance/destroy_media_worker.rb View File

@ -0,0 +1,14 @@
# frozen_string_literal: true
class Maintenance::DestroyMediaWorker
include Sidekiq::Worker
sidekiq_options queue: 'pull'
def perform(media_attachment_id)
media = MediaAttachment.find(media_attachment_id)
media.destroy
rescue ActiveRecord::RecordNotFound
true
end
end

+ 16
- 0
app/workers/maintenance/redownload_account_media_worker.rb View File

@ -0,0 +1,16 @@
# frozen_string_literal: true
class Maintenance::RedownloadAccountMediaWorker
include Sidekiq::Worker
sidekiq_options queue: 'pull', retry: false
def perform(account_id)
account = Account.find(account_id)
account.reset_avatar!
account.reset_header!
account.save
rescue ActiveRecord::RecordNotFound
true
end
end

+ 18
- 0
app/workers/maintenance/uncache_media_worker.rb View File

@ -0,0 +1,18 @@
# frozen_string_literal: true
class Maintenance::UncacheMediaWorker
include Sidekiq::Worker
sidekiq_options queue: 'pull'
def perform(media_attachment_id)
media = MediaAttachment.find(media_attachment_id)
return unless media.file.exists?
media.file.destroy
media.save
rescue ActiveRecord::RecordNotFound
true
end
end

+ 9
- 16
lib/tasks/mastodon.rake View File

@ -502,18 +502,17 @@ namespace :mastodon do
desc 'Remove media attachments attributed to silenced accounts'
task remove_silenced: :environment do
MediaAttachment.where(account: Account.silenced).find_each(&:destroy)
MediaAttachment.where(account: Account.silenced).select(:id).find_in_batches do |media_attachments|
Maintenance::DestroyMediaWorker.push_bulk(media_attachments.map(&:id))
end
end
desc 'Remove cached remote media attachments that are older than NUM_DAYS. By default 7 (week)'
task remove_remote: :environment do
time_ago = ENV.fetch('NUM_DAYS') { 7 }.to_i.days.ago
MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).find_each do |media|
next unless media.file.exists?
media.file.destroy
media.save
MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).find_in_batches do |media_attachments|
Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id))
end
end
@ -529,14 +528,8 @@ namespace :mastodon do
accounts = Account.remote
accounts = accounts.where(domain: ENV['DOMAIN']) if ENV['DOMAIN'].present?
accounts.find_each do |account|
begin
account.reset_avatar!
account.reset_header!
account.save
rescue Paperclip::Error
puts "Error resetting avatar and header for account #{username}@#{domain}"
end
accounts.select(:id).find_in_batches do |accounts_batch|
Maintenance::RedownloadAccountMediaWorker.push_bulk(accounts_batch.map(&:id))
end
end
end
@ -568,8 +561,8 @@ namespace :mastodon do
desc 'Generates home timelines for users who logged in in the past two weeks'
task build: :environment do
User.active.includes(:account).find_each do |u|
PrecomputeFeedService.new.call(u.account)
User.active.select(:account_id).find_in_batches do |users|
RegenerationWorker.push_bulk(users.map(&:account_id))
end
end
end

Loading…
Cancel
Save