You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
238 lines
9.1 KiB
238 lines
9.1 KiB
# frozen_string_literal: true
|
|
|
|
require_relative '../../config/boot'
|
|
require_relative '../../config/environment'
|
|
require_relative 'cli_helper'
|
|
|
|
module Mastodon
|
|
class MediaCLI < Thor
|
|
include ActionView::Helpers::NumberHelper
|
|
include CLIHelper
|
|
|
|
def self.exit_on_failure?
|
|
true
|
|
end
|
|
|
|
option :days, type: :numeric, default: 7, aliases: [:d]
|
|
option :concurrency, type: :numeric, default: 5, aliases: [:c]
|
|
option :verbose, type: :boolean, default: false, aliases: [:v]
|
|
option :dry_run, type: :boolean, default: false
|
|
desc 'remove', 'Remove remote media files'
|
|
long_desc <<-DESC
|
|
Removes locally cached copies of media attachments from other servers.
|
|
|
|
The --days option specifies how old media attachments have to be before
|
|
they are removed. It defaults to 7 days.
|
|
DESC
|
|
def remove
|
|
time_ago = options[:days].days.ago
|
|
dry_run = options[:dry_run] ? '(DRY RUN)' : ''
|
|
|
|
processed, aggregate = parallelize_with_progress(MediaAttachment.cached.where.not(remote_url: '').where('created_at < ?', time_ago)) do |media_attachment|
|
|
next if media_attachment.file.blank?
|
|
|
|
size = media_attachment.file_file_size
|
|
|
|
unless options[:dry_run]
|
|
media_attachment.file.destroy
|
|
media_attachment.save
|
|
end
|
|
|
|
size
|
|
end
|
|
|
|
say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)}) #{dry_run}", :green, true)
|
|
end
|
|
|
|
option :start_after
|
|
option :dry_run, type: :boolean, default: false
|
|
desc 'remove-orphans', 'Scan storage and check for files that do not belong to existing media attachments'
|
|
long_desc <<~LONG_DESC
|
|
Scans file storage for files that do not belong to existing media attachments. Because this operation
|
|
requires iterating over every single file individually, it will be slow.
|
|
|
|
Please mind that some storage providers charge for the necessary API requests to list objects.
|
|
LONG_DESC
|
|
def remove_orphans
|
|
progress = create_progress_bar(nil)
|
|
reclaimed_bytes = 0
|
|
removed = 0
|
|
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
|
|
|
|
case Paperclip::Attachment.default_options[:storage]
|
|
when :s3
|
|
paperclip_instance = MediaAttachment.new.file
|
|
s3_interface = paperclip_instance.s3_interface
|
|
bucket = s3_interface.bucket(Paperclip::Attachment.default_options[:s3_credentials][:bucket])
|
|
last_key = options[:start_after]
|
|
|
|
loop do
|
|
objects = begin
|
|
begin
|
|
bucket.objects(start_after: last_key, prefix: 'media_attachments/files/').limit(1000).map { |x| x }
|
|
rescue => e
|
|
progress.log(pastel.red("Error fetching list of files: #{e}"))
|
|
progress.log("If you want to continue from this point, add --start-after=#{last_key} to your command") if last_key
|
|
break
|
|
end
|
|
end
|
|
|
|
break if objects.empty?
|
|
|
|
last_key = objects.last.key
|
|
attachments_map = MediaAttachment.where(id: objects.map { |object| object.key.split('/')[2..-2].join.to_i }).each_with_object({}) { |attachment, map| map[attachment.id] = attachment }
|
|
|
|
objects.each do |object|
|
|
attachment_id = object.key.split('/')[2..-2].join.to_i
|
|
filename = object.key.split('/').last
|
|
|
|
progress.increment
|
|
|
|
next unless attachments_map[attachment_id].nil? || !attachments_map[attachment_id].variant?(filename)
|
|
|
|
begin
|
|
object.delete unless options[:dry_run]
|
|
|
|
reclaimed_bytes += object.size
|
|
removed += 1
|
|
|
|
progress.log("Found and removed orphan: #{object.key}")
|
|
rescue => e
|
|
progress.log(pastel.red("Error processing #{object.key}: #{e}"))
|
|
end
|
|
end
|
|
end
|
|
when :fog
|
|
say('The fog storage driver is not supported for this operation at this time', :red)
|
|
exit(1)
|
|
when :filesystem
|
|
require 'find'
|
|
|
|
root_path = ENV.fetch('RAILS_ROOT_PATH', File.join(':rails_root', 'public', 'system')).gsub(':rails_root', Rails.root.to_s)
|
|
|
|
Find.find(File.join(root_path, 'media_attachments', 'files')) do |path|
|
|
next if File.directory?(path)
|
|
|
|
key = path.gsub("#{root_path}#{File::SEPARATOR}", '')
|
|
attachment_id = key.split(File::SEPARATOR)[2..-2].join.to_i
|
|
filename = key.split(File::SEPARATOR).last
|
|
attachment = MediaAttachment.find_by(id: attachment_id)
|
|
|
|
progress.increment
|
|
|
|
next unless attachment.nil? || !attachment.variant?(filename)
|
|
|
|
begin
|
|
size = File.size(path)
|
|
|
|
File.delete(path) unless options[:dry_run]
|
|
|
|
reclaimed_bytes += size
|
|
removed += 1
|
|
|
|
progress.log("Found and removed orphan: #{key}")
|
|
rescue => e
|
|
progress.log(pastel.red("Error processing #{key}: #{e}"))
|
|
end
|
|
end
|
|
end
|
|
|
|
progress.total = progress.progress
|
|
progress.finish
|
|
|
|
say("Removed #{removed} orphans (approx. #{number_to_human_size(reclaimed_bytes)})#{dry_run}", :green, true)
|
|
end
|
|
|
|
option :account, type: :string
|
|
option :domain, type: :string
|
|
option :status, type: :numeric
|
|
option :concurrency, type: :numeric, default: 5, aliases: [:c]
|
|
option :verbose, type: :boolean, default: false, aliases: [:v]
|
|
option :dry_run, type: :boolean, default: false
|
|
option :force, type: :boolean, default: false
|
|
desc 'refresh', 'Fetch remote media files'
|
|
long_desc <<-DESC
|
|
Re-downloads media attachments from other servers. You must specify the
|
|
source of media attachments with one of the following options:
|
|
|
|
Use the --status option to download attachments from a specific status,
|
|
using the status local numeric ID.
|
|
|
|
Use the --account option to download attachments from a specific account,
|
|
using username@domain handle of the account.
|
|
|
|
Use the --domain option to download attachments from a specific domain.
|
|
|
|
By default, attachments that are believed to be already downloaded will
|
|
not be re-downloaded. To force re-download of every URL, use --force.
|
|
DESC
|
|
def refresh
|
|
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
|
|
|
|
if options[:status]
|
|
scope = MediaAttachment.where(status_id: options[:status])
|
|
elsif options[:account]
|
|
username, domain = username.split('@')
|
|
account = Account.find_remote(username, domain)
|
|
|
|
if account.nil?
|
|
say('No such account', :red)
|
|
exit(1)
|
|
end
|
|
|
|
scope = MediaAttachment.where(account_id: account.id)
|
|
elsif options[:domain]
|
|
scope = MediaAttachment.joins(:account).merge(Account.by_domain_and_subdomains(options[:domain]))
|
|
else
|
|
exit(1)
|
|
end
|
|
|
|
processed, aggregate = parallelize_with_progress(scope) do |media_attachment|
|
|
next if media_attachment.remote_url.blank? || (!options[:force] && media_attachment.file_file_name.present?)
|
|
|
|
unless options[:dry_run]
|
|
media_attachment.reset_file!
|
|
media_attachment.save
|
|
end
|
|
|
|
media_attachment.file_file_size
|
|
end
|
|
|
|
say("Downloaded #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true)
|
|
end
|
|
|
|
desc 'usage', 'Calculate disk space consumed by Mastodon'
|
|
def usage
|
|
say("Attachments:\t#{number_to_human_size(MediaAttachment.sum(:file_file_size))} (#{number_to_human_size(MediaAttachment.where(account: Account.local).sum(:file_file_size))} local)")
|
|
say("Custom emoji:\t#{number_to_human_size(CustomEmoji.sum(:image_file_size))} (#{number_to_human_size(CustomEmoji.local.sum(:image_file_size))} local)")
|
|
say("Preview cards:\t#{number_to_human_size(PreviewCard.sum(:image_file_size))}")
|
|
say("Avatars:\t#{number_to_human_size(Account.sum(:avatar_file_size))} (#{number_to_human_size(Account.local.sum(:avatar_file_size))} local)")
|
|
say("Headers:\t#{number_to_human_size(Account.sum(:header_file_size))} (#{number_to_human_size(Account.local.sum(:header_file_size))} local)")
|
|
say("Backups:\t#{number_to_human_size(Backup.sum(:dump_file_size))}")
|
|
say("Imports:\t#{number_to_human_size(Import.sum(:data_file_size))}")
|
|
say("Settings:\t#{number_to_human_size(SiteUpload.sum(:file_file_size))}")
|
|
end
|
|
|
|
desc 'lookup', 'Lookup where media is displayed by passing a media URL'
|
|
def lookup
|
|
prompt = TTY::Prompt.new
|
|
|
|
url = prompt.ask('Please enter a URL to the media to lookup:', required: true)
|
|
|
|
attachment_id = url
|
|
.split('/')[0..-2]
|
|
.grep(/\A\d+\z/)
|
|
.join('')
|
|
|
|
if url.split('/')[0..-2].include? 'media_attachments'
|
|
model = MediaAttachment.find(attachment_id).status
|
|
prompt.say(ActivityPub::TagManager.instance.url_for(model))
|
|
elsif url.split('/')[0..-2].include? 'accounts'
|
|
model = Account.find(attachment_id)
|
|
prompt.say(ActivityPub::TagManager.instance.url_for(model))
|
|
else
|
|
prompt.say('Not found')
|
|
end
|
|
end
|
|
end
|
|
end
|