|
|
|
@ -22,285 +22,268 @@ class ProcessFeedService < BaseService |
|
|
|
class ProcessEntry |
|
|
|
def call(xml, account) |
|
|
|
@account = account |
|
|
|
@fetched = Activity.new(xml) |
|
|
|
|
|
|
|
return unless [:activity, :note, :comment].include?(@fetched.type) |
|
|
|
|
|
|
|
klass = case @fetched.verb |
|
|
|
when :post |
|
|
|
PostActivity |
|
|
|
when :share |
|
|
|
ShareActivity |
|
|
|
when :delete |
|
|
|
DeletionActivity |
|
|
|
else |
|
|
|
return |
|
|
|
end |
|
|
|
|
|
|
|
@fetched = klass.new(xml, account) |
|
|
|
@fetched.perform |
|
|
|
@xml = xml |
|
|
|
|
|
|
|
return if skip_unsupported_type? |
|
|
|
|
|
|
|
case verb |
|
|
|
when :post, :share |
|
|
|
return create_status |
|
|
|
when :delete |
|
|
|
return delete_status |
|
|
|
end |
|
|
|
rescue ActiveRecord::RecordInvalid => e |
|
|
|
Rails.logger.debug "Nothing was saved for #{id} because: #{e}" |
|
|
|
nil |
|
|
|
end |
|
|
|
|
|
|
|
class Activity |
|
|
|
def initialize(xml, account = nil) |
|
|
|
@xml = xml |
|
|
|
@account = account |
|
|
|
end |
|
|
|
private |
|
|
|
|
|
|
|
def verb |
|
|
|
raw = @xml.at_xpath('./activity:verb', activity: TagManager::AS_XMLNS).content |
|
|
|
TagManager::VERBS.key(raw) |
|
|
|
rescue |
|
|
|
:post |
|
|
|
def create_status |
|
|
|
if redis.exists("delete_upon_arrival:#{@account.id}:#{id}") |
|
|
|
Rails.logger.debug "Delete for status #{id} was queued, ignoring" |
|
|
|
return |
|
|
|
end |
|
|
|
|
|
|
|
def type |
|
|
|
raw = @xml.at_xpath('./activity:object-type', activity: TagManager::AS_XMLNS).content |
|
|
|
TagManager::TYPES.key(raw) |
|
|
|
rescue |
|
|
|
:activity |
|
|
|
end |
|
|
|
status, just_created = nil |
|
|
|
|
|
|
|
def id |
|
|
|
@xml.at_xpath('./xmlns:id', xmlns: TagManager::XMLNS).content |
|
|
|
end |
|
|
|
Rails.logger.debug "Creating remote status #{id}" |
|
|
|
|
|
|
|
def url |
|
|
|
link = @xml.at_xpath('./xmlns:link[@rel="alternate"]', xmlns: TagManager::XMLNS) |
|
|
|
link.nil? ? nil : link['href'] |
|
|
|
if verb == :share |
|
|
|
original_status = shared_status_from_xml(@xml.at_xpath('.//activity:object', activity: TagManager::AS_XMLNS)) |
|
|
|
return nil if original_status.nil? |
|
|
|
end |
|
|
|
|
|
|
|
private |
|
|
|
ApplicationRecord.transaction do |
|
|
|
status, just_created = status_from_xml(@xml) |
|
|
|
|
|
|
|
def find_status(uri) |
|
|
|
if TagManager.instance.local_id?(uri) |
|
|
|
local_id = TagManager.instance.unique_tag_to_local_id(uri, 'Status') |
|
|
|
return Status.find_by(id: local_id) |
|
|
|
return if status.nil? |
|
|
|
return status unless just_created |
|
|
|
|
|
|
|
if verb == :share |
|
|
|
status.reblog = original_status.reblog? ? original_status.reblog : original_status |
|
|
|
end |
|
|
|
|
|
|
|
Status.find_by(uri: uri) |
|
|
|
status.save! |
|
|
|
end |
|
|
|
|
|
|
|
def redis |
|
|
|
Redis.current |
|
|
|
if thread?(@xml) && status.thread.nil? |
|
|
|
Rails.logger.debug "Trying to attach #{status.id} (#{id(@xml)}) to #{thread(@xml).first}" |
|
|
|
ThreadResolveWorker.perform_async(status.id, thread(@xml).second) |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
class CreationActivity < Activity |
|
|
|
def perform |
|
|
|
if redis.exists("delete_upon_arrival:#{@account.id}:#{id}") |
|
|
|
Rails.logger.debug "Delete for status #{id} was queued, ignoring" |
|
|
|
return |
|
|
|
end |
|
|
|
notify_about_mentions!(status) unless status.reblog? |
|
|
|
notify_about_reblog!(status) if status.reblog? && status.reblog.account.local? |
|
|
|
|
|
|
|
Rails.logger.debug "Creating remote status #{id}" |
|
|
|
# Return early if status already exists in db |
|
|
|
status = find_status(id) |
|
|
|
|
|
|
|
return [status, false] unless status.nil? |
|
|
|
|
|
|
|
return [nil, false] if @account.suspended? |
|
|
|
|
|
|
|
status = Status.create!( |
|
|
|
uri: id, |
|
|
|
url: url, |
|
|
|
account: @account, |
|
|
|
reblog: reblog, |
|
|
|
text: content, |
|
|
|
spoiler_text: content_warning, |
|
|
|
created_at: published, |
|
|
|
reply: thread?, |
|
|
|
language: content_language, |
|
|
|
visibility: visibility_scope, |
|
|
|
conversation: converstation_to_persistent&.first, |
|
|
|
thread: thread? ? find_status(thread.first) : nil |
|
|
|
) |
|
|
|
|
|
|
|
save_mentions(status) |
|
|
|
save_hashtags(status) |
|
|
|
save_media(status) |
|
|
|
|
|
|
|
if thread? && status.thread.nil? |
|
|
|
Rails.logger.debug "Trying to attach #{status.id} (#{id}) to #{thread.first}" |
|
|
|
ThreadResolveWorker.perform_async(status.id, thread.second) |
|
|
|
end |
|
|
|
Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution" |
|
|
|
|
|
|
|
Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution" |
|
|
|
LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text? |
|
|
|
DistributionWorker.perform_async(status.id) |
|
|
|
|
|
|
|
LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text? |
|
|
|
DistributionWorker.perform_async(status.id) |
|
|
|
status |
|
|
|
end |
|
|
|
|
|
|
|
[status, true] |
|
|
|
def notify_about_mentions!(status) |
|
|
|
status.mentions.includes(:account).each do |mention| |
|
|
|
mentioned_account = mention.account |
|
|
|
next unless mentioned_account.local? |
|
|
|
NotifyService.new.call(mentioned_account, mention) |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
def content |
|
|
|
@xml.at_xpath('./xmlns:content', xmlns: TagManager::XMLNS).content |
|
|
|
end |
|
|
|
def notify_about_reblog!(status) |
|
|
|
NotifyService.new.call(status.reblog.account, status) |
|
|
|
end |
|
|
|
|
|
|
|
def content_language |
|
|
|
@xml.at_xpath('./xmlns:content', xmlns: TagManager::XMLNS)['xml:lang']&.presence || 'en'; |
|
|
|
end |
|
|
|
def delete_status |
|
|
|
Rails.logger.debug "Deleting remote status #{id}" |
|
|
|
status = Status.find_by(uri: id, account: @account) |
|
|
|
|
|
|
|
def content_warning |
|
|
|
@xml.at_xpath('./xmlns:summary', xmlns: TagManager::XMLNS)&.content || '' |
|
|
|
end |
|
|
|
|
|
|
|
def visibility_scope |
|
|
|
@xml.at_xpath('./mastodon:scope', mastodon: TagManager::MTDN_XMLNS)&.content&.to_sym || :public |
|
|
|
if status.nil? |
|
|
|
redis.setex("delete_upon_arrival:#{@account.id}:#{id}", 6 * 3_600, id) |
|
|
|
else |
|
|
|
RemoveStatusService.new.call(status) |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
def published |
|
|
|
@xml.at_xpath('./xmlns:published', xmlns: TagManager::XMLNS).content |
|
|
|
end |
|
|
|
def skip_unsupported_type? |
|
|
|
!([:post, :share, :delete].include?(verb) && [:activity, :note, :comment].include?(type)) |
|
|
|
end |
|
|
|
|
|
|
|
def thread? |
|
|
|
!@xml.at_xpath('./thr:in-reply-to', thr: TagManager::THR_XMLNS).nil? |
|
|
|
end |
|
|
|
def shared_status_from_xml(entry) |
|
|
|
status = find_status(id(entry)) |
|
|
|
|
|
|
|
def thread |
|
|
|
thr = @xml.at_xpath('./thr:in-reply-to', thr: TagManager::THR_XMLNS) |
|
|
|
[thr['ref'], thr['href']] |
|
|
|
end |
|
|
|
return status unless status.nil? |
|
|
|
|
|
|
|
private |
|
|
|
FetchRemoteStatusService.new.call(url(entry)) |
|
|
|
end |
|
|
|
|
|
|
|
def converstation_to_persistent |
|
|
|
uri = @xml.at_xpath('./ostatus:conversation', ostatus: TagManager::OS_XMLNS)&.attribute('ref')&.content |
|
|
|
return if uri.nil? |
|
|
|
def status_from_xml(entry) |
|
|
|
# Return early if status already exists in db |
|
|
|
status = find_status(id(entry)) |
|
|
|
|
|
|
|
if TagManager.instance.local_id?(uri) |
|
|
|
local_id = TagManager.instance.unique_tag_to_local_id(uri, 'Conversation') |
|
|
|
return Conversation.find_by(id: local_id) |
|
|
|
end |
|
|
|
return [status, false] unless status.nil? |
|
|
|
|
|
|
|
found = Conversation.find_by(uri: uri) |
|
|
|
found ? [found, false] : [Conversation.create!(uri: uri), true] |
|
|
|
end |
|
|
|
account = @account |
|
|
|
|
|
|
|
def save_mentions(parent) |
|
|
|
processed_account_ids = [] |
|
|
|
return [nil, false] if account.suspended? |
|
|
|
|
|
|
|
@xml.xpath('./xmlns:link[@rel="mentioned"]', xmlns: TagManager::XMLNS).each do |link| |
|
|
|
next if [TagManager::TYPES[:group], TagManager::TYPES[:collection]].include? link['ostatus:object-type'] |
|
|
|
status = Status.create!( |
|
|
|
uri: id(entry), |
|
|
|
url: url(entry), |
|
|
|
account: account, |
|
|
|
text: content(entry), |
|
|
|
spoiler_text: content_warning(entry), |
|
|
|
created_at: published(entry), |
|
|
|
reply: thread?(entry), |
|
|
|
language: content_language(entry), |
|
|
|
visibility: visibility_scope(entry), |
|
|
|
conversation: find_or_create_conversation(entry), |
|
|
|
thread: thread?(entry) ? find_status(thread(entry).first) : nil |
|
|
|
) |
|
|
|
|
|
|
|
mentioned_account = account_from_href(link['href']) |
|
|
|
mentions_from_xml(status, entry) |
|
|
|
hashtags_from_xml(status, entry) |
|
|
|
media_from_xml(status, entry) |
|
|
|
|
|
|
|
next if mentioned_account.nil? || processed_account_ids.include?(mentioned_account.id) |
|
|
|
[status, true] |
|
|
|
end |
|
|
|
|
|
|
|
mentioned_account.mentions.where(status: parent).first_or_create(status: parent) |
|
|
|
def find_or_create_conversation(xml) |
|
|
|
uri = xml.at_xpath('./ostatus:conversation', ostatus: TagManager::OS_XMLNS)&.attribute('ref')&.content |
|
|
|
return if uri.nil? |
|
|
|
|
|
|
|
# So we can skip duplicate mentions |
|
|
|
processed_account_ids << mentioned_account.id |
|
|
|
end |
|
|
|
if TagManager.instance.local_id?(uri) |
|
|
|
local_id = TagManager.instance.unique_tag_to_local_id(uri, 'Conversation') |
|
|
|
return Conversation.find_by(id: local_id) |
|
|
|
end |
|
|
|
|
|
|
|
def save_hashtags(parent) |
|
|
|
tags = @xml.xpath('./xmlns:category', xmlns: TagManager::XMLNS).map { |category| category['term'] }.select(&:present?) |
|
|
|
ProcessHashtagsService.new.call(parent, tags) |
|
|
|
Conversation.find_by(uri: uri) || Conversation.create!(uri: uri) |
|
|
|
end |
|
|
|
|
|
|
|
def find_status(uri) |
|
|
|
if TagManager.instance.local_id?(uri) |
|
|
|
local_id = TagManager.instance.unique_tag_to_local_id(uri, 'Status') |
|
|
|
return Status.find_by(id: local_id) |
|
|
|
end |
|
|
|
|
|
|
|
def save_media(parent) |
|
|
|
do_not_download = DomainBlock.find_by(domain: parent.account.domain)&.reject_media? |
|
|
|
Status.find_by(uri: uri) |
|
|
|
end |
|
|
|
|
|
|
|
@xml.xpath('./xmlns:link[@rel="enclosure"]', xmlns: TagManager::XMLNS).each do |link| |
|
|
|
next unless link['href'] |
|
|
|
def mentions_from_xml(parent, xml) |
|
|
|
processed_account_ids = [] |
|
|
|
|
|
|
|
media = MediaAttachment.where(status: parent, remote_url: link['href']).first_or_initialize(account: parent.account, status: parent, remote_url: link['href']) |
|
|
|
parsed_url = Addressable::URI.parse(link['href']).normalize |
|
|
|
xml.xpath('./xmlns:link[@rel="mentioned"]', xmlns: TagManager::XMLNS).each do |link| |
|
|
|
next if [TagManager::TYPES[:group], TagManager::TYPES[:collection]].include? link['ostatus:object-type'] |
|
|
|
|
|
|
|
next if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.empty? |
|
|
|
mentioned_account = account_from_href(link['href']) |
|
|
|
|
|
|
|
media.save |
|
|
|
next if mentioned_account.nil? || processed_account_ids.include?(mentioned_account.id) |
|
|
|
|
|
|
|
next if do_not_download |
|
|
|
mentioned_account.mentions.where(status: parent).first_or_create(status: parent) |
|
|
|
|
|
|
|
begin |
|
|
|
media.file_remote_url = link['href'] |
|
|
|
media.save! |
|
|
|
rescue ActiveRecord::RecordInvalid |
|
|
|
next |
|
|
|
end |
|
|
|
end |
|
|
|
# So we can skip duplicate mentions |
|
|
|
processed_account_ids << mentioned_account.id |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
def account_from_href(href) |
|
|
|
url = Addressable::URI.parse(href).normalize |
|
|
|
def account_from_href(href) |
|
|
|
url = Addressable::URI.parse(href).normalize |
|
|
|
|
|
|
|
if TagManager.instance.web_domain?(url.host) |
|
|
|
Account.find_local(url.path.gsub('/users/', '')) |
|
|
|
else |
|
|
|
Account.where(uri: href).or(Account.where(url: href)).first || FetchRemoteAccountService.new.call(href) |
|
|
|
end |
|
|
|
if TagManager.instance.web_domain?(url.host) |
|
|
|
Account.find_local(url.path.gsub('/users/', '')) |
|
|
|
else |
|
|
|
Account.where(uri: href).or(Account.where(url: href)).first || FetchRemoteAccountService.new.call(href) |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
class ShareActivity < CreationActivity |
|
|
|
def perform |
|
|
|
status, just_created = super |
|
|
|
NotifyService.new.call(reblog.account, status) if reblog&.account&.local? && just_created |
|
|
|
status |
|
|
|
end |
|
|
|
def hashtags_from_xml(parent, xml) |
|
|
|
tags = xml.xpath('./xmlns:category', xmlns: TagManager::XMLNS).map { |category| category['term'] }.select(&:present?) |
|
|
|
ProcessHashtagsService.new.call(parent, tags) |
|
|
|
end |
|
|
|
|
|
|
|
def object |
|
|
|
@xml.at_xpath('.//activity:object', activity: TagManager::AS_XMLNS) |
|
|
|
end |
|
|
|
def media_from_xml(parent, xml) |
|
|
|
do_not_download = DomainBlock.find_by(domain: parent.account.domain)&.reject_media? |
|
|
|
|
|
|
|
private |
|
|
|
xml.xpath('./xmlns:link[@rel="enclosure"]', xmlns: TagManager::XMLNS).each do |link| |
|
|
|
next unless link['href'] |
|
|
|
|
|
|
|
def status |
|
|
|
reblog && super |
|
|
|
end |
|
|
|
media = MediaAttachment.where(status: parent, remote_url: link['href']).first_or_initialize(account: parent.account, status: parent, remote_url: link['href']) |
|
|
|
parsed_url = Addressable::URI.parse(link['href']).normalize |
|
|
|
|
|
|
|
next if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.empty? |
|
|
|
|
|
|
|
media.save |
|
|
|
|
|
|
|
def reblog |
|
|
|
return @reblog if defined? @reblog |
|
|
|
next if do_not_download |
|
|
|
|
|
|
|
original_status = RemoteActivity.new(object).perform |
|
|
|
@reblog = original_status.reblog? ? original_status.reblog : original_status |
|
|
|
begin |
|
|
|
media.file_remote_url = link['href'] |
|
|
|
media.save! |
|
|
|
rescue ActiveRecord::RecordInvalid |
|
|
|
next |
|
|
|
end |
|
|
|
end |
|
|
|
end |
|
|
|
|
|
|
|
class PostActivity < CreationActivity |
|
|
|
def perform |
|
|
|
status, just_created = super |
|
|
|
def id(xml = @xml) |
|
|
|
xml.at_xpath('./xmlns:id', xmlns: TagManager::XMLNS).content |
|
|
|
end |
|
|
|
|
|
|
|
if just_created |
|
|
|
status.mentions.includes(:account).each do |mention| |
|
|
|
mentioned_account = mention.account |
|
|
|
next unless mentioned_account.local? |
|
|
|
NotifyService.new.call(mentioned_account, mention) |
|
|
|
end |
|
|
|
end |
|
|
|
def verb(xml = @xml) |
|
|
|
raw = xml.at_xpath('./activity:verb', activity: TagManager::AS_XMLNS).content |
|
|
|
TagManager::VERBS.key(raw) |
|
|
|
rescue |
|
|
|
:post |
|
|
|
end |
|
|
|
|
|
|
|
status |
|
|
|
end |
|
|
|
def type(xml = @xml) |
|
|
|
raw = xml.at_xpath('./activity:object-type', activity: TagManager::AS_XMLNS).content |
|
|
|
TagManager::TYPES.key(raw) |
|
|
|
rescue |
|
|
|
:activity |
|
|
|
end |
|
|
|
|
|
|
|
def url(xml = @xml) |
|
|
|
link = xml.at_xpath('./xmlns:link[@rel="alternate"]', xmlns: TagManager::XMLNS) |
|
|
|
link.nil? ? nil : link['href'] |
|
|
|
end |
|
|
|
|
|
|
|
private |
|
|
|
def content(xml = @xml) |
|
|
|
xml.at_xpath('./xmlns:content', xmlns: TagManager::XMLNS).content |
|
|
|
end |
|
|
|
|
|
|
|
def reblog |
|
|
|
nil |
|
|
|
end |
|
|
|
def content_language(xml = @xml) |
|
|
|
xml.at_xpath('./xmlns:content', xmlns: TagManager::XMLNS)['xml:lang']&.presence || 'en' |
|
|
|
end |
|
|
|
|
|
|
|
class DeletionActivity < Activity |
|
|
|
def perform |
|
|
|
Rails.logger.debug "Deleting remote status #{id}" |
|
|
|
status = Status.find_by(uri: id, account: @account) |
|
|
|
def content_warning(xml = @xml) |
|
|
|
xml.at_xpath('./xmlns:summary', xmlns: TagManager::XMLNS)&.content || '' |
|
|
|
end |
|
|
|
|
|
|
|
if status.nil? |
|
|
|
redis.setex("delete_upon_arrival:#{@account.id}:#{id}", 6 * 3_600, id) |
|
|
|
else |
|
|
|
RemoveStatusService.new.call(status) |
|
|
|
end |
|
|
|
end |
|
|
|
def visibility_scope(xml = @xml) |
|
|
|
xml.at_xpath('./mastodon:scope', mastodon: TagManager::MTDN_XMLNS)&.content&.to_sym || :public |
|
|
|
end |
|
|
|
|
|
|
|
class RemoteActivity < Activity |
|
|
|
def perform |
|
|
|
find_status(id) || FetchRemoteStatusService.new.call(url) |
|
|
|
end |
|
|
|
def published(xml = @xml) |
|
|
|
xml.at_xpath('./xmlns:published', xmlns: TagManager::XMLNS).content |
|
|
|
end |
|
|
|
|
|
|
|
def thread?(xml = @xml) |
|
|
|
!xml.at_xpath('./thr:in-reply-to', thr: TagManager::THR_XMLNS).nil? |
|
|
|
end |
|
|
|
|
|
|
|
def thread(xml = @xml) |
|
|
|
thr = xml.at_xpath('./thr:in-reply-to', thr: TagManager::THR_XMLNS) |
|
|
|
[thr['ref'], thr['href']] |
|
|
|
end |
|
|
|
|
|
|
|
def account?(xml = @xml) |
|
|
|
!xml.at_xpath('./xmlns:author', xmlns: TagManager::XMLNS).nil? |
|
|
|
end |
|
|
|
|
|
|
|
def redis |
|
|
|
Redis.current |
|
|
|
end |
|
|
|
end |
|
|
|
end |
nightpool
7 years ago
Eugen Rochko