You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

163 lines
5.7 KiB

  1. # frozen_string_literal: true
  2. class FetchLinkCardService < BaseService
  3. URL_PATTERN = %r{
  4. ( # $1 URL
  5. (https?:\/\/) # $2 Protocol (required)
  6. (#{Twitter::Regex[:valid_domain]}) # $3 Domain(s)
  7. (?::(#{Twitter::Regex[:valid_port_number]}))? # $4 Port number (optional)
  8. (/#{Twitter::Regex[:valid_url_path]}*)? # $5 URL Path and anchor
  9. (\?#{Twitter::Regex[:valid_url_query_chars]}*#{Twitter::Regex[:valid_url_query_ending_chars]})? # $6 Query String
  10. )
  11. }iox
  12. def call(status)
  13. @status = status
  14. @url = parse_urls
  15. return if @url.nil? || @status.preview_cards.any?
  16. @url = @url.to_s
  17. RedisLock.acquire(lock_options) do |lock|
  18. if lock.acquired?
  19. @card = PreviewCard.find_by(url: @url)
  20. process_url if @card.nil? || @card.updated_at <= 2.weeks.ago
  21. end
  22. end
  23. attach_card if @card&.persisted?
  24. rescue HTTP::Error, Addressable::URI::InvalidURIError => e
  25. Rails.logger.debug "Error fetching link #{@url}: #{e}"
  26. nil
  27. end
  28. private
  29. def process_url
  30. @card ||= PreviewCard.new(url: @url)
  31. failed = Request.new(:head, @url).perform do |res|
  32. res.code != 405 && (res.code != 200 || res.mime_type != 'text/html')
  33. end
  34. return if failed
  35. Request.new(:get, @url).perform do |res|
  36. if res.code == 200 && res.mime_type == 'text/html'
  37. @html = res.to_s
  38. @html_charset = res.charset
  39. else
  40. @html = nil
  41. @html_charset = nil
  42. end
  43. end
  44. return if @html.nil?
  45. attempt_oembed || attempt_opengraph
  46. end
  47. def attach_card
  48. @status.preview_cards << @card
  49. end
  50. def parse_urls
  51. if @status.local?
  52. urls = @status.text.scan(URL_PATTERN).map { |array| Addressable::URI.parse(array[0]).normalize }
  53. else
  54. html = Nokogiri::HTML(@status.text)
  55. links = html.css('a')
  56. urls = links.map { |a| Addressable::URI.parse(a['href']).normalize unless skip_link?(a) }.compact
  57. end
  58. urls.reject { |uri| bad_url?(uri) }.first
  59. end
  60. def bad_url?(uri)
  61. # Avoid local instance URLs and invalid URLs
  62. uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme)
  63. end
  64. def skip_link?(a)
  65. # Avoid links for hashtags and mentions (microformats)
  66. a['rel']&.include?('tag') || a['class']&.include?('u-url')
  67. end
  68. def attempt_oembed
  69. embed = OEmbed::Providers.get(@url, html: @html)
  70. return false unless embed.respond_to?(:type)
  71. @card.type = embed.type
  72. @card.title = embed.respond_to?(:title) ? embed.title : ''
  73. @card.author_name = embed.respond_to?(:author_name) ? embed.author_name : ''
  74. @card.author_url = embed.respond_to?(:author_url) ? embed.author_url : ''
  75. @card.provider_name = embed.respond_to?(:provider_name) ? embed.provider_name : ''
  76. @card.provider_url = embed.respond_to?(:provider_url) ? embed.provider_url : ''
  77. @card.width = 0
  78. @card.height = 0
  79. case @card.type
  80. when 'link'
  81. @card.image_remote_url = embed.thumbnail_url if embed.respond_to?(:thumbnail_url)
  82. when 'photo'
  83. return false unless embed.respond_to?(:url)
  84. @card.embed_url = embed.url
  85. @card.image_remote_url = embed.url
  86. @card.width = embed.width.presence || 0
  87. @card.height = embed.height.presence || 0
  88. when 'video'
  89. @card.width = embed.width.presence || 0
  90. @card.height = embed.height.presence || 0
  91. @card.html = Formatter.instance.sanitize(embed.html, Sanitize::Config::MASTODON_OEMBED)
  92. @card.image_remote_url = embed.thumbnail_url if embed.respond_to?(:thumbnail_url)
  93. when 'rich'
  94. # Most providers rely on <script> tags, which is a no-no
  95. return false
  96. end
  97. @card.save_with_optional_image!
  98. rescue OEmbed::NotFound
  99. false
  100. end
  101. def attempt_opengraph
  102. detector = CharlockHolmes::EncodingDetector.new
  103. detector.strip_tags = true
  104. guess = detector.detect(@html, @html_charset)
  105. page = Nokogiri::HTML(@html, nil, guess&.fetch(:encoding, nil))
  106. if meta_property(page, 'twitter:player')
  107. @card.type = :video
  108. @card.width = meta_property(page, 'twitter:player:width') || 0
  109. @card.height = meta_property(page, 'twitter:player:height') || 0
  110. @card.html = content_tag(:iframe, nil, src: meta_property(page, 'twitter:player'),
  111. width: @card.width,
  112. height: @card.height,
  113. allowtransparency: 'true',
  114. scrolling: 'no',
  115. frameborder: '0')
  116. else
  117. @card.type = :link
  118. end
  119. @card.title = meta_property(page, 'og:title').presence || page.at_xpath('//title')&.content || ''
  120. @card.description = meta_property(page, 'og:description').presence || meta_property(page, 'description') || ''
  121. @card.image_remote_url = meta_property(page, 'og:image') if meta_property(page, 'og:image')
  122. return if @card.title.blank? && @card.html.blank?
  123. @card.save_with_optional_image!
  124. end
  125. def meta_property(page, property)
  126. page.at_xpath("//meta[@property=\"#{property}\"]")&.attribute('content')&.value || page.at_xpath("//meta[@name=\"#{property}\"]")&.attribute('content')&.value
  127. end
  128. def lock_options
  129. { redis: Redis.current, key: "fetch:#{@url}" }
  130. end
  131. end