Merge branch 'main' into glitch-soc/merge-upstream

Conflicts:
- `app/views/admin/tags/index.html.haml`:
  Removed upstream while it had changes in glitch-soc to accomodate for the
  theming system.

Additional changes to accomodate for the theming system:
- `app/views/admin/trends/links/preview_card_providers/index.html.haml`
- `app/views/admin/trends/links/index.html.haml`
- `app/views/admin/trends/tags/index.html.haml`
- `app/views/admin/tags/show.html.haml`

app/chewy/tags_index.rb

@@ -31,7 +31,7 @@ class TagsIndex < Chewy::Index
     end
 
     field :reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? }
-    field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day[:accounts].to_i } }
+    field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day.accounts } }
     field :last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at }
   end
 end

app/controllers/admin/dashboard_controller.rb

@@ -4,7 +4,7 @@ module Admin
   class DashboardController < BaseController
     def index
       @system_checks         = Admin::SystemCheck.perform
-      @time_period           = (1.month.ago.to_date...Time.now.utc.to_date)
+      @time_period           = (29.days.ago.to_date...Time.now.utc.to_date)
       @pending_users_count   = User.pending.count
       @pending_reports_count = Report.unresolved.count
       @pending_tags_count    = Tag.pending_review.count

app/controllers/admin/tags_controller.rb

@@ -2,38 +2,12 @@
 
 module Admin
   class TagsController < BaseController
-    before_action :set_tag, except: [:index, :batch, :approve_all, :reject_all]
-    before_action :set_usage_by_domain, except: [:index, :batch, :approve_all, :reject_all]
-    before_action :set_counters, except: [:index, :batch, :approve_all, :reject_all]
-
-    def index
-      authorize :tag, :index?
-
-      @tags = filtered_tags.page(params[:page])
-      @form = Form::TagBatch.new
-    end
-
-    def batch
-      @form = Form::TagBatch.new(form_tag_batch_params.merge(current_account: current_account, action: action_from_button))
-      @form.save
-    rescue ActionController::ParameterMissing
-      flash[:alert] = I18n.t('admin.accounts.no_account_selected')
-    ensure
-      redirect_to admin_tags_path(filter_params)
-    end
-
-    def approve_all
-      Form::TagBatch.new(current_account: current_account, tag_ids: Tag.pending_review.pluck(:id), action: 'approve').save
-      redirect_to admin_tags_path(filter_params)
-    end
-
-    def reject_all
-      Form::TagBatch.new(current_account: current_account, tag_ids: Tag.pending_review.pluck(:id), action: 'reject').save
-      redirect_to admin_tags_path(filter_params)
-    end
+    before_action :set_tag
 
     def show
       authorize @tag, :show?
+
+      @time_period = (6.days.ago.to_date...Time.now.utc.to_date)
     end
 
     def update
@@ -52,52 +26,8 @@ module Admin
       @tag = Tag.find(params[:id])
     end
 
-    def set_usage_by_domain
-      @usage_by_domain = @tag.statuses
-                             .with_public_visibility
-                             .excluding_silenced_accounts
-                             .where(Status.arel_table[:id].gteq(Mastodon::Snowflake.id_at(Time.now.utc.beginning_of_day)))
-                             .joins(:account)
-                             .group('accounts.domain')
-                             .reorder(statuses_count: :desc)
-                             .pluck(Arel.sql('accounts.domain, count(*) AS statuses_count'))
-    end
-
-    def set_counters
-      @accounts_today = @tag.history.first[:accounts]
-      @accounts_week  = Redis.current.pfcount(*current_week_days.map { |day| "activity:tags:#{@tag.id}:#{day}:accounts" })
-    end
-
-    def filtered_tags
-      TagFilter.new(filter_params).results
-    end
-
-    def filter_params
-      params.slice(:page, *TagFilter::KEYS).permit(:page, *TagFilter::KEYS)
-    end
-
     def tag_params
       params.require(:tag).permit(:name, :trendable, :usable, :listable)
     end
-
-    def current_week_days
-      now = Time.now.utc.beginning_of_day.to_date
-
-      (Date.commercial(now.cwyear, now.cweek)..now).map do |date|
-        date.to_time(:utc).beginning_of_day.to_i
-      end
-    end
-
-    def form_tag_batch_params
-      params.require(:form_tag_batch).permit(:action, tag_ids: [])
-    end
-
-    def action_from_button
-      if params[:approve]
-        'approve'
-      elsif params[:reject]
-        'reject'
-      end
-    end
   end
 end

app/controllers/admin/trends/links/preview_card_providers_controller.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class Admin::Trends::Links::PreviewCardProvidersController < Admin::BaseController
+  def index
+    authorize :preview_card_provider, :index?
+
+    @preview_card_providers = filtered_preview_card_providers.page(params[:page])
+    @form = Form::PreviewCardProviderBatch.new
+  end
+
+  def batch
+    @form = Form::PreviewCardProviderBatch.new(form_preview_card_provider_batch_params.merge(current_account: current_account, action: action_from_button))
+    @form.save
+  rescue ActionController::ParameterMissing
+    flash[:alert] = I18n.t('admin.accounts.no_account_selected')
+  ensure
+    redirect_to admin_trends_links_preview_card_providers_path(filter_params)
+  end
+
+  private
+
+  def filtered_preview_card_providers
+    PreviewCardProviderFilter.new(filter_params).results
+  end
+
+  def filter_params
+    params.slice(:page, *PreviewCardProviderFilter::KEYS).permit(:page, *PreviewCardProviderFilter::KEYS)
+  end
+
+  def form_preview_card_provider_batch_params
+    params.require(:form_preview_card_provider_batch).permit(:action, preview_card_provider_ids: [])
+  end
+
+  def action_from_button
+    if params[:approve]
+      'approve'
+    elsif params[:reject]
+      'reject'
+    end
+  end
+end

app/controllers/admin/trends/links_controller.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+class Admin::Trends::LinksController < Admin::BaseController
+  def index
+    authorize :preview_card, :index?
+
+    @preview_cards = filtered_preview_cards.page(params[:page])
+    @form          = Form::PreviewCardBatch.new
+  end
+
+  def batch
+    @form = Form::PreviewCardBatch.new(form_preview_card_batch_params.merge(current_account: current_account, action: action_from_button))
+    @form.save
+  rescue ActionController::ParameterMissing
+    flash[:alert] = I18n.t('admin.accounts.no_account_selected')
+  ensure
+    redirect_to admin_trends_links_path(filter_params)
+  end
+
+  private
+
+  def filtered_preview_cards
+    PreviewCardFilter.new(filter_params.with_defaults(trending: 'all')).results
+  end
+
+  def filter_params
+    params.slice(:page, *PreviewCardFilter::KEYS).permit(:page, *PreviewCardFilter::KEYS)
+  end
+
+  def form_preview_card_batch_params
+    params.require(:form_preview_card_batch).permit(:action, preview_card_ids: [])
+  end
+
+  def action_from_button
+    if params[:approve]
+      'approve'
+    elsif params[:approve_all]
+      'approve_all'
+    elsif params[:reject]
+      'reject'
+    elsif params[:reject_all]
+      'reject_all'
+    end
+  end
+end

app/controllers/admin/trends/tags_controller.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class Admin::Trends::TagsController < Admin::BaseController
+  def index
+    authorize :tag, :index?
+
+    @tags = filtered_tags.page(params[:page])
+    @form = Form::TagBatch.new
+  end
+
+  def batch
+    @form = Form::TagBatch.new(form_tag_batch_params.merge(current_account: current_account, action: action_from_button))
+    @form.save
+  rescue ActionController::ParameterMissing
+    flash[:alert] = I18n.t('admin.accounts.no_account_selected')
+  ensure
+    redirect_to admin_trends_tags_path(filter_params)
+  end
+
+  private
+
+  def filtered_tags
+    TagFilter.new(filter_params).results
+  end
+
+  def filter_params
+    params.slice(:page, *TagFilter::KEYS).permit(:page, *TagFilter::KEYS)
+  end
+
+  def form_tag_batch_params
+    params.require(:form_tag_batch).permit(:action, tag_ids: [])
+  end
+
+  def action_from_button
+    if params[:approve]
+      'approve'
+    elsif params[:reject]
+      'reject'
+    end
+  end
+end

app/controllers/api/v1/admin/dimensions_controller.rb

@@ -17,7 +17,8 @@ class Api::V1::Admin::DimensionsController < Api::BaseController
       params[:keys],
       params[:start_at],
       params[:end_at],
-      params[:limit]
+      params[:limit],
+      params
     )
   end
 end

app/controllers/api/v1/admin/measures_controller.rb

@@ -16,7 +16,8 @@ class Api::V1::Admin::MeasuresController < Api::BaseController
     @measures = Admin::Metrics::Measure.retrieve(
       params[:keys],
       params[:start_at],
-      params[:end_at]
+      params[:end_at],
+      params
     )
   end
 end

app/controllers/api/v1/admin/trends/tags_controller.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class Api::V1::Admin::Trends::TagsController < Api::BaseController
+  before_action :require_staff!
+  before_action :set_tags
+
+  def index
+    render json: @tags, each_serializer: REST::Admin::TagSerializer
+  end
+
+  private
+
+  def set_tags
+    @tags = Trends.tags.get(false, limit_param(10))
+  end
+end

app/controllers/api/v1/admin/trends_controller.rb

@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-class Api::V1::Admin::TrendsController < Api::BaseController
-  before_action :require_staff!
-  before_action :set_trends
-
-  def index
-    render json: @trends, each_serializer: REST::Admin::TagSerializer
-  end
-
-  private
-
-  def set_trends
-    @trends = TrendingTags.get(10, filtered: false)
-  end
-end

app/controllers/api/v1/trends/links_controller.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class Api::V1::Trends::LinksController < Api::BaseController
+  before_action :set_links
+
+  def index
+    render json: @links, each_serializer: REST::Trends::LinkSerializer
+  end
+
+  private
+
+  def set_links
+    @links = begin
+      if Setting.trends
+        Trends.links.get(true, limit_param(10))
+      else
+        []
+      end
+    end
+  end
+end

app/controllers/api/v1/trends/tags_controller.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class Api::V1::Trends::TagsController < Api::BaseController
+  before_action :set_tags
+
+  def index
+    render json: @tags, each_serializer: REST::TagSerializer
+  end
+
+  private
+
+  def set_tags
+    @tags = begin
+      if Setting.trends
+        Trends.tags.get(true, limit_param(10))
+      else
+        []
+      end
+    end
+  end
+end

app/controllers/api/v1/trends_controller.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-class Api::V1::TrendsController < Api::BaseController
-  before_action :set_tags
-
-  def index
-    render json: @tags, each_serializer: REST::TagSerializer
-  end
-
-  private
-
-  def set_tags
-    @tags = TrendingTags.get(limit_param(10))
-  end
-end

app/helpers/admin/filter_helper.rb

@@ -6,6 +6,8 @@ module Admin::FilterHelper
     CustomEmojiFilter::KEYS,
     ReportFilter::KEYS,
     TagFilter::KEYS,
+    PreviewCardProviderFilter::KEYS,
+    PreviewCardFilter::KEYS,
     InstanceFilter::KEYS,
     InviteFilter::KEYS,
     RelationshipFilter::KEYS,

app/helpers/languages_helper.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module LanguagesHelper
+  HUMAN_LOCALES = {
+    af: 'Afrikaans',
+    ar: 'العربية',
+    ast: 'Asturianu',
+    bg: 'Български',
+    bn: 'বাংলা',
+    br: 'Breton',
+    ca: 'Català',
+    co: 'Corsu',
+    cs: 'Čeština',
+    cy: 'Cymraeg',
+    da: 'Dansk',
+    de: 'Deutsch',
+    el: 'Ελληνικά',
+    en: 'English',
+    eo: 'Esperanto',
+    'es-AR': 'Español (Argentina)',
+    'es-MX': 'Español (México)',
+    es: 'Español',
+    et: 'Eesti',
+    eu: 'Euskara',
+    fa: 'فارسی',
+    fi: 'Suomi',
+    fr: 'Français',
+    ga: 'Gaeilge',
+    gd: 'Gàidhlig',
+    gl: 'Galego',
+    he: 'עברית',
+    hi: 'हिन्दी',
+    hr: 'Hrvatski',
+    hu: 'Magyar',
+    hy: 'Հայերեն',
+    id: 'Bahasa Indonesia',
+    io: 'Ido',
+    is: 'Íslenska',
+    it: 'Italiano',
+    ja: '日本語',
+    ka: 'ქართული',
+    kab: 'Taqbaylit',
+    kk: 'Қазақша',
+    kmr: 'Kurmancî',
+    kn: 'ಕನ್ನಡ',
+    ko: '한국어',
+    ku: 'سۆرانی',
+    lt: 'Lietuvių',
+    lv: 'Latviešu',
+    mk: 'Македонски',
+    ml: 'മലയാളം',
+    mr: 'मराठी',
+    ms: 'Bahasa Melayu',
+    nl: 'Nederlands',
+    nn: 'Nynorsk',
+    no: 'Norsk',
+    oc: 'Occitan',
+    pl: 'Polski',
+    'pt-BR': 'Português (Brasil)',
+    'pt-PT': 'Português (Portugal)',
+    pt: 'Português',
+    ro: 'Română',
+    ru: 'Русский',
+    sa: 'संस्कृतम्',
+    sc: 'Sardu',
+    si: 'සිංහල',
+    sk: 'Slovenčina',
+    sl: 'Slovenščina',
+    sq: 'Shqip',
+    'sr-Latn': 'Srpski (latinica)',
+    sr: 'Српски',
+    sv: 'Svenska',
+    ta: 'தமிழ்',
+    te: 'తెలుగు',
+    th: 'ไทย',
+    tr: 'Türkçe',
+    uk: 'Українська',
+    ur: 'اُردُو',
+    vi: 'Tiếng Việt',
+    zgh: 'ⵜⴰⵎⴰⵣⵉⵖⵜ',
+    'zh-CN': '简体中文',
+    'zh-HK': '繁體中文（香港）',
+    'zh-TW': '繁體中文（臺灣）',
+    zh: '中文',
+  }.freeze
+
+  def human_locale(locale)
+    if locale == 'und'
+      I18n.t('generic.none')
+    else
+      HUMAN_LOCALES[locale.to_sym] || locale
+    end
+  end
+end

app/helpers/settings_helper.rb

@@ -1,95 +1,8 @@
 # frozen_string_literal: true
 
 module SettingsHelper
-  HUMAN_LOCALES = {
-    af: 'Afrikaans',
-    ar: 'العربية',
-    ast: 'Asturianu',
-    bg: 'Български',
-    bn: 'বাংলা',
-    br: 'Breton',
-    ca: 'Català',
-    co: 'Corsu',
-    cs: 'Čeština',
-    cy: 'Cymraeg',
-    da: 'Dansk',
-    de: 'Deutsch',
-    el: 'Ελληνικά',
-    en: 'English',
-    eo: 'Esperanto',
-    'es-AR': 'Español (Argentina)',
-    'es-MX': 'Español (México)',
-    es: 'Español',
-    et: 'Eesti',
-    eu: 'Euskara',
-    fa: 'فارسی',
-    fi: 'Suomi',
-    fr: 'Français',
-    ga: 'Gaeilge',
-    gd: 'Gàidhlig',
-    gl: 'Galego',
-    he: 'עברית',
-    hi: 'हिन्दी',
-    hr: 'Hrvatski',
-    hu: 'Magyar',
-    hy: 'Հայերեն',
-    id: 'Bahasa Indonesia',
-    io: 'Ido',
-    is: 'Íslenska',
-    it: 'Italiano',
-    ja: '日本語',
-    ka: 'ქართული',
-    kab: 'Taqbaylit',
-    kk: 'Қазақша',
-    kmr: 'Kurmancî',
-    kn: 'ಕನ್ನಡ',
-    ko: '한국어',
-    ku: 'سۆرانی',
-    lt: 'Lietuvių',
-    lv: 'Latviešu',
-    mk: 'Македонски',
-    ml: 'മലയാളം',
-    mr: 'मराठी',
-    ms: 'Bahasa Melayu',
-    nl: 'Nederlands',
-    nn: 'Nynorsk',
-    no: 'Norsk',
-    oc: 'Occitan',
-    pl: 'Polski',
-    'pt-BR': 'Português (Brasil)',
-    'pt-PT': 'Português (Portugal)',
-    pt: 'Português',
-    ro: 'Română',
-    ru: 'Русский',
-    sa: 'संस्कृतम्',
-    sc: 'Sardu',
-    si: 'සිංහල',
-    sk: 'Slovenčina',
-    sl: 'Slovenščina',
-    sq: 'Shqip',
-    'sr-Latn': 'Srpski (latinica)',
-    sr: 'Српски',
-    sv: 'Svenska',
-    ta: 'தமிழ்',
-    te: 'తెలుగు',
-    th: 'ไทย',
-    tr: 'Türkçe',
-    uk: 'Українська',
-    ur: 'اُردُو',
-    vi: 'Tiếng Việt',
-    zgh: 'ⵜⴰⵎⴰⵣⵉⵖⵜ',
-    'zh-CN': '简体中文',
-    'zh-HK': '繁體中文（香港）',
-    'zh-TW': '繁體中文（臺灣）',
-    zh: '中文',
-  }.freeze
-
-  def human_locale(locale)
-    HUMAN_LOCALES[locale]
-  end
-
   def filterable_languages
-    LanguageDetector.instance.language_names.select(&HUMAN_LOCALES.method(:key?))
+    LanguageDetector.instance.language_names.select(&LanguagesHelper::HUMAN_LOCALES.method(:key?))
   end
 
   def hash_to_object(hash)

app/javascript/mastodon/components/admin/Counter.js

@@ -32,6 +32,7 @@ export default class Counter extends React.PureComponent {
     end_at: PropTypes.string.isRequired,
     label: PropTypes.string.isRequired,
     href: PropTypes.string,
+    params: PropTypes.object,
   };
 
   state = {
@@ -40,9 +41,9 @@ export default class Counter extends React.PureComponent {
   };
 
   componentDidMount () {
-    const { measure, start_at, end_at } = this.props;
+    const { measure, start_at, end_at, params } = this.props;
 
-    api().post('/api/v1/admin/measures', { keys: [measure], start_at, end_at }).then(res => {
+    api().post('/api/v1/admin/measures', { keys: [measure], start_at, end_at, [measure]: params }).then(res => {
       this.setState({
         loading: false,
         data: res.data,

app/javascript/mastodon/components/admin/Dimension.js

@@ -13,6 +13,7 @@ export default class Dimension extends React.PureComponent {
     end_at: PropTypes.string.isRequired,
     limit: PropTypes.number.isRequired,
     label: PropTypes.string.isRequired,
+    params: PropTypes.object,
   };
 
   state = {
@@ -21,9 +22,9 @@ export default class Dimension extends React.PureComponent {
   };
 
   componentDidMount () {
-    const { start_at, end_at, dimension, limit } = this.props;
+    const { start_at, end_at, dimension, limit, params } = this.props;
 
-    api().post('/api/v1/admin/dimensions', { keys: [dimension], start_at, end_at, limit }).then(res => {
+    api().post('/api/v1/admin/dimensions', { keys: [dimension], start_at, end_at, limit, [dimension]: params }).then(res => {
       this.setState({
         loading: false,
         data: res.data,

app/javascript/mastodon/components/admin/Trends.js

@@ -19,7 +19,7 @@ export default class Trends extends React.PureComponent {
   componentDidMount () {
     const { limit } = this.props;
 
-    api().get('/api/v1/admin/trends', { params: { limit } }).then(res => {
+    api().get('/api/v1/admin/trends/tags', { params: { limit } }).then(res => {
       this.setState({
         loading: false,
         data: res.data,

app/javascript/mastodon/features/status/index.js

@@ -83,7 +83,7 @@ const makeMapStateToProps = () => {
     ancestorsIds = ancestorsIds.withMutations(mutable => {
       let id = statusId;
 
-      while (id) {
+      while (id && !mutable.includes(id)) {
         mutable.unshift(id);
         id = inReplyTos.get(id);
       }
@@ -101,7 +101,7 @@ const makeMapStateToProps = () => {
     const ids = [statusId];
 
     while (ids.length > 0) {
-      let id        = ids.shift();
+      let id        = ids.pop();
       const replies = contextReplies.get(id);
 
       if (statusId !== id) {
@@ -110,7 +110,7 @@ const makeMapStateToProps = () => {
 
       if (replies) {
         replies.reverse().forEach(reply => {
-          ids.unshift(reply);
+          if (!ids.includes(reply) && !descendantsIds.includes(reply) && statusId !== reply) ids.push(reply);
         });
       }
     }

app/javascript/styles/mastodon/accounts.scss

@@ -325,3 +325,19 @@
     margin-top: 10px;
   }
 }
+
+.batch-table__row--muted .pending-account__header {
+  &,
+  a,
+  strong {
+    color: lighten($ui-base-color, 26%);
+  }
+}
+
+.batch-table__row--attention .pending-account__header {
+  &,
+  a,
+  strong {
+    color: $gold-star;
+  }
+}

app/javascript/styles/mastodon/dashboard.scss

@@ -100,6 +100,16 @@
       transition: all 200ms ease-out;
     }
 
+    &.positive {
+      background: lighten($ui-base-color, 4%);
+      color: $valid-value-color;
+    }
+
+    &.negative {
+      background: lighten($ui-base-color, 4%);
+      color: $error-value-color;
+    }
+
     span {
       flex: 1 1 auto;
     }

app/lib/activitypub/activity.rb

@@ -129,8 +129,6 @@ class ActivityPub::Activity
   end
 
   def crawl_links(status)
-    return if status.spoiler_text?
-
     # Spread out crawling randomly to avoid DDoSing the link
     LinkCrawlWorker.perform_in(rand(1..59).seconds, status.id)
   end

app/lib/activitypub/activity/announce.rb

@@ -22,9 +22,8 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
         visibility: visibility_from_audience
       )
 
-      original_status.tags.each do |tag|
-        tag.use!(@account)
-      end
+      Trends.tags.register(@status)
+      Trends.links.register(@status)
 
       distribute(@status)
     end

app/lib/activitypub/activity/create.rb

@@ -164,9 +164,14 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def attach_tags(status)
     @tags.each do |tag|
       status.tags << tag
-      tag.use!(@account, status: status, at_time: status.created_at) if status.public_visibility?
+      tag.update(last_status_at: status.created_at) if tag.last_status_at.nil? || (tag.last_status_at < status.created_at && tag.last_status_at < 12.hours.ago)
     end
 
+    # If we're processing an old status, this may register tags as being used now
+    # as opposed to when the status was really published, but this is probably
+    # not a big deal
+    Trends.tags.register(status)
+
     @mentions.each do |mention|
       mention.status = status
       mention.save

app/lib/admin/metrics/dimension.rb

@@ -7,9 +7,14 @@ class Admin::Metrics::Dimension
     servers: Admin::Metrics::Dimension::ServersDimension,
     space_usage: Admin::Metrics::Dimension::SpaceUsageDimension,
     software_versions: Admin::Metrics::Dimension::SoftwareVersionsDimension,
+    tag_servers: Admin::Metrics::Dimension::TagServersDimension,
+    tag_languages: Admin::Metrics::Dimension::TagLanguagesDimension,
   }.freeze
 
-  def self.retrieve(dimension_keys, start_at, end_at, limit)
-    Array(dimension_keys).map { |key| DIMENSIONS[key.to_sym]&.new(start_at, end_at, limit) }.compact
+  def self.retrieve(dimension_keys, start_at, end_at, limit, params)
+    Array(dimension_keys).map do |key|
+      klass = DIMENSIONS[key.to_sym]
+      klass&.new(start_at, end_at, limit, klass.with_params? ? params.require(key.to_sym) : nil)
+    end.compact
   end
 end

app/lib/admin/metrics/dimension/base_dimension.rb

@@ -1,10 +1,15 @@
 # frozen_string_literal: true
 
 class Admin::Metrics::Dimension::BaseDimension
-  def initialize(start_at, end_at, limit)
+  def self.with_params?
+    false
+  end
+
+  def initialize(start_at, end_at, limit, params)
     @start_at = start_at&.to_datetime
     @end_at   = end_at&.to_datetime
     @limit    = limit&.to_i
+    @params   = params
   end
 
   def key
@@ -26,6 +31,10 @@ class Admin::Metrics::Dimension::BaseDimension
   protected
 
   def time_period
-    (@start_at...@end_at)
+    (@start_at..@end_at)
+  end
+
+  def params
+    raise NotImplementedError
   end
 end

app/lib/admin/metrics/dimension/languages_dimension.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension::BaseDimension
+  include LanguagesHelper
+
   def key
     'languages'
   end
@@ -18,6 +20,6 @@ class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension:
 
     rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at], [nil, @limit]])
 
-    rows.map { |row| { key: row['locale'], human_key: SettingsHelper::HUMAN_LOCALES[row['locale'].to_sym], value: row['value'].to_s } }
+    rows.map { |row| { key: row['locale'], human_key: human_locale(row['locale']), value: row['value'].to_s } }
   end
 end

app/lib/admin/metrics/dimension/tag_languages_dimension.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class Admin::Metrics::Dimension::TagLanguagesDimension < Admin::Metrics::Dimension::BaseDimension
+  include LanguagesHelper
+
+  def self.with_params?
+    true
+  end
+
+  def key
+    'tag_languages'
+  end
+
+  def data
+    sql = <<-SQL.squish
+      SELECT COALESCE(statuses.language, 'und') AS language, count(*) AS value
+      FROM statuses
+      INNER JOIN statuses_tags ON statuses_tags.status_id = statuses.id
+      WHERE statuses_tags.tag_id = $1
+        AND statuses.id BETWEEN $2 AND $3
+      GROUP BY COALESCE(statuses.language, 'und')
+      ORDER BY count(*) DESC
+      LIMIT $4
+    SQL
+
+    rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]])
+
+    rows.map { |row| { key: row['language'], human_key: human_locale(row['language']), value: row['value'].to_s } }
+  end
+
+  private
+
+  def params
+    @params.permit(:id)
+  end
+end

app/lib/admin/metrics/dimension/tag_servers_dimension.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class Admin::Metrics::Dimension::TagServersDimension < Admin::Metrics::Dimension::BaseDimension
+  def self.with_params?
+    true
+  end
+
+  def key
+    'tag_servers'
+  end
+
+  def data
+    sql = <<-SQL.squish
+      SELECT accounts.domain, count(*) AS value
+      FROM statuses
+      INNER JOIN accounts ON accounts.id = statuses.account_id
+      INNER JOIN statuses_tags ON statuses_tags.status_id = statuses.id
+      WHERE statuses_tags.tag_id = $1
+        AND statuses.id BETWEEN $2 AND $3
+      GROUP BY accounts.domain
+      ORDER BY count(*) DESC
+      LIMIT $4
+    SQL
+
+    rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]])
+
+    rows.map { |row| { key: row['domain'] || Rails.configuration.x.local_domain, human_key: row['domain'] || Rails.configuration.x.local_domain, value: row['value'].to_s } }
+  end
+
+  private
+
+  def params
+    @params.permit(:id)
+  end
+end

app/lib/admin/metrics/measure.rb

@@ -7,9 +7,15 @@ class Admin::Metrics::Measure
     interactions: Admin::Metrics::Measure::InteractionsMeasure,
     opened_reports: Admin::Metrics::Measure::OpenedReportsMeasure,
     resolved_reports: Admin::Metrics::Measure::ResolvedReportsMeasure,
+    tag_accounts: Admin::Metrics::Measure::TagAccountsMeasure,
+    tag_uses: Admin::Metrics::Measure::TagUsesMeasure,
+    tag_servers: Admin::Metrics::Measure::TagServersMeasure,
   }.freeze
 
-  def self.retrieve(measure_keys, start_at, end_at)
-    Array(measure_keys).map { |key| MEASURES[key.to_sym]&.new(start_at, end_at) }.compact
+  def self.retrieve(measure_keys, start_at, end_at, params)
+    Array(measure_keys).map do |key|
+      klass = MEASURES[key.to_sym]
+      klass&.new(start_at, end_at, klass.with_params? ? params.require(key.to_sym) : nil)
+    end.compact
   end
 end

app/lib/admin/metrics/measure/active_users_measure.rb

@@ -24,10 +24,10 @@ class Admin::Metrics::Measure::ActiveUsersMeasure < Admin::Metrics::Measure::Bas
   end
 
   def time_period
-    (@start_at.to_date...@end_at.to_date)
+    (@start_at.to_date..@end_at.to_date)
   end
 
   def previous_time_period
-    ((@start_at.to_date - length_of_period)...(@end_at.to_date - length_of_period))
+    ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
   end
 end

app/lib/admin/metrics/measure/base_measure.rb

@@ -1,9 +1,14 @@
 # frozen_string_literal: true
 
 class Admin::Metrics::Measure::BaseMeasure
-  def initialize(start_at, end_at)
+  def self.with_params?
+    false
+  end
+
+  def initialize(start_at, end_at, params)
     @start_at = start_at&.to_datetime
     @end_at   = end_at&.to_datetime
+    @params   = params
   end
 
   def key
@@ -33,14 +38,18 @@ class Admin::Metrics::Measure::BaseMeasure
   protected
 
   def time_period
-    (@start_at...@end_at)
+    (@start_at..@end_at)
   end
 
   def previous_time_period
-    ((@start_at - length_of_period)...(@end_at - length_of_period))
+    ((@start_at - length_of_period)..(@end_at - length_of_period))
   end
 
   def length_of_period
     @length_of_period ||= @end_at - @start_at
   end
+
+  def params
+    raise NotImplementedError
+  end
 end

app/lib/admin/metrics/measure/interactions_measure.rb

@@ -24,10 +24,10 @@ class Admin::Metrics::Measure::InteractionsMeasure < Admin::Metrics::Measure::Ba
   end
 
   def time_period
-    (@start_at.to_date...@end_at.to_date)
+    (@start_at.to_date..@end_at.to_date)
   end
 
   def previous_time_period
-    ((@start_at.to_date - length_of_period)...(@end_at.to_date - length_of_period))
+    ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
   end
 end

app/lib/admin/metrics/measure/tag_accounts_measure.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class Admin::Metrics::Measure::TagAccountsMeasure < Admin::Metrics::Measure::BaseMeasure
+  def self.with_params?
+    true
+  end
+
+  def key
+    'tag_accounts'
+  end
+
+  def total
+    tag.history.aggregate(time_period).accounts
+  end
+
+  def previous_total
+    tag.history.aggregate(previous_time_period).accounts
+  end
+
+  def data
+    time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).accounts.to_s } }
+  end
+
+  protected
+
+  def tag
+    @tag ||= Tag.find(params[:id])
+  end
+
+  def time_period
+    (@start_at.to_date..@end_at.to_date)
+  end
+
+  def previous_time_period
+    ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
+  end
+
+  def params
+    @params.permit(:id)
+  end
+end

app/lib/admin/metrics/measure/tag_servers_measure.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+class Admin::Metrics::Measure::TagServersMeasure < Admin::Metrics::Measure::BaseMeasure
+  def self.with_params?
+    true
+  end
+
+  def key
+    'tag_servers'
+  end
+
+  def total
+    tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at, with_random: false), Mastodon::Snowflake.id_at(@end_at, with_random: false)).joins(:account).count('distinct accounts.domain')
+  end
+
+  def previous_total
+    tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at - length_of_period, with_random: false), Mastodon::Snowflake.id_at(@end_at - length_of_period, with_random: false)).joins(:account).count('distinct accounts.domain')
+  end
+
+  def data
+    sql = <<-SQL.squish
+      SELECT axis.*, (
+        SELECT count(*) AS value
+        FROM statuses
+        WHERE statuses.id BETWEEN $1 AND $2
+          AND date_trunc('day', statuses.created_at)::date = axis.day
+      )
+      FROM (
+        SELECT generate_series(date_trunc('day', $3::timestamp)::date, date_trunc('day', $4::timestamp)::date, ('1 day')::interval) AS day
+      ) as axis
+    SQL
+
+    rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @start_at], [nil, @end_at]])
+
+    rows.map { |row| { date: row['day'], value: row['value'].to_s } }
+  end
+
+  protected
+
+  def tag
+    @tag ||= Tag.find(params[:id])
+  end
+
+  def params
+    @params.permit(:id)
+  end
+end

app/lib/admin/metrics/measure/tag_uses_measure.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class Admin::Metrics::Measure::TagUsesMeasure < Admin::Metrics::Measure::BaseMeasure
+  def self.with_params?
+    true
+  end
+
+  def key
+    'tag_uses'
+  end
+
+  def total
+    tag.history.aggregate(time_period).uses
+  end
+
+  def previous_total
+    tag.history.aggregate(previous_time_period).uses
+  end
+
+  def data
+    time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).uses.to_s } }
+  end
+
+  protected
+
+  def tag
+    @tag ||= Tag.find(params[:id])
+  end
+
+  def time_period
+    (@start_at.to_date..@end_at.to_date)
+  end
+
+  def previous_time_period
+    ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
+  end
+
+  def params
+    @params.permit(:id)
+  end
+end

app/lib/link_details_extractor.rb

@@ -4,6 +4,11 @@ class LinkDetailsExtractor
   include ActionView::Helpers::TagHelper
 
   class StructuredData
+    SUPPORTED_TYPES = %w(
+      NewsArticle
+      WebPage
+    ).freeze
+
     def initialize(data)
       @data = data
     end
@@ -16,6 +21,14 @@ class LinkDetailsExtractor
       json['description']
     end
 
+    def language
+      json['inLanguage']
+    end
+
+    def type
+      json['@type']
+    end
+
     def image
       obj = first_of_value(json['image'])
 
@@ -44,6 +57,10 @@ class LinkDetailsExtractor
       publisher['name']
     end
 
+    def publisher_logo
+      publisher.dig('logo', 'url')
+    end
+
     private
 
     def author
@@ -58,8 +75,12 @@ class LinkDetailsExtractor
       arr.is_a?(Array) ? arr.first : arr
     end
 
+    def root_array(root)
+      root.is_a?(Array) ? root : [root]
+    end
+
     def json
-      @json ||= first_of_value(Oj.load(@data))
+      @json ||= root_array(Oj.load(@data)).find { |obj| SUPPORTED_TYPES.include?(obj['@type']) } || {}
     end
   end
 
@@ -75,6 +96,7 @@ class LinkDetailsExtractor
       description: description || '',
       image_remote_url: image,
       type: type,
+      link_type: link_type,
       width: width || 0,
       height: height || 0,
       html: html || '',
@@ -83,6 +105,7 @@ class LinkDetailsExtractor
       author_name: author_name || '',
       author_url: author_url || '',
       embed_url: embed_url || '',
+      language: language,
     }
   end
 
@@ -90,6 +113,14 @@ class LinkDetailsExtractor
     player_url.present? ? :video : :link
   end
 
+  def link_type
+    if structured_data&.type == 'NewsArticle' || opengraph_tag('og:type') == 'article'
+      :article
+    else
+      :unknown
+    end
+  end
+
   def html
     player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil
   end
@@ -138,6 +169,14 @@ class LinkDetailsExtractor
     valid_url_or_nil(opengraph_tag('twitter:player:stream'))
   end
 
+  def language
+    valid_locale_or_nil(structured_data&.language || opengraph_tag('og:locale') || document.xpath('//html').map { |element| element['lang'] }.first)
+  end
+
+  def icon
+    valid_url_or_nil(structured_data&.publisher_icon || link_tag('apple-touch-icon') || link_tag('shortcut icon'))
+  end
+
   private
 
   def player_url
@@ -162,6 +201,14 @@ class LinkDetailsExtractor
     nil
   end
 
+  def valid_locale_or_nil(str)
+    return nil if str.blank?
+
+    code,  = str.split(/_-/) # Strip out the region from e.g. en_US or ja-JA
+    locale = ISO_639.find(code)
+    locale&.alpha2
+  end
+
   def link_tag(name)
     document.xpath("//link[@rel=\"#{name}\"]").map { |link| link['href'] }.first
   end

app/mailers/admin_mailer.rb

@@ -25,13 +25,25 @@ class AdminMailer < ApplicationMailer
     end
   end
 
-  def new_trending_tag(recipient, tag)
-    @tag      = tag
-    @me       = recipient
-    @instance = Rails.configuration.x.local_domain
+  def new_trending_tags(recipient, tags)
+    @tags                = tags
+    @me                  = recipient
+    @instance            = Rails.configuration.x.local_domain
+    @lowest_trending_tag = Trends.tags.get(true, Trends::Tags::REVIEW_THRESHOLD).last
+
+    locale_for_account(@me) do
+      mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tags.subject', instance: @instance)
+    end
+  end
+
+  def new_trending_links(recipient, links)
+    @links                = links
+    @me                   = recipient
+    @instance             = Rails.configuration.x.local_domain
+    @lowest_trending_link = Trends.links.get(true, Trends::Links::REVIEW_THRESHOLD).last
 
     locale_for_account(@me) do
-      mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tag.subject', instance: @instance, name: @tag.name)
+      mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_links.subject', instance: @instance)
     end
   end
 end

app/models/account_statuses_cleanup_policy.rb

@@ -4,8 +4,8 @@
 #
 # Table name: account_statuses_cleanup_policies
 #
-#  id                 :bigint           not null, primary key
-#  account_id         :bigint           not null
+#  id                 :bigint(8)        not null, primary key
+#  account_id         :bigint(8)        not null
 #  enabled            :boolean          default(TRUE), not null
 #  min_status_age     :integer          default(1209600), not null
 #  keep_direct        :boolean          default(TRUE), not null

app/models/form/preview_card_batch.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+class Form::PreviewCardBatch
+  include ActiveModel::Model
+  include Authorization
+
+  attr_accessor :preview_card_ids, :action, :current_account, :precision
+
+  def save
+    case action
+    when 'approve'
+      approve!
+    when 'approve_all'
+      approve_all!
+    when 'reject'
+      reject!
+    when 'reject_all'
+      reject_all!
+    end
+  end
+
+  private
+
+  def preview_cards
+    @preview_cards ||= PreviewCard.where(id: preview_card_ids)
+  end
+
+  def preview_card_providers
+    @preview_card_providers ||= preview_cards.map(&:domain).uniq.map { |domain| PreviewCardProvider.matching_domain(domain) || PreviewCardProvider.new(domain: domain) }
+  end
+
+  def approve!
+    preview_cards.each { |preview_card| authorize(preview_card, :update?) }
+    preview_cards.update_all(trendable: true)
+  end
+
+  def approve_all!
+    preview_card_providers.each do |provider|
+      authorize(provider, :update?)
+      provider.update(trendable: true, reviewed_at: action_time)
+    end
+
+    # Reset any individual overrides
+    preview_cards.update_all(trendable: nil)
+  end
+
+  def reject!
+    preview_cards.each { |preview_card| authorize(preview_card, :update?) }
+    preview_cards.update_all(trendable: false)
+  end
+
+  def reject_all!
+    preview_card_providers.each do |provider|
+      authorize(provider, :update?)
+      provider.update(trendable: false, reviewed_at: action_time)
+    end
+
+    # Reset any individual overrides
+    preview_cards.update_all(trendable: nil)
+  end
+
+  def action_time
+    @action_time ||= Time.now.utc
+  end
+end

app/models/form/preview_card_provider_batch.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+class Form::PreviewCardProviderBatch
+  include ActiveModel::Model
+  include Authorization
+
+  attr_accessor :preview_card_provider_ids, :action, :current_account
+
+  def save
+    case action
+    when 'approve'
+      approve!
+    when 'reject'
+      reject!
+    end
+  end
+
+  private
+
+  def preview_card_providers
+    PreviewCardProvider.where(id: preview_card_provider_ids)
+  end
+
+  def approve!
+    preview_card_providers.each { |provider| authorize(provider, :update?) }
+    preview_card_providers.update_all(trendable: true, reviewed_at: Time.now.utc)
+  end
+
+  def reject!
+    preview_card_providers.each { |provider| authorize(provider, :update?) }
+    preview_card_providers.update_all(trendable: false, reviewed_at: Time.now.utc)
+  end
+end

app/models/form/tag_batch.rb

@@ -23,11 +23,15 @@ class Form::TagBatch
 
   def approve!
     tags.each { |tag| authorize(tag, :update?) }
-    tags.update_all(trendable: true, reviewed_at: Time.now.utc)
+    tags.update_all(trendable: true, reviewed_at: action_time)
   end
 
   def reject!
     tags.each { |tag| authorize(tag, :update?) }
-    tags.update_all(trendable: false, reviewed_at: Time.now.utc)
+    tags.update_all(trendable: false, reviewed_at: action_time)
+  end
+
+  def action_time
+    @action_time ||= Time.now.utc
   end
 end

app/models/preview_card.rb

@@ -24,6 +24,11 @@
 #  embed_url                    :string           default(""), not null
 #  image_storage_schema_version :integer
 #  blurhash                     :string
+#  language                     :string
+#  max_score                    :float
+#  max_score_at                 :datetime
+#  trendable                    :boolean
+#  link_type                    :integer
 #
 
 class PreviewCard < ApplicationRecord
@@ -40,6 +45,7 @@ class PreviewCard < ApplicationRecord
   self.inheritance_column = false
 
   enum type: [:link, :photo, :video, :rich]
+  enum link_type: [:unknown, :article]
 
   has_and_belongs_to_many :statuses
 
@@ -54,6 +60,32 @@ class PreviewCard < ApplicationRecord
 
   before_save :extract_dimensions, if: :link?
 
+  def appropriate_for_trends?
+    link? && article? && title.present? && description.present? && image.present? && provider_name.present?
+  end
+
+  def domain
+    @domain ||= Addressable::URI.parse(url).normalized_host
+  end
+
+  def provider
+    @provider ||= PreviewCardProvider.matching_domain(domain)
+  end
+
+  def trendable?
+    if attributes['trendable'].nil?
+      provider&.trendable?
+    else
+      attributes['trendable']
+    end
+  end
+
+  def requires_review_notification?
+    attributes['trendable'].nil? && (provider.nil? || provider.requires_review_notification?)
+  end
+
+  attr_writer :provider
+
   def local?
     false
   end
@@ -69,11 +101,14 @@ class PreviewCard < ApplicationRecord
     save!
   end
 
+  def history
+    @history ||= Trends::History.new('links', id)
+  end
+
   class << self
     private
 
-    # rubocop:disable Naming/MethodParameterName
-    def image_styles(f)
+    def image_styles(file)
       styles = {
         original: {
           geometry: '400x400>',
@@ -83,10 +118,9 @@ class PreviewCard < ApplicationRecord
         },
       }
 
-      styles[:original][:format] = 'jpg' if f.instance.image_content_type == 'image/gif'
+      styles[:original][:format] = 'jpg' if file.instance.image_content_type == 'image/gif'
       styles
     end
-    # rubocop:enable Naming/MethodParameterName
   end
 
   private

app/models/preview_card_filter.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+class PreviewCardFilter
+  KEYS = %i(
+    trending
+  ).freeze
+
+  attr_reader :params
+
+  def initialize(params)
+    @params = params
+  end
+
+  def results
+    scope = PreviewCard.unscoped
+
+    params.each do |key, value|
+      next if key.to_s == 'page'
+
+      scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
+    end
+
+    scope
+  end
+
+  private
+
+  def scope_for(key, value)
+    case key.to_s
+    when 'trending'
+      trending_scope(value)
+    else
+      raise "Unknown filter: #{key}"
+    end
+  end
+
+  def trending_scope(value)
+    ids = begin
+      case value.to_s
+      when 'allowed'
+        Trends.links.currently_trending_ids(true, -1)
+      else
+        Trends.links.currently_trending_ids(false, -1)
+      end
+    end
+
+    if ids.empty?
+      PreviewCard.none
+    else
+      PreviewCard.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id").order('x.ordering')
+    end
+  end
+end

app/models/preview_card_provider.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: preview_card_providers
+#
+#  id                  :bigint(8)        not null, primary key
+#  domain              :string           default(""), not null
+#  icon_file_name      :string
+#  icon_content_type   :string
+#  icon_file_size      :bigint(8)
+#  icon_updated_at     :datetime
+#  trendable           :boolean
+#  reviewed_at         :datetime
+#  requested_review_at :datetime
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#
+
+class PreviewCardProvider < ApplicationRecord
+  include DomainNormalizable
+  include Attachmentable
+
+  ICON_MIME_TYPES = %w(image/x-icon image/vnd.microsoft.icon image/png).freeze
+  LIMIT = 1.megabyte
+
+  validates :domain, presence: true, uniqueness: true, domain: true
+
+  has_attached_file :icon, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } }, validate_media_type: false
+  validates_attachment :icon, content_type: { content_type: ICON_MIME_TYPES }, size: { less_than: LIMIT }
+  remotable_attachment :icon, LIMIT
+
+  scope :trendable, -> { where(trendable: true) }
+  scope :not_trendable, -> { where(trendable: false) }
+  scope :reviewed, -> { where.not(reviewed_at: nil) }
+  scope :pending_review, -> { where(reviewed_at: nil) }
+
+  def requires_review?
+    reviewed_at.nil?
+  end
+
+  def reviewed?
+    reviewed_at.present?
+  end
+
+  def requested_review?
+    requested_review_at.present?
+  end
+
+  def requires_review_notification?
+    requires_review? && !requested_review?
+  end
+
+  def self.matching_domain(domain)
+    segments = domain.split('.')
+    where(domain: segments.map.with_index { |_, i| segments[i..-1].join('.') }).order(Arel.sql('char_length(domain) desc')).first
+  end
+end

app/models/preview_card_provider_filter.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+class PreviewCardProviderFilter
+  KEYS = %i(
+    status
+  ).freeze
+
+  attr_reader :params
+
+  def initialize(params)
+    @params = params
+  end
+
+  def results
+    scope = PreviewCardProvider.unscoped
+
+    params.each do |key, value|
+      next if key.to_s == 'page'
+
+      scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
+    end
+
+    scope.order(domain: :asc)
+  end
+
+  private
+
+  def scope_for(key, value)
+    case key.to_s
+    when 'status'
+      status_scope(value)
+    else
+      raise "Unknown filter: #{key}"
+    end
+  end
+
+  def status_scope(value)
+    case value.to_s
+    when 'approved'
+      PreviewCardProvider.trendable
+    when 'rejected'
+      PreviewCardProvider.not_trendable
+    when 'pending_review'
+      PreviewCardProvider.pending_review
+    else
+      raise "Unknown status: #{value}"
+    end
+  end
+end

app/models/tag.rb

@@ -36,6 +36,7 @@ class Tag < ApplicationRecord
   scope :usable, -> { where(usable: [true, nil]) }
   scope :listable, -> { where(listable: [true, nil]) }
   scope :trendable, -> { Setting.trendable_by_default ? where(trendable: [true, nil]) : where(trendable: true) }
+  scope :not_trendable, -> { where(trendable: false) }
   scope :recently_used, ->(account) { joins(:statuses).where(statuses: { id: account.statuses.select(:id).limit(1000) }).group(:id).order(Arel.sql('count(*) desc')) }
   scope :matches_name, ->(term) { where(arel_table[:name].lower.matches(arel_table.lower("#{sanitize_sql_like(Tag.normalize(term))}%"), nil, true)) } # Search with case-sensitive to use B-tree index
 
@@ -75,28 +76,12 @@ class Tag < ApplicationRecord
     requested_review_at.present?
   end
 
-  def use!(account, status: nil, at_time: Time.now.utc)
-    TrendingTags.record_use!(self, account, status: status, at_time: at_time)
-  end
-
-  def trending?
-    TrendingTags.trending?(self)
+  def requires_review_notification?
+    requires_review? && !requested_review?
   end
 
   def history
-    days = []
-
-    7.times do |i|
-      day = i.days.ago.beginning_of_day.to_i
-
-      days << {
-        day: day.to_s,
-        uses: Redis.current.get("activity:tags:#{id}:#{day}") || '0',
-        accounts: Redis.current.pfcount("activity:tags:#{id}:#{day}:accounts").to_s,
-      }
-    end
-
-    days
+    @history ||= Trends::History.new('tags', id)
   end
 
   class << self

app/models/tag_filter.rb

@@ -2,13 +2,8 @@
 
 class TagFilter
   KEYS = %i(
-    directory
-    reviewed
-    unreviewed
-    pending_review
-    popular
-    active
-    name
+    trending
+    status
   ).freeze
 
   attr_reader :params
@@ -18,7 +13,13 @@ class TagFilter
   end
 
   def results
-    scope = Tag.unscoped
+    scope = begin
+      if params[:status] == 'pending_review'
+        Tag.unscoped
+      else
+        trending_scope
+      end
+    end
 
     params.each do |key, value|
       next if key.to_s == 'page'
@@ -26,27 +27,40 @@ class TagFilter
       scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
     end
 
-    scope.order(id: :desc)
+    scope
   end
 
   private
 
   def scope_for(key, value)
     case key.to_s
-    when 'reviewed'
-      Tag.reviewed.order(reviewed_at: :desc)
-    when 'unreviewed'
-      Tag.unreviewed
-    when 'pending_review'
-      Tag.pending_review.order(requested_review_at: :desc)
-    when 'popular'
-      Tag.order('max_score DESC NULLS LAST')
-    when 'active'
-      Tag.order('last_status_at DESC NULLS LAST')
-    when 'name'
-      Tag.matches_name(value)
+    when 'status'
+      status_scope(value)
     else
       raise "Unknown filter: #{key}"
     end
   end
+
+  def trending_scope
+    ids = Trends.tags.currently_trending_ids(false, -1)
+
+    if ids.empty?
+      Tag.none
+    else
+      Tag.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id").order('x.ordering')
+    end
+  end
+
+  def status_scope(value)
+    case value.to_s
+    when 'approved'
+      Tag.trendable
+    when 'rejected'
+      Tag.not_trendable
+    when 'pending_review'
+      Tag.pending_review
+    else
+      raise "Unknown status: #{value}"
+    end
+  end
 end

app/models/trending_tags.rb

@@ -1,128 +0,0 @@
-# frozen_string_literal: true
-
-class TrendingTags
-  KEY                  = 'trending_tags'
-  EXPIRE_HISTORY_AFTER = 7.days.seconds
-  EXPIRE_TRENDS_AFTER  = 1.day.seconds
-  THRESHOLD            = 5
-  LIMIT                = 10
-  REVIEW_THRESHOLD     = 3
-  MAX_SCORE_COOLDOWN   = 2.days.freeze
-  MAX_SCORE_HALFLIFE   = 2.hours.freeze
-
-  class << self
-    include Redisable
-
-    def record_use!(tag, account, status: nil, at_time: Time.now.utc)
-      return unless tag.usable? && !account.silenced?
-
-      # Even if a tag is not allowed to trend, we still need to
-      # record the stats since they can be displayed in other places
-      increment_historical_use!(tag.id, at_time)
-      increment_unique_use!(tag.id, account.id, at_time)
-      increment_use!(tag.id, at_time)
-
-      # Only update when the tag was last used once every 12 hours
-      # and only if a status is given (lets use ignore reblogs)
-      tag.update(last_status_at: at_time) if status.present? && (tag.last_status_at.nil? || (tag.last_status_at < at_time && tag.last_status_at < 12.hours.ago))
-    end
-
-    def update!(at_time = Time.now.utc)
-      tag_ids = redis.smembers("#{KEY}:used:#{at_time.beginning_of_day.to_i}") + redis.zrange(KEY, 0, -1)
-      tags    = Tag.trendable.where(id: tag_ids.uniq)
-
-      # First pass to calculate scores and update the set
-
-      tags.each do |tag|
-        expected  = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
-        expected  = 1.0 if expected.zero?
-        observed  = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
-        max_time  = tag.max_score_at
-        max_score = tag.max_score
-        max_score = 0 if max_time.nil? || max_time < (at_time - MAX_SCORE_COOLDOWN)
-
-        score = begin
-          if expected > observed || observed < THRESHOLD
-            0
-          else
-            ((observed - expected)**2) / expected
-          end
-        end
-
-        if score > max_score
-          max_score = score
-          max_time  = at_time
-
-          # Not interested in triggering any callbacks for this
-          tag.update_columns(max_score: max_score, max_score_at: max_time)
-        end
-
-        decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / MAX_SCORE_HALFLIFE.to_f))
-
-        if decaying_score.zero?
-          redis.zrem(KEY, tag.id)
-        else
-          redis.zadd(KEY, decaying_score, tag.id)
-        end
-      end
-
-      users_for_review = User.staff.includes(:account).to_a.select(&:allows_trending_tag_emails?)
-
-      # Second pass to notify about previously unreviewed trends
-
-      tags.each do |tag|
-        current_rank              = redis.zrevrank(KEY, tag.id)
-        needs_review_notification = tag.requires_review? && !tag.requested_review?
-        rank_passes_threshold     = current_rank.present? && current_rank <= REVIEW_THRESHOLD
-
-        next unless !tag.trendable? && rank_passes_threshold && needs_review_notification
-
-        tag.touch(:requested_review_at)
-
-        users_for_review.each do |user|
-          AdminMailer.new_trending_tag(user.account, tag).deliver_later!
-        end
-      end
-
-      # Trim older items
-
-      redis.zremrangebyrank(KEY, 0, -(LIMIT + 1))
-      redis.zremrangebyscore(KEY, '(0.3', '-inf')
-    end
-
-    def get(limit, filtered: true)
-      tag_ids = redis.zrevrange(KEY, 0, LIMIT - 1).map(&:to_i)
-
-      tags = Tag.where(id: tag_ids)
-      tags = tags.trendable if filtered
-      tags = tags.index_by(&:id)
-
-      tag_ids.map { |tag_id| tags[tag_id] }.compact.take(limit)
-    end
-
-    def trending?(tag)
-      rank = redis.zrevrank(KEY, tag.id)
-      rank.present? && rank < LIMIT
-    end
-
-    private
-
-    def increment_historical_use!(tag_id, at_time)
-      key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}"
-      redis.incrby(key, 1)
-      redis.expire(key, EXPIRE_HISTORY_AFTER)
-    end
-
-    def increment_unique_use!(tag_id, account_id, at_time)
-      key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts"
-      redis.pfadd(key, account_id)
-      redis.expire(key, EXPIRE_HISTORY_AFTER)
-    end
-
-    def increment_use!(tag_id, at_time)
-      key = "#{KEY}:used:#{at_time.beginning_of_day.to_i}"
-      redis.sadd(key, tag_id)
-      redis.expire(key, EXPIRE_HISTORY_AFTER)
-    end
-  end
-end

app/models/trends.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Trends
+  def self.table_name_prefix
+    'trends_'
+  end
+
+  def self.links
+    @links ||= Trends::Links.new
+  end
+
+  def self.tags
+    @tags ||= Trends::Tags.new
+  end
+
+  def self.refresh!
+    [links, tags].each(&:refresh)
+  end
+
+  def self.request_review!
+    [links, tags].each(&:request_review) if enabled?
+  end
+
+  def self.enabled?
+    Setting.trends
+  end
+end

app/models/trends/base.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+class Trends::Base
+  include Redisable
+
+  class_attribute :default_options
+
+  attr_reader :options
+
+  # @param [Hash] options
+  # @option options [Integer] :threshold Minimum amount of uses by unique accounts to begin calculating the score
+  # @option options [Integer] :review_threshold Minimum rank (lower = better) before requesting a review
+  # @option options [ActiveSupport::Duration] :max_score_cooldown For this amount of time, the peak score (if bigger than current score) is decayed-from
+  # @option options [ActiveSupport::Duration] :max_score_halflife How quickly a peak score decays
+  def initialize(options = {})
+    @options = self.class.default_options.merge(options)
+  end
+
+  def register(_status)
+    raise NotImplementedError
+  end
+
+  def add(*)
+    raise NotImplementedError
+  end
+
+  def refresh(*)
+    raise NotImplementedError
+  end
+
+  def request_review
+    raise NotImplementedError
+  end
+
+  def get(*)
+    raise NotImplementedError
+  end
+
+  def score(id)
+    redis.zscore("#{key_prefix}:all", id) || 0
+  end
+
+  def rank(id)
+    redis.zrevrank("#{key_prefix}:allowed", id)
+  end
+
+  def currently_trending_ids(allowed, limit)
+    redis.zrevrange(allowed ? "#{key_prefix}:allowed" : "#{key_prefix}:all", 0, limit.positive? ? limit - 1 : limit).map(&:to_i)
+  end
+
+  protected
+
+  def key_prefix
+    raise NotImplementedError
+  end
+
+  def recently_used_ids(at_time = Time.now.utc)
+    redis.smembers(used_key(at_time)).map(&:to_i)
+  end
+
+  def record_used_id(id, at_time = Time.now.utc)
+    redis.sadd(used_key(at_time), id)
+    redis.expire(used_key(at_time), 1.day.seconds)
+  end
+
+  def trim_older_items
+    redis.zremrangebyscore("#{key_prefix}:all", '-inf', '(1')
+    redis.zremrangebyscore("#{key_prefix}:allowed", '-inf', '(1')
+  end
+
+  def score_at_rank(rank)
+    redis.zrevrange("#{key_prefix}:allowed", 0, rank, with_scores: true).last&.last || 0
+  end
+
+  private
+
+  def used_key(at_time)
+    "#{key_prefix}:used:#{at_time.beginning_of_day.to_i}"
+  end
+end

app/models/trends/history.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+class Trends::History
+  include Enumerable
+
+  class Aggregate
+    include Redisable
+
+    def initialize(prefix, id, date_range)
+      @days = date_range.map { |date| Day.new(prefix, id, date.to_time(:utc)) }
+    end
+
+    def uses
+      redis.mget(*@days.map { |day| day.key_for(:uses) }).map(&:to_i).sum
+    end
+
+    def accounts
+      redis.pfcount(*@days.map { |day| day.key_for(:accounts) })
+    end
+  end
+
+  class Day
+    include Redisable
+
+    EXPIRE_AFTER = 14.days.seconds
+
+    def initialize(prefix, id, day)
+      @prefix = prefix
+      @id     = id
+      @day    = day.beginning_of_day
+    end
+
+    attr_reader :day
+
+    def accounts
+      redis.pfcount(key_for(:accounts))
+    end
+
+    def uses
+      redis.get(key_for(:uses))&.to_i || 0
+    end
+
+    def add(account_id)
+      redis.pipelined do
+        redis.incrby(key_for(:uses), 1)
+        redis.pfadd(key_for(:accounts), account_id)
+        redis.expire(key_for(:uses), EXPIRE_AFTER)
+        redis.expire(key_for(:accounts), EXPIRE_AFTER)
+      end
+    end
+
+    def as_json
+      { day: day.to_i.to_s, accounts: accounts.to_s, uses: uses.to_s }
+    end
+
+    def key_for(suffix)
+      case suffix
+      when :accounts
+        "#{key_prefix}:#{suffix}"
+      when :uses
+        key_prefix
+      end
+    end
+
+    def key_prefix
+      "activity:#{@prefix}:#{@id}:#{day.to_i}"
+    end
+  end
+
+  def initialize(prefix, id)
+    @prefix = prefix
+    @id     = id
+  end
+
+  def get(date)
+    Day.new(@prefix, @id, date)
+  end
+
+  def add(account_id, at_time = Time.now.utc)
+    Day.new(@prefix, @id, at_time).add(account_id)
+  end
+
+  def aggregate(date_range)
+    Aggregate.new(@prefix, @id, date_range)
+  end
+
+  def each(&block)
+    if block_given?
+      (0...7).map { |i| block.call(get(i.days.ago)) }
+    else
+      to_enum(:each)
+    end
+  end
+
+  def as_json(*)
+    map(&:as_json)
+  end
+end

app/models/trends/links.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+class Trends::Links < Trends::Base
+  PREFIX = 'trending_links'
+
+  self.default_options = {
+    threshold: 15,
+    review_threshold: 10,
+    max_score_cooldown: 2.days.freeze,
+    max_score_halflife: 8.hours.freeze,
+  }
+
+  def register(status, at_time = Time.now.utc)
+    original_status = status.reblog? ? status.reblog : status
+
+    return unless original_status.public_visibility? && status.public_visibility? &&
+                  !original_status.account.silenced? && !status.account.silenced? &&
+                  !original_status.spoiler_text?
+
+    original_status.preview_cards.each do |preview_card|
+      add(preview_card, status.account_id, at_time) if preview_card.appropriate_for_trends?
+    end
+  end
+
+  def add(preview_card, account_id, at_time = Time.now.utc)
+    preview_card.history.add(account_id, at_time)
+    record_used_id(preview_card.id, at_time)
+  end
+
+  def get(allowed, limit)
+    preview_card_ids = currently_trending_ids(allowed, limit)
+    preview_cards = PreviewCard.where(id: preview_card_ids).index_by(&:id)
+    preview_card_ids.map { |id| preview_cards[id] }.compact
+  end
+
+  def refresh(at_time = Time.now.utc)
+    preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq)
+    calculate_scores(preview_cards, at_time)
+    trim_older_items
+  end
+
+  def request_review
+    preview_cards = PreviewCard.where(id: currently_trending_ids(false, -1))
+
+    preview_cards_requiring_review = preview_cards.filter_map do |preview_card|
+      next unless would_be_trending?(preview_card.id) && !preview_card.trendable? && preview_card.requires_review_notification?
+
+      if preview_card.provider.nil?
+        preview_card.provider = PreviewCardProvider.create(domain: preview_card.domain, requested_review_at: Time.now.utc)
+      else
+        preview_card.provider.touch(:requested_review_at)
+      end
+
+      preview_card
+    end
+
+    return if preview_cards_requiring_review.empty?
+
+    User.staff.includes(:account).find_each do |user|
+      AdminMailer.new_trending_links(user.account, preview_cards_requiring_review).deliver_later! if user.allows_trending_tag_emails?
+    end
+  end
+
+  protected
+
+  def key_prefix
+    PREFIX
+  end
+
+  private
+
+  def calculate_scores(preview_cards, at_time)
+    preview_cards.each do |preview_card|
+      expected  = preview_card.history.get(at_time - 1.day).accounts.to_f
+      expected  = 1.0 if expected.zero?
+      observed  = preview_card.history.get(at_time).accounts.to_f
+      max_time  = preview_card.max_score_at
+      max_score = preview_card.max_score
+      max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown])
+
+      score = begin
+        if expected > observed || observed < options[:threshold]
+          0
+        else
+          ((observed - expected)**2) / expected
+        end
+      end
+
+      if score > max_score
+        max_score = score
+        max_time  = at_time
+
+        # Not interested in triggering any callbacks for this
+        preview_card.update_columns(max_score: max_score, max_score_at: max_time)
+      end
+
+      decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
+
+      if decaying_score.zero?
+        redis.zrem("#{PREFIX}:all", preview_card.id)
+        redis.zrem("#{PREFIX}:allowed", preview_card.id)
+      else
+        redis.zadd("#{PREFIX}:all", decaying_score, preview_card.id)
+
+        if preview_card.trendable?
+          redis.zadd("#{PREFIX}:allowed", decaying_score, preview_card.id)
+        else
+          redis.zrem("#{PREFIX}:allowed", preview_card.id)
+        end
+      end
+    end
+  end
+
+  def would_be_trending?(id)
+    score(id) > score_at_rank(options[:review_threshold] - 1)
+  end
+end

app/models/trends/tags.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+class Trends::Tags < Trends::Base
+  PREFIX = 'trending_tags'
+
+  self.default_options = {
+    threshold: 15,
+    review_threshold: 10,
+    max_score_cooldown: 2.days.freeze,
+    max_score_halflife: 4.hours.freeze,
+  }
+
+  def register(status, at_time = Time.now.utc)
+    original_status = status.reblog? ? status.reblog : status
+
+    return unless original_status.public_visibility? && status.public_visibility? &&
+                  !original_status.account.silenced? && !status.account.silenced?
+
+    original_status.tags.each do |tag|
+      add(tag, status.account_id, at_time) if tag.usable?
+    end
+  end
+
+  def add(tag, account_id, at_time = Time.now.utc)
+    tag.history.add(account_id, at_time)
+    record_used_id(tag.id, at_time)
+  end
+
+  def refresh(at_time = Time.now.utc)
+    tags = Tag.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq)
+    calculate_scores(tags, at_time)
+    trim_older_items
+  end
+
+  def get(allowed, limit)
+    tag_ids = currently_trending_ids(allowed, limit)
+    tags = Tag.where(id: tag_ids).index_by(&:id)
+    tag_ids.map { |id| tags[id] }.compact
+  end
+
+  def request_review
+    tags = Tag.where(id: currently_trending_ids(false, -1))
+
+    tags_requiring_review = tags.filter_map do |tag|
+      next unless would_be_trending?(tag.id) && !tag.trendable? && tag.requires_review_notification?
+
+      tag.touch(:requested_review_at)
+      tag
+    end
+
+    return if tags_requiring_review.empty?
+
+    User.staff.includes(:account).find_each do |user|
+      AdminMailer.new_trending_tags(user.account, tags_requiring_review).deliver_later! if user.allows_trending_tag_emails?
+    end
+  end
+
+  protected
+
+  def key_prefix
+    PREFIX
+  end
+
+  private
+
+  def calculate_scores(tags, at_time)
+    tags.each do |tag|
+      expected  = tag.history.get(at_time - 1.day).accounts.to_f
+      expected  = 1.0 if expected.zero?
+      observed  = tag.history.get(at_time).accounts.to_f
+      max_time  = tag.max_score_at
+      max_score = tag.max_score
+      max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown])
+
+      score = begin
+        if expected > observed || observed < options[:threshold]
+          0
+        else
+          ((observed - expected)**2) / expected
+        end
+      end
+
+      if score > max_score
+        max_score = score
+        max_time  = at_time
+
+        # Not interested in triggering any callbacks for this
+        tag.update_columns(max_score: max_score, max_score_at: max_time)
+      end
+
+      decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
+
+      if decaying_score.zero?
+        redis.zrem("#{PREFIX}:all", tag.id)
+        redis.zrem("#{PREFIX}:allowed", tag.id)
+      else
+        redis.zadd("#{PREFIX}:all", decaying_score, tag.id)
+
+        if tag.trendable?
+          redis.zadd("#{PREFIX}:allowed", decaying_score, tag.id)
+        else
+          redis.zrem("#{PREFIX}:allowed", tag.id)
+        end
+      end
+    end
+  end
+
+  def would_be_trending?(id)
+    score(id) > score_at_rank(options[:review_threshold] - 1)
+  end
+end

app/policies/preview_card_policy.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class PreviewCardPolicy < ApplicationPolicy
+  def index?
+    staff?
+  end
+
+  def update?
+    staff?
+  end
+end

app/policies/preview_card_provider_policy.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class PreviewCardProviderPolicy < ApplicationPolicy
+  def index?
+    staff?
+  end
+
+  def update?
+    staff?
+  end
+end

app/serializers/rest/trends/link_serializer.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class REST::Trends::LinkSerializer < REST::PreviewCardSerializer
+  attributes :history
+end

app/services/fetch_link_card_service.rb

@@ -50,7 +50,7 @@ class FetchLinkCardService < BaseService
       # We follow redirects, and ideally we want to save the preview card for
       # the destination URL and not any link shortener in-between, so here
       # we set the URL to the one of the last response in the redirect chain
-      @url  = res.request.uri.to_s.to_s
+      @url  = res.request.uri.to_s
       @card = PreviewCard.find_or_initialize_by(url: @url) if @card.url != @url
 
       if res.code == 200 && res.mime_type == 'text/html'
@@ -66,6 +66,7 @@ class FetchLinkCardService < BaseService
   def attach_card
     @status.preview_cards << @card
     Rails.cache.delete(@status)
+    Trends.links.register(@status)
   end
 
   def parse_urls

app/services/notify_service.rb

@@ -67,8 +67,49 @@ class NotifyService < BaseService
     message? && @notification.target_status.direct_visibility?
   end
 
+  # Returns true if the sender has been mentionned by the recipient up the thread
   def response_to_recipient?
-    @notification.target_status.in_reply_to_account_id == @recipient.id && @notification.target_status.thread&.direct_visibility?
+    return false if @notification.target_status.in_reply_to_id.nil?
+
+    # Using an SQL CTE to avoid unneeded back-and-forth with SQL server in case of long threads
+    !Status.count_by_sql([<<-SQL.squish, id: @notification.target_status.in_reply_to_id, recipient_id: @recipient.id, sender_id: @notification.from_account.id]).zero?
+      WITH RECURSIVE ancestors(id, in_reply_to_id, replying_to_sender) AS (
+          SELECT
+            s.id, s.in_reply_to_id, (CASE
+              WHEN s.account_id = :recipient_id THEN
+                EXISTS (
+                  SELECT *
+                  FROM mentions m
+                  WHERE m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id
+                )
+              ELSE
+                FALSE
+             END)
+          FROM statuses s
+          WHERE s.id = :id
+        UNION ALL
+          SELECT
+            s.id,
+            s.in_reply_to_id,
+            (CASE
+              WHEN s.account_id = :recipient_id THEN
+                EXISTS (
+                  SELECT *
+                  FROM mentions m
+                  WHERE m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id
+                )
+              ELSE
+                FALSE
+             END)
+          FROM ancestors st
+          JOIN statuses s ON s.id = st.in_reply_to_id
+          WHERE st.replying_to_sender IS FALSE
+      )
+      SELECT COUNT(*)
+      FROM ancestors st
+      JOIN statuses s ON s.id = st.id
+      WHERE st.replying_to_sender IS TRUE AND s.visibility = 3
+    SQL
   end
 
   def from_staff?

app/services/post_status_service.rb

@@ -100,7 +100,8 @@ class PostStatusService < BaseService
   end
 
   def postprocess_status!
-    LinkCrawlWorker.perform_async(@status.id) unless @status.spoiler_text?
+    Trends.tags.register(@status)
+    LinkCrawlWorker.perform_async(@status.id)
     DistributionWorker.perform_async(@status.id)
     ActivityPub::DistributionWorker.perform_async(@status.id) unless @status.local_only?
     PollExpirationNotifyWorker.perform_at(@status.poll.expires_at, @status.poll.id) if @status.poll

app/services/process_hashtags_service.rb

@@ -8,7 +8,7 @@ class ProcessHashtagsService < BaseService
     Tag.find_or_create_by_names(tags) do |tag|
       status.tags << tag
       records << tag
-      tag.use!(status.account, status: status, at_time: status.created_at) if status.public_visibility?
+      tag.update(last_status_at: status.created_at) if tag.last_status_at.nil? || (tag.last_status_at < status.created_at && tag.last_status_at < 12.hours.ago)
     end
 
     return unless status.distributable?

app/services/reblog_service.rb

@@ -30,12 +30,13 @@ class ReblogService < BaseService
 
     reblog = account.statuses.create!(reblog: reblogged_status, text: '', visibility: visibility, rate_limit: options[:with_rate_limit])
 
+    Trends.tags.register(reblog)
+    Trends.links.register(reblog)
     DistributionWorker.perform_async(reblog.id)
     ActivityPub::DistributionWorker.perform_async(reblog.id) unless reblogged_status.local_only?
 
     create_notification(reblog)
     bump_potential_friendship(account, reblog)
-    record_use(account, reblog)
 
     reblog
   end
@@ -60,16 +61,6 @@ class ReblogService < BaseService
     PotentialFriendshipTracker.record(account.id, reblog.reblog.account_id, :reblog)
   end
 
-  def record_use(account, reblog)
-    return unless reblog.public_visibility?
-
-    original_status = reblog.reblog
-
-    original_status.tags.each do |tag|
-      tag.use!(account)
-    end
-  end
-
   def build_json(reblog)
     Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(reblog), ActivityPub::ActivitySerializer, signer: reblog.account))
   end

app/views/admin/dashboard/index.html.haml

@@ -39,7 +39,7 @@
       %span= t('admin.dashboard.pending_users_html', count: @pending_users_count)
       = fa_icon 'chevron-right fw'
 
-    = link_to admin_tags_path(pending_review: '1'), class: 'dashboard__quick-access' do
+    = link_to admin_trends_tags_path(status: 'pending_review'), class: 'dashboard__quick-access' do
       %span= t('admin.dashboard.pending_tags_html', count: @pending_tags_count)
       = fa_icon 'chevron-right fw'
 

app/views/admin/tags/_tag.html.haml

@@ -1,19 +0,0 @@
-.batch-table__row
-  - if batch_available
-    %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
-      = f.check_box :tag_ids, { multiple: true, include_hidden: false }, tag.id
-
-  .directory__tag
-    = link_to admin_tag_path(tag.id) do
-      %h4
-        = fa_icon 'hashtag'
-        = tag.name
-
-        %small
-          = t('admin.tags.unique_uses_today', count: tag.history.first[:accounts])
-
-          - if tag.trending?
-            = fa_icon 'fire fw'
-            = t('admin.tags.trending_right_now')
-
-      .trends__item__current= friendly_number_to_human tag.history.first[:uses]

app/views/admin/tags/index.html.haml

@@ -1,71 +0,0 @@
-- content_for :page_title do
-  = t('admin.tags.title')
-
-.filters
-  .filter-subset
-    %strong= t('admin.tags.review')
-    %ul
-      %li= filter_link_to t('generic.all'), reviewed: nil, unreviewed: nil, pending_review: nil
-      %li= filter_link_to t('admin.tags.unreviewed'), unreviewed: '1', reviewed: nil, pending_review: nil
-      %li= filter_link_to t('admin.tags.reviewed'), reviewed: '1', unreviewed: nil, pending_review: nil
-      %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), pending_review: '1', reviewed: nil, unreviewed: nil
-
-  .filter-subset
-    %strong= t('generic.order_by')
-    %ul
-      %li= filter_link_to t('admin.tags.most_recent'), popular: nil, active: nil
-      %li= filter_link_to t('admin.tags.last_active'), active: '1', popular: nil
-      %li= filter_link_to t('admin.tags.most_popular'), popular: '1', active: nil
-
-
-= form_tag admin_tags_url, method: 'GET', class: 'simple_form' do
-  .fields-group
-    - TagFilter::KEYS.each do |key|
-      = hidden_field_tag key, params[key] if params[key].present?
-
-    - %i(name).each do |key|
-      .input.string.optional
-        = text_field_tag key, params[key], class: 'string optional', placeholder: I18n.t("admin.tags.#{key}")
-
-    .actions
-      %button.button= t('admin.accounts.search')
-      = link_to t('admin.accounts.reset'), admin_tags_path, class: 'button negative'
-
-%hr.spacer/
-
-= form_for(@form, url: batch_admin_tags_path) do |f|
-  = hidden_field_tag :page, params[:page] || 1
-
-  - TagFilter::KEYS.each do |key|
-    = hidden_field_tag key, params[key] if params[key].present?
-
-  .batch-table.optional
-    .batch-table__toolbar
-      - if params[:pending_review] == '1' || params[:unreviewed] == '1'
-        %label.batch-table__toolbar__select.batch-checkbox-all
-          = check_box_tag :batch_checkbox_all, nil, false
-        .batch-table__toolbar__actions
-          = f.button safe_join([fa_icon('check'), t('admin.accounts.approve')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
-
-          = f.button safe_join([fa_icon('times'), t('admin.accounts.reject')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
-      - else
-        .batch-table__toolbar__actions
-          %span.neutral-hint= t('generic.no_batch_actions_available')
-
-    .batch-table__body
-      - if @tags.empty?
-        = nothing_here 'nothing-here--under-tabs'
-      - else
-        = render partial: 'tag', collection: @tags, locals: { f: f, batch_available: params[:pending_review] == '1' || params[:unreviewed] == '1' }
-
-= paginate @tags
-
-- if params[:pending_review] == '1' || params[:unreviewed] == '1'
-  %hr.spacer/
-
-  %div.action-buttons
-    %div
-      = link_to t('admin.accounts.approve_all'), approve_all_admin_tags_path, method: :post, data: { confirm: t('admin.accounts.are_you_sure') }, class: 'button'
-
-    %div
-      = link_to t('admin.accounts.reject_all'), reject_all_admin_tags_path, method: :post, data: { confirm: t('admin.accounts.are_you_sure') }, class: 'button button--destructive'

app/views/admin/tags/show.html.haml

@@ -1,15 +1,47 @@
 - content_for :page_title do
   = "##{@tag.name}"
 
-.dashboard__counters
-  %div
-    = link_to tag_url(@tag), target: '_blank', rel: 'noopener noreferrer' do
-      .dashboard__counters__num= number_with_delimiter @accounts_today
-      .dashboard__counters__label= t 'admin.tags.accounts_today'
-  %div
-    %div
-      .dashboard__counters__num= number_with_delimiter @accounts_week
-      .dashboard__counters__label= t 'admin.tags.accounts_week'
+- content_for :heading_actions do
+  = l(@time_period.first)
+  = ' - '
+  = l(@time_period.last)
+
+.dashboard
+  .dashboard__item
+    = react_admin_component :counter, measure: 'tag_accounts', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, label: t('admin.trends.tags.dashboard.tag_accounts_measure')
+  .dashboard__item
+    = react_admin_component :counter, measure: 'tag_uses', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, label: t('admin.trends.tags.dashboard.tag_uses_measure')
+  .dashboard__item
+    = react_admin_component :counter, measure: 'tag_servers', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, label: t('admin.trends.tags.dashboard.tag_servers_measure')
+  .dashboard__item
+    = react_admin_component :dimension, dimension: 'tag_servers', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, limit: 8, label: t('admin.trends.tags.dashboard.tag_servers_dimension')
+  .dashboard__item
+    = react_admin_component :dimension, dimension: 'tag_languages', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, limit: 8, label: t('admin.trends.tags.dashboard.tag_languages_dimension')
+  .dashboard__item
+    = link_to admin_tag_path(@tag.id), class: ['dashboard__quick-access', @tag.usable? ? 'positive' : 'negative'] do
+      - if @tag.usable?
+        %span= t('admin.trends.tags.usable')
+        = fa_icon 'check fw'
+      - else
+        %span= t('admin.trends.tags.not_usable')
+        = fa_icon 'lock fw'
+
+    = link_to admin_tag_path(@tag.id), class: ['dashboard__quick-access', @tag.trendable? ? 'positive' : 'negative'] do
+      - if @tag.trendable?
+        %span= t('admin.trends.tags.trendable')
+        = fa_icon 'check fw'
+      - else
+        %span= t('admin.trends.tags.not_trendable')
+        = fa_icon 'lock fw'
+
+
+    = link_to admin_tag_path(@tag.id), class: ['dashboard__quick-access', @tag.listable? ? 'positive' : 'negative'] do
+      - if @tag.listable?
+        %span= t('admin.trends.tags.listable')
+        = fa_icon 'check fw'
+      - else
+        %span= t('admin.trends.tags.not_listable')
+        = fa_icon 'lock fw'
 
 %hr.spacer/
 
@@ -26,18 +58,3 @@
 
   .actions
     = f.button :button, t('generic.save_changes'), type: :submit
-
-%hr.spacer/
-
-%h3= t 'admin.tags.breakdown'
-
-.table-wrapper
-  %table.table
-    %tbody
-      - total = @usage_by_domain.sum(&:last).to_f
-
-      - @usage_by_domain.each do |(domain, count)|
-        %tr
-          %th= domain || site_hostname
-          %td= number_to_percentage((count / total) * 100, precision: 1)
-          %td= number_with_delimiter count

app/views/admin/trends/links/_preview_card.html.haml

@@ -0,0 +1,30 @@
+.batch-table__row{ class: [preview_card.provider&.requires_review? && 'batch-table__row--attention', !preview_card.provider&.requires_review? && !preview_card.trendable? && 'batch-table__row--muted'] }
+  %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
+    = f.check_box :preview_card_ids, { multiple: true, include_hidden: false }, preview_card.id
+
+  .batch-table__row__content.pending-account
+    .pending-account__header
+      = link_to preview_card.title, preview_card.url
+
+      %br/
+
+      - if preview_card.provider_name.present?
+        = preview_card.provider_name
+        •
+
+      - if preview_card.language.present?
+        = human_locale(preview_card.language)
+        •
+
+      = t('admin.trends.links.shared_by_over_week', count: preview_card.history.reduce(0) { |sum, day| sum + day.accounts })
+
+      - if preview_card.trendable? && (rank = Trends.links.rank(preview_card.id))
+        •
+        %abbr{ title: t('admin.trends.tags.current_score', score: Trends.links.score(preview_card.id)) }= t('admin.trends.tags.trending_rank', rank: rank + 1)
+
+        - if preview_card.max_score_at && preview_card.max_score_at >= Trends::Links::MAX_SCORE_COOLDOWN.ago && preview_card.max_score_at < 1.day.ago
+          •
+          = t('admin.trends.tags.peaked_on_and_decaying', date: l(preview_card.max_score_at.to_date, format: :short))
+      - elsif preview_card.provider&.requires_review?
+        •
+        = t('admin.trends.pending_review')

app/views/admin/trends/links/index.html.haml

@@ -0,0 +1,38 @@
+- content_for :page_title do
+  = t('admin.trends.links.title')
+
+.filters
+  .filter-subset
+    %strong= t('admin.trends.trending')
+    %ul
+      %li= filter_link_to t('generic.all'), trending: nil
+      %li= filter_link_to t('admin.trends.only_allowed'), trending: 'allowed'
+  .back-link
+    = link_to admin_trends_links_preview_card_providers_path do
+      = t('admin.trends.preview_card_providers.title')
+      = fa_icon 'chevron-right fw'
+
+%hr.spacer/
+
+= form_for(@form, url: batch_admin_trends_links_path) do |f|
+  = hidden_field_tag :page, params[:page] || 1
+
+  - PreviewCardFilter::KEYS.each do |key|
+    = hidden_field_tag key, params[key] if params[key].present?
+
+  .batch-table
+    .batch-table__toolbar
+      %label.batch-table__toolbar__select.batch-checkbox-all
+        = check_box_tag :batch_checkbox_all, nil, false
+      .batch-table__toolbar__actions
+        = f.button safe_join([fa_icon('check'), t('admin.trends.links.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+        = f.button safe_join([fa_icon('check'), t('admin.trends.links.allow_provider')]), name: :approve_all, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+        = f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+        = f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow_provider')]), name: :reject_all, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+    .batch-table__body
+      - if @preview_cards.empty?
+        = nothing_here 'nothing-here--under-tabs'
+      - else
+        = render partial: 'preview_card', collection: @preview_cards, locals: { f: f }
+
+= paginate @preview_cards

app/views/admin/trends/links/preview_card_providers/_preview_card_provider.html.haml

@@ -0,0 +1,16 @@
+.batch-table__row{ class: [preview_card_provider.requires_review? && 'batch-table__row--attention', !preview_card_provider.requires_review? && !preview_card_provider.trendable? && 'batch-table__row--muted'] }
+  %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
+    = f.check_box :preview_card_provider_ids, { multiple: true, include_hidden: false }, preview_card_provider.id
+
+  .batch-table__row__content.pending-account
+    .pending-account__header
+      %strong= preview_card_provider.domain
+
+      %br/
+
+      - if preview_card_provider.requires_review?
+        = t('admin.trends.pending_review')
+      - elsif preview_card_provider.trendable?
+        = t('admin.trends.preview_card_providers.allowed')
+      - else
+        = t('admin.trends.preview_card_providers.rejected')

app/views/admin/trends/links/preview_card_providers/index.html.haml

@@ -0,0 +1,40 @@
+- content_for :page_title do
+  = t('admin.trends.preview_card_providers.title')
+
+.filters
+  .filter-subset
+    %strong= t('admin.tags.review')
+    %ul
+      %li= filter_link_to t('generic.all'), status: nil
+      %li= filter_link_to t('admin.trends.approved'), status: 'approved'
+      %li= filter_link_to t('admin.trends.rejected'), status: 'rejected'
+      %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{PreviewCardProvider.pending_review.count})"], ' '), status: 'pending_review'
+  .back-link
+    = link_to admin_trends_links_path do
+      = fa_icon 'chevron-left fw'
+      = t('admin.trends.links.title')
+
+
+%hr.spacer/
+
+= form_for(@form, url: batch_admin_trends_links_preview_card_providers_path) do |f|
+  = hidden_field_tag :page, params[:page] || 1
+
+  - PreviewCardProviderFilter::KEYS.each do |key|
+    = hidden_field_tag key, params[key] if params[key].present?
+
+  .batch-table.optional
+    .batch-table__toolbar
+      %label.batch-table__toolbar__select.batch-checkbox-all
+        = check_box_tag :batch_checkbox_all, nil, false
+      .batch-table__toolbar__actions
+        = f.button safe_join([fa_icon('check'), t('admin.trends.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+        = f.button safe_join([fa_icon('times'), t('admin.trends.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+
+    .batch-table__body
+      - if @preview_card_providers.empty?
+        = nothing_here 'nothing-here--under-tabs'
+      - else
+        = render partial: 'preview_card_provider', collection: @preview_card_providers, locals: { f: f }
+
+= paginate @preview_card_providers

app/views/admin/trends/tags/_tag.html.haml

@@ -0,0 +1,24 @@
+.batch-table__row{ class: [tag.requires_review? && 'batch-table__row--attention', !tag.requires_review? && !tag.trendable? && 'batch-table__row--muted'] }
+  %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
+    = f.check_box :tag_ids, { multiple: true, include_hidden: false }, tag.id
+
+  .batch-table__row__content.pending-account
+    .pending-account__header
+      = link_to admin_tag_path(tag.id) do
+        = fa_icon 'hashtag'
+        = tag.name
+
+      %br/
+
+      = t('admin.trends.tags.used_by_over_week', count: tag.history.reduce(0) { |sum, day| sum + day.accounts })
+
+      - if tag.trendable? && (rank = Trends.tags.rank(tag.id))
+        •
+        %abbr{ title: t('admin.trends.tags.current_score', score: Trends.tags.score(tag.id)) }= t('admin.trends.tags.trending_rank', rank: rank + 1)
+
+        - if tag.max_score_at && tag.max_score_at >= Trends::Tags::MAX_SCORE_COOLDOWN.ago && tag.max_score_at < 1.day.ago
+          •
+          = t('admin.trends.tags.peaked_on_and_decaying', date: l(tag.max_score_at.to_date, format: :short))
+      - elsif tag.requires_review?
+        •
+        = t('admin.trends.pending_review')

app/views/admin/trends/tags/index.html.haml

@@ -0,0 +1,35 @@
+- content_for :page_title do
+  = t('admin.trends.tags.title')
+
+.filters
+  .filter-subset
+    %strong= t('admin.tags.review')
+    %ul
+      %li= filter_link_to t('generic.all'), status: nil
+      %li= filter_link_to t('admin.trends.approved'), status: 'approved'
+      %li= filter_link_to t('admin.trends.rejected'), status: 'rejected'
+      %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), status: 'pending_review'
+
+%hr.spacer/
+
+= form_for(@form, url: batch_admin_trends_tags_path) do |f|
+  = hidden_field_tag :page, params[:page] || 1
+
+  - TagFilter::KEYS.each do |key|
+    = hidden_field_tag key, params[key] if params[key].present?
+
+  .batch-table.optional
+    .batch-table__toolbar
+      %label.batch-table__toolbar__select.batch-checkbox-all
+        = check_box_tag :batch_checkbox_all, nil, false
+      .batch-table__toolbar__actions
+        = f.button safe_join([fa_icon('check'), t('admin.trends.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+        = f.button safe_join([fa_icon('times'), t('admin.trends.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+
+    .batch-table__body
+      - if @tags.empty?
+        = nothing_here 'nothing-here--under-tabs'
+      - else
+        = render partial: 'tag', collection: @tags, locals: { f: f }
+
+= paginate @tags

app/views/admin_mailer/new_trending_links.text.erb

@@ -0,0 +1,16 @@
+<%= raw t('application_mailer.salutation', name: display_name(@me)) %>
+
+<%= raw t('admin_mailer.new_trending_links.body') %>
+
+<% @links.each do |link| %>
+- <%= link.title %> • <%= link.url %>
+  <%= t('admin.trends.links.usage_comparison', today: link.history.get(Time.now.utc).accounts, yesterday: link.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.links.score(link.id).round(2)) %>
+<% end %>
+
+<% if @lowest_trending_link %>
+<%= t('admin_mailer.new_trending_links.requirements', lowest_link_title: @lowest_trending_link.title, lowest_link_score: Trends.links.score(@lowest_trending_link.id).round(2)) %>
+<% else %>
+<%= t('admin_mailer.new_trending_links.no_approved_links') %>
+<% end %>
+
+<%= raw t('application_mailer.view')%> <%= admin_trends_links_url %>

app/views/admin_mailer/new_trending_tag.text.erb

@@ -1,5 +0,0 @@
-<%= raw t('application_mailer.salutation', name: display_name(@me)) %>
-
-<%= raw t('admin_mailer.new_trending_tag.body', name: @tag.name) %>
-
-<%= raw t('application_mailer.view')%> <%= admin_tags_url(pending_review: '1') %>

app/views/admin_mailer/new_trending_tags.text.erb

@@ -0,0 +1,16 @@
+<%= raw t('application_mailer.salutation', name: display_name(@me)) %>
+
+<%= raw t('admin_mailer.new_trending_tags.body') %>
+
+<% @tags.each do |tag| %>
+- #<%= tag.name %>
+  <%= t('admin.trends.tags.usage_comparison', today: tag.history.get(Time.now.utc).accounts, yesterday: tag.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.tags.score(tag.id).round(2)) %>
+<% end %>
+
+<% if @lowest_trending_tag %>
+<%= t('admin_mailer.new_trending_tags.requirements', lowest_tag_name: @lowest_trending_tag.name, lowest_tag_score: Trends.tags.score(@lowest_trending_tag.id).round(2)) %>
+<% else %>
+<%= t('admin_mailer.new_trending_tags.no_approved_tags') %>
+<% end %>
+
+<%= raw t('application_mailer.view')%> <%= admin_trends_tags_url(pending_review: '1') %>

app/views/application/_sidebar.html.haml

@@ -6,7 +6,7 @@
     %p= @instance_presenter.site_short_description.html_safe.presence || t('about.about_mastodon_html')
 
 - if Setting.trends && !(user_signed_in? && !current_user.setting_trends)
-  - trends = TrendingTags.get(3)
+  - trends = Trends.tags.get(true, 3)
 
   - unless trends.empty?
     .endorsements-widget.trends-widget

app/workers/scheduler/trending_tags_scheduler.rb → app/workers/scheduler/trends/refresh_scheduler.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
-class Scheduler::TrendingTagsScheduler
+class Scheduler::Trends::RefreshScheduler
   include Sidekiq::Worker
 
   sidekiq_options retry: 0
 
   def perform
-    TrendingTags.update! if Setting.trends
+    Trends.refresh!
   end
 end

app/workers/scheduler/trends/review_notifications_scheduler.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class Scheduler::Trends::ReviewNotificationsScheduler
+  include Sidekiq::Worker
+
+  sidekiq_options retry: 0
+
+  def perform
+    Trends.request_review!
+  end
+end

config/brakeman.ignore

@@ -67,7 +67,7 @@
       "check_name": "SQL",
       "message": "Possible SQL injection",
       "file": "app/models/account.rb",
-      "line": 479,
+      "line": 484,
       "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
       "code": "find_by_sql([\"          WITH first_degree AS (\\n            SELECT target_account_id\\n            FROM follows\\n            WHERE account_id = ?\\n            UNION ALL\\n            SELECT ?\\n          )\\n          SELECT\\n            accounts.*,\\n            (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n          FROM accounts\\n          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?)\\n          WHERE accounts.id IN (SELECT * FROM first_degree)\\n            AND #{query} @@ #{textsearch}\\n            AND accounts.suspended_at IS NULL\\n            AND accounts.moved_to_account_id IS NULL\\n          GROUP BY accounts.id\\n          ORDER BY rank DESC\\n          LIMIT ? OFFSET ?\\n\".squish, account.id, account.id, account.id, limit, offset])",
       "render_path": null,
@@ -100,6 +100,26 @@
       "confidence": "Weak",
       "note": ""
     },
+    {
+      "warning_type": "SQL Injection",
+      "warning_code": 0,
+      "fingerprint": "75fcd147b7611763ab6915faf8c5b0709e612b460f27c05c72d8b9bd0a6a77f8",
+      "check_name": "SQL",
+      "message": "Possible SQL injection",
+      "file": "lib/mastodon/snowflake.rb",
+      "line": 87,
+      "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
+      "code": "connection.execute(\"CREATE OR REPLACE FUNCTION timestamp_id(table_name text)\\nRETURNS bigint AS\\n$$\\n  DECLARE\\n    time_part bigint;\\n    sequence_base bigint;\\n    tail bigint;\\n  BEGIN\\n    time_part := (\\n      -- Get the time in milliseconds\\n      ((date_part('epoch', now()) * 1000))::bigint\\n      -- And shift it over two bytes\\n      << 16);\\n\\n    sequence_base := (\\n      'x' ||\\n      -- Take the first two bytes (four hex characters)\\n      substr(\\n        -- Of the MD5 hash of the data we documented\\n        md5(table_name || '#{SecureRandom.hex(16)}' || time_part::text),\\n        1, 4\\n      )\\n    -- And turn it into a bigint\\n    )::bit(16)::bigint;\\n\\n    -- Finally, add our sequence number to our base, and chop\\n    -- it to the last two bytes\\n    tail := (\\n      (sequence_base + nextval(table_name || '_id_seq'))\\n      & 65535);\\n\\n    -- Return the time part and the sequence part. OR appears\\n    -- faster here than addition, but they're equivalent:\\n    -- time_part has no trailing two bytes, and tail is only\\n    -- the last two bytes.\\n    RETURN time_part | tail;\\n  END\\n$$ LANGUAGE plpgsql VOLATILE;\\n\")",
+      "render_path": null,
+      "location": {
+        "type": "method",
+        "class": "Mastodon::Snowflake",
+        "method": "define_timestamp_id"
+      },
+      "user_input": "SecureRandom.hex(16)",
+      "confidence": "Medium",
+      "note": ""
+    },
     {
       "warning_type": "Mass Assignment",
       "warning_code": 105,
@@ -143,40 +163,40 @@
     {
       "warning_type": "SQL Injection",
       "warning_code": 0,
-      "fingerprint": "9251d682c4e2840e1b2fea91e7d758efe2097ecb7f6255c065e3750d25eb178c",
+      "fingerprint": "8c1d8c4b76c1cd3960e90dff999f854a6ff742fcfd8de6c7184ac5a1b1a4d7dd",
       "check_name": "SQL",
       "message": "Possible SQL injection",
-      "file": "app/models/account.rb",
-      "line": 448,
+      "file": "app/models/preview_card_filter.rb",
+      "line": 50,
       "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
-      "code": "find_by_sql([\"        SELECT\\n          accounts.*,\\n          ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n        FROM accounts\\n        WHERE #{query} @@ #{textsearch}\\n          AND accounts.suspended_at IS NULL\\n          AND accounts.moved_to_account_id IS NULL\\n        ORDER BY rank DESC\\n        LIMIT ? OFFSET ?\\n\".squish, limit, offset])",
+      "code": "PreviewCard.joins(\"join unnest(array[#{(Trends.links.currently_trending_ids(true, -1) or Trends.links.currently_trending_ids(false, -1)).map(&:to_i).join(\",\")}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id\")",
       "render_path": null,
       "location": {
         "type": "method",
-        "class": "Account",
-        "method": "search_for"
+        "class": "PreviewCardFilter",
+        "method": "trending_scope"
       },
-      "user_input": "textsearch",
+      "user_input": "(Trends.links.currently_trending_ids(true, -1) or Trends.links.currently_trending_ids(false, -1)).map(&:to_i).join(\",\")",
       "confidence": "Medium",
       "note": ""
     },
     {
       "warning_type": "SQL Injection",
       "warning_code": 0,
-      "fingerprint": "9ccb9ba6a6947400e187d515e0bf719d22993d37cfc123c824d7fafa6caa9ac3",
+      "fingerprint": "9251d682c4e2840e1b2fea91e7d758efe2097ecb7f6255c065e3750d25eb178c",
       "check_name": "SQL",
       "message": "Possible SQL injection",
-      "file": "lib/mastodon/snowflake.rb",
-      "line": 87,
+      "file": "app/models/account.rb",
+      "line": 453,
       "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
-      "code": "connection.execute(\"        CREATE OR REPLACE FUNCTION timestamp_id(table_name text)\\n        RETURNS bigint AS\\n        $$\\n          DECLARE\\n            time_part bigint;\\n            sequence_base bigint;\\n            tail bigint;\\n          BEGIN\\n            time_part := (\\n              -- Get the time in milliseconds\\n              ((date_part('epoch', now()) * 1000))::bigint\\n              -- And shift it over two bytes\\n              << 16);\\n\\n            sequence_base := (\\n              'x' ||\\n              -- Take the first two bytes (four hex characters)\\n              substr(\\n                -- Of the MD5 hash of the data we documented\\n                md5(table_name ||\\n                  '#{SecureRandom.hex(16)}' ||\\n                  time_part::text\\n                ),\\n                1, 4\\n              )\\n            -- And turn it into a bigint\\n            )::bit(16)::bigint;\\n\\n            -- Finally, add our sequence number to our base, and chop\\n            -- it to the last two bytes\\n            tail := (\\n              (sequence_base + nextval(table_name || '_id_seq'))\\n              & 65535);\\n\\n            -- Return the time part and the sequence part. OR appears\\n            -- faster here than addition, but they're equivalent:\\n            -- time_part has no trailing two bytes, and tail is only\\n            -- the last two bytes.\\n            RETURN time_part | tail;\\n          END\\n        $$ LANGUAGE plpgsql VOLATILE;\\n\")",
+      "code": "find_by_sql([\"        SELECT\\n          accounts.*,\\n          ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n        FROM accounts\\n        WHERE #{query} @@ #{textsearch}\\n          AND accounts.suspended_at IS NULL\\n          AND accounts.moved_to_account_id IS NULL\\n        ORDER BY rank DESC\\n        LIMIT ? OFFSET ?\\n\".squish, limit, offset])",
       "render_path": null,
       "location": {
         "type": "method",
-        "class": "Mastodon::Snowflake",
-        "method": "define_timestamp_id"
+        "class": "Account",
+        "method": "search_for"
       },
-      "user_input": "SecureRandom.hex(16)",
+      "user_input": "textsearch",
       "confidence": "Medium",
       "note": ""
     },
@@ -201,23 +221,53 @@
       "note": ""
     },
     {
-      "warning_type": "Redirect",
-      "warning_code": 18,
-      "fingerprint": "ba699ddcc6552c422c4ecd50d2cd217f616a2446659e185a50b05a0f2dad8d33",
-      "check_name": "Redirect",
-      "message": "Possible unprotected redirect",
-      "file": "app/controllers/media_controller.rb",
-      "line": 20,
-      "link": "https://brakemanscanner.org/docs/warning_types/redirect/",
-      "code": "redirect_to(MediaAttachment.attached.find_by!(:shortcode => ((params[:id] or params[:medium_id]))).file.url(:original))",
+      "warning_type": "SQL Injection",
+      "warning_code": 0,
+      "fingerprint": "c32a484ccd9da46abd3bc93d08b72029d7dbc0576ccf4e878a9627e9a83cad2e",
+      "check_name": "SQL",
+      "message": "Possible SQL injection",
+      "file": "app/models/tag_filter.rb",
+      "line": 50,
+      "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
+      "code": "Tag.joins(\"join unnest(array[#{Trends.tags.currently_trending_ids(false, -1).map(&:to_i).join(\",\")}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id\")",
       "render_path": null,
       "location": {
         "type": "method",
-        "class": "MediaController",
-        "method": "show"
+        "class": "TagFilter",
+        "method": "trending_scope"
       },
-      "user_input": "MediaAttachment.attached.find_by!(:shortcode => ((params[:id] or params[:medium_id]))).file.url(:original)",
-      "confidence": "High",
+      "user_input": "Trends.tags.currently_trending_ids(false, -1).map(&:to_i).join(\",\")",
+      "confidence": "Medium",
+      "note": ""
+    },
+    {
+      "warning_type": "Cross-Site Scripting",
+      "warning_code": 4,
+      "fingerprint": "cd5cfd7f40037fbfa753e494d7129df16e358bfc43ef0da3febafbf4ee1ed3ac",
+      "check_name": "LinkToHref",
+      "message": "Potentially unsafe model attribute in `link_to` href",
+      "file": "app/views/admin/trends/links/_preview_card.html.haml",
+      "line": 7,
+      "link": "https://brakemanscanner.org/docs/warning_types/link_to_href",
+      "code": "link_to((Unresolved Model).new.title, (Unresolved Model).new.url)",
+      "render_path": [
+        {
+          "type": "template",
+          "name": "admin/trends/links/index",
+          "line": 37,
+          "file": "app/views/admin/trends/links/index.html.haml",
+          "rendered": {
+            "name": "admin/trends/links/_preview_card",
+            "file": "app/views/admin/trends/links/_preview_card.html.haml"
+          }
+        }
+      ],
+      "location": {
+        "type": "template",
+        "template": "admin/trends/links/_preview_card"
+      },
+      "user_input": "(Unresolved Model).new.url",
+      "confidence": "Weak",
       "note": ""
     },
     {
@@ -227,7 +277,7 @@
       "check_name": "SQL",
       "message": "Possible SQL injection",
       "file": "app/models/account.rb",
-      "line": 495,
+      "line": 500,
       "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
       "code": "find_by_sql([\"          SELECT\\n            accounts.*,\\n            (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n          FROM accounts\\n          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?) OR (accounts.id = f.target_account_id AND f.account_id = ?)\\n          WHERE #{query} @@ #{textsearch}\\n            AND accounts.suspended_at IS NULL\\n            AND accounts.moved_to_account_id IS NULL\\n          GROUP BY accounts.id\\n          ORDER BY rank DESC\\n          LIMIT ? OFFSET ?\\n\".squish, account.id, account.id, limit, offset])",
       "render_path": null,
@@ -261,6 +311,6 @@
       "note": ""
     }
   ],
-  "updated": "2021-05-11 20:22:27 +0900",
-  "brakeman_version": "5.0.1"
+  "updated": "2021-11-14 05:26:09 +0100",
+  "brakeman_version": "5.1.2"
 }

config/locales/en.yml

@@ -674,8 +674,8 @@ en:
         desc_html: Affects hashtags that have not been previously disallowed
         title: Allow hashtags to trend without prior review
       trends:
-        desc_html: Publicly display previously reviewed hashtags that are currently trending
-        title: Trending hashtags
+        desc_html: Publicly display previously reviewed content that is currently trending
+        title: Trends
     site_uploads:
       delete: Delete uploaded file
       destroyed_msg: Site upload successfully deleted!
@@ -702,21 +702,51 @@ en:
       sidekiq_process_check:
         message_html: No Sidekiq process running for the %{value} queue(s). Please review your Sidekiq configuration
     tags:
-      accounts_today: Unique uses today
-      accounts_week: Unique uses this week
-      breakdown: Breakdown of today's usage by source
-      last_active: Recently used
-      most_popular: Most popular
-      most_recent: Recently created
-      name: Hashtag
       review: Review status
-      reviewed: Reviewed
-      title: Hashtags
-      trending_right_now: Trending right now
-      unique_uses_today: "%{count} posting today"
-      unreviewed: Not reviewed
       updated_msg: Hashtag settings updated successfully
     title: Administration
+    trends:
+      allow: Allow
+      approved: Approved
+      disallow: Disallow
+      links:
+        allow: Allow link
+        allow_provider: Allow publisher
+        disallow: Disallow link
+        disallow_provider: Disallow publisher
+        shared_by_over_week:
+          one: Shared by one person over the last week
+          other: Shared by %{count} people over the last week
+        title: Trending links
+        usage_comparison: Shared %{today} times today, compared to %{yesterday} yesterday
+      pending_review: Pending review
+      preview_card_providers:
+        allowed: Links from this publisher can trend
+        rejected: Links from this publisher won't trend
+        title: Publishers
+      rejected: Rejected
+      tags:
+        current_score: Current score %{score}
+        dashboard:
+          tag_accounts_measure: unique uses
+          tag_languages_dimension: Top languages
+          tag_servers_dimension: Top servers
+          tag_servers_measure: different servers
+          tag_uses_measure: total uses
+        listable: Can be suggested
+        not_listable: Won't be suggested
+        not_trendable: Won't appear under trends
+        not_usable: Cannot be used
+        peaked_on_and_decaying: Peaked on %{date}, now decaying
+        title: Trending hashtags
+        trendable: Can appear under trends
+        trending_rank: 'Trending #%{rank}'
+        usable: Can be used
+        usage_comparison: Used %{today} times today, compared to %{yesterday} yesterday
+        used_by_over_week:
+          one: Used by one person over the last week
+          other: Used by %{count} people over the last week
+      title: Trends
     warning_presets:
       add_new: Add new
       delete: Delete
@@ -731,9 +761,16 @@ en:
       body: "%{reporter} has reported %{target}"
       body_remote: Someone from %{domain} has reported %{target}
       subject: New report for %{instance} (#%{id})
-    new_trending_tag:
-      body: 'The hashtag #%{name} is trending today, but has not been previously reviewed. It will not be displayed publicly unless you allow it to, or just save the form as it is to never hear about it again.'
-      subject: New hashtag up for review on %{instance} (#%{name})
+    new_trending_links:
+      body: The following links are trending today, but their publishers have not been previously reviewed. They will not be displayed publicly unless you approve them. Further notifications from the same publishers will not be generated.
+      no_approved_links: There are currently no approved trending links.
+      requirements: The lowest approved trending link is currently "%{lowest_link_title}" with a score of %{lowest_link_score}.
+      subject: New trending links up for review on %{instance}
+    new_trending_tags:
+      body: 'The following hashtags are trending today, but they have not been previously reviewed. They will not be displayed publicly unless you approve them:'
+      no_approved_tags: There are currently no approved trending hashtags.
+      requirements: 'The lowest approved trending hashtag is currently #%{lowest_tag_name} with a score of %{lowest_tag_score}.'
+      subject: New trending hashtags up for review on %{instance}
   aliases:
     add_new: Create alias
     created_msg: Successfully created a new alias. You can now initiate the move from the old account.
@@ -940,7 +977,7 @@ en:
     changes_saved_msg: Changes successfully saved!
     copy: Copy
     delete: Delete
-    no_batch_actions_available: No batch actions available on this page
+    none: None
     order_by: Order by
     save_changes: Save changes
     validation_errors:

config/locales/simple_form.en.yml

@@ -204,8 +204,8 @@ en:
         mention: Someone mentioned you
         pending_account: New account needs review
         reblog: Someone boosted your post
-        report: New report is submitted
-        trending_tag: An unreviewed hashtag is trending
+        report: A new report is submitted
+        trending_tag: A new trend requires approval
       rule:
         text: Rule
       tag:

config/navigation.rb

@@ -40,12 +40,16 @@ SimpleNavigation::Configuration.run do |navigation|
     n.item :invites, safe_join([fa_icon('user-plus fw'), t('invites.title')]), invites_path, if: proc { Setting.min_invite_role == 'user' && current_user.functional? }
     n.item :development, safe_join([fa_icon('code fw'), t('settings.development')]), settings_applications_url, if: -> { current_user.functional? }
 
+    n.item :trends, safe_join([fa_icon('fire fw'), t('admin.trends.title')]), admin_trends_tags_path, if: proc { current_user.staff? } do |s|
+      s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.trends.tags.title')]), admin_trends_tags_path, highlights_on: %r{/admin/tags|/admin/trends/tags}
+      s.item :links, safe_join([fa_icon('newspaper-o fw'), t('admin.trends.links.title')]), admin_trends_links_path, highlights_on: %r{/admin/trends/links}
+    end
+
     n.item :moderation, safe_join([fa_icon('gavel fw'), t('moderation.title')]), admin_reports_url, if: proc { current_user.staff? } do |s|
       s.item :action_logs, safe_join([fa_icon('bars fw'), t('admin.action_logs.title')]), admin_action_logs_url
       s.item :reports, safe_join([fa_icon('flag fw'), t('admin.reports.title')]), admin_reports_url, highlights_on: %r{/admin/reports}
       s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_url, highlights_on: %r{/admin/accounts|/admin/pending_accounts}
       s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path
-      s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.tags.title')]), admin_tags_path, highlights_on: %r{/admin/tags}
       s.item :follow_recommendations, safe_join([fa_icon('user-plus fw'), t('admin.follow_recommendations.title')]), admin_follow_recommendations_path, highlights_on: %r{/admin/follow_recommendations}
       s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: whitelist_mode? ? nil : '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks|/admin/domain_allows}, if: -> { current_user.admin? }
       s.item :email_domain_blocks, safe_join([fa_icon('envelope fw'), t('admin.email_domain_blocks.title')]), admin_email_domain_blocks_url, highlights_on: %r{/admin/email_domain_blocks}, if: -> { current_user.admin? }

config/routes.rb

@@ -303,12 +303,27 @@ Rails.application.routes.draw do
 
     resources :account_moderation_notes, only: [:create, :destroy]
     resource :follow_recommendations, only: [:show, :update]
+    resources :tags, only: [:show, :update]
 
-    resources :tags, only: [:index, :show, :update] do
-      collection do
-        post :approve_all
-        post :reject_all
-        post :batch
+    namespace :trends do
+      resources :links, only: [:index] do
+        collection do
+          post :batch
+        end
+      end
+
+      resources :tags, only: [:index] do
+        collection do
+          post :batch
+        end
+      end
+
+      namespace :links do
+        resources :preview_card_providers, only: [:index], path: :publishers do
+          collection do
+            post :batch
+          end
+        end
       end
     end
   end
@@ -402,7 +417,7 @@ Rails.application.routes.draw do
       resources :favourites,   only: [:index]
       resources :bookmarks,    only: [:index]
       resources :reports,      only: [:create]
-      resources :trends,       only: [:index]
+      resources :trends,       only: [:index], controller: 'trends/tags'
       resources :filters,      only: [:index, :create, :show, :update, :destroy]
       resources :endorsements, only: [:index]
       resources :markers,      only: [:index, :create]
@@ -413,6 +428,11 @@ Rails.application.routes.draw do
 
       resources :apps, only: [:create]
 
+      namespace :trends do
+        resources :links, only: [:index]
+        resources :tags, only: [:index]
+      end
+
       namespace :emails do
         resources :confirmations, only: [:create]
       end
@@ -516,7 +536,9 @@ Rails.application.routes.draw do
           end
         end
 
-        resources :trends, only: [:index]
+        namespace :trends do
+          resources :tags, only: [:index]
+        end
 
         post :measures, to: 'measures#create'
         post :dimensions, to: 'dimensions#create'

config/sidekiq.yml

@@ -13,9 +13,13 @@
     every: '5m'
     class: Scheduler::ScheduledStatusesScheduler
     queue: scheduler
-  trending_tags_scheduler:
+  trends_refresh_scheduler:
     every: '5m'
-    class: Scheduler::TrendingTagsScheduler
+    class: Scheduler::Trends::RefreshScheduler
+    queue: scheduler
+  trends_review_notifications_scheduler:
+    every: '2h'
+    class: Scheduler::Trends::ReviewNotificationsScheduler
     queue: scheduler
   media_cleanup_scheduler:
     cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *'

db/migrate/20211031031021_create_preview_card_providers.rb

@@ -0,0 +1,12 @@
+class CreatePreviewCardProviders < ActiveRecord::Migration[6.1]
+  def change
+    create_table :preview_card_providers do |t|
+      t.string :domain, null: false, default: '', index: { unique: true }
+      t.attachment :icon
+      t.boolean :trendable
+      t.datetime :reviewed_at
+      t.datetime :requested_review_at
+      t.timestamps
+    end
+  end
+end

db/migrate/20211112011713_add_language_to_preview_cards.rb

@@ -0,0 +1,7 @@
+class AddLanguageToPreviewCards < ActiveRecord::Migration[6.1]
+  def change
+    add_column :preview_cards, :language, :string
+    add_column :preview_cards, :max_score, :float
+    add_column :preview_cards, :max_score_at, :datetime
+  end
+end

db/migrate/20211115032527_add_trendable_to_preview_cards.rb

@@ -0,0 +1,5 @@
+class AddTrendableToPreviewCards < ActiveRecord::Migration[6.1]
+  def change
+    add_column :preview_cards, :trendable, :boolean
+  end
+end

db/migrate/20211123212714_add_link_type_to_preview_cards.rb

@@ -0,0 +1,5 @@
+class AddLinkTypeToPreviewCards < ActiveRecord::Migration[6.1]
+  def change
+    add_column :preview_cards, :link_type, :int
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2021_08_08_071221) do
+ActiveRecord::Schema.define(version: 2021_11_23_212714) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -689,6 +689,20 @@ ActiveRecord::Schema.define(version: 2021_08_08_071221) do
     t.index ["status_id"], name: "index_polls_on_status_id"
   end
 
+  create_table "preview_card_providers", force: :cascade do |t|
+    t.string "domain", default: "", null: false
+    t.string "icon_file_name"
+    t.string "icon_content_type"
+    t.bigint "icon_file_size"
+    t.datetime "icon_updated_at"
+    t.boolean "trendable"
+    t.datetime "reviewed_at"
+    t.datetime "requested_review_at"
+    t.datetime "created_at", precision: 6, null: false
+    t.datetime "updated_at", precision: 6, null: false
+    t.index ["domain"], name: "index_preview_card_providers_on_domain", unique: true
+  end
+
   create_table "preview_cards", force: :cascade do |t|
     t.string "url", default: "", null: false
     t.string "title", default: "", null: false
@@ -710,6 +724,11 @@ ActiveRecord::Schema.define(version: 2021_08_08_071221) do
     t.string "embed_url", default: "", null: false
     t.integer "image_storage_schema_version"
     t.string "blurhash"
+    t.string "language"
+    t.float "max_score"
+    t.datetime "max_score_at"
+    t.boolean "trendable"
+    t.integer "link_type"
     t.index ["url"], name: "index_preview_cards_on_url", unique: true
   end
 

lib/mastodon/snowflake.rb

@@ -84,10 +84,7 @@ module Mastodon::Snowflake
               -- Take the first two bytes (four hex characters)
               substr(
                 -- Of the MD5 hash of the data we documented
-                md5(table_name ||
-                  '#{SecureRandom.hex(16)}' ||
-                  time_part::text
-                ),
+                md5(table_name || '#{SecureRandom.hex(16)}' || time_part::text),
                 1, 4
               )
             -- And turn it into a bigint

lib/tasks/repo.rake

@@ -96,7 +96,7 @@ namespace :repo do
     end.uniq.compact
 
     missing_available_locales = locales_in_files - I18n.available_locales
-    missing_locale_names = I18n.available_locales.reject { |locale| SettingsHelper::HUMAN_LOCALES.key?(locale) }
+    missing_locale_names = I18n.available_locales.reject { |locale| LanguagesHelper::HUMAN_LOCALES.key?(locale) }
 
     critical = false
 

spec/controllers/admin/tags_controller_spec.rb

@@ -9,18 +9,6 @@ RSpec.describe Admin::TagsController, type: :controller do
     sign_in Fabricate(:user, admin: true)
   end
 
-  describe 'GET #index' do
-    let!(:tag) { Fabricate(:tag) }
-
-    before do
-      get :index
-    end
-
-    it 'returns status 200' do
-      expect(response).to have_http_status(200)
-    end
-  end
-
   describe 'GET #show' do
     let!(:tag) { Fabricate(:tag) }
 

spec/controllers/api/v1/trends/tags_controller_spec.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe Api::V1::Trends::TagsController, type: :controller do
+  render_views
+
+  describe 'GET #index' do
+    before do
+      trending_tags = double()
+
+      allow(trending_tags).to receive(:get).and_return(Fabricate.times(10, :tag))
+      allow(Trends).to receive(:tags).and_return(trending_tags)
+
+      get :index
+    end
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+  end
+end

spec/controllers/api/v1/trends_controller_spec.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-RSpec.describe Api::V1::TrendsController, type: :controller do
-  render_views
-
-  describe 'GET #index' do
-    before do
-      allow(TrendingTags).to receive(:get).and_return(Fabricate.times(10, :tag))
-      get :index
-    end
-
-    it 'returns http success' do
-      expect(response).to have_http_status(200)
-    end
-  end
-end

spec/helpers/settings_helper_spec.rb → spec/helpers/languages_helper_spec.rb

@@ -2,20 +2,15 @@
 
 require 'rails_helper'
 
-describe SettingsHelper do
+describe LanguagesHelper do
   describe 'the HUMAN_LOCALES constant' do
     it 'includes all I18n locales' do
-      options = I18n.available_locales
-
-      expect(described_class::HUMAN_LOCALES.keys).to include(*options)
+      expect(described_class::HUMAN_LOCALES.keys).to include(*I18n.available_locales)
     end
   end
 
   describe 'human_locale' do
     it 'finds the human readable local description from a key' do
-      # Ensure the value is as we expect
-      expect(described_class::HUMAN_LOCALES[:en]).to eq('English')
-
       expect(helper.human_locale(:en)).to eq('English')
     end
   end

spec/mailers/previews/admin_mailer_preview.rb

@@ -5,4 +5,14 @@ class AdminMailerPreview < ActionMailer::Preview
   def new_pending_account
     AdminMailer.new_pending_account(Account.first, User.pending.first)
   end
+
+  # Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_trending_tags
+  def new_trending_tags
+    AdminMailer.new_trending_tags(Account.first, Tag.limit(3))
+  end
+
+  # Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_trending_links
+  def new_trending_links
+    AdminMailer.new_trending_links(Account.first, PreviewCard.limit(3))
+  end
 end

spec/models/trending_tags_spec.rb

@@ -1,68 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe TrendingTags do
-  describe '.record_use!' do
-    pending
-  end
-
-  describe '.update!' do
-    let!(:at_time) { Time.now.utc }
-    let!(:tag1) { Fabricate(:tag, name: 'Catstodon', trendable: true) }
-    let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon', trendable: true) }
-    let!(:tag3) { Fabricate(:tag, name: 'OCs', trendable: true) }
-
-    before do
-      allow(Redis.current).to receive(:pfcount) do |key|
-        case key
-        when "activity:tags:#{tag1.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts"
-          2
-        when "activity:tags:#{tag1.id}:#{at_time.beginning_of_day.to_i}:accounts"
-          16
-        when "activity:tags:#{tag2.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts"
-          0
-        when "activity:tags:#{tag2.id}:#{at_time.beginning_of_day.to_i}:accounts"
-          4
-        when "activity:tags:#{tag3.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts"
-          13
-        end
-      end
-
-      Redis.current.zadd('trending_tags', 0.9, tag3.id)
-      Redis.current.sadd("trending_tags:used:#{at_time.beginning_of_day.to_i}", [tag1.id, tag2.id])
-
-      tag3.update(max_score: 0.9, max_score_at: (at_time - 1.day).beginning_of_day + 12.hours)
-
-      described_class.update!(at_time)
-    end
-
-    it 'calculates and re-calculates scores' do
-      expect(described_class.get(10, filtered: false)).to eq [tag1, tag3]
-    end
-
-    it 'omits hashtags below threshold' do
-      expect(described_class.get(10, filtered: false)).to_not include(tag2)
-    end
-
-    it 'decays scores' do
-      expect(Redis.current.zscore('trending_tags', tag3.id)).to be < 0.9
-    end
-  end
-
-  describe '.trending?' do
-    let(:tag) { Fabricate(:tag) }
-
-    before do
-      10.times { |i| Redis.current.zadd('trending_tags', i + 1, Fabricate(:tag).id) }
-    end
-
-    it 'returns true if the hashtag is within limit' do
-      Redis.current.zadd('trending_tags', 11, tag.id)
-      expect(described_class.trending?(tag)).to be true
-    end
-
-    it 'returns false if the hashtag is outside the limit' do
-      Redis.current.zadd('trending_tags', 0, tag.id)
-      expect(described_class.trending?(tag)).to be false
-    end
-  end
-end

spec/models/trends/tags_spec.rb

@@ -0,0 +1,67 @@
+require 'rails_helper'
+
+RSpec.describe Trends::Tags do
+  subject { described_class.new(threshold: 5, review_threshold: 10) }
+
+  let!(:at_time) { DateTime.new(2021, 11, 14, 10, 15, 0) }
+
+  describe '#add' do
+    let(:tag) { Fabricate(:tag) }
+
+    before do
+      subject.add(tag, 1, at_time)
+    end
+
+    it 'records history' do
+      expect(tag.history.get(at_time).accounts).to eq 1
+    end
+
+    it 'records use' do
+      expect(subject.send(:recently_used_ids, at_time)).to eq [tag.id]
+    end
+  end
+
+  describe '#get' do
+    pending
+  end
+
+  describe '#refresh' do
+    let!(:today) { at_time }
+    let!(:yesterday) { today - 1.day }
+
+    let!(:tag1) { Fabricate(:tag, name: 'Catstodon', trendable: true) }
+    let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon', trendable: true) }
+    let!(:tag3) { Fabricate(:tag, name: 'OCs', trendable: true) }
+
+    before do
+      2.times  { |i| subject.add(tag1, i, yesterday) }
+      13.times { |i| subject.add(tag3, i, yesterday) }
+      16.times { |i| subject.add(tag1, i, today) }
+      4.times  { |i| subject.add(tag2, i, today) }
+    end
+
+    context do
+      before do
+        subject.refresh(yesterday + 12.hours)
+        subject.refresh(at_time)
+      end
+
+      it 'calculates and re-calculates scores' do
+        expect(subject.get(false, 10)).to eq [tag1, tag3]
+      end
+
+      it 'omits hashtags below threshold' do
+        expect(subject.get(false, 10)).to_not include(tag2)
+      end
+    end
+
+    it 'decays scores' do
+      subject.refresh(yesterday + 12.hours)
+      original_score = subject.score(tag3.id)
+      expect(original_score).to eq 144.0
+      subject.refresh(yesterday + 12.hours + subject.options[:max_score_halflife])
+      decayed_score = subject.score(tag3.id)
+      expect(decayed_score).to be <= original_score / 2
+    end
+  end
+end

spec/services/notify_service_spec.rb

@@ -64,8 +64,9 @@ RSpec.describe NotifyService, type: :service do
         is_expected.to_not change(Notification, :count)
       end
 
-      context 'if the message chain initiated by recipient, but is not direct message' do
+      context 'if the message chain is initiated by recipient, but is not direct message' do
         let(:reply_to) { Fabricate(:status, account: recipient) }
+        let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) }
         let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: reply_to)) }
 
         it 'does not notify' do
@@ -73,8 +74,20 @@ RSpec.describe NotifyService, type: :service do
         end
       end
 
-      context 'if the message chain initiated by recipient and is direct message' do
+      context 'if the message chain is initiated by recipient, but without a mention to the sender, even if the sender sends multiple messages in a row' do
+        let(:reply_to) { Fabricate(:status, account: recipient) }
+        let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) }
+        let(:dummy_reply) { Fabricate(:status, account: sender, visibility: :direct, thread: reply_to) }
+        let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: dummy_reply)) }
+
+        it 'does not notify' do
+          is_expected.to_not change(Notification, :count)
+        end
+      end
+
+      context 'if the message chain is initiated by the recipient with a mention to the sender' do
         let(:reply_to) { Fabricate(:status, account: recipient, visibility: :direct) }
+        let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) }
         let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: reply_to)) }
 
         it 'does notify' do