|
|
@ -7,7 +7,7 @@ class Scheduler::AccountsStatusesCleanupScheduler |
|
|
|
# This limit is mostly to be nice to the fediverse at large and not |
|
|
|
# generate too much traffic. |
|
|
|
# This also helps limiting the running time of the scheduler itself. |
|
|
|
MAX_BUDGET = 50 |
|
|
|
MAX_BUDGET = 150 |
|
|
|
|
|
|
|
# This is an attempt to spread the load across instances, as various |
|
|
|
# accounts are likely to have various followers. |
|
|
@ -15,28 +15,22 @@ class Scheduler::AccountsStatusesCleanupScheduler |
|
|
|
|
|
|
|
# This is an attempt to limit the workload generated by status removal |
|
|
|
# jobs to something the particular instance can handle. |
|
|
|
PER_THREAD_BUDGET = 5 |
|
|
|
PER_THREAD_BUDGET = 6 |
|
|
|
|
|
|
|
# Those avoid loading an instance that is already under load |
|
|
|
MAX_DEFAULT_SIZE = 2 |
|
|
|
MAX_DEFAULT_SIZE = 200 |
|
|
|
MAX_DEFAULT_LATENCY = 5 |
|
|
|
MAX_PUSH_SIZE = 5 |
|
|
|
MAX_PUSH_SIZE = 500 |
|
|
|
MAX_PUSH_LATENCY = 10 |
|
|
|
|
|
|
|
# 'pull' queue has lower priority jobs, and it's unlikely that pushing |
|
|
|
# deletes would cause much issues with this queue if it didn't cause issues |
|
|
|
# with default and push. Yet, do not enqueue deletes if the instance is |
|
|
|
# lagging behind too much. |
|
|
|
MAX_PULL_SIZE = 500 |
|
|
|
MAX_PULL_LATENCY = 300 |
|
|
|
|
|
|
|
# This is less of an issue in general, but deleting old statuses is likely |
|
|
|
# to cause delivery errors, and thus increase the number of jobs to be retried. |
|
|
|
# This doesn't directly translate to load, but connection errors and a high |
|
|
|
# number of dead instances may lead to this spiraling out of control if |
|
|
|
# unchecked. |
|
|
|
MAX_RETRY_SIZE = 50_000 |
|
|
|
MAX_PULL_SIZE = 10_000 |
|
|
|
MAX_PULL_LATENCY = 5.minutes.to_i |
|
|
|
|
|
|
|
sidekiq_options retry: 0, lock: :until_executed |
|
|
|
sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i |
|
|
|
|
|
|
|
def perform |
|
|
|
return if under_load? |
|
|
@ -73,8 +67,6 @@ class Scheduler::AccountsStatusesCleanupScheduler |
|
|
|
end |
|
|
|
|
|
|
|
def under_load? |
|
|
|
return true if Sidekiq::Stats.new.retry_size > MAX_RETRY_SIZE |
|
|
|
|
|
|
|
queue_under_load?('default', MAX_DEFAULT_SIZE, MAX_DEFAULT_LATENCY) || queue_under_load?('push', MAX_PUSH_SIZE, MAX_PUSH_LATENCY) || queue_under_load?('pull', MAX_PULL_SIZE, MAX_PULL_LATENCY) |
|
|
|
end |
|
|
|
|
|
|
|