Browse Source

Fix opening and closing Redis connections instead of using a pool (#18171)

* Fix opening and closing Redis connections instead of using a pool

* Fix Redis connections not being returned to the pool in CLI commands
closed-social-glitch-2
Eugen Rochko 2 years ago
committed by GitHub
parent
commit
7b0fe4aef9
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 28 additions and 12 deletions
  1. +6
    -1
      app/lib/redis_configuration.rb
  2. +1
    -1
      app/models/concerns/redisable.rb
  3. +4
    -2
      config/initializers/stoplight.rb
  4. +9
    -3
      lib/mastodon/cli_helper.rb
  5. +1
    -1
      lib/mastodon/rack_middleware.rb
  6. +4
    -3
      lib/mastodon/search_cli.rb
  7. +1
    -1
      lib/mastodon/sidekiq_middleware.rb
  8. +2
    -0
      spec/services/resolve_account_service_spec.rb

+ 6
- 1
app/lib/redis_configuration.rb View File

@ -2,12 +2,17 @@
class RedisConfiguration
class << self
def establish_pool(new_pool_size)
@pool&.shutdown(&:close)
@pool = ConnectionPool.new(size: new_pool_size) { new.connection }
end
def with
pool.with { |redis| yield redis }
end
def pool
@pool ||= ConnectionPool.new(size: pool_size) { new.connection }
@pool ||= establish_pool(pool_size)
end
def pool_size

+ 1
- 1
app/models/concerns/redisable.rb View File

@ -6,6 +6,6 @@ module Redisable
private
def redis
Thread.current[:redis] ||= RedisConfiguration.new.connection
Thread.current[:redis] ||= RedisConfiguration.pool.checkout
end
end

+ 4
- 2
config/initializers/stoplight.rb View File

@ -1,4 +1,6 @@
require 'stoplight'
Stoplight::Light.default_data_store = Stoplight::DataStore::Redis.new(RedisConfiguration.new.connection)
Stoplight::Light.default_notifiers = [Stoplight::Notifier::Logger.new(Rails.logger)]
Rails.application.reloader.to_prepare do
Stoplight::Light.default_data_store = Stoplight::DataStore::Redis.new(RedisConfiguration.new.connection)
Stoplight::Light.default_notifiers = [Stoplight::Notifier::Logger.new(Rails.logger)]
end

+ 9
- 3
lib/mastodon/cli_helper.rb View File

@ -19,15 +19,18 @@ module Mastodon
ProgressBar.create(total: total, format: '%c/%u |%b%i| %e')
end
def reset_connection_pools!
ActiveRecord::Base.establish_connection(ActiveRecord::Base.configurations[Rails.env].dup.tap { |config| config['pool'] = options[:concurrency] + 1 })
RedisConfiguration.establish_pool(options[:concurrency])
end
def parallelize_with_progress(scope)
if options[:concurrency] < 1
say('Cannot run with this concurrency setting, must be at least 1', :red)
exit(1)
end
db_config = ActiveRecord::Base.configurations[Rails.env].dup
db_config['pool'] = options[:concurrency] + 1
ActiveRecord::Base.establish_connection(db_config)
reset_connection_pools!
progress = create_progress_bar(scope.count)
pool = Concurrent::FixedThreadPool.new(options[:concurrency])
@ -52,6 +55,9 @@ module Mastodon
result = ActiveRecord::Base.connection_pool.with_connection do
yield(item)
ensure
RedisConfiguration.pool.checkin if Thread.current[:redis]
Thread.current[:redis] = nil
end
aggregate.increment(result) if result.is_a?(Integer)

+ 1
- 1
lib/mastodon/rack_middleware.rb View File

@ -19,7 +19,7 @@ class Mastodon::RackMiddleware
end
def clean_up_redis_socket!
Thread.current[:redis]&.close
RedisConfiguration.pool.checkin if Thread.current[:redis]
Thread.current[:redis] = nil
end

+ 4
- 3
lib/mastodon/search_cli.rb View File

@ -59,9 +59,7 @@ module Mastodon
index.specification.lock!
end
db_config = ActiveRecord::Base.configurations[Rails.env].dup
db_config['pool'] = options[:concurrency] + 1
ActiveRecord::Base.establish_connection(db_config)
reset_connection_pools!
pool = Concurrent::FixedThreadPool.new(options[:concurrency])
added = Concurrent::AtomicFixnum.new(0)
@ -139,6 +137,9 @@ module Mastodon
sleep 1
rescue => e
progress.log pastel.red("Error importing #{index}: #{e}")
ensure
RedisConfiguration.pool.checkin if Thread.current[:redis]
Thread.current[:redis] = nil
end
end
end

+ 1
- 1
lib/mastodon/sidekiq_middleware.rb View File

@ -26,7 +26,7 @@ class Mastodon::SidekiqMiddleware
end
def clean_up_redis_socket!
Thread.current[:redis]&.close
RedisConfiguration.pool.checkin if Thread.current[:redis]
Thread.current[:redis] = nil
end

+ 2
- 0
spec/services/resolve_account_service_spec.rb View File

@ -220,6 +220,8 @@ RSpec.describe ResolveAccountService, type: :service do
return_values << described_class.new.call('foo@ap.example.com')
rescue ActiveRecord::RecordNotUnique
fail_occurred = true
ensure
RedisConfiguration.pool.checkin if Thread.current[:redis]
end
end
end

Loading…
Cancel
Save