Refactor (ruby) redis configuration (#31694)

app/lib/redis_configuration.rb → app/lib/redis_connection.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class RedisConfiguration
+class RedisConnection
   class << self
     def establish_pool(new_pool_size)
       @pool&.shutdown(&:close)
@@ -22,33 +22,24 @@ def pool_size
     end
   end
 
+  attr_reader :config
+
+  def initialize
+    @config = REDIS_CONFIGURATION.base
+  end
+
   def connection
-    if namespace?
+    namespace = config[:namespace]
+    if namespace.present?
       Redis::Namespace.new(namespace, redis: raw_connection)
     else
       raw_connection
     end
   end
 
-  def namespace?
-    namespace.present?
-  end
-
-  def namespace
-    ENV.fetch('REDIS_NAMESPACE', nil)
-  end
-
-  def url
-    ENV['REDIS_URL']
-  end
-
-  def redis_driver
-    ENV.fetch('REDIS_DRIVER', 'hiredis') == 'ruby' ? :ruby : :hiredis
-  end
-
   private
 
   def raw_connection
-    Redis.new(url: url, driver: redis_driver)
+    Redis.new(**config)
   end
 end

app/models/concerns/redisable.rb

@@ -2,10 +2,10 @@
 
 module Redisable
   def redis
-    Thread.current[:redis] ||= RedisConfiguration.pool.checkout
+    Thread.current[:redis] ||= RedisConnection.pool.checkout
   end
 
   def with_redis(&block)
-    RedisConfiguration.with(&block)
+    RedisConnection.with(&block)
   end
 end

config/application.rb

@@ -101,7 +101,8 @@ class Application < Rails::Application
     end
 
     config.before_configuration do
-      require 'mastodon/redis_config'
+      require 'mastodon/redis_configuration'
+      ::REDIS_CONFIGURATION = Mastodon::RedisConfiguration.new
 
       config.x.use_vips = ENV['MASTODON_USE_LIBVIPS'] == 'true'
 

config/environments/development.rb

@@ -25,7 +25,7 @@
     config.action_controller.perform_caching = true
     config.action_controller.enable_fragment_cache_logging = true
 
-    config.cache_store = :redis_cache_store, REDIS_CACHE_PARAMS
+    config.cache_store = :redis_cache_store, REDIS_CONFIGURATION.cache
     config.public_file_server.headers = {
       'Cache-Control' => "public, max-age=#{2.days.to_i}",
     }

config/environments/production.rb

@@ -57,7 +57,7 @@
   config.log_tags = [:request_id]
 
   # Use a different cache store in production.
-  config.cache_store = :redis_cache_store, REDIS_CACHE_PARAMS
+  config.cache_store = :redis_cache_store, REDIS_CONFIGURATION.cache
 
   # Use a real queuing backend for Active Job (and separate queues per environment).
   # config.active_job.queue_adapter     = :resque

config/initializers/sidekiq.rb

@@ -3,7 +3,7 @@
 require_relative '../../lib/mastodon/sidekiq_middleware'
 
 Sidekiq.configure_server do |config|
-  config.redis = REDIS_SIDEKIQ_PARAMS
+  config.redis = REDIS_CONFIGURATION.sidekiq
 
   # This is used in Kubernetes setups, to signal that the Sidekiq process has started and will begin processing jobs
   # This comes from https://github.com/sidekiq/sidekiq/wiki/Kubernetes#sidekiq
@@ -51,7 +51,7 @@
 end
 
 Sidekiq.configure_client do |config|
-  config.redis = REDIS_SIDEKIQ_PARAMS
+  config.redis = REDIS_CONFIGURATION.sidekiq
 
   config.client_middleware do |chain|
     chain.add SidekiqUniqueJobs::Middleware::Client

config/initializers/stoplight.rb

@@ -3,6 +3,6 @@
 require 'stoplight'
 
 Rails.application.reloader.to_prepare do
-  Stoplight.default_data_store = Stoplight::DataStore::Redis.new(RedisConfiguration.new.connection)
+  Stoplight.default_data_store = Stoplight::DataStore::Redis.new(RedisConnection.new.connection)
   Stoplight.default_notifiers  = [Stoplight::Notifier::Logger.new(Rails.logger)]
 end

db/migrate/20170920032311_fix_reblogs_in_feeds.rb

@@ -2,7 +2,7 @@
 
 class FixReblogsInFeeds < ActiveRecord::Migration[5.1]
   def up
-    redis = RedisConfiguration.pool.checkout
+    redis = RedisConnection.pool.checkout
     fm = FeedManager.instance
 
     # Old scheme:

db/migrate/20200407202420_migrate_unavailable_inboxes.rb

@@ -4,7 +4,7 @@ class MigrateUnavailableInboxes < ActiveRecord::Migration[5.2]
   disable_ddl_transaction!
 
   def up
-    redis = RedisConfiguration.pool.checkout
+    redis = RedisConnection.pool.checkout
     urls = redis.smembers('unavailable_inboxes')
 
     hosts = urls.filter_map do |url|

lib/chewy/strategy/mastodon.rb

@@ -14,7 +14,7 @@ def update(type, objects, _options = {})
       end
 
       def leave
-        RedisConfiguration.with do |redis|
+        RedisConnection.with do |redis|
           redis.pipelined do |pipeline|
             @stash.each do |type, ids|
               pipeline.sadd("chewy:queue:#{type.name}", ids)

lib/mastodon/cli/base.rb

@@ -40,7 +40,7 @@ def reset_connection_pools!
             .dup
             .tap { |config| config['pool'] = options[:concurrency] + 1 }
         )
-        RedisConfiguration.establish_pool(options[:concurrency])
+        RedisConnection.establish_pool(options[:concurrency])
       end
     end
   end

lib/mastodon/cli/progress_helper.rb

@@ -51,7 +51,7 @@ def parallelize_with_progress(scope)
               result = ActiveRecord::Base.connection_pool.with_connection do
                 yield(item)
               ensure
-                RedisConfiguration.pool.checkin if Thread.current[:redis]
+                RedisConnection.pool.checkin if Thread.current[:redis]
                 Thread.current[:redis] = nil
               end
 

lib/mastodon/rack_middleware.rb

@@ -19,7 +19,7 @@ def clean_up_sockets!
   end
 
   def clean_up_redis_socket!
-    RedisConfiguration.pool.checkin if Thread.current[:redis]
+    RedisConnection.pool.checkin if Thread.current[:redis]
     Thread.current[:redis] = nil
   end
 

lib/mastodon/redis_configuration.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+class Mastodon::RedisConfiguration
+  def base
+    @base ||= {
+      url: setup_base_redis_url,
+      driver: driver,
+      namespace: base_namespace,
+    }
+  end
+
+  def sidekiq
+    @sidekiq ||= {
+      url: setup_prefixed_redis_url(:sidekiq),
+      driver: driver,
+      namespace: sidekiq_namespace,
+    }
+  end
+
+  def cache
+    @cache ||= {
+      url: setup_prefixed_redis_url(:cache),
+      driver: driver,
+      namespace: cache_namespace,
+      expires_in: 10.minutes,
+      connect_timeout: 5,
+      pool: {
+        size: Sidekiq.server? ? Sidekiq[:concurrency] : Integer(ENV['MAX_THREADS'] || 5),
+        timeout: 5,
+      },
+    }
+  end
+
+  private
+
+  def driver
+    ENV['REDIS_DRIVER'] == 'ruby' ? :ruby : :hiredis
+  end
+
+  def namespace
+    @namespace ||= ENV.fetch('REDIS_NAMESPACE', nil)
+  end
+
+  def base_namespace
+    return "mastodon_test#{ENV.fetch('TEST_ENV_NUMBER', nil)}" if Rails.env.test?
+
+    namespace
+  end
+
+  def sidekiq_namespace
+    namespace
+  end
+
+  def cache_namespace
+    namespace ? "#{namespace}_cache" : 'cache'
+  end
+
+  def setup_base_redis_url
+    url = ENV.fetch('REDIS_URL', nil)
+    return url if url.present?
+
+    user     = ENV.fetch('REDIS_USER', '')
+    password = ENV.fetch('REDIS_PASSWORD', '')
+    host     = ENV.fetch('REDIS_HOST', 'localhost')
+    port     = ENV.fetch('REDIS_PORT', 6379)
+    db       = ENV.fetch('REDIS_DB', 0)
+
+    construct_uri(host, port, db, user, password)
+  end
+
+  def setup_prefixed_redis_url(prefix)
+    prefix = "#{prefix.to_s.upcase}_"
+    url = ENV.fetch("#{prefix}REDIS_URL", nil)
+
+    return url if url.present?
+
+    user     = ENV.fetch("#{prefix}REDIS_USER", nil)
+    password = ENV.fetch("#{prefix}REDIS_PASSWORD", nil)
+    host     = ENV.fetch("#{prefix}REDIS_HOST", nil)
+    port     = ENV.fetch("#{prefix}REDIS_PORT", nil)
+    db       = ENV.fetch("#{prefix}REDIS_DB", nil)
+
+    if host.nil?
+      base[:url]
+    else
+      construct_uri(host, port, db, user, password)
+    end
+  end
+
+  def construct_uri(host, port, db, user, password)
+    Addressable::URI.parse("redis://#{host}:#{port}/#{db}").tap do |uri|
+      uri.user = user if user.present?
+      uri.password = password if password.present?
+    end.normalize.to_str
+  end
+end

lib/mastodon/sidekiq_middleware.rb

@@ -53,7 +53,7 @@ def clean_up_elasticsearch_connections!
   end
 
   def clean_up_redis_socket!
-    RedisConfiguration.pool.checkin if Thread.current[:redis]
+    RedisConnection.pool.checkin if Thread.current[:redis]
     Thread.current[:redis] = nil
   end