Merge pull request #17 from paladinsoftware/rework-2.0

Rework 2.0

.github/gemfiles/sidekiq-6.5.8.gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gem "sidekiq", "= 6.5.8"
+
+gemspec path: '../../'

.github/gemfiles/sidekiq-7.0.6.gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gem "sidekiq", "= 7.0.6"
+
+gemspec path: '../../'

.github/workflows/tests.yml

@@ -14,16 +14,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        sidekiq: [ "5.0.5", "5.1.3", "5.2.7", "6.0.2", "6.2.2", "6.3.1", "7.0.0" ]
+        sidekiq: [ "6.5.8", "7.0.6" ]
+        ruby: [ "2.7.7", "3.0.5", "3.1.3", "3.2.1" ]
     env:
       RAILS_ENV: test
       SIDEKIQ_VERSION: ${{ matrix.sidekiq }}
-      BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/sidekiq-${{ matrix.sidekiq }}.gemfile
+      BUNDLE_GEMFILE: ${{ github.workspace }}/.github/gemfiles/sidekiq-${{ matrix.sidekiq }}.gemfile
+      RUBY_VERSION: ${{ matrix.ruby }}
     steps:
     - uses: actions/checkout@v2
-    - run: |
-        version=$(cat gemfiles/sidekiq-${{ matrix.sidekiq }}.gemfile | grep "ruby \".*\"" | grep -o "[0-9]\.[0-9]\.[0-9]")
-        echo "RUBY_VERSION=$version" >> $GITHUB_ENV
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:

.gitignore

@@ -9,11 +9,11 @@ InstalledFiles
 _yardoc
 coverage
 doc/
-gemfiles/*.gemfile.lock
+.github/gemfiles/*.gemfile.lock
 lib/bundler/man
 pkg
 rdoc
 spec/reports
 test/tmp
 test/version_tmp
-tmp
+tmp

CHANGELOG.md

@@ -0,0 +1,12 @@
+## [2.0.0] - 2023-02-28
+Complete rewrite of the library:  
+- Instead of iterating through whole schedule set, sidekiq-debouncer will now cache debounce key in redis with a reference to the job.
+Thanks to that there is a huge performance boost compared to V1. With 1k jobs in schedule set it's over 100x faster.
+The difference is even bigger with larger amount of jobs.
+- Debouncing is now handled by Lua script instead of pure ruby so it's process safe.
+
+Breaking changes:
+- Including `Sidekiq::Debouncer` in the workers and using `debounce` method is now deprecated. Use `perform_async` instead.
+- Setup requires middlewares to be added in sidekiq configuration.
+- `by` attribute is now required
+- dropped support for Ruby < 2.7 and Sidekiq < 6.5

Gemfile

@@ -1,3 +1,3 @@
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
 gemspec

LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2012 Gabriel Evans
+Copyright (c) 2023 Karol Bąk
 
 MIT License
 
@@ -19,4 +19,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -2,7 +2,9 @@
 
 Sidekiq extension that adds the ability to debounce job execution.
 
-Worker will postpone its execution after `wait time` have elapsed since the last time it was invoked. Useful for implementing behavior that should only happen after the input has stopped arriving. For example: sending group email to the user after he stopped interacting with the application.
+Worker will postpone its execution after `wait time` have elapsed since the last time it was invoked. 
+Useful for implementing behavior that should only happen after the input has stopped arriving. 
+For example: sending one email with multiple notifications.
 
 ## Installation
 
@@ -18,39 +20,35 @@ And then execute:
 
 ## Basic usage
 
-In a worker, include `Sidekiq::Debouncer` module, specify debounce wait time (in seconds):
+Add middlewares to sidekiq:
 
 ```ruby
-class MyWorker
-  include Sidekiq::Worker
-  include Sidekiq::Debouncer
-
-  sidekiq_options(
-    debounce: {
-      time: 5 * 60
-    }
-  )
+Sidekiq.configure_client do |config|
+  config.client_middleware do |chain|
+    chain.add Sidekiq::Debouncer::Middleware::Client
+  end
+end
 
-  def perform(group)
-    group.each do
-      # do some work with group
-    end
+Sidekiq.configure_server do |config|
+  config.client_middleware do |chain|
+    chain.add Sidekiq::Debouncer::Middleware::Client
+  end
+
+  config.server_middleware do |chain|
+    chain.add Sidekiq::Debouncer::Middleware::Server
   end
 end
 ```
 
-You can specify your own debounce method. In this case worker will be debounced if first argument matches.
+Add debounce option to worker with `by` and `time` keys:
 ```ruby
 class MyWorker
   include Sidekiq::Worker
-  include Sidekiq::Debouncer
 
   sidekiq_options(
     debounce: {
-      time: 5 * 60,
-      by: -> (job_args) {
-        job_args[0]
-      }
+      by: -> (args) { args[0] }, # debounce by first argument only
+      time: 5 * 60
     }
   )
 
@@ -62,11 +60,10 @@ class MyWorker
 end
 ```
 
-You can also pass symbol as `debounce_by` matching class method.
+You can also pass symbol as `debounce.by` matching class method.
 ```ruby
 class MyWorker
   include Sidekiq::Worker
-  include Sidekiq::Debouncer
 
   sidekiq_options(
     debounce: {
@@ -87,16 +84,24 @@ class MyWorker
 end
 ```
 
+Keep in mind that the result of the debounce method will be converted to string, so make sure it doesn't return any objects that don't implement `to_s` method.
 
-In the application, call `MyWorker.debounce(...)`. Everytime you call this function, `MyWorker`'s execution will be postponed by 5 minutes. After that time `MyWorker` will receive a method call `perform` with an array of arguments that were provided to the `MyWorker.debounce(...)`.
+In the application, call `MyWorker.perform_async(...)` as usual. Everytime you call this function, `MyWorker`'s execution will be postponed by 5 minutes. After that time `MyWorker` will receive a method call `perform` with an array of arguments that were provided to the `MyWorker.perform_async(...)` calls.
 
-## Testing
+To avoid keeping leftover keys in redis (for example, when job was manually removed from schedule set), all additional keys are created with TTL.
+It's 7 days by default and should be ok in most of the cases. If you are debouncing your jobs in higher interval than that, you can overwrite this setting:
 
-From https://github.com/mperham/sidekiq/wiki/Testing#api:
+```ruby
+Sidekiq.configure_client do |config|
+  config.client_middleware do |chain|
+    chain.add Sidekiq::Debouncer::Middleware::Client, ttl: 60 * 60 * 24 * 30 # 30 days
+  end
+end
+```
 
-> Sidekiq's API does not have a testing mode, e.g. something like Sidekiq::ScheduledSet.new.each(...) will always hit Redis. You can use Sidekiq::Testing.disable! to set up jobs in order to use the API in your tests against a real Redis instance.
+## Testing
 
-In order to test the behavior of `sidekiq-debouncer` it is necessary to disable testing mode. It is the limitation of internal implementation and Sidekiq API.
+In order to test the behavior of `sidekiq-debouncer` it is necessary to disable testing mode. It is the limitation of internal implementation.
 
 ## License
 

lib/sidekiq-debouncer.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require "sidekiq"
+require "sidekiq/debouncer"

lib/sidekiq/debouncer.rb

@@ -1,7 +1,9 @@
-require 'sidekiq'
-require 'sidekiq/api'
+# frozen_string_literal: true
 
-require 'sidekiq/debouncer/version'
+require "sidekiq/debouncer/version"
+require "sidekiq/debouncer/errors"
+require "sidekiq/debouncer/middleware/client"
+require "sidekiq/debouncer/middleware/server"
 
 module Sidekiq
   module Debouncer
@@ -10,46 +12,9 @@ def self.included(base)
     end
 
     module ClassMethods
-      DEFAULT_DEBOUNCE_FOR = 5 * 60 # 5.minutes
-      DEFAULT_DEBOUNCE_BY = -> (job_args) { 0 }
-
-      def debounce(*args)
-        sidekiq_options["debounce"] ||= {}
-
-        debounce_for = sidekiq_options["debounce"][:time] || DEFAULT_DEBOUNCE_FOR
-        debounce_by = sidekiq_options["debounce"][:by] || DEFAULT_DEBOUNCE_BY
-        debounce_by_value = debounce_by.is_a?(Symbol) ? send(debounce_by, args) : debounce_by.call(args)
-
-        jobs = jobs_to_debounce(debounce_by, debounce_by_value)
-
-        debounce_job(jobs, debounce_for, args)
-      end
-
-      private
-
-      def jobs_to_debounce(debounce_by, debounce_by_value)
-        ss = Sidekiq::ScheduledSet.new
-        ss.select { |job| job.klass == self.to_s }.select do |job|
-          debounce_by_value_job = debounce_by.is_a?(Symbol) ? send(debounce_by, job.args[0][0]) : debounce_by.call(job.args[0][0])
-
-          debounce_by_value_job == debounce_by_value
-        end
-      end
-
-      def debounce_job(jobs, debounce_for, args)
-        time_from_now = Time.now + debounce_for
-        jobs_to_group = []
-
-        jobs.each do |job|
-          if job.at > Time.now && job.at < time_from_now
-            jobs_to_group += job.args[0]
-            job.delete
-          end
-        end
-
-        jobs_to_group << args
-
-        perform_in(debounce_for, jobs_to_group)
+      def debounce(...)
+        warn "WARNING: debounce method is deprecated, use perform_async instead"
+        perform_async(...)
       end
     end
   end

lib/sidekiq/debouncer/errors.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Debouncer
+    Error = Class.new(StandardError)
+    NotSupportedError = Class.new(Error)
+    MissingArgumentError = Class.new(Error)
+  end
+end

lib/sidekiq/debouncer/lua/debounce.lua

@@ -0,0 +1,22 @@
+local set = KEYS[1]
+local debounce_key = KEYS[2]
+
+local job = ARGV[1]
+local time = ARGV[2]
+local ttl = ARGV[3]
+
+local existing_debounce = redis.call("GET", debounce_key)
+
+if existing_debounce then
+    redis.call("DEL", debounce_key)
+    -- skip if job wasn't found in schedule set
+    if redis.call("ZREM", set, existing_debounce) > 0 then
+        local new_args = cjson.decode(job)['args'][1]
+        local new_job = cjson.decode(existing_debounce)
+        table.insert(new_job['args'], new_args)
+        job = cjson.encode(new_job)
+    end
+end
+
+redis.call("SET", debounce_key, job, "EX", ttl)
+redis.call("ZADD", set, time, job)