From 540ab48e46f71949644c4b080bafa0c6c357f24e Mon Sep 17 00:00:00 2001 From: drsnuggles Date: Fri, 8 Mar 2013 19:27:51 +1000 Subject: [PATCH] Sidekiq looks for workers in /workers not /jobs. Everything now works 100%. --- Gemfile | 3 +- Gemfile.lock | 39 +++++++++---------- Rakefile | 1 - app/{jobs => workers}/delete_genotype.rb | 2 - app/{jobs => workers}/fitbit_dump.rb | 2 +- app/{jobs => workers}/fitbit_edit.rb | 2 +- .../fitbit_endsubscription.rb | 2 +- app/{jobs => workers}/fitbit_init.rb | 2 +- app/{jobs => workers}/fitbit_notification.rb | 2 +- app/{jobs => workers}/fixphenotypes.rb | 8 ++-- app/{jobs => workers}/frequency.rb | 2 +- app/{jobs => workers}/genomegov.rb | 2 - app/{jobs => workers}/mailnewphenotype.rb | 4 +- app/{jobs => workers}/mendeley_search.rb | 4 +- app/{jobs => workers}/mendeleydetails.rb | 1 - app/{jobs => workers}/parsing.rb | 4 +- app/{jobs => workers}/pgpannotation.rb | 3 +- app/{jobs => workers}/plos.rb | 5 +-- app/{jobs => workers}/plosdetails.rb | 1 - app/{jobs => workers}/preparsing.rb | 6 +-- app/{jobs => workers}/recommendphenotypes.rb | 2 +- app/{jobs => workers}/recommendvariations.rb | 2 +- app/{jobs => workers}/snpedia.rb | 2 - app/{jobs => workers}/zipfulldata.rb | 4 +- app/{jobs => workers}/zipgenotypingfiles.rb | 2 +- config.ru | 4 +- config/initializers/resque.rb | 4 -- config/routes.rb | 4 ++ lib/tasks/resque.rake | 6 --- serverscript | 2 +- turnoffserver | 2 +- 31 files changed, 54 insertions(+), 75 deletions(-) rename app/{jobs => workers}/delete_genotype.rb (97%) rename app/{jobs => workers}/fitbit_dump.rb (99%) rename app/{jobs => workers}/fitbit_edit.rb (99%) rename app/{jobs => workers}/fitbit_endsubscription.rb (97%) rename app/{jobs => workers}/fitbit_init.rb (97%) rename app/{jobs => workers}/fitbit_notification.rb (99%) rename app/{jobs => workers}/fixphenotypes.rb (87%) rename app/{jobs => workers}/frequency.rb (98%) rename app/{jobs => workers}/genomegov.rb (97%) rename app/{jobs => workers}/mailnewphenotype.rb (78%) rename app/{jobs => workers}/mendeley_search.rb (96%) rename app/{jobs => workers}/mendeleydetails.rb (98%) rename app/{jobs => workers}/parsing.rb (99%) rename app/{jobs => workers}/pgpannotation.rb (97%) rename app/{jobs => workers}/plos.rb (95%) rename app/{jobs => workers}/plosdetails.rb (98%) rename app/{jobs => workers}/preparsing.rb (98%) rename app/{jobs => workers}/recommendphenotypes.rb (98%) rename app/{jobs => workers}/recommendvariations.rb (98%) rename app/{jobs => workers}/snpedia.rb (98%) rename app/{jobs => workers}/zipfulldata.rb (99%) rename app/{jobs => workers}/zipgenotypingfiles.rb (98%) delete mode 100644 config/initializers/resque.rb delete mode 100644 lib/tasks/resque.rake diff --git a/Gemfile b/Gemfile index 36dbb62f..3b47a842 100644 --- a/Gemfile +++ b/Gemfile @@ -42,8 +42,9 @@ gem 'friendly_id', :git => 'git://github.com/norman/friendly_id.git' gem 'recommendify',:git => 'git://github.com/paulasmuth/recommendify.git', :ref => "34308c4" # for jobs -gem 'resque', '1.23.0' gem 'sidekiq' +gem 'slim' +gem 'sinatra' #group :production do # gem 'rpm_contrib' diff --git a/Gemfile.lock b/Gemfile.lock index 295061d8..fcbd8fc5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,12 +1,3 @@ -GIT - remote: git://github.com/nevans/resque-pool.git - revision: 96218b25fabbcbe58a109135706611a33158ba13 - specs: - resque-pool (0.4.0.dev) - rake - resque (~> 1.20) - trollop (~> 1.16) - GIT remote: git://github.com/norman/friendly_id.git revision: cb0c987be9cc36fea581ba56b321f8790f0b1c50 @@ -90,11 +81,15 @@ GEM net-sftp (>= 2.0.0) net-ssh (>= 2.0.14) net-ssh-gateway (>= 1.1.0) + celluloid (0.12.4) + facter (>= 1.6.12) + timers (>= 1.0.0) climate_control (0.0.3) activesupport (>= 3.0) cocaine (0.5.1) climate_control (>= 0.0.3, < 1.0) columnize (0.3.6) + connection_pool (1.0.0) debugger (1.3.3) columnize (>= 0.3.1) debugger-linecache (~> 1.1.1) @@ -104,6 +99,7 @@ GEM debugger-ruby_core_source (1.2.0) dynamic_form (1.1.4) erubis (2.7.0) + facter (1.6.17) factory_girl (4.2.0) activesupport (>= 3.0.0) fitgem (0.6.0) @@ -181,13 +177,6 @@ GEM redis (3.0.2) redis-namespace (1.2.1) redis (~> 3.0.0) - resque (1.23.0) - multi_json (~> 1.0) - redis-namespace (~> 1.0) - sinatra (>= 0.9.2) - vegas (~> 0.1.2) - resque-loner (1.2.1) - resque (~> 1.0) rest-client (1.6.7) mime-types (>= 1.16) rsolr (1.0.8) @@ -198,6 +187,12 @@ GEM sanitize (2.0.3) nokogiri (>= 1.4.4, < 1.6) shoulda-context (1.0.2) + sidekiq (2.8.0) + celluloid (~> 0.12.0) + connection_pool (~> 1.0) + multi_json (~> 1) + redis (~> 3) + redis-namespace simplecov (0.7.1) multi_json (~> 1.0) simplecov-html (~> 0.7.1) @@ -206,6 +201,9 @@ GEM rack (~> 1.4) rack-protection (~> 1.3) tilt (~> 1.3, >= 1.3.3) + slim (1.3.6) + temple (~> 0.5.5) + tilt (~> 1.3.3) sprockets (2.2.2) hike (~> 1.2) multi_json (~> 1.0) @@ -218,12 +216,13 @@ GEM nokogiri sunspot (= 1.3.3) sunspot_solr (1.3.3) + temple (0.5.5) thor (0.17.0) tilt (1.3.3) + timers (1.1.0) treetop (1.4.12) polyglot polyglot (>= 0.3.1) - trollop (1.16.2) tzinfo (0.3.35) vegas (0.1.11) rack (>= 1.0.0) @@ -257,14 +256,14 @@ DEPENDENCIES rails3-generators recaptcha recommendify! - resque (= 1.23.0) - resque-loner - resque-pool! rubyzip (= 0.9.5) rvm-capistrano sanitize shoulda-context + sidekiq simplecov + sinatra + slim sunspot_rails sunspot_solr sunspot_test! diff --git a/Rakefile b/Rakefile index 46a860a9..bf881069 100644 --- a/Rakefile +++ b/Rakefile @@ -1,7 +1,6 @@ # Add your own tasks in files placed in lib/tasks ending in .rake, # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake. -require 'resque/tasks' require File.expand_path('../config/application', __FILE__) require 'rake' diff --git a/app/jobs/delete_genotype.rb b/app/workers/delete_genotype.rb similarity index 97% rename from app/jobs/delete_genotype.rb rename to app/workers/delete_genotype.rb index 86b9a278..3e1686fa 100644 --- a/app/jobs/delete_genotype.rb +++ b/app/workers/delete_genotype.rb @@ -1,5 +1,3 @@ -require 'resque' - class DeleteGenotype include Sidekiq::Worker sidekiq_options :queue => :deletegenotype diff --git a/app/jobs/fitbit_dump.rb b/app/workers/fitbit_dump.rb similarity index 99% rename from app/jobs/fitbit_dump.rb rename to app/workers/fitbit_dump.rb index 7f399900..8f4182ef 100644 --- a/app/jobs/fitbit_dump.rb +++ b/app/workers/fitbit_dump.rb @@ -1,4 +1,4 @@ -require 'resque' + class FitbitDump include Sidekiq::Worker diff --git a/app/jobs/fitbit_edit.rb b/app/workers/fitbit_edit.rb similarity index 99% rename from app/jobs/fitbit_edit.rb rename to app/workers/fitbit_edit.rb index 2f49962d..54af0424 100644 --- a/app/jobs/fitbit_edit.rb +++ b/app/workers/fitbit_edit.rb @@ -1,4 +1,4 @@ -require 'resque' + class FitbitEdit include Sidekiq::Worker diff --git a/app/jobs/fitbit_endsubscription.rb b/app/workers/fitbit_endsubscription.rb similarity index 97% rename from app/jobs/fitbit_endsubscription.rb rename to app/workers/fitbit_endsubscription.rb index d942effe..a75fb0ee 100644 --- a/app/jobs/fitbit_endsubscription.rb +++ b/app/workers/fitbit_endsubscription.rb @@ -1,4 +1,4 @@ -require 'resque' + class FitbitEndSubscription include Sidekiq::Worker diff --git a/app/jobs/fitbit_init.rb b/app/workers/fitbit_init.rb similarity index 97% rename from app/jobs/fitbit_init.rb rename to app/workers/fitbit_init.rb index 574dc25e..abacae0a 100644 --- a/app/jobs/fitbit_init.rb +++ b/app/workers/fitbit_init.rb @@ -1,4 +1,4 @@ -require 'resque' + class FitbitInit include Sidekiq::Worker diff --git a/app/jobs/fitbit_notification.rb b/app/workers/fitbit_notification.rb similarity index 99% rename from app/jobs/fitbit_notification.rb rename to app/workers/fitbit_notification.rb index eec9bb5e..4ff89ef1 100644 --- a/app/jobs/fitbit_notification.rb +++ b/app/workers/fitbit_notification.rb @@ -1,4 +1,4 @@ -require 'resque' + class FitbitNotification include Sidekiq::Worker diff --git a/app/jobs/fixphenotypes.rb b/app/workers/fixphenotypes.rb similarity index 87% rename from app/jobs/fixphenotypes.rb rename to app/workers/fixphenotypes.rb index 8fbea0c7..eef85c41 100644 --- a/app/jobs/fixphenotypes.rb +++ b/app/workers/fixphenotypes.rb @@ -1,4 +1,4 @@ -require 'resque' + class Fixphenotypes include Sidekiq::Worker @@ -14,8 +14,8 @@ def perform() # delete! log "Deleting phenotype '" + p.characteristic Phenotype.destroy(p) - Resque.enqueue(Recommendvariations) - Resque.enqueue(Recommendphenotypes) + Sidekiq::Client.enqueue(Recommendvariations) + Sidekiq::Client.enqueue(Recommendphenotypes) next end @@ -30,7 +30,7 @@ def perform() end - def self.log msg + def log msg Rails.logger.info "#{DateTime.now}: #{msg}" end end diff --git a/app/jobs/frequency.rb b/app/workers/frequency.rb similarity index 98% rename from app/jobs/frequency.rb rename to app/workers/frequency.rb index 602d7027..52b8de7d 100644 --- a/app/jobs/frequency.rb +++ b/app/workers/frequency.rb @@ -1,4 +1,4 @@ -require 'resque' + class Frequency include Sidekiq::Worker diff --git a/app/jobs/genomegov.rb b/app/workers/genomegov.rb similarity index 97% rename from app/jobs/genomegov.rb rename to app/workers/genomegov.rb index b8323d2b..819a9c31 100644 --- a/app/jobs/genomegov.rb +++ b/app/workers/genomegov.rb @@ -1,10 +1,8 @@ -require 'resque' require 'open-uri' require 'iconv' class GenomeGov include Sidekiq::Worker - include Resque::Plugins::UniqueJob sidekiq_options :queue => :genomegov def perform() diff --git a/app/jobs/mailnewphenotype.rb b/app/workers/mailnewphenotype.rb similarity index 78% rename from app/jobs/mailnewphenotype.rb rename to app/workers/mailnewphenotype.rb index 7bfae591..26c5f1d5 100644 --- a/app/jobs/mailnewphenotype.rb +++ b/app/workers/mailnewphenotype.rb @@ -1,8 +1,8 @@ -require 'resque' + class Mailnewphenotype include Sidekiq::Worker - sidekiq_options :queue => :mailnewgenotype + sidekiq_options :queue => :mailnewgenotype, :retry => false # avoid spam on breakage def perform(phenotype_id,user_id) @phenotype = Phenotype.find_by_id(phenotype_id) diff --git a/app/jobs/mendeley_search.rb b/app/workers/mendeley_search.rb similarity index 96% rename from app/jobs/mendeley_search.rb rename to app/workers/mendeley_search.rb index 116213d2..d1107b13 100644 --- a/app/jobs/mendeley_search.rb +++ b/app/workers/mendeley_search.rb @@ -1,11 +1,9 @@ -require "resque" require "rubygems" require "net/http" require "json" class MendeleySearch include Sidekiq::Worker - include Resque::Plugins::UniqueJob sidekiq_options :queue => :mendeley def perform(snp_id) @@ -83,7 +81,7 @@ def perform(snp_id) ) end end - Resque.enqueue(MendeleyDetails, @mendeley_paper.id) + Sidekiq::Client.enqueue(MendeleyDetails, @mendeley_paper.id) end else puts "mendeley: No papers found" diff --git a/app/jobs/mendeleydetails.rb b/app/workers/mendeleydetails.rb similarity index 98% rename from app/jobs/mendeleydetails.rb rename to app/workers/mendeleydetails.rb index e9076513..4676dc1f 100644 --- a/app/jobs/mendeleydetails.rb +++ b/app/workers/mendeleydetails.rb @@ -1,4 +1,3 @@ -require "resque" require "rubygems" require "net/http" require "json" diff --git a/app/jobs/parsing.rb b/app/workers/parsing.rb similarity index 99% rename from app/jobs/parsing.rb rename to app/workers/parsing.rb index 828b2070..4c2b59ac 100644 --- a/app/jobs/parsing.rb +++ b/app/workers/parsing.rb @@ -1,4 +1,4 @@ -require 'resque' + class Parsing include Sidekiq::Worker @@ -132,7 +132,7 @@ def perform(genotype_id, temp_file) end end - def self.log msg + def log msg Rails.logger.info "#{DateTime.now}: #{msg}" end end diff --git a/app/jobs/pgpannotation.rb b/app/workers/pgpannotation.rb similarity index 97% rename from app/jobs/pgpannotation.rb rename to app/workers/pgpannotation.rb index 75c30393..b4513c22 100644 --- a/app/jobs/pgpannotation.rb +++ b/app/workers/pgpannotation.rb @@ -1,9 +1,8 @@ -require 'resque' + require 'open-uri' class PgpAnnotationjob include Sidekiq::Worker - include Resque::Plugins::UniqueJob sidekiq_options :queue => :pgp def perform() diff --git a/app/jobs/plos.rb b/app/workers/plos.rb similarity index 95% rename from app/jobs/plos.rb rename to app/workers/plos.rb index 6c18f16e..52e54e9c 100644 --- a/app/jobs/plos.rb +++ b/app/workers/plos.rb @@ -1,10 +1,9 @@ -require 'resque' + require 'net/http' require 'rexml/document' class Plos include Sidekiq::Worker - include Resque::Plugins::UniqueJob sidekiq_options :queue => :plos def perform(snp_id) @@ -44,7 +43,7 @@ def perform(snp_id) print "-> paper is old" @plos_paper = PlosPaper.find_by_doi(doi) end - Resque.enqueue(PlosDetails,@plos_paper) + Sidekiq::Client.enqueue(PlosDetails,@plos_paper) end else print "plos: none found\n" diff --git a/app/jobs/plosdetails.rb b/app/workers/plosdetails.rb similarity index 98% rename from app/jobs/plosdetails.rb rename to app/workers/plosdetails.rb index dfce92ca..12f4ca2c 100644 --- a/app/jobs/plosdetails.rb +++ b/app/workers/plosdetails.rb @@ -1,4 +1,3 @@ -require "resque" require "rubygems" require "net/http" require "json" diff --git a/app/jobs/preparsing.rb b/app/workers/preparsing.rb similarity index 98% rename from app/jobs/preparsing.rb rename to app/workers/preparsing.rb index f54cf6dd..b84c434f 100644 --- a/app/jobs/preparsing.rb +++ b/app/workers/preparsing.rb @@ -1,4 +1,4 @@ -require 'resque' + require 'digest' class Preparsing @@ -149,11 +149,11 @@ def perform(genotype_id) temp_files = Dir.glob("tmp/#{@genotype.id}_tmpfile*") temp_files.each do |single_temp_file| - Resque.enqueue(Parsing, @genotype.id, single_temp_file) + Sidekiq::Client.enqueue(Parsing, @genotype.id, single_temp_file) end end end - def self.log msg + def log msg Rails.logger.info "#{DateTime.now}: #{msg}" end end diff --git a/app/jobs/recommendphenotypes.rb b/app/workers/recommendphenotypes.rb similarity index 98% rename from app/jobs/recommendphenotypes.rb rename to app/workers/recommendphenotypes.rb index a87b6722..d09b0c1d 100644 --- a/app/jobs/recommendphenotypes.rb +++ b/app/workers/recommendphenotypes.rb @@ -1,4 +1,4 @@ -require 'resque' + require "recommendify" Recommendify.redis = Redis.new diff --git a/app/jobs/recommendvariations.rb b/app/workers/recommendvariations.rb similarity index 98% rename from app/jobs/recommendvariations.rb rename to app/workers/recommendvariations.rb index 1777974d..a1196d53 100644 --- a/app/jobs/recommendvariations.rb +++ b/app/workers/recommendvariations.rb @@ -1,4 +1,4 @@ -require 'resque' + require "recommendify" Recommendify.redis = Redis.new diff --git a/app/jobs/snpedia.rb b/app/workers/snpedia.rb similarity index 98% rename from app/jobs/snpedia.rb rename to app/workers/snpedia.rb index e1c8d9d5..bdb04c1a 100644 --- a/app/jobs/snpedia.rb +++ b/app/workers/snpedia.rb @@ -1,11 +1,9 @@ -require 'resque' require 'net/http' require 'rexml/document' require 'media_wiki' class Snpedia include Sidekiq::Worker - include Resque::Plugins::UniqueJob sidekiq_options :queue => :snpedia def perform(snp_id) diff --git a/app/jobs/zipfulldata.rb b/app/workers/zipfulldata.rb similarity index 99% rename from app/jobs/zipfulldata.rb rename to app/workers/zipfulldata.rb index 9da51714..508cfdfa 100644 --- a/app/jobs/zipfulldata.rb +++ b/app/workers/zipfulldata.rb @@ -1,4 +1,4 @@ -require 'resque' + require 'csv' class Zipfulldata @@ -261,7 +261,7 @@ def log(msg) self.class.log(msg) end - def self.log(msg) + def log(msg) Rails.logger.info "#{DateTime.now}: #{msg}" end end diff --git a/app/jobs/zipgenotypingfiles.rb b/app/workers/zipgenotypingfiles.rb similarity index 98% rename from app/jobs/zipgenotypingfiles.rb rename to app/workers/zipgenotypingfiles.rb index 690be995..8a9b534b 100644 --- a/app/jobs/zipgenotypingfiles.rb +++ b/app/workers/zipgenotypingfiles.rb @@ -1,4 +1,4 @@ -require 'resque' + class Zipgenotypingfiles include Sidekiq::Worker diff --git a/config.ru b/config.ru index 2046ac16..128017d2 100644 --- a/config.ru +++ b/config.ru @@ -1,6 +1,4 @@ require ::File.expand_path('../config/environment', __FILE__) -require 'resque/server' run Rack::URLMap.new \ - "/" => Snpr::Application, - "/resque" => Resque::Server.new + "/" => Snpr::Application diff --git a/config/initializers/resque.rb b/config/initializers/resque.rb deleted file mode 100644 index 073306eb..00000000 --- a/config/initializers/resque.rb +++ /dev/null @@ -1,4 +0,0 @@ -Dir["#{::Rails.root.to_s}/app/jobs/*.rb"].each { |file| require file } - -# FIX for: "PG::Error: ERROR: prepared statement "a1" already exists" -Resque.before_fork = Proc.new { ActiveRecord::Base.establish_connection } diff --git a/config/routes.rb b/config/routes.rb index 0c2f06d2..2714abf1 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,3 +1,4 @@ +require 'sidekiq/web' Snpr::Application.routes.draw do resources :static resources :phenotypes do @@ -73,6 +74,9 @@ match '/user_picture_phenotypes/:id/delete', :to => 'user_picture_phenotypes#delete' root :to => 'index#index' # change this, maybe + + mount Sidekiq::Web, at: '/sidekiq' + # The priority is based upon order of creation: # first created -> highest priority. diff --git a/lib/tasks/resque.rake b/lib/tasks/resque.rake deleted file mode 100644 index b8dde56c..00000000 --- a/lib/tasks/resque.rake +++ /dev/null @@ -1,6 +0,0 @@ -require 'resque' - -task "resque:setup" => :environment do - # FIX FOR "PG::Error: ERROR: prepared statement "a1" already exists" - Resque.before_fork = Proc.new { ActiveRecord::Base.establish_connection } -end diff --git a/serverscript b/serverscript index 5d1c8249..f9254d6b 100755 --- a/serverscript +++ b/serverscript @@ -3,4 +3,4 @@ screen -d -m -S "Server" bundle exec rails s screen -d -m -S "Solr" bundle exec rake sunspot:solr:run screen -d -m -S "Redis" redis-server -screen -d -m -S "Resque-worker" bundle exec rake environment resque:work QUEUE=* +screen -d -m -S "Sidekiq" bundle exec sidekiq -q preparse,2 -q parse,2 -q deletegenotype -q fitbitdump -q fitbitedit -q fitbitendsubscription -q fitbitinit -q fitbitnotification -q fixphenotypes -q frequency -q genomegov -q mailnewgenotype -q mendeley_details -q mendeley -q pgp -q plos_details -q plos -q zipfulldata -q snpedia -q zipgenotyping diff --git a/turnoffserver b/turnoffserver index 6817e81f..f00fa349 100755 --- a/turnoffserver +++ b/turnoffserver @@ -2,4 +2,4 @@ screen -X -S Server quit screen -X -S Solr quit killall -9 redis-server -ps aux | grep [r]esque-pool | kill -9 `awk '{print $2}'` # ugly :D +screen -X -S Sidekiq quit