From aa4c73dbda26a05710463865a74e3211b2ecb472 Mon Sep 17 00:00:00 2001 From: drsnuggles Date: Sat, 9 Mar 2013 12:53:21 +1000 Subject: [PATCH] Merged fitbit-queues into one. Slight refactor to find out why mendeley_details breaks (still broken). Currently can't install Ruby 2.0 as debugger-gem breaks, not supported yet. --- app/workers/fitbit_dump.rb | 2 +- app/workers/fitbit_edit.rb | 2 +- app/workers/fitbit_endsubscription.rb | 2 +- app/workers/fitbit_init.rb | 2 +- app/workers/fitbit_notification.rb | 2 +- app/workers/mendeleydetails.rb | 83 +++++++++++++++------------ app/workers/plos.rb | 4 +- serverscript | 2 +- 8 files changed, 53 insertions(+), 46 deletions(-) diff --git a/app/workers/fitbit_dump.rb b/app/workers/fitbit_dump.rb index 8f4182ef..b69ec4a7 100644 --- a/app/workers/fitbit_dump.rb +++ b/app/workers/fitbit_dump.rb @@ -2,7 +2,7 @@ class FitbitDump include Sidekiq::Worker - sidekiq_options :queue => :fitbitdump + sidekiq_options :queue => :fitbit def perform(target_address,fitbit_profile_id) fp = FitbitProfile.find_by_id(fitbit_profile_id) diff --git a/app/workers/fitbit_edit.rb b/app/workers/fitbit_edit.rb index 54af0424..7353ee73 100644 --- a/app/workers/fitbit_edit.rb +++ b/app/workers/fitbit_edit.rb @@ -2,7 +2,7 @@ class FitbitEdit include Sidekiq::Worker - sidekiq_options :queue => :fitbitedit + sidekiq_options :queue => :fitbit def perform(fitbit_profile_id) @fitbit_profile = FitbitProfile.find_by_id(fitbit_profile_id) diff --git a/app/workers/fitbit_endsubscription.rb b/app/workers/fitbit_endsubscription.rb index a75fb0ee..e4e19f5d 100644 --- a/app/workers/fitbit_endsubscription.rb +++ b/app/workers/fitbit_endsubscription.rb @@ -2,7 +2,7 @@ class FitbitEndSubscription include Sidekiq::Worker - sidekiq_options :queue => :fitbitendsubscription + sidekiq_options :queue => :fitbit def perform(fitbit_profile_id) @fitbit_profile = FitbitProfile.find_by_id(fitbit_profile_id) diff --git a/app/workers/fitbit_init.rb b/app/workers/fitbit_init.rb index abacae0a..05aa2f4e 100644 --- a/app/workers/fitbit_init.rb +++ b/app/workers/fitbit_init.rb @@ -2,7 +2,7 @@ class FitbitInit include Sidekiq::Worker - sidekiq_options :queue => :fitbitinit + sidekiq_options :queue => :fitbit def perform(fitbit_profile_id) @client = Fitgem::Client.new(:consumer_key => APP_CONFIG[:fitbit_consumer_key], :consumer_secret => APP_CONFIG[:fitbit_consumer_secret]) diff --git a/app/workers/fitbit_notification.rb b/app/workers/fitbit_notification.rb index 4ff89ef1..c85daef2 100644 --- a/app/workers/fitbit_notification.rb +++ b/app/workers/fitbit_notification.rb @@ -2,7 +2,7 @@ class FitbitNotification include Sidekiq::Worker - sidekiq_options :queue => :fitbitnotification + sidekiq_options :queue => :fitbit def perform(notification) puts notification diff --git a/app/workers/mendeleydetails.rb b/app/workers/mendeleydetails.rb index 4676dc1f..c1a86fde 100644 --- a/app/workers/mendeleydetails.rb +++ b/app/workers/mendeleydetails.rb @@ -3,42 +3,49 @@ require "json" class MendeleyDetails - include Sidekiq::Worker - sidekiq_options :queue => :mendeley_details - - def perform(mendeley_paper_id) - @mendeley_paper = MendeleyPaper.find_by_id(mendeley_paper_id.to_i) - - key_handle = File.open(::Rails.root.to_s+"/key_mendeley.txt") - api_key = key_handle.readline.rstrip - - detail_url = "http://api.mendeley.com/oapi/documents/details/" + @mendeley_paper.uuid + "/?consumer_key="+api_key - begin - detail_resp = Net::HTTP.get_response(URI.parse(detail_url)) - rescue - retry - end - - detail_data = detail_resp.body - detail_result = JSON.parse(detail_data) - - if detail_result["oa_journal"] != false - @mendeley_paper.open_access = true - else - @mendeley_paper.open_access = false - end - - print "mendeley details: updated oa- and reader-status\n" - if detail_result["stats"] - @mendeley_paper.reader = detail_result["stats"]["readers"] - elsif detail_result["reader"] - @mendeley_paper.reader = detail_result["reader"] - else - @mendeley_paper.reader = "Unknown" - end - - @mendeley_paper.save - print "-> sleep for 5 secs\n" - sleep(5) - end + include Sidekiq::Worker + sidekiq_options :queue => :mendeley_details + + def perform(mendeley_paper_id) + # Logging stuff + Rails.logger.level = 0 + Rails.logger = Logger.new("#{Rails.root}/log/mendeleydetails_#{Rails.env}.log") + @mendeley_paper = MendeleyPaper.find_by_id(mendeley_paper_id.to_i) + + key_handle = File.open(::Rails.root.to_s+"/key_mendeley.txt") + api_key = key_handle.readline.rstrip + + detail_url = "http://api.mendeley.com/oapi/documents/details/" + @mendeley_paper.uuid + "/?consumer_key="+api_key + begin + detail_resp = Net::HTTP.get_response(URI.parse(detail_url)) + rescue + retry + end + + detail_data = detail_resp.body + detail_result = JSON.parse(detail_data) + + if detail_result["oa_journal"] != false + @mendeley_paper.open_access = true + else + @mendeley_paper.open_access = false + end + + log "mendeley details: updated oa- and reader-status\n" + if detail_result["stats"] + @mendeley_paper.reader = detail_result["stats"]["readers"] + elsif detail_result["reader"] + @mendeley_paper.reader = detail_result["reader"] + else + @mendeley_paper.reader = "Unknown" + end + + @mendeley_paper.save + log "-> sleep for 5 secs\n" + sleep(5) + end + + def log msg + Rails.logger.info "#{DateTime.now}: #{msg}" + end end diff --git a/app/workers/plos.rb b/app/workers/plos.rb index 0a335df0..29267a6e 100644 --- a/app/workers/plos.rb +++ b/app/workers/plos.rb @@ -5,9 +5,9 @@ class Plos include Sidekiq::Worker sidekiq_options :queue => :plos - + def is_illegal_snp(name) - # we don't need mitochondrial or VG-SNPs as these just result in noise + # we don't need mitochondrial or VG-SNPs as these just result in noise # from the PLOS API forbidden_names = ["mt-", "vg"] if forbidden_names.any? { |part| name[part] } diff --git a/serverscript b/serverscript index f9254d6b..d6dcb381 100755 --- a/serverscript +++ b/serverscript @@ -3,4 +3,4 @@ screen -d -m -S "Server" bundle exec rails s screen -d -m -S "Solr" bundle exec rake sunspot:solr:run screen -d -m -S "Redis" redis-server -screen -d -m -S "Sidekiq" bundle exec sidekiq -q preparse,2 -q parse,2 -q deletegenotype -q fitbitdump -q fitbitedit -q fitbitendsubscription -q fitbitinit -q fitbitnotification -q fixphenotypes -q frequency -q genomegov -q mailnewgenotype -q mendeley_details -q mendeley -q pgp -q plos_details -q plos -q zipfulldata -q snpedia -q zipgenotyping +screen -d -m -S "Sidekiq" bundle exec sidekiq -q preparse,2 -q parse,2 -q deletegenotype -q fitbit -q fixphenotypes -q frequency -q genomegov -q mailnewgenotype -q mendeley_details -q mendeley -q pgp -q plos_details -q plos -q zipfulldata -q snpedia -q zipgenotyping