From 12dee111b4cda4e0f21117d08886648c4e51e180 Mon Sep 17 00:00:00 2001
From: spaquet <176050+spaquet@users.noreply.github.com>
Date: Sat, 22 Jul 2023 12:58:55 -0700
Subject: [PATCH] Crontasks (#95)
* Adding support Faraday-retry
* Annontate is not loaded by default
* Upgrading to report close_events task status
* Improved db_cleanup task
* Improved clean_active_sessions task
* Adding Whenever gem
* Setting required:false
* Initial schedule file used by whenever
* First scheduled jobs
* Code cleaning and misc fixes to better deploy
* Testing a new Capistrano config
* Making sure whenever write jobs that have proper access to the bundler
* Experimenting with Capistrano and Sidekiq
* Better Poll list display on small screens
* Fixing a typo
---
Capfile | 13 ++++----
Gemfile | 21 ++++++++----
Gemfile.lock | 12 +++++++
Puma-Service.md | 6 ++--
app/views/polls/_poll.html.erb | 4 +--
app/views/polls/index.html.erb | 6 ++--
config/deploy.rb | 24 ++++++--------
config/deploy/production.rb | 2 +-
config/schedule.rb | 35 ++++++++++++++++++++
lib/capistrano/tasks/sidekiq.rake | 20 ++++++++++++
lib/tasks/clean_active_sessions.rake | 27 +++++++++++++++-
lib/tasks/close_events.rake | 48 ++++++++++++++++++++++++----
lib/tasks/db_cleanup.rake | 25 ++++++++++++++-
13 files changed, 199 insertions(+), 44 deletions(-)
create mode 100644 config/schedule.rb
create mode 100644 lib/capistrano/tasks/sidekiq.rake
diff --git a/Capfile b/Capfile
index c0d12eb4..5d9077b6 100644
--- a/Capfile
+++ b/Capfile
@@ -19,19 +19,20 @@ install_plugin Capistrano::SCM::Git
# https://github.com/capistrano/rails
# https://github.com/capistrano/passenger
#
-# require "capistrano/rvm"
require "capistrano/rails"
require "capistrano/rbenv"
-# require "capistrano/chruby"
require "capistrano/bundler"
require "capistrano/rails/assets"
require "capistrano/rails/migrations"
-# require "capistrano/passenger"
+require "whenever/capistrano"
-require "capistrano/sidekiq"
-install_plugin Capistrano::Sidekiq
-install_plugin Capistrano::Sidekiq::Systemd
+# require "capistrano/sidekiq"
+# install_plugin Capistrano::Sidekiq
+# install_plugin Capistrano::Sidekiq::Systemd
+require 'capistrano/puma'
+install_plugin Capistrano::Puma # Default puma tasks
+# install_plugin Capistrano::Puma::Systemd
set :rbenv_type, :user
set :rbenv_ruby, "3.2.2"
diff --git a/Gemfile b/Gemfile
index 611bb025..ccb59864 100644
--- a/Gemfile
+++ b/Gemfile
@@ -153,6 +153,12 @@ gem 'pg_search', '~> 2.3.6'
# Stripe (payment, subscription processing) [https://github.com/stripe/stripe-ruby]
gem 'stripe', '~> 8.6.0'
+# To enable retry in Faraday v2.0+
+gem 'faraday-retry', '~> 2.2.0'
+
+# Whenever gem to mamage crontab & tasks [https://github.com/javan/whenever]
+gem 'whenever', '~> 1.0.0', require: false
+
group :development, :test do
# See https://guides.rubyonrails.org/debugging_rails_applications.html#debugging-with-the-debug-gem
gem 'debug', platforms: %i[mri mingw x64_mingw]
@@ -180,8 +186,8 @@ group :development do
# Vulnerability scanner
gem 'brakeman', '~> 6.0.1'
- # Add Model annotations
- gem 'annotate', '~>3.2.0'
+ # Add Model annotations [https://github.com/ctran/annotate_models]
+ gem 'annotate', '~>3.2.0', require: false
# Add Bullet to monitor and help fix N+1 DB queries
gem 'bullet'
@@ -191,11 +197,12 @@ group :development do
# facilitate the deployment to Digital #
# Ocean.
# [https://gorails.com/deploy/ubuntu/22.04]
- gem 'capistrano', '~> 3.17', require: false
- gem 'capistrano-rails', '~> 1.6.3', require: false
- gem 'capistrano-rbenv', '~> 2.2.0', require: false
- gem 'capistrano-bundler', require: false
- gem 'capistrano-sidekiq', '~> 3.0.0.alpha.2', require: false
+ gem 'capistrano', '~> 3.17', require: false
+ gem 'capistrano-rails', '~> 1.6.3', require: false
+ gem 'capistrano-rbenv', '~> 2.2.0', require: false
+ gem 'capistrano-bundler', require: false
+ gem 'capistrano-sidekiq', '~> 3.0.0.alpha.2', require: false
+ gem 'capistrano3-puma', '~> 6.0.0.beta.1', require: false
end
group :test do
diff --git a/Gemfile.lock b/Gemfile.lock
index 1efaf575..0dca93a2 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -140,6 +140,10 @@ GEM
capistrano (>= 3.9.0)
capistrano-bundler
sidekiq (>= 6.0.6)
+ capistrano3-puma (6.0.0.beta.1)
+ capistrano (~> 3.7)
+ capistrano-bundler
+ puma (>= 5.1, < 7.0)
capybara (3.39.2)
addressable
matrix
@@ -158,6 +162,7 @@ GEM
actionpack (>= 3.1)
caxlsx (>= 3.0)
chartkick (5.0.2)
+ chronic (0.10.2)
concurrent-ruby (1.2.2)
connection_pool (2.4.1)
countries (5.5.0)
@@ -182,6 +187,8 @@ GEM
faraday-multipart (1.0.4)
multipart-post (~> 2)
faraday-net_http (3.0.2)
+ faraday-retry (2.2.0)
+ faraday (~> 2.0)
ffi (1.15.5)
ffi-compiler (1.0.1)
ffi (>= 1.0.0)
@@ -457,6 +464,8 @@ GEM
websocket-driver (0.7.5)
websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.5)
+ whenever (1.0.0)
+ chronic (>= 0.6.3)
xpath (3.2.0)
nokogiri (~> 1.8)
xsv (1.2.1)
@@ -482,6 +491,7 @@ DEPENDENCIES
capistrano-rails (~> 1.6.3)
capistrano-rbenv (~> 2.2.0)
capistrano-sidekiq (~> 3.0.0.alpha.2)
+ capistrano3-puma (~> 6.0.0.beta.1)
capybara
caxlsx
caxlsx_rails
@@ -491,6 +501,7 @@ DEPENDENCIES
debug
down (~> 5.0)
faker!
+ faraday-retry (~> 2.2.0)
groupdate
hiredis (~> 0.6.3)
honeybadger (~> 5.0)
@@ -532,6 +543,7 @@ DEPENDENCIES
view_component
web-console
webdrivers
+ whenever (~> 1.0.0)
xsv
RUBY VERSION
diff --git a/Puma-Service.md b/Puma-Service.md
index f78299b0..c35f3399 100644
--- a/Puma-Service.md
+++ b/Puma-Service.md
@@ -23,9 +23,9 @@ sudo systemctl daemon-reload
sudo systemctl enable puma.service
sudo systemctl start puma.service
-sudo systemctl start puma-thepew51
-sudo systemctl stop puma-thepew51
-sudo systemctl status puma-thepew51
+sudo systemctl start puma
+sudo systemctl stop puma
+sudo systemctl status puma
ps -eo pid,comm,euser,supgrp | grep nginx
When using SSL add the following to ExecStart to enable Puma over HTTPS
diff --git a/app/views/polls/_poll.html.erb b/app/views/polls/_poll.html.erb
index 0e9e888f..1c89e221 100644
--- a/app/views/polls/_poll.html.erb
+++ b/app/views/polls/_poll.html.erb
@@ -2,10 +2,10 @@
|
<%= poll.title %>
|
-
+ |
<%= poll.user.profile.nickname %>
|
-
+ |
<%= poll.created_at.strftime('%b %d, %y') %>
|
diff --git a/app/views/polls/index.html.erb b/app/views/polls/index.html.erb
index 02bbaf3c..0ddfb0ab 100644
--- a/app/views/polls/index.html.erb
+++ b/app/views/polls/index.html.erb
@@ -15,13 +15,13 @@
|
Poll name
|
-
+ |
Created by
|
-
+ |
Created on
|
-
+ |
Participants
|
diff --git a/config/deploy.rb b/config/deploy.rb
index e38f4730..4a314842 100644
--- a/config/deploy.rb
+++ b/config/deploy.rb
@@ -11,17 +11,12 @@
# changing the branch... as master no longer exists on GitHub
set :branch, proc { `git rev-parse --abbrev-ref HEAD`.chomp }
+set :rails_env, "production"
+set :linked_dirs, fetch(:linked_dirs, []).push('log', 'tmp/pids', 'tmp/cache', 'tmp/sockets', 'vendor/bundle', 'public/system')
+
# Default deploy_to directory is /var/www/my_app_name
-# set :deploy_to, "/var/www/my_app_name"
set :deploy_to, "/home/deploy/#{fetch :application}"
-# Default value for :format is :airbrussh.
-# set :format, :airbrussh
-
-# You can configure the Airbrussh format using :format_options.
-# These are the defaults.
-# set :format_options, command_output: true, log_file: "log/capistrano.log", color: :auto, truncate: :auto
-
# Default value for :pty is false
# set :pty, true
@@ -42,15 +37,16 @@
# set :keep_releases, 5
set :keep_releases, 2
+set :use_sudo, true
+
# Uncomment the following to require manually verifying the host key before first deploy.
# set :ssh_options, verify_host_key: :secure
-# Puma configuration
-# set :use_sudo, true
-# set :linked_files, %w{config/master.key config/database.yml}
-set :rails_env, "production"
-set :linked_dirs, fetch(:linked_dirs, []).push('log', 'tmp/pids', 'tmp/cache', 'tmp/sockets', 'vendor/bundle', 'public/system')
-# set :linked_files, %w{config/database.yml config/master.key}
+# Puma
+# Commented out as we control puma manually for the moment.
+# set :puma_service_unit_name, "puma.service"
+# set :puma_user, fetch(:user)
+# set :puma_role, :web
# Sidekiq
# set :sidekiq_service_unit_name, "sidekiq"
diff --git a/config/deploy/production.rb b/config/deploy/production.rb
index 871fecf0..fe2181a4 100644
--- a/config/deploy/production.rb
+++ b/config/deploy/production.rb
@@ -7,7 +7,7 @@
# server "example.com", user: "deploy", roles: %w{app web}, other_property: :other_value
# server "db.example.com", user: "deploy", roles: %w{db}
-server 'demo.thepew.io', user: 'deploy', roles: %w{app web}
+server 'demo.thepew.io', user: 'deploy', roles: %w{app db web}
# role-based syntax
# ==================
diff --git a/config/schedule.rb b/config/schedule.rb
new file mode 100644
index 00000000..9e272c37
--- /dev/null
+++ b/config/schedule.rb
@@ -0,0 +1,35 @@
+# Use this file to easily define all of your cron jobs.
+#
+# It's helpful, but not entirely necessary to understand cron before proceeding.
+# http://en.wikipedia.org/wiki/Cron
+
+# Example:
+#
+# set :output, "/path/to/my/cron_log.log"
+#
+# every 2.hours do
+# command "/usr/bin/some_great_command"
+# runner "MyModel.some_method"
+# rake "some:great:rake:task"
+# end
+#
+# every 4.days do
+# runner "AnotherModel.prune_old_records"
+# end
+
+# Learn more: http://github.com/javan/whenever
+
+# Fix an issue when using rbenv (at least)
+env :PATH, ENV['PATH']
+
+set :output, "log/cron.log"
+
+# Clear Active Sessions from sessions that are tooooo long
+every 6.hours do
+ rake 'clean_active_sessions:clean[false]'
+end
+
+# Close Events
+every 1.day do
+ rake 'close_events:close_events[false]'
+end
\ No newline at end of file
diff --git a/lib/capistrano/tasks/sidekiq.rake b/lib/capistrano/tasks/sidekiq.rake
new file mode 100644
index 00000000..b669cee0
--- /dev/null
+++ b/lib/capistrano/tasks/sidekiq.rake
@@ -0,0 +1,20 @@
+namespace :sidekiq do
+ after 'deploy:starting', 'sidekiq:stop'
+ after 'deploy:finished', 'sidekiq:start'
+
+ task :stop do
+ on roles(:app) do
+ within current_path do
+ # execute(:sudo, 'systemctl kill -s TSTP sidekiq')
+ execute(:sudo, 'systemctl stop sidekiq')
+ end
+ end
+ end
+
+ task :start do
+ on roles(:app) do |host|
+ execute(:sudo, 'systemctl start sidekiq')
+ info "Host #{host} (#{host.roles.to_a.join(', ')}):\t#{capture(:uptime)}"
+ end
+ end
+end
\ No newline at end of file
diff --git a/lib/tasks/clean_active_sessions.rake b/lib/tasks/clean_active_sessions.rake
index 623d29a5..eafd7599 100644
--- a/lib/tasks/clean_active_sessions.rake
+++ b/lib/tasks/clean_active_sessions.rake
@@ -7,15 +7,40 @@ namespace :clean_active_sessions do
# - rake "clean_active_sessions:clean[false]"
desc 'Remove session that are older than 20 days'
task :clean, [:dry_run] => :environment do |_t, args|
+ start_at= Time.now.utc
+ failed_list = []
+ session_count = 0
dry_run = true unless args[:dry_run] == 'false'
+ Rails.logger.error "[clean_active_sessions:clean] Starting at #{start_at}."
puts("[#{Time.now.utc}] Running clean session :: INI#{' (dry_run activated)' if dry_run}")
ActiveSession.where('active_sessions.created_at <= ?', 2.days.ago).find_each do |session|
- puts("Deleting Session: #{session.id}#{' (dry_run activated)' if dry_run}")
+ Rails.logger.error "[clean_active_sessions:clean] Deleting Session: #{session.id}#{' (dry_run activated)' if dry_run}."
+
session.destroy! unless dry_run
+
+ session_count += 1
+
+ rescue StandardError => e
+ failed_list.push({ session_id: session.id, reason: session.inspect })
+
+ Rails.logger.error "[clean_active_sessions:clean] session_id: #{session.id} failed to update user name."
+ Rails.logger.error "[clean_active_sessions:clean] session_id: #{session.id} failed reason: #{e.inspect}"
end
+ if failed_list.count > 0
+ Rails.logger.info "[clean_active_sessions:clean] Failed list: #{failed_list}"
+ p "[#{Time.now.utc}] [clean_active_sessions:clean] Failed list: #{failed_list}"
+ end
+
+ # End the task
+ # Compute task duration
+ end_at= Time.now.utc
+ duration = ((end_at - start_at) / 60.seconds).to_i
+
+ # Display closing messages and report to Rails logger for centralized logs
+ Rails.logger.error "[clean_active_sessions:clean] Ending at #{end_at}. Sessions terminated: #{session_count} in #{duration}"
puts("[#{Time.now.utc}] Running clean session :: END#{' (dry_run activated)' if dry_run}")
end
end
diff --git a/lib/tasks/close_events.rake b/lib/tasks/close_events.rake
index deec1a51..c4d53878 100644
--- a/lib/tasks/close_events.rake
+++ b/lib/tasks/close_events.rake
@@ -8,8 +8,12 @@ namespace :close_events do
# - rake "close_events:close_events[false]"
desc 'Close events'
task :close_events, [:dry_run] => :environment do |_t, args|
+ start_at= Time.now.utc
dry_run = true unless args[:dry_run] == 'false'
+ failed_list = []
+ event_closed_count = 0
+ Rails.logger.error "[close_events] Starting at #{start_at}."
puts("[#{Time.now.utc}] Running close_events :: INI#{' (dry_run activated)' if dry_run}")
# List all the opened events and can be closed, meaning the always_on flag is set to false
@@ -17,14 +21,46 @@ namespace :close_events do
# Current rule is to close an event 24h after its end_date.
@events = Event.where("always_on = false AND end_date < ?", 1.days.ago).opened
- @events.each do |event|
- # Closing the event
- event.closed!
+ puts("[#{Time.now.utc}] Events to be closed: #{@events.count}")
+
+ # If there is no event to be closed we simply exit the task.
+ if @events.count > 0 then
+ @events.each do |event|
+ # Closing the event
+ event.closed!
+
+ # Removing the PIN
+ event.pin = nil
+
+ # Updating the event
+ event.update!
+
+ # Update the counter
+ event_closed_count += 1
+
+ # Rescue used to log errors and report to Rails looger
+ rescue StandardError => e
+ failed_list.push({ event_id: event.id, reason: event.inspect })
+
+ Rails.logger.error "[close_events] event_id: #{event.id} failed to update user name."
+ Rails.logger.error "[close_events] event_id: #{event.id} failed reason: #{e.inspect}"
+ end
- # Removing the PIN
- event.pin = nil
+ if failed_list.count > 0
+ Rails.logger.info "[close_events] Failed list: #{failed_list}"
+ p "[#{Time.now.utc}] [close_events] Failed list: #{failed_list}"
+ end
+
+ puts("[#{Time.now.utc}] #{event_closed_count} events have been closed")
end
-
+
+ # End the task
+ # Compute task duration
+ end_at= Time.now.utc
+ duration = ((end_at - start_at) / 60.seconds).to_i
+ # Display closing messages and report to Rails logger for centralized logs
+ Rails.logger.error "[close_events] Ending at #{end_at}. Closed #{event_closed_count} event(s) in #{duration}"
+ puts("[#{Time.now.utc}] Running close_events :: duration: #{duration} :: END#{' (dry_run activated)' if dry_run}")
puts("[#{Time.now.utc}] Running close_events :: END#{' (dry_run activated)' if dry_run}")
end
diff --git a/lib/tasks/db_cleanup.rake b/lib/tasks/db_cleanup.rake
index 40087adc..892622b4 100644
--- a/lib/tasks/db_cleanup.rake
+++ b/lib/tasks/db_cleanup.rake
@@ -8,8 +8,12 @@ namespace :db_cleanup do
# - rake "db_cleanup:remove_orphan_members_and_accounts[false]"
desc 'Remove accounts that have no user'
task :remove_orphan_members_and_accounts, [:dry_run] => :environment do |_t, args|
+ start_at= Time.now.utc
+ failed_list = []
+ member_count = 0
dry_run = true unless args[:dry_run] == 'false'
+ Rails.logger.error "[remove_orphan_members_and_accounts] Starting at #{start_at}."
puts("[#{Time.now.utc}] Running remove_orphan_members_and_accounts :: INI#{' (dry_run activated)' if dry_run}")
# List all the rows in members which do not have a matching user
@@ -24,9 +28,28 @@ namespace :db_cleanup do
# Delete the orphan member entry
Member.destroy!(member.id)
+
+ # Increment the counter
+ member_count += 1
+ rescue StandardError => e
+ failed_list.push({ member_id: member.id, reason: member.inspect })
+
+ Rails.logger.error "[remove_orphan_members_and_accounts] member_id: #{member.id} failed to update user name."
+ Rails.logger.error "[remove_orphan_members_and_accounts] member_id: #{member.id} failed reason: #{e.inspect}"
end
+ if failed_list.count > 0
+ Rails.logger.info "[remove_orphan_members_and_accounts] Failed list: #{failed_list}"
+ p "[#{Time.now.utc}] [remove_orphan_members_and_accounts] Failed list: #{failed_list}"
+ end
+ # End the task
+ # Compute task duration
+ end_at= Time.now.utc
+ duration = ((end_at - start_at) / 60.seconds).to_i
+
+ # Display closing messages and report to Rails logger for centralized logs
+ Rails.logger.error "[remove_orphan_members_and_accounts] Ending at #{end_at}. Members removed: #{member_count} in #{duration}"
puts("[#{Time.now.utc}] Running remove_orphan_members_and_accounts :: END#{' (dry_run activated)' if dry_run}")
end
@@ -39,7 +62,7 @@ namespace :db_cleanup do
puts("[#{Time.now.utc}] Running remove_orphan_accounts :: INI#{' (dry_run activated)' if dry_run}")
- # List all the rows in members which do not have a matching user
+ # List all the rows in accounts which do not have a matching user
# In order to avoid deleting objects that are freshly created and might still be in review
# it has been decided to only check objects for which the created_at value is greater or equal
# to 7 days from the current date.
|