Added AL activity syncing

.rubocop.yml

@@ -38,6 +38,10 @@ Performance:
 Rails:
   Enabled: true
 
+# Allow upsert
+Rails/SkipsModelValidations:
+  Enabled: false
+
 # Correct usage of Date methods in Rails. Use Time.zone.today over Date.today
 Rails/Date:
   Enabled: true

Gemfile

@@ -112,3 +112,7 @@ gem "persistent_httparty", "~> 0.1.2"
 gem "sorbet-runtime"
 
 gem "brotli", "~> 0.5.0"
+
+gem "sidekiq", "~> 7.2"
+
+gem "sidekiq-cron", "~> 1.12"

Gemfile.lock

@@ -124,12 +124,17 @@ GEM
     erb-formatter (0.7.2)
       syntax_tree (~> 6.0)
     erubi (1.12.0)
+    et-orbi (1.2.11)
+      tzinfo
     event_emitter (0.2.6)
     faraday (2.9.0)
       faraday-net_http (>= 2.0, < 3.2)
     faraday-net_http (3.1.0)
       net-http
     ffi (1.16.3)
+    fugit (1.10.1)
+      et-orbi (~> 1, >= 1.2.7)
+      raabro (~> 1.4)
     gene_pool (1.5.0)
       concurrent-ruby (>= 1.0)
     globalid (1.2.1)
@@ -233,6 +238,7 @@ GEM
     public_suffix (5.0.4)
     puma (6.4.2)
       nio4r (~> 2.0)
+    raabro (1.4.0)
     racc (1.7.3)
     rack (2.2.9)
     rack-brotli (1.2.0)
@@ -349,6 +355,15 @@ GEM
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)
+    sidekiq (7.2.2)
+      concurrent-ruby (< 2)
+      connection_pool (>= 2.3.0)
+      rack (>= 2.2.4)
+      redis-client (>= 0.19.0)
+    sidekiq-cron (1.12.0)
+      fugit (~> 1.8)
+      globalid (>= 1.0.1)
+      sidekiq (>= 6)
     sorbet (0.5.11305)
       sorbet-static (= 0.5.11305)
     sorbet-runtime (0.5.11305)
@@ -456,6 +471,8 @@ DEPENDENCIES
   ruby-lsp
   ruby-lsp-rails
   selenium-webdriver
+  sidekiq (~> 7.2)
+  sidekiq-cron (~> 1.12)
   sorbet
   sorbet-runtime
   tapioca

app/models/anilist_activity.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class AnilistActivity
+  include Mongoid::Document
+
+  field :_id, type: Integer
+  field :__typename, type: String
+  field :media, type: Hash
+  field :createdAt, type: Integer
+  field :status, type: String
+  field :progress, type: String
+  field :siteUrl, type: String
+
+  index({ _id: 1 }, { unique: true })
+end

app/sidekiq/anilist_activities_sync_job.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+# typed: true
+
+require "anilist"
+
+class AnilistActivitiesSyncJob
+  include Sidekiq::Job
+  IGNORED_USER_STATUS = ["plans to watch", "paused watching", "dropped"]
+
+  def perform(date)
+    user_id = ENV.fetch("ANILIST_USER_ID")
+    user_activity = []
+    page = 1
+
+    loop do
+      response = AniList::Client.execute(AniList::UserAnimeActivitiesQuery, date:, user_id:, page:, per_page: 50)
+      has_next_page = response.data&.page&.page_info&.has_next_page? || false
+
+      user_activity.push(*response.data.page.activities.to_a.map(&:to_h))
+
+      break if has_next_page == false
+
+      sleep 20
+      page += 1
+    end
+
+    user_activity = user_activity.reverse.select { |x| IGNORED_USER_STATUS.exclude? x["status"] }
+    user_activity.each do |activity|
+      activity = AnilistActivity.new(activity)
+      activity.upsert(replace: false)
+    end
+  end
+end

bin/sidekiq

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'sidekiq' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__)
+
+bundle_binstub = File.expand_path("bundle", __dir__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300).include?("This file was generated by Bundler")
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("sidekiq", "sidekiq")

bin/sidekiqmon

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'sidekiqmon' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__)
+
+bundle_binstub = File.expand_path("bundle", __dir__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300).include?("This file was generated by Bundler")
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("sidekiq", "sidekiqmon")

config/routes.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "sidekiq/web"
+require "sidekiq/cron/web"
+
 Rails.application.routes.draw do
   # For details on the DSL available within this file, see https://guides.rubyonrails.org/routing.html
   constraints subdomain: "admin" do
@@ -22,6 +25,8 @@
         password: "/forgot",
         sign_out: "/logout"
       }
+
+    mount Sidekiq::Web => "/sidekiq"
   end
 
   root to: "home#index"

config/schedule.yml

@@ -0,0 +1,6 @@
+anilist_activities_sync_job:
+  cron: "0 * * * *"
+  class: "AnilistActivitiesSyncJob"
+  queue: default
+  args:
+    date: <%= Time.zone.now.beginning_of_month.to_i %>

config/sidekiq.yml

@@ -0,0 +1,7 @@
+:verbose: false
+:concurrency: 5
+:timeout: 25
+:queues:
+  - critical
+  - default
+  - low

docker-compose.yml

@@ -53,6 +53,7 @@ services:
     depends_on:
       - db
       - redis
+      - sidekiq
     ports:
       - 3000:3000
     environment:
@@ -65,6 +66,36 @@ services:
       - RAILS_ASSET_HOST
       - OCCSON_ACCESS_TOKEN
       - OCCSON_PASSPHRASE
+  sidekiq:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      secrets:
+        - TZ
+        - RAILS_ENV
+        - NODE_ENV
+        - OCCSON_ACCESS_TOKEN
+        - OCCSON_PASSPHRASE
+        - RAILS_MASTER_KEY
+        - RAILS_ASSET_HOST
+    container_name: project-cyan-sidekiq
+    image: aldwin7894/project-cyan:prod
+    entrypoint: ./entrypoint-sidekiq.sh
+    command: "bin/sidekiq"
+    restart: always
+    depends_on:
+      - db
+      - redis
+    environment:
+      - DATABASE_URL=mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@db:27017/${MONGO_DB}
+      - REDIS_URL=redis://redis:6379
+      - TZ=Asia/Manila
+      - RAILS_MASTER_KEY
+      - RAILS_ENV
+      - NODE_ENV
+      - RAILS_ASSET_HOST
+      - OCCSON_ACCESS_TOKEN
+      - OCCSON_PASSPHRASE
 
 volumes:
   dbdata:

entrypoint-sidekiq.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -e
+
+# Then exec the container's main process (what's set as CMD in the Dockerfile).
+exec "$@"

lib/anilist.rb

@@ -92,6 +92,7 @@ module AniList
         }
         activities(userId: $user_id, createdAt_greater: $date, sort: ID, type: ANIME_LIST) {
           ... on ListActivity {
+            id
             media {
               format
               bannerImage