From 3da3b6cf100b1409efe1b621e352b83e8ac37c39 Mon Sep 17 00:00:00 2001 From: Braxen Date: Wed, 20 Apr 2022 09:59:35 +0200 Subject: [PATCH 1/3] remove cron container, integrate into server --- docker-compose.yml | 6 +++--- server/package.json | 2 ++ server/src/Controllers/Cron.ts | 23 +++++++++++++------- server/src/Core/Scheduler.ts | 37 +++++++++++++++++++++++++++++++++ server/src/Core/TwitchConfig.ts | 6 ++++++ server/yarn.lock | 27 ++++++++++++++++++++++++ 6 files changed, 90 insertions(+), 11 deletions(-) create mode 100644 server/src/Core/Scheduler.ts diff --git a/docker-compose.yml b/docker-compose.yml index b420ef0b..60984e5d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,9 +21,9 @@ services: # - ./docker/saved_vods:/var/www/twitchautomator/public/saved_vods environment: - NODE_ENV=production - cron: - build: ./docker/cron-container/ - image: mrbrax/twitchautomator:latest-cron + # cron: + # build: ./docker/cron-container/ + # image: mrbrax/twitchautomator:latest-cron # broker: # build: ./client-broker/ # image: mrbrax/twitchautomator:latest-broker \ No newline at end of file diff --git a/server/package.json b/server/package.json index f1e88a5f..fabda67c 100644 --- a/server/package.json +++ b/server/package.json @@ -16,6 +16,7 @@ "license": "ISC", "devDependencies": { "@types/basic-auth": "^1.1.3", + "@types/cron": "^1.7.3", "@types/express": "^4.17.13", "@types/jest": "^27.4.1", "@types/minimist": "^1.2.2", @@ -41,6 +42,7 @@ "axios": "^0.26.0", "basic-auth": "^2.0.1", "chalk": "4", + "cron": "^1.8.2", "date-fns": "^2.28.0", "express": "^4.17.3", "minimist": "^1.2.6", diff --git a/server/src/Controllers/Cron.ts b/server/src/Controllers/Cron.ts index 9d95525c..c4fdf9be 100644 --- a/server/src/Controllers/Cron.ts +++ b/server/src/Controllers/Cron.ts @@ -4,9 +4,7 @@ import { ClientBroker } from "../Core/ClientBroker"; import { LOGLEVEL, TwitchLog } from "../Core/TwitchLog"; import { generateStreamerList } from "../Helpers/StreamerList"; -export async function CheckDeletedVods(req: express.Request, res: express.Response): Promise { - - // const force = req.query.force; +export async function fCheckDeletedVods(): Promise { const streamerList = generateStreamerList(); @@ -41,18 +39,21 @@ export async function CheckDeletedVods(req: express.Request, res: express.Respon } - res.send(output || "No deleted vods found"); + return output; +} +export async function CheckDeletedVods(req: express.Request, res: express.Response): Promise { + // const force = req.query.force; + const output = await fCheckDeletedVods(); + res.send(output || "No deleted vods found"); } -export async function CheckMutedVods(req: express.Request, res: express.Response): Promise { +export async function fCheckMutedVods(force = false): Promise { const streamerList = generateStreamerList(); let output = ""; - const force = req.query.force !== undefined; - for (const channel of streamerList.channels) { if (!channel.vods_list) continue; @@ -104,6 +105,12 @@ export async function CheckMutedVods(req: express.Request, res: express.Response } - res.send(output || "No muted vods found"); + return output; + +} +export async function CheckMutedVods(req: express.Request, res: express.Response): Promise { + const force = req.query.force !== undefined; + const output = await fCheckMutedVods(force); + res.send(output || "No muted vods found"); } \ No newline at end of file diff --git a/server/src/Core/Scheduler.ts b/server/src/Core/Scheduler.ts new file mode 100644 index 00000000..5b9b5087 --- /dev/null +++ b/server/src/Core/Scheduler.ts @@ -0,0 +1,37 @@ +import cron from "cron"; +import * as CronController from "../Controllers/Cron"; +import { TwitchConfig } from "./TwitchConfig"; + +export class Scheduler { + + public static jobs: cron.CronJob[] = []; + + public static schedule(cronTime: string, callback: () => void): cron.CronJob { + const job = new cron.CronJob(cronTime, callback); + Scheduler.jobs.push(job); + job.start(); + return job; + } + + public static defaultJobs() { + // # 0 5 * * 1 curl http://localhost:8080/api/v0/cron/sub + // 0 */12 * * * curl http://localhost:8080/api/v0/cron/check_muted_vods + // 10 */12 * * * curl http://localhost:8080/api/v0/cron/check_deleted_vods + + this.schedule("0 */12 * * *", () => { + if (!TwitchConfig.cfg("schedule_muted_vods")) return; + CronController.fCheckMutedVods(); + }); + + this.schedule("10 */12 * * *", () => { + if (!TwitchConfig.cfg("schedule_deleted_vods")) return; + CronController.fCheckDeletedVods(); + }); + + // this.schedule("* * * * *", () => { + // console.log("Cronjob ran", new Date().toISOString()); + // }); + + } + +} \ No newline at end of file diff --git a/server/src/Core/TwitchConfig.ts b/server/src/Core/TwitchConfig.ts index aa8a848f..eb6ac170 100644 --- a/server/src/Core/TwitchConfig.ts +++ b/server/src/Core/TwitchConfig.ts @@ -14,6 +14,7 @@ import path from "path"; import { ClientBroker } from "./ClientBroker"; import minimist from "minimist"; import { TwitchVOD } from "./TwitchVOD"; +import { Scheduler } from "./Scheduler"; const argv = minimist(process.argv.slice(2)); @@ -120,6 +121,9 @@ export class TwitchConfig { { "key": "discord_enabled", "group": "Notifications", "text": "Enable Discord notifications", "type": "boolean", "default": false }, { "key": "discord_webhook", "group": "Notifications", "text": "Discord webhook", "type": "string" }, + { "key": "schedule_muted_vods", "group": "Schedules", "text": "Check muted vods", "type": "boolean", "default": true }, + { "key": "schedule_deleted_vods", "group": "Schedules", "text": "Check deleted vods", "type": "boolean", "default": true }, + ]; static watcher: fs.FSWatcher | undefined; @@ -488,6 +492,8 @@ export class TwitchConfig { this.startWatchingConfig(); + Scheduler.defaultJobs(); + // monitor for program exit // let saidGoobye = false; // const goodbye = () => { diff --git a/server/yarn.lock b/server/yarn.lock index 2a26c110..c5ed92fc 100644 --- a/server/yarn.lock +++ b/server/yarn.lock @@ -593,6 +593,14 @@ dependencies: "@types/node" "*" +"@types/cron@^1.7.3": + version "1.7.3" + resolved "https://registry.yarnpkg.com/@types/cron/-/cron-1.7.3.tgz#993db7d54646f61128c851607b64ba4495deae93" + integrity sha512-iPmUXyIJG1Js+ldPYhOQcYU3kCAQ2FWrSkm1FJPoii2eYSn6wEW6onPukNTT0bfiflexNSRPl6KWmAIqS+36YA== + dependencies: + "@types/node" "*" + moment ">=2.14.0" + "@types/express-serve-static-core@^4.17.18": version "4.17.28" resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.17.28.tgz#c47def9f34ec81dc6328d0b1b5303d1ec98d86b8" @@ -1256,6 +1264,13 @@ create-require@^1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" +cron@^1.8.2: + version "1.8.2" + resolved "https://registry.yarnpkg.com/cron/-/cron-1.8.2.tgz#4ac5e3c55ba8c163d84f3407bde94632da8370ce" + integrity sha512-Gk2c4y6xKEO8FSAUTklqtfSr7oTq0CiPQeLBG5Fl0qoXpZyMcj1SG59YL+hqq04bu6/IuEA7lMkYDAplQNKkyg== + dependencies: + moment-timezone "^0.5.x" + cross-spawn@^7.0.2, cross-spawn@^7.0.3: version "7.0.3" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" @@ -3047,6 +3062,18 @@ minimist@^1.2.6: resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44" integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q== +moment-timezone@^0.5.x: + version "0.5.34" + resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.34.tgz#a75938f7476b88f155d3504a9343f7519d9a405c" + integrity sha512-3zAEHh2hKUs3EXLESx/wsgw6IQdusOT8Bxm3D9UrHPQR7zlMmzwybC8zHEM1tQ4LJwP7fcxrWr8tuBg05fFCbg== + dependencies: + moment ">= 2.9.0" + +"moment@>= 2.9.0", moment@>=2.14.0: + version "2.29.3" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.3.tgz#edd47411c322413999f7a5940d526de183c031f3" + integrity sha512-c6YRvhEo//6T2Jz/vVtYzqBzwvPT95JBQ+smCytzf7c50oMZRsR/a4w88aD34I+/QVSfnoAnSBFPJHItlOMJVw== + morgan@^1.10.0: version "1.10.0" resolved "https://registry.yarnpkg.com/morgan/-/morgan-1.10.0.tgz#091778abc1fc47cd3509824653dae1faab6b17d7" From 641eea7a4568a788f0f7ad60a3ae355186f34fc6 Mon Sep 17 00:00:00 2001 From: braxen Date: Wed, 20 Apr 2022 12:09:33 +0200 Subject: [PATCH 2/3] Update Scheduler.ts --- server/src/Core/Scheduler.ts | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/server/src/Core/Scheduler.ts b/server/src/Core/Scheduler.ts index 5b9b5087..e8633dab 100644 --- a/server/src/Core/Scheduler.ts +++ b/server/src/Core/Scheduler.ts @@ -4,11 +4,14 @@ import { TwitchConfig } from "./TwitchConfig"; export class Scheduler { - public static jobs: cron.CronJob[] = []; + public static jobs: Record = {}; - public static schedule(cronTime: string, callback: () => void): cron.CronJob { + public static schedule(name: string, cronTime: string, callback: () => void): cron.CronJob { + if (this.hasJob(name)) { + this.removeJob(name); + } const job = new cron.CronJob(cronTime, callback); - Scheduler.jobs.push(job); + this.jobs[name] = job; job.start(); return job; } @@ -18,12 +21,12 @@ export class Scheduler { // 0 */12 * * * curl http://localhost:8080/api/v0/cron/check_muted_vods // 10 */12 * * * curl http://localhost:8080/api/v0/cron/check_deleted_vods - this.schedule("0 */12 * * *", () => { + this.schedule("check_muted_vods", "0 */12 * * *", () => { if (!TwitchConfig.cfg("schedule_muted_vods")) return; CronController.fCheckMutedVods(); }); - this.schedule("10 */12 * * *", () => { + this.schedule("check_deleted_vods", "10 */12 * * *", () => { if (!TwitchConfig.cfg("schedule_deleted_vods")) return; CronController.fCheckDeletedVods(); }); @@ -34,4 +37,15 @@ export class Scheduler { } + public static hasJob(name: string) { + return this.jobs[name] !== undefined; + } + + public static removeJob(name: string) { + if (this.hasJob(name)) { + this.jobs[name].stop(); + delete this.jobs[name]; + } + } + } \ No newline at end of file From a66c16bb81de12e31b4c5485e7120a67b3cf4755 Mon Sep 17 00:00:00 2001 From: braxen Date: Wed, 20 Apr 2022 14:57:52 +0200 Subject: [PATCH 3/3] pack script, new start now works --- .dockerignore | 3 ++- .gitignore | 3 ++- pack.js | 43 +++++++++++++++++++++++++++++++++ server/package.json | 2 +- server/src/Core/BaseConfig.ts | 7 +++--- server/src/Core/TwitchHelper.ts | 13 +++------- 6 files changed, 55 insertions(+), 16 deletions(-) create mode 100644 pack.js diff --git a/.dockerignore b/.dockerignore index c00a00ae..3ab3b334 100644 --- a/.dockerignore +++ b/.dockerignore @@ -28,4 +28,5 @@ storage data **/*.mp4 **/*.chatdump -**/*.chatdump.line \ No newline at end of file +**/*.chatdump.line +release/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index f767f3aa..ac0c36a8 100644 --- a/.gitignore +++ b/.gitignore @@ -28,4 +28,5 @@ vendor/ data/storage data/cache data/config -data/payloads \ No newline at end of file +data/payloads +release/ \ No newline at end of file diff --git a/pack.js b/pack.js new file mode 100644 index 00000000..e24a110d --- /dev/null +++ b/pack.js @@ -0,0 +1,43 @@ +const util = require("util"); +const fs = require("fs"); +const exec = util.promisify(require("child_process").exec); +const client_version = require("./client-vue/package.json").version; +const server_version = require("./server/package.json").version; + +console.log(`Client version: ${client_version}`); +console.log(`Server version: ${server_version}`); + +const release_name = `TwitchAutomator-s${server_version}-c${client_version}`; + +if (fs.existsSync(`./release/${release_name}.zip`)) { + fs.unlinkSync(`./release/${release_name}.zip`); +} + +// build client +exec('cd client-vue && yarn install && yarn run build').then(() => { + console.log("Client built"); + + // build server + exec('cd server && yarn install && yarn run build').then(() => { + console.log("Server built"); + + // package files + exec( + `7za a -tzip -r -xr!node_modules ./release/${release_name}.zip ` + + `client-vue/dist ` + + `client-vue/package.json ` + + `server/build ` + + `server/package.json ` + + `server/tsconfig.json ` + + `requirements.txt ` + + `Pipfile ` + + `Pipfile.lock ` + + `vodplayer ` + + `twitch-chat-dumper ` + + `README.md ` + + `LICENSE ` + ).then(() => { + console.log("Files packaged"); + }); + }); +}); \ No newline at end of file diff --git a/server/package.json b/server/package.json index fabda67c..69357b74 100644 --- a/server/package.json +++ b/server/package.json @@ -1,6 +1,6 @@ { "name": "twitchautomator-server", - "version": "0.0.1f", + "version": "0.0.2", "description": "", "main": "index.ts", "scripts": { diff --git a/server/src/Core/BaseConfig.ts b/server/src/Core/BaseConfig.ts index 568e6fc6..891cc721 100644 --- a/server/src/Core/BaseConfig.ts +++ b/server/src/Core/BaseConfig.ts @@ -35,10 +35,9 @@ export const DataRoot = if (argv.home && !fs.existsSync(HomeRoot)) { fs.mkdirSync(HomeRoot, { recursive: true }); -} - -if (!fs.existsSync(DataRoot)) { - throw new Error(`DataRoot does not exist: ${DataRoot}`); +} else if (!argv.home && !fs.existsSync(DataRoot)) { // create data root, is this a good idea? + // throw new Error(`DataRoot does not exist: ${DataRoot}`); + fs.mkdirSync(DataRoot, { recursive: true }); } console.log(`DataRoot: ${DataRoot}`); diff --git a/server/src/Core/TwitchHelper.ts b/server/src/Core/TwitchHelper.ts index 18c50d6a..abcb945d 100644 --- a/server/src/Core/TwitchHelper.ts +++ b/server/src/Core/TwitchHelper.ts @@ -253,8 +253,7 @@ export class TwitchHelper { } public static path_streamlink(): string | false { - // $path = TwitchConfig::cfg('bin_dir') . DIRECTORY_SEPARATOR . "streamlink" . (self::is_windows() ? '.exe' : ''); - // return file_exists($path) ? $path : false; + if (!TwitchConfig.cfg("bin_dir")) return false; const full_path = path.join(TwitchConfig.cfg("bin_dir"), `streamlink${this.is_windows() ? ".exe" : ""}`); const exists = fs.existsSync(full_path); @@ -267,8 +266,7 @@ export class TwitchHelper { } public static path_youtubedl(): string | false { - // $path = TwitchConfig::cfg('bin_dir') . DIRECTORY_SEPARATOR . "youtube-dl" . (self::is_windows() ? '.exe' : ''); - // return file_exists($path) ? $path : false; + if (!TwitchConfig.cfg("bin_dir")) return false; const full_path = path.join(TwitchConfig.cfg("bin_dir"), `yt-dlp${this.is_windows() ? ".exe" : ""}`); const exists = fs.existsSync(full_path); @@ -281,8 +279,7 @@ export class TwitchHelper { } public static path_tcd(): string | false { - // $path = TwitchConfig::cfg('bin_dir') . DIRECTORY_SEPARATOR . "tcd" . (self::is_windows() ? '.exe' : ''); - // return file_exists($path) ? $path : false; + if (!TwitchConfig.cfg("bin_dir")) return false; const full_path = path.join(TwitchConfig.cfg("bin_dir"), `tcd${this.is_windows() ? ".exe" : ""}`); const exists = fs.existsSync(full_path); @@ -295,8 +292,7 @@ export class TwitchHelper { } public static path_pipenv(): string | false { - // $path = TwitchConfig::cfg('bin_dir') . DIRECTORY_SEPARATOR . "pipenv" . (self::is_windows() ? '.exe' : ''); - // return file_exists($path) ? $path : false; + if (!TwitchConfig.cfg("bin_dir")) return false; const full_path = path.join(TwitchConfig.cfg("bin_dir"), `pipenv${this.is_windows() ? ".exe" : ""}`); const exists = fs.existsSync(full_path); @@ -310,7 +306,6 @@ export class TwitchHelper { public static path_twitchdownloader(): string | false { if (TwitchConfig.cfg("twitchdownloader_path")) return TwitchConfig.cfg("twitchdownloader_path"); - return false; }