From 2cb2ded2d9805b68acc9588d143f1076b27e1588 Mon Sep 17 00:00:00 2001 From: Sascha Hahne Date: Wed, 10 Nov 2021 20:08:41 +0100 Subject: [PATCH] Implement Update of Library/Category (#235) * Implement Update Controller tests * Basic Threading and notify * WIP * Reworked using coroutines * Use Map for JobSummary Tracking * Change Tests * Clean up * Changes based on review * Rethrow cancellationexception * Clean up * Fix Merge Error * Actually handle messages * Clean up * Remove useless annotation --- server/build.gradle.kts | 2 + .../suwayomi/tachidesk/manga/MangaAPI.kt | 3 + .../manga/controller/UpdateController.kt | 64 +++++++++++++ .../suwayomi/tachidesk/manga/impl/Category.kt | 8 ++ .../tachidesk/manga/impl/update/IUpdater.kt | 10 +++ .../tachidesk/manga/impl/update/UpdateJob.kt | 17 ++++ .../manga/impl/update/UpdateStatus.kt | 33 +++++++ .../tachidesk/manga/impl/update/Updater.kt | 76 ++++++++++++++++ .../manga/impl/update/UpdaterSocket.kt | 65 ++++++++++++++ .../tachidesk/manga/impl/update/Websocket.kt | 21 +++++ .../suwayomi/tachidesk/server/ServerSetup.kt | 3 + .../manga/controller/UpdateControllerTest.kt | 90 +++++++++++++++++++ .../manga/impl/update/TestUpdater.kt | 24 +++++ .../tachidesk/test/ApplicationTest.kt | 3 + 14 files changed, 419 insertions(+) create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/IUpdater.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateJob.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateStatus.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdaterSocket.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Websocket.kt create mode 100644 server/src/test/kotlin/suwayomi/tachidesk/manga/controller/UpdateControllerTest.kt create mode 100644 server/src/test/kotlin/suwayomi/tachidesk/manga/impl/update/TestUpdater.kt diff --git a/server/build.gradle.kts b/server/build.gradle.kts index 620ce0868..cf987c3c3 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -70,6 +70,8 @@ dependencies { // uncomment to test extensions directly // implementation(fileTree("lib/")) implementation(kotlin("script-runtime")) + + testImplementation("io.mockk:mockk:1.9.3") } application { diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/MangaAPI.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/MangaAPI.kt index b650364c6..9cc822b4c 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/MangaAPI.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/MangaAPI.kt @@ -113,6 +113,9 @@ object MangaAPI { path("update") { get("recentChapters/{pageNum}", UpdateController::recentChapters) + post("fetch", UpdateController::categoryUpdate) + get("summary", UpdateController::updateSummary) + ws("", UpdateController::categoryUpdateWS) } } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/controller/UpdateController.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/controller/UpdateController.kt index 57609f5df..9d3a12e34 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/controller/UpdateController.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/controller/UpdateController.kt @@ -1,7 +1,19 @@ package suwayomi.tachidesk.manga.controller import io.javalin.http.Context +import io.javalin.http.HttpCode +import io.javalin.websocket.WsConfig +import kotlinx.coroutines.runBlocking +import mu.KotlinLogging +import org.kodein.di.DI +import org.kodein.di.conf.global +import org.kodein.di.instance +import suwayomi.tachidesk.manga.impl.Category +import suwayomi.tachidesk.manga.impl.CategoryManga import suwayomi.tachidesk.manga.impl.Chapter +import suwayomi.tachidesk.manga.impl.update.IUpdater +import suwayomi.tachidesk.manga.impl.update.UpdaterSocket +import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass import suwayomi.tachidesk.server.JavalinSetup.future /* @@ -12,6 +24,8 @@ import suwayomi.tachidesk.server.JavalinSetup.future * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ object UpdateController { + private val logger = KotlinLogging.logger { } + /** get recently updated manga chapters */ fun recentChapters(ctx: Context) { val pageNum = ctx.pathParam("pageNum").toInt() @@ -22,4 +36,54 @@ object UpdateController { } ) } + + fun categoryUpdate(ctx: Context) { + val categoryId = ctx.formParam("category")?.toIntOrNull() + val categoriesForUpdate = ArrayList() + if (categoryId == null) { + logger.info { "Adding Library to Update Queue" } + categoriesForUpdate.addAll(Category.getCategoryList()) + } else { + val category = Category.getCategoryById(categoryId) + if (category != null) { + categoriesForUpdate.add(category) + } else { + logger.info { "No Category found" } + ctx.status(HttpCode.BAD_REQUEST) + return + } + } + addCategoriesToUpdateQueue(categoriesForUpdate, true) + ctx.status(HttpCode.OK) + } + + private fun addCategoriesToUpdateQueue(categories: List, clear: Boolean = false) { + val updater by DI.global.instance() + if (clear) { + runBlocking { updater.reset() } + } + categories.forEach { category -> + val mangas = CategoryManga.getCategoryMangaList(category.id) + mangas.forEach { manga -> + updater.addMangaToQueue(manga) + } + } + } + + fun categoryUpdateWS(ws: WsConfig) { + ws.onConnect { ctx -> + UpdaterSocket.addClient(ctx) + } + ws.onMessage { ctx -> + UpdaterSocket.handleRequest(ctx) + } + ws.onClose { ctx -> + UpdaterSocket.removeClient(ctx) + } + } + + fun updateSummary(ctx: Context) { + val updater by DI.global.instance() + ctx.json(updater.getStatus().value.getJsonSummary()) + } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Category.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Category.kt index 7f93c8b22..46bb71108 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Category.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Category.kt @@ -109,4 +109,12 @@ object Category { addDefaultIfNecessary(categories) } } + + fun getCategoryById(categoryId: Int): CategoryDataClass? { + return transaction { + CategoryTable.select { CategoryTable.id eq categoryId }.firstOrNull()?.let { + CategoryTable.toDataClass(it) + } + } + } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/IUpdater.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/IUpdater.kt new file mode 100644 index 000000000..0b18e30bb --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/IUpdater.kt @@ -0,0 +1,10 @@ +package suwayomi.tachidesk.manga.impl.update + +import kotlinx.coroutines.flow.StateFlow +import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass + +interface IUpdater { + fun addMangaToQueue(manga: MangaDataClass) + fun getStatus(): StateFlow + suspend fun reset(): Unit +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateJob.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateJob.kt new file mode 100644 index 000000000..b5463ad63 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateJob.kt @@ -0,0 +1,17 @@ +package suwayomi.tachidesk.manga.impl.update + +import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass + +enum class JobStatus { + PENDING, + RUNNING, + COMPLETE, + FAILED +} + +class UpdateJob(val manga: MangaDataClass, var status: JobStatus = JobStatus.PENDING) { + + override fun toString(): String { + return "UpdateJob(status=$status, manga=${manga.title})" + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateStatus.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateStatus.kt new file mode 100644 index 000000000..3dd78cff8 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateStatus.kt @@ -0,0 +1,33 @@ +package suwayomi.tachidesk.manga.impl.update + +import mu.KotlinLogging +import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass + +var logger = KotlinLogging.logger {} +class UpdateStatus( + var statusMap: MutableMap> = mutableMapOf>(), + var running: Boolean = false, +) { + var numberOfJobs: Int = 0 + + constructor(jobs: List, running: Boolean) : this( + mutableMapOf>(), + running + ) { + this.numberOfJobs = jobs.size + jobs.forEach { + val list = statusMap.getOrDefault(it.status, mutableListOf()) + list.add(it.manga) + statusMap[it.status] = list + } + } + + override fun toString(): String { + return "UpdateStatus(statusMap=${statusMap.map { "${it.key} : ${it.value.size}" }.joinToString("; ")}, running=$running)" + } + + // serialize to summary json + fun getJsonSummary(): String { + return """{"statusMap":{${statusMap.map { "\"${it.key}\" : ${it.value.size}" }.joinToString(",")}}, "running":$running}""" + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt new file mode 100644 index 000000000..35f51e2c5 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt @@ -0,0 +1,76 @@ +package suwayomi.tachidesk.manga.impl.update + +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.launch +import mu.KotlinLogging +import suwayomi.tachidesk.manga.impl.Chapter +import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass + +class Updater : IUpdater { + private val logger = KotlinLogging.logger {} + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + + private var tracker = mutableMapOf() + private var updateChannel = Channel() + private val statusChannel = MutableStateFlow(UpdateStatus()) + private var updateJob: Job? = null + + init { + updateJob = createUpdateJob() + } + + private fun createUpdateJob(): Job { + return scope.launch { + while (true) { + val job = updateChannel.receive() + process(job) + statusChannel.value = UpdateStatus(tracker.values.toList(), !updateChannel.isEmpty) + } + } + } + + private suspend fun process(job: UpdateJob) { + job.status = JobStatus.RUNNING + tracker["${job.manga.id}"] = job + statusChannel.value = UpdateStatus(tracker.values.toList(), true) + try { + logger.info { "Updating ${job.manga.title}" } + Chapter.getChapterList(job.manga.id, true) + job.status = JobStatus.COMPLETE + } catch (e: Exception) { + if (e is CancellationException) throw e + logger.error(e) { "Error while updating ${job.manga.title}" } + job.status = JobStatus.FAILED + } + tracker["${job.manga.id}"] = job + } + + override fun addMangaToQueue(manga: MangaDataClass) { + scope.launch { + updateChannel.send(UpdateJob(manga)) + } + tracker["${manga.id}"] = UpdateJob(manga) + statusChannel.value = UpdateStatus(tracker.values.toList(), true) + } + + override fun getStatus(): StateFlow { + return statusChannel + } + + override suspend fun reset() { + tracker.clear() + updateChannel.cancel() + statusChannel.value = UpdateStatus() + updateJob?.cancel("Reset") + updateChannel = Channel() + updateJob = createUpdateJob() + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdaterSocket.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdaterSocket.kt new file mode 100644 index 000000000..38e692a09 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdaterSocket.kt @@ -0,0 +1,65 @@ +package suwayomi.tachidesk.manga.impl.update + +import io.javalin.websocket.WsContext +import io.javalin.websocket.WsMessageContext +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.flow.collectLatest +import kotlinx.coroutines.launch +import mu.KotlinLogging +import org.kodein.di.DI +import org.kodein.di.conf.global +import org.kodein.di.instance + +object UpdaterSocket : Websocket() { + private val logger = KotlinLogging.logger {} + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + private val updater by DI.global.instance() + private var job: Job? = null + + override fun notifyClient(ctx: WsContext) { + ctx.send(updater.getStatus().value.getJsonSummary()) + } + + override fun handleRequest(ctx: WsMessageContext) { + when (ctx.message()) { + "STATUS" -> notifyClient(ctx) + else -> ctx.send( + """ + |Invalid command. + |Supported commands are: + | - STATUS + | sends the current update status + |""".trimMargin() + ) + } + } + + override fun addClient(ctx: WsContext) { + logger.info { ctx.sessionId } + super.addClient(ctx) + if (job == null) { + job = start() + } + } + + override fun removeClient(ctx: WsContext) { + super.removeClient(ctx) + if (clients.isEmpty()) { + job?.cancel() + job = null + } + } + + fun start(): Job { + return scope.launch { + while (true) { + updater.getStatus().collectLatest { + notifyAllClients() + } + } + } + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Websocket.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Websocket.kt new file mode 100644 index 000000000..e83675dd0 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Websocket.kt @@ -0,0 +1,21 @@ +package suwayomi.tachidesk.manga.impl.update + +import io.javalin.websocket.WsContext +import io.javalin.websocket.WsMessageContext +import java.util.concurrent.ConcurrentHashMap + +abstract class Websocket { + protected val clients = ConcurrentHashMap() + open fun addClient(ctx: WsContext) { + clients[ctx.sessionId] = ctx + notifyClient(ctx) + } + open fun removeClient(ctx: WsContext) { + clients.remove(ctx.sessionId) + } + open fun notifyAllClients() { + clients.values.forEach { notifyClient(it) } + } + abstract fun notifyClient(ctx: WsContext) + abstract fun handleRequest(ctx: WsMessageContext) +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt index 2b788331f..d7116b417 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt @@ -16,6 +16,8 @@ import org.kodein.di.DI import org.kodein.di.bind import org.kodein.di.conf.global import org.kodein.di.singleton +import suwayomi.tachidesk.manga.impl.update.IUpdater +import suwayomi.tachidesk.manga.impl.update.Updater import suwayomi.tachidesk.server.database.databaseUp import suwayomi.tachidesk.server.util.AppMutex.handleAppMutex import suwayomi.tachidesk.server.util.SystemTray.systemTray @@ -55,6 +57,7 @@ fun applicationSetup() { DI.global.addImport( DI.Module("Server") { bind() with singleton { applicationDirs } + bind() with singleton { Updater() } bind() with singleton { JavalinJackson() } } ) diff --git a/server/src/test/kotlin/suwayomi/tachidesk/manga/controller/UpdateControllerTest.kt b/server/src/test/kotlin/suwayomi/tachidesk/manga/controller/UpdateControllerTest.kt new file mode 100644 index 000000000..cce77e7aa --- /dev/null +++ b/server/src/test/kotlin/suwayomi/tachidesk/manga/controller/UpdateControllerTest.kt @@ -0,0 +1,90 @@ +package suwayomi.tachidesk.manga.controller + +import io.javalin.http.Context +import io.javalin.http.HttpCode +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import kotlinx.coroutines.runBlocking +import org.jetbrains.exposed.sql.insertAndGetId +import org.jetbrains.exposed.sql.transactions.transaction +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.kodein.di.DI +import org.kodein.di.conf.global +import org.kodein.di.instance +import suwayomi.tachidesk.manga.impl.Category +import suwayomi.tachidesk.manga.impl.CategoryManga +import suwayomi.tachidesk.manga.impl.update.IUpdater +import suwayomi.tachidesk.manga.model.table.CategoryMangaTable +import suwayomi.tachidesk.manga.model.table.CategoryTable +import suwayomi.tachidesk.manga.model.table.MangaTable +import suwayomi.tachidesk.test.ApplicationTest +import suwayomi.tachidesk.test.clearTables + +internal class UpdateControllerTest : ApplicationTest() { + private val ctx = mockk(relaxed = true) + + @Test + fun `POST non existent Category Id should give error`() { + every { ctx.formParam("category") } returns "1" + UpdateController.categoryUpdate(ctx) + verify { ctx.status(HttpCode.BAD_REQUEST) } + val updater by DI.global.instance() + assertEquals(0, updater.getStatus().value.numberOfJobs) + } + + @Test + fun `POST existent Category Id should give success`() { + Category.createCategory("foo") + createLibraryManga("bar") + CategoryManga.addMangaToCategory(1, 1) + every { ctx.formParam("category") } returns "1" + UpdateController.categoryUpdate(ctx) + verify { ctx.status(HttpCode.OK) } + val updater by DI.global.instance() + assertEquals(1, updater.getStatus().value.numberOfJobs) + } + + @Test + fun `POST null or empty category should update library`() { + val fooCatId = Category.createCategory("foo") + val fooMangaId = createLibraryManga("foo") + CategoryManga.addMangaToCategory(fooMangaId, fooCatId) + val barCatId = Category.createCategory("bar") + val barMangaId = createLibraryManga("bar") + CategoryManga.addMangaToCategory(barMangaId, barCatId) + createLibraryManga("mangaInDefault") + every { ctx.formParam("category") } returns null + UpdateController.categoryUpdate(ctx) + verify { ctx.status(HttpCode.OK) } + val updater by DI.global.instance() + assertEquals(3, updater.getStatus().value.numberOfJobs) + } + + private fun createLibraryManga( + _title: String + ): Int { + return transaction { + MangaTable.insertAndGetId { + it[title] = _title + it[url] = _title + it[sourceReference] = 1 + it[defaultCategory] = true + it[inLibrary] = true + }.value + } + } + + @AfterEach + internal fun tearDown() { + clearTables( + CategoryMangaTable, + MangaTable, + CategoryTable + ) + val updater by DI.global.instance() + runBlocking { updater.reset() } + } +} diff --git a/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/update/TestUpdater.kt b/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/update/TestUpdater.kt new file mode 100644 index 000000000..dbff40c45 --- /dev/null +++ b/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/update/TestUpdater.kt @@ -0,0 +1,24 @@ +package suwayomi.tachidesk.manga.impl.update + +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass + +class TestUpdater : IUpdater { + private val updateQueue = ArrayList() + private var isRunning = false + + override fun addMangaToQueue(manga: MangaDataClass) { + updateQueue.add(UpdateJob(manga)) + isRunning = true + } + + override fun getStatus(): StateFlow { + return MutableStateFlow(UpdateStatus(updateQueue, isRunning)) + } + + override suspend fun reset() { + updateQueue.clear() + isRunning = false + } +} diff --git a/server/src/test/kotlin/suwayomi/tachidesk/test/ApplicationTest.kt b/server/src/test/kotlin/suwayomi/tachidesk/test/ApplicationTest.kt index d49338cf3..ee3039a29 100644 --- a/server/src/test/kotlin/suwayomi/tachidesk/test/ApplicationTest.kt +++ b/server/src/test/kotlin/suwayomi/tachidesk/test/ApplicationTest.kt @@ -18,6 +18,8 @@ import org.kodein.di.DI import org.kodein.di.bind import org.kodein.di.conf.global import org.kodein.di.singleton +import suwayomi.tachidesk.manga.impl.update.IUpdater +import suwayomi.tachidesk.manga.impl.update.TestUpdater import suwayomi.tachidesk.server.ApplicationDirs import suwayomi.tachidesk.server.JavalinSetup import suwayomi.tachidesk.server.ServerConfig @@ -61,6 +63,7 @@ open class ApplicationTest { DI.Module("Server") { bind() with singleton { applicationDirs } bind() with singleton { JavalinJackson() } + bind() with singleton { TestUpdater() } } )