feat: GithubDownload only downloads once per org/repo/branch for matc…

…hing templates

.github/workflows/build.yml

@@ -19,7 +19,7 @@ jobs:
         uses: actions/setup-java@v3
         with:
           distribution: temurin
-          java-version: 17
+          java-version: 21
           cache: gradle
 
       - name: Build

src/main/kotlin/com/mineinabyss/packy/PackyCommands.kt

@@ -30,8 +30,9 @@ class PackyCommands : IdofrontCommandExecutor(), TabCompleter {
                     })
                     action {
                         packy.plugin.launch(packy.plugin.asyncDispatcher) {
+                            val templates = packy.templates.filter { it.value.githubDownload?.key() == template.githubDownload!!.key() }.values.sortedBy { it.id }.toTypedArray()
                             sender.info("Downloading template ${template.id}...")
-                            PackyDownloader.updateGithubTemplate(template)
+                            PackyDownloader.updateGithubTemplate(*templates)
                             sender.success("Downloaded template ${template.id}!")
                         }
                     }

src/main/kotlin/com/mineinabyss/packy/PackyDownloader.kt

@@ -5,6 +5,7 @@ import com.github.shynixn.mccoroutine.bukkit.launch
 import com.google.gson.JsonParser
 import com.mineinabyss.idofront.textcomponents.miniMsg
 import com.mineinabyss.packy.config.PackyTemplate
+import com.mineinabyss.packy.config.PackyTemplates
 import com.mineinabyss.packy.config.packy
 import com.mineinabyss.packy.helpers.downloadZipFromGithubResponse
 import kotlinx.coroutines.Job
@@ -16,27 +17,30 @@ import kotlin.io.path.*
 object PackyDownloader {
     var startupJob: Job? = null
 
-    fun updateGithubTemplate(template: PackyTemplate): Boolean {
+    fun updateGithubTemplate(vararg templates: PackyTemplate): Boolean {
+        val template = templates.first()
+        val templateIds = templates.joinToString("|") { it.id }
+        val regex = "$templateIds=.*".toRegex()
         val hashFile = packy.plugin.dataFolder.toPath() / "templates" / "localHashes.txt"
         hashFile.createParentDirectories()
         hashFile.toFile().createNewFile()
 
-        val templateExists = template.path.exists()
+        val templatesExists = templates.all { it.path.exists() }
         val latestHash = latestCommitHash(template.githubDownload ?: return false) ?: return false
-        val localHash = hashFile.readLines().find { it.matches("${template.id}=.*".toRegex()) }?.substringAfter("=")
+        val localHash = hashFile.readLines().find { it.matches(regex) }?.substringAfter("=")
 
         when {
-            !templateExists || localHash == null || localHash != latestHash -> {
-                downloadAndExtractGithub(template)
-                val lines = hashFile.readLines().toMutableSet().apply { removeIf { it.startsWith(template.id) } }
-                lines += "${template.id}=$latestHash"
+            !templatesExists || localHash == null || localHash != latestHash -> {
+                downloadAndExtractGithub(*templates)
+                val lines = hashFile.readLines().toMutableSet().apply { removeIf { it.matches(regex) } }
+                lines += "$templateIds=$latestHash"
                 hashFile.writeLines(lines)
-                if (templateExists) packy.logger.s("Updated hash for ${template.id}")
+                if (templatesExists) packy.logger.s("Updated hash for $templateIds")
             }
 
-            else -> packy.logger.s("Template up to date: <dark_gray>${template.id}".miniMsg())
+            else -> packy.logger.s("Template up to date: <dark_gray>$templateIds".miniMsg())
         }
-        return !templateExists || localHash == null || localHash != latestHash
+        return !templatesExists || localHash == null || localHash != latestHash
     }
 
     private fun latestCommitHash(githubDownload: PackyTemplate.GithubDownload): String? {
@@ -61,27 +65,36 @@ object PackyDownloader {
 
     fun downloadTemplates() {
         startupJob = packy.plugin.launch(packy.plugin.asyncDispatcher) {
-            packy.templates.entries.filter { it.value.githubDownload != null }.map { (id, template) ->
-                launch {
-                    packy.logger.i("Checking updates (GitHub): <dark_gray>$id".miniMsg())
-                    if (updateGithubTemplate(template)) {
-                        packy.logger.s("Successfully downloaded ${id}-template!")
-                        PackyGenerator.cachedPacks.keys.removeIf { id in it }
-                        PackyGenerator.cachedPacksByteArray.keys.removeIf { id in it }
+            packy.templates.values.filter { it.githubDownload != null }
+                .sortedBy { it.id }
+                .groupBy { it.githubDownload!!.key() }
+                .map { (_, templates) ->
+                    val templateIds = templates.joinToString { it.id }
+                    val suffix = "template" + if (templates.size > 1) "s" else ""
+                    launch {
+                        packy.logger.i("Checking updates (GitHub): <dark_gray>$templateIds".miniMsg())
+                        if (updateGithubTemplate(*templates.toTypedArray())) {
+                            packy.logger.s("Successfully downloaded ${templateIds}-$suffix!")
+                            templates.forEach { template ->
+                                PackyGenerator.cachedPacks.keys.removeIf { template.id in it }
+                                PackyGenerator.cachedPacksByteArray.keys.removeIf { template.id in it }
+                            }
+                        }
+                        if (packy.config.packSquash.enabled) {
+                            packy.logger.i("Starting PackSquash process for $templateIds-$suffix...")
+                            templates.forEach(PackySquash::squashPackyTemplate)
+                            packy.logger.s("Finished PackSquash process for $templateIds-$suffix")
+                        }
                     }
-                    if (packy.config.packSquash.enabled) {
-                        packy.logger.i("Starting PackSquash process for $id-template...")
-                        PackySquash.squashPackyTemplate(template)
-                        packy.logger.s("Finished PackSquash process for $id-template")
-                    }
-                }
             }
         }
     }
 
-    fun downloadAndExtractGithub(template: PackyTemplate) {
-        val (owner, repo, branch, _) = template.githubDownload
-            ?: return packy.logger.e("${template.id} has no githubDownload, skipping...")
+    fun downloadAndExtractGithub(vararg templates: PackyTemplate) {
+        val firstTemplate = templates.firstOrNull() ?: return
+        val templateIds = templates.joinToString { it.id }
+        val (owner, repo, branch, _) = firstTemplate.githubDownload
+            ?: return packy.logger.e("$templateIds has no githubDownload, skipping...")
         val url = "https://api.github.com/repos/${owner}/${repo}/zipball/${branch}"
 
 
@@ -93,8 +106,8 @@ object PackyDownloader {
             .build()
 
         client.newCall(request).execute().use { response ->
-            if (!response.isSuccessful) return@use packy.logger.e("Failed to download template ${template.id} via $url")
-            response.downloadZipFromGithubResponse(template)
+            if (!response.isSuccessful) return@use packy.logger.e("Failed to download template $templateIds via $url")
+            response.downloadZipFromGithubResponse(*templates)
         }
     }
 }

src/main/kotlin/com/mineinabyss/packy/config/PackyTemplate.kt

@@ -15,6 +15,8 @@ import kotlin.io.path.*
 data class PackyTemplates(val templates: List<PackyTemplate> = listOf()) {
     @EncodeDefault(NEVER)
     val templateMap: Map<String, PackyTemplate> = templates.associateBy { it.name }
+
+    fun component2(): Map<String, PackyTemplate> = templateMap
 }
 
 @Serializable
@@ -44,7 +46,10 @@ data class PackyTemplate(
         val repo: String,
         val branch: String,
         @EncodeDefault(NEVER) val subFolder: String? = null
-    )
+    ) {
+        fun key(): GithubDownloadKey = GithubDownloadKey(org, repo, branch)
+        data class GithubDownloadKey(val org: String, val repo: String, val branch: String)
+    }
 }
 
 @Serializable

src/main/kotlin/com/mineinabyss/packy/helpers/PackyHelpers.kt

@@ -3,11 +3,10 @@ package com.mineinabyss.packy.helpers
 import com.mineinabyss.packy.config.PackyTemplate
 import com.mineinabyss.packy.config.packy
 import okhttp3.Response
+import org.apache.commons.io.IOUtils
 import team.unnamed.creative.ResourcePack
 import team.unnamed.creative.serialize.minecraft.MinecraftResourcePackReader
-import java.io.File
-import java.io.FileInputStream
-import java.io.FileOutputStream
+import java.io.*
 import java.util.*
 import java.util.zip.ZipEntry
 import java.util.zip.ZipInputStream
@@ -27,61 +26,63 @@ fun File.readPack(): ResourcePack? {
 }
 
 @OptIn(ExperimentalPathApi::class)
-fun Response.downloadZipFromGithubResponse(template: PackyTemplate) {
-    val (owner, repo, _, subPath) = template.githubDownload ?: return packy.logger.e("${template.id} has no githubDownload, skipping...")
-    val zipStream = ZipInputStream(body!!.byteStream())
-
-    runCatching {
-        template.path.deleteRecursively()
+fun Response.downloadZipFromGithubResponse(vararg templates: PackyTemplate) {
+    // Read the entire response body into a byte array
+    val responseBody = body?.byteStream()?.use { inputStream ->
+        ByteArrayOutputStream().apply {
+            inputStream.copyTo(this)
+        }.toByteArray()
+    } ?: run {
+        packy.logger.e("${templates.joinToString { it.name }} has no response body, skipping...")
+        return
+    }
 
-        ZipOutputStream(FileOutputStream(template.path.toFile())).use { zipOutputStream ->
-            var entry = zipStream.nextEntry
+    templates.forEach { template ->
+        val githubDownload = template.githubDownload
+        if (githubDownload == null) {
+            packy.logger.e("${template.name} has no githubDownload, skipping...")
+            return@forEach
+        }
 
-            while (entry != null) {
-                if ( entry.name.startsWith("$owner-$repo", true) && !entry.isDirectory) {
-                    entry.name.substringAfter("/${subPath?.let { "$it/" } ?: ""}").takeIf { it != entry!!.name }?.let { fileName ->
-                        zipOutputStream.putNextEntry(ZipEntry(fileName))
+        val (owner, repo, _, subPath) = githubDownload
 
-                        val buffer = ByteArray(1024)
-                        var len: Int
-                        while (zipStream.read(buffer).also { len = it } > 0)
-                            zipOutputStream.write(buffer, 0, len)
-
-                        zipOutputStream.closeEntry()
-                    }
-                }
-                zipStream.closeEntry()
-                entry = zipStream.nextEntry
-            }
-        }
-    }.onFailure { it.printStackTrace() }.also { zipStream.close() }
-}
+        // Create a new ZipInputStream from the byte array for each template
+        val zipStream = ZipInputStream(ByteArrayInputStream(responseBody))
 
-typealias TemplateIds = SortedSet<String>
+        runCatching {
+            // Ensure the template path is deleted before extraction
+            template.path.deleteRecursively()
 
-fun zipDirectory(directory: File, zipDest: File) {
-    val files = directory.walkTopDown().toList()
-    val buffer = ByteArray(1024)
+            // Use a ZipOutputStream to write the extracted files
+            ZipOutputStream(FileOutputStream(template.path.toFile())).use { zipOutputStream ->
+                var entry = zipStream.nextEntry
 
-    ZipOutputStream(FileOutputStream(zipDest)).use { zipOutputStream ->
-        files.forEach { file ->
-            val entryName = directory.toPath().relativize(file.toPath()).toString()
-            val entry = ZipEntry(entryName)
-            zipOutputStream.putNextEntry(entry)
+                while (entry != null) {
+                    if (entry.name.startsWith("$owner-$repo", ignoreCase = true) && !entry.isDirectory) {
+                        val entryName = entry.name.substringAfter("/${subPath?.let { "$it/" } ?: ""}")
+                        if (entryName != entry.name) {
+                            zipOutputStream.putNextEntry(ZipEntry(entryName))
 
-            if (file.isDirectory) return@forEach
+                            val buffer = ByteArray(1024)
+                            var len: Int
+                            while (zipStream.read(buffer).also { len = it } > 0) {
+                                zipOutputStream.write(buffer, 0, len)
+                            }
 
-            FileInputStream(file).use { inputStream ->
-                var len: Int
-                while (inputStream.read(buffer).also { len = it } > 0) {
-                    zipOutputStream.write(buffer, 0, len)
+                            zipOutputStream.closeEntry()
+                        }
+                    }
+                    zipStream.closeEntry()
+                    entry = zipStream.nextEntry
                 }
             }
-            zipOutputStream.closeEntry()
-        }
+        }.onFailure { it.printStackTrace() }
+        zipStream.close() // Ensure the ZipInputStream is closed properly
     }
 }
 
+typealias TemplateIds = SortedSet<String>
+
 fun unzip(zipFile: File, destDir: File) {
     runCatching {
         destDir.deleteRecursively()