CM-29952 - Fix creation of many scan jobs on many on-save events for a short period of time (#31)

Author: MarshalX

CHANGELOG.md

@@ -4,9 +4,14 @@
 
 ## [Unreleased]
 
+## [1.0.1] - 2023-12-12
+
+Fix the creation of many scan jobs on many on-save events for a short period of time.
+
 ## [1.0.0] - 2023-12-05
 
 The first public release of the plugin.
 
+[1.0.1]: https://github.com/cycodehq/intellij-platform-plugin/releases/tag/v1.0.1
 [1.0.0]: https://github.com/cycodehq/intellij-platform-plugin/releases/tag/v1.0.0
 [Unreleased]: https://github.com/cycodehq/intellij-platform-plugin/compare/v1.0.0...HEAD

gradle.properties

@@ -4,7 +4,7 @@ pluginGroup = com.cycode.plugin
 pluginName = Cycode (beta)
 pluginRepositoryUrl = https://github.com/cycodehq/intellij-platform-plugin
 # SemVer format -> https://semver.org
-pluginVersion = 1.0.0
+pluginVersion = 1.0.1
 
 # Supported build number ranges and IntelliJ Platform versions -> https://plugins.jetbrains.com/docs/intellij/build-number-ranges.html
 pluginSinceBuild = 211.1

src/main/kotlin/com/cycode/plugin/Consts.kt

@@ -9,7 +9,9 @@ class Consts {
         val DEFAULT_CLI_PATH = "$PLUGIN_PATH/cycode" + if (SystemInfo.isWindows) ".exe" else ""
         const val CLI_GITHUB_ORG = "cycodehq"
         const val CLI_GITHUB_REPO = "cycode-cli"
-        const val CLI_GITHUB_TAG = "v1.5.0"
+        const val CLI_GITHUB_TAG = "v1.6.0"
         const val CLI_CHECK_NEW_VERSION_EVERY_SEC = 24 * 60 * 60 // 24 hours
+        const val PLUGIN_AUTO_SAVE_FLUSH_INITIAL_DELAY_SEC = 0L
+        const val PLUGIN_AUTO_SAVE_FLUSH_DELAY_SEC = 5L
     }
 }

src/main/kotlin/com/cycode/plugin/listeners/FileSaveListener.kt

@@ -1,5 +1,6 @@
 package com.cycode.plugin.listeners
 
+import com.cycode.plugin.Consts
 import com.cycode.plugin.CycodeBundle
 import com.cycode.plugin.services.cli
 import com.cycode.plugin.services.pluginSettings
@@ -11,31 +12,65 @@ import com.intellij.openapi.fileEditor.FileDocumentManagerListener
 import com.intellij.openapi.progress.ProgressIndicator
 import com.intellij.openapi.progress.Task
 import com.intellij.openapi.project.Project
+import com.intellij.util.concurrency.AppExecutorUtil
+import java.io.File
+import java.util.concurrent.TimeUnit
 
 
 class FileSaveListener(private val project: Project) : FileDocumentManagerListener {
     private val cliService = cli(project)
     private val pluginState = pluginState()
     private val pluginSettings = pluginSettings()
+    private val collectedPathsToScan = mutableSetOf<String>() // we use a set to avoid duplicates
 
-    override fun beforeDocumentSaving(document: Document) {
-        thisLogger().debug("FileSaveListener.beforeDocumentSaving")
+    init {
+        scheduleScanPathsFlush()
+    }
 
-        if (!pluginSettings.scanOnSave || !pluginState.cliAuthed) {
+    private fun scanPathsFlush() {
+        val pathsToScan = excludeNotExistingPaths(collectedPathsToScan.toMutableList())
+        collectedPathsToScan.clear()
+
+        if (pathsToScan.isEmpty() || !pluginState.cliAuthed) {
             return
         }
 
-        val file = FileDocumentManager.getInstance().getFile(document) ?: return
-        val filePath = file.canonicalFile?.canonicalPath ?: return
-
         object : Task.Backgroundable(project, CycodeBundle.message("fileScanning"), true) {
             override fun run(indicator: ProgressIndicator) {
                 cliService.cliShouldDestroyCallback = { indicator.isCanceled }
 
-                thisLogger().debug("Start scanning file: $filePath")
-                cliService.scanFileSecrets(filePath, false)
-                thisLogger().debug("Finish scanning file: $filePath")
+                thisLogger().debug("Start scanning paths: $pathsToScan")
+                cliService.scanPathsSecrets(pathsToScan, false)
+                thisLogger().debug("Finish scanning paths: $pathsToScan")
             }
         }.queue()
     }
+
+    private fun excludeNotExistingPaths(paths: List<String>): List<String> {
+        return paths.filter { File(it).exists() }
+    }
+
+    private fun scheduleScanPathsFlush() {
+        AppExecutorUtil.getAppScheduledExecutorService().scheduleWithFixedDelay(
+            {
+                scanPathsFlush()
+            },
+            Consts.PLUGIN_AUTO_SAVE_FLUSH_INITIAL_DELAY_SEC,
+            Consts.PLUGIN_AUTO_SAVE_FLUSH_DELAY_SEC,
+            TimeUnit.SECONDS
+        )
+    }
+
+    override fun beforeDocumentSaving(document: Document) {
+        thisLogger().debug("FileSaveListener.beforeDocumentSaving")
+
+        if (!pluginSettings.scanOnSave || !pluginState.cliAuthed) {
+            return
+        }
+
+        val file = FileDocumentManager.getInstance().getFile(document) ?: return
+        val filePath = file.canonicalFile?.canonicalPath ?: return
+
+        collectedPathsToScan.add(filePath)
+    }
 }

src/main/kotlin/com/cycode/plugin/services/CliService.kt

@@ -147,25 +147,29 @@ class CliService(private val project: Project) {
         }
     }
 
-    private inline fun <reified T> scanFile(filePath: String, scanType: CliScanType): CliResult<T>? {
+    private inline fun <reified T> scanPaths(paths: List<String>, scanType: CliScanType): CliResult<T>? {
         val scanTypeString = scanType.name.toLowerCase()
         val result = cli
             .executeCommand<T>(
                 "scan",
                 "-t",
                 scanTypeString,
                 "path",
-                filePath,
+                *getPathsAsArguments(paths),
                 shouldDestroyCallback = cliShouldDestroyCallback
             )
 
         return processResult(result)
     }
 
-    fun scanFileSecrets(filePath: String, onDemand: Boolean = true) {
-        val results = scanFile<SecretScanResult>(filePath, CliScanType.Secret)
+    private fun getPathsAsArguments(paths: List<String>): Array<String> {
+        return paths.toTypedArray()
+    }
+
+    fun scanPathsSecrets(paths: List<String>, onDemand: Boolean = true) {
+        val results = scanPaths<SecretScanResult>(paths, CliScanType.Secret)
         if (results == null) {
-            thisLogger().warn("Failed to scan file: $filePath")
+            thisLogger().warn("Failed to scan paths: $paths")
             return
         }
 

src/main/kotlin/com/cycode/plugin/services/CycodeService.kt

@@ -67,7 +67,7 @@ class CycodeService(val project: Project) {
                 }
 
                 cliService.cliShouldDestroyCallback = { indicator.isCanceled }
-                cliService.scanFileSecrets(filepath)
+                cliService.scanPathsSecrets(listOf(filepath))
             }
         }.queue()
     }
@@ -87,7 +87,7 @@ class CycodeService(val project: Project) {
                 // the right way: apply "ignore rules" in the local results db of the plugin
                 // the disadvantage of the right way: we rewrite code that already exists in CLI in every plugin...
                 // TODO(MarshalX): think about what we can do from CLI side
-                cliService.scanFileSecrets(filepath)
+                cliService.scanPathsSecrets(listOf(filepath))
             }
         }.queue()
     }