优化完善输出节点及输出项配置

kenuoseclab · Jul 24, 2024 · ecc00e1 · ecc00e1
1 parent e6b4373
commit ecc00e1
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 16 deletions.
diff --git a/kspider-core/src/main/java/top/kangert/kspider/executor/node/OutputExecutor.java b/kspider-core/src/main/java/top/kangert/kspider/executor/node/OutputExecutor.java
@@ -1,6 +1,7 @@
 package top.kangert.kspider.executor.node;
 
 import cn.hutool.core.exceptions.ExceptionUtil;
+import cn.hutool.core.lang.TypeReference;
 import cn.hutool.json.JSONConfig;
 import cn.hutool.json.JSONUtil;
 import top.kangert.kspider.constant.Constants;
@@ -33,24 +34,24 @@
 public class OutputExecutor implements NodeExecutor, SpiderListener {
 
     /**
-     * 输出方式
+     * 输出其他变量
      */
-    String OUTPUT_TYPE = "output-type";
+    String OUTPUT_OTHERS = "output-others";
 
     /**
-     * 输出其他变量
+     * 输出项
      */
-    String OUTPUT_OTHERS = "output-others";
+    String OUTPUT_ITEM = "output-item";
 
     /**
      * 输出项的名称
      */
-    String OUTPUT_NAME = "output-name";
+    String OUTPUT_NAME = "key";
 
     /**
      * 输出项的值
      */
-    String OUTPUT_VALUE = "output-value";
+    String OUTPUT_VALUE = "value";
 
     @Resource
     private ExpressionEngine expressionEngine;
@@ -99,11 +100,13 @@ private List<SpiderOutput.OutputItem> getOutputItems(Map<String, Object> variabl
             SpiderNode node) {
         List<SpiderOutput.OutputItem> outputItems = new ArrayList<>();
         // 获取用户设置的所有输出项
-        List<Map<String, String>> items = node.getJsonArrayProperty(OUTPUT_NAME, OUTPUT_VALUE);
+        List<Map<String, String>> items = node.getJsonArrayProperty(OUTPUT_ITEM);
         for (Map<String, String> item : items) {
+            String outputItem = item.get(OUTPUT_ITEM);
+            Map<String, String> kvMap = JSONUtil.toBean(outputItem, new TypeReference<Map<String, String>>() {}, false);
             Object value = null;
-            String itemName = item.get(OUTPUT_NAME);
-            String itemValue = item.get(OUTPUT_VALUE);
+            String itemName = kvMap.get(OUTPUT_NAME);
+            String itemValue = kvMap.get(OUTPUT_VALUE);
             try {
                 value = expressionEngine.execute(itemValue, variables);
                 context.pause(node.getNodeId(), WebSocketEvent.COMMON_EVENT, itemName, value);
@@ -185,20 +188,32 @@ public List<ConfigItem> configItems() {
         outputTypeConfigItem.add(csvItem);
 
         // 输出方式
-        ConfigItem outputType = new ConfigItem("输出方式", ConfigItem.ComponentType.EL_MULT_SELECT, ConfigItem.DataType.LIST_STRING, OUTPUT_TYPE, "请选择输出方式", Arrays.asList(OutputType.CSV.getVariableName()), null, outputTypeConfigItem, true);
+        ConfigItem outputType = new ConfigItem("输出方式", ConfigItem.ComponentType.EL_MULT_SELECT, ConfigItem.DataType.LIST_STRING, Constants.OUTPUT_TYPE, "请选择输出方式", Arrays.asList(OutputType.CSV.getVariableName()), null, outputTypeConfigItem, true);
         configItemList.add(outputType);
 
         // 输出文件名称（输出方式为CSV时生效）
-        ConfigItem csvName = new ConfigItem("文件名称", ConfigItem.ComponentType.EL_INPUT, ConfigItem.DataType.LIST_STRING, "csvName", "请输入CSV文件名称", "", null, outputTypeConfigItem, false);
+        ConfigItem csvName = new ConfigItem("文件名称", ConfigItem.ComponentType.EL_INPUT, ConfigItem.DataType.STRING, "csvName", "请输入CSV文件名称", "", null, outputTypeConfigItem, false);
         configItemList.add(csvName);
 
+        // 输出文件编码（输出方式为CSV时生效）
+        List<ConfigItem.SelectItem> csvEncodingChoice = new ArrayList<>();
+        csvEncodingChoice.add(new ConfigItem.SelectItem("UTF-8", "UTF-8", ConfigItem.DataType.STRING));
+        csvEncodingChoice.add(new ConfigItem.SelectItem("UTF-8BOM", "UTF-8BOM", ConfigItem.DataType.STRING));
+        csvEncodingChoice.add(new ConfigItem.SelectItem("GBK", "GBK", ConfigItem.DataType.STRING));
+        ConfigItem csvEncoding = new ConfigItem("文件编码", ConfigItem.ComponentType.EL_SELECT, ConfigItem.DataType.STRING, "csvEncoding", "请选择CSV文件编码", csvEncodingChoice.get(0).getValue(), null, csvEncodingChoice, false);
+        configItemList.add(csvEncoding);
+
         // 是否输出所有变量
         List<ConfigItem.SelectItem> outputAllVar = new ArrayList<>();
         outputAllVar.add(new ConfigItem.SelectItem("是", "true", ConfigItem.DataType.BOOLEAN));
         outputAllVar.add(new ConfigItem.SelectItem("否", "false", ConfigItem.DataType.BOOLEAN));
         ConfigItem outputAllVarConfigItem = new ConfigItem("输出全部参数", ConfigItem.ComponentType.EL_SWITCH, ConfigItem.DataType.BOOLEAN, OUTPUT_OTHERS, "", false, null, outputAllVar);
         configItemList.add(outputAllVarConfigItem);
 
+        // 输出项配置（键值对）
+        ConfigItem outputItem = new ConfigItem("输出项", ConfigItem.ComponentType.CUSTOM_MULT_KEY_VALUE, ConfigItem.DataType.LIST_MAP, OUTPUT_ITEM, "请输入输出项", new ArrayList<>(), null, outputTypeConfigItem, false);
+        configItemList.add(outputItem);
+
         return configItemList;
     }
 

diff --git a/kspider-core/src/main/java/top/kangert/kspider/executor/node/event/OutputEventHandler.java b/kspider-core/src/main/java/top/kangert/kspider/executor/node/event/OutputEventHandler.java
@@ -15,6 +15,7 @@
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Component;
 
+import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.exceptions.ExceptionUtil;
 import cn.hutool.core.util.RandomUtil;
 import cn.hutool.core.util.StrUtil;
@@ -140,9 +141,12 @@ private void outputCSV(OutputEventBean eventBean) {
                     printer = cachePrinter.get(key);
                     if (printer == null) {
                         CSVFormat format = CSVFormat.DEFAULT.withHeader(headers.toArray(new String[headers.size()]));
-                        String fileName = spiderConfig.getWorkspace() + File.separator + "files" + File.separator
-                                + context.getFlowId() + "_" + context.getTaskId() + File.separator + csvName + ".csv";
-                        FileOutputStream os = new FileOutputStream(fileName);
+                        String fileName = spiderConfig.getWorkspace() + File.separator + "files" + File.separator + node.getNodeId() + "_" + DateUtil.format(DateUtil.date(), "yyyyMMddHHmmss") + File.separator + csvName + ".csv";
+                        File csvFile = new File(fileName);
+                        if (csvFile.getParentFile() != null) {
+                            csvFile.getParentFile().mkdirs();
+                        }
+                        FileOutputStream os = new FileOutputStream(csvFile);
                         String csvEncoding = node.getJsonProperty(OUTPUT_CSV_ENCODING);
                         if ("UTF-8BOM".equals(csvEncoding)) {
                             csvEncoding = csvEncoding.substring(0, 5);

diff --git a/kspider-core/src/main/java/top/kangert/kspider/executor/node/event/OutputEventPublisher.java b/kspider-core/src/main/java/top/kangert/kspider/executor/node/event/OutputEventPublisher.java
@@ -13,6 +13,8 @@
 import org.springframework.context.ApplicationEventPublisher;
 import org.springframework.stereotype.Component;
 
+import cn.hutool.core.convert.Convert;
+
 import java.util.List;
 
 @Component
@@ -30,9 +32,12 @@ public class OutputEventPublisher {
      */
     public void publish(SpiderContext context, SpiderNode node, List<SpiderOutput.OutputItem> outputItems) {
         OutputType[] outputTypes = OutputType.values();
+        List<String> outputTypeArr = Convert.toList(String.class, node.getJsonProperty(Constants.OUTPUT_TYPE));
         for (OutputType outputType : outputTypes) {
-            if (Constants.YES.equals(node.getJsonProperty(outputType.getVariableName()))) {
-                eventPublisher.publishEvent(new OutputEventBean(context, node, outputItems, outputType.getVariableName()));
+            for (String userSelected : outputTypeArr) {
+                if (userSelected.equals(outputType.getVariableName())) {
+                    eventPublisher.publishEvent(new OutputEventBean(context, node, outputItems, outputType.getVariableName()));
+                }
             }
         }
     }