From ecc00e1b601db1b50a9788f1764d2d1875f88c21 Mon Sep 17 00:00:00 2001 From: kangert Date: Thu, 25 Jul 2024 00:58:35 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=20=E5=AE=8C=E5=96=84?= =?UTF-8?q?=E8=BE=93=E5=87=BA=E8=8A=82=E7=82=B9=E5=8F=8A=E8=BE=93=E5=87=BA?= =?UTF-8?q?=E9=A1=B9=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kspider/executor/node/OutputExecutor.java | 37 +++++++++++++------ .../node/event/OutputEventHandler.java | 10 +++-- .../node/event/OutputEventPublisher.java | 9 ++++- 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/kspider-core/src/main/java/top/kangert/kspider/executor/node/OutputExecutor.java b/kspider-core/src/main/java/top/kangert/kspider/executor/node/OutputExecutor.java index 2d0b1a7..c1c9f89 100644 --- a/kspider-core/src/main/java/top/kangert/kspider/executor/node/OutputExecutor.java +++ b/kspider-core/src/main/java/top/kangert/kspider/executor/node/OutputExecutor.java @@ -1,6 +1,7 @@ package top.kangert.kspider.executor.node; import cn.hutool.core.exceptions.ExceptionUtil; +import cn.hutool.core.lang.TypeReference; import cn.hutool.json.JSONConfig; import cn.hutool.json.JSONUtil; import top.kangert.kspider.constant.Constants; @@ -33,24 +34,24 @@ public class OutputExecutor implements NodeExecutor, SpiderListener { /** - * 输出方式 + * 输出其他变量 */ - String OUTPUT_TYPE = "output-type"; + String OUTPUT_OTHERS = "output-others"; /** - * 输出其他变量 + * 输出项 */ - String OUTPUT_OTHERS = "output-others"; + String OUTPUT_ITEM = "output-item"; /** * 输出项的名称 */ - String OUTPUT_NAME = "output-name"; + String OUTPUT_NAME = "key"; /** * 输出项的值 */ - String OUTPUT_VALUE = "output-value"; + String OUTPUT_VALUE = "value"; @Resource private ExpressionEngine expressionEngine; @@ -99,11 +100,13 @@ private List getOutputItems(Map variabl SpiderNode node) { List outputItems = new ArrayList<>(); // 获取用户设置的所有输出项 - List> items = node.getJsonArrayProperty(OUTPUT_NAME, OUTPUT_VALUE); + List> items = node.getJsonArrayProperty(OUTPUT_ITEM); for (Map item : items) { + String outputItem = item.get(OUTPUT_ITEM); + Map kvMap = JSONUtil.toBean(outputItem, new TypeReference>() {}, false); Object value = null; - String itemName = item.get(OUTPUT_NAME); - String itemValue = item.get(OUTPUT_VALUE); + String itemName = kvMap.get(OUTPUT_NAME); + String itemValue = kvMap.get(OUTPUT_VALUE); try { value = expressionEngine.execute(itemValue, variables); context.pause(node.getNodeId(), WebSocketEvent.COMMON_EVENT, itemName, value); @@ -185,13 +188,21 @@ public List configItems() { outputTypeConfigItem.add(csvItem); // 输出方式 - ConfigItem outputType = new ConfigItem("输出方式", ConfigItem.ComponentType.EL_MULT_SELECT, ConfigItem.DataType.LIST_STRING, OUTPUT_TYPE, "请选择输出方式", Arrays.asList(OutputType.CSV.getVariableName()), null, outputTypeConfigItem, true); + ConfigItem outputType = new ConfigItem("输出方式", ConfigItem.ComponentType.EL_MULT_SELECT, ConfigItem.DataType.LIST_STRING, Constants.OUTPUT_TYPE, "请选择输出方式", Arrays.asList(OutputType.CSV.getVariableName()), null, outputTypeConfigItem, true); configItemList.add(outputType); // 输出文件名称（输出方式为CSV时生效） - ConfigItem csvName = new ConfigItem("文件名称", ConfigItem.ComponentType.EL_INPUT, ConfigItem.DataType.LIST_STRING, "csvName", "请输入CSV文件名称", "", null, outputTypeConfigItem, false); + ConfigItem csvName = new ConfigItem("文件名称", ConfigItem.ComponentType.EL_INPUT, ConfigItem.DataType.STRING, "csvName", "请输入CSV文件名称", "", null, outputTypeConfigItem, false); configItemList.add(csvName); + // 输出文件编码（输出方式为CSV时生效） + List csvEncodingChoice = new ArrayList<>(); + csvEncodingChoice.add(new ConfigItem.SelectItem("UTF-8", "UTF-8", ConfigItem.DataType.STRING)); + csvEncodingChoice.add(new ConfigItem.SelectItem("UTF-8BOM", "UTF-8BOM", ConfigItem.DataType.STRING)); + csvEncodingChoice.add(new ConfigItem.SelectItem("GBK", "GBK", ConfigItem.DataType.STRING)); + ConfigItem csvEncoding = new ConfigItem("文件编码", ConfigItem.ComponentType.EL_SELECT, ConfigItem.DataType.STRING, "csvEncoding", "请选择CSV文件编码", csvEncodingChoice.get(0).getValue(), null, csvEncodingChoice, false); + configItemList.add(csvEncoding); + // 是否输出所有变量 List outputAllVar = new ArrayList<>(); outputAllVar.add(new ConfigItem.SelectItem("是", "true", ConfigItem.DataType.BOOLEAN)); @@ -199,6 +210,10 @@ public List configItems() { ConfigItem outputAllVarConfigItem = new ConfigItem("输出全部参数", ConfigItem.ComponentType.EL_SWITCH, ConfigItem.DataType.BOOLEAN, OUTPUT_OTHERS, "", false, null, outputAllVar); configItemList.add(outputAllVarConfigItem); + // 输出项配置（键值对） + ConfigItem outputItem = new ConfigItem("输出项", ConfigItem.ComponentType.CUSTOM_MULT_KEY_VALUE, ConfigItem.DataType.LIST_MAP, OUTPUT_ITEM, "请输入输出项", new ArrayList<>(), null, outputTypeConfigItem, false); + configItemList.add(outputItem); + return configItemList; } diff --git a/kspider-core/src/main/java/top/kangert/kspider/executor/node/event/OutputEventHandler.java b/kspider-core/src/main/java/top/kangert/kspider/executor/node/event/OutputEventHandler.java index cdcbbc9..37ab6c9 100644 --- a/kspider-core/src/main/java/top/kangert/kspider/executor/node/event/OutputEventHandler.java +++ b/kspider-core/src/main/java/top/kangert/kspider/executor/node/event/OutputEventHandler.java @@ -15,6 +15,7 @@ import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; +import cn.hutool.core.date.DateUtil; import cn.hutool.core.exceptions.ExceptionUtil; import cn.hutool.core.util.RandomUtil; import cn.hutool.core.util.StrUtil; @@ -140,9 +141,12 @@ private void outputCSV(OutputEventBean eventBean) { printer = cachePrinter.get(key); if (printer == null) { CSVFormat format = CSVFormat.DEFAULT.withHeader(headers.toArray(new String[headers.size()])); - String fileName = spiderConfig.getWorkspace() + File.separator + "files" + File.separator - + context.getFlowId() + "_" + context.getTaskId() + File.separator + csvName + ".csv"; - FileOutputStream os = new FileOutputStream(fileName); + String fileName = spiderConfig.getWorkspace() + File.separator + "files" + File.separator + node.getNodeId() + "_" + DateUtil.format(DateUtil.date(), "yyyyMMddHHmmss") + File.separator + csvName + ".csv"; + File csvFile = new File(fileName); + if (csvFile.getParentFile() != null) { + csvFile.getParentFile().mkdirs(); + } + FileOutputStream os = new FileOutputStream(csvFile); String csvEncoding = node.getJsonProperty(OUTPUT_CSV_ENCODING); if ("UTF-8BOM".equals(csvEncoding)) { csvEncoding = csvEncoding.substring(0, 5); diff --git a/kspider-core/src/main/java/top/kangert/kspider/executor/node/event/OutputEventPublisher.java b/kspider-core/src/main/java/top/kangert/kspider/executor/node/event/OutputEventPublisher.java index 13ade13..5ba1650 100644 --- a/kspider-core/src/main/java/top/kangert/kspider/executor/node/event/OutputEventPublisher.java +++ b/kspider-core/src/main/java/top/kangert/kspider/executor/node/event/OutputEventPublisher.java @@ -13,6 +13,8 @@ import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Component; +import cn.hutool.core.convert.Convert; + import java.util.List; @Component @@ -30,9 +32,12 @@ public class OutputEventPublisher { */ public void publish(SpiderContext context, SpiderNode node, List outputItems) { OutputType[] outputTypes = OutputType.values(); + List outputTypeArr = Convert.toList(String.class, node.getJsonProperty(Constants.OUTPUT_TYPE)); for (OutputType outputType : outputTypes) { - if (Constants.YES.equals(node.getJsonProperty(outputType.getVariableName()))) { - eventPublisher.publishEvent(new OutputEventBean(context, node, outputItems, outputType.getVariableName())); + for (String userSelected : outputTypeArr) { + if (userSelected.equals(outputType.getVariableName())) { + eventPublisher.publishEvent(new OutputEventBean(context, node, outputItems, outputType.getVariableName())); + } } } }