diff --git a/src/main/kotlin/cz/dynawest/csvcruncher/Cruncher.kt b/src/main/kotlin/cz/dynawest/csvcruncher/Cruncher.kt index 5900c71..3bbec69 100644 --- a/src/main/kotlin/cz/dynawest/csvcruncher/Cruncher.kt +++ b/src/main/kotlin/cz/dynawest/csvcruncher/Cruncher.kt @@ -190,6 +190,10 @@ class Cruncher(private val options: Options2) { if (output.inputTableName != null) { sql = sql.replace(SQL_TABLE_PLACEHOLDER, quote(output.inputTableName)) } + else if (export.sqlQuery == null) { + throw CsvCruncherException("Default SQL is used (no -sql set), but no single input table name " + + "was determined automatically for output: $output\nTherefore the table placeholder can't be replaced.") + } // Create the parent dir. diff --git a/src/test/data/sample-multiFile-json+csv/report-2024-08-21-part2.csv b/src/test/data/sample-multiFile-json+csv/report-2024-08-21-part2.csv new file mode 100644 index 0000000..2af73e4 --- /dev/null +++ b/src/test/data/sample-multiFile-json+csv/report-2024-08-21-part2.csv @@ -0,0 +1,22 @@ +# formatted_address, product, address.country, address.road, address.city, address.state, device_id, groups, asset_id, battery, type, last_message_ts, nfc_id, protocol_version, environment, asset_name, first_paired_ts, temperature, service_status.health_status.modified_ts, service_status.health_status.status_comment, service_status.health_status.modified_by, service_status.health_status.status, manufacture_ts, location.lat, location.lon, tenant, paired, service_status.billing_status.status, service_status.health_status.status_code, serial_number, service_status.health_status.status_reason, service_status.decommissioning_status.modified_ts, service_status.decommissioning_status.status_comment, service_status.decommissioning_status.modified_by, service_status.decommissioning_status.status, service_status.billing_status.modified_ts, service_status.billing_status.status_comment, service_status.billing_status.modified_by, service_status.billing_status.status_reason, service_status.replacement_status.modified_ts, service_status.replacement_status.modified_by, service_status.replacement_status.status +"D 716, Brieux, Normandie, FR",AX.1C,FR,D 716,Brieux,Normandie,RVCF74,[...],VdHmp6,100,Asset,1724204321,04AA0432215284,11,ftx,33 84 6771 223-2,1551442919,9.5,1677834342,"Inherent connectivity problems or device is leaving area of bad connectivity +28-02-2023 04:19 to 02-03-2023 13:41",XnNmDs,Normal,1524434400,48.840759277344,-0.10459040105343,ftx,true,,,,,,,,,,,,,,, +"An der Bahn, Syke, Niedersachsen, DE",AX.1C,DE,An der Bahn,Syke,Niedersachsen,RVEup0,[...],ItqdI8,100,Asset,1724235712,04AA0432215284,11,ftx,31 85 4557 770-8,1524219408,20.5,1678964465,"Inherent connectivity problems or device is leaving area of bad connectivity +13-03-2023 08:29 to 15-03-2023 18:50",XnNmDs,Normal,1517266800,52.955226898193,8.824854850769,ftx,true,Billed,E01,,,,,,,,,,,,, +"Avenue de Quenneport, Val-de-la-Haye, Normandie, FR",AX.2A,FR,Avenue de Quenneport,Val-de-la-Haye,Normandie,RVLIF1,[...],giM7x5,86.152794,Asset,1724230476,04AA0432215284,11,ftx,37 80 7849 214-7,1574239859,42,1677491456,"Inherent connectivity problems or device is leaving area of bad connectivity +23-02-2023 21:59 to 26-02-2023 20:00",XnNmDs,Normal,1559858400,49.390979766846,1.0089439153671,ftx,true,,,ABCD5919@000040266,,,,,,,,,,,, +"Guessling-Hémering, Alsace–Champagne-Ardenne–Lorraine, FR",AX.2A,FR,,Guessling-Hémering,Alsace–Champagne-Ardenne–Lorraine,RVVEB3,[...],,84.37926,Device,1718179520,04AA0432215284,11,ftx,,1576581460,17.5,1722515486,"Inherent connectivity problems or device is leaving area of bad connectivity +17-03-2023 01:14 to 19-03-2023 16:32",XnNmDs,SuspectedMalfunction,1565042400,49.03865814209,6.6821365356445,ftx,false,,,ABCD5919@000053009,Unknown,,,,,,,,,,, +"Via Monte Sei Busi, Ronchi dei Legionari, FVG, IT",AX.2A,IT,Via Monte Sei Busi,Ronchi dei Legionari,FVG,RVXdm4,[...],sg0xl5,84.10641,Asset,1724241666,04AA0432215284,11,ftx,33 56 4576 336-8,1617782257,36,1674402289,"Inherent connectivity problems or device is leaving area of bad connectivity +19-01-2023 11:18 to 21-01-2023 17:23",XnNmDs,Normal,1602626400,45.837436676025,13.500262260437,ftx,true,,,ABCD5919@000102723,,,,,,,,,,,, +"Potsdamer Ring, Wittenberg, Sachsen-Anhalt, DE",AX.2A,DE,Potsdamer Ring,Wittenberg,Sachsen-Anhalt,RVepE1,[...],8KQjx8,85.470665,Asset,1724241308,04AA0432215284,11,ftx,37 80 6643 284-0,1611045290,27,1601556598,,XnNmDs,Normal,1588629600,51.868202209473,12.662959098816,ftx,true,,,ABCD5919@000083423,,,,,,,,,,,, +"Strada Ana Ipătescu, Târgu Jiu, RO",AX.1C,RO,Strada Ana Ipătescu,Târgu Jiu,,RVhfs0,[...],g1Ox49,98.56753,Asset,1724219819,04AA0432215284,11,ftx,37 80 0693 202-4,1564031118,19,1671107784,"Inherent connectivity problems or device is leaving area of bad connectivity +12-12-2022 08:26 to 15-12-2022 00:32",XnNmDs,Normal,1525644000,45.029064178467,23.282018661499,ftx,true,,,,,,,,,,,,,,, +"A 5, Rastatt, Baden-Württemberg, DE",AX.2A,DE,A 5,Rastatt,Baden-Württemberg,RVhzc2,[...],bHfye8,78.37653,Asset,1723616827,04AA0432215284,11,ftx,33 81 7837 810-9,1594719620,22.5,1692353208,"Inherent connectivity problems or device is leaving area of bad connectivity +19-03-2023 08:25 to 21-03-2023 02:36",XnNmDs,SuspectedMalfunction,1571090400,48.857955932617,8.2423343658447,ftx,true,,,ABCD5919@000063570,Unknown,,,,,,,,,,, +"B6415, Midlothian, Scotland, GB",AX.1C,GB,B6415,Midlothian,Scotland,RVl751,[...],KmQE6x,1.9781718,Asset,1724227619,04AA0432215284,11,ftx,12421,1528366632,12,1703839694,"Extremely low battery (2.66%), device might stop sending",XnNmDs,SuspectedMalfunction,1516057200,55.917793273926,-3.089676618576,ftx,true,Billed,E01,,Battery,,,,,,,,,,, +"Állomás utca, Pécs, HU",AX.3A,HU,Állomás utca,Pécs,,RVmxx2,[...],mbfxG7,61.882355,Asset,1724224658,E0XX2466C3C12347,11,ftx,33 80 0764 565-2,1710401285,36.5,1696165090,,XnNmDs,Normal,1692741600,46.059791564941,18.148193359375,ftx,true,,,ABCD5923@000011082,,,,,,,,,,,, +"Bridge Street, Litherland, England, GB",AX.2A,GB,Bridge Street,Litherland,England,RVnSn0,[...],,88.74488,Device,1679391051,04AA0432215284,11,ftx,,,10.5,1598964043,,XnNmDs,SuspectedMalfunction,1581980400,53.445652008057,-2.9961693286896,ftx,false,,,ABCD5919@000082339,Unknown,,,,,,,,,,, +"26, Berente, HU",AX.2A,HU,26,Berente,,RVnyo4,[...],iXaVk6,85.334236,Asset,1724241018,04AA0432215284,11,ftx,33 80 7844 032-2,1578833640,53,1721807233,"Inherent connectivity problems or device is leaving area of bad connectivity +31-03-2022 01:41 to 02-04-2022 19:33",XnNmDs,Normal,1565301600,48.246627807617,20.660512924194,ftx,true,,,ABCD5919@000056178,,,,,,,,,,,, +"SP115, Cellole, CAM, IT",AX.2A,IT,SP115,Cellole,CAM,RVoNx3,[...],,32.40109,Device,1671769918,04AA0432215284,11,ftx,,1601891676,11.5,1675858655,"Inherent connectivity problems or device is leaving area of bad connectivity diff --git a/src/test/data/sample-multiFile-json+csv/report-2024-08-21-part3.csv b/src/test/data/sample-multiFile-json+csv/report-2024-08-21-part3.csv new file mode 100644 index 0000000..7a1afa1 --- /dev/null +++ b/src/test/data/sample-multiFile-json+csv/report-2024-08-21-part3.csv @@ -0,0 +1,18 @@ +# formatted_address, product, address.country, address.road, address.city, address.state, device_id, groups, serial_number, asset_id, battery, type, last_message_ts, nfc_id, protocol_version, environment, asset_name, first_paired_ts, temperature, service_status.health_status.modified_ts, service_status.health_status.status_comment, service_status.health_status.modified_by, service_status.health_status.status, manufacture_ts, location.lat, location.lon, tenant, paired, service_status.billing_status.status, service_status.health_status.status_code, service_status.health_status.status_reason, service_status.billing_status.modified_ts, service_status.billing_status.modified_by, service_status.billing_status.status_reason, service_status.decommissioning_status.modified_ts, service_status.decommissioning_status.status_comment, service_status.decommissioning_status.modified_by, service_status.decommissioning_status.status, service_status.billing_status.status_comment +"Plitzenweg, Gemeinde Weißenstein, Kärnten, AT",AX.2A,AT,Plitzenweg,Gemeinde Weißenstein,Kärnten,xcoQ02,[...],ABCD5919@000092916,dlXiw8,85.879944,Asset,1724204260,04AEC455556280,11,ftx,22 81 7837 589-9,1601640837,13.5,1679994490,"Inherent connectivity problems or device is leaving area of bad connectivity +07-12-2022 05:18 to 09-12-2022 23:27",tnNmDs,Normal,1587420000,46.654972076416,13.768341064453,ftx,true,,,,,,,,,,, +"Rotenbrook, Itzehoe, Schleswig-Holstein, DE",AX.1C,DE,Rotenbrook,Itzehoe,Schleswig-Holstein,xcpgI0,[...],,16qKt8,99.931786,Asset,1724201870,04AEC455556280,11,ftx,22 81 7837 589-9,1516267113,18.5,1679302532,"Inherent connectivity problems or device is leaving area of bad connectivity +16-03-2023 21:03 to 19-03-2023 20:50",tnNmDs,Normal,1513378800,53.923515319824,9.5094804763794,ftx,true,Billed,E01,,,,,,,,, +"Straker Street, South Tyneside, England, GB",AX.2A,GB,Straker Street,South Tyneside,England,xcuZE0,[...],ABCD5919@000057967,,0,Device,1711194601,04AEC455556280,11,ftx,,,7.5,1706788919,"Extremely low battery (2.8%), device might stop sending",tnNmDs,SuspectedMalfunction,1567116000,54.977432250977,-1.4763457775116,ftx,false,,,Battery,,,,,,,, +"Via Fontevivo, La Spezia, LIG, IT",AX.2A,IT,Via Fontevivo,La Spezia,LIG,xcw7i3,[...],ABCD5919@000074206,hnIiL5,85.197815,Asset,1724239941,04AEC455556280,11,ftx,22 81 7837 589-9,1595494281,35,1658845136,Lost 60% battery charge in 24h (once in last 30 days),tnNmDs,SuspectedMalfunction,1573081200,44.123149871826,9.8390140533447,ftx,true,,,Battery,,,,,,,, +"Rue Jacques Callot, Forbach, Alsace–Champagne-Ardenne–Lorraine, FR",AX.2A,FR,Rue Jacques Callot,Forbach,Alsace–Champagne-Ardenne–Lorraine,xcwsc1,[...],ABCD5919@000069112,3PCyM6,84.652115,Asset,1724241207,04AEC455556280,11,ftx,22 81 7837 589-9,1589535415,22.5,1629047403,"Inherent connectivity problems or device is leaving area of bad connectivity +12-08-2021 05:10 to 15-08-2021 05:08",tnNmDs,Normal,1575414000,49.178470611572,6.8637938499451,ftx,true,,,,,,,,,,, +"Austraße, Vohburg an der Donau, Bayern, DE",AX.1C,DE,Austraße,Vohburg an der Donau,Bayern,xd1J64,[...],,3pzNY5,100,Asset,1724229006,04AEC455556280,11,ftx,22 81 7837 589-9,1553754466,20.5,1678356629,"Inherent connectivity problems or device is leaving area of bad connectivity +06-03-2023 04:51 to 08-03-2023 16:52",tnNmDs,Normal,1523397600,48.763473510742,11.600967407227,ftx,true,,,,,,,,,,, +"Hurn Road, Marholm, England, GB",AX.2A,GB,Hurn Road,Marholm,England,xd4iE4,[...],ABCD5919@000060781,HJiJz8,26.398361,Asset,1724084998,04AEC455556280,11,ftx,22 81 7837 589-9,1612185199,17.5,1720159586,"Inherent connectivity problems or device is leaving area of bad connectivity +07-01-2023 04:42 to 09-01-2023 16:28",tnNmDs,SuspectedMalfunction,1569967200,52.619388580322,-0.29891300201416,ftx,true,,,Unknown,,,,,,,, +"Church Lane, Lichfield, England, GB",AX.2A,GB,Church Lane,Lichfield,England,xd62a1,[...],ABCD5919@000057579,J5JnE7,86.289215,Asset,1724241789,04AEC455556280,11,ftx,22 81 7837 589-9,1583227580,19.5,1678611945,"Inherent connectivity problems or device is leaving area of bad connectivity +08-03-2023 21:24 to 11-03-2023 10:04",tnNmDs,Normal,1567548000,52.746757507324,-1.8866622447968,ftx,true,,,,,,,,,,, +"Straße 1100, Marl, Nordrhein-Westfalen, DE",AX.2A,DE,Straße 1100,Marl,Nordrhein-Westfalen,xdDGa0,[...],ABCD5919@000047881,0AvJA7,67.46248,Asset,1724235392,04AEC455556280,11,ftx,AXCU3500758,1597908723,23,1597994896,,tnNmDs,Normal,1562536800,51.680908203125,7.1065511703491,ftx,true,,,,,,,,,,, +"Vratimov, Moravskoslezsko, CZ",AX.3A,CZ,,Vratimov,Moravskoslezsko,xdE381,[...],ABCD5923@000014219,r23Vl8,88,Asset,1722327015,E0022403445619C,11,ftx,22 81 7837 589-9,1705744552,16.5,1714633361,,tnNmDs,UnderMonitoring,1694728800,49.784133911133,18.323900222778,ftx,true,,,,,,,,,,, +"Anilinfabrikstraße, Ludwigshafen am Rhein, Rheinland-Pfalz, DE",AX.2A,DE,Anilinfabrikstraße,Ludwigshafen am Rhein,Rheinland-Pfalz,xdFnL2,[...],ABCD5919@000069617,jsOzg7,86.698494,Asset,1724199832,04AEC455556280,11,ftx,22 81 7837 589-9,1587289775,19,1699353470,"Inherent connectivity problems or device is leaving area of bad connectivity diff --git a/src/test/kotlin/cz/dynawest/csvcruncher/it/OptionsCombinationsTest.kt b/src/test/kotlin/cz/dynawest/csvcruncher/it/OptionsCombinationsTest.kt index e3a28c5..b2e7c32 100644 --- a/src/test/kotlin/cz/dynawest/csvcruncher/it/OptionsCombinationsTest.kt +++ b/src/test/kotlin/cz/dynawest/csvcruncher/it/OptionsCombinationsTest.kt @@ -87,7 +87,7 @@ class OptionsCombinationsTest { @Test fun combineInputFile() { - val command = //"--json=entries" + + val command = " | --rowNumbers" + " | --combineInputs" + " | -in | src/test/data/eapBuilds.csv" + @@ -103,6 +103,34 @@ class OptionsCombinationsTest { assertEquals(columnNames[8].lowercase(), "warmupslower") } + @Test + fun combineInputDir_defaultSql() { + val command = + " | -in | src/test/data/sample-multiFile-all" + + " | --combineInputs" + + " | -out | target/testResults/combineInputDir.csv" + CsvCruncherTestUtils.runCruncherWithArguments(command) + val resultCsv = Paths.get("target/testResults/combineInputDir.csv").toFile() + assertTrue(resultCsv.exists()) + val columnNames = parseColumnsFromFirstCsvLine(resultCsv) + assertEquals(columnNames.size.toLong(), 9, "Column names fit") + assertEquals(columnNames[8].lowercase(), "warmupslower") + } + + @Test + fun combineInputDir_JsonAndCsv_defaultSql_issue149() { + val command = + " | -in | src/test/data/sample-multiFile-json+csv" + + " | --combineInputs" + + " | -out | target/testResults/combineInputDir_json+csv.csv" + CsvCruncherTestUtils.runCruncherWithArguments(command) + val resultCsv = Paths.get("target/testResults/combineInputDir_json+csv.csv").toFile() + assertTrue(resultCsv.exists()) + val columnNames = parseColumnsFromFirstCsvLine(resultCsv) + assertEquals(columnNames.size.toLong(), 9, "Column names fit") + assertEquals(columnNames[8].lowercase(), "warmupslower") + } + @Test fun combineInputFiles_sort() { val command = "--json=entries" +