From 0c28a3abd9b388913ed10e5b43eb0ea659406c21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Wed, 14 Feb 2024 16:01:23 -0500 Subject: [PATCH] Trip stops generating > OOM > use SQL prepared statements --- build.gradle | 2 + .../java/org/mtransit/parser/FileUtils.java | 23 + src/main/java/org/mtransit/parser/MTLog.java | 13 + .../java/org/mtransit/parser/db/DBUtils.kt | 469 ++++++++++-------- .../org/mtransit/parser/db/DumpDbUtils.kt | 31 +- .../java/org/mtransit/parser/db/SQLUtils.kt | 17 +- .../org/mtransit/parser/gtfs/GReader.java | 205 ++++---- .../mtransit/parser/gtfs/data/GDropOffType.kt | 2 +- .../org/mtransit/parser/gtfs/data/GIDs.kt | 21 +- .../org/mtransit/parser/gtfs/data/GSpec.java | 10 +- .../org/mtransit/parser/gtfs/data/GTrip.kt | 25 +- 11 files changed, 465 insertions(+), 353 deletions(-) diff --git a/build.gradle b/build.gradle index 1c731ea..06d2ad6 100644 --- a/build.gradle +++ b/build.gradle @@ -26,6 +26,8 @@ dependencies { //noinspection DependencyNotationArgument // fixed in AGP 8.3.0 https://issuetracker.google.com/issues/295282939 implementation(libs.bundles.kotlin) + implementation(libs.androidx.collection) + api(libs.bundles.commons) api(libs.xerial.SQLiteJDBC) diff --git a/src/main/java/org/mtransit/parser/FileUtils.java b/src/main/java/org/mtransit/parser/FileUtils.java index 794ac25..0722ed3 100644 --- a/src/main/java/org/mtransit/parser/FileUtils.java +++ b/src/main/java/org/mtransit/parser/FileUtils.java @@ -1,8 +1,11 @@ package org.mtransit.parser; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; @SuppressWarnings("unused") public final class FileUtils { @@ -30,4 +33,24 @@ public static void delete(@NotNull File file) { throw new MTLog.Fatal("Deletion of the directory '%s' failed!", file); } } + + @Nullable + public static Long size(@NotNull File file) { + try { + return Files.size(file.toPath()); + } catch (IOException ioe) { + MTLog.logNonFatal(ioe, "Error while reading size of %s!", file); + return null; + } + } + + @Nullable + public static String sizeToDiplayString(@Nullable Long size) { + try { + return org.apache.commons.io.FileUtils.byteCountToDisplaySize(size == null ? 0L : size); + } catch (Exception e) { + MTLog.logNonFatal(e, "Error while converting size of %s!", size); + return null; + } + } } diff --git a/src/main/java/org/mtransit/parser/MTLog.java b/src/main/java/org/mtransit/parser/MTLog.java index 5b0ef81..0505a3d 100644 --- a/src/main/java/org/mtransit/parser/MTLog.java +++ b/src/main/java/org/mtransit/parser/MTLog.java @@ -102,6 +102,19 @@ public static void logFatal(@NotNull Throwable t, @NotNull String format, @Nulla System.exit(-1); } + public static void waitForEnter() { + if (DefaultAgencyTools.IS_CI) { + return; + } + System.out.println("\nPress Enter to continue..."); + try { + //noinspection ResultOfMethodCallIgnored + System.in.read(new byte[2]); // requires run { standardInput = System.in } in build.gradle + } catch (Exception e) { + throw new MTLog.Fatal(e, "Error while waiting for Enter!"); + } + } + public static class Fatal extends RuntimeException { public Fatal(@NotNull String format, @Nullable Object... args) { super(); diff --git a/src/main/java/org/mtransit/parser/db/DBUtils.kt b/src/main/java/org/mtransit/parser/db/DBUtils.kt index 9bb35ed..4ccfba7 100644 --- a/src/main/java/org/mtransit/parser/db/DBUtils.kt +++ b/src/main/java/org/mtransit/parser/db/DBUtils.kt @@ -7,9 +7,12 @@ import org.mtransit.parser.MTLog import org.mtransit.parser.gtfs.data.GStopTime import org.mtransit.parser.gtfs.data.GTripStop import org.mtransit.parser.mt.data.MSchedule +import org.sqlite.SQLiteException import java.io.File import java.sql.Connection import java.sql.DriverManager +import java.sql.PreparedStatement +import java.sql.SQLException import org.mtransit.commons.sql.SQLUtils as SQLUtilsCommons object DBUtils { @@ -23,11 +26,13 @@ object DBUtils { private const val SQL_RESULT_ALIAS = "result" private const val SQL_NULL = "null" + private val IS_USING_FILE_INSTEAD_OF_MEMORY = DefaultAgencyTools.IS_CI + private val connection: Connection by lazy { FileUtils.deleteIfExist(File(FILE_PATH)) // delete previous - MTLog.log("DB connection > IS_CI: ${DefaultAgencyTools.IS_CI}") + MTLog.log("DB connection > IS_USING_FILE_INSTEAD_OF_MEMORY: $IS_USING_FILE_INSTEAD_OF_MEMORY") DriverManager.getConnection( - if (DefaultAgencyTools.IS_CI) { + if (IS_USING_FILE_INSTEAD_OF_MEMORY) { SQLUtils.getJDBCSQLiteFile(FILE_PATH) } else { SQLUtils.JDBC_SQLITE_MEMORY // faster @@ -45,58 +50,63 @@ object DBUtils { private var deletedRowCount = 0 init { - val statement = connection.createStatement() - - SQLUtils.execute(statement, "PRAGMA synchronous = OFF") - SQLUtils.execute(statement, "PRAGMA journal_mode = MEMORY") - SQLUtils.execute(statement, SQLUtilsCommons.PRAGMA_AUTO_VACUUM_NONE) - SQLUtils.executeUpdate(statement, SQLUtilsCommons.getSQLDropIfExistsQuery(STOP_TIMES_TABLE_NAME)) - SQLUtils.executeUpdate(statement, SQLUtilsCommons.getSQLDropIfExistsQuery(TRIP_STOPS_TABLE_NAME)) - SQLUtils.executeUpdate(statement, SQLUtilsCommons.getSQLDropIfExistsQuery(SCHEDULES_TABLE_NAME)) - SQLUtils.executeUpdate( - statement, - SQLCreateBuilder.getNew(STOP_TIMES_TABLE_NAME) - .appendColumn(GStopTime.TRIP_ID, SQLUtilsCommons.INT) - .appendColumn(GStopTime.STOP_ID, SQLUtilsCommons.INT) - .appendColumn(GStopTime.STOP_SEQUENCE, SQLUtilsCommons.INT) - .appendColumn(GStopTime.ARRIVAL_TIME, SQLUtilsCommons.INT) - .appendColumn(GStopTime.DEPARTURE_TIME, SQLUtilsCommons.INT) - .appendColumn(GStopTime.STOP_HEADSIGN, SQLUtilsCommons.TXT) // string ?? - .appendColumn(GStopTime.PICKUP_TYPE, SQLUtilsCommons.INT) - .appendColumn(GStopTime.DROP_OFF_TYPE, SQLUtilsCommons.INT) - .appendColumn(GStopTime.TIME_POINT, SQLUtilsCommons.INT) - .appendPrimaryKeys( - GStopTime.TRIP_ID, - GStopTime.STOP_SEQUENCE, - ) - .build() - ) - SQLUtils.executeUpdate( - statement, - SQLCreateBuilder.getNew(TRIP_STOPS_TABLE_NAME) - .appendColumn(GTripStop.ROUTE_ID, SQLUtilsCommons.INT) - .appendColumn(GTripStop.TRIP_ID, SQLUtilsCommons.INT) - .appendColumn(GTripStop.STOP_ID, SQLUtilsCommons.INT) - .appendColumn(GTripStop.STOP_SEQUENCE, SQLUtilsCommons.INT) - .build() - ) - SQLUtils.executeUpdate( - statement, - SQLCreateBuilder.getNew(SCHEDULES_TABLE_NAME) - .appendColumn(MSchedule.ROUTE_ID, SQLUtilsCommons.INT) - .appendColumn(MSchedule.SERVICE_ID, SQLUtilsCommons.INT) - .appendColumn(MSchedule.TRIP_ID, SQLUtilsCommons.INT) - .appendColumn(MSchedule.STOP_ID, SQLUtilsCommons.INT) - .appendColumn(MSchedule.ARRIVAL, SQLUtilsCommons.INT) - .appendColumn(MSchedule.DEPARTURE, SQLUtilsCommons.INT) - .appendColumn(MSchedule.PATH_ID, SQLUtilsCommons.INT) - .appendColumn(MSchedule.WHEELCHAIR_BOARDING, SQLUtilsCommons.INT) - .appendColumn(MSchedule.HEADSIGN_TYPE, SQLUtilsCommons.INT) - .appendColumn(MSchedule.HEADSIGN_VALUE, SQLUtilsCommons.TXT) // string ?? - .build() - ) + connection.createStatement().use { statement -> + SQLUtils.execute(statement, "PRAGMA synchronous = OFF") + SQLUtils.execute(statement, "PRAGMA journal_mode = MEMORY") + SQLUtils.execute(statement, SQLUtilsCommons.PRAGMA_AUTO_VACUUM_NONE) + SQLUtils.executeUpdate(statement, SQLUtilsCommons.getSQLDropIfExistsQuery(STOP_TIMES_TABLE_NAME)) + SQLUtils.executeUpdate(statement, SQLUtilsCommons.getSQLDropIfExistsQuery(TRIP_STOPS_TABLE_NAME)) + SQLUtils.executeUpdate(statement, SQLUtilsCommons.getSQLDropIfExistsQuery(SCHEDULES_TABLE_NAME)) + SQLUtils.executeUpdate( + statement, + SQLCreateBuilder.getNew(STOP_TIMES_TABLE_NAME) + .appendColumn(GStopTime.TRIP_ID, SQLUtilsCommons.INT) + .appendColumn(GStopTime.STOP_ID, SQLUtilsCommons.INT) + .appendColumn(GStopTime.STOP_SEQUENCE, SQLUtilsCommons.INT) + .appendColumn(GStopTime.ARRIVAL_TIME, SQLUtilsCommons.INT) + .appendColumn(GStopTime.DEPARTURE_TIME, SQLUtilsCommons.INT) + .appendColumn(GStopTime.STOP_HEADSIGN, SQLUtilsCommons.TXT) // string ?? + .appendColumn(GStopTime.PICKUP_TYPE, SQLUtilsCommons.INT) + .appendColumn(GStopTime.DROP_OFF_TYPE, SQLUtilsCommons.INT) + .appendColumn(GStopTime.TIME_POINT, SQLUtilsCommons.INT) + .appendPrimaryKeys( + GStopTime.TRIP_ID, + GStopTime.STOP_SEQUENCE, + ) + .build() + ) + SQLUtils.executeUpdate( + statement, + SQLCreateBuilder.getNew(TRIP_STOPS_TABLE_NAME) + .appendColumn(GTripStop.ROUTE_ID, SQLUtilsCommons.INT) + .appendColumn(GTripStop.TRIP_ID, SQLUtilsCommons.INT) + .appendColumn(GTripStop.STOP_ID, SQLUtilsCommons.INT) + .appendColumn(GTripStop.STOP_SEQUENCE, SQLUtilsCommons.INT) + .build() + ) + SQLUtils.executeUpdate( + statement, + SQLCreateBuilder.getNew(SCHEDULES_TABLE_NAME) + .appendColumn(MSchedule.ROUTE_ID, SQLUtilsCommons.INT) + .appendColumn(MSchedule.SERVICE_ID, SQLUtilsCommons.INT) + .appendColumn(MSchedule.TRIP_ID, SQLUtilsCommons.INT) + .appendColumn(MSchedule.STOP_ID, SQLUtilsCommons.INT) + .appendColumn(MSchedule.ARRIVAL, SQLUtilsCommons.INT) + .appendColumn(MSchedule.DEPARTURE, SQLUtilsCommons.INT) + .appendColumn(MSchedule.PATH_ID, SQLUtilsCommons.INT) + .appendColumn(MSchedule.WHEELCHAIR_BOARDING, SQLUtilsCommons.INT) + .appendColumn(MSchedule.HEADSIGN_TYPE, SQLUtilsCommons.INT) + .appendColumn(MSchedule.HEADSIGN_VALUE, SQLUtilsCommons.TXT) // string ?? + .build() + ) + } } + @JvmStatic + fun getDBSize() = if (IS_USING_FILE_INSTEAD_OF_MEMORY) { + FileUtils.size(File(FILE_PATH)) + } else null + @Suppress("unused") @JvmStatic fun beginTransaction() = SQLUtils.beginTransaction(this.connection) @@ -113,64 +123,121 @@ object DBUtils { fun commit() = SQLUtils.commit(this.connection) @JvmStatic - @JvmOverloads - fun insertStopTime(gStopTime: GStopTime, allowUpdate: Boolean = false): Boolean { - val rs = SQLUtils.executeUpdate( - connection.createStatement(), + fun prepareInsertStopTime(allowUpdate: Boolean = false): PreparedStatement { + return connection.prepareStatement( (if (allowUpdate) SQLUtilsCommons.INSERT_OR_REPLACE_INTO else SQLUtilsCommons.INSERT_INTO) + STOP_TIMES_TABLE_NAME + SQLUtilsCommons.VALUES_P1 + - "${gStopTime.tripIdInt}," + - "${gStopTime.stopIdInt}," + - "${gStopTime.stopSequence}," + - "${gStopTime.arrivalTime}," + - "${gStopTime.departureTime}," + - "${gStopTime.stopHeadsign?.let { SQLUtils.quotes(SQLUtils.escape(it)) }}," + - "${gStopTime.pickupType.id}," + - "${gStopTime.dropOffType.id}," + - "${gStopTime.timePoint.id}" + + "?," + // trip ID + "?," + // stop ID + "?," + // stop sequence + "?," + // arrival time + "?," + // departure time + "?," + // stop head-sign + "?," + // pickup type + "?," + // drop off type + "?" + // time point SQLUtilsCommons.P2 ) - insertRowCount++ - insertCount++ - return rs > 0 + } + + @JvmStatic + fun insertStopTime(gStopTime: GStopTime, preparedStatement: PreparedStatement) { + try { + var idx = 1 + with(preparedStatement) { + setInt(idx++, gStopTime.tripIdInt) + setInt(idx++, gStopTime.stopIdInt) + setInt(idx++, gStopTime.stopSequence) + setInt(idx++, gStopTime.arrivalTime) + setInt(idx++, gStopTime.departureTime) + setString(idx++, "${gStopTime.stopHeadsign?.let { SQLUtils.quotes(SQLUtils.escape(it)) }}") + setInt(idx++, gStopTime.pickupType.id) + setInt(idx++, gStopTime.dropOffType.id) + setInt(idx++, gStopTime.timePoint.id) + addBatch() + } + insertRowCount++ + insertCount++ + } catch (e: SQLiteException) { + throw MTLog.Fatal(e, "SQL lite error while inserting '$gStopTime'!") + } catch (e: SQLException) { + throw MTLog.Fatal(e, "SQL error while inserting '$gStopTime'!") + } catch (e: Exception) { + throw MTLog.Fatal(e, "Error while inserting '$gStopTime'!") + } + } + + @JvmStatic + fun executeInsertStopTime(preparedStatement: PreparedStatement): Boolean { + val rs = preparedStatement.executeBatch() + return rs.isNotEmpty() + } + + @JvmStatic + @JvmOverloads + fun insertStopTime(gStopTime: GStopTime, allowUpdate: Boolean = false): Boolean { + connection.createStatement().use { statement -> + val rs = SQLUtils.executeUpdate( + statement, + (if (allowUpdate) SQLUtilsCommons.INSERT_OR_REPLACE_INTO else SQLUtilsCommons.INSERT_INTO) + + STOP_TIMES_TABLE_NAME + SQLUtilsCommons.VALUES_P1 + + "${gStopTime.tripIdInt}," + + "${gStopTime.stopIdInt}," + + "${gStopTime.stopSequence}," + + "${gStopTime.arrivalTime}," + + "${gStopTime.departureTime}," + + "${gStopTime.stopHeadsign?.let { SQLUtils.quotes(SQLUtils.escape(it)) }}," + + "${gStopTime.pickupType.id}," + + "${gStopTime.dropOffType.id}," + + "${gStopTime.timePoint.id}" + + SQLUtilsCommons.P2 + ) + insertRowCount++ + insertCount++ + return rs > 0 + } } @JvmStatic fun insertTripStop(gTripStop: GTripStop): Boolean { - val rs = SQLUtils.executeUpdate( - connection.createStatement(), - SQLUtilsCommons.INSERT_INTO + TRIP_STOPS_TABLE_NAME + SQLUtilsCommons.VALUES_P1 + - "${gTripStop.routeIdInt}," + - "${gTripStop.tripIdInt}," + - "${gTripStop.stopIdInt}," + - "${gTripStop.stopSequence}" + - SQLUtilsCommons.P2 - ) - insertRowCount++ - insertCount++ - return rs > 0 + connection.createStatement().use { statement -> + val rs = SQLUtils.executeUpdate( + statement, + SQLUtilsCommons.INSERT_INTO + TRIP_STOPS_TABLE_NAME + SQLUtilsCommons.VALUES_P1 + + "${gTripStop.routeIdInt}," + + "${gTripStop.tripIdInt}," + + "${gTripStop.stopIdInt}," + + "${gTripStop.stopSequence}" + + SQLUtilsCommons.P2 + ) + insertRowCount++ + insertCount++ + return rs > 0 + } } @JvmStatic fun insertSchedule(mSchedule: MSchedule): Boolean { - val rs = SQLUtils.executeUpdate( - connection.createStatement(), - SQLUtilsCommons.INSERT_INTO + SCHEDULES_TABLE_NAME + SQLUtilsCommons.VALUES_P1 + - "${mSchedule.routeId}," + - "${mSchedule.serviceIdInt}," + - "${mSchedule.tripId}," + - "${mSchedule.stopId}," + - "${mSchedule.arrival}," + - "${mSchedule.departure}," + - "${mSchedule.pathIdInt}," + - "${mSchedule.accessible}," + - "${mSchedule.headsignType}," + - "${mSchedule.headsignValue?.let { SQLUtils.quotes(SQLUtils.escape(it)) }}" + - SQLUtilsCommons.P2 - ) - insertRowCount++ - insertCount++ - return rs > 0 + connection.createStatement().use { statement -> + val rs = SQLUtils.executeUpdate( + statement, + SQLUtilsCommons.INSERT_INTO + SCHEDULES_TABLE_NAME + SQLUtilsCommons.VALUES_P1 + + "${mSchedule.routeId}," + + "${mSchedule.serviceIdInt}," + + "${mSchedule.tripId}," + + "${mSchedule.stopId}," + + "${mSchedule.arrival}," + + "${mSchedule.departure}," + + "${mSchedule.pathIdInt}," + + "${mSchedule.accessible}," + + "${mSchedule.headsignType}," + + "${mSchedule.headsignValue?.let { SQLUtils.quotes(SQLUtils.escape(it)) }}" + + SQLUtilsCommons.P2 + ) + insertRowCount++ + insertCount++ + return rs > 0 + } } @JvmStatic @@ -180,7 +247,6 @@ object DBUtils { limitMaxNbRow: Int? = null, limitOffset: Int? = null ): List { - MTLog.log("selectStopTimes($tripId, $tripIds, $limitMaxNbRow, $limitOffset)") var query = "SELECT * FROM $STOP_TIMES_TABLE_NAME" tripId?.let { query += " WHERE ${GStopTime.TRIP_ID} = $tripId" @@ -206,35 +272,32 @@ object DBUtils { query += " OFFSET $limitOffset" } } - MTLog.log("selectStopTimes() > $query") val result = ArrayList() - val rs = SQLUtils.executeQuery(connection.createStatement(), query) - MTLog.log("selectStopTimes() > query executed") - val selectRowCountBefore = selectRowCount - while (rs.next()) { - var stopHeadSign: String? = rs.getString(GStopTime.STOP_HEADSIGN) - if (stopHeadSign == SQL_NULL) { - stopHeadSign = null - } - result.add( - GStopTime( - rs.getInt(GStopTime.TRIP_ID), - rs.getInt(GStopTime.ARRIVAL_TIME), - rs.getInt(GStopTime.DEPARTURE_TIME), - rs.getInt(GStopTime.STOP_ID), - rs.getInt(GStopTime.STOP_SEQUENCE), - stopHeadSign, - rs.getInt(GStopTime.PICKUP_TYPE), - rs.getInt(GStopTime.DROP_OFF_TYPE), - rs.getInt(GStopTime.TIME_POINT), + connection.createStatement().use { statement -> + val rs = SQLUtils.executeQuery(statement, query) + while (rs.next()) { + var stopHeadSign: String? = rs.getString(GStopTime.STOP_HEADSIGN) + if (stopHeadSign == SQL_NULL) { + stopHeadSign = null + } + result.add( + GStopTime( + rs.getInt(GStopTime.TRIP_ID), + rs.getInt(GStopTime.ARRIVAL_TIME), + rs.getInt(GStopTime.DEPARTURE_TIME), + rs.getInt(GStopTime.STOP_ID), + rs.getInt(GStopTime.STOP_SEQUENCE), + stopHeadSign, + rs.getInt(GStopTime.PICKUP_TYPE), + rs.getInt(GStopTime.DROP_OFF_TYPE), + rs.getInt(GStopTime.TIME_POINT), + ) ) - ) - selectRowCount++ + selectRowCount++ + } + selectCount++ + return result } - MTLog.log("selectStopTimes() > loaded: ${selectRowCount - selectRowCountBefore}") - selectCount++ - MTLog.log("selectStopTimes() > selectCount: $selectCount") - return result } @JvmStatic @@ -266,20 +329,22 @@ object DBUtils { } } val result = ArrayList() - val rs = SQLUtils.executeQuery(connection.createStatement(), query) - while (rs.next()) { - result.add( - GTripStop( - rs.getInt(GTripStop.ROUTE_ID), - rs.getInt(GTripStop.TRIP_ID), - rs.getInt(GTripStop.STOP_ID), - rs.getInt(GTripStop.STOP_SEQUENCE) + connection.createStatement().use { statement -> + val rs = SQLUtils.executeQuery(statement, query) + while (rs.next()) { + result.add( + GTripStop( + rs.getInt(GTripStop.ROUTE_ID), + rs.getInt(GTripStop.TRIP_ID), + rs.getInt(GTripStop.STOP_ID), + rs.getInt(GTripStop.STOP_SEQUENCE) + ) ) - ) - selectRowCount++ + selectRowCount++ + } + selectCount++ + return result } - selectCount++ - return result } @JvmStatic @@ -416,30 +481,32 @@ object DBUtils { } } val result = ArrayList() - val rs = SQLUtils.executeQuery(connection.createStatement(), query) - while (rs.next()) { - var headsignValue: String? = rs.getString(MSchedule.HEADSIGN_VALUE) - if (headsignValue == SQL_NULL) { - headsignValue = null - } - result.add( - MSchedule( - rs.getLong(MSchedule.ROUTE_ID), - rs.getInt(MSchedule.SERVICE_ID), - rs.getLong(MSchedule.TRIP_ID), - rs.getInt(MSchedule.STOP_ID), - rs.getInt(MSchedule.ARRIVAL), - rs.getInt(MSchedule.DEPARTURE), - rs.getInt(MSchedule.PATH_ID), - rs.getInt(MSchedule.WHEELCHAIR_BOARDING), - rs.getInt(MSchedule.HEADSIGN_TYPE), - headsignValue + connection.createStatement().use { statement -> + val rs = SQLUtils.executeQuery(statement, query) + while (rs.next()) { + var headsignValue: String? = rs.getString(MSchedule.HEADSIGN_VALUE) + if (headsignValue == SQL_NULL) { + headsignValue = null + } + result.add( + MSchedule( + rs.getLong(MSchedule.ROUTE_ID), + rs.getInt(MSchedule.SERVICE_ID), + rs.getLong(MSchedule.TRIP_ID), + rs.getInt(MSchedule.STOP_ID), + rs.getInt(MSchedule.ARRIVAL), + rs.getInt(MSchedule.DEPARTURE), + rs.getInt(MSchedule.PATH_ID), + rs.getInt(MSchedule.WHEELCHAIR_BOARDING), + rs.getInt(MSchedule.HEADSIGN_TYPE), + headsignValue + ) ) - ) - selectRowCount++ + selectRowCount++ + } + selectCount++ + return result } - selectCount++ - return result } @Suppress("unused") @@ -452,13 +519,15 @@ object DBUtils { "${GStopTime.STOP_ID} = ${gStopTime.stopIdInt}" + " AND " + "${GStopTime.STOP_SEQUENCE} = ${gStopTime.stopSequence}" - val rs = SQLUtils.executeUpdate(connection.createStatement(), query) - deletedRowCount += rs - if (rs > 1) { - throw MTLog.Fatal("Deleted too many stop times!") + connection.createStatement().use { statement -> + val rs = SQLUtils.executeUpdate(statement, query) + deletedRowCount += rs + if (rs > 1) { + throw MTLog.Fatal("Deleted too many stop times!") + } + deleteCount++ + return rs > 0 } - deleteCount++ - return rs > 0 } @JvmStatic @@ -469,9 +538,11 @@ object DBUtils { "${GStopTime.TRIP_ID} = $tripId" } deleteCount++ - val rs = SQLUtils.executeUpdate(connection.createStatement(), query) - deletedRowCount += rs - return rs + connection.createStatement().use { statement -> + val rs = SQLUtils.executeUpdate(statement, query) + deletedRowCount += rs + return rs + } } @Suppress("unused") @@ -536,49 +607,57 @@ object DBUtils { query += " ${MSchedule.DEPARTURE} = $departure" } deleteCount++ - val rs = SQLUtils.executeUpdate(connection.createStatement(), query) - deletedRowCount += rs - return rs + connection.createStatement().use { statement -> + val rs = SQLUtils.executeUpdate(statement, query) + deletedRowCount += rs + return rs + } } @JvmStatic fun countStopTimes(): Int { - val rs = SQLUtils.executeQuery( - connection.createStatement(), - "SELECT COUNT(*) AS $SQL_RESULT_ALIAS FROM $STOP_TIMES_TABLE_NAME" - ) - selectCount++ - if (rs.next()) { - selectRowCount++ - return rs.getInt(SQL_RESULT_ALIAS) + connection.createStatement().use { statement -> + val rs = SQLUtils.executeQuery( + statement, + "SELECT COUNT(*) AS $SQL_RESULT_ALIAS FROM $STOP_TIMES_TABLE_NAME" + ) + selectCount++ + if (rs.next()) { + selectRowCount++ + return rs.getInt(SQL_RESULT_ALIAS) + } } throw MTLog.Fatal("Error while counting stop times!") } @JvmStatic fun countTripStops(): Int { - val rs = SQLUtils.executeQuery( - connection.createStatement(), - "SELECT COUNT(*) AS $SQL_RESULT_ALIAS FROM $TRIP_STOPS_TABLE_NAME" - ) - selectCount++ - if (rs.next()) { - selectRowCount++ - return rs.getInt(SQL_RESULT_ALIAS) + connection.createStatement().use { statement -> + val rs = SQLUtils.executeQuery( + statement, + "SELECT COUNT(*) AS $SQL_RESULT_ALIAS FROM $TRIP_STOPS_TABLE_NAME" + ) + selectCount++ + if (rs.next()) { + selectRowCount++ + return rs.getInt(SQL_RESULT_ALIAS) + } } throw MTLog.Fatal("Error while counting trip stops!") } @JvmStatic fun countSchedule(): Int { - val rs = SQLUtils.executeQuery( - connection.createStatement(), - "SELECT COUNT(*) AS $SQL_RESULT_ALIAS FROM $SCHEDULES_TABLE_NAME" - ) - selectCount++ - if (rs.next()) { - selectRowCount++ - return rs.getInt(SQL_RESULT_ALIAS) + connection.createStatement().use { statement -> + val rs = SQLUtils.executeQuery( + statement, + "SELECT COUNT(*) AS $SQL_RESULT_ALIAS FROM $SCHEDULES_TABLE_NAME" + ) + selectCount++ + if (rs.next()) { + selectRowCount++ + return rs.getInt(SQL_RESULT_ALIAS) + } } throw MTLog.Fatal("Error while counting schedules!") } diff --git a/src/main/java/org/mtransit/parser/db/DumpDbUtils.kt b/src/main/java/org/mtransit/parser/db/DumpDbUtils.kt index ff68f86..80da006 100644 --- a/src/main/java/org/mtransit/parser/db/DumpDbUtils.kt +++ b/src/main/java/org/mtransit/parser/db/DumpDbUtils.kt @@ -23,20 +23,21 @@ object DumpDbUtils { @JvmStatic fun init(connection: Connection) { - val statement = connection.createStatement() - SQLUtils.execute(statement, "PRAGMA auto_vacuum = NONE") - // DROP IF EXIST - SQLUtils.executeUpdate(statement, GTFSCommons.T_TRIP_STOPS_SQL_DROP) - SQLUtils.executeUpdate(statement, GTFSCommons.T_STOP_SQL_DROP) - SQLUtils.executeUpdate(statement, GTFSCommons.T_TRIP_SQL_DROP) - SQLUtils.executeUpdate(statement, GTFSCommons.T_ROUTE_SQL_DROP) - SQLUtils.executeUpdate(statement, GTFSCommons.T_SERVICE_DATES_SQL_DROP) - SQLUtils.executeUpdate(statement, GTFSCommons.T_ROUTE_SQL_DROP) - // CREATE - SQLUtils.executeUpdate(statement, GTFSCommons.T_ROUTE_SQL_CREATE) - SQLUtils.executeUpdate(statement, GTFSCommons.T_TRIP_SQL_CREATE) - SQLUtils.executeUpdate(statement, GTFSCommons.T_STOP_SQL_CREATE) - SQLUtils.executeUpdate(statement, GTFSCommons.T_TRIP_STOPS_SQL_CREATE) - SQLUtils.executeUpdate(statement, GTFSCommons.T_SERVICE_DATES_SQL_CREATE) + connection.createStatement().use { statement -> + SQLUtils.execute(statement, "PRAGMA auto_vacuum = NONE") + // DROP IF EXIST + SQLUtils.executeUpdate(statement, GTFSCommons.T_TRIP_STOPS_SQL_DROP) + SQLUtils.executeUpdate(statement, GTFSCommons.T_STOP_SQL_DROP) + SQLUtils.executeUpdate(statement, GTFSCommons.T_TRIP_SQL_DROP) + SQLUtils.executeUpdate(statement, GTFSCommons.T_ROUTE_SQL_DROP) + SQLUtils.executeUpdate(statement, GTFSCommons.T_SERVICE_DATES_SQL_DROP) + SQLUtils.executeUpdate(statement, GTFSCommons.T_ROUTE_SQL_DROP) + // CREATE + SQLUtils.executeUpdate(statement, GTFSCommons.T_ROUTE_SQL_CREATE) + SQLUtils.executeUpdate(statement, GTFSCommons.T_TRIP_SQL_CREATE) + SQLUtils.executeUpdate(statement, GTFSCommons.T_STOP_SQL_CREATE) + SQLUtils.executeUpdate(statement, GTFSCommons.T_TRIP_STOPS_SQL_CREATE) + SQLUtils.executeUpdate(statement, GTFSCommons.T_SERVICE_DATES_SQL_CREATE) + } } } \ No newline at end of file diff --git a/src/main/java/org/mtransit/parser/db/SQLUtils.kt b/src/main/java/org/mtransit/parser/db/SQLUtils.kt index 5328420..fad159b 100644 --- a/src/main/java/org/mtransit/parser/db/SQLUtils.kt +++ b/src/main/java/org/mtransit/parser/db/SQLUtils.kt @@ -9,6 +9,7 @@ import org.mtransit.parser.MTLog import org.sqlite.SQLiteException import java.sql.Connection import java.sql.ResultSet +import java.sql.SQLException import java.sql.Statement object SQLUtils { @@ -46,6 +47,8 @@ object SQLUtils { try { return statement.execute(query) } catch (e: SQLiteException) { + throw MTLog.Fatal(e, "SQL lite error while executing '$query'!") + } catch (e: SQLException) { throw MTLog.Fatal(e, "SQL error while executing '$query'!") } } @@ -58,6 +61,8 @@ object SQLUtils { try { return statement.executeQuery(query) } catch (e: SQLiteException) { + throw MTLog.Fatal(e, "SQL lite error while executing '$query'!") + } catch (e: SQLException) { throw MTLog.Fatal(e, "SQL error while executing '$query'!") } } @@ -70,20 +75,24 @@ object SQLUtils { try { return statement.executeUpdate(query) } catch (e: SQLiteException) { + throw MTLog.Fatal(e, "SQL lite error while executing '$query'!") + } catch (e: SQLException) { throw MTLog.Fatal(e, "SQL error while executing '$query'!") } } @JvmStatic fun beginTransaction(connection: Connection) { - val statement = connection.createStatement() - executeQuery(statement, "BEGIN TRANSACTION") + connection.createStatement().use { statement -> + executeQuery(statement, "BEGIN TRANSACTION") + } } @JvmStatic fun endTransaction(connection: Connection) { - val statement = connection.createStatement() - executeQuery(statement, "END TRANSACTION") + connection.createStatement().use { statement -> + executeQuery(statement, "END TRANSACTION") + } } @JvmStatic diff --git a/src/main/java/org/mtransit/parser/gtfs/GReader.java b/src/main/java/org/mtransit/parser/gtfs/GReader.java index 00988bf..aacbd67 100644 --- a/src/main/java/org/mtransit/parser/gtfs/GReader.java +++ b/src/main/java/org/mtransit/parser/gtfs/GReader.java @@ -7,7 +7,6 @@ import org.apache.commons.csv.CSVRecord; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import org.mtransit.commons.CloseableUtils; import org.mtransit.commons.StringUtils; import org.mtransit.parser.MTLog; import org.mtransit.parser.Utils; @@ -18,6 +17,7 @@ import org.mtransit.parser.gtfs.data.GCalendarDatesExceptionType; import org.mtransit.parser.gtfs.data.GDropOffType; import org.mtransit.parser.gtfs.data.GFrequency; +import org.mtransit.parser.gtfs.data.GIDs; import org.mtransit.parser.gtfs.data.GPickupType; import org.mtransit.parser.gtfs.data.GRoute; import org.mtransit.parser.gtfs.data.GSpec; @@ -28,10 +28,11 @@ import java.io.BufferedReader; import java.io.File; -import java.io.FileReader; import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.sql.PreparedStatement; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -59,142 +60,120 @@ public static GSpec readGtfsZipFile(@NotNull String gtfsFile, if (!gtfsDirF.exists()) { throw new MTLog.Fatal("'%s' GTFS directory does not exist!", gtfsDirF); } - FileReader fr = null; - BufferedReader reader = null; try { // AGENCY if (!calendarsOnly) { - File agencyFile = new File(gtfsDir, GAgency.FILENAME); - if (!agencyFile.exists()) { - throw new MTLog.Fatal("'%s' agency file does not exist!", agencyFile); - } else { - fr = new FileReader(agencyFile); - reader = new BufferedReader(fr); - readCsv(agencyFile.getName(), reader, line -> - processAgency(agencyTools, gSpec, line) - ); - } - } - // CALENDAR DATES - boolean hasCalendar = false; - File calendarDateFile = new File(gtfsDir, GCalendarDate.FILENAME); - if (!calendarDateFile.exists()) { - MTLog.log("Reading file '%s'... SKIP (non-existing).", calendarDateFile.getName()); - } else { - hasCalendar = true; - fr = new FileReader(calendarDateFile); - reader = new BufferedReader(fr); - readCsv(calendarDateFile.getName(), reader, line -> - processCalendarDate(agencyTools, gSpec, line) + readFile(agencyTools, gtfsDir, GAgency.FILENAME, true, line -> + processAgency(agencyTools, gSpec, line) ); } + // CALENDAR DATES + boolean hasCalendarDates = readFile(agencyTools, gtfsDir, GCalendarDate.FILENAME, false, line -> + processCalendarDate(agencyTools, gSpec, line) + ); // CALENDAR - File calendarFile = new File(gtfsDir, GCalendar.FILENAME); - if (!calendarFile.exists()) { - MTLog.log("Reading file '%s'... SKIP (non-existing).", calendarFile.getName()); - } else { - hasCalendar = true; - fr = new FileReader(calendarFile); - reader = new BufferedReader(fr); - readCsv(calendarFile.getName(), reader, line -> - processCalendar(agencyTools, gSpec, line) - ); - } + boolean hasCalendars = readFile(agencyTools, gtfsDir, GCalendar.FILENAME, false, line -> + processCalendar(agencyTools, gSpec, line) + ); + boolean hasCalendar = hasCalendarDates || hasCalendars; if (!hasCalendar) { throw new MTLog.Fatal("'%s' & '%s' file do not exist!", GCalendar.FILENAME, GCalendarDate.FILENAME); } // ROUTES if (!calendarsOnly) { - File routeFile = new File(gtfsDir, GRoute.FILENAME); - if (!routeFile.exists()) { - throw new MTLog.Fatal("'%s' route file does not exist!", routeFile); - } else { - fr = new FileReader(routeFile); - reader = new BufferedReader(fr); - readCsv(routeFile.getName(), reader, line -> - processRoute(agencyTools, gSpec, line) - ); - } + readFile(agencyTools, gtfsDir, GRoute.FILENAME, true, line -> + processRoute(agencyTools, gSpec, line) + ); } // TRIPS if (!calendarsOnly) { - File tripFile = new File(gtfsDir, GTrip.FILENAME); - if (!tripFile.exists()) { - throw new MTLog.Fatal("'%s' trip file does not exist!", tripFile); - } else { - fr = new FileReader(tripFile); - reader = new BufferedReader(fr); - readCsv(tripFile.getName(), reader, line -> - processTrip(agencyTools, gSpec, line) - ); - } + readFile(agencyTools, gtfsDir, GTrip.FILENAME, true, line -> + processTrip(agencyTools, gSpec, line) + ); } // FREQUENCIES if (!calendarsOnly && !routeTripCalendarsOnly) { - File frequencyFile = new File(gtfsDir, GFrequency.FILENAME); - if (!frequencyFile.exists()) { - MTLog.log("Reading file '%s'... SKIP (non-existing).", frequencyFile.getName()); - } else { - fr = new FileReader(frequencyFile); - reader = new BufferedReader(fr); - readCsv(frequencyFile.getName(), reader, line -> - processFrequency(agencyTools, gSpec, line) - ); - } + readFile(agencyTools, gtfsDir, GFrequency.FILENAME, false, line -> + processFrequency(agencyTools, gSpec, line) + ); } // STOPS if (!calendarsOnly && !routeTripCalendarsOnly) { - File stopFile = new File(gtfsDir, GStop.FILENAME); - if (!stopFile.exists()) { - throw new MTLog.Fatal("'%s' stop file does not exist!", stopFile); - } else { - fr = new FileReader(stopFile); - reader = new BufferedReader(fr); - readCsv(stopFile.getName(), reader, line -> - processStop(agencyTools, gSpec, line) - ); - } + readFile(agencyTools, gtfsDir, GStop.FILENAME, true, line -> + processStop(agencyTools, gSpec, line) + ); } // STOP TIMES if (!calendarsOnly && !routeTripCalendarsOnly) { - File stopTimeFile = new File(gtfsDir, GStopTime.FILENAME); - if (!stopTimeFile.exists()) { - MTLog.log("Reading file '%s'... SKIP (non-existing).", stopTimeFile.getName()); - } else { - fr = new FileReader(stopTimeFile); - reader = new BufferedReader(fr); - DBUtils.setAutoCommit(false); - readCsv(stopTimeFile.getName(), reader, - line -> processStopTime(agencyTools, gSpec, line), - columnNames -> { - if (!columnNames.contains(GStopTime.PICKUP_TYPE)) { - agencyTools.setForceStopTimeLastNoPickupType(true); // pickup types not provided - } - if (!columnNames.contains(GStopTime.DROP_OFF_TYPE)) { - agencyTools.setForceStopTimeFirstNoDropOffType(true); // drop-off types not provided - } - }); - if (!agencyTools.stopTimesHasPickupTypeNotRegular()) { - agencyTools.setForceStopTimeLastNoPickupType(true); // all provided pickup type are REGULAR == not provided - } - if (!agencyTools.stopTimesHasDropOffTypeNotRegular()) { - agencyTools.setForceStopTimeFirstNoDropOffType(true); // all provided drop-off type are REGULAR == not provided - } - DBUtils.setAutoCommit(true); // true => commit() + MTLog.log("- IDs: %d", GIDs.count()); + DBUtils.setAutoCommit(false); + final PreparedStatement insertStopTimePrepared = DBUtils.prepareInsertStopTime(false); + readFile(agencyTools, gtfsDir, GStopTime.FILENAME, false, + line -> processStopTime(agencyTools, gSpec, line, insertStopTimePrepared), + columnNames -> { + if (!columnNames.contains(GStopTime.PICKUP_TYPE)) { + agencyTools.setForceStopTimeLastNoPickupType(true); // pickup types not provided + } + if (!columnNames.contains(GStopTime.DROP_OFF_TYPE)) { + agencyTools.setForceStopTimeFirstNoDropOffType(true); // drop-off types not provided + } + } + ); + if (!agencyTools.stopTimesHasPickupTypeNotRegular()) { + agencyTools.setForceStopTimeLastNoPickupType(true); // all provided pickup type are REGULAR == not provided } + if (!agencyTools.stopTimesHasDropOffTypeNotRegular()) { + agencyTools.setForceStopTimeFirstNoDropOffType(true); // all provided drop-off type are REGULAR == not provided + } + DBUtils.executeInsertStopTime(insertStopTimePrepared); + DBUtils.commit(); + DBUtils.setAutoCommit(true); // true => commit() + MTLog.log("- IDs: %d", GIDs.count()); } - // TODO OTHER FILE TYPE - } catch (IOException ioe) { + // TODO OTHER FILES TYPE + } catch (Exception ioe) { throw new MTLog.Fatal(ioe, "I/O Error while reading GTFS file!"); - } finally { - CloseableUtils.closeQuietly(reader); - CloseableUtils.closeQuietly(fr); } MTLog.log("Reading GTFS file '%1$s'... DONE in %2$s.", gtfsFile, Utils.getPrettyDuration(System.currentTimeMillis() - start)); gSpec.print(calendarsOnly, false); return gSpec; } + private static boolean readFile( + @NotNull final GAgencyTools agencyTools, + @NotNull String gtfsDir, + @NotNull String fileName, + boolean fileRequired, + @NotNull LineProcessor lineProcessor + ) { + return readFile(agencyTools, gtfsDir, fileName, fileRequired, lineProcessor, null); + } + + private static boolean readFile( + @NotNull final GAgencyTools agencyTools, + @NotNull String gtfsDir, + @NotNull String fileName, + boolean fileRequired, + @NotNull LineProcessor lineProcessor, + @Nullable OnColumnNamesFound onColumnNamesFoundCallback + ) { + final File gtfsFile = new File(gtfsDir, fileName); + if (!gtfsFile.exists()) { + if (fileRequired) { + throw new MTLog.Fatal("'%s' file does not exist!", gtfsFile); + } else { + MTLog.log("Reading file '%s'... SKIP (non-existing).", gtfsFile.getName()); + return false; + } + } + try (BufferedReader br = Files.newBufferedReader(gtfsFile.toPath())) { + readCsv(gtfsFile.getName(), br, lineProcessor, onColumnNamesFoundCallback); + } catch (IOException ioe) { + throw new MTLog.Fatal(ioe, "I/O Error while reading GTFS file %s!", gtfsFile); + } + return true; + } + private static final CSVFormat CSV_FORMAT = CSVFormat.RFC4180.builder().setIgnoreSurroundingSpaces(true).build(); private static final CSVFormat CSV_FORMAT_NO_QUOTE = CSV_FORMAT.builder().setQuote(null).build(); @@ -283,18 +262,18 @@ private static void readCsv(String filename, BufferedReader reader, MTLog.log("Reading file '%s' (lines: %s)... DONE", filename, l); } - private static void processStopTime(GAgencyTools agencyTools, GSpec gSpec, HashMap line) { + private static void processStopTime(GAgencyTools agencyTools, GSpec gSpec, HashMap line, PreparedStatement insertStopTimePrepared) { try { - GStopTime gStopTime = new GStopTime( + final GStopTime gStopTime = new GStopTime( line.get(GStopTime.TRIP_ID), line.get(GStopTime.ARRIVAL_TIME).trim(), line.get(GStopTime.DEPARTURE_TIME).trim(), line.get(GStopTime.STOP_ID).trim(), Integer.parseInt(line.get(GStopTime.STOP_SEQUENCE).trim()), - line.get(GStopTime.STOP_HEADSIGN), // - GPickupType.parse(line.get(GStopTime.PICKUP_TYPE)), // - GDropOffType.parse(line.get(GStopTime.DROP_OFF_TYPE)), // - GTimePoint.parse(line.get(GStopTime.TIME_POINT)) // + line.get(GStopTime.STOP_HEADSIGN), + GPickupType.parse(line.get(GStopTime.PICKUP_TYPE)), + GDropOffType.parse(line.get(GStopTime.DROP_OFF_TYPE)), + GTimePoint.parse(line.get(GStopTime.TIME_POINT)) ); if (agencyTools.excludeTripNullable(gSpec.getTrip(gStopTime.getTripIdInt()))) { return; @@ -311,7 +290,7 @@ private static void processStopTime(GAgencyTools agencyTools, GSpec gSpec, HashM if (agencyTools.excludeStopTime(gStopTime)) { return; } - gSpec.addStopTime(gStopTime, false); + gSpec.addStopTime(gStopTime, insertStopTimePrepared); } catch (Exception e) { throw new MTLog.Fatal(e, "Error while parsing: '%s'!", line); } diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GDropOffType.kt b/src/main/java/org/mtransit/parser/gtfs/data/GDropOffType.kt index ae2b5f5..8f341f6 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GDropOffType.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GDropOffType.kt @@ -29,7 +29,7 @@ enum class GDropOffType(val id: Int) { @JvmStatic fun parse(id: String?): GDropOffType { - return if (id == null || id.isEmpty()) { // that's OK + return if (id.isNullOrEmpty()) { // that's OK REGULAR // default } else try { parse(id.toInt()) diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GIDs.kt b/src/main/java/org/mtransit/parser/gtfs/data/GIDs.kt index e8d93be..0da4c88 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GIDs.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GIDs.kt @@ -1,36 +1,33 @@ package org.mtransit.parser.gtfs.data +import androidx.collection.SparseArrayCompat import org.mtransit.parser.Constants.EMPTY import org.mtransit.parser.MTLog object GIDs { - private var increment = 0 - - private val integerToString = hashMapOf() - private val stringToInteger = hashMapOf() + private val intToString = SparseArrayCompat() @JvmStatic fun getString(integer: Int): String { - return integerToString[integer] ?: throw MTLog.Fatal("Unexpected GID integer $integer!") + return intToString[integer] ?: throw MTLog.Fatal("Unexpected GID integer $integer!") } @JvmStatic fun getInt(string: String): Int { - return stringToInteger[string] ?: add(string) + return add(string) } private fun add(newString: String): Int { - val newInteger = increment - integerToString[newInteger] = newString - stringToInteger[newString] = newInteger - increment++ // ready for next call - return newInteger + return newString.hashCode() + .also { + intToString.put(it, newString) + } } @JvmStatic fun count(): Int { - return increment + return intToString.size() } @Suppress("unused") diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java b/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java index cd9c879..8e69582 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java +++ b/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java @@ -4,12 +4,14 @@ import org.jetbrains.annotations.Nullable; import org.mtransit.parser.Constants; import org.mtransit.parser.DefaultAgencyTools; +import org.mtransit.parser.FileUtils; import org.mtransit.parser.MTLog; import org.mtransit.parser.Pair; import org.mtransit.parser.db.DBUtils; import org.mtransit.parser.gtfs.GAgencyTools; import org.mtransit.parser.mt.GenerateMObjectsTask; +import java.sql.PreparedStatement; import java.util.ArrayList; import java.util.Calendar; import java.util.Collection; @@ -272,6 +274,10 @@ public void addStopTime(@NotNull GStopTime gStopTime, boolean allowUpdate) { DBUtils.insertStopTime(gStopTime, allowUpdate); } + public void addStopTime(@NotNull GStopTime gStopTime, @NotNull PreparedStatement insertStopTimePrepared) { + DBUtils.insertStopTime(gStopTime, insertStopTimePrepared); + } + private int removeTripStopTimes(@NotNull Integer gTripId) { int r = 0; r += DBUtils.deleteStopTimes(gTripId); @@ -398,7 +404,8 @@ public void generateTripStops() { final int stopTimesCount = readStopTimesCount(); int tripStopsCount = 0; int offset = 0; - final int maxRowNumber = DefaultAgencyTools.IS_CI ? 50_000 : 1_000_000; + final int maxRowNumber = DefaultAgencyTools.IS_CI ? 100_000 : 1_000_000; + MTLog.log("Generating GTFS trip stops from stop times... (DB size: %s)", FileUtils.sizeToDiplayString(DBUtils.getDBSize())); DBUtils.setAutoCommit(false); while (offset < stopTimesCount) { MTLog.log("Generating GTFS trip stops from stop times... (%d -> %d)", offset, offset + maxRowNumber); @@ -426,6 +433,7 @@ public void generateTripStops() { MTLog.log("Generating GTFS trip stops from stop times... > commit to DB..."); DBUtils.setAutoCommit(true); // true => commit() MTLog.log("Generating GTFS trip stops from stop times... > commit to DB... DONE"); + MTLog.log("Generating GTFS trip stops from stop times... (DB size: %s)", FileUtils.sizeToDiplayString(DBUtils.getDBSize())); MTLog.log("Generating GTFS trip stops from stop times... DONE"); MTLog.log("- Trip stops: %d", readTripStopsCount()); MTLog.log("- IDs: %d", GIDs.count()); diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt b/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt index 86ae1b2..be920da 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt @@ -1,7 +1,9 @@ package org.mtransit.parser.gtfs.data import org.mtransit.commons.StringUtils +import org.mtransit.parser.MTLog import org.mtransit.parser.gtfs.GAgencyTools +import java.lang.Exception // https://developers.google.com/transit/gtfs/reference#tripstxt // https://gtfs.org/reference/static/#tripstxt @@ -111,31 +113,30 @@ data class GTrip( const val DIRECTION_ID = "direction_id" const val WHEELCHAIR_ACCESSIBLE = "wheelchair_accessible" + private const val UID_SEPARATOR = "+" // int IDs can be negative + @Suppress("unused") @JvmStatic - fun extractRouteIdInt(tripUID: String): Int { - val (_, routeIdInt) = split(tripUID) - return routeIdInt - } + fun extractRouteIdInt(tripUID: String) = split(tripUID).second @Suppress("unused") @JvmStatic - fun extractTripIdInt(tripUID: String): Int { - val (tripIdInt, _) = split(tripUID) - return tripIdInt - } + fun extractTripIdInt(tripUID: String) = split(tripUID).first @JvmStatic - fun split(tripUID: String): Pair { - val s = tripUID.split("-") - return Pair(s[0].toInt(), s[1].toInt()) + fun split(tripUID: String) = try { + tripUID.split(UID_SEPARATOR).let { s -> + Pair(s[0].toInt(), s[1].toInt()) + } + } catch (e: Exception) { + throw MTLog.Fatal(e, "Error while trying to split $tripUID!") } @JvmStatic fun getNewUID( routeIdInt: Int, tripIdInt: Int - ) = "${routeIdInt}-${tripIdInt}" + ) = "${routeIdInt}$UID_SEPARATOR${tripIdInt}" @JvmStatic fun longestFirst(tripList: List, tripStopListGetter: (Int) -> List?): List {