From c64c20016ddff10ba8d8a813d9a20fda07b0fb9d Mon Sep 17 00:00:00 2001 From: cka-y Date: Fri, 29 Sep 2023 10:00:58 -0400 Subject: [PATCH 1/6] fix: comments on #1587 --- .../TripAndShapeDistanceValidator.java | 53 ++++++++++--------- .../harvest_latest_versions.py | 1 - 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java index 13196dd68b..e52420a325 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java @@ -59,32 +59,33 @@ public void validate(NoticeContainer noticeContainer) { .distinct() .collect(Collectors.toList()); - uniqueShapeIds.forEach( - shapeId -> { - double maxShapeDist = - shapeTable.getEntities().stream() - .filter(s -> s.shapeId().equals(shapeId)) - .mapToDouble(GtfsShape::shapeDistTraveled) - .max() - .orElse(Double.NEGATIVE_INFINITY); - - tripTable - .byShapeId(shapeId) - .forEach( - trip -> { - double maxStopTimeDist = - stopTimeTable.byTripId(trip.tripId()).stream() - .mapToDouble(GtfsStopTime::shapeDistTraveled) - .max() - .orElse(Double.NEGATIVE_INFINITY); - - if (maxStopTimeDist > maxShapeDist) { - noticeContainer.addValidationNotice( - new TripDistanceExceedsShapeDistanceNotice( - trip.tripId(), shapeId, maxStopTimeDist, maxShapeDist)); - } - }); - }); + shapeTable + .byShapeIdMap() + .forEach( + (shapeId, shape) -> { + double maxShapeDist = + shapeTable.byShapeId(shapeId).stream() + .mapToDouble(GtfsShape::shapeDistTraveled) + .max() + .orElse(Double.NEGATIVE_INFINITY); + + tripTable + .byShapeId(shapeId) + .forEach( + trip -> { + double maxStopTimeDist = + stopTimeTable.byTripId(trip.tripId()).stream() + .mapToDouble(GtfsStopTime::shapeDistTraveled) + .max() + .orElse(Double.NEGATIVE_INFINITY); + + if (maxStopTimeDist > maxShapeDist) { + noticeContainer.addValidationNotice( + new TripDistanceExceedsShapeDistanceNotice( + trip.tripId(), shapeId, maxStopTimeDist, maxShapeDist)); + } + }); + }); } /** The distance traveled by a trip should be less or equal to the max length of its shape. */ diff --git a/scripts/mobility-database-harvester/harvest_latest_versions.py b/scripts/mobility-database-harvester/harvest_latest_versions.py index 81f93f031a..6f6cd03516 100644 --- a/scripts/mobility-database-harvester/harvest_latest_versions.py +++ b/scripts/mobility-database-harvester/harvest_latest_versions.py @@ -33,7 +33,6 @@ # Sources to exclude because they are too big for the workflow. SOURCES_TO_EXCLUDE = [ "de-unknown-rursee-schifffahrt-kg-gtfs-784", - "de-unknown-ulmer-eisenbahnfreunde-gtfs-1081", "no-unknown-agder-kollektivtrafikk-as-gtfs-1078", "hu-unknown-volanbusz-gtfs-1836", "de-baden-wurttemberg-verkehrsverbund-rhein-neckar-gtfs-1173", From 1101b0c69b47f3a6305c2de6825e8d10fd6eec8a Mon Sep 17 00:00:00 2001 From: cka-y Date: Fri, 29 Sep 2023 10:04:55 -0400 Subject: [PATCH 2/6] fix: removed unused variable --- .../validator/TripAndShapeDistanceValidator.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java index e52420a325..efcdc9d616 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java @@ -53,12 +53,6 @@ public class TripAndShapeDistanceValidator extends FileValidator { @Override public void validate(NoticeContainer noticeContainer) { - List uniqueShapeIds = - shapeTable.getEntities().stream() - .map(GtfsShape::shapeId) - .distinct() - .collect(Collectors.toList()); - shapeTable .byShapeIdMap() .forEach( From fe5e51604dd5b98064d8545da03b844c829c98d2 Mon Sep 17 00:00:00 2001 From: cka-y Date: Fri, 29 Sep 2023 10:08:00 -0400 Subject: [PATCH 3/6] fix: formatting --- .../gtfsvalidator/validator/TripAndShapeDistanceValidator.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java index efcdc9d616..5725ef6e43 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java @@ -17,8 +17,6 @@ import static org.mobilitydata.gtfsvalidator.notice.SeverityLevel.ERROR; -import java.util.List; -import java.util.stream.Collectors; import javax.inject.Inject; import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice; import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.FileRefs; From 42ed0e9cbd59a9d72d96525030110217706d6796 Mon Sep 17 00:00:00 2001 From: cka-y Date: Tue, 10 Oct 2023 15:41:49 -0400 Subject: [PATCH 4/6] fix: rollback sources for acceptance test --- .../mobility-database-harvester/harvest_latest_versions.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/scripts/mobility-database-harvester/harvest_latest_versions.py b/scripts/mobility-database-harvester/harvest_latest_versions.py index 6f6cd03516..1c1dd75f6f 100644 --- a/scripts/mobility-database-harvester/harvest_latest_versions.py +++ b/scripts/mobility-database-harvester/harvest_latest_versions.py @@ -32,12 +32,7 @@ # Sources to exclude because they are too big for the workflow. SOURCES_TO_EXCLUDE = [ - "de-unknown-rursee-schifffahrt-kg-gtfs-784", - "no-unknown-agder-kollektivtrafikk-as-gtfs-1078", - "hu-unknown-volanbusz-gtfs-1836", - "de-baden-wurttemberg-verkehrsverbund-rhein-neckar-gtfs-1173", - "de-baden-wurttemberg-db-zugbus-regionalverkehr-alb-bodensee-gtfs-773", - "au-new-south-wales-train-replacement-bus-operators-gtfs-1322" + "de-unknown-rursee-schifffahrt-kg-gtfs-784" ] # Google Cloud constants From 4b5fe913137f4211b1a9f649d21e1f16ea0689b9 Mon Sep 17 00:00:00 2001 From: cka-y Date: Tue, 10 Oct 2023 16:45:18 -0400 Subject: [PATCH 5/6] fix: removing source causing timeout --- scripts/mobility-database-harvester/harvest_latest_versions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/mobility-database-harvester/harvest_latest_versions.py b/scripts/mobility-database-harvester/harvest_latest_versions.py index 1c1dd75f6f..fd4549749c 100644 --- a/scripts/mobility-database-harvester/harvest_latest_versions.py +++ b/scripts/mobility-database-harvester/harvest_latest_versions.py @@ -32,7 +32,8 @@ # Sources to exclude because they are too big for the workflow. SOURCES_TO_EXCLUDE = [ - "de-unknown-rursee-schifffahrt-kg-gtfs-784" + "de-unknown-rursee-schifffahrt-kg-gtfs-784", + "de-unknown-ulmer-eisenbahnfreunde-gtfs-1081" ] # Google Cloud constants From c00c6a07dea38f47e44fdb44a84862693d44564a Mon Sep 17 00:00:00 2001 From: cka-y Date: Tue, 10 Oct 2023 17:48:14 -0400 Subject: [PATCH 6/6] fix: removing source causing timeout --- scripts/mobility-database-harvester/harvest_latest_versions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/mobility-database-harvester/harvest_latest_versions.py b/scripts/mobility-database-harvester/harvest_latest_versions.py index fd4549749c..0ddef96fba 100644 --- a/scripts/mobility-database-harvester/harvest_latest_versions.py +++ b/scripts/mobility-database-harvester/harvest_latest_versions.py @@ -33,7 +33,8 @@ # Sources to exclude because they are too big for the workflow. SOURCES_TO_EXCLUDE = [ "de-unknown-rursee-schifffahrt-kg-gtfs-784", - "de-unknown-ulmer-eisenbahnfreunde-gtfs-1081" + "de-unknown-ulmer-eisenbahnfreunde-gtfs-1081", + "no-unknown-agder-kollektivtrafikk-as-gtfs-1078" ] # Google Cloud constants