fix(CalendarDatesMergeLineContext): Add missing converter for GTFS+ calendar_attributes.

Author: binh-dam-ibigroup

src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarAttributesMergeLineContext.java

@@ -0,0 +1,90 @@
+package com.conveyal.datatools.manager.jobs.feedmerge;
+
+import com.conveyal.datatools.manager.jobs.MergeFeedsJob;
+import com.conveyal.gtfs.error.NewGTFSError;
+import com.conveyal.gtfs.loader.Table;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Set;
+import java.util.zip.ZipOutputStream;
+
+import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD;
+import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getTableScopedValue;
+import static com.conveyal.datatools.manager.utils.MergeFeedUtils.hasDuplicateError;
+
+public class CalendarAttributesMergeLineContext extends MergeLineContext {
+    private static final Logger LOG = LoggerFactory.getLogger(CalendarAttributesMergeLineContext.class);
+
+    public CalendarAttributesMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException {
+        super(job, table, out);
+    }
+
+    @Override
+    public boolean checkFieldsForMergeConflicts(Set<NewGTFSError> idErrors, FieldContext fieldContext) {
+        return checkCalendarIds(idErrors, fieldContext);
+    }
+
+    @Override
+    public void afterRowWrite() throws IOException {
+        // If the current row is for a calendar service_id that is marked for cloning/renaming, clone the
+        // values, change the ID, extend the start/end dates to the feed's full range, and write the
+        // additional line to the file.
+        addClonedServiceId();
+    }
+
+    private boolean checkCalendarIds(Set<NewGTFSError> idErrors, FieldContext fieldContext) {
+        boolean shouldSkipRecord = false;
+
+        // If any service_id in the active feed matches with the future
+        // feed, it should be modified and all associated trip records
+        // must also be changed with the modified service_id.
+        // TODO How can we check that calendar_dates entries are
+        //  duplicates? I think we would need to consider the
+        //  service_id:exception_type:date as the unique key and include any
+        //  all entries as long as they are unique on this key.
+        if (isHandlingActiveFeed() && hasDuplicateError(idErrors)) {
+            // Modify service_id and ensure that referencing trips
+            // have service_id updated.
+            updateAndRemapOutput(fieldContext);
+        }
+
+        // Skip record (based on remapped id if necessary) if it was skipped in the calendar table.
+        String keyInCalendarTable = getTableScopedValue(Table.CALENDAR, getIdScope(), keyValue);
+        if (mergeFeedsResult.skippedIds.contains(keyInCalendarTable)) {
+            LOG.warn(
+                "Skipping calendar entry {} because it was skipped in the merged calendar table.",
+                keyValue);
+            shouldSkipRecord = true;
+        }
+
+        return !shouldSkipRecord;
+    }
+
+    /**
+     * Adds a cloned service id for trips with the same signature in both the active & future feeds.
+     * The cloned service id spans from the start date in the active feed until the end date in the future feed.
+     * @throws IOException
+     */
+    public void addClonedServiceId() throws IOException {
+        if (isHandlingFutureFeed() && job.mergeType.equals(SERVICE_PERIOD)) {
+            String originalServiceId = keyValue;
+            if (job.serviceIdsToCloneRenameAndExtend.contains(originalServiceId)) {
+                String[] clonedValues = getOriginalRowValues().clone();
+                String newServiceId = clonedValues[keyFieldIndex] = String.join(":", getIdScope(), originalServiceId);
+
+                referenceTracker.checkReferencesAndUniqueness(
+                    keyValue,
+                    getLineNumber(),
+                    table.fields[0],
+                    newServiceId,
+                    table,
+                    keyField,
+                    table.getOrderFieldName()
+                );
+                writeValuesToTable(clonedValues, true);
+            }
+        }
+    }
+}

src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java

@@ -84,7 +84,7 @@ private boolean checkCalendarDatesIds(FieldContext fieldContext) throws IOExcept
         }
 
 
-    // Track service ID because we want to avoid removing trips that may reference this
+        // Track service ID because we want to avoid removing trips that may reference this
         // service_id when the service_id is used by calendar.txt records that operate in
         // the valid date range, i.e., before the future feed's first date.
         if (!shouldSkipRecord && fieldContext.nameEquals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(fieldContext.getValueToWrite());

src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java

@@ -84,6 +84,8 @@ public static MergeLineContext create(MergeFeedsJob job, Table table, ZipOutputS
                 return new AgencyMergeLineContext(job, table, out);
             case "calendar":
                 return new CalendarMergeLineContext(job, table, out);
+            case "calendar_attributes":
+                return new CalendarAttributesMergeLineContext(job, table, out);
             case "calendar_dates":
                 return new CalendarDatesMergeLineContext(job, table, out);
             case "routes":

src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java

@@ -458,6 +458,13 @@ void mergeMTCShouldHandleMatchingTripIdsAndDropUnusedFutureCalendar() throws SQL
         // The calendar_dates entry should be preserved, but remapped to a different id.
         assertRowCountInTable(mergedNamespace, "calendar_dates", 1);
 
+        // The GTFS+ calendar_attributes table should contain the same number of entries as the calendar table.
+        assertEquals(
+            3,
+            mergeFeedsJob.mergeFeedsResult.linesPerTable.get("calendar_attributes").intValue(),
+            "Merged calendar_dates table count should equal expected value."
+        );
+
         assertNoUnusedServiceIds(mergedNamespace);
         assertNoRefIntegrityErrors(mergedNamespace);
     }

src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/calendar_attributes.txt

@@ -0,0 +1,3 @@
+service_id,service_description
+common_id,Description for common_id (added trips)
+only_calendar_id,Description for only_calendar_id (added trips)

src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/calendar_attributes.txt

@@ -0,0 +1,3 @@
+service_id,service_description
+common_id,Description for common_id
+only_calendar_id,Description for only_calendar_id