fix(FeedUpdater): Handle unknown feed ids from MTC.

Author: binh-dam-ibigroup

src/main/java/com/conveyal/datatools/manager/jobs/FeedUpdater.java

@@ -162,17 +162,17 @@ public Map<String, String> checkForUpdatedFeeds() {
             String filename = keyName.split("/")[1];
             String feedId = filename.replace(".zip", "");
             FeedSource feedSource = getFeedSource(feedId);
+            if (feedSource == null) {
+                LOG.error("No feed source found for feed ID {}", feedId);
+                continue;
+            }
 
             if (shouldMarkFeedAsProcessed(eTag, feedSource)) {
                 // Don't add object if it is a dir
                 // Skip object if the filename is null
                 if ("null".equals(feedId)) continue;
                 try {
                     LOG.info("New version found for {} at s3://{}/{}. ETag = {}.", feedId, feedBucket, keyName, eTag);
-                    if (feedSource == null) {
-                        LOG.error("No feed source found for feed ID {}", feedId);
-                        continue;
-                    }
                     updatePublishedFeedVersion(feedId, feedSource);
                     // TODO: Explore if MD5 checksum can be used to find matching feed version.
                     // findMatchingFeedVersion(md5, feedId, feedSource);

src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java

@@ -12,7 +12,6 @@
 import com.google.common.collect.Lists;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
@@ -32,6 +31,7 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 /**
@@ -131,8 +131,9 @@ private static Stream<Arguments> createPublishFeedCases() {
         );
     }
 
-    @Test
-    void shouldUpdateFeedInfoAfterPublishComplete() {
+    @ParameterizedTest
+    @MethodSource("createUpdateFeedInfoCases")
+    void shouldUpdateFeedInfoAfterPublishComplete(String agencyId, boolean isUnknownFeedId) {
         // Add the version to the feed source
         FeedVersion createdVersion = createFeedVersionFromGtfsZip(feedSource, "bart_new_lite.zip");
 
@@ -149,7 +150,7 @@ void shouldUpdateFeedInfoAfterPublishComplete() {
         assertNotNull(updatedFeedVersion.sentToExternalPublisher);
 
         // Create a test FeedUpdater instance, and simulate running the task.
-        TestCompletedFeedRetriever completedFeedRetriever = new TestCompletedFeedRetriever();
+        TestCompletedFeedRetriever completedFeedRetriever = new TestCompletedFeedRetriever(agencyId);
         FeedUpdater feedUpdater = FeedUpdater.createForTest(completedFeedRetriever);
 
         // The list of feeds processed externally (completed) should be empty at this point.
@@ -163,38 +164,66 @@ void shouldUpdateFeedInfoAfterPublishComplete() {
         // If a feed has been republished since last check, it will have a new etag/file hash,
         // and the scenario below should apply.
         Map<String, String> etagsAfter = feedUpdater.checkForUpdatedFeeds();
-        assertEquals(1, etagsAfter.size());
-        assertTrue(etagsAfter.containsValue("test-etag"));
 
-        // Make sure that the publish-complete attribute has been set for the feed version in Mongo.
         FeedVersion updatedFeedVersionAfter = Persistence.feedVersions.getById(createdVersion.id);
         Date updatedDate = updatedFeedVersionAfter.processedByExternalPublisher;
         String namespace = updatedFeedVersionAfter.namespace;
-        assertNotNull(updatedDate);
-
-        // At the next check for updates, the metadata for the feeds completed above
-        // should not be updated again.
-        feedUpdater.checkForUpdatedFeeds();
-        FeedVersion updatedFeedVersionAfter2 = Persistence.feedVersions.getById(createdVersion.id);
-        assertEquals(updatedDate, updatedFeedVersionAfter2.processedByExternalPublisher);
-        assertEquals(namespace, updatedFeedVersionAfter2.namespace);
+
+        if (!isUnknownFeedId) {
+            // Regular scenario: updating a known/existing feed.
+            assertEquals(1, etagsAfter.size());
+            assertTrue(etagsAfter.containsValue("test-etag"));
+
+            // Make sure that the publish-complete attribute has been set for the feed version in Mongo.
+            assertNotNull(updatedDate);
+
+            // At the next check for updates, the metadata for the feeds completed above
+            // should not be updated again.
+            feedUpdater.checkForUpdatedFeeds();
+            FeedVersion updatedFeedVersionAfter2 = Persistence.feedVersions.getById(createdVersion.id);
+            assertEquals(updatedDate, updatedFeedVersionAfter2.processedByExternalPublisher);
+            assertEquals(namespace, updatedFeedVersionAfter2.namespace);
+        } else {
+            // Edge case: an unknown feed id was provided,
+            // so no update of the feed should be happening (and there should not be an exception).
+            assertEquals(0, etagsAfter.size());
+            assertNull(updatedDate);
+        }
+    }
+
+    private static Stream<Arguments> createUpdateFeedInfoCases() {
+        return Stream.of(
+            Arguments.of(
+                TEST_AGENCY,
+                false
+            ),
+            Arguments.of(
+                "12345",
+                true
+            )
+        );
     }
 
     /**
      * Mocks the results of an {@link S3ObjectSummary} retrieval before/after the
      * external MTC publishing process is complete.
      */
     private static class TestCompletedFeedRetriever implements FeedUpdater.CompletedFeedRetriever {
+        private final String agencyId;
         public boolean isPublishingComplete;
 
+        public TestCompletedFeedRetriever(String agencyId) {
+            this.agencyId = agencyId;
+        }
+
         @Override
         public List<S3ObjectSummary> retrieveCompletedFeeds() {
             if (!isPublishingComplete) {
                 return new ArrayList<>();
             } else {
                 S3ObjectSummary objSummary = new S3ObjectSummary();
                 objSummary.setETag("test-etag");
-                objSummary.setKey(String.format("%s/%s", TEST_COMPLETED_FOLDER, TEST_AGENCY));
+                objSummary.setKey(String.format("%s/%s", TEST_COMPLETED_FOLDER, agencyId));
                 return Lists.newArrayList(objSummary);
             }
         }