Get mod info in 3-page batches to reduce call count

README.md

@@ -48,7 +48,7 @@ Get Maven, then run the following command at the project root:
 mvn clean package
 ```
 
-This will build the project to `target/update-checker-0.0.14.jar`.
+This will build the project to `target/update-checker-0.0.15.jar`.
 
 ## Running the project
 
@@ -61,7 +61,7 @@ First, follow these steps to set it up:
 Then, to run the project, browse to where the JAR is in a terminal / command prompt, then run
 
 ```
-java -jar update-checker-0.0.14.jar [port] [minutes]
+java -jar update-checker-0.0.15.jar [port] [minutes]
 ```
 
 [port] is the HTTP port for the server. If you don't provide any, there won't be any server hosted (useful if you already have something else hosting the files).

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.max480.everest.updatechecker</groupId>
     <artifactId>update-checker</artifactId>
-    <version>0.0.14</version>
+    <version>0.0.15</version>
 
     <properties>
         <jdk.version>1.8</jdk.version>
@@ -73,7 +73,7 @@
         <dependency>
             <groupId>org.yaml</groupId>
             <artifactId>snakeyaml</artifactId>
-            <version>1.27</version>
+            <version>1.28</version>
         </dependency>
 
         <!-- https://mvnrepository.com/artifact/commons-io/commons-io -->

src/main/java/com/max480/everest/updatechecker/DatabaseUpdater.java

@@ -22,6 +22,9 @@
 class DatabaseUpdater {
     private static final Logger log = LoggerFactory.getLogger(DatabaseUpdater.class);
 
+    // the update checker will load X pages, then get the mod info for those X pages in 1 call.
+    private static final int PAGE_GROUP_COUNT = 3;
+
     private Map<String, Mod> database = new HashMap<>();
     private Map<String, String> databaseExcludedFiles = new HashMap<>();
     private Set<String> databaseNoYamlFiles = new HashSet<>();
@@ -42,7 +45,7 @@ void updateDatabaseYaml() throws IOException {
 
         int i = 1;
         while (loadPage(i)) {
-            i++;
+            i += PAGE_GROUP_COUNT;
         }
 
         checkForModDeletion();
@@ -132,14 +135,27 @@ private QueriedModInfo(String itemtype, int itemid) {
     private boolean loadPage(int page) throws IOException {
         log.debug("Loading page {} of the list of mods from GameBanana", page);
 
-        List<List<Object>> mods = runWithRetry(() -> {
-            try (InputStream is = openStreamWithTimeout(new URL("https://api.gamebanana.com/Core/List/New?page=" + page + "&gameid=6460&format=yaml"))) {
-                return Optional.ofNullable(new Yaml().<List<List<Object>>>load(is))
-                        .orElseThrow(() -> new IOException("Ended up with a null value when loading a mod page"));
+        boolean reachedEmptyPage = false;
+
+        // we are going to get [PAGE_GROUP_COUNT] pages and merge them all in a mods list.
+        List<List<Object>> mods = new LinkedList<>();
+        for (int i = 0; i < PAGE_GROUP_COUNT && !reachedEmptyPage; i++) {
+            final int pageOffset = page + i;
+            List<List<Object>> modsPage = runWithRetry(() -> {
+                try (InputStream is = openStreamWithTimeout(new URL("https://api.gamebanana.com/Core/List/New?page=" + pageOffset + "&gameid=6460&format=yaml"))) {
+                    return Optional.ofNullable(new Yaml().<List<List<Object>>>load(is))
+                            .orElseThrow(() -> new IOException("Ended up with a null value when loading a mod page"));
+                }
+            });
+            mods.addAll(modsPage);
+
+            if (modsPage.isEmpty()) {
+                reachedEmptyPage = true;
             }
-        });
+        }
 
-        if (mods.isEmpty()) return false;
+        // stop here if no mod was retrieved.
+        if (mods.isEmpty()) return !reachedEmptyPage;
 
         /*
             List of GameBanana types with whether or not they accept files:
@@ -190,7 +206,7 @@ private boolean loadPage(int page) throws IOException {
         String urlModInfo = getModInfoCallUrl(queriedModInfo);
         loadPageModInfo(urlModInfo, queriedModInfo);
 
-        return true;
+        return !reachedEmptyPage;
     }
 
     /**