Implement multithreading

Author: Meeples10

README.md

@@ -31,7 +31,7 @@ java -jar mc-resource-analyzer-x.x.x.jar [-hHmsStV] [-B=PATH] [-M=PATH] [-o=STRI
 - `-T`, `--table-template`: When used in conjunction with `table`, the generated table will replace any instances of the string `{{{TABLE}}}` in a copy of the template file. Note that the table will not include `<table></table>` tags when using this argument.
 - `-s`, `--statistics`: Outputs a file with statistics about the analysis.
 - `-o`, `--output-prefix`: Use this argument to add a prefix to the program's output files. For example, using `-o abc` would result in the files `abc.csv` and `abc_table.html`.
-- `-v`, `--version-select`: Use this argument if you want to analyze a world that was not generated with the latest version of Minecraft. Selecting a version that does not match the version in which the regions were generated may result in unexpected behavior. The following versions are supported:
+- `-v`, `--version-select`: Use this argument if you want to analyze a world that was not generated with the latest version of Minecraft. Selecting a version that does not match the version with which the regions were generated may result in unexpected behavior. The following versions are supported:
   - `ANVIL_118` for 1.18
   - `ANVIL_2021` for 1.16 to 1.17
   - `ANVIL_2018` for 1.13 to 1.15
@@ -43,6 +43,7 @@ java -jar mc-resource-analyzer-x.x.x.jar [-hHmsStV] [-B=PATH] [-M=PATH] [-o=STRI
 - `-B`, `--block-ids`: When using the `--modernize-ids` option on a world with block IDs outside the range of 0-255, use this to specify the path to a file containing block IDs in the same format as [blocks.properties](https://github.com/Meeples10/MCResourceAnalyzer/blob/master/src/main/resources/blocks.properties).
 - `-M`, `--merge-ids`: When analyzing a world with block IDs outside the range of 0-255, use this to specify the path to a file containing block IDs in the same format as [merge.properties](https://github.com/Meeples10/MCResourceAnalyzer/blob/master/src/main/resources/merge.properties). Any block with an ID listed in this file will have all of its variants merged into a single value.
 - `-H`, `--no-hack`: The program attempts to compensate for the aforementioned inaccuracies at high Y values by assuming that empty chunk sections are filled with air. Use this argument to disable this hack.
+- `-n`, `--num-threads` (default: `8`): The maximum number of threads to use for analysis.
 - `-S`, `--silent`: Prevents the program from printing output, other than errors. This may result in marginally improved performance.
 
 ### Version compatibility

src/main/java/io/github/meeples10/mcresourceanalyzer/Analysis.java

@@ -0,0 +1,9 @@
+package io.github.meeples10.mcresourceanalyzer;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class Analysis {
+    public Map<String, Long> blocks = new HashMap<String, Long>();
+    public Map<String, HashMap<Integer, Long>> heights = new HashMap<String, HashMap<Integer, Long>>();
+}

src/main/java/io/github/meeples10/mcresourceanalyzer/AnalyzerThread.java

@@ -1,12 +1,15 @@
 package io.github.meeples10.mcresourceanalyzer;
 
-public abstract class AnalyzerThread implements Runnable {
-    Region region;
-    Chunk chunk;
+import java.util.HashSet;
+import java.util.Set;
 
-    public AnalyzerThread(Region region, Chunk chunk) {
-        this.region = region;
-        this.chunk = chunk;
+public abstract class AnalyzerThread extends Thread {
+    final Region r;
+    final Set<Analysis> analyses = new HashSet<>();
+
+    public AnalyzerThread(Region region) {
+        r = region;
+        setName(r.name);
     }
 
     public abstract void run();

src/main/java/io/github/meeples10/mcresourceanalyzer/Chunk.java

@@ -1,3 +1,3 @@
 package io.github.meeples10.mcresourceanalyzer;
 
-public record Chunk(int x, int z) {}
+record Chunk(int x, int z) {}

src/main/java/io/github/meeples10/mcresourceanalyzer/Main.java

@@ -25,7 +25,6 @@
 import picocli.CommandLine.ParseResult;
 
 public class Main {
-    public static final int THREAD_COUNT = 8;
     public static final DateFormat DATE_FORMAT = new SimpleDateFormat("dd MMM yyyy 'at' hh:mm:ss a zzz");
     public static final FilenameFilter DS_STORE_FILTER = new FilenameFilter() {
         @Override
@@ -46,6 +45,7 @@ public boolean accept(File dir, String name) {
     static String outputPrefix = "";
     static String tableTemplatePath = "";
     static String tableTemplate = "";
+    static int numThreads = 8;
 
     public static void main(String[] args) {
         CommandLine commandLine = new CommandLine(createCommandSpec());
@@ -124,19 +124,21 @@ private static CommandSpec createCommandSpec() {
                 "The program attempts to compensate for inaccuracies at high Y values by assuming that empty chunk"
                         + "sections are filled with air. Use this option to disable this hack.")
                 .build());
+        spec.addOption(OptionSpec.builder("-n", "--num-threads").paramLabel("COUNT").type(int.class)
+                .description("The maximum number of threads to use for analysis. (default: 8)").build());
         spec.addPositional(PositionalParamSpec.builder().paramLabel("INPUT").arity("0..1").type(String.class)
-                .description("The region directory or .mclevel file to analyze.").build());
+                .description("The region directory or .mclevel file to analyze. (default: 'region')").build());
         return spec;
     }
 
     private static int parseArgs(ParseResult pr) {
+        if(pr.hasMatchedPositional(0)) inputFile = new File((String) pr.matchedPositional(0).getValue());
         saveStatistics = pr.hasMatchedOption('s');
-        allowHack = pr.hasMatchedOption('H');
+        allowHack = !pr.hasMatchedOption('H');
         generateTable = pr.hasMatchedOption('t');
         modernizeIDs = pr.hasMatchedOption('m');
         silent = pr.hasMatchedOption('S');
         if(pr.hasMatchedOption('v')) selectedVersion = pr.matchedOption('v').getValue();
-        if(pr.hasMatchedPositional(0)) inputFile = new File((String) pr.matchedPositional(0).getValue());
         if(pr.hasMatchedOption('o')) outputPrefix = pr.matchedOption('o').getValue();
         if(pr.hasMatchedOption('B')) {
             try {
@@ -173,6 +175,9 @@ private static int parseArgs(ParseResult pr) {
                 System.exit(1);
             }
         }
+        if(pr.hasMatchedOption('n')) {
+            numThreads = pr.matchedOption('n').getValue();
+        }
         return 0;
     }
 
@@ -283,4 +288,8 @@ public static void print(Object s) {
     public static void println(Object s) {
         if(!silent) System.out.println(s);
     }
+
+    public static void printf(String format, Object... args) {
+        if(!silent) System.out.printf(format, args);
+    }
 }

src/main/java/io/github/meeples10/mcresourceanalyzer/Region.java

@@ -4,7 +4,7 @@
 import java.util.HashSet;
 import java.util.Set;
 
-public class Region {
+class Region {
     public final RegionFile file;
     public final String name;
     public final Set<Chunk> chunks = new HashSet<>();

src/main/java/io/github/meeples10/mcresourceanalyzer/RegionAnalyzer.java

@@ -4,23 +4,28 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
 public abstract class RegionAnalyzer {
     private Version version;
-    public long chunkCount = 0;
-    public Map<String, Long> blockCounter = new ConcurrentHashMap<String, Long>();
-    public Map<String, ConcurrentHashMap<Integer, Long>> heightCounter = new ConcurrentHashMap<String, ConcurrentHashMap<Integer, Long>>();
+    public int chunkCount = 0;
+    public Map<String, Long> blockCounter = new HashMap<String, Long>();
+    public Map<String, HashMap<Integer, Long>> heightCounter = new HashMap<String, HashMap<Integer, Long>>();
     private long firstStartTime;
     public long duration;
     List<Region> regions = new ArrayList<>();
-    int maxThreads = Main.THREAD_COUNT;
-    private AtomicInteger complete = new AtomicInteger(0);
+    Set<AnalyzerThread> threads = new HashSet<>();
+    private AtomicInteger completed = new AtomicInteger(0);
 
     public RegionAnalyzer() {
         firstStartTime = System.currentTimeMillis();
@@ -32,16 +37,52 @@ public void setVersion(Version version) {
 
     public void run(File input) {
         validateInput(input);
+
+        Main.print("Scanning for chunks... ");
         findChunks(input);
 
         for(Region r : regions) {
             chunkCount += r.size();
         }
-        System.out.printf("%d region%s, %d chunk%s found\n", regions.size(), regions.size() == 1 ? "s" : "", chunkCount,
-                chunkCount == 1 ? "s" : "");
-        maxThreads = Math.min(Main.THREAD_COUNT, regions.size());
+        Main.printf("%d region%s, %d chunk%s found\n", regions.size(), regions.size() == 1 ? "" : "s", chunkCount,
+                chunkCount == 1 ? "" : "s");
 
-        analyze(input);
+        analyze();
+
+        if(threads.size() > 0) {
+            System.out.println();
+            ExecutorService pool = Executors.newFixedThreadPool(Math.min(Main.numThreads, 1024));
+            for(AnalyzerThread t : threads) {
+                pool.submit(t);
+            }
+            pool.shutdown();
+            try {
+                pool.awaitTermination(100, TimeUnit.DAYS);
+            } catch(InterruptedException e) {
+                e.printStackTrace();
+            }
+            for(AnalyzerThread t : threads) {
+                for(Analysis a : t.analyses) {
+                    for(String s : a.blocks.keySet()) {
+                        blockCounter.put(s, blockCounter.getOrDefault(s, 0L) + a.blocks.get(s));
+                    }
+                    for(String s : a.heights.keySet()) {
+                        if(heightCounter.containsKey(s)) {
+                            Map<Integer, Long> existing = heightCounter.get(s);
+                            Map<Integer, Long> heights = a.heights.get(s);
+                            for(int i : heights.keySet()) {
+                                existing.put(i, existing.getOrDefault(i, 0L) + heights.get(i));
+                            }
+                        } else {
+                            heightCounter.put(s, a.heights.get(s));
+                        }
+                    }
+                }
+            }
+        }
+
+        duration = System.currentTimeMillis() - getStartTime();
+        Main.println("Completed analysis in " + Main.millisToHMS(duration));
 
         long totalBlocks = 0L;
         for(String key : blockCounter.keySet()) {
@@ -72,12 +113,7 @@ public int compare(String arg0, String arg1) {
             data.append(",");
         }
         data.append("total,percent_of_total,percent_excluding_air\n");
-        int digits = String.valueOf(blockCounter.size()).length();
-        String completionFormat = "[%0" + digits + "d/%0" + digits + "d]";
-        int keyIndex = 0;
         for(String key : heightCounter.keySet()) {
-            keyIndex += 1;
-            Main.print("\rGenerating CSV... " + String.format(completionFormat, keyIndex, blockCounter.size()));
             data.append(Main.modernizeIDs ? Main.getStringID(key) : key);
             data.append(",");
             for(int i = minY; i <= maxY; i++) {
@@ -100,10 +136,11 @@ public int compare(String arg0, String arg1) {
             }
             data.append("\n");
         }
+        Main.println("Done");
         try {
             File out = new File(Main.getOutputPrefix() + ".csv");
             Main.writeStringToFile(out, data.toString());
-            Main.println("\nData written to " + out.getAbsolutePath());
+            Main.println("CSV written to " + out.getAbsolutePath());
         } catch(IOException e) {
             e.printStackTrace();
             System.exit(1);
@@ -134,13 +171,20 @@ public int compare(String arg0, String arg1) {
 
     public abstract void findChunks(File input);
 
-    public abstract void analyze(File input);
+    public abstract void analyze();
+
+    public synchronized void updateProgress() {
+        int c = completed.incrementAndGet() + 1;
+        if(c > chunkCount) return;
+        Main.print("Analyzing chunks [" + c + "/" + chunkCount + "]\r");
+        if(c == chunkCount) System.out.println("\n");
+    }
 
-    public void updateProgress() {
-        int c = complete.incrementAndGet();
-        if(c + 1 == regions.size()) {
-            System.out.println("DONE");
+    public synchronized void halt() {
+        for(AnalyzerThread t : threads) {
+            t.interrupt();
         }
+        System.exit(1);
     }
 
     public String generateTable(double totalBlocks, double totalExcludingAir) {
@@ -209,17 +253,17 @@ static boolean mergeStates(byte id) {
     }
 
     /* THIS IS A HACK TO ACCOUNT FOR NONEXISTENT SECTIONS AT HIGH Y VALUES */
-    void airHack(int sectionY, String airID) {
+    static void airHack(Analysis a, int sectionY, String airID) {
         if(Main.allowHack && sectionY < 15) {
-            if(!blockCounter.containsKey(airID)) blockCounter.put(airID, 0L);
-            if(!heightCounter.containsKey(airID)) heightCounter.put(airID, new ConcurrentHashMap<Integer, Long>());
+            if(!a.blocks.containsKey(airID)) a.blocks.put(airID, 0L);
+            if(!a.heights.containsKey(airID)) a.heights.put(airID, new HashMap<Integer, Long>());
             for(; sectionY < 16; sectionY++) {
-                blockCounter.put(airID, blockCounter.get(airID) + 4096L);
+                a.blocks.put(airID, a.blocks.get(airID) + 4096L);
                 for(int y = sectionY * 16; y < sectionY * 16 + 16; y++) {
-                    if(heightCounter.get(airID).containsKey(y)) {
-                        heightCounter.get(airID).put(y, heightCounter.get(airID).get(y) + 256L);
+                    if(a.heights.get(airID).containsKey(y)) {
+                        a.heights.get(airID).put(y, a.heights.get(airID).get(y) + 256L);
                     } else {
-                        heightCounter.get(airID).put(y, 256L);
+                        a.heights.get(airID).put(y, 256L);
                     }
                 }
             }

src/main/java/io/github/meeples10/mcresourceanalyzer/RegionAnalyzerAlpha.java

@@ -3,13 +3,15 @@
 import java.io.File;
 import java.io.FileInputStream;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
 
 import net.minecraft.nbt.CompressedStreamTools;
 import net.minecraft.nbt.NBTTagCompound;
 
 public class RegionAnalyzerAlpha extends RegionAnalyzer {
+    final List<File> chunkFiles = new ArrayList<>();
+    private Analysis a = new Analysis();
 
     @Override
     public void validateInput(File world) {
@@ -20,13 +22,7 @@ public void validateInput(File world) {
     }
 
     @Override
-    public void findChunks(File input) {
-        // TODO
-    }
-
-    @Override
-    public void analyze(File world) {
-        List<File> chunkFiles = new ArrayList<>();
+    public void findChunks(File world) {
         for(File f : world.listFiles(Main.DS_STORE_FILTER)) {
             if(!f.isDirectory()) continue;
             chunkFiles.addAll(traverseSubdirectories(f));
@@ -36,24 +32,23 @@ public void analyze(File world) {
             System.exit(1);
         }
         Main.println(chunkFiles.size() + " chunks found");
-        int cnum = 1;
+    }
+
+    @Override
+    public void analyze() {
+        int i = 1;
         for(File f : chunkFiles) {
-            long startTime = System.currentTimeMillis();
-            String name = f.getName();
-            System.out.print("Scanning chunk " + name + " [" + cnum + "/" + chunkFiles.size() + "]... ");
+            System.out.print("Scanning chunks [" + i + "/" + chunkFiles.size() + "]\r");
 
             try {
                 processChunk(f);
             } catch(Exception e) {
                 e.printStackTrace();
             }
-
-            Main.println(
-                    "Done (" + String.format("%.2f", (double) (System.currentTimeMillis() - startTime) / 1000) + "s)");
-            cnum++;
+            i++;
         }
-        duration = System.currentTimeMillis() - getStartTime();
-        Main.println(("Completed analysis in " + Main.millisToHMS(duration) + " (" + chunkCount + " chunks)"));
+        blockCounter.putAll(a.blocks);
+        heightCounter.putAll(a.heights);
     }
 
     private void processChunk(File chunkFile) throws Exception {
@@ -88,18 +83,18 @@ private void analyzeChunk(byte[] blocks, byte[] data) {
                                 : Byte.toString(blockID) + ":" + Byte.toString(blockData);
                     }
 
-                    if(blockCounter.containsKey(blockName)) {
-                        blockCounter.put(blockName, blockCounter.get(blockName) + 1L);
+                    if(a.blocks.containsKey(blockName)) {
+                        a.blocks.put(blockName, a.blocks.get(blockName) + 1L);
                     } else {
-                        blockCounter.put(blockName, 1L);
+                        a.blocks.put(blockName, 1L);
                     }
-                    if(!heightCounter.containsKey(blockName)) {
-                        heightCounter.put(blockName, new ConcurrentHashMap<Integer, Long>());
+                    if(!a.heights.containsKey(blockName)) {
+                        a.heights.put(blockName, new HashMap<Integer, Long>());
                     }
-                    if(heightCounter.get(blockName).containsKey(y)) {
-                        heightCounter.get(blockName).put(y, heightCounter.get(blockName).get(y) + 1L);
+                    if(a.heights.get(blockName).containsKey(y)) {
+                        a.heights.get(blockName).put(y, a.heights.get(blockName).get(y) + 1L);
                     } else {
-                        heightCounter.get(blockName).put(y, 1L);
+                        a.heights.get(blockName).put(y, 1L);
                     }
                 }
             }