Merge pull request #125 from jgi-kbase/dev-mongo7_upgrade

Mongo7 upgrade

.github/workflows/test.yml

@@ -22,13 +22,10 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          # the current production setup
           - java: '8'
-            mongo: 'mongodb-linux-x86_64-3.6.13'
-            gradle_test: 'test'
+            mongo: 'mongodb-linux-x86_64-ubuntu2204-7.0.4'
           - java: '11'
-            mongo: 'mongodb-linux-x86_64-3.6.23'
-            gradle_test: 'test'
+            mongo: 'mongodb-linux-x86_64-ubuntu2004-4.4.29'
     steps:
       - uses: actions/checkout@v3
 
@@ -70,7 +67,7 @@ jobs:
         run: |
           PATH=$PATH:$MASH_PATH
           echo $PATH
-          ./gradlew ${{matrix.gradle_test}}
+          ./gradlew test
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3

build.gradle

@@ -114,7 +114,10 @@ dependencies {
 	implementation 'org.ini4j:ini4j:0.5.2'
 	implementation 'commons-io:commons-io:2.4'
 	implementation 'com.beust:jcommander:1.72'
-	implementation 'org.mongodb:mongo-java-driver:3.10.1'
+	implementation 'org.mongodb:mongodb-driver-core:4.11.1'
+	implementation 'org.mongodb:mongodb-driver-sync:4.11.1'
+	implementation 'org.mongodb:bson-record-codec:4.11.1'
+	implementation 'org.mongodb:bson:4.11.1'
 	implementation 'com.google.guava:guava:18.0'
 	implementation "com.github.kbase:auth2_client_java:0.5.0"
 	implementation('com.github.kbase:java_common:0.3.0') {

src/main/java/us/kbase/assemblyhomology/build/AssemblyHomologyBuilder.java

@@ -9,8 +9,9 @@
 
 import org.slf4j.LoggerFactory;
 
-import com.mongodb.MongoClient;
-import com.mongodb.MongoClientOptions;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
+import com.mongodb.MongoClientSettings;
 import com.mongodb.MongoCredential;
 import com.mongodb.MongoException;
 import com.mongodb.ServerAddress;
@@ -79,15 +80,16 @@ public AssemblyHomologyBuilder(
 
 	private MongoClient buildMongo(final AssemblyHomologyConfig c) throws StorageInitException {
 		//TODO ZLATER MONGO handle shards & replica sets
+		final MongoClientSettings.Builder mongoBuilder = MongoClientSettings.builder().applyToClusterSettings(
+				builder -> builder.hosts(Arrays.asList(new ServerAddress(c.getMongoHost()))));
 		try {
 			if (c.getMongoUser().isPresent()) {
 				final MongoCredential creds = MongoCredential.createCredential(
 						c.getMongoUser().get(), c.getMongoDatabase(), c.getMongoPwd().get());
 				// unclear if and when it's safe to clear the password
-				return new MongoClient(new ServerAddress(c.getMongoHost()), creds,
-						MongoClientOptions.builder().build());
+				return MongoClients.create(mongoBuilder.credential(creds).build());
 			} else {
-				return new MongoClient(new ServerAddress(c.getMongoHost()));
+				return MongoClients.create(mongoBuilder.build());
 			}
 		} catch (MongoException e) {
 			LoggerFactory.getLogger(getClass()).error(

src/main/java/us/kbase/assemblyhomology/cli/AssemblyHomologyCLI.java

@@ -14,7 +14,7 @@
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.ParameterException;
 import com.beust.jcommander.Parameters;
-import com.mongodb.MongoClient;
+import com.mongodb.client.MongoClient;
 
 import ch.qos.logback.classic.Level;
 import ch.qos.logback.classic.Logger;

src/main/java/us/kbase/assemblyhomology/service/AssemblyHomologyService.java

@@ -8,7 +8,7 @@
 import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.jaxrs.json.JacksonJaxbJsonProvider;
-import com.mongodb.MongoClient;
+import com.mongodb.client.MongoClient;
 
 import ch.qos.logback.classic.Level;
 import ch.qos.logback.classic.Logger;

src/test/java/us/kbase/test/assemblyhomology/MongoStorageTestManager.java

@@ -9,7 +9,8 @@
 import org.bson.Document;
 
 import com.github.zafarkhaja.semver.Version;
-import com.mongodb.MongoClient;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
 import com.mongodb.client.MongoDatabase;
 
 import us.kbase.assemblyhomology.storage.mongo.MongoAssemblyHomologyStorage;
@@ -35,7 +36,7 @@ public MongoStorageTestManager(final String dbName) throws Exception {
 		wiredTiger = TestCommon.useWiredTigerEngine();
 		System.out.println(String.format("Testing against mongo executable %s on port %s",
 				TestCommon.getMongoExe(), mongo.getServerPort()));
-		mc = new MongoClient("localhost:" + mongo.getServerPort());
+		mc = MongoClients.create("mongodb://localhost:" + mongo.getServerPort());
 		db = mc.getDatabase(dbName);
 
 		final Document bi = db.runCommand(new Document("buildinfo", 1));

src/test/java/us/kbase/test/assemblyhomology/storage/MongoAssemblyHomologyStorageStartupTest.java

@@ -99,9 +99,9 @@ public void startUpWith2ConfigDocs() throws Exception {
 
 		final Pattern errorPattern = Pattern.compile(
 				"Failed to create index: Write failed with error code 11000 and error message " +
-				"'(exception: )?E11000 duplicate key error (index|collection): " +
+				"'(exception: )?(.*)E11000 duplicate key error (index|collection): " +
 				"startUpWith2ConfigDocs.config( index: |\\.\\$)schema_1\\s+dup key: " +
-				"\\{ : \"schema\" \\}'");
+				"(\\{ schema: \"schema\" \\}'|\\{ : \"schema\" \\}')");
 		try {
 			new MongoAssemblyHomologyStorage(db);
 			fail("started mongo with bad config");
@@ -174,56 +174,59 @@ public void checkCollectionNames() throws Exception {
 
 	@Test
 	public void indexesConfig() {
-		final Set<Document> indexes = new HashSet<>();
-		manager.db.getCollection("config").listIndexes()
-				.forEach((Consumer<Document>) indexes::add);
+		final Set<Document> indexes = getAndNormalizeIndexes("config");
 		assertThat("incorrect indexes", indexes, is(set(
 				new Document("v", manager.indexVer)
 						.append("unique", true)
 						.append("key", new Document("schema", 1))
-						.append("name", "schema_1")
-						.append("ns", "test_mongoahstorage.config"),
+						.append("name", "schema_1"),
 				new Document("v", manager.indexVer)
 						.append("key", new Document("_id", 1))
 						.append("name", "_id_")
-						.append("ns", "test_mongoahstorage.config")
 				)));
 	}
 
 	@Test
 	public void indexesNamespace() {
-		final Set<Document> indexes = new HashSet<>();
-		manager.db.getCollection("namesp").listIndexes()
-				.forEach((Consumer<Document>) indexes::add);
+		final Set<Document> indexes = getAndNormalizeIndexes("namesp");
 		assertThat("incorrect indexes", indexes, is(set(
 				new Document("v", manager.indexVer)
 						.append("unique", true)
 						.append("key", new Document("id", 1))
-						.append("name", "id_1")
-						.append("ns", "test_mongoahstorage.namesp"),
+						.append("name", "id_1"),
 				new Document("v", manager.indexVer)
 						.append("key", new Document("_id", 1))
 						.append("name", "_id_")
-						.append("ns", "test_mongoahstorage.namesp")
 				)));
 	}
 
 	@Test
 	public void indexesSeqMeta() {
-		final Set<Document> indexes = new HashSet<>();
-		manager.db.getCollection("seqmeta").listIndexes()
-				.forEach((Consumer<Document>) indexes::add);
+		final Set<Document> indexes = getAndNormalizeIndexes("seqmeta");
 		assertThat("incorrect indexes", indexes, is(set(
 				new Document("v", manager.indexVer)
 						.append("unique", true)
 						.append("key", new Document("nsid", 1)
 								.append("load", 1).append("seqid", 1))
-						.append("name", "nsid_1_load_1_seqid_1")
-						.append("ns", "test_mongoahstorage.seqmeta"),
+						.append("name", "nsid_1_load_1_seqid_1"),
 				new Document("v", manager.indexVer)
 						.append("key", new Document("_id", 1))
 						.append("name", "_id_")
-						.append("ns", "test_mongoahstorage.seqmeta")
 				)));
 	}
+
+	private Set<Document> getAndNormalizeIndexes(final String collectionName) {
+		final Set<Document> indexes = new HashSet<>();
+		for (Document index: manager.db.getCollection(collectionName).listIndexes()) {
+			// In MongoDB 4.4, the listIndexes and the mongo shell helper method db.collection.getIndexes()
+			// no longer returns the namespace ns field in the index specification documents.
+			index.remove("ns");
+			// some versions of Mongo return ints, some longs. Convert all to longs.
+			if (index.containsKey("expireAfterSeconds")) {
+				index.put("expireAfterSeconds", ((Number) index.get("expireAfterSeconds")).longValue());
+			}
+			indexes.add(index);
+		}
+		return indexes;
+	}
 }