Merge related HashMaps in FieldInfos#FieldNumbers into one map (#13460)

Merges all immutable attributes in FieldInfos.FieldNumbers into one hashmap saving memory when 
writing big indices. Fixes an exotic bug when calling clear where not all attributes were cleared.

Author: iverase

lucene/CHANGES.txt

@@ -265,7 +265,9 @@ Bug Fixes
 
 Other
 ---------------------
-(No changes)
+* GITHUB#13459: Merges all immutable attributes in FieldInfos.FieldNumbers into one Hashmap saving
+  memory when writing big indices. Fixes an exotic bug when calling clear where not all attributes
+  were cleared. (Ignacio Vera)
 
 ======================== Lucene 9.11.0 =======================
 

lucene/core/src/java/org/apache/lucene/index/FieldInfos.java

@@ -346,48 +346,31 @@ public FieldInfo fieldInfo(int fieldNumber) {
     return fieldNumber >= byNumber.length ? null : byNumber[fieldNumber];
   }
 
-  static final class FieldDimensions {
-    public final int dimensionCount;
-    public final int indexDimensionCount;
-    public final int dimensionNumBytes;
-
-    public FieldDimensions(int dimensionCount, int indexDimensionCount, int dimensionNumBytes) {
-      this.dimensionCount = dimensionCount;
-      this.indexDimensionCount = indexDimensionCount;
-      this.dimensionNumBytes = dimensionNumBytes;
-    }
-  }
-
-  static final class FieldVectorProperties {
-    final int numDimensions;
-    final VectorEncoding vectorEncoding;
-    final VectorSimilarityFunction similarityFunction;
-
-    FieldVectorProperties(
-        int numDimensions,
-        VectorEncoding vectorEncoding,
-        VectorSimilarityFunction similarityFunction) {
-      this.numDimensions = numDimensions;
-      this.vectorEncoding = vectorEncoding;
-      this.similarityFunction = similarityFunction;
-    }
-  }
+  private record FieldDimensions(
+      int dimensionCount, int indexDimensionCount, int dimensionNumBytes) {}
+
+  private record FieldVectorProperties(
+      int numDimensions,
+      VectorEncoding vectorEncoding,
+      VectorSimilarityFunction similarityFunction) {}
+
+  private record IndexOptionsProperties(boolean storeTermVectors, boolean omitNorms) {}
+
+  // We use this to enforce that a given field never
+  // changes DV type, even across segments / IndexWriter
+  // sessions:
+  private record FieldProperties(
+      int number,
+      IndexOptions indexOptions,
+      IndexOptionsProperties indexOptionsProperties,
+      DocValuesType docValuesType,
+      FieldDimensions fieldDimensions,
+      FieldVectorProperties fieldVectorProperties) {}
 
   static final class FieldNumbers {
 
     private final IntObjectHashMap<String> numberToName;
-    private final Map<String, Integer> nameToNumber;
-    private final Map<String, IndexOptions> indexOptions;
-    // We use this to enforce that a given field never
-    // changes DV type, even across segments / IndexWriter
-    // sessions:
-    private final Map<String, DocValuesType> docValuesType;
-
-    private final Map<String, FieldDimensions> dimensions;
-
-    private final Map<String, FieldVectorProperties> vectorProps;
-    private final Map<String, Boolean> omitNorms;
-    private final Map<String, Boolean> storeTermVectors;
+    private final Map<String, FieldProperties> fieldProperties;
 
     // TODO: we should similarly catch an attempt to turn
     // norms back on after they were already committed; today
@@ -401,14 +384,8 @@ static final class FieldNumbers {
     private final String parentFieldName;
 
     FieldNumbers(String softDeletesFieldName, String parentFieldName) {
-      this.nameToNumber = new HashMap<>();
       this.numberToName = new IntObjectHashMap<>();
-      this.indexOptions = new HashMap<>();
-      this.docValuesType = new HashMap<>();
-      this.dimensions = new HashMap<>();
-      this.vectorProps = new HashMap<>();
-      this.omitNorms = new HashMap<>();
-      this.storeTermVectors = new HashMap<>();
+      this.fieldProperties = new HashMap<>();
       this.softDeletesFieldName = softDeletesFieldName;
       this.parentFieldName = parentFieldName;
       if (softDeletesFieldName != null
@@ -425,7 +402,7 @@ synchronized void verifyFieldInfo(FieldInfo fi) {
       String fieldName = fi.getName();
       verifySoftDeletedFieldName(fieldName, fi.isSoftDeletesField());
       verifyParentFieldName(fieldName, fi.isParentField());
-      if (nameToNumber.containsKey(fieldName)) {
+      if (fieldProperties.containsKey(fieldName)) {
         verifySameSchema(fi);
       }
     }
@@ -439,15 +416,15 @@ synchronized int addOrGet(FieldInfo fi) {
       String fieldName = fi.getName();
       verifySoftDeletedFieldName(fieldName, fi.isSoftDeletesField());
       verifyParentFieldName(fieldName, fi.isParentField());
-      Integer fieldNumber = nameToNumber.get(fieldName);
+      var fieldProperties = this.fieldProperties.get(fieldName);
 
-      if (fieldNumber != null) {
+      if (fieldProperties != null) {
         verifySameSchema(fi);
       } else { // first time we see this field in this index
-        final Integer preferredBoxed = Integer.valueOf(fi.number);
-        if (fi.number != -1 && !numberToName.containsKey(preferredBoxed)) {
+        int fieldNumber;
+        if (fi.number != -1 && numberToName.containsKey(fi.number) == false) {
           // cool - we can use this number globally
-          fieldNumber = preferredBoxed;
+          fieldNumber = fi.number;
         } else {
           // find a new FieldNumber
           while (numberToName.containsKey(++lowestUnassignedFieldNumber)) {
@@ -457,25 +434,25 @@ synchronized int addOrGet(FieldInfo fi) {
         }
         assert fieldNumber >= 0;
         numberToName.put(fieldNumber, fieldName);
-        nameToNumber.put(fieldName, fieldNumber);
-        this.indexOptions.put(fieldName, fi.getIndexOptions());
-        if (fi.getIndexOptions() != IndexOptions.NONE) {
-          this.storeTermVectors.put(fieldName, fi.hasVectors());
-          this.omitNorms.put(fieldName, fi.omitsNorms());
-        }
-        docValuesType.put(fieldName, fi.getDocValuesType());
-        dimensions.put(
-            fieldName,
-            new FieldDimensions(
-                fi.getPointDimensionCount(),
-                fi.getPointIndexDimensionCount(),
-                fi.getPointNumBytes()));
-        vectorProps.put(
-            fieldName,
-            new FieldVectorProperties(
-                fi.getVectorDimension(), fi.getVectorEncoding(), fi.getVectorSimilarityFunction()));
+        fieldProperties =
+            new FieldProperties(
+                fieldNumber,
+                fi.getIndexOptions(),
+                fi.getIndexOptions() != IndexOptions.NONE
+                    ? new IndexOptionsProperties(fi.hasVectors(), fi.omitsNorms())
+                    : null,
+                fi.getDocValuesType(),
+                new FieldDimensions(
+                    fi.getPointDimensionCount(),
+                    fi.getPointIndexDimensionCount(),
+                    fi.getPointNumBytes()),
+                new FieldVectorProperties(
+                    fi.getVectorDimension(),
+                    fi.getVectorEncoding(),
+                    fi.getVectorSimilarityFunction()));
+        this.fieldProperties.put(fieldName, fieldProperties);
       }
-      return fieldNumber.intValue();
+      return fieldProperties.number;
     }
 
     private void verifySoftDeletedFieldName(String fieldName, boolean isSoftDeletesField) {
@@ -532,19 +509,20 @@ private void verifyParentFieldName(String fieldName, boolean isParentField) {
 
     private void verifySameSchema(FieldInfo fi) {
       String fieldName = fi.getName();
-      IndexOptions currentOpts = this.indexOptions.get(fieldName);
+      FieldProperties fieldProperties = this.fieldProperties.get(fieldName);
+      IndexOptions currentOpts = fieldProperties.indexOptions;
       verifySameIndexOptions(fieldName, currentOpts, fi.getIndexOptions());
       if (currentOpts != IndexOptions.NONE) {
-        boolean curStoreTermVector = this.storeTermVectors.get(fieldName);
+        boolean curStoreTermVector = fieldProperties.indexOptionsProperties.storeTermVectors;
         verifySameStoreTermVectors(fieldName, curStoreTermVector, fi.hasVectors());
-        boolean curOmitNorms = this.omitNorms.get(fieldName);
+        boolean curOmitNorms = fieldProperties.indexOptionsProperties.omitNorms;
         verifySameOmitNorms(fieldName, curOmitNorms, fi.omitsNorms());
       }
 
-      DocValuesType currentDVType = docValuesType.get(fieldName);
+      DocValuesType currentDVType = fieldProperties.docValuesType;
       verifySameDocValuesType(fieldName, currentDVType, fi.getDocValuesType());
 
-      FieldDimensions dims = dimensions.get(fieldName);
+      FieldDimensions dims = fieldProperties.fieldDimensions;
       verifySamePointsOptions(
           fieldName,
           dims.dimensionCount,
@@ -554,7 +532,7 @@ private void verifySameSchema(FieldInfo fi) {
           fi.getPointIndexDimensionCount(),
           fi.getPointNumBytes());
 
-      FieldVectorProperties props = vectorProps.get(fieldName);
+      FieldVectorProperties props = fieldProperties.fieldVectorProperties;
       verifySameVectorOptions(
           fieldName,
           props.numDimensions,
@@ -579,7 +557,7 @@ private void verifySameSchema(FieldInfo fi) {
      */
     synchronized void verifyOrCreateDvOnlyField(
         String fieldName, DocValuesType dvType, boolean fieldMustExist) {
-      if (nameToNumber.containsKey(fieldName) == false) {
+      if (fieldProperties.containsKey(fieldName) == false) {
         if (fieldMustExist) {
           throw new IllegalArgumentException(
               "Can't update ["
@@ -612,7 +590,8 @@ synchronized void verifyOrCreateDvOnlyField(
         }
       } else {
         // verify that field is doc values only field with the give doc values type
-        DocValuesType fieldDvType = docValuesType.get(fieldName);
+        FieldProperties fieldProperties = this.fieldProperties.get(fieldName);
+        DocValuesType fieldDvType = fieldProperties.docValuesType;
         if (dvType != fieldDvType) {
           throw new IllegalArgumentException(
               "Can't update ["
@@ -623,7 +602,7 @@ synchronized void verifyOrCreateDvOnlyField(
                   + fieldDvType
                   + "].");
         }
-        FieldDimensions fdimensions = dimensions.get(fieldName);
+        FieldDimensions fdimensions = fieldProperties.fieldDimensions;
         if (fdimensions != null && fdimensions.dimensionCount != 0) {
           throw new IllegalArgumentException(
               "Can't update ["
@@ -632,7 +611,7 @@ synchronized void verifyOrCreateDvOnlyField(
                   + fieldName
                   + "] must be doc values only field, but is also indexed with points.");
         }
-        IndexOptions ioptions = indexOptions.get(fieldName);
+        IndexOptions ioptions = fieldProperties.indexOptions;
         if (ioptions != null && ioptions != IndexOptions.NONE) {
           throw new IllegalArgumentException(
               "Can't update ["
@@ -641,7 +620,7 @@ synchronized void verifyOrCreateDvOnlyField(
                   + fieldName
                   + "] must be doc values only field, but is also indexed with postings.");
         }
-        FieldVectorProperties fvp = vectorProps.get(fieldName);
+        FieldVectorProperties fvp = fieldProperties.fieldVectorProperties;
         if (fvp != null && fvp.numDimensions != 0) {
           throw new IllegalArgumentException(
               "Can't update ["
@@ -664,14 +643,13 @@ synchronized void verifyOrCreateDvOnlyField(
      *     {@code dvType} returns a new FieldInfo based based on the options in global field numbers
      */
     FieldInfo constructFieldInfo(String fieldName, DocValuesType dvType, int newFieldNumber) {
-      Integer fieldNumber;
+      FieldProperties fieldProperties;
       synchronized (this) {
-        fieldNumber = nameToNumber.get(fieldName);
+        fieldProperties = this.fieldProperties.get(fieldName);
       }
-      if (fieldNumber == null) return null;
-      DocValuesType dvType0 = docValuesType.get(fieldName);
+      if (fieldProperties == null) return null;
+      DocValuesType dvType0 = fieldProperties.docValuesType;
       if (dvType != dvType0) return null;
-
       boolean isSoftDeletesField = fieldName.equals(softDeletesFieldName);
       boolean isParentField = fieldName.equals(parentFieldName);
       return new FieldInfo(
@@ -695,15 +673,12 @@ FieldInfo constructFieldInfo(String fieldName, DocValuesType dvType, int newFiel
     }
 
     synchronized Set<String> getFieldNames() {
-      return Set.copyOf(nameToNumber.keySet());
+      return Set.copyOf(fieldProperties.keySet());
     }
 
     synchronized void clear() {
       numberToName.clear();
-      nameToNumber.clear();
-      indexOptions.clear();
-      docValuesType.clear();
-      dimensions.clear();
+      fieldProperties.clear();
       lowestUnassignedFieldNumber = -1;
     }
   }