[GR-51211] Preparations for multiple image heaps at runtime.

substratevm/src/com.oracle.objectfile/src/com/oracle/objectfile/debugentry/DebugInfoBase.java

@@ -36,7 +36,6 @@
 
 import org.graalvm.collections.EconomicMap;
 import org.graalvm.collections.EconomicSet;
-import jdk.graal.compiler.debug.DebugContext;
 
 import com.oracle.objectfile.debugentry.range.PrimaryRange;
 import com.oracle.objectfile.debugentry.range.Range;
@@ -49,6 +48,7 @@
 import com.oracle.objectfile.debuginfo.DebugInfoProvider.DebugTypeInfo.DebugTypeKind;
 import com.oracle.objectfile.elf.dwarf.DwarfDebugInfo;
 
+import jdk.graal.compiler.debug.DebugContext;
 import jdk.vm.ci.meta.ResolvedJavaType;
 
 /**
@@ -351,10 +351,10 @@ public void installDebugInfo(DebugInfoProvider debugInfoProvider) {
                 String provenance = debugDataInfo.getProvenance();
                 String typeName = debugDataInfo.getTypeName();
                 String partitionName = debugDataInfo.getPartition();
-                /* Address is heap-register relative pointer. */
-                long address = debugDataInfo.getAddress();
+                /* Offset is relative to heap-base register. */
+                long offset = debugDataInfo.getOffset();
                 long size = debugDataInfo.getSize();
-                debugContext.log(DebugContext.INFO_LEVEL, "Data: address 0x%x size 0x%x type %s partition %s provenance %s ", address, size, typeName, partitionName, provenance);
+                debugContext.log(DebugContext.INFO_LEVEL, "Data: offset 0x%x size 0x%x type %s partition %s provenance %s ", offset, size, typeName, partitionName, provenance);
             }
         }));
         // populate a file and dir list and associated index for each class entry

substratevm/src/com.oracle.objectfile/src/com/oracle/objectfile/debuginfo/DebugInfoProvider.java

@@ -31,11 +31,11 @@
 import java.util.function.Consumer;
 import java.util.stream.Stream;
 
+import jdk.graal.compiler.debug.DebugContext;
 import jdk.vm.ci.meta.JavaConstant;
 import jdk.vm.ci.meta.JavaKind;
 import jdk.vm.ci.meta.ResolvedJavaMethod;
 import jdk.vm.ci.meta.ResolvedJavaType;
-import jdk.graal.compiler.debug.DebugContext;
 
 /**
  * Interfaces used to allow a native image to communicate details of types, code and data to the
@@ -356,8 +356,6 @@ interface DebugDataInfo {
 
         long getOffset();
 
-        long getAddress();
-
         long getSize();
     }
 

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapLayouter.java

@@ -35,7 +35,6 @@
 import com.oracle.svm.core.genscavenge.ChunkedImageHeapAllocator.Chunk;
 import com.oracle.svm.core.genscavenge.ChunkedImageHeapAllocator.UnalignedChunk;
 import com.oracle.svm.core.genscavenge.remset.RememberedSet;
-import com.oracle.svm.core.heap.Heap;
 import com.oracle.svm.core.hub.DynamicHub;
 import com.oracle.svm.core.image.ImageHeap;
 import com.oracle.svm.core.image.ImageHeapLayoutInfo;
@@ -83,10 +82,9 @@ public class ChunkedImageHeapLayouter implements ImageHeapLayouter {
     private final long hugeObjectThreshold;
     private ChunkedImageHeapAllocator allocator;
 
+    /** @param startOffset Offset relative to the heap base. */
     @SuppressWarnings("this-escape")
     public ChunkedImageHeapLayouter(ImageHeapInfo heapInfo, long startOffset) {
-        assert startOffset == 0 || startOffset >= Heap.getHeap().getImageHeapOffsetInAddressSpace() : "must be relative to the heap base";
-
         this.partitions = new ChunkedImageHeapPartition[PARTITION_COUNT];
         this.partitions[READ_ONLY_PRIMITIVE] = new ChunkedImageHeapPartition("readOnlyPrimitive", false, false);
         this.partitions[READ_ONLY_REFERENCE] = new ChunkedImageHeapPartition("readOnlyReference", false, false);
@@ -209,34 +207,15 @@ private ImageHeapLayoutInfo populateInfoObjects(int dynamicHubCount) {
             break;
         }
 
-        initializeHeapInfo(dynamicHubCount, offsetOfFirstWritableAlignedChunk, offsetOfFirstWritableUnalignedChunk);
-        return createLayoutInfo(startOffset, offsetOfFirstWritableAlignedChunk);
-    }
-
-    private void initializeHeapInfo(int dynamicHubCount, long offsetOfFirstWritableAlignedChunk, long offsetOfFirstWritableUnalignedChunk) {
-        long writableAligned = offsetOfFirstWritableAlignedChunk;
-        long writableUnaligned = offsetOfFirstWritableUnalignedChunk;
-
-        if (startOffset == 0) {
-            // Adjust all offsets by the offset of the image heap in the address space.
-            int imageHeapOffsetInAddressSpace = Heap.getHeap().getImageHeapOffsetInAddressSpace();
-            writableAligned += imageHeapOffsetInAddressSpace;
-            if (writableUnaligned >= 0) {
-                writableUnaligned += imageHeapOffsetInAddressSpace;
-            }
-        }
-
         heapInfo.initialize(getReadOnlyPrimitive().firstObject, getReadOnlyPrimitive().lastObject, getReadOnlyReference().firstObject, getReadOnlyReference().lastObject,
                         getReadOnlyRelocatable().firstObject, getReadOnlyRelocatable().lastObject, getWritablePrimitive().firstObject, getWritablePrimitive().lastObject,
                         getWritableReference().firstObject, getWritableReference().lastObject, getWritableHuge().firstObject, getWritableHuge().lastObject,
-                        getReadOnlyHuge().firstObject, getReadOnlyHuge().lastObject, writableAligned, writableUnaligned, dynamicHubCount);
-    }
+                        getReadOnlyHuge().firstObject, getReadOnlyHuge().lastObject, offsetOfFirstWritableAlignedChunk, offsetOfFirstWritableUnalignedChunk, dynamicHubCount);
 
-    private ImageHeapLayoutInfo createLayoutInfo(long heapStartOffset, long writableBeginOffset) {
         long writableEnd = getWritableHuge().getStartOffset() + getWritableHuge().getSize();
-        long writableSize = writableEnd - writableBeginOffset;
-        long imageHeapSize = getReadOnlyHuge().getStartOffset() + getReadOnlyHuge().getSize() - heapStartOffset;
-        return new ImageHeapLayoutInfo(writableBeginOffset, writableSize, getReadOnlyRelocatable().getStartOffset(), getReadOnlyRelocatable().getSize(), imageHeapSize);
+        long writableSize = writableEnd - offsetOfFirstWritableAlignedChunk;
+        long imageHeapSize = getReadOnlyHuge().getStartOffset() + getReadOnlyHuge().getSize() - startOffset;
+        return new ImageHeapLayoutInfo(startOffset, offsetOfFirstWritableAlignedChunk, writableSize, getReadOnlyRelocatable().getStartOffset(), getReadOnlyRelocatable().getSize(), imageHeapSize);
     }
 
     @Override

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java

@@ -912,8 +912,9 @@ private void blackenDirtyImageHeapRoots() {
 
         Timer blackenImageHeapRootsTimer = timers.blackenImageHeapRoots.open();
         try {
-            ImageHeapInfo info = HeapImpl.getImageHeapInfo();
-            blackenDirtyImageHeapChunkRoots(info.getFirstWritableAlignedChunk(), info.getFirstWritableUnalignedChunk());
+            for (ImageHeapInfo info = HeapImpl.getFirstImageHeapInfo(); info != null; info = info.next) {
+                blackenDirtyImageHeapChunkRoots(info.getFirstWritableAlignedChunk(), info.getFirstWritableUnalignedChunk());
+            }
 
             if (AuxiliaryImageHeap.isPresent()) {
                 ImageHeapInfo auxInfo = AuxiliaryImageHeap.singleton().getImageHeapInfo();
@@ -959,7 +960,10 @@ private void blackenImageHeapRoots() {
 
         Timer blackenImageHeapRootsTimer = timers.blackenImageHeapRoots.open();
         try {
-            blackenImageHeapRoots(HeapImpl.getImageHeapInfo());
+            for (ImageHeapInfo info = HeapImpl.getFirstImageHeapInfo(); info != null; info = info.next) {
+                blackenImageHeapRoots(info);
+            }
+
             if (AuxiliaryImageHeap.isPresent()) {
                 ImageHeapInfo auxImageHeapInfo = AuxiliaryImageHeap.singleton().getImageHeapInfo();
                 if (auxImageHeapInfo != null) {

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java

@@ -108,7 +108,7 @@ public final class HeapImpl extends Heap {
     private final ObjectHeaderImpl objectHeaderImpl = new ObjectHeaderImpl();
     private final GCImpl gcImpl;
     private final RuntimeCodeInfoGCSupportImpl runtimeCodeInfoGcSupport;
-    private final ImageHeapInfo imageHeapInfo = new ImageHeapInfo();
+    private final ImageHeapInfo firstImageHeapInfo = new ImageHeapInfo();
     private final HeapAccounting accounting = new HeapAccounting();
 
     /** Head of the linked list of currently pending (ready to be enqueued) {@link Reference}s. */
@@ -143,8 +143,8 @@ public static HeapImpl getHeapImpl() {
     }
 
     @Fold
-    public static ImageHeapInfo getImageHeapInfo() {
-        return getHeapImpl().imageHeapInfo;
+    public static ImageHeapInfo getFirstImageHeapInfo() {
+        return getHeapImpl().firstImageHeapInfo;
     }
 
     @Fold
@@ -179,7 +179,12 @@ public boolean isInPrimaryImageHeap(Object obj) {
     @Override
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public boolean isInPrimaryImageHeap(Pointer objPointer) {
-        return imageHeapInfo.isInImageHeap(objPointer);
+        for (ImageHeapInfo info = firstImageHeapInfo; info != null; info = info.next) {
+            if (info.isInImageHeap(objPointer)) {
+                return true;
+            }
+        }
+        return false;
     }
 
     @Override
@@ -283,7 +288,9 @@ void logChunks(Log log) {
 
     void logImageHeapPartitionBoundaries(Log log) {
         log.string("Native image heap boundaries:").indent(true);
-        imageHeapInfo.print(log);
+        for (ImageHeapInfo info = firstImageHeapInfo; info != null; info = info.next) {
+            info.print(log);
+        }
         log.indent(false);
 
         if (AuxiliaryImageHeap.isPresent()) {
@@ -326,22 +333,28 @@ static void logZapValues(Log log) {
     @Override
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public int getClassCount() {
-        return imageHeapInfo.dynamicHubCount;
+        int count = firstImageHeapInfo.dynamicHubCount;
+        for (ImageHeapInfo info = firstImageHeapInfo.next; info != null; info = info.next) {
+            count += info.dynamicHubCount;
+        }
+        return count;
     }
 
     @Override
     protected List<Class<?>> getAllClasses() {
         /* Two threads might race to set classList, but they compute the same result. */
         if (classList == null) {
             ArrayList<Class<?>> list = new ArrayList<>(1024);
-            ImageHeapWalker.walkRegions(imageHeapInfo, new ClassListBuilderVisitor(list));
+            for (ImageHeapInfo info = firstImageHeapInfo; info != null; info = info.next) {
+                ImageHeapWalker.walkRegions(info, new ClassListBuilderVisitor(list));
+            }
             list.trimToSize();
 
             /* Ensure that other threads see consistent values once the list is published. */
             MembarNode.memoryBarrier(MembarNode.FenceKind.STORE_STORE);
             classList = list;
         }
-        assert classList.size() == imageHeapInfo.dynamicHubCount;
+        assert classList.size() == getClassCount();
         return classList;
     }
 
@@ -429,11 +442,15 @@ public int getImageHeapOffsetInAddressSpace() {
     @Override
     public boolean walkImageHeapObjects(ObjectVisitor visitor) {
         VMOperation.guaranteeInProgressAtSafepoint("Must only be called at a safepoint");
-        if (visitor != null) {
-            return ImageHeapWalker.walkImageHeapObjects(imageHeapInfo, visitor) &&
-                            (!AuxiliaryImageHeap.isPresent() || AuxiliaryImageHeap.singleton().walkObjects(visitor));
+        if (visitor == null) {
+            return true;
         }
-        return true;
+        for (ImageHeapInfo info = firstImageHeapInfo; info != null; info = info.next) {
+            if (!ImageHeapWalker.walkImageHeapObjects(info, visitor)) {
+                return false;
+            }
+        }
+        return !AuxiliaryImageHeap.isPresent() || AuxiliaryImageHeap.singleton().walkObjects(visitor);
     }
 
     @Override
@@ -694,28 +711,32 @@ static Pointer getImageHeapStart() {
     }
 
     private boolean printLocationInfo(Log log, Pointer ptr, boolean allowJavaHeapAccess, boolean allowUnsafeOperations) {
-        if (imageHeapInfo.isInReadOnlyPrimitivePartition(ptr)) {
-            log.string("points into the image heap (read-only primitives)");
-            return true;
-        } else if (imageHeapInfo.isInReadOnlyReferencePartition(ptr)) {
-            log.string("points into the image heap (read-only references)");
-            return true;
-        } else if (imageHeapInfo.isInReadOnlyRelocatablePartition(ptr)) {
-            log.string("points into the image heap (read-only relocatables)");
-            return true;
-        } else if (imageHeapInfo.isInWritablePrimitivePartition(ptr)) {
-            log.string("points into the image heap (writable primitives)");
-            return true;
-        } else if (imageHeapInfo.isInWritableReferencePartition(ptr)) {
-            log.string("points into the image heap (writable references)");
-            return true;
-        } else if (imageHeapInfo.isInWritableHugePartition(ptr)) {
-            log.string("points into the image heap (writable huge)");
-            return true;
-        } else if (imageHeapInfo.isInReadOnlyHugePartition(ptr)) {
-            log.string("points into the image heap (read-only huge)");
-            return true;
-        } else if (AuxiliaryImageHeap.isPresent() && AuxiliaryImageHeap.singleton().containsObject(ptr)) {
+        for (ImageHeapInfo info = firstImageHeapInfo; info != null; info = info.next) {
+            if (info.isInReadOnlyPrimitivePartition(ptr)) {
+                log.string("points into the image heap (read-only primitives)");
+                return true;
+            } else if (info.isInReadOnlyReferencePartition(ptr)) {
+                log.string("points into the image heap (read-only references)");
+                return true;
+            } else if (info.isInReadOnlyRelocatablePartition(ptr)) {
+                log.string("points into the image heap (read-only relocatables)");
+                return true;
+            } else if (info.isInWritablePrimitivePartition(ptr)) {
+                log.string("points into the image heap (writable primitives)");
+                return true;
+            } else if (info.isInWritableReferencePartition(ptr)) {
+                log.string("points into the image heap (writable references)");
+                return true;
+            } else if (info.isInWritableHugePartition(ptr)) {
+                log.string("points into the image heap (writable huge)");
+                return true;
+            } else if (info.isInReadOnlyHugePartition(ptr)) {
+                log.string("points into the image heap (read-only huge)");
+                return true;
+            }
+        }
+
+        if (AuxiliaryImageHeap.isPresent() && AuxiliaryImageHeap.singleton().containsObject(ptr)) {
             log.string("points into the auxiliary image heap");
             return true;
         } else if (printTlabInfo(log, ptr, CurrentIsolate.getCurrentThread())) {

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java

@@ -68,9 +68,10 @@ public static boolean verify(Occasion occasion) {
 
     private static boolean verifyImageHeap() {
         boolean success = true;
-        ImageHeapInfo info = HeapImpl.getImageHeapInfo();
-        success &= verifyAlignedChunks(null, info.getFirstWritableAlignedChunk());
-        success &= verifyUnalignedChunks(null, info.getFirstWritableUnalignedChunk());
+        for (ImageHeapInfo info = HeapImpl.getFirstImageHeapInfo(); info != null; info = info.next) {
+            success &= verifyAlignedChunks(null, info.getFirstWritableAlignedChunk());
+            success &= verifyUnalignedChunks(null, info.getFirstWritableUnalignedChunk());
+        }
         return success;
     }
 
@@ -145,9 +146,10 @@ private static boolean verifyRememberedSets() {
          * For the image heap, we can't verify that all cards are clean after a GC because the GC
          * itself may result in dirty cards.
          */
-        ImageHeapInfo info = HeapImpl.getImageHeapInfo();
-        success &= rememberedSet.verify(info.getFirstWritableAlignedChunk());
-        success &= rememberedSet.verify(info.getFirstWritableUnalignedChunk());
+        for (ImageHeapInfo info = HeapImpl.getFirstImageHeapInfo(); info != null; info = info.next) {
+            success &= rememberedSet.verify(info.getFirstWritableAlignedChunk());
+            success &= rememberedSet.verify(info.getFirstWritableUnalignedChunk());
+        }
 
         OldGeneration oldGeneration = HeapImpl.getHeapImpl().getOldGeneration();
         Space toSpace = oldGeneration.getToSpace();

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java

@@ -79,7 +79,14 @@ public final class ImageHeapInfo {
 
     @UnknownPrimitiveField(availability = AfterHeapLayout.class) public int dynamicHubCount;
 
+    public final ImageHeapInfo next;
+
     public ImageHeapInfo() {
+        this(null);
+    }
+
+    public ImageHeapInfo(ImageHeapInfo next) {
+        this.next = next;
     }
 
     @SuppressWarnings("hiding")

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java

@@ -142,13 +142,13 @@ public void beforeAnalysis(BeforeAnalysisAccess access) {
 
     @Override
     public void afterAnalysis(AfterAnalysisAccess access) {
-        ImageHeapLayouter heapLayouter = new ChunkedImageHeapLayouter(HeapImpl.getImageHeapInfo(), 0);
+        ImageHeapLayouter heapLayouter = new ChunkedImageHeapLayouter(HeapImpl.getFirstImageHeapInfo(), Heap.getHeap().getImageHeapOffsetInAddressSpace());
         ImageSingletons.add(ImageHeapLayouter.class, heapLayouter);
     }
 
     @Override
     public void beforeCompilation(BeforeCompilationAccess access) {
-        ImageHeapInfo imageHeapInfo = HeapImpl.getImageHeapInfo();
+        ImageHeapInfo imageHeapInfo = HeapImpl.getFirstImageHeapInfo();
         access.registerAsImmutable(imageHeapInfo);
     }