Add Xgc options for suballocator heap size and quick allocation

Signed-off-by: Nathan Henderson <nathan.henderson@ibm.com>

fvtest/porttest/omrmemTest.cpp

@@ -56,7 +56,7 @@ extern PortTestEnvironment *portTestEnv;
 #endif
 
 #if defined(OMR_ENV_DATA64)
-/* this macro corresponds to the one defined in omrmem32helpers */
+/* This macro corresponds to SUBALLOCATOR_INCREMENT_SIZE defined in omrgcconsts.h. */
 #define HEAP_SIZE_BYTES 8*1024*1024
 #endif
 

gc/base/GCExtensionsBase.hpp

@@ -744,6 +744,10 @@ class MM_GCExtensionsBase : public MM_BaseVirtual {
 	bool scavengerAlignHotFields; /**< True if the scavenger is to check the hot field description for an object in order to better cache align it when tenuring (enabled with the -Xgc:hotAlignment option) */
 	uintptr_t suballocatorInitialSize; /**< the initial chunk size in bytes for the J9Heap suballocator (enabled with the -Xgc:suballocatorInitialSize option) */
 	uintptr_t suballocatorCommitSize; /**< the commit size in bytes for the J9Heap suballocator (enabled with the -Xgc:suballocatorCommitSize option) */
+	uintptr_t suballocatorIncrementSize;
+#if defined(OMR_ENV_DATA64) && defined(LINUX)
+	bool suballocatorQuickAlloc;
+#endif /* defined(OMR_ENV_DATA64) && defined(LINUX) */
 
 #if defined(OMR_GC_COMPRESSED_POINTERS)
 	bool shouldAllowShiftingCompression; /**< temporary option to enable compressed reference scaling by shifting pointers */
@@ -1868,6 +1872,10 @@ class MM_GCExtensionsBase : public MM_BaseVirtual {
 		, scavengerAlignHotFields(true) /* VM Design 1774: hot field alignment is on by default */
 		, suballocatorInitialSize(SUBALLOCATOR_INITIAL_SIZE) /* default for J9Heap suballocator initial size is 200 MB */
 		, suballocatorCommitSize(SUBALLOCATOR_COMMIT_SIZE) /* default for J9Heap suballocator commit size is 50 MB */
+		, suballocatorIncrementSize(SUBALLOCATOR_INCREMENT_SIZE) /* default for J9Heap suballocator commit size is 8 MB or 256 MB for AIX */
+#if defined(OMR_ENV_DATA64) && defined(LINUX)
+		, suballocatorQuickAlloc(true) /* use mmap-based allocation by default for the J9Heap suballoctor */
+#endif /* defined(OMR_ENV_DATA64) && defined(LINUX) */
 #if defined(OMR_GC_COMPRESSED_POINTERS)
 		, shouldAllowShiftingCompression(true) /* VM Design 1810: shifting compression enabled, by default, for compressed refs */
 		, shouldForceSpecifiedShiftingCompression(0)

gc/base/MemoryManager.cpp

@@ -174,6 +174,14 @@ MM_MemoryManager::createVirtualMemoryForHeap(MM_EnvironmentBase *env, MM_MemoryH
 			/* Set the commit size for the sub allocator. This needs to be completed before the call to omrmem_ensure_capacity32 */
 			omrport_control(OMRPORT_CTLDATA_ALLOCATE32_COMMIT_SIZE, extensions->suballocatorCommitSize);
 
+			/* Set the increment size for the sub allocator. This needs to be completed before the call to omrmem_ensure_capacity32 */
+			omrport_control(OMRPORT_CTLDATA_ALLOCATE32_INCREMENT_SIZE, extensions->suballocatorIncrementSize);
+
+#if defined(OMR_ENV_DATA64) && defined(LINUX)
+			/* Set if the sub allocator should use ALLOC_QUICK. This needs to be completed before the call to omrmem_ensure_capacity32 */
+			omrport_control(OMRPORT_CTLDATA_ALLOCATE32_QUICK_ALLOC, extensions->suballocatorQuickAlloc);
+#endif /* defined(OMR_ENV_DATA64) && defined(LINUX) */
+
 			if (!shouldHeapBeAllocatedFirst) {
 				if (OMRPORT_ENSURE_CAPACITY_FAILED == omrmem_ensure_capacity32(extensions->suballocatorInitialSize)) {
 					extensions->heapInitializationFailureReason = MM_GCExtensionsBase::HEAP_INITIALIZATION_FAILURE_REASON_CAN_NOT_ALLOCATE_LOW_MEMORY_RESERVE;

include_core/omrgcconsts.h

@@ -564,6 +564,17 @@ typedef enum {
 #define SUBALLOCATOR_ALIGNMENT (8*1024*1024)
 #endif /* defined(AIXPPC) */
 
+/* VMDESIGN 1761 The size of a suballocation heap.
+ * See VMDESIGN 1761 for the rationale behind the selection of this size.
+ * We use a 8MB heap to give us more room in case an application loads a larger amount of classes than usual.
+ * For testing purposes, this value is mirrored in port library test. If we tune this value, we should also adjust it in omrmemTest.cpp
+ */
+#if defined(AIXPPC) && defined(OMR_GC_COMPRESSED_POINTERS)
+#define SUBALLOCATOR_INCREMENT_SIZE (256*1024*1024)
+#else /* defined(AIXPPC) && defined(OMR_GC_COMPRESSED_POINTERS) */
+#define SUBALLOCATOR_INCREMENT_SIZE (8*1024*1024)
+#endif /* defined(AIXPPC) && defined(OMR_GC_COMPRESSED_POINTERS) */
+
 #define MAXIMUM_HEAP_SIZE_RECOMMENDED_FOR_COMPRESSEDREFS            ((U_64)57 * 1024 * 1024 * 1024)
 #define MAXIMUM_HEAP_SIZE_RECOMMENDED_FOR_3BIT_SHIFT_COMPRESSEDREFS ((U_64)25 * 1024 * 1024 * 1024)
 

include_core/omrport.h

@@ -877,6 +877,10 @@ typedef struct J9ProcessorInfos {
 #define OMRPORT_CTLDATA_MEM_CATEGORIES_SET  "MEM_CATEGORIES_SET"
 #define OMRPORT_CTLDATA_AIX_PROC_ATTR  "AIX_PROC_ATTR"
 #define OMRPORT_CTLDATA_ALLOCATE32_COMMIT_SIZE  "ALLOCATE32_COMMIT_SIZE"
+#define OMRPORT_CTLDATA_ALLOCATE32_INCREMENT_SIZE  "ALLOCATE32_INCREMENT_SIZE"
+#if defined(OMR_ENV_DATA64) && defined(LINUX)
+#define OMRPORT_CTLDATA_ALLOCATE32_QUICK_ALLOC  "ALLOCATE32_QUICK_ALLOC"
+#endif /* defined(OMR_ENV_DATA64) && defined(LINUX) */
 #define OMRPORT_CTLDATA_NOSUBALLOC32BITMEM  "NOSUBALLOC32BITMEM"
 #define OMRPORT_CTLDATA_VMEM_ADVISE_OS_ONFREE  "VMEM_ADVISE_OS_ONFREE"
 #define OMRPORT_CTLDATA_VECTOR_REGS_SUPPORT_ON  "VECTOR_REGS_SUPPORT_ON"
@@ -2483,6 +2487,8 @@ typedef struct OMRPortLibrary {
 	int32_t (*port_startup_library)(struct OMRPortLibrary *portLibrary) ;
 	/** see @ref omrport.c::omrport_create_library "omrport_create_library"*/
 	int32_t (*port_create_library)(struct OMRPortLibrary *portLibrary, uintptr_t size) ;
+	/** see @ref omrport.c::omrport_copy_suballocator_heap_globals "omrport_copy_suballocator_heap_globals"*/
+	void (*port_copy_suballocator_heap_globals)(struct OMRPortLibrary *destPortLibrary, struct OMRPortLibrary *srcPortLibrary) ;
 	/** see @ref omrsyslog.c::omrsyslog_write "omrsyslog_write"*/
 	uintptr_t (*syslog_write)(struct OMRPortLibrary *portLibrary, uintptr_t flags, const char *message) ;
 	/** see @ref omrintrospect.c::omrintrospect_startup "omrintrospect_startup"*/

port/common/omrmem32helpers.c

@@ -37,26 +37,14 @@ static void *reserveAndCommitRegion(struct OMRPortLibrary *portLibrary, uintptr_
 #define VMEM_MODE_COMMIT OMRPORT_VMEM_MEMORY_MODE_READ | OMRPORT_VMEM_MEMORY_MODE_WRITE | OMRPORT_VMEM_MEMORY_MODE_COMMIT
 #define VMEM_MODE_WITHOUT_COMMIT OMRPORT_VMEM_MEMORY_MODE_READ | OMRPORT_VMEM_MEMORY_MODE_WRITE
 
+#define MEM32_LIMIT 0XFFFFFFFF
 struct {
 	uintptr_t base;
 	uintptr_t limit;
 } regions[] = {
-	{0x0, 0xFFFFFFFF}
+	{0x0, MEM32_LIMIT}
 };
 
-#define MEM32_LIMIT 0XFFFFFFFF
-
-/* VMDESIGN 1761 The size of a suballocation heap.
- * See VMDESIGN 1761 for the rationale behind the selection of this size.
- * We use a 8MB heap to give us more room in case an application loads a larger amount of classes than usual.
- * For testing purposes, this value is mirrored in port library test. If we tune this value, we should also adjust it in omrmemTest.cpp
- */
-#if defined(AIXPPC) && defined(OMR_GC_COMPRESSED_POINTERS)
-/* virtual memory is allocated in 256M segments on AIX, so grab the whole segment */
-#define HEAP_SIZE_BYTES (256 * 1024 * 1024)
-#else
-#define HEAP_SIZE_BYTES (8 * 1024 * 1024)
-#endif
 /* Creates any of the resources required to use allocate_memory32
  *
  * Note: Any resources created here need to be cleaned up in shutdown_memory32_using_vmem
@@ -72,6 +60,10 @@ startup_memory32(struct OMRPortLibrary *portLibrary)
 	PPG_mem_mem32_subAllocHeapMem32.subCommitHeapWrapper = NULL;
 	PPG_mem_mem32_subAllocHeapMem32.suballocator_initialSize = 0;
 	PPG_mem_mem32_subAllocHeapMem32.suballocator_commitSize = 0;
+	PPG_mem_mem32_subAllocHeapMem32.suballocator_incrementSize = 0;
+#if defined(OMR_ENV_DATA64) && defined(LINUX)
+	PPG_mem_mem32_subAllocHeapMem32.suballocator_quickAlloc = TRUE;
+#endif /* defined(OMR_ENV_DATA64) && defined(LINUX) */
 
 	/* initialize the monitor in subAllocHeap32 */
 	if (0 != omrthread_monitor_init(&(PPG_mem_mem32_subAllocHeapMem32.monitor), 0)) {
@@ -438,15 +430,28 @@ allocate_memory32(struct OMRPortLibrary *portLibrary, uintptr_t byteAmount, cons
 #endif
 		omrthread_monitor_enter(PPG_mem_mem32_subAllocHeapMem32.monitor);
 
-		/* Check if byteAmount is larger than HEAP_SIZE_BYTES.
+		/* Check if byteAmount is larger than PPG_mem_mem32_subAllocHeapMem32.suballocator_incrementSize.
 		 * The majority of size requests will typically be much smaller.
 		 */
 		returnPtr = iterateHeapsAndSubAllocate(portLibrary, byteAmount);
 		if (NULL == returnPtr) {
-			if (byteAmount >= HEAP_SIZE_BYTES) {
-				returnPtr = allocateLargeRegion(portLibrary, byteAmount, callSite, 0);
+			if (byteAmount >= PPG_mem_mem32_subAllocHeapMem32.suballocator_incrementSize) {
+				returnPtr = allocateLargeRegion(portLibrary,
+								byteAmount,
+								callSite,
+								0);
 			} else {
-				returnPtr = allocateRegion(portLibrary, HEAP_SIZE_BYTES, byteAmount, callSite, 0);
+#if defined(OMR_ENV_DATA64) && defined(LINUX)
+				/* For 64 bit linux, use the OMRPORT_VMEM_ALLOC_QUICK flag if it has not been disabled. */
+				uintptr_t vmemAllocOptions = PPG_mem_mem32_subAllocHeapMem32.suballocator_quickAlloc ? OMRPORT_VMEM_ALLOC_QUICK : 0;
+#else /* defined(OMR_ENV_DATA64) && defined(LINUX) */
+				uintptr_t vmemAllocOptions = 0;
+#endif /* defined(OMR_ENV_DATA64) && defined(LINUX) */
+				returnPtr = allocateRegion(portLibrary,
+							   PPG_mem_mem32_subAllocHeapMem32.suballocator_incrementSize,
+							   byteAmount,
+							   callSite,
+							   vmemAllocOptions);
 			}
 		}
 
@@ -465,12 +470,12 @@ ensure_capacity32(struct OMRPortLibrary *portLibrary, uintptr_t byteAmount)
 {
 	J9HeapWrapper *heapWrapperCursor = NULL;
 	uintptr_t returnValue = OMRPORT_ENSURE_CAPACITY_FAILED;
-#if defined(OMR_ENV_DATA64)
-	/* For 64 bit os, use flag OMRPORT_VMEM_ALLOC_QUICK as it is in the startup period. */
+#if defined(OMR_ENV_DATA64) && defined(LINUX)
+	/* For 64 bit linux, use flag OMRPORT_VMEM_ALLOC_QUICK as it is in the startup period. */
 	uintptr_t vmemAllocOptions = OMRPORT_VMEM_ALLOC_QUICK;
-#else
+#else /* defined(OMR_ENV_DATA64) && defined(LINUX) */
 	uintptr_t vmemAllocOptions = 0;
-#endif
+#endif /* defined(OMR_ENV_DATA64) && defined(LINUX) */
 
 	Trc_PRT_mem_ensure_capacity32_Entry(byteAmount);
 
@@ -481,9 +486,9 @@ ensure_capacity32(struct OMRPortLibrary *portLibrary, uintptr_t byteAmount)
 	}
 #endif
 
-	/* Ensured byte amount should be at least HEAP_SIZE_BYTES large */
-	if (byteAmount < HEAP_SIZE_BYTES) {
-		byteAmount = HEAP_SIZE_BYTES;
+	/* Ensured byte amount should be at least PPG_mem_mem32_subAllocHeapMem32.suballocator_incrementSize large. */
+	if (byteAmount < PPG_mem_mem32_subAllocHeapMem32.suballocator_incrementSize) {
+		byteAmount = PPG_mem_mem32_subAllocHeapMem32.suballocator_incrementSize;
 	}
 
 	omrthread_monitor_enter(PPG_mem_mem32_subAllocHeapMem32.monitor);

port/common/omrmem32struct.h

@@ -41,6 +41,10 @@ typedef struct J9SubAllocateHeapMem32 {
 	J9HeapWrapper *subCommitHeapWrapper;
 	uintptr_t suballocator_initialSize;
 	uintptr_t suballocator_commitSize;
+	uintptr_t suballocator_incrementSize;
+#if defined(OMR_ENV_DATA64) && defined(LINUX)
+	uintptr_t suballocator_quickAlloc;
+#endif /* defined(OMR_ENV_DATA64) && defined(LINUX) */
 } J9SubAllocateHeapMem32;
 
 #endif	/* omrmem32struct_h */

port/common/omrport.c

@@ -339,6 +339,7 @@ static OMRPortLibrary MainPortLibraryTable = {
 	omrport_init_library, /* port_init_library */
 	omrport_startup_library, /* port_startup_library */
 	omrport_create_library, /* port_create_library */
+	omrport_copy_suballocator_heap_globals, /* port_copy_suballocator_heap_globals */
 	omrsyslog_write, /* syslog_write */
 	omrintrospect_startup, /* introspect_startup */
 	omrintrospect_shutdown, /* introspect_shutdown */
@@ -589,6 +590,27 @@ omrport_create_library(struct OMRPortLibrary *portLibrary, uintptr_t size)
 	return 0;
 }
 
+/**
+ * Copy the subAllocHeapMem32 values from a source OMRPortLibrary to a destination OMRPortLibrary.
+ *
+ * This is a helper for ensuring memCheckPortLib picks up the subAllocHeapMem32 default and/or -Xgc
+ * specified values.
+ *
+ * @param[in,out] destPortLibrary The port library to copy the port globals to.
+ * @param[in,out] sourcePortLibrary The port library to copy the port globals from.
+ *
+ */
+void
+omrport_copy_suballocator_heap_globals(struct OMRPortLibrary *destPortLibrary, struct OMRPortLibrary *srcPortLibrary) {
+	J9SubAllocateHeapMem32 subAllocGlobals = srcPortLibrary->portGlobals->platformGlobals.subAllocHeapMem32;
+	omrport_control(destPortLibrary, OMRPORT_CTLDATA_ALLOCATE32_COMMIT_SIZE, subAllocGlobals.suballocator_commitSize);
+	omrport_control(destPortLibrary, OMRPORT_CTLDATA_ALLOCATE32_INCREMENT_SIZE, subAllocGlobals.suballocator_incrementSize);
+#if defined(LINUX)
+	omrport_control(destPortLibrary, OMRPORT_CTLDATA_ALLOCATE32_QUICK_ALLOC, subAllocGlobals.suballocator_quickAlloc);
+#endif /* defined(LINUX) */
+	return;
+}
+
 /**
  * PortLibrary startup.
  *

port/common/omrportcontrol.c

@@ -72,8 +72,39 @@ omrport_control(struct OMRPortLibrary *portLibrary, const char *key, uintptr_t v
 			return (int32_t)PPG_mem_mem32_subAllocHeapMem32.suballocator_commitSize;
 		}
 		return 0;
+	} else if (!strcmp(OMRPORT_CTLDATA_ALLOCATE32_INCREMENT_SIZE, key)) {
+		if (0 != value) {
+			/* IncrementSize is immutable. It can only be set once. */
+			if (0 == PPG_mem_mem32_subAllocHeapMem32.suballocator_incrementSize) {
+				/* Round up the increment size to the page size and set it to global variable */
+				uintptr_t pageSize = portLibrary->vmem_supported_page_sizes(portLibrary)[0];
+				uintptr_t roundedIncrementSize = pageSize * (value / pageSize);
+				if (roundedIncrementSize < value) {
+					roundedIncrementSize += pageSize;
+				}
+				PPG_mem_mem32_subAllocHeapMem32.suballocator_incrementSize = roundedIncrementSize;
+			} else {
+				return 1;
+			}
+		} else {
+			return (int32_t)PPG_mem_mem32_subAllocHeapMem32.suballocator_incrementSize;
+		}
+		return 0;
 	}
-#endif
+#if defined(LINUX)
+	else if (!strcmp(OMRPORT_CTLDATA_ALLOCATE32_QUICK_ALLOC, key)) {
+		if (FALSE == value) {
+			/* QuickAlloc is immutable. It can only be set once. */
+			if (TRUE == PPG_mem_mem32_subAllocHeapMem32.suballocator_quickAlloc) {
+				PPG_mem_mem32_subAllocHeapMem32.suballocator_quickAlloc = FALSE;
+			} else {
+				return 1;
+			}
+		}
+		return 0;
+	}
+#endif /* defined(LINUX) */
+#endif /* defined(OMR_ENV_DATA64) */
 
 #if defined(OMR_RAS_TDF_TRACE)
 	if (!strcmp(OMRPORT_CTLDATA_TRACE_START, key) && value) {

port/omrportpriv.h

@@ -931,6 +931,8 @@ extern J9_CFUNC int32_t
 omrport_startup_library(struct OMRPortLibrary *portLibrary);
 extern J9_CFUNC int32_t
 omrport_create_library(struct OMRPortLibrary *portLibrary, uintptr_t size);
+extern J9_CFUNC void
+omrport_copy_suballocator_heap_globals(struct OMRPortLibrary *destPortLibrary, struct OMRPortLibrary *srcPortLibrary);
 
 /* J9Syslog */
 extern J9_CFUNC uintptr_t