Track the total usage of the code cache

compiler/control/OMROptions.cpp

@@ -1002,6 +1002,7 @@ TR::OptionTable OMR::Options::_jitOptions[] = {
    {"poisonDeadSlots",    "O\tpaints all dead slots with deadf00d", SET_OPTION_BIT(TR_PoisonDeadSlots), "F"},
    {"prepareForOSREvenIfThatDoesNothing",   "O\temit the call to prepareForOSR even if there is no slot sharing", SET_OPTION_BIT(TR_EnablePrepareForOSREvenIfThatDoesNothing), "F"},
    {"printAbsoluteTimestampInVerboseLog", "O\tPrint Absolute Timestamp in vlog", SET_OPTION_BIT(TR_PrintAbsoluteTimestampInVerboseLog), "F", NOT_IN_SUBSET},
+   {"printCodeCacheUsage",        "M\tPrint code cache usage at shutdown", SET_OPTION_BIT(TR_PrintCodeCacheUsage), "F", NOT_IN_SUBSET},
    {"printErrorInfoOnCompFailure",        "O\tPrint compilation error info to stderr", SET_OPTION_BIT(TR_PrintErrorInfoOnCompFailure), "F", NOT_IN_SUBSET},
    {"privatizeOverlaps",  "O\tif BCD storageRefs are going to overlap then do the move through a temp", SET_OPTION_BIT(TR_PrivatizeOverlaps), "F"},
    {"profile",            "O\tcompile a profiling method body", SET_OPTION_BIT(TR_Profile), "F"},

compiler/control/OMROptions.hpp

@@ -293,7 +293,7 @@ enum TR_CompilationOptions
    TR_UseSamplingJProfilingForAllFirstTimeComps   = 0x02000000 + 6,
    TR_NoStoreAOT                          = 0x04000000 + 6,
    TR_NoLoadAOT                           = 0x08000000 + 6,
-   // Available                           = 0x10000000 + 6,
+   TR_PrintCodeCacheUsage                 = 0x10000000 + 6,
    TR_UseSamplingJProfilingForDLT                 = 0x20000000 + 6,
    TR_UseSamplingJProfilingForInterpSampledMethods= 0x40000000 + 6,
    TR_EmitRelocatableELFFile              = 0x80000000 + 6,

compiler/runtime/OMRCodeCache.cpp

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2000, 2019 IBM Corp. and others
+ * Copyright (c) 2000, 2020 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -229,7 +229,7 @@ OMR::CodeCache::trimCodeMemoryAllocation(void *codeMemoryStart, size_t actualSiz
 
    if (expectedHeapAlloc == _warmCodeAlloc)
       {
-      _manager->increaseFreeSpaceInCodeCacheRepository(shrinkage);
+      _manager->decreaseCurrTotalUsedInBytes(shrinkage);
       _warmCodeAlloc -= shrinkage;
       cacheHeader->_size = actualSizeInBytes;
       return true;
@@ -402,9 +402,8 @@ OMR::CodeCache::initialize(TR::CodeCacheManager *manager,
 
    // Before returning, let's adjust the free space seen by VM.
    // Usable space is between _warmCodeAlloc and _trampolineBase. Everything else is overhead
-   // Only relevant if code cache repository is used
    size_t spaceLost = (_warmCodeAlloc - _segment->segmentBase()) + (_segment->segmentTop() - _trampolineBase);
-   _manager->decreaseFreeSpaceInCodeCacheRepository(spaceLost);
+   _manager->increaseCurrTotalUsedInBytes(spaceLost);
 
    return true;
    }
@@ -1065,6 +1064,8 @@ OMR::CodeCache::addFreeBlock2WithCallSite(uint8_t *start,
 
    self()->updateMaxSizeOfFreeBlocks(link, link->_size);
 
+   _manager->decreaseCurrTotalUsedInBytes(size);
+
    if (config.verboseReclamation())
       {
       TR_VerboseLog::writeLineLocked(TR_Vlog_CODECACHE,"--ccr-- addFreeBlock2WithCallSite CC=%p start=%p end=%p mergedBlock=%p link=%p link->_size=%u, _sizeOfLargestFreeWarmBlock=%d _sizeOfLargestFreeColdBlock=%d warmCodeAlloc=%p coldBlockAlloc=%p",
@@ -1215,6 +1216,8 @@ OMR::CodeCache::findFreeBlock(size_t size, bool isCold, bool isMethodHeaderNeede
          {
          TR_VerboseLog::writeLineLocked(TR_Vlog_CODECACHE,"--ccr- findFreeBlock: CodeCache=%p size=%u isCold=%d bestFitLink=%p bestFitLink->size=%u leftBlock=%p", this, size, isCold, bestFitLink, bestFitLink->_size, leftBlock);
          }
+
+      _manager->increaseCurrTotalUsedInBytes(bestFitLink->_size);
       }
    // Because we call this method only after we made sure a free block exists
    // this function can never return NULL
@@ -1288,6 +1291,7 @@ OMR::CodeCache::printOccupancyStats()
    fprintf(stderr, "Code Cache @%p flags=0x%x almostFull=%d\n", this, _flags, _almostFull);
    fprintf(stderr, "   cold-warm hole size        = %8" OMR_PRIuSIZE " bytes\n", self()->getFreeContiguousSpace());
    fprintf(stderr, "   warmCodeAlloc=%p coldCodeAlloc=%p\n", (void*)_warmCodeAlloc, (void*)_coldCodeAlloc);
+   size_t totalReclaimed = 0;
    if (_freeBlockList)
       {
       fprintf(stderr, "   sizeOfLargestFreeColdBlock = %8" OMR_PRIuSIZE " bytes\n", _sizeOfLargestFreeColdBlock);
@@ -1297,7 +1301,10 @@ OMR::CodeCache::printOccupancyStats()
          {
          CacheCriticalSection resolveAndCreateTrampoline(self());
          for (CodeCacheFreeCacheBlock *currLink = _freeBlockList; currLink; currLink = currLink->_next)
+            {
             fprintf(stderr, " %" OMR_PRIuSIZE, currLink->_size);
+            totalReclaimed += currLink->_size;
+            }
          }
       fprintf(stderr, "\n");
       }
@@ -1309,6 +1316,12 @@ OMR::CodeCache::printOccupancyStats()
          (int32_t)(_trampolineReservationMark - _trampolineBase),
          (int32_t)(_tempTrampolineNext - _tempTrampolineBase));
       }
+
+   size_t totalConfigSizeInBytes = config.codeCacheKB() * 1024;
+   size_t totalFreeSizeInBytes = self()->getFreeContiguousSpace() + totalReclaimed;
+   fprintf(stderr, "   config size     = %8" OMR_PRIuSIZE " bytes\n", totalConfigSizeInBytes);
+   fprintf(stderr, "   total free size = %8" OMR_PRIuSIZE " bytes\n", totalFreeSizeInBytes);
+   fprintf(stderr, "   total used size = %8" OMR_PRIuSIZE " bytes\n", totalConfigSizeInBytes - totalFreeSizeInBytes);
    }
 
 
@@ -1559,7 +1572,7 @@ OMR::CodeCache::allocateCodeMemory(size_t warmCodeSize,
 
          // _warmCodeAlloc will change to its new value 'cacheHeapAlloc'
          // Thus the free code cache space decreases by (cacheHeapAlloc-_warmCodeAlloc)
-         _manager->decreaseFreeSpaceInCodeCacheRepository(cacheHeapAlloc-_warmCodeAlloc);
+         _manager->increaseCurrTotalUsedInBytes(cacheHeapAlloc-_warmCodeAlloc);
          _warmCodeAlloc = cacheHeapAlloc;
          if (isMethodHeaderNeeded)
             self()->writeMethodHeader(warmCodeAddress, warmSize, false);
@@ -1598,7 +1611,7 @@ OMR::CodeCache::allocateCodeMemory(size_t warmCodeSize,
                }
             return NULL;
             }
-         _manager->decreaseFreeSpaceInCodeCacheRepository(_coldCodeAlloc - cacheHeapAlloc);
+         _manager->increaseCurrTotalUsedInBytes(_coldCodeAlloc - cacheHeapAlloc);
          _coldCodeAlloc = cacheHeapAlloc;
          coldCodeAddress = cacheHeapAlloc;
          if (isMethodHeaderNeeded)

compiler/runtime/OMRCodeCacheManager.cpp

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2000, 2019 IBM Corp. and others
+ * Copyright (c) 2000, 2020 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -56,7 +56,9 @@ TR::CodeCacheSymbolContainer * OMR::CodeCacheManager::_symbolContainer = NULL;
 OMR::CodeCacheManager::CodeCacheManager(TR::RawAllocator rawAllocator) :
    _rawAllocator(rawAllocator),
    _initialized(false),
-   _codeCacheFull(false)
+   _codeCacheFull(false),
+   _currTotalUsedInBytes(0),
+   _maxUsedInBytes(0)
    {
    }
 
@@ -78,6 +80,10 @@ OMR::CodeCacheManager::RepositoryMonitorCriticalSection::RepositoryMonitorCritic
    {
    }
 
+OMR::CodeCacheManager::UsageMonitorCriticalSection::UsageMonitorCriticalSection(TR::CodeCacheManager *mgr)
+   : CriticalSection(mgr->_usageMonitor)
+   {
+   }
 
 TR::CodeCache *
 OMR::CodeCacheManager::initialize(
@@ -141,6 +147,9 @@ OMR::CodeCacheManager::initialize(
    if (_codeCacheList._mutex == NULL)
       return NULL;
 
+   if (!(_usageMonitor = TR::Monitor::create("CodeCacheUsageMonitor")))
+      return NULL;
+
 #if defined(TR_HOST_POWER)
    #define REACHEABLE_RANGE_KB (32*1024)
 #else
@@ -958,6 +967,31 @@ OMR::CodeCacheManager::chooseCacheStartAddress(size_t repositorySize)
    }
 
 
+void
+OMR::CodeCacheManager::increaseCurrTotalUsedInBytes(size_t size)
+   {
+   self()->decreaseFreeSpaceInCodeCacheRepository(size);
+
+      {
+      UsageMonitorCriticalSection updateCodeCacheUsage(self());
+      _currTotalUsedInBytes += size;
+      _maxUsedInBytes = std::max(_maxUsedInBytes, _currTotalUsedInBytes);
+      }
+   }
+
+
+void
+OMR::CodeCacheManager::decreaseCurrTotalUsedInBytes(size_t size)
+   {
+   self()->increaseFreeSpaceInCodeCacheRepository(size);
+
+      {
+      UsageMonitorCriticalSection updateCodeCacheUsage(self());
+      _currTotalUsedInBytes = (size < _currTotalUsedInBytes) ? (_currTotalUsedInBytes - size) : 0;
+      }
+   }
+
+
 void
 OMR::CodeCacheManager::increaseFreeSpaceInCodeCacheRepository(size_t size)
    {

compiler/runtime/OMRCodeCacheManager.hpp

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2000, 2019 IBM Corp. and others
+ * Copyright (c) 2000, 2020 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -128,6 +128,12 @@ class OMR_EXTENSIBLE CodeCacheManager
       RepositoryMonitorCriticalSection(TR::CodeCacheManager *mgr);
       };
 
+   class UsageMonitorCriticalSection : public CriticalSection
+      {
+      public:
+      UsageMonitorCriticalSection(TR::CodeCacheManager *mgr);
+      };
+
    TR::CodeCacheConfig & codeCacheConfig() { return _config; }
 
    /**
@@ -274,6 +280,11 @@ class OMR_EXTENSIBLE CodeCacheManager
     */
    void freeCodeCacheSegment(TR::CodeCacheMemorySegment * memSegment) {}
 
+   void increaseCurrTotalUsedInBytes(size_t size);
+   void decreaseCurrTotalUsedInBytes(size_t size);
+   size_t getCurrTotalUsedInBytes() const { return _currTotalUsedInBytes; }
+   size_t getMaxUsedInBytes() const { return _maxUsedInBytes; }
+
 protected:
 
    TR::RawAllocator               _rawAllocator;
@@ -291,6 +302,9 @@ class OMR_EXTENSIBLE CodeCacheManager
    bool                           _lowCodeCacheSpaceThresholdReached; /*!< true if close to exhausting available code cache */
    bool                           _codeCacheFull;
 
+   TR::Monitor                   *_usageMonitor;
+   size_t                         _currTotalUsedInBytes;
+   size_t                         _maxUsedInBytes;
 #if (HOST_OS == OMR_LINUX)
    public:
    /**