Merge pull request #17217 from cjjdespres/counter-relocation-pr

Enable EDO during AOT compilation

doc/compiler/aot/RelocationRecords.md

@@ -134,4 +134,5 @@ exact type of the API class for each relocation kind can be found in
 |`TR_SymbolFromManager`|Relocates a pointer materialized by using its SVM ID.|
 |`TR_DiscontiguousSymbolFromManager`|Relocates a discontiguous pointer materialized by using its SVM ID.|
 |`TR_MethodCallAddress`|Relocates the address of a call target. Only used in JitServer (in AOT, all other methods are assumed to be interpreted).|
-
+|`TR_CatchBlockCounter`|Relocates the address of the catch block counter in the `TR_PersistentMethodInfo` of the method being compiled.|
+|`TR_StartPC`|Relocates the startPC of the method being compiled. Only implemented and used on Power.|

runtime/compiler/aarch64/codegen/J9TreeEvaluator.cpp

@@ -107,7 +107,7 @@ void VMgenerateCatchBlockBBStartPrologue(TR::Node *node, TR::Instruction *fenceI
       TR::Register *biAddrReg = cg->allocateRegister();
       TR::Register *recompCounterReg = cg->allocateRegister();
       intptr_t addr = (intptr_t)(comp->getRecompilationInfo()->getCounterAddress());
-      loadAddressConstant(cg, node, addr, biAddrReg);
+      loadAddressConstant(cg, node, addr, biAddrReg, NULL, false, TR_BodyInfoAddressLoad);
       TR::MemoryReference *loadbiMR = TR::MemoryReference::createWithDisplacement(cg, biAddrReg, 0);
       TR::MemoryReference *storebiMR = TR::MemoryReference::createWithDisplacement(cg, biAddrReg, 0);
       generateTrg1MemInstruction(cg, TR::InstOpCode::ldrimmx, node, recompCounterReg, loadbiMR);

runtime/compiler/codegen/J9AheadOfTimeCompile.cpp

@@ -335,6 +335,8 @@ J9::AheadOfTimeCompile::initializeCommonAOTRelocationHeader(TR::IteratedExternal
       case TR_ArrayCopyToc:
       case TR_BodyInfoAddressLoad:
       case TR_RecompQueuedFlag:
+      case TR_CatchBlockCounter:
+      case TR_StartPC:
          {
          // Nothing to do
          }
@@ -1416,6 +1418,8 @@ J9::AheadOfTimeCompile::dumpRelocationHeaderData(uint8_t *cursor, bool isVerbose
       case TR_ArrayCopyToc:
       case TR_BodyInfoAddressLoad:
       case TR_RecompQueuedFlag:
+      case TR_CatchBlockCounter:
+      case TR_StartPC:
          {
          self()->traceRelocationOffsets(startOfOffsets, offsetSize, endOfCurrentRecord, orderedPair);
          }

runtime/compiler/codegen/J9CodeGenerator.cpp

@@ -4886,6 +4886,12 @@ J9::CodeGenerator::needRelocationsForCurrentMethodPC()
    return self()->fej9()->needRelocationsForCurrentMethodPC();
    }
 
+bool
+J9::CodeGenerator::needRelocationsForCurrentMethodStartPC()
+   {
+   return self()->fej9()->needRelocationsForCurrentMethodStartPC();
+   }
+
 bool
 J9::CodeGenerator::needRelocationsForHelpers()
    {

runtime/compiler/codegen/J9CodeGenerator.hpp

@@ -183,6 +183,7 @@ class OMR_EXTENSIBLE CodeGenerator : public OMR::CodeGeneratorConnector
    bool needRelocationsForStatics();
    bool needRelocationsForHelpers();
    bool needRelocationsForCurrentMethodPC();
+   bool needRelocationsForCurrentMethodStartPC();
 #if defined(J9VM_OPT_JITSERVER)
    bool needRelocationsForBodyInfoData();
    bool needRelocationsForPersistentInfoData();

runtime/compiler/control/CompilationThread.cpp

@@ -8762,7 +8762,6 @@ TR::CompilationInfoPerThreadBase::wrappedCompile(J9PortLibrary *portLib, void *
                options->setOption(TR_DisableOnDemandLiteralPoolRegister);
 
                options->setOption(TR_DisableIPA);
-               options->setOption(TR_DisableEDO);
                options->setDisabled(OMR::invariantArgumentPreexistence, true);
                options->setOption(TR_DisableHierarchyInlining);
                options->setOption(TR_DisableKnownObjectTable);

runtime/compiler/control/J9Options.cpp

@@ -2679,7 +2679,6 @@ J9::Options::setupJITServerOptions()
       {
       self()->setOption(TR_DisableSamplingJProfiling);
       self()->setOption(TR_DisableProfiling); // JITServer limitation, JIT profiling data is not available to remote compiles yet
-      self()->setOption(TR_DisableEDO); // JITServer limitation, EDO counters are not relocatable yet
       self()->setOption(TR_DisableMethodIsCold); // Shady heuristic; better to disable to reduce client/server traffic
       self()->setOption(TR_DisableJProfilerThread);
       self()->setOption(TR_EnableJProfiling, false);

runtime/compiler/env/VMJ9.h

@@ -285,6 +285,7 @@ class TR_J9VMBase : public TR_FrontEnd
    virtual bool needClassAndMethodPointerRelocations() { return false; }
    virtual bool needRelocationsForStatics() { return false; }
    virtual bool needRelocationsForCurrentMethodPC() { return false; }
+   virtual bool needRelocationsForCurrentMethodStartPC() { return false; }
    virtual bool needRelocationsForBodyInfoData() { return false; }
    virtual bool needRelocationsForPersistentInfoData() { return false; }
    virtual bool needRelocationsForLookupEvaluationData() { return false; }
@@ -1497,6 +1498,7 @@ class TR_J9SharedCacheVM : public TR_J9VM
    virtual bool               supportsFastNanoTime()                          { return false; }
    virtual bool               needRelocationsForStatics()                     { return true; }
    virtual bool               needRelocationsForCurrentMethodPC()             { return true; }
+   virtual bool               needRelocationsForCurrentMethodStartPC()        { return true; }
    virtual bool               needRelocationsForLookupEvaluationData()        { return true; }
    virtual bool               needRelocationsForBodyInfoData()                { return true; }
    virtual bool               needRelocationsForPersistentInfoData()          { return true; }

runtime/compiler/env/VMJ9Server.hpp

@@ -54,7 +54,8 @@ class TR_J9ServerVM: public TR_J9VM
    virtual bool needRelocationsForBodyInfoData() override                { return true; }
    virtual bool needRelocationsForPersistentInfoData() override          { return true; }
    virtual bool needRelocationsForLookupEvaluationData() override        { return true; }
-   virtual bool needRelocationsForCurrentMethodPC() override                     { return true; }
+   virtual bool needRelocationsForCurrentMethodPC() override             { return true; }
+   virtual bool needRelocationsForCurrentMethodStartPC() override        { return true; }
    virtual bool isPortableSCCEnabled() override;
    virtual void markHotField(TR::Compilation *, TR::SymbolReference *, TR_OpaqueClassBlock *, bool) override { return; }
    virtual bool isClassLibraryMethod(TR_OpaqueMethodBlock *method, bool vettedForAOT) override;
@@ -289,6 +290,7 @@ class TR_J9SharedCacheServerVM: public TR_J9ServerVM
    virtual bool       supportsFastNanoTime() override                          { return false; }
    virtual bool       needRelocationsForStatics() override                     { return true; }
    virtual bool       needRelocationsForCurrentMethodPC() override             { return true; }
+   virtual bool       needRelocationsForCurrentMethodStartPC() override        { return true; }
    virtual bool       needRelocationsForLookupEvaluationData() override        { return true; }
    virtual bool       needRelocationsForBodyInfoData() override                { return true; }
    virtual bool       needRelocationsForPersistentInfoData() override          { return true; }

runtime/compiler/net/CommunicationStream.hpp

@@ -118,7 +118,7 @@ class CommunicationStream
    // likely to lose an increment when merging/rebasing/etc.
    //
    static const uint8_t MAJOR_NUMBER = 1;
-   static const uint16_t MINOR_NUMBER = 44; // ID: EMimrZgcKwDZhSWK3DD5
+   static const uint16_t MINOR_NUMBER = 45; // ID: xE92Epd5mxpOTwnwiRt7
    static const uint8_t PATCH_NUMBER = 0;
    static uint32_t CONFIGURATION_FLAGS;
 

runtime/compiler/optimizer/CatchBlockProfiler.cpp

@@ -65,6 +65,8 @@ int32_t TR::CatchBlockProfiler::perform()
                {
                uint32_t *catchBlockCounterAddress = recompilation->getMethodInfo()->getCatchBlockCounterAddress();
                _catchBlockCounterSymRef = comp()->getSymRefTab()->createKnownStaticDataSymbolRef(catchBlockCounterAddress, TR::Int32);
+               _catchBlockCounterSymRef->getSymbol()->setIsCatchBlockCounter();
+               _catchBlockCounterSymRef->getSymbol()->setNotDataAddress();
                }
             TR::TreeTop *profilingTree = TR::TreeTop::createIncTree(comp(), b->getEntry()->getNode(), _catchBlockCounterSymRef, 1, b->getEntry());
             profilingTree->getNode()->setIsProfilingCode();

runtime/compiler/p/codegen/ForceRecompilationSnippet.cpp

@@ -47,6 +47,10 @@ uint8_t *TR::PPCForceRecompilationSnippet::emitSnippetBody()
    uint8_t             *buffer = cg()->getBinaryBufferCursor();
    TR::SymbolReference  *induceRecompilationSymRef = cg()->symRefTab()->findOrCreateRuntimeHelper(TR_PPCinduceRecompilation);
    intptr_t startPC = (intptr_t)((uint8_t*)cg()->getCodeStart());
+   // The relo runtime expects this kind of address to be zero in a relocatable binary, since it uses
+   // |= to update the address chunks. See TR_PPC64RelocationTarget::storeAddressSequence.
+   intptr_t startPCRegValue = cg()->needRelocationsForCurrentMethodStartPC() &&
+                              !cg()->canEmitDataForExternallyRelocatableInstructions() ? 0 : startPC;
 
    getSnippetLabel()->setCodeLocation(buffer);
 
@@ -57,12 +61,14 @@ uint8_t *TR::PPCForceRecompilationSnippet::emitSnippetBody()
 
    if (cg()->comp()->target().is64Bit())
       {
+      uint8_t *firstInstruction = buffer;
+
       // put jit entry point address in startPCReg
       uint32_t hhval, hlval, lhval, llval;
-      hhval = startPC >> 48 & 0xffff;
-      hlval = (startPC >>32) & 0xffff;
-      lhval = (startPC >>16) & 0xffff;
-      llval = startPC & 0xffff;
+      hhval = startPCRegValue >> 48 & 0xffff;
+      hlval = (startPCRegValue >>32) & 0xffff;
+      lhval = (startPCRegValue >>16) & 0xffff;
+      llval = startPCRegValue & 0xffff;
 
       opcode.setOpCodeValue(TR::InstOpCode::lis);
       buffer = opcode.copyBinaryToBuffer(buffer);
@@ -100,22 +106,57 @@ uint8_t *TR::PPCForceRecompilationSnippet::emitSnippetBody()
       startPCReg->setRegisterFieldRS((uint32_t *)buffer);
       *(uint32_t *)buffer |= llval;
       buffer += PPC_INSTRUCTION_LENGTH;
+
+      if (cg()->needRelocationsForCurrentMethodStartPC())
+         {
+         TR::Relocation *startPCRelo = new (cg()->trHeapMemory()) TR::ExternalRelocation(firstInstruction,
+                                                                                        NULL,
+                                                                                        (uint8_t *)fixedSequence1,
+                                                                                        TR_StartPC,
+                                                                                        cg());
+         cg()->addExternalRelocation(startPCRelo,
+                                     __FILE__,
+                                     __LINE__,
+                                     getNode());
+         }
       }
    else
       {
+      uint8_t *firstInstruction = buffer;
+
       // put jit entry point address in startPCReg
       opcode.setOpCodeValue(TR::InstOpCode::lis);
       buffer = opcode.copyBinaryToBuffer(buffer);
       startPCReg->setRegisterFieldRS((uint32_t *)buffer);
-      *(uint32_t *)buffer |= (uint32_t) (startPC >> 16 & 0xffff);
+      *(uint32_t *)buffer |= (uint32_t) (startPCRegValue >> 16 & 0xffff);
       buffer += PPC_INSTRUCTION_LENGTH;
 
+      uint8_t *secondInstruction = buffer;
+
       opcode.setOpCodeValue(TR::InstOpCode::ori);
       buffer = opcode.copyBinaryToBuffer(buffer);
       startPCReg->setRegisterFieldRT((uint32_t *)buffer);
       startPCReg->setRegisterFieldRA((uint32_t *)buffer);
-      *(uint32_t *)buffer |= (uint32_t) (startPC & 0xffff);
+      *(uint32_t *)buffer |= (uint32_t) (startPCRegValue & 0xffff);
       buffer += PPC_INSTRUCTION_LENGTH;
+
+      if (cg()->needRelocationsForCurrentMethodStartPC())
+         {
+         TR_RelocationRecordInformation *recordInfo =
+            (TR_RelocationRecordInformation *)cg()->comp()->trMemory()->allocateMemory(sizeof(TR_RelocationRecordInformation), heapAlloc);
+         recordInfo->data1 = (uintptr_t)NULL;
+         recordInfo->data3 = orderedPairSequence2;
+         TR::Relocation *startPCRelo = new (cg()->trHeapMemory()) TR::ExternalOrderedPair32BitRelocation(firstInstruction,
+                                                                                                         secondInstruction,
+                                                                                                         (uint8_t *)recordInfo,
+                                                                                                         TR_StartPC,
+                                                                                                         cg());
+         cg()->addExternalRelocation(startPCRelo,
+                                     __FILE__,
+                                     __LINE__,
+                                     getNode());
+         }
+
       }
 
    intptr_t helperAddress = (intptr_t)induceRecompilationSymRef->getMethodAddress();

runtime/compiler/p/codegen/J9AheadOfTimeCompile.cpp

@@ -185,6 +185,8 @@ J9::Power::AheadOfTimeCompile::initializePlatformSpecificAOTRelocationHeader(TR:
       case TR_ArrayCopyHelper:
       case TR_ArrayCopyToc:
       case TR_BodyInfoAddressLoad:
+      case TR_CatchBlockCounter:
+      case TR_StartPC:
       case TR_RecompQueuedFlag:
          {
          TR_RelocationRecord *rRecord = reinterpret_cast<TR_RelocationRecord *>(reloRecord);

runtime/compiler/p/codegen/J9TreeEvaluator.cpp

@@ -11960,7 +11960,7 @@ void VMgenerateCatchBlockBBStartPrologue(TR::Node *node, TR::Instruction *fenceI
       TR::Register *biAddrReg = cg->allocateRegister();
       TR::Register *recompCounterReg = cg->allocateRegister();
       intptr_t addr = (intptr_t) (comp->getRecompilationInfo()->getCounterAddress());
-      TR::Instruction *cursor = loadAddressConstant(cg, comp->compileRelocatableCode(), node, addr, biAddrReg);
+      TR::Instruction *cursor = loadAddressConstant(cg, cg->needRelocationsForBodyInfoData(), node, addr, biAddrReg, NULL, false, TR_BodyInfoAddressLoad);
       TR::MemoryReference *loadbiMR = TR::MemoryReference::createWithDisplacement(cg, biAddrReg, 0, TR::Compiler->om.sizeofReferenceAddress());
       TR::MemoryReference *storebiMR = TR::MemoryReference::createWithDisplacement(cg, biAddrReg, 0, TR::Compiler->om.sizeofReferenceAddress());
       cursor = generateTrg1MemInstruction(cg,TR::InstOpCode::Op_load, node, recompCounterReg, loadbiMR);

runtime/compiler/p/runtime/PPCRelocationTarget.cpp

@@ -308,6 +308,8 @@ TR_PPC32RelocationTarget::isOrderedPairRelocation(TR_RelocationRecord *reloRecor
       case TR_GlobalValue:
       case TR_RamMethodSequence:
       case TR_BodyInfoAddressLoad:
+      case TR_CatchBlockCounter:
+      case TR_StartPC:
       case TR_DataAddress:
       case TR_DebugCounter:
          return true;

runtime/compiler/runtime/RelocationRecord.cpp

@@ -860,6 +860,12 @@ TR_RelocationRecord::create(TR_RelocationRecord *storage, TR_RelocationRuntime *
       case TR_ValidateIsClassVisible:
          reloRecord = new (storage) TR_RelocationRecordValidateIsClassVisible(reloRuntime, record);
          break;
+      case TR_CatchBlockCounter:
+         reloRecord = new (storage) TR_RelocationRecordCatchBlockCounter(reloRuntime, record);
+         break;
+      case TR_StartPC:
+         reloRecord = new (storage) TR_RelocationRecordStartPC(reloRuntime, record);
+         break;
       default:
          // TODO: error condition
          printf("Unexpected relo record: %d\n", reloType);fflush(stdout);
@@ -6435,6 +6441,86 @@ TR_RelocationRecordStaticDefaultValueInstance::applyRelocation(TR_RelocationRunt
    return TR_RelocationErrorCode::relocationOK;
    }
 
+// TR_CatchBlockCounter
+//
+char *
+TR_RelocationRecordCatchBlockCounter::name()
+   {
+   return "TR_CatchBlockCounter";
+   }
+
+void
+TR_RelocationRecordCatchBlockCounter::preparePrivateData(TR_RelocationRuntime *reloRuntime, TR_RelocationTarget *reloTarget)
+   {
+   TR_RelocationRecordWithOffsetPrivateData *reloPrivateData = &(privateData()->offset);
+   reloPrivateData->_addressToPatch = NULL;
+
+   TR_PersistentJittedBodyInfo *bodyInfo = reinterpret_cast<TR_PersistentJittedBodyInfo *>(reloRuntime->exceptionTable()->bodyInfo);
+   if (bodyInfo)
+      {
+      TR_PersistentMethodInfo *methodInfo = bodyInfo->getMethodInfo();
+      if (methodInfo)
+         reloPrivateData->_addressToPatch = (uint8_t *)methodInfo->getCatchBlockCounterAddress();
+      }
+   RELO_LOG(reloRuntime->reloLogger(), 6, "\tpreparePrivateData: addressToPatch: %p \n", reloPrivateData->_addressToPatch);
+   }
+
+TR_RelocationErrorCode
+TR_RelocationRecordCatchBlockCounter::applyRelocation(TR_RelocationRuntime *reloRuntime, TR_RelocationTarget *reloTarget, uint8_t *reloLocation)
+   {
+   TR_RelocationRecordWithOffsetPrivateData *reloPrivateData = &(privateData()->offset);
+   if (!reloPrivateData->_addressToPatch)
+      {
+      return TR_RelocationErrorCode::catchBlockCounterRelocationFailure;
+      }
+   reloTarget->storeAddressSequence(reloPrivateData->_addressToPatch, reloLocation, reloFlags(reloTarget));
+   return TR_RelocationErrorCode::relocationOK;
+   }
+
+TR_RelocationErrorCode
+TR_RelocationRecordCatchBlockCounter::applyRelocation(TR_RelocationRuntime *reloRuntime, TR_RelocationTarget *reloTarget, uint8_t *reloLocationHigh, uint8_t *reloLocationLow)
+   {
+   TR_RelocationRecordWithOffsetPrivateData *reloPrivateData = &(privateData()->offset);
+   if (!reloPrivateData->_addressToPatch)
+      {
+      return TR_RelocationErrorCode::catchBlockCounterRelocationFailure;
+      }
+   reloTarget->storeAddress(reloPrivateData->_addressToPatch, reloLocationHigh, reloLocationLow, reloFlags(reloTarget));
+   return TR_RelocationErrorCode::relocationOK;
+   }
+
+// TR_StartPC
+//
+char *
+TR_RelocationRecordStartPC::name()
+   {
+   return "TR_StartPC";
+   }
+
+void
+TR_RelocationRecordStartPC::preparePrivateData(TR_RelocationRuntime *reloRuntime, TR_RelocationTarget *reloTarget)
+   {
+   TR_RelocationRecordWithOffsetPrivateData *reloPrivateData = &(privateData()->offset);
+   reloPrivateData->_addressToPatch = reinterpret_cast<uint8_t *>(reloRuntime->exceptionTable()->startPC);
+   RELO_LOG(reloRuntime->reloLogger(), 6, "\tpreparePrivateData: addressToPatch: %p \n", reloPrivateData->_addressToPatch);
+   }
+
+TR_RelocationErrorCode
+TR_RelocationRecordStartPC::applyRelocation(TR_RelocationRuntime *reloRuntime, TR_RelocationTarget *reloTarget, uint8_t *reloLocation)
+   {
+   TR_RelocationRecordWithOffsetPrivateData *reloPrivateData = &(privateData()->offset);
+   reloTarget->storeAddressSequence(reloPrivateData->_addressToPatch, reloLocation, reloFlags(reloTarget));
+   return TR_RelocationErrorCode::relocationOK;
+   }
+
+TR_RelocationErrorCode
+TR_RelocationRecordStartPC::applyRelocation(TR_RelocationRuntime *reloRuntime, TR_RelocationTarget *reloTarget, uint8_t *reloLocationHigh, uint8_t *reloLocationLow)
+   {
+   TR_RelocationRecordWithOffsetPrivateData *reloPrivateData = &(privateData()->offset);
+   reloTarget->storeAddress(reloPrivateData->_addressToPatch, reloLocationHigh, reloLocationLow, reloFlags(reloTarget));
+   return TR_RelocationErrorCode::relocationOK;
+   }
+
 // The _relocationRecordHeaderSizeTable table should be the last thing in this file
 uint32_t TR_RelocationRecord::_relocationRecordHeaderSizeTable[TR_NumExternalRelocationKinds] =
    {
@@ -6551,5 +6637,7 @@ uint32_t TR_RelocationRecord::_relocationRecordHeaderSizeTable[TR_NumExternalRel
    sizeof(TR_RelocationRecordValidateJ2IThunkFromMethodBinaryTemplate),              // TR_ValidateJ2IThunkFromMethod                   = 110
    sizeof(TR_RelocationRecordConstantPoolWithIndexBinaryTemplate),                   // TR_StaticDefaultValueInstance                   = 111
    sizeof(TR_RelocationRecordValidateIsClassVisibleBinaryTemplate),                  // TR_ValidateIsClassVisible                       = 112
+   sizeof(TR_RelocationRecordBinaryTemplate),                                        // TR_CatchBlockCounter                            = 113
+   sizeof(TR_RelocationRecordBinaryTemplate),                                        // TR_StartPC                                      = 114
    };
 // The _relocationRecordHeaderSizeTable table should be the last thing in this file