From 0f8ef7e7e214b61df1dbb7078abfc42befae2b47 Mon Sep 17 00:00:00 2001 From: Filip Krikava Date: Thu, 23 Nov 2023 15:11:02 +0100 Subject: [PATCH] Extract recording (#1245) The type feedback is no longer in the byte code instruction, instead it is at a function, currently only at the baseline. --- .gdbinit | 6 + .gitignore | 4 +- documentation/debugging.md | 4 + rir/src/api.cpp | 12 +- rir/src/bc/BC.cpp | 75 +---- rir/src/bc/BC.h | 18 +- rir/src/bc/BC_inc.h | 43 +-- rir/src/bc/Compiler.cpp | 64 ++-- rir/src/bc/Compiler.h | 1 + rir/src/bc/insns.h | 2 +- rir/src/compiler/analysis/verifier.cpp | 2 +- rir/src/compiler/backend.cpp | 15 +- rir/src/compiler/compiler.cpp | 29 +- rir/src/compiler/compiler.h | 4 +- rir/src/compiler/native/allocator.cpp | 4 +- rir/src/compiler/native/builtins.cpp | 72 ++--- rir/src/compiler/native/builtins.h | 3 +- .../compiler/native/lower_function_llvm.cpp | 46 +-- rir/src/compiler/opt/eager_calls.cpp | 16 +- rir/src/compiler/opt/type_test.h | 4 +- rir/src/compiler/opt/typefeedback_cleanup.cpp | 7 +- rir/src/compiler/osr.cpp | 3 + rir/src/compiler/parameter.h | 1 + rir/src/compiler/pir/builder.cpp | 13 +- rir/src/compiler/pir/instruction.cpp | 2 +- rir/src/compiler/rir2pir/rir2pir.cpp | 56 ++-- rir/src/compiler/rir2pir/rir2pir.h | 10 +- rir/src/interpreter/interp.cpp | 36 ++- rir/src/runtime/Code.cpp | 21 +- rir/src/runtime/DispatchTable.h | 73 ++++- rir/src/runtime/Function.cpp | 17 +- rir/src/runtime/Function.h | 44 ++- rir/src/runtime/PirTypeFeedback.cpp | 31 +- rir/src/runtime/PirTypeFeedback.h | 13 +- rir/src/runtime/TypeFeedback.cpp | 304 +++++++++++++++--- rir/src/runtime/TypeFeedback.h | 281 ++++++++++++---- rir/src/utils/FunctionWriter.h | 8 +- rir/tests/pir_check.R | 11 + 38 files changed, 926 insertions(+), 429 deletions(-) diff --git a/.gdbinit b/.gdbinit index 457dc7b31..ff193a7e0 100644 --- a/.gdbinit +++ b/.gdbinit @@ -179,6 +179,12 @@ define ds dumpsxp $arg0 1 end +define ninja + shell ninja + python gdb.execute("file " + gdb.current_progspace().filename) + directory +end + # source .pirpp.py diff --git a/.gitignore b/.gitignore index 01d6a2c5b..47c2f580e 100644 --- a/.gitignore +++ b/.gitignore @@ -36,4 +36,6 @@ benchmarks/ *.DS_Store external/* !external/custom-r -.history \ No newline at end of file +.history +.cache +compile_commands.json diff --git a/documentation/debugging.md b/documentation/debugging.md index c44463199..545689d6e 100644 --- a/documentation/debugging.md +++ b/documentation/debugging.md @@ -68,6 +68,10 @@ complete. PIR_WARMUP= number: after how many invocations a function is (re-) optimized + PIR_OPT_BC_SIZE= + number: controls the maximum byte code size of a callee that could + trigger OSR in the caller (cf. https://github.com/reactorlabs/rir/issues/1252#issuecomment-1775517529) + #### Extended debug flags RIR_CHECK_PIR_TYPES= diff --git a/rir/src/api.cpp b/rir/src/api.cpp index 205066999..22570da6d 100644 --- a/rir/src/api.cpp +++ b/rir/src/api.cpp @@ -4,6 +4,7 @@ #include "api.h" #include "R/Serialize.h" +#include "Rinternals.h" #include "bc/BC.h" #include "bc/Compiler.h" #include "compiler/backend.h" @@ -15,6 +16,7 @@ #include "compiler/test/PirCheck.h" #include "compiler/test/PirTests.h" #include "interpreter/interp_incl.h" +#include "runtime/DispatchTable.h" #include "utils/measuring.h" #include @@ -56,13 +58,9 @@ REXPORT SEXP rirDisassemble(SEXP what, SEXP verbose) { if (!t) Rf_error("Not a rir compiled code (CLOSXP but not DispatchTable)"); - std::cout << "== closure " << what << " (dispatch table " << t << ", env " - << CLOENV(what) << ") ==\n"; - for (size_t entry = 0; entry < t->size(); ++entry) { - Function* f = t->get(entry); - std::cout << "= version " << entry << " (" << f << ") =\n"; - f->disassemble(std::cout); - } + std::cout << "== closure " << what << " (env " << CLOENV(what) << ") ==\n"; + + t->print(std::cout, Rf_asLogical(verbose)); return R_NilValue; } diff --git a/rir/src/bc/BC.cpp b/rir/src/bc/BC.cpp index 1dea8a49a..3e5d462b5 100644 --- a/rir/src/bc/BC.cpp +++ b/rir/src/bc/BC.cpp @@ -24,23 +24,6 @@ void BC::write(CodeStream& cs) const { cs.insert(immediate.cacheIdx); return; - case Opcode::record_call_: - // Call feedback targets are stored in the code extra pool. We don't - // have access to them here, so we can't write a call feedback with - // preseeded values. - assert(immediate.callFeedback.numTargets == 0 && - "cannot write call feedback targets"); - cs.insert(immediate.callFeedback); - return; - - case Opcode::record_test_: - cs.insert(immediate.testFeedback); - break; - - case Opcode::record_type_: - cs.insert(immediate.typeFeedback); - break; - case Opcode::push_: case Opcode::ldfun_: case Opcode::ldddvar_: @@ -96,6 +79,9 @@ void BC::write(CodeStream& cs) const { case Opcode::pull_: case Opcode::is_: case Opcode::put_: + case Opcode::record_call_: + case Opcode::record_test_: + case Opcode::record_type_: cs.insert(immediate.i); return; @@ -331,9 +317,7 @@ void BC::printOpcode(std::ostream& out) const { out << name(bc) << " "; } void BC::print(std::ostream& out) const { out << " "; - if (bc != Opcode::record_call_ && bc != Opcode::record_type_ && - bc != Opcode::record_test_) - printOpcode(out); + printOpcode(out); switch (bc) { case Opcode::invalid_: @@ -402,54 +386,11 @@ void BC::print(std::ostream& out) const { case Opcode::is_: out << (BC::RirTypecheck)immediate.i; break; - case Opcode::record_call_: { - ObservedCallees prof = immediate.callFeedback; - out << "[ "; - if (prof.taken == ObservedCallees::CounterOverflow) - out << "*, <"; - else - out << prof.taken << ", <"; - if (prof.numTargets == ObservedCallees::MaxTargets) - out << "*>, "; - else - out << prof.numTargets << ">, "; - - out << (prof.invalid ? "invalid" : "valid"); - out << (prof.numTargets ? ", " : " "); - - for (int i = 0; i < prof.numTargets; ++i) - out << callFeedbackExtra().targets[i] << "(" - << Rf_type2char(TYPEOF(callFeedbackExtra().targets[i])) << ") "; - out << "]"; - break; - } - - case Opcode::record_test_: { - out << "[ "; - switch (immediate.testFeedback.seen) { - case ObservedTest::None: - out << "_"; - break; - case ObservedTest::OnlyTrue: - out << "T"; - break; - case ObservedTest::OnlyFalse: - out << "F"; - break; - case ObservedTest::Both: - out << "?"; - break; - } - out << " ]"; - break; - } - - case Opcode::record_type_: { - out << "[ "; - immediate.typeFeedback.print(out); - out << " ]"; + case Opcode::record_test_: + case Opcode::record_type_: + case Opcode::record_call_: + out << "#" << immediate.i; break; - } #define V(NESTED, name, name_) case Opcode::name_##_: BC_NOARGS(V, _) diff --git a/rir/src/bc/BC.h b/rir/src/bc/BC.h index d24169f2e..1bb6e23f7 100644 --- a/rir/src/bc/BC.h +++ b/rir/src/bc/BC.h @@ -23,11 +23,23 @@ class CodeStream; BC_NOARGS(V, _) #undef V -BC BC::recordCall() { return BC(Opcode::record_call_); } +BC BC::recordCall(uint32_t idx) { + ImmediateArguments i; + i.i = idx; + return BC(Opcode::record_call_, i); +} -BC BC::recordType() { return BC(Opcode::record_type_); } +BC BC::recordType(uint32_t idx) { + ImmediateArguments i; + i.i = idx; + return BC(Opcode::record_type_, i); +} -BC BC::recordTest() { return BC(Opcode::record_test_); } +BC BC::recordTest(uint32_t idx) { + ImmediateArguments i; + i.i = idx; + return BC(Opcode::record_test_, i); +} BC BC::asSwitchIdx() { return BC(Opcode::as_switch_idx_); } diff --git a/rir/src/bc/BC_inc.h b/rir/src/bc/BC_inc.h index 20653494d..933ed7d25 100644 --- a/rir/src/bc/BC_inc.h +++ b/rir/src/bc/BC_inc.h @@ -152,9 +152,6 @@ class BC { uint32_t i; RirTypecheck typecheck; NumLocals loc; - ObservedCallees callFeedback; - ObservedValues typeFeedback; - ObservedTest testFeedback; PoolAndCachePositionRange poolAndCache; CachePositionRange cacheIdx; ImmediateArguments() { @@ -267,6 +264,11 @@ class BC { bool isJmp() const { return isCondJmp() || isUncondJmp(); } + bool isRecord() const { + return bc == Opcode::record_call_ || bc == Opcode::record_test_ || + bc == Opcode::record_type_; + } + bool isExit() const { return bc == Opcode::ret_ || bc == Opcode::return_; } // This code performs the same as `BC::decode(pc).size()`, but for @@ -309,10 +311,10 @@ class BC { #define V(NESTED, name, name_) inline static BC name(); BC_NOARGS(V, _) #undef V - inline static BC recordCall(); + inline static BC recordCall(uint32_t idx); inline static BC recordBinop(); - inline static BC recordType(); - inline static BC recordTest(); + inline static BC recordType(uint32_t idx); + inline static BC recordTest(uint32_t idx); inline static BC asSwitchIdx(); inline static BC popn(unsigned n); inline static BC push(SEXP constant); @@ -406,14 +408,6 @@ class BC { extraInformation.get()); } - CallFeedbackExtraInformation& callFeedbackExtra() const { - assert(bc == Opcode::record_call_ && "not a record call instruction"); - assert(extraInformation.get() && - "missing extra information. created through decodeShallow?"); - return *static_cast( - extraInformation.get()); - } - private: void allocExtraInformation() { assert(extraInformation == nullptr); @@ -428,10 +422,6 @@ class BC { extraInformation.reset(new CallInstructionExtraInformation); break; } - case Opcode::record_call_: { - extraInformation.reset(new CallFeedbackExtraInformation); - break; - } default: { } } @@ -455,13 +445,6 @@ class BC { break; } - case Opcode::record_call_: { - // Read call target feedback from the extra pool - for (size_t i = 0; i < immediate.callFeedback.numTargets; ++i) - callFeedbackExtra().targets.push_back( - immediate.callFeedback.getTarget(code, i)); - break; - } default: { } } @@ -580,18 +563,10 @@ class BC { case Opcode::pull_: case Opcode::is_: case Opcode::put_: - memcpy(&immediate.i, pc, sizeof(immediate.i)); - break; case Opcode::record_call_: - memcpy(&immediate.callFeedback, pc, sizeof(ObservedCallees)); - break; case Opcode::record_test_: - memcpy(reinterpret_cast(&immediate.testFeedback), pc, - sizeof(ObservedValues)); - break; case Opcode::record_type_: - memcpy(reinterpret_cast(&immediate.typeFeedback), pc, - sizeof(ObservedValues)); + memcpy(&immediate.i, pc, sizeof(immediate.i)); break; #define V(NESTED, name, name_) case Opcode::name_##_: BC_NOARGS(V, _) diff --git a/rir/src/bc/Compiler.cpp b/rir/src/bc/Compiler.cpp index 2ad133d76..250120af8 100644 --- a/rir/src/bc/Compiler.cpp +++ b/rir/src/bc/Compiler.cpp @@ -3,6 +3,7 @@ #include "R/RList.h" #include "R/Symbols.h" #include "R/r.h" +#include "Rinternals.h" #include "bc/BC.h" #include "bc/CodeStream.h" #include "bc/CodeVerifier.h" @@ -10,6 +11,7 @@ #include "interpreter/interp.h" #include "interpreter/interp_incl.h" #include "interpreter/safe_force.h" +#include "runtime/TypeFeedback.h" #include "simple_instruction_list.h" #include "utils/Pool.h" @@ -136,6 +138,7 @@ class CompilerContext { FunctionWriter& fun; Preserve& preserve; + TypeFeedback::Builder typeFeedbackBuilder; CompilerContext(FunctionWriter& fun, Preserve& preserve) : fun(fun), preserve(preserve) {} @@ -201,6 +204,12 @@ class CompilerContext { << BC::callBuiltin(4, ast, getBuiltinFun("warning")) << BC::pop(); } + BC recordType() { return BC::recordType(typeFeedbackBuilder.addType()); } + + BC recordCall() { return BC::recordCall(typeFeedbackBuilder.addCallee()); } + + BC recordTest() { return BC::recordTest(typeFeedbackBuilder.addTest()); } + private: unsigned int pushedPromiseContexts = 0; }; @@ -257,7 +266,7 @@ void compileWhile(CompilerContext& ctx, std::function compileCond, // loop peel is a copy of the condition and body, with no backwards jumps if (Compiler::loopPeelingEnabled && peelLoop) { compileCond(); - cs << BC::recordTest() << BC::brfalse(breakBranch); + cs << ctx.recordTest() << BC::brfalse(breakBranch); compileBody(); } @@ -348,7 +357,7 @@ bool compileSimpleFor(CompilerContext& ctx, SEXP fullAst, SEXP sym, SEXP seq, // branch cs << BC::pop(); } else { - cs << BC::recordTest() << BC::brtrue(skipRegularForBranch); + cs << ctx.recordTest() << BC::brtrue(skipRegularForBranch); // // Note that we call the builtin `for` and pass the body as a // promise to lower the bytecode size @@ -378,7 +387,7 @@ bool compileSimpleFor(CompilerContext& ctx, SEXP fullAst, SEXP sym, SEXP seq, if (voidContext) cs << BC::pop(); else if (Compiler::profile) - cs << BC::recordType(); + cs << ctx.recordType(); cs << BC::br(endBranch); cs << skipRegularForBranch; @@ -386,16 +395,16 @@ bool compileSimpleFor(CompilerContext& ctx, SEXP fullAst, SEXP sym, SEXP seq, // } else { // m' <- colonCastLhs(m') - cs << BC::swap() << BC::colonCastLhs() << BC::recordType() + cs << BC::swap() << BC::colonCastLhs() << ctx.recordType() << BC::ensureNamed() << BC::swap(); // n' <- colonCastRhs(m', n') - cs << BC::colonCastRhs() << BC::ensureNamed() << BC::recordType(); + cs << BC::colonCastRhs() << BC::ensureNamed() << ctx.recordType(); // step <- if (m' <= n') 1L else -1L cs << BC::dup2() << BC::le(); cs.addSrc(R_NilValue); - cs << BC::recordTest() << BC::brfalse(stepElseBranch) << BC::push(1) + cs << ctx.recordTest() << BC::brfalse(stepElseBranch) << BC::push(1) << BC::br(stepEndBranch) << stepElseBranch << BC::push(-1) << stepEndBranch; @@ -539,7 +548,7 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, if (voidContext) cs << BC::pop(); else if (Compiler::profile) - cs << BC::recordType(); + cs << ctx.recordType(); return true; } @@ -794,7 +803,7 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, } if (Compiler::profile) - cs << BC::recordType(); + cs << ctx.recordType(); if (maybeChanges(target, *idx) || (dims > 1 && maybeChanges(target, *(idx + 1))) || @@ -943,7 +952,7 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, cs << BC::ldfun(farrow_sym); if (Compiler::profile) - cs << BC::recordCall(); + cs << ctx.recordCall(); // prepare x, yk, z as promises LoadArgsResult load_arg_res; @@ -1017,7 +1026,7 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, // The return value, RHS, is TOS cs << BC::invisible(); if (Compiler::profile) { - cs << BC::recordType(); + cs << ctx.recordType(); } } @@ -1157,7 +1166,7 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, BC::Label contBranch = cs.mkLabel(); cs << BC::dup() << BC::is(BC::RirTypecheck::isNonObject) - << BC::recordTest() << BC::brfalse(objBranch) + << ctx.recordTest() << BC::brfalse(objBranch) << BC::br(nonObjBranch); cs << objBranch; @@ -1198,7 +1207,7 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, cs.addSrc(ast); if (!voidContext) { if (Compiler::profile) - cs << BC::recordType(); + cs << ctx.recordType(); cs << BC::visible(); } else { cs << BC::pop(); @@ -1307,7 +1316,7 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, cs.addSrc(R_NilValue); if (record) - cs << BC::recordTest(); + cs << ctx.recordTest(); // If outside bound, branch, otherwise index into the vector cs << BC::brtrue(breakBranch) << BC::pull(2) << BC::pull(1) @@ -1493,14 +1502,14 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, // !isVector(x) cs << BC::dup() << BC::is(BC::RirTypecheck::isVector) - << BC::recordTest() << BC::brtrue(vecArityBr); + << ctx.recordTest() << BC::brtrue(vecArityBr); cs << BC::br(vecErrorBr); // ... || LENGTH(x) != 1 cs << vecArityBr << BC::dup() << BC::length_() << BC::push(1) << BC::eq(); cs.addSrc(R_NilValue); // to make code verifier happy - cs << BC::recordTest() << BC::brtrue(vecEContBr); + cs << ctx.recordTest() << BC::brtrue(vecEContBr); cs << vecErrorBr; ctx.emitError("EXPR must be a length 1 vector", ast); @@ -1509,7 +1518,7 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, cs << vecEContBr; cs << BC::dup() << BC::is(BC::RirTypecheck::isFactor) - << BC::recordTest() << BC::brfalse(facWContBr); + << ctx.recordTest() << BC::brfalse(facWContBr); ctx.emitWarning("EXPR is a \"factor\", treated as integer.\n Consider " "using 'switch(as.character( * ), ...)' instead.", @@ -1522,14 +1531,14 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, cs << BC::br(nilBr); } cs << BC::dup() << BC::is(BC::RirTypecheck::isSTRSXP) - << BC::recordTest() << BC::brtrue(strBr); + << ctx.recordTest() << BC::brtrue(strBr); cs << BC::asSwitchIdx(); // currently stack is [arg[0]] (converted to integer) for (size_t i = 0; i < labels.size(); ++i) { cs << BC::dup() << BC::push(Rf_ScalarInteger(i + 1)) << BC::eq(); cs.addSrc(R_NilValue); // call argument for builtin - cs << BC::asbool() << BC::recordTest() << BC::brtrue(labels[i]); + cs << BC::asbool() << ctx.recordTest() << BC::brtrue(labels[i]); } cs << BC::br(nilBr) << strBr; if (dupDflt) { @@ -1543,14 +1552,14 @@ bool compileSpecialCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args_, // BC::asbool to compare the cases. cs << BC::dup() << BC::callBuiltin(1, R_NilValue, getBuiltinFun("is.na")) - << BC::asbool() << BC::recordTest() << BC::brfalse(strNAContBr) + << BC::asbool() << ctx.recordTest() << BC::brfalse(strNAContBr) << BC::pop() << BC::push(Rf_mkString("NA")) << strNAContBr; for (size_t i = 0; i < expressions.size(); ++i) { for (auto& n : groups[i]) { cs << BC::dup() << BC::push(n) << BC::eq(); cs.addSrc(R_NilValue); // call argument for builtin - cs << BC::asbool() << BC::recordTest() + cs << BC::asbool() << ctx.recordTest() << BC::brtrue(groupLabels[i]); } } @@ -1858,7 +1867,7 @@ void compileCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args, theEnd = cs.mkLabel(); cs << BC::push(builtin) << BC::dup() << BC::ldvarNoForce(fun) << BC::identicalNoforce() - << BC::recordTest() << BC::brtrue(eager); + << ctx.recordTest() << BC::brtrue(eager); cs << BC::pop(); } @@ -1872,7 +1881,7 @@ void compileCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args, } if (Compiler::profile) - cs << BC::recordCall(); + cs << ctx.recordCall(); auto compileCall = [&](LoadArgsResult& info) { if (info.hasDots) { @@ -1892,7 +1901,7 @@ void compileCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args, compileLoadOneArg(ctx, args, ArgType::RAW_VALUE, info); compileLoadOneArg(ctx, CDR(args), ArgType::RAW_VALUE, info); if (Compiler::profile) - cs << BC::recordCall(); + cs << ctx.recordCall(); // Load the rest of the args compileLoadArgs(ctx, ast, fun, args, info, voidContext, 2, 0); } else { @@ -1915,7 +1924,7 @@ void compileCall(CompilerContext& ctx, SEXP ast, SEXP fun, SEXP args, if (voidContext) cs << BC::pop(); else if (Compiler::profile) - cs << BC::recordType(); + cs << ctx.recordType(); } // Lookup @@ -1933,7 +1942,7 @@ void compileGetvar(CompilerContext& ctx, SEXP name) { cs << BC::ldvar(name); } if (Compiler::profile) - cs << BC::recordType(); + cs << ctx.recordType(); } } @@ -2052,7 +2061,10 @@ SEXP Compiler::finalize() { compileExpr(ctx, exp); ctx.cs() << BC::ret(); Code* body = ctx.pop(); - function.finalize(body, signature, Context()); + TypeFeedback* feedback = ctx.typeFeedbackBuilder.build(); + PROTECT(feedback->container()); + function.finalize(body, signature, Context(), feedback); + UNPROTECT(1); #ifdef ENABLE_SLOWASSERT CodeVerifier::verifyFunctionLayout(function.function()->container()); diff --git a/rir/src/bc/Compiler.h b/rir/src/bc/Compiler.h index 6032b856e..31c8af95c 100644 --- a/rir/src/bc/Compiler.h +++ b/rir/src/bc/Compiler.h @@ -5,6 +5,7 @@ #include "R/Protect.h" #include "R/r.h" #include "runtime/DispatchTable.h" +#include "runtime/TypeFeedback.h" #include "utils/FunctionWriter.h" #include "utils/Pool.h" diff --git a/rir/src/bc/insns.h b/rir/src/bc/insns.h index dcdf864c0..a9bcc91ee 100644 --- a/rir/src/bc/insns.h +++ b/rir/src/bc/insns.h @@ -443,7 +443,7 @@ DEF_INSTR(ret_, 0, 1, 0) * They keep a struct from RuntimeFeedback.h inline, that's why they are quite * heavy in size. */ -DEF_INSTR(record_call_, 4, 1, 1) +DEF_INSTR(record_call_, 1, 1, 1) DEF_INSTR(record_type_, 1, 1, 1) DEF_INSTR(record_test_, 1, 1, 1) diff --git a/rir/src/compiler/analysis/verifier.cpp b/rir/src/compiler/analysis/verifier.cpp index 4cedab71d..4002054fe 100644 --- a/rir/src/compiler/analysis/verifier.cpp +++ b/rir/src/compiler/analysis/verifier.cpp @@ -284,7 +284,7 @@ class TheVerifier { } if (auto assume = Assume::Cast(i)) { if (IsType::Cast(assume->arg(0).val())) { - if (!assume->reason.pc()) { + if (!assume->reason.origin.hasSlot()) { std::cerr << "Error: instruction '"; i->print(std::cerr); std::cerr << "' typecheck without origin information\n"; diff --git a/rir/src/compiler/backend.cpp b/rir/src/compiler/backend.cpp index e85d012bd..f4be0ffbe 100644 --- a/rir/src/compiler/backend.cpp +++ b/rir/src/compiler/backend.cpp @@ -16,6 +16,7 @@ #include "compiler/util/visitor.h" #include "interpreter/instance.h" #include "runtime/DispatchTable.h" +#include "runtime/TypeFeedback.h" #include "simple_instruction_list.h" #include "utils/FunctionWriter.h" #include "utils/measuring.h" @@ -97,7 +98,8 @@ static void approximateNeedsLdVarForUpdate( // the same lexical scope. apply(i, i->arg(1).val()->followCastsAndForce()); break; - default: {} + default: { + } } }); Visitor::run(code->entry, [&](Instruction* i) { @@ -404,11 +406,20 @@ rir::Function* Backend::doCompile(ClosureVersion* cls, ClosureLog& log) { } log.finalPIR(); - function.finalize(body, signature, cls->context()); + + // the type feedback is only used at the baseline + // here we only set the current version used to compile this function + auto feedback = rir::TypeFeedback::empty(); + PROTECT(feedback->container()); + feedback->version( + cls->optFunction->dispatchTable()->currentTypeFeedbackVersion()); + + function.finalize(body, signature, cls->context(), feedback); for (auto& c : done) c.second->function(function.function()); function.function()->inheritFlags(cls->owner()->rirFunction()); + UNPROTECT(1); return function.function(); } diff --git a/rir/src/compiler/compiler.cpp b/rir/src/compiler/compiler.cpp index b4c367e28..7dad3bd49 100644 --- a/rir/src/compiler/compiler.cpp +++ b/rir/src/compiler/compiler.cpp @@ -3,6 +3,7 @@ #include "pir/continuation.h" #include "pir/pir_impl.h" #include "rir2pir/rir2pir.h" +#include "runtime/TypeFeedback.h" #include "utils/Map.h" #include "utils/measuring.h" @@ -55,7 +56,8 @@ void Compiler::compileClosure(SEXP closure, const std::string& name, tbl->userDefinedContext()); Context context(assumptions); compileClosure(pirClosure, tbl->dispatch(assumptions), context, root, - success, fail, outerFeedback); + success, fail, outerFeedback, + tbl->baseline()->typeFeedback()); } void Compiler::compileFunction(rir::DispatchTable* src, const std::string& name, @@ -71,7 +73,7 @@ void Compiler::compileFunction(rir::DispatchTable* src, const std::string& name, auto closure = module->getOrDeclareRirFunction( name, srcFunction, formals, srcRef, src->userDefinedContext()); compileClosure(closure, src->dispatch(assumptions), context, false, success, - fail, outerFeedback); + fail, outerFeedback, src->baseline()->typeFeedback()); } void Compiler::compileContinuation(SEXP closure, rir::Function* curFun, @@ -89,7 +91,8 @@ void Compiler::compileContinuation(SEXP closure, rir::Function* curFun, Builder builder(version, pirClosure->closureEnv()); auto& log = logger.open(version); - Rir2Pir rir2pir(*this, version, log, pirClosure->name(), {}); + auto typeFeedback = tbl->baseline()->typeFeedback(); + Rir2Pir rir2pir(*this, version, log, pirClosure->name(), {}, typeFeedback); if (rir2pir.tryCompileContinuation(builder, ctx->pc(), ctx->stack())) { log.flush(); @@ -105,7 +108,8 @@ void Compiler::compileContinuation(SEXP closure, rir::Function* curFun, void Compiler::compileClosure(Closure* closure, rir::Function* optFunction, const Context& ctx, bool root, MaybeCls success, Maybe fail, - std::list outerFeedback) { + std::list outerFeedback, + rir::TypeFeedback* typeFeedback) { if (!ctx.includes(minimalContext)) { for (const auto a : minimalContext) { @@ -119,10 +123,11 @@ void Compiler::compileClosure(Closure* closure, rir::Function* optFunction, } // Currently dots args are not supported in PIR. Unless if we statically - // matched all arguments correctly and are therefore guaranteed to receive a + // matched all arguments correctly and are therefore guaranteed to + // receive a // `...` list as DOTSXP in the correct location, we can support them. - // TODO: extend call instruction to do the necessary argument shuffling to - // support it in all cases + // TODO: extend call instruction to do the necessary argument shuffling + // to support it in all cases if (!ctx.includes(Assumption::StaticallyArgmatched) && closure->formals().hasDots()) { logger.warn("no support for ..."); @@ -141,7 +146,8 @@ void Compiler::compileClosure(Closure* closure, rir::Function* optFunction, auto version = closure->declareVersion(ctx, root, optFunction); Builder builder(version, closure->closureEnv()); auto& log = logger.open(version); - Rir2Pir rir2pir(*this, version, log, closure->name(), outerFeedback); + Rir2Pir rir2pir(*this, version, log, closure->name(), outerFeedback, + typeFeedback); auto& context = version->context(); bool failedToCompileDefaultArgs = false; @@ -175,8 +181,8 @@ void Compiler::compileClosure(Closure* closure, rir::Function* optFunction, for (unsigned i = 0; i < closure->nargs() - context.numMissing(); ++i) { if (closure->formals().defaultArgs()[i] != R_MissingArg) { - // If this arg has a default, then test if the argument is - // missing and if so, load the default arg. + // If this arg has a default, then test if the argument + // is missing and if so, load the default arg. auto a = builder(new LdArg(i)); auto testMissing = builder(new Identical( a, MissingArg::instance(), PirType::any())); @@ -270,7 +276,8 @@ static void findUnreachable(Module* m, Log& log, const std::string& where) { if (!call->tryDispatch()) { std::stringstream msg; msg << "After pass " << where - << " found a broken static call. Available " + << " found a broken static call. " + "Available " "versions:\n"; call->cls()->eachVersion( [&](ClosureVersion* v) { diff --git a/rir/src/compiler/compiler.h b/rir/src/compiler/compiler.h index 628d4abc2..30f700aad 100644 --- a/rir/src/compiler/compiler.h +++ b/rir/src/compiler/compiler.h @@ -4,6 +4,7 @@ #include "R/Preserve.h" #include "compiler/log/log.h" #include "pir/pir.h" +#include "runtime/TypeFeedback.h" #include "utils/FormalArgs.h" #include @@ -57,7 +58,8 @@ class Compiler { void compileClosure(Closure* closure, rir::Function* optFunction, const Context& ctx, bool root, MaybeCls success, - Maybe fail, std::list outerFeedback); + Maybe fail, std::list outerFeedback, + rir::TypeFeedback* typeFeedback); Preserve preserve_; diff --git a/rir/src/compiler/native/allocator.cpp b/rir/src/compiler/native/allocator.cpp index d63e89987..d92fd90ec 100644 --- a/rir/src/compiler/native/allocator.cpp +++ b/rir/src/compiler/native/allocator.cpp @@ -22,8 +22,8 @@ void NativeAllocator::compute() { // them accessible to the runtime profiler. // TODO: this needs to be replaced by proper mapping of slots. if (RuntimeProfiler::enabled() && a != b && - (a->typeFeedback().feedbackOrigin.pc() || - b->typeFeedback().feedbackOrigin.pc())) + (a->typeFeedback().feedbackOrigin.hasSlot() || + b->typeFeedback().feedbackOrigin.hasSlot())) return true; return livenessIntervals.interfere(a, b); }; diff --git a/rir/src/compiler/native/builtins.cpp b/rir/src/compiler/native/builtins.cpp index 934709157..bd3f627c9 100644 --- a/rir/src/compiler/native/builtins.cpp +++ b/rir/src/compiler/native/builtins.cpp @@ -821,8 +821,9 @@ static SEXP deoptSentinelContainer = []() { PROTECT(c->container()); SEXP store = Rf_allocVector(EXTERNALSXP, sizeof(Function)); R_PreserveObject(store); - deoptSentinel = new (INTEGER(store)) - Function(0, c->container(), {}, deoptSentinelSig, Context()); + deoptSentinel = + new (INTEGER(store)) Function(0, c->container(), {}, deoptSentinelSig, + Context(), rir::TypeFeedback::empty()); deoptSentinel->registerDeopt(); UNPROTECT(1); return store; @@ -954,39 +955,29 @@ void deoptImpl(rir::Code* c, SEXP cls, DeoptMetadata* m, R_bcstack_t* args, assert(false); } -void recordTypefeedbackImpl(Opcode* pos, rir::Code* code, SEXP value) { - switch (*pos) { - case Opcode::record_test_: { - ObservedTest* feedback = (ObservedTest*)(pos + 1); - feedback->record(value); - break; - } - case Opcode::record_type_: { - assert(*pos == Opcode::record_type_); - ObservedValues* feedback = (ObservedValues*)(pos + 1); - feedback->record(value); +void recordTypeFeedbackImpl(rir::TypeFeedback* feedback, uint32_t idx, + SEXP value) { + feedback->record_type(idx, value); + // FIXME: cf. 1260 + feedback->record_type(idx, [&](auto& slot) { if (TYPEOF(value) == PROMSXP) { if (PRVALUE(value) == R_UnboundValue && - feedback->stateBeforeLastForce < ObservedValues::promise) - feedback->stateBeforeLastForce = ObservedValues::promise; - else if (feedback->stateBeforeLastForce < - ObservedValues::evaluatedPromise) - feedback->stateBeforeLastForce = - ObservedValues::evaluatedPromise; + slot.stateBeforeLastForce < ObservedValues::promise) { + slot.stateBeforeLastForce = ObservedValues::promise; + } else if (slot.stateBeforeLastForce < + ObservedValues::evaluatedPromise) { + slot.stateBeforeLastForce = ObservedValues::evaluatedPromise; + } } else { - if (feedback->stateBeforeLastForce < ObservedValues::value) - feedback->stateBeforeLastForce = ObservedValues::value; + if (slot.stateBeforeLastForce < ObservedValues::value) + slot.stateBeforeLastForce = ObservedValues::value; } - break; - } - case Opcode::record_call_: { - ObservedCallees* feedback = (ObservedCallees*)(pos + 1); - feedback->record(code, value); - break; - } - default: - assert(false); - } + }); +} + +void recordCallFeedbackImpl(rir::TypeFeedback* feedback, uint32_t idx, + SEXP value) { + feedback->record_callee(idx, feedback->owner(), value); } void assertFailImpl(const char* msg) { @@ -1450,8 +1441,8 @@ static SEXP nativeCallTrampolineImpl(ArglistOrder::CallId callId, rir::Code* c, RCNTXT cntxt; - // This code needs to be protected, because its slot in the dispatch table - // could get overwritten while we are executing it. + // This code needs to be protected, because its slot in the dispatch + // table could get overwritten while we are executing it. PROTECT(fun->container()); initClosureContext(ast, &cntxt, symbol::delayedEnv, env, lazyArgs.asSexp(), @@ -2423,10 +2414,17 @@ void NativeBuiltins::initializeBuiltins() { (void*)&lengthImpl, llvm::FunctionType::get(t::Int, {t::SEXP}, false), {}}; - get_(Id::recordTypefeedback) = { - "recordTypefeedback", - (void*)&recordTypefeedbackImpl, - llvm::FunctionType::get(t::t_void, {t::i64, t::i64, t::SEXP}, false), + get_(Id::recordTypeFeedback) = { + "recordTypeFeedback", + (void*)&recordTypeFeedbackImpl, + llvm::FunctionType::get(t::t_void, {t::voidPtr, t::i32, t::SEXP}, + false), + {}}; + get_(Id::recordCallFeedback) = { + "recordCallFeedback", + (void*)&recordCallFeedbackImpl, + llvm::FunctionType::get(t::t_void, {t::voidPtr, t::i32, t::SEXP}, + false), {}}; get_(Id::deopt) = {"deopt", (void*)&deoptImpl, diff --git a/rir/src/compiler/native/builtins.h b/rir/src/compiler/native/builtins.h index dfaf0dadc..1becc9ed5 100644 --- a/rir/src/compiler/native/builtins.h +++ b/rir/src/compiler/native/builtins.h @@ -85,7 +85,8 @@ struct NativeBuiltins { checkTrueFalse, asLogicalBlt, length, - recordTypefeedback, + recordTypeFeedback, + recordCallFeedback, deopt, assertFail, printValue, diff --git a/rir/src/compiler/native/lower_function_llvm.cpp b/rir/src/compiler/native/lower_function_llvm.cpp index 75fd4eed8..01e1cd912 100644 --- a/rir/src/compiler/native/lower_function_llvm.cpp +++ b/rir/src/compiler/native/lower_function_llvm.cpp @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -418,13 +419,15 @@ llvm::Value* LowerFunctionLLVM::load(Value* val, PirType type, Rep needed) { auto srcAddr = (Constant*)builder.CreateIntToPtr( llvm::ConstantInt::get( PirJitLLVM::getContext(), - llvm::APInt(64, - reinterpret_cast(dr->reason.srcCode()), - false)), + llvm::APInt( + 64, + reinterpret_cast(dr->reason.origin.function()), + false)), t::voidPtr); auto drs = llvm::ConstantStruct::get( - t::DeoptReason, {c(dr->reason.reason, 32), - c(dr->reason.origin.offset(), 32), srcAddr}); + t::DeoptReason, + {c(dr->reason.reason, 32), + c(dr->reason.origin.index().asInteger(), 32), srcAddr}); res = globalConst(drs); } else { val->printRef(std::cerr); @@ -6129,21 +6132,25 @@ void LowerFunctionLLVM::compile() { if (cls->isContinuation() && Rep::Of(i) == Rep::SEXP && variables_.count(i) && !cls->isContinuation()->continuationContext->asDeoptContext()) { - if (i->hasTypeFeedback() && - i->typeFeedback().feedbackOrigin.pc()) { - call(NativeBuiltins::get( - NativeBuiltins::Id::recordTypefeedback), - {c((void*)i->typeFeedback().feedbackOrigin.pc()), - c((void*)i->typeFeedback().feedbackOrigin.srcCode()), - load(i)}); + if (i->hasTypeFeedback()) { + auto& origin = i->typeFeedback().feedbackOrigin; + if (origin.hasSlot()) { + call( + NativeBuiltins::get( + NativeBuiltins::Id::recordTypeFeedback), + {convertToPointer(origin.function()->typeFeedback(), + t::i8, true), + c(origin.index().idx, 32), load(i)}); + } } if (i->hasCallFeedback()) { - assert(i->callFeedback().feedbackOrigin.pc()); + auto& origin = i->callFeedback().feedbackOrigin; + assert(origin.hasSlot()); call(NativeBuiltins::get( - NativeBuiltins::Id::recordTypefeedback), - {c((void*)i->callFeedback().feedbackOrigin.pc()), - c((void*)i->callFeedback().feedbackOrigin.srcCode()), - load(i)}); + NativeBuiltins::Id::recordCallFeedback), + {convertToPointer(origin.function()->typeFeedback(), + t::i8, true), + c(origin.index().idx, 32), load(i)}); } } @@ -6249,12 +6256,13 @@ void LowerFunctionLLVM::compile() { auto i = var.first; if (Rep::Of(i) != Rep::SEXP) continue; - if (!i->typeFeedback().feedbackOrigin.pc()) + if (!i->typeFeedback().feedbackOrigin.hasSlot()) continue; if (!var.second.initialized) continue; if (var.second.stackSlot < PirTypeFeedback::MAX_SLOT_IDX) { - codes.insert(i->typeFeedback().feedbackOrigin.srcCode()); + codes.insert( + i->typeFeedback().feedbackOrigin.function()->body()); variableMapping.emplace(var.second.stackSlot, i->typeFeedback()); #ifdef DEBUG_REGISTER_MAP diff --git a/rir/src/compiler/opt/eager_calls.cpp b/rir/src/compiler/opt/eager_calls.cpp index a0504a217..326d223ed 100644 --- a/rir/src/compiler/opt/eager_calls.cpp +++ b/rir/src/compiler/opt/eager_calls.cpp @@ -26,7 +26,7 @@ bool EagerCalls::apply(Compiler& cmp, ClosureVersion* cls, Code* code, Speculation() {} Speculation(SEXP builtin, Checkpoint* cp, const FeedbackOrigin& origin) : builtin(builtin), cp(cp), origin(origin) { - assert(origin.pc()); + assert(origin.hasSlot()); } }; @@ -140,7 +140,8 @@ bool EagerCalls::apply(Compiler& cmp, ClosureVersion* cls, Code* code, break; VECTOR_RW_INSTRUCTIONS(V); #undef V - default: {} + default: { + } } if (auto call = Call::Cast(*ip)) { @@ -210,7 +211,7 @@ bool EagerCalls::apply(Compiler& cmp, ClosureVersion* cls, Code* code, } if (!inBase && ldfun->typeFeedback() - .feedbackOrigin.pc()) + .feedbackOrigin.hasSlot()) needsGuard[ldfun] = { builtin, cp, ldfun->typeFeedback() @@ -262,7 +263,8 @@ bool EagerCalls::apply(Compiler& cmp, ClosureVersion* cls, Code* code, } break; VECTOR_RW_INSTRUCTIONS(V) #undef V - default: {} + default: { + } } // Look for static calls, where we statically know that all (or @@ -409,9 +411,9 @@ bool EagerCalls::apply(Compiler& cmp, ClosureVersion* cls, Code* code, } next = ip + 1; - // This might fire back, since we don't know if we really have no - // objects... We should have some profiling. It's still sound, since - // static_call_ will check the assumptions + // This might fire back, since we don't know if we really have + // no objects... We should have some profiling. It's still + // sound, since static_call_ will check the assumptions for (size_t i = 0; i < call->nCallArgs(); ++i) if (!newAssumptions.isNotObj(i) && newAssumptions.isEager(i)) diff --git a/rir/src/compiler/opt/type_test.h b/rir/src/compiler/opt/type_test.h index cb6501e43..47ede220f 100644 --- a/rir/src/compiler/opt/type_test.h +++ b/rir/src/compiler/opt/type_test.h @@ -35,10 +35,10 @@ class TypeTest { return failed(); } - if (!feedback.feedbackOrigin.pc()) + if (!feedback.feedbackOrigin.hasSlot()) return failed(); - assert(feedback.feedbackOrigin.pc()); + assert(feedback.feedbackOrigin.hasSlot()); // First try to refine the type if (!expected.maybeObj() && // TODO: Is this right? (expected.noAttribsOrObject().isA(RType::integer) || diff --git a/rir/src/compiler/opt/typefeedback_cleanup.cpp b/rir/src/compiler/opt/typefeedback_cleanup.cpp index 1f8489994..48c981047 100644 --- a/rir/src/compiler/opt/typefeedback_cleanup.cpp +++ b/rir/src/compiler/opt/typefeedback_cleanup.cpp @@ -25,7 +25,8 @@ bool TypefeedbackCleanup::apply(Compiler& cmp, ClosureVersion* cls, Code* code, std::unordered_set affected; if (deoptCtx) { - if (deoptCtx->reason().srcCode() != cls->rirSrc()) { + if (deoptCtx->reason().origin.function() != + cls->owner()->rirFunction()) { Visitor::run(version->entry, [&](Instruction* i) { if (!i->hasTypeFeedback()) return; @@ -36,8 +37,8 @@ bool TypefeedbackCleanup::apply(Compiler& cmp, ClosureVersion* cls, Code* code, if (!i->hasTypeFeedback()) return; - if (i->typeFeedback().feedbackOrigin.pc() == - deoptCtx->reason().pc()) { + if (i->typeFeedback().feedbackOrigin == + deoptCtx->reason().origin) { if (deoptCtx->reason().reason == DeoptReason::Typecheck) { i->updateTypeFeedback().type = deoptCtx->typeCheckTrigger(); diff --git a/rir/src/compiler/osr.cpp b/rir/src/compiler/osr.cpp index c6ac62c88..76149fc9d 100644 --- a/rir/src/compiler/osr.cpp +++ b/rir/src/compiler/osr.cpp @@ -4,6 +4,7 @@ #include "compiler/compiler.h" #include "pir/deopt_context.h" #include "pir/pir_impl.h" +#include "runtime/DispatchTable.h" namespace rir { namespace pir { @@ -26,6 +27,8 @@ Function* OSR::compile(SEXP closure, rir::Code* c, [&](Continuation* cnt) { cmp.optimizeModule(); fun = backend.getOrCompile(cnt); + auto dt = DispatchTable::unpack(BODY(closure)); + fun->dispatchTable(dt); }, [&]() { std::cerr << "Continuation compilation failed\n"; }); diff --git a/rir/src/compiler/parameter.h b/rir/src/compiler/parameter.h index 3a474abdf..1cda46864 100644 --- a/rir/src/compiler/parameter.h +++ b/rir/src/compiler/parameter.h @@ -16,6 +16,7 @@ struct Parameter { static const unsigned PIR_WARMUP; static const unsigned PIR_OPT_TIME; static const unsigned PIR_REOPT_TIME; + static const unsigned PIR_OPT_BC_SIZE; static const unsigned DEOPT_ABANDON; static size_t PROMISE_INLINER_MAX_SIZE; diff --git a/rir/src/compiler/pir/builder.cpp b/rir/src/compiler/pir/builder.cpp index e72412ae2..fdb875de6 100644 --- a/rir/src/compiler/pir/builder.cpp +++ b/rir/src/compiler/pir/builder.cpp @@ -54,7 +54,8 @@ void Builder::add(Instruction* i) { assert(false && "Invalid instruction"); case Tag::PirCopy: assert(false && "This instruction is only allowed during lowering"); - default: {} + default: { + } } bb->append(i); } @@ -125,8 +126,10 @@ Builder::Builder(Continuation* cnt, Value* closureEnv) } auto mkenv = new MkEnv(closureEnv, names, args.data(), miss); - auto rirCode = cnt->owner()->rirFunction()->body(); - mkenv->updateTypeFeedback().feedbackOrigin.srcCode(rirCode); + // FIXME: (cf. #1260) what does this mean, we need both rirFun and we + // need idx + mkenv->updateTypeFeedback().feedbackOrigin.function( + cnt->owner()->rirFunction()); add(mkenv); this->env = mkenv; } else { @@ -172,7 +175,9 @@ Builder::Builder(ClosureVersion* version, Value* closureEnv) auto rirFun = version->owner()->rirFunction(); if (rirFun->flags.contains(rir::Function::NeedsFullEnv)) mkenv->neverStub = true; - mkenv->updateTypeFeedback().feedbackOrigin.srcCode(rirFun->body()); + // FIXME: (cf. #1260) what does this mean, we need both rirFun and we need + // idx + mkenv->updateTypeFeedback().feedbackOrigin.function(rirFun); add(mkenv); this->env = mkenv; } diff --git a/rir/src/compiler/pir/instruction.cpp b/rir/src/compiler/pir/instruction.cpp index c0644b955..0acb340a5 100644 --- a/rir/src/compiler/pir/instruction.cpp +++ b/rir/src/compiler/pir/instruction.cpp @@ -211,7 +211,7 @@ void Instruction::print(std::ostream& out, bool tty) const { typeFeedback().value->printRef(out); else if (!typeFeedback().type.isVoid()) out << typeFeedback().type; - if (!typeFeedback().feedbackOrigin.pc()) + if (!typeFeedback().feedbackOrigin.hasSlot()) out << "@?"; out << ">"; } diff --git a/rir/src/compiler/rir2pir/rir2pir.cpp b/rir/src/compiler/rir2pir/rir2pir.cpp index e157fedc8..ea193fd63 100644 --- a/rir/src/compiler/rir2pir/rir2pir.cpp +++ b/rir/src/compiler/rir2pir/rir2pir.cpp @@ -17,6 +17,7 @@ #include "compiler/util/visitor.h" #include "insert_cast.h" #include "runtime/ArglistOrder.h" +#include "runtime/TypeFeedback.h" #include "simple_instruction_list.h" #include "utils/FormalArgs.h" @@ -144,9 +145,10 @@ namespace pir { Rir2Pir::Rir2Pir(Compiler& cmp, ClosureVersion* cls, ClosureLog& log, const std::string& name, - const std::list& outerFeedback) + const std::list& outerFeedback, + rir::TypeFeedback* typeFeedback) : compiler(cmp), cls(cls), log(log), name(name), - outerFeedback(outerFeedback) { + outerFeedback(outerFeedback), typeFeedback(typeFeedback) { if (cls->optFunction && cls->optFunction->body()->pirTypeFeedback()) this->outerFeedback.push_back( cls->optFunction->body()->pirTypeFeedback()); @@ -370,7 +372,9 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, } case Opcode::record_test_: { - auto feedback = bc.immediate.testFeedback; + uint32_t idx = bc.immediate.i; + auto& feedback = typeFeedback->test(idx); + if (feedback.seen == ObservedTest::OnlyTrue || feedback.seen == ObservedTest::OnlyFalse) { if (auto i = Instruction::Cast(at(0))) { @@ -380,7 +384,8 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, if (!i->typeFeedback().value) { auto& t = i->updateTypeFeedback(); t.value = v; - t.feedbackOrigin = FeedbackOrigin(srcCode, pos); + t.feedbackOrigin = FeedbackOrigin(srcCode->function(), + FeedbackIndex::test(idx)); } else if (i->typeFeedback().value != v) { i->updateTypeFeedback().value = nullptr; } @@ -395,7 +400,9 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, } case Opcode::record_type_: { - auto feedback = bc.immediate.typeFeedback; + uint32_t idx = bc.immediate.i; + auto& feedback = typeFeedback->types(idx); + if (auto i = Instruction::Cast(at(0))) { // Search for the most specific feedback for this location for (auto fb : outerFeedback) { @@ -404,8 +411,9 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, fb->forEachSlot( [&](size_t i, const PirTypeFeedback::MDEntry& mdEntry) { found = true; - auto origin = fb->getOriginOfSlot(i); - if (origin == pos && mdEntry.readyForReopt) { + auto origin = fb->rirIdx(i); + if (origin == FeedbackIndex::type(idx) && + mdEntry.readyForReopt) { feedback = mdEntry.feedback; } }); @@ -414,7 +422,8 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, } // TODO: deal with multiple locations auto& t = i->updateTypeFeedback(); - t.feedbackOrigin = FeedbackOrigin(srcCode, pos); + t.feedbackOrigin = + FeedbackOrigin(srcCode->function(), FeedbackIndex::type(idx)); if (feedback.numTypes) { t.type.merge(feedback); if (auto force = Force::Cast(i)) { @@ -431,9 +440,10 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, } case Opcode::record_call_: { + uint32_t idx = bc.immediate.i; Value* target = top(); - auto feedback = bc.immediate.callFeedback; + auto& feedback = typeFeedback->callees(bc.immediate.i); // If this call was never executed we might as well compile an // unconditional deopt. @@ -443,8 +453,9 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, auto sp = insert.registerFrameState(srcCode, pos, stack, inPromise()); - DeoptReason reason = DeoptReason(FeedbackOrigin(srcCode, pos), - DeoptReason::DeadCall); + DeoptReason reason = DeoptReason( + FeedbackOrigin(srcCode->function(), FeedbackIndex::call(idx)), + DeoptReason::DeadCall); auto d = insert(new Deopt(sp)); d->setDeoptReason(compiler.module->deoptReasonValue(reason), @@ -453,17 +464,17 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, } else if (auto i = Instruction::Cast(target)) { // See if the call feedback suggests a monomorphic target // TODO: Deopts in promises are not supported by the promise - // inliner. So currently it does not pay off to put any deopts in - // there. + // inliner. So currently it does not pay off to put any deopts + // in there. // auto& f = i->updateCallFeedback(); - const auto& feedback = bc.immediate.callFeedback; f.taken = feedback.taken; - f.feedbackOrigin = FeedbackOrigin(srcCode, pos); + f.feedbackOrigin = + FeedbackOrigin(srcCode->function(), FeedbackIndex::call(idx)); if (feedback.numTargets == 1) { assert(!feedback.invalid && "feedback can't be invalid if numTargets is 1"); - f.monomorphic = feedback.getTarget(srcCode, 0); + f.monomorphic = feedback.getTarget(srcCode->function(), 0); f.type = TYPEOF(f.monomorphic); f.stableEnv = true; } else if (feedback.numTargets > 1) { @@ -472,7 +483,7 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, bool stableBody = !feedback.invalid; bool stableEnv = !feedback.invalid; for (size_t i = 0; i < feedback.numTargets; ++i) { - SEXP b = feedback.getTarget(srcCode, i); + SEXP b = feedback.getTarget(srcCode->function(), i); if (!first) { first = b; } else { @@ -492,11 +503,12 @@ bool Rir2Pir::compileBC(const BC& bc, Opcode* pos, Opcode* nextPos, if (auto c = cls->isContinuation()) { if (auto d = c->continuationContext->asDeoptContext()) { if (d->reason().reason == DeoptReason::CallTarget) { - if (d->reason().pc() == pos) { + if (d->reason().origin.idx() == idx) { auto deoptCallTarget = d->callTargetTrigger(); for (size_t i = 0; i < feedback.numTargets; ++i) { - SEXP b = feedback.getTarget(srcCode, i); + SEXP b = feedback.getTarget( + srcCode->function(), i); if (b != deoptCallTarget) deoptedCallTargets.insert(b); } @@ -1347,12 +1359,14 @@ bool Rir2Pir::tryCompile(rir::Code* srcCode, Builder& insert, Opcode* start, } bool Rir2Pir::tryCompilePromise(rir::Code* prom, Builder& insert) { - return PromiseRir2Pir(compiler, cls, log, name, outerFeedback, false) + return PromiseRir2Pir(compiler, cls, log, name, outerFeedback, typeFeedback, + false) .tryCompile(prom, insert); } Value* Rir2Pir::tryInlinePromise(rir::Code* srcCode, Builder& insert) { - return PromiseRir2Pir(compiler, cls, log, name, outerFeedback, true) + return PromiseRir2Pir(compiler, cls, log, name, outerFeedback, typeFeedback, + true) .tryTranslate(srcCode, insert); } diff --git a/rir/src/compiler/rir2pir/rir2pir.h b/rir/src/compiler/rir2pir/rir2pir.h index afe022161..1f463b383 100644 --- a/rir/src/compiler/rir2pir/rir2pir.h +++ b/rir/src/compiler/rir2pir/rir2pir.h @@ -3,6 +3,7 @@ #include "compiler/compiler.h" #include "compiler/pir/builder.h" +#include "runtime/TypeFeedback.h" #include #include @@ -17,7 +18,8 @@ class Rir2Pir { public: Rir2Pir(Compiler& cmp, ClosureVersion* cls, ClosureLog& log, const std::string& name, - const std::list& outerFeedback); + const std::list& outerFeedback, + rir::TypeFeedback* typeFeedback); bool tryCompile(Builder& insert) __attribute__((warn_unused_result)); bool tryCompileContinuation(Builder& insert, Opcode* start, @@ -58,6 +60,7 @@ class Rir2Pir { ClosureLog& log; std::string name; std::list outerFeedback; + rir::TypeFeedback* typeFeedback; std::unordered_map localFuns; std::unordered_set deoptedCallTargets; @@ -88,8 +91,9 @@ class PromiseRir2Pir : public Rir2Pir { PromiseRir2Pir(Compiler& cmp, ClosureVersion* cls, ClosureLog& log, const std::string& name, const std::list& outerFeedback, - bool inlining) - : Rir2Pir(cmp, cls, log, name, outerFeedback), inlining_(inlining) {} + rir::TypeFeedback* feedback, bool inlining) + : Rir2Pir(cmp, cls, log, name, outerFeedback, feedback), + inlining_(inlining) {} private: bool inlining_; diff --git a/rir/src/interpreter/interp.cpp b/rir/src/interpreter/interp.cpp index 7e25478d7..8e97ec978 100644 --- a/rir/src/interpreter/interp.cpp +++ b/rir/src/interpreter/interp.cpp @@ -823,6 +823,8 @@ const unsigned pir::Parameter::PIR_REOPT_TIME = getenv("PIR_REOPT_TIME") ? atoi(getenv("PIR_REOPT_TIME")) : 5e7; const unsigned pir::Parameter::DEOPT_ABANDON = getenv("PIR_DEOPT_ABANDON") ? atoi(getenv("PIR_DEOPT_ABANDON")) : 12; +const unsigned pir::Parameter::PIR_OPT_BC_SIZE = + getenv("PIR_OPT_BC_SIZE") ? atoi(getenv("PIR_OPT_BC_SIZE")) : 20; static unsigned serializeCounter = 0; @@ -1002,7 +1004,8 @@ SEXP doCall(CallContext& call, bool popArgs) { !call.caller->isCompiled() && !call.caller->function()->disabled() && call.caller->size() < pir::Parameter::MAX_INPUT_SIZE && - fun->body()->codeSize < 20) { + fun->body()->codeSize < + pir::Parameter::PIR_OPT_BC_SIZE) { call.triggerOsr = true; } DoRecompile(fun, call.ast, call.callee, given); @@ -1996,11 +1999,18 @@ SEXP evalRirCode(Code* c, SEXP env, const CallContext* callCtxt, state = ObservedValues::StateBeforeLastForce::promise; } - ObservedValues* feedback = (ObservedValues*)(pc + 1); - if (feedback->stateBeforeLastForce < state) - feedback->stateBeforeLastForce = state; + auto idx = *(Immediate*)(pc + 1); + // FIXME: cf. #1260 + c->function()->typeFeedback()->record_type(idx, [&](auto& feedback) { + if (feedback.stateBeforeLastForce < state) { + feedback.stateBeforeLastForce = state; + } + }); }; + auto function = c->function(); + auto typeFeedback = function->typeFeedback(); + // main loop BEGIN_MACHINE { @@ -2306,26 +2316,26 @@ SEXP evalRirCode(Code* c, SEXP env, const CallContext* callCtxt, } INSTRUCTION(record_call_) { - ObservedCallees* feedback = (ObservedCallees*)pc; + Immediate idx = readImmediate(); + advanceImmediate(); SEXP callee = ostack_top(); - feedback->record(c, callee); - pc += sizeof(ObservedCallees); + typeFeedback->record_callee(idx, function, callee); NEXT(); } INSTRUCTION(record_test_) { - ObservedTest* feedback = (ObservedTest*)pc; + Immediate idx = readImmediate(); + advanceImmediate(); SEXP t = ostack_top(); - feedback->record(t); - pc += sizeof(ObservedTest); + typeFeedback->record_test(idx, t); NEXT(); } INSTRUCTION(record_type_) { - ObservedValues* feedback = (ObservedValues*)pc; + Immediate idx = readImmediate(); + advanceImmediate(); SEXP t = ostack_top(); - feedback->record(t); - pc += sizeof(ObservedValues); + typeFeedback->record_type(idx, t); NEXT(); } diff --git a/rir/src/runtime/Code.cpp b/rir/src/runtime/Code.cpp index b7ca453cd..e65ebcccd 100644 --- a/rir/src/runtime/Code.cpp +++ b/rir/src/runtime/Code.cpp @@ -3,7 +3,9 @@ #include "R/Printing.h" #include "R/Serialize.h" #include "bc/BC.h" +#include "bc/BC_inc.h" #include "compiler/native/pir_jit_llvm.h" +#include "runtime/TypeFeedback.h" #include "utils/Pool.h" #include @@ -187,7 +189,7 @@ void Code::disassemble(std::ostream& out, const std::string& prefix) const { map->forEachSlot( [&](size_t i, const PirTypeFeedback::MDEntry& mdEntry) { auto feedback = mdEntry.feedback; - out << " - slot #" << i << ": " << mdEntry.offset << " : ["; + out << " - slot #" << i << ": " << mdEntry.rirIdx << " : ["; feedback.print(out); out << "] (" << mdEntry.sampleCount << " records - " << (mdEntry.readyForReopt ? "ready" : "not ready") @@ -197,6 +199,8 @@ void Code::disassemble(std::ostream& out, const std::string& prefix) const { switch (kind) { case Kind::Bytecode: { + Function* fun = function(); + TypeFeedback* typeFeedback = fun->typeFeedback(); Opcode* pc = code(); size_t label = 0; std::map targets; @@ -257,10 +261,23 @@ void Code::disassemble(std::ostream& out, const std::string& prefix) const { bc.printOpcode(out); formatLabel(targets[BC::jmpTarget(pc)]); out << "\n"; + } else if (bc.isRecord()) { + out << " " + << "[ "; + if (bc.bc == Opcode::record_call_) { + typeFeedback->callees(bc.immediate.i).print(out, fun); + out << " ] Call#"; + } else if (bc.bc == Opcode::record_test_) { + typeFeedback->test(bc.immediate.i).print(out); + out << " ] Test#"; + } else { + typeFeedback->types(bc.immediate.i).print(out); + out << " ] Type#"; + } + out << bc.immediate.i << "\n"; } else { bc.print(out); } - pc = BC::next(pc); } diff --git a/rir/src/runtime/DispatchTable.h b/rir/src/runtime/DispatchTable.h index 7043f603e..6aeefc899 100644 --- a/rir/src/runtime/DispatchTable.h +++ b/rir/src/runtime/DispatchTable.h @@ -4,7 +4,9 @@ #include "Function.h" #include "R/Serialize.h" #include "RirRuntimeObject.h" +#include "TypeFeedback.h" #include "utils/random.h" +#include namespace rir { @@ -92,6 +94,7 @@ struct DispatchTable assert(baseline()->signature().optimization == FunctionSignature::OptimizationLevel::Baseline); setEntry(0, f->container()); + f->dispatchTable(this); } bool contains(const Context& assumptions) const { @@ -104,8 +107,11 @@ struct DispatchTable void remove(Code* funCode) { size_t i = 1; for (; i < size(); ++i) { - if (get(i)->body() == funCode) + auto fun = get(i); + if (fun->body() == funCode) { + fun->dispatchTable(nullptr); break; + } } if (i == size()) return; @@ -122,18 +128,18 @@ struct DispatchTable assert(size() > 0); assert(fun->signature().optimization != FunctionSignature::OptimizationLevel::Baseline); + fun->dispatchTable(this); auto assumptions = fun->context(); size_t i; for (i = size() - 1; i > 0; --i) { auto old = get(i); if (old->context() == assumptions) { - if (i != 0) { - // Remember deopt counts across recompilation to avoid - // deopt loops - fun->addDeoptCount(old->deoptCount()); - setEntry(i, fun->container()); - assert(get(i) == fun); - } + // Remember deopt counts across recompilation to avoid + // deopt loops + fun->addDeoptCount(old->deoptCount()); + setEntry(i, fun->container()); + assert(get(i) == fun); + // old->dispatchTable(nullptr); return; } if (!(assumptions < get(i)->context())) { @@ -199,8 +205,9 @@ struct DispatchTable AddReadRef(refTable, table->container()); table->size_ = InInteger(inp); for (size_t i = 0; i < table->size(); i++) { - table->setEntry(i, - Function::deserialize(refTable, inp)->container()); + auto fun = Function::deserialize(refTable, inp); + fun->dispatchTable(table); + table->setEntry(i, fun->container()); } UNPROTECT(1); return table; @@ -235,14 +242,56 @@ struct DispatchTable return userDefinedContext_ | anotherContext; } + void print(std::ostream& out, bool verbose) const { + std::cout << "== dispatch table " << this << " ==\n"; + + for (size_t entry = 0; entry < size(); ++entry) { + Function* f = get(entry); + std::cout << "= version " << entry << " (" << f << ") =\n"; + f->disassemble(std::cout); + } + + if (verbose) { + auto code = baseline()->body(); + auto pc = code->code(); + auto print_header = true; + + Opcode* prev = NULL; + Opcode* pprev = NULL; + + while (pc < code->endCode()) { + auto bc = BC::decode(pc, code); + if (bc.bc == Opcode::close_) { + if (print_header) { + out << "== nested closures ==\n"; + print_header = false; + } + + // prev is the push_ of srcref + // pprev is the push_ of body + auto body = BC::decodeShallow(pprev).immediateConst(); + auto dt = DispatchTable::unpack(body); + dt->print(std::cout, verbose); + } + pprev = prev; + prev = pc; + pc = bc.next(pc); + } + } + } + + size_t currentTypeFeedbackVersion() { + return baseline()->typeFeedback()->version(); + } + private: DispatchTable() = delete; - explicit DispatchTable(size_t cap) + explicit DispatchTable(size_t capacity) : RirRuntimeObject( // GC area starts at the end of the DispatchTable sizeof(DispatchTable), // GC area is just the pointers in the entry array - cap) {} + capacity) {} size_t size_ = 0; Context userDefinedContext_; diff --git a/rir/src/runtime/Function.cpp b/rir/src/runtime/Function.cpp index 1e6d9ba50..dc7a5e003 100644 --- a/rir/src/runtime/Function.cpp +++ b/rir/src/runtime/Function.cpp @@ -1,6 +1,8 @@ #include "Function.h" #include "R/Serialize.h" +#include "Rinternals.h" #include "compiler/compiler.h" +#include "runtime/TypeFeedback.h" namespace rir { @@ -10,18 +12,25 @@ Function* Function::deserialize(SEXP refTable, R_inpstream_t inp) { const Context as = Context::deserialize(refTable, inp); SEXP store = Rf_allocVector(EXTERNALSXP, functionSize); void* payload = DATAPTR(store); - Function* fun = new (payload) Function(functionSize, nullptr, {}, sig, as); + Function* fun = + new (payload) Function(functionSize, nullptr, {}, sig, as, nullptr); fun->numArgs_ = InInteger(inp); fun->info.gc_area_length += fun->numArgs_; - for (unsigned i = 0; i < fun->numArgs_ + 1; i++) { + // What this loop does is that it sets the function owned (yet not + // deserialized) SEXPs to something reasonable so it will not confuse the GC + // which might run while they are deserialized. + for (unsigned i = 0; i < fun->numArgs_ + NUM_PTRS; i++) { fun->setEntry(i, R_NilValue); } PROTECT(store); AddReadRef(refTable, store); + TypeFeedback* feedback = TypeFeedback::deserialize(refTable, inp); + PROTECT(feedback->container()); + fun->typeFeedback(feedback); SEXP body = Code::deserialize(refTable, inp)->container(); fun->body(body); PROTECT(body); - int protectCount = 2; + int protectCount = 3; for (unsigned i = 0; i < fun->numArgs_; i++) { if ((bool)InInteger(inp)) { SEXP arg = Code::deserialize(refTable, inp)->container(); @@ -42,6 +51,7 @@ void Function::serialize(SEXP refTable, R_outpstream_t out) const { context_.serialize(refTable, out); OutInteger(out, numArgs_); HashAdd(container(), refTable); + typeFeedback()->serialize(refTable, out); body()->serialize(refTable, out); for (unsigned i = 0; i < numArgs_; i++) { Code* arg = defaultArg(i); @@ -58,6 +68,7 @@ void Function::disassemble(std::ostream& out) { if (!context_.empty()) out << "| context: [" << context_ << "]"; out << "\n"; + out << "[type feedback version] " << typeFeedback()->version() << "\n"; out << "[flags] "; #define V(F) \ if (flags.includes(F)) \ diff --git a/rir/src/runtime/Function.h b/rir/src/runtime/Function.h index 04d3ea0d3..022c74524 100644 --- a/rir/src/runtime/Function.h +++ b/rir/src/runtime/Function.h @@ -5,13 +5,15 @@ #include "FunctionSignature.h" #include "R/r.h" #include "RirRuntimeObject.h" +#include "runtime/TypeFeedback.h" namespace rir { +struct DispatchTable; + /** * Aliases for readability. */ -typedef SEXP FunctionSEXP; // Function magic constant is designed to help to distinguish between Function // objects and normal EXTERNALSXPs. Normally this is not necessary, but a very @@ -39,24 +41,42 @@ struct Function : public RirRuntimeObject { friend class FunctionCodeIterator; friend class ConstFunctionCodeIterator; - static constexpr size_t NUM_PTRS = 1; + // In its entries, a function ows two SEXP pointers + a variable length of + // default arguments code: + static constexpr size_t NUM_PTRS = 2; + // 0: body (Code*) + static constexpr size_t BODY_IDX = 0; + // 1: type feedback (TypeFeedback*) + static constexpr size_t TYPE_FEEDBACK_IDX = 1; Function(size_t functionSize, SEXP body_, const std::vector& defaultArgs, - const FunctionSignature& signature, const Context& ctx) + const FunctionSignature& signature, const Context& ctx, + TypeFeedback* feedback) : RirRuntimeObject( // GC area starts at &locals and goes to the end of defaultArg_ - sizeof(Function) - NUM_PTRS * sizeof(FunctionSEXP), + sizeof(Function) - NUM_PTRS * sizeof(SEXP), NUM_PTRS + defaultArgs.size()), size(functionSize), numArgs_(defaultArgs.size()), signature_(signature), context_(ctx) { for (size_t i = 0; i < numArgs_; ++i) setEntry(NUM_PTRS + i, defaultArgs[i]); body(body_); + if (feedback) { + typeFeedback(feedback); + } } - Code* body() const { return Code::unpack(getEntry(0)); } - void body(SEXP body) { setEntry(0, body); } + Code* body() const { return Code::unpack(getEntry(BODY_IDX)); } + void body(SEXP body) { setEntry(BODY_IDX, body); } + + TypeFeedback* typeFeedback() const { + return TypeFeedback::unpack(getEntry(TYPE_FEEDBACK_IDX)); + } + void typeFeedback(TypeFeedback* typeFeedback) { + typeFeedback->owner_ = this; + setEntry(TYPE_FEEDBACK_IDX, typeFeedback->container()); + } static Function* deserialize(SEXP refTable, R_inpstream_t inp); void serialize(SEXP refTable, R_outpstream_t out) const; @@ -135,7 +155,7 @@ struct Function : public RirRuntimeObject { RIR_FUNCTION_FLAGS(V) #undef V - FIRST = Deopt, + FIRST = Deopt, LAST = DisableNumArgumentsSpezialization }; EnumSet flags; @@ -186,6 +206,9 @@ struct Function : public RirRuntimeObject { return deadCallReached_; } + void dispatchTable(DispatchTable* dt) { dispatchTable_ = dt; } + DispatchTable* dispatchTable() { return dispatchTable_; } + private: unsigned numArgs_; @@ -199,11 +222,12 @@ struct Function : public RirRuntimeObject { FunctionSignature signature_; /// pointer to this version's signature Context context_; + DispatchTable* dispatchTable_; // !!! SEXPs traceable by the GC must be declared here !!! - // locals contains: body - CodeSEXP locals[NUM_PTRS]; - CodeSEXP defaultArg_[]; + // locals contains: body (BODY_IDX) and typeFeedback (TYPE_FEEDBACK_IDX) + SEXP locals[NUM_PTRS]; + SEXP defaultArg_[]; }; #pragma pack(pop) diff --git a/rir/src/runtime/PirTypeFeedback.cpp b/rir/src/runtime/PirTypeFeedback.cpp index 66e0e0bdb..733938731 100644 --- a/rir/src/runtime/PirTypeFeedback.cpp +++ b/rir/src/runtime/PirTypeFeedback.cpp @@ -1,6 +1,7 @@ #include "PirTypeFeedback.h" #include "Code.h" #include "compiler/pir/instruction.h" +#include "runtime/TypeFeedback.h" #include #include @@ -23,46 +24,42 @@ PirTypeFeedback::PirTypeFeedback( // TODO, is this really needed? or is there any guarantee that my baseline // and all inlinee's baseline code objects stay live? also this should // probably be a weak map instead... - std::unordered_map srcCodeMap; + std::unordered_map functionMap; size_t idx = 0; for (auto c : codes) { - srcCodeMap[c] = idx; - setEntry(idx++, c->container()); + functionMap[c->function()] = idx; + setEntry(idx++, c->function()->container()); } idx = 0; - std::unordered_map reverseMapping; + std::unordered_map reverseMapping; for (auto s : slots) { auto slot = s.first; auto typeFeedback = s.second; assert(slot < MAX_SLOT_IDX); - auto e = reverseMapping.find(typeFeedback.feedbackOrigin.pc()); + auto e = reverseMapping.find(typeFeedback.feedbackOrigin); + if (e != reverseMapping.end()) { entry[slot] = e->second; assert(mdEntries()[e->second].previousType == typeFeedback.type); } else { - assert(codes.count(typeFeedback.feedbackOrigin.srcCode())); + assert(codes.count(typeFeedback.feedbackOrigin.function()->body())); new (&mdEntries()[idx]) MDEntry; - mdEntries()[idx].srcCode = - srcCodeMap.at(typeFeedback.feedbackOrigin.srcCode()); - mdEntries()[idx].offset = typeFeedback.feedbackOrigin.offset(); + mdEntries()[idx].funIdx = + functionMap.at(typeFeedback.feedbackOrigin.function()); + mdEntries()[idx].rirIdx = typeFeedback.feedbackOrigin.index(); mdEntries()[idx].previousType = typeFeedback.type; - reverseMapping[typeFeedback.feedbackOrigin.pc()] = idx; + reverseMapping[typeFeedback.feedbackOrigin] = idx; entry[slot] = idx++; } } } -Code* PirTypeFeedback::getSrcCodeOfSlot(size_t slot) { - auto code = getEntry(getMDEntryOfSlot(slot).srcCode); - return Code::unpack(code); -} - -Opcode* PirTypeFeedback::getOriginOfSlot(size_t slot) { - return getSrcCodeOfSlot(slot)->code() + getBCOffsetOfSlot(slot); +FeedbackIndex PirTypeFeedback::rirIdx(size_t slot) { + return getMDEntryOfSlot(slot).rirIdx; } } // namespace rir diff --git a/rir/src/runtime/PirTypeFeedback.h b/rir/src/runtime/PirTypeFeedback.h index 43168e640..62c5baeaa 100644 --- a/rir/src/runtime/PirTypeFeedback.h +++ b/rir/src/runtime/PirTypeFeedback.h @@ -21,7 +21,7 @@ struct Code; namespace pir { struct TypeFeedback; struct CallFeedback; -} +} // namespace pir struct PirTypeFeedback : public RirRuntimeObject { @@ -45,11 +45,8 @@ struct PirTypeFeedback ObservedValues& getSampleOfSlot(size_t slot) { return getMDEntryOfSlot(slot).feedback; } - unsigned getBCOffsetOfSlot(size_t slot) { - return getMDEntryOfSlot(slot).offset; - } - Code* getSrcCodeOfSlot(size_t slot); - Opcode* getOriginOfSlot(size_t slot); + + FeedbackIndex rirIdx(size_t slot); static size_t requiredSize(size_t origins, size_t entries) { return sizeof(PirTypeFeedback) + sizeof(SEXP) * origins + @@ -57,8 +54,8 @@ struct PirTypeFeedback } struct MDEntry { - uint8_t srcCode; - unsigned offset; + uint8_t funIdx; + FeedbackIndex rirIdx; ObservedValues feedback; pir::PirType previousType; unsigned sampleCount = 0; diff --git a/rir/src/runtime/TypeFeedback.cpp b/rir/src/runtime/TypeFeedback.cpp index 8a0ce4074..ff7402d5e 100644 --- a/rir/src/runtime/TypeFeedback.cpp +++ b/rir/src/runtime/TypeFeedback.cpp @@ -1,108 +1,97 @@ #include "TypeFeedback.h" +#include "R/Serialize.h" #include "R/Symbols.h" #include "R/r.h" #include "runtime/Code.h" #include "runtime/Function.h" #include +#include +#include namespace rir { -void ObservedCallees::record(Code* caller, SEXP callee, +bool ObservedCallees::record(Function* function, SEXP callee, bool invalidateWhenFull) { if (taken < CounterOverflow) taken++; - if (numTargets < MaxTargets) { int i = 0; + auto caller = function->body(); for (; i < numTargets; ++i) if (caller->getExtraPoolEntry(targets[i]) == callee) break; if (i == numTargets) { auto idx = caller->addExtraPoolEntry(callee); targets[numTargets++] = idx; + return true; } } else { - if (invalidateWhenFull) + if (invalidateWhenFull) { invalid = true; + return true; + } } + return false; } -SEXP ObservedCallees::getTarget(const Code* code, size_t pos) const { +SEXP ObservedCallees::getTarget(const Function* function, size_t pos) const { assert(pos < numTargets); - return code->getExtraPoolEntry(targets[pos]); + return function->body()->getExtraPoolEntry(targets[pos]); } -FeedbackOrigin::FeedbackOrigin(rir::Code* src, Opcode* p) - : offset_((uintptr_t)p - (uintptr_t)src), srcCode_(src) { - if (p) { - assert(p >= src->code()); - assert(p < src->endCode()); - assert(pc() == p); - } +FeedbackOrigin::FeedbackOrigin(rir::Function* function, FeedbackIndex index) + : index_(index), function_(function) { + assert(function->typeFeedback()->isValid(index)); } DeoptReason::DeoptReason(const FeedbackOrigin& origin, DeoptReason::Reason reason) - : reason(reason), origin(origin) { - switch (reason) { - case DeoptReason::Typecheck: - case DeoptReason::DeadCall: - case DeoptReason::CallTarget: - case DeoptReason::ForceAndCall: - case DeoptReason::DeadBranchReached: { - assert(pc()); - auto o = *pc(); - assert(o == Opcode::record_call_ || o == Opcode::record_type_ || - o == Opcode::record_test_); - break; - } - case DeoptReason::Unknown: - case DeoptReason::EnvStubMaterialized: - break; - } -} + : reason(reason), origin(origin) {} void DeoptReason::record(SEXP val) const { - srcCode()->function()->registerDeoptReason(reason); + origin.function()->registerDeoptReason(reason); switch (reason) { case DeoptReason::Unknown: break; case DeoptReason::DeadBranchReached: { - assert(*pc() == Opcode::record_test_); - ObservedTest* feedback = (ObservedTest*)(pc() + 1); - feedback->seen = ObservedTest::Both; + auto& feedback = origin.function()->typeFeedback()->test(origin.idx()); + feedback.seen = ObservedTest::Both; break; } case DeoptReason::Typecheck: { - assert(*pc() == Opcode::record_type_); if (val == symbol::UnknownDeoptTrigger) break; - ObservedValues* feedback = (ObservedValues*)(pc() + 1); - feedback->record(val); - if (TYPEOF(val) == PROMSXP) { - if (PRVALUE(val) == R_UnboundValue && - feedback->stateBeforeLastForce < ObservedValues::promise) - feedback->stateBeforeLastForce = ObservedValues::promise; - else if (feedback->stateBeforeLastForce < - ObservedValues::evaluatedPromise) - feedback->stateBeforeLastForce = - ObservedValues::evaluatedPromise; - } + auto feedback = origin.function()->typeFeedback(); + + // FIXME: (cf. #1260) very similar code is in the recordTypeFeedbackImpl + // IMHO the one there is more correct. Would it make sense + // to pull this into the TypeFeedback::record_type()? + // and get rid of the overload that takes lambda? + feedback->record_type(origin.idx(), val); + feedback->record_type(origin.idx(), [&](auto& slot) { + if (TYPEOF(val) == PROMSXP) { + if (PRVALUE(val) == R_UnboundValue && + slot.stateBeforeLastForce < ObservedValues::promise) + slot.stateBeforeLastForce = ObservedValues::promise; + else if (slot.stateBeforeLastForce < + ObservedValues::evaluatedPromise) + slot.stateBeforeLastForce = + ObservedValues::evaluatedPromise; + } + }); break; } case DeoptReason::DeadCall: case DeoptReason::ForceAndCall: case DeoptReason::CallTarget: { - assert(*pc() == Opcode::record_call_); if (val == symbol::UnknownDeoptTrigger) break; - ObservedCallees* feedback = (ObservedCallees*)(pc() + 1); - feedback->record(srcCode(), val, true); - assert(feedback->taken > 0); + auto feedback = origin.function()->typeFeedback(); + feedback->record_callee(origin.idx(), origin.function(), val, true); break; } case DeoptReason::EnvStubMaterialized: { @@ -111,4 +100,219 @@ void DeoptReason::record(SEXP val) const { } } +void ObservedCallees::print(std::ostream& out, const Function* function) const { + if (taken == ObservedCallees::CounterOverflow) + out << "*, <"; + else + out << taken << ", <"; + if (numTargets == ObservedCallees::MaxTargets) + out << "*>, "; + else + out << numTargets << ">, "; + + out << (invalid ? "invalid" : "valid"); + out << (numTargets ? ", " : " "); + + for (unsigned i = 0; i < numTargets; ++i) { + auto target = getTarget(function, i); + out << target << "(" << Rf_type2char(TYPEOF(target)) << ") "; + } +} + +void TypeFeedback::serialize(SEXP refTable, R_outpstream_t out) const { + OutInteger(out, callees_size_); + for (size_t i = 0; i < callees_size_; i++) { + OutBytes(out, callees_ + i, sizeof(ObservedCallees)); + } + + OutInteger(out, tests_size_); + for (size_t i = 0; i < tests_size_; i++) { + OutBytes(out, tests_ + i, sizeof(ObservedTest)); + } + + OutInteger(out, types_size_); + for (size_t i = 0; i < types_size_; i++) { + OutBytes(out, types_ + i, sizeof(ObservedValues)); + } + OutInteger(out, version_); +} + +TypeFeedback* TypeFeedback::deserialize(SEXP refTable, R_inpstream_t inp) { + auto size = InInteger(inp); + std::vector callees; + callees.reserve(size); + for (auto i = 0; i < size; ++i) { + ObservedCallees tmp; + InBytes(inp, &tmp, sizeof(ObservedCallees)); + callees.push_back(std::move(tmp)); + } + + size = InInteger(inp); + std::vector tests; + tests.reserve(size); + for (auto i = 0; i < size; ++i) { + ObservedTest tmp; + InBytes(inp, &tmp, sizeof(ObservedTest)); + tests.push_back(std::move(tmp)); + } + + size = InInteger(inp); + std::vector types; + types.reserve(size); + for (auto i = 0; i < size; ++i) { + ObservedValues tmp; + InBytes(inp, &tmp, sizeof(ObservedValues)); + types.push_back(std::move(tmp)); + } + + auto res = TypeFeedback::create(callees, tests, types); + res->version_ = InInteger(inp); + + return res; +} + +ObservedCallees& TypeFeedback::callees(uint32_t idx) { + return this->callees_[idx]; +} + +ObservedTest& TypeFeedback::test(uint32_t idx) { return this->tests_[idx]; } + +ObservedValues& TypeFeedback::types(uint32_t idx) { return this->types_[idx]; } + +void ObservedTest::print(std::ostream& out) const { + switch (seen) { + case ObservedTest::None: + out << "_"; + break; + case ObservedTest::OnlyTrue: + out << "T"; + break; + case ObservedTest::OnlyFalse: + out << "F"; + break; + case ObservedTest::Both: + out << "?"; + break; + } +} + +void ObservedValues::print(std::ostream& out) const { + if (numTypes) { + for (size_t i = 0; i < numTypes; ++i) { + out << Rf_type2char(seen[i]); + if (i != (unsigned)numTypes - 1) + out << ", "; + } + out << " (" << (object ? "o" : "") << (attribs ? "a" : "") + << (notFastVecelt ? "v" : "") << (!notScalar ? "s" : "") << ")"; + if (stateBeforeLastForce != + ObservedValues::StateBeforeLastForce::unknown) { + out << " | " + << ((stateBeforeLastForce == + ObservedValues::StateBeforeLastForce::value) + ? "value" + : (stateBeforeLastForce == + ObservedValues::StateBeforeLastForce::evaluatedPromise) + ? "evaluatedPromise" + : "promise"); + } + } else { + out << ""; + } +} + +bool FeedbackOrigin::hasSlot() const { return !index_.isUndefined(); } + +uint32_t TypeFeedback::Builder::addCallee() { return ncallees_++; } + +uint32_t TypeFeedback::Builder::addTest() { return ntests_++; } + +uint32_t TypeFeedback::Builder::addType() { return ntypes_++; } + +TypeFeedback* TypeFeedback::Builder::build() { + std::vector callees(ncallees_, ObservedCallees{}); + std::vector tests(ntests_, ObservedTest{}); + std::vector types(ntypes_, ObservedValues{}); + + return TypeFeedback::create(callees, tests, types); +} + +TypeFeedback* TypeFeedback::empty() { return TypeFeedback::create({}, {}, {}); } + +void FeedbackOrigin::function(Function* fun) { + assert(!hasSlot() || fun->typeFeedback()->isValid(index_)); + function_ = fun; +} +bool TypeFeedback::isValid(const FeedbackIndex& index) const { + switch (index.kind) { + case FeedbackKind::Call: + return index.idx < callees_size_; + case FeedbackKind::Test: + return index.idx < tests_size_; + case FeedbackKind::Type: + return index.idx < types_size_; + default: + return false; + } +} + +TypeFeedback* TypeFeedback::create(const std::vector& callees, + const std::vector& tests, + const std::vector& types) { + size_t dataSize = callees.size() * sizeof(ObservedCallees) + + tests.size() * sizeof(ObservedTest) + + types.size() * sizeof(ObservedValues); + + size_t objSize = sizeof(TypeFeedback) + dataSize; + + SEXP store = Rf_allocVector(EXTERNALSXP, objSize); + + TypeFeedback* res = + new (INTEGER(store)) TypeFeedback(callees, tests, types); + + return res; +} + +TypeFeedback::TypeFeedback(const std::vector& callees, + const std::vector& tests, + const std::vector& types) + : RirRuntimeObject(0, 0), version_(0), owner_(nullptr), + callees_size_(callees.size()), tests_size_(tests.size()), + types_size_(types.size()) { + + size_t callees_mem_size = callees_size_ * sizeof(ObservedCallees); + size_t tests_mem_size = tests_size_ * sizeof(ObservedTest); + size_t types_mem_size = types_size_ * sizeof(ObservedValues); + + callees_ = (ObservedCallees*)slots_; + tests_ = (ObservedTest*)(slots_ + callees_mem_size); + types_ = (ObservedValues*)(slots_ + callees_mem_size + tests_mem_size); + + if (callees_size_) { + memcpy(callees_, callees.data(), callees_mem_size); + } + + if (tests_size_) { + memcpy(tests_, tests.data(), tests_mem_size); + } + + if (types_size_) { + memcpy(types_, types.data(), types_mem_size); + } +} +const char* FeedbackIndex::name() const { + switch (kind) { + case FeedbackKind::Call: + return "Call"; + break; + case FeedbackKind::Test: + return "Test"; + break; + case FeedbackKind::Type: + return "Type"; + break; + default: + assert(false); + } +} } // namespace rir diff --git a/rir/src/runtime/TypeFeedback.h b/rir/src/runtime/TypeFeedback.h index ccf3821b5..e1bb8c0f3 100644 --- a/rir/src/runtime/TypeFeedback.h +++ b/rir/src/runtime/TypeFeedback.h @@ -2,19 +2,87 @@ #define RIR_RUNTIME_FEEDBACK #include "R/r.h" +#include "Rinternals.h" #include "common.h" +#include "interpreter/profiler.h" +#include "runtime/RirRuntimeObject.h" #include +#include #include +#include #include +#include +#include +#include +#include namespace rir { struct Code; +struct Function; +class TypeFeedback; + +enum class FeedbackKind : uint8_t { + Call, + Test, + Type, +}; + +class FeedbackIndex { + private: + static constexpr unsigned IdxBits = 24; + static constexpr unsigned Undefined = (1 << IdxBits) - 1; + + FeedbackIndex(FeedbackKind kind_, uint32_t idx_) : kind(kind_), idx(idx_) {} + friend struct std::hash; + + public: + FeedbackKind kind; + uint32_t idx : IdxBits; + + FeedbackIndex() : kind(FeedbackKind::Call), idx(Undefined) {} + + static FeedbackIndex call(uint32_t idx) { + return FeedbackIndex(FeedbackKind::Call, idx); + } + static FeedbackIndex test(uint32_t idx) { + return FeedbackIndex(FeedbackKind::Test, idx); + } + static FeedbackIndex type(uint32_t idx) { + return FeedbackIndex(FeedbackKind::Type, idx); + } + + bool isUndefined() const { return idx == Undefined; } + + const char* name() const; + + uint32_t asInteger() const { return *((uint32_t*)this); } + + bool operator==(const FeedbackIndex& other) const { + return idx == other.idx && kind == other.kind; + } + + friend std::ostream& operator<<(std::ostream& out, + const FeedbackIndex& index) { + out << index.name() << "#"; + if (index.isUndefined()) { + out << "unknown"; + } else { + out << index.idx; + } + return out; + } +}; + +static_assert(sizeof(FeedbackIndex) == sizeof(uint32_t), + "Size needs to fit inside in integer for the llvm transition"); #pragma pack(push) #pragma pack(1) struct ObservedCallees { + friend TypeFeedback; + static constexpr unsigned CounterBits = 29; static constexpr unsigned CounterOverflow = (1 << CounterBits) - 1; static constexpr unsigned TargetBits = 2; @@ -28,12 +96,16 @@ struct ObservedCallees { uint32_t numTargets : TargetBits; uint32_t taken : CounterBits; uint32_t invalid : 1; + std::array targets; - void record(Code* caller, SEXP callee, bool invalidateWhenFull = false); - SEXP getTarget(const Code* code, size_t pos) const; + SEXP getTarget(const Function* function, size_t pos) const; + void print(std::ostream& out, const Function* function) const; - std::array targets; + private: + bool record(Function* function, SEXP callee, + bool invalidateWhenFull = false); }; + static_assert(sizeof(ObservedCallees) == 4 * sizeof(uint32_t), "Size needs to fit inside a record_ bc immediate args"); @@ -44,34 +116,44 @@ inline bool fastVeceltOk(SEXP vec) { } struct ObservedTest { + friend TypeFeedback; + enum { None, OnlyTrue, OnlyFalse, Both }; uint32_t seen : 2; uint32_t unused : 30; ObservedTest() : seen(0), unused(0) {} - inline void record(SEXP e) { + void print(std::ostream& out) const; + + private: + inline bool record(const SEXP e) { + uint32_t old; + memcpy(&old, this, sizeof(ObservedTest)); + if (e == R_TrueValue) { if (seen == None) seen = OnlyTrue; else if (seen != OnlyTrue) seen = Both; - return; - } - if (e == R_FalseValue) { + } else if (e == R_FalseValue) { if (seen == None) seen = OnlyFalse; else if (seen != OnlyFalse) seen = Both; - return; + } else { + seen = Both; } - seen = Both; + + return memcmp(&old, this, sizeof(ObservedTest)); } }; static_assert(sizeof(ObservedTest) == sizeof(uint32_t), "Size needs to fit inside a record_ bc immediate args"); struct ObservedValues { + friend TypeFeedback; + friend RuntimeProfiler; enum StateBeforeLastForce { unknown, @@ -97,33 +179,12 @@ struct ObservedValues { void reset() { *this = ObservedValues(); } - void print(std::ostream& out) const { - if (numTypes) { - for (size_t i = 0; i < numTypes; ++i) { - out << Rf_type2char(seen[i]); - if (i != (unsigned)numTypes - 1) - out << ", "; - } - out << " (" << (object ? "o" : "") << (attribs ? "a" : "") - << (notFastVecelt ? "v" : "") << (!notScalar ? "s" : "") << ")"; - if (stateBeforeLastForce != - ObservedValues::StateBeforeLastForce::unknown) { - out << " | " - << ((stateBeforeLastForce == - ObservedValues::StateBeforeLastForce::value) - ? "value" - : (stateBeforeLastForce == - ObservedValues::StateBeforeLastForce:: - evaluatedPromise) - ? "evaluatedPromise" - : "promise"); - } - } else { - out << ""; - } - } + void print(std::ostream& out) const; - inline void record(SEXP e) { + private: + inline bool record(SEXP e) { + uint32_t old; + memcpy(&old, this, sizeof(ObservedValues)); // Set attribs flag for every object even if the SEXP does not // have attributes. The assumption used to be that e having no @@ -149,6 +210,8 @@ struct ObservedValues { if (i == numTypes) seen[numTypes++] = type; } + + return memcmp(&old, this, sizeof(ObservedValues)); } }; static_assert(sizeof(ObservedValues) == sizeof(uint32_t), @@ -156,26 +219,28 @@ static_assert(sizeof(ObservedValues) == sizeof(uint32_t), enum class Opcode : uint8_t; -struct FeedbackOrigin { - private: - uint32_t offset_ = 0; - Code* srcCode_ = nullptr; +class FeedbackOrigin { + FeedbackIndex index_; + Function* function_ = nullptr; public: FeedbackOrigin() {} - FeedbackOrigin(rir::Code* src, Opcode* pc); + FeedbackOrigin(rir::Function* fun, FeedbackIndex idx); - Opcode* pc() const { - if (offset_ == 0) - return nullptr; - return (Opcode*)((uintptr_t)srcCode() + offset_); - } - uint32_t offset() const { return offset_; } - Code* srcCode() const { return srcCode_; } - void srcCode(Code* src) { srcCode_ = src; } + bool hasSlot() const; + FeedbackIndex index() const { return index_; } + uint32_t idx() const { return index_.idx; } + Function* function() const { return function_; } + void function(Function* fun); bool operator==(const FeedbackOrigin& other) const { - return offset_ == other.offset_ && srcCode_ == other.srcCode_; + return index_ == other.index_ && function_ == other.function_; + } + + friend std::ostream& operator<<(std::ostream& out, + const FeedbackOrigin& origin) { + out << (void*)origin.function_ << "[" << origin.index_ << "]"; + return out; } }; @@ -196,9 +261,6 @@ struct DeoptReason { DeoptReason(const FeedbackOrigin& origin, DeoptReason::Reason reason); - Code* srcCode() const { return origin.srcCode(); } - Opcode* pc() const { return origin.pc(); } - bool operator==(const DeoptReason& other) const { return reason == other.reason && origin == other.origin; } @@ -228,13 +290,11 @@ struct DeoptReason { out << "Unknown"; break; } - out << "@" << (void*)reason.pc(); + out << "@" << reason.origin; return out; } - static DeoptReason unknown() { - return DeoptReason(FeedbackOrigin(0, 0), Unknown); - } + static DeoptReason unknown() { return DeoptReason({}, Unknown); } void record(SEXP val) const; @@ -246,15 +306,122 @@ struct DeoptReason { static_assert(sizeof(DeoptReason) == 4 * sizeof(uint32_t), "Size needs to fit inside a record_deopt_ bc immediate args"); +#define TYPEFEEDBACK_MAGIC (unsigned)0xfeedbac0 + +class TypeFeedback : public RirRuntimeObject { + private: + friend Function; + + size_t version_; + Function* owner_; + size_t callees_size_; + size_t tests_size_; + size_t types_size_; + ObservedCallees* callees_; + ObservedTest* tests_; + ObservedValues* types_; + // All the data are stored in this array: callees, tests and types in this + // order. The constructors sets the above pointers to point at the + // appropriate locations. + uint8_t slots_[]; + + explicit TypeFeedback(const std::vector& callees, + const std::vector& tests, + const std::vector& types); + + public: + static TypeFeedback* create(const std::vector& callees, + const std::vector& tests, + const std::vector& types); + + static TypeFeedback* empty(); + static TypeFeedback* deserialize(SEXP refTable, R_inpstream_t inp); + + class Builder { + unsigned ncallees_ = 0; + unsigned ntests_ = 0; + unsigned ntypes_ = 0; + + public: + uint32_t addCallee(); + uint32_t addTest(); + uint32_t addType(); + TypeFeedback* build(); + }; + + ObservedCallees& callees(uint32_t idx); + ObservedTest& test(uint32_t idx); + ObservedValues& types(uint32_t idx); + + void record_callee(uint32_t idx, Function* function, SEXP callee, + bool invalidateWhenFull = false) { + if (callees(idx).record(function, callee, invalidateWhenFull)) { + version_++; + } + } + + void record_test(uint32_t idx, const SEXP e) { + if (test(idx).record(e)) { + version_++; + } + } + + void record_type(uint32_t idx, const SEXP e) { + if (types(idx).record(e)) { + version_++; + } + } + + void record_type(uint32_t idx, std::function f) { + ObservedValues& slot = types(idx); + uint32_t o, n; + memcpy(&o, &slot, sizeof(ObservedValues)); + f(slot); + memcpy(&n, &slot, sizeof(ObservedValues)); + if (memcmp(&o, &n, sizeof(ObservedValues))) { + version_++; + } + } + + void print(std::ostream& out) const; + + void serialize(SEXP refTable, R_outpstream_t out) const; + + bool isValid(const FeedbackIndex& index) const; + + Function* owner() const { return owner_; } + + // Type feedback is versioned. Each time new feedback + // in any of the slot is recorded, its version increased. + // The new is important, if we record already known + // information, the version is left unchnaged. + size_t version() const { return version_; } + void version(size_t version) { version_ = version; } +}; + #pragma pack(pop) } // namespace rir namespace std { +template <> +struct hash { + std::size_t operator()(const rir::FeedbackIndex& v) const { + return hash_combine(hash_combine(0, v.kind), v.idx); + } +}; + +template <> +struct hash { + std::size_t operator()(const rir::FeedbackOrigin& v) const { + return hash_combine(hash_combine(0, v.index()), v.function()); + } +}; + template <> struct hash { std::size_t operator()(const rir::DeoptReason& v) const { - return hash_combine(hash_combine(0, v.pc()), v.reason); + return hash_combine(hash_combine(0, v.origin), v.reason); } }; } // namespace std diff --git a/rir/src/utils/FunctionWriter.h b/rir/src/utils/FunctionWriter.h index d32699888..d6fab2f94 100644 --- a/rir/src/utils/FunctionWriter.h +++ b/rir/src/utils/FunctionWriter.h @@ -5,6 +5,7 @@ #include "bc/BC_inc.h" #include "interpreter/instance.h" #include "runtime/Function.h" +#include "runtime/TypeFeedback.h" #include #include @@ -40,7 +41,7 @@ class FunctionWriter { } void finalize(Code* body, const FunctionSignature& signature, - const Context& context) { + const Context& context, TypeFeedback* feedback) { assert(function_ == nullptr && "Trying to finalize a second time"); size_t dataSize = defaultArgs.size() * sizeof(SEXP); @@ -48,8 +49,9 @@ class FunctionWriter { SEXP store = Rf_allocVector(EXTERNALSXP, functionSize); void* payload = INTEGER(store); - Function* fun = new (payload) Function(functionSize, body->container(), - defaultArgs, signature, context); + Function* fun = + new (payload) Function(functionSize, body->container(), defaultArgs, + signature, context, feedback); preserve(store); assert(fun->info.magic == FUNCTION_MAGIC); diff --git a/rir/tests/pir_check.R b/rir/tests/pir_check.R index f160ce659..a5023f49e 100644 --- a/rir/tests/pir_check.R +++ b/rir/tests/pir_check.R @@ -7,6 +7,17 @@ if (!jitOn) if (Sys.getenv("PIR_GLOBAL_SPECIALIZATION_LEVEL") != "") q() +deoptChaos <- as.numeric(Sys.getenv("PIR_DEOPT_CHAOS", unset=0)) + +# If this test runs in the with deopt chaos, weird things might +# happen (cf. #1258): a function might be compiled with the +# chaos on but run in a context with chaos off making the +# assert in deoptChaosTriggerImpl fail. +if (deoptChaos != 0) { + warning("skipping due to PIR_DEOPT_CHAOS=", deoptChaos, " set") + q() +} + # Sanity check for loop peeling, and testing that enabling/disabling works # These loop peeling tests may be a bit brittle. # Loop peeling should be enabled by default