[OpaqueValues] Follow-up to #61846.

include/swift/AST/Builtins.def

@@ -915,6 +915,24 @@ BUILTIN_MISC_OPERATION_WITH_SILGEN(EndAsyncLet, "endAsyncLet", "", Special)
 /// until the endAsyncLet.
 BUILTIN_MISC_OPERATION(EndAsyncLetLifetime, "endAsyncLetLifetime", "", Special)
 
+/// addressOfBorrowOpaque (__shared T) -> Builtin.RawPointer
+/// Returns a RawPointer pointing to a borrowed rvalue.  The returned pointer is
+/// only valid within the scope of the borrow.  
+///
+/// Differs from addressOfBorrow only in that it is not lowered until
+/// AddressLowering.
+BUILTIN_MISC_OPERATION(AddressOfBorrowOpaque, "addressOfBorrowOpaque", "", Special)
+
+/// unprotectedAddressOfBorrowOpaque (__shared T) -> Builtin.RawPointer
+/// Returns a RawPointer pointing to a borrowed rvalue. The returned pointer is only
+/// valid within the scope of the borrow.
+/// In contrast to `addressOfBorrowOpaque`, this builtin doesn't trigger an
+/// insertion of stack protectors.
+///
+/// Differs from unprotectedAddressOfBorrow only in that it is not lowered until
+/// AddressLowering.
+BUILTIN_MISC_OPERATION(UnprotectedAddressOfBorrowOpaque, "unprotectedAddressOfBorrowOpaque", "", Special)
+
 /// createAsyncTask(): (
 ///     Int,                            // task-creation flags
 ///     @escaping () async throws -> T  // function

include/swift/SIL/ApplySite.h

@@ -485,17 +485,21 @@ class ApplySite {
   ///
   /// NOTE: We pass std::next() for begin_apply. If one wishes to insert code
   /// /after/ the end_apply/abort_apply, please use instead
-  /// insertAfterFullEvaluation.
+  /// insertAfterApplication.
   void insertAfterInvocation(function_ref<void(SILBuilder &)> func) const;
 
   /// Pass a builder with insertion points that are guaranteed to be immediately
-  /// after this full apply site has completely finished executing.
+  /// after this apply site has been applied.
+  ///
+  /// For apply and try_apply, that means after the apply.  For partial_apply,
+  /// that means after the partial_apply.  For begin_apply, that means after its
+  /// end_apply and abort_apply instructions.
   ///
   /// This is just like insertAfterInvocation except that if the full apply site
   /// is a begin_apply, we pass the insertion points after the end_apply,
   /// abort_apply rather than an insertion point right after the
   /// begin_apply. For such functionality, please invoke insertAfterInvocation.
-  void insertAfterFullEvaluation(function_ref<void(SILBuilder &)> func) const;
+  void insertAfterApplication(function_ref<void(SILBuilder &)> func) const;
 
   /// Return whether the given apply is of a formally-throwing function
   /// which is statically known not to throw.

lib/AST/Builtins.cpp

@@ -2701,7 +2701,9 @@ ValueDecl *swift::getBuiltinValueDecl(ASTContext &Context, Identifier Id) {
     return getLegacyCondFailOperation(Context, Id);
 
   case BuiltinValueKind::AddressOfBorrow:
+  case BuiltinValueKind::AddressOfBorrowOpaque:
   case BuiltinValueKind::UnprotectedAddressOfBorrow:
+  case BuiltinValueKind::UnprotectedAddressOfBorrowOpaque:
     if (!Types.empty()) return nullptr;
     return getAddressOfBorrowOperation(Context, Id);
 

lib/SIL/IR/ApplySite.cpp

@@ -20,7 +20,7 @@ void ApplySite::insertAfterInvocation(function_ref<void(SILBuilder &)> func) con
   SILBuilderWithScope::insertAfter(getInstruction(), func);
 }
 
-void ApplySite::insertAfterFullEvaluation(
+void ApplySite::insertAfterApplication(
     function_ref<void(SILBuilder &)> func) const {
   switch (getKind()) {
   case ApplySiteKind::ApplyInst:

lib/SIL/IR/OperandOwnership.cpp

@@ -678,6 +678,8 @@ struct OperandOwnershipBuiltinClassifier
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, ErrorInMain)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, UnexpectedError)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, WillThrow)
+BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, AddressOfBorrowOpaque)
+BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, UnprotectedAddressOfBorrowOpaque)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, AShr)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, GenericAShr)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, Add)

lib/SIL/IR/ValueOwnership.cpp

@@ -382,6 +382,8 @@ struct ValueOwnershipKindBuiltinVisitor
 // This returns a value at +1 that is destroyed strictly /after/ the
 // UnsafeGuaranteedEnd. This provides the guarantee that we want.
 CONSTANT_OWNERSHIP_BUILTIN(Owned, COWBufferForReading)
+CONSTANT_OWNERSHIP_BUILTIN(None, AddressOfBorrowOpaque)
+CONSTANT_OWNERSHIP_BUILTIN(None, UnprotectedAddressOfBorrowOpaque)
 CONSTANT_OWNERSHIP_BUILTIN(None, AShr)
 CONSTANT_OWNERSHIP_BUILTIN(None, GenericAShr)
 CONSTANT_OWNERSHIP_BUILTIN(None, Add)

lib/SILGen/ResultPlan.cpp

@@ -991,7 +991,7 @@ ResultPlanPtr ResultPlanBuilder::buildForTuple(Initialization *init,
   // If the tuple is address-only, we'll get much better code if we
   // emit into a single buffer.
   auto &substTL = SGF.getTypeLowering(substType);
-  if (substTL.isAddressOnly()) {
+  if (substTL.isAddressOnly() && SGF.F.getConventions().useLoweredAddresses()) {
     // Create a temporary.
     auto temporary = SGF.emitTemporary(loc, substTL);
 

lib/SILGen/SILGenBuiltin.cpp

@@ -478,6 +478,17 @@ static ManagedValue emitBuiltinAddressOfBorrowBuiltins(SILGenFunction &SGF,
   // naturally emitted borrowed in memory.
   auto borrow = SGF.emitRValue(argument, SGFContext::AllowGuaranteedPlusZero)
      .getAsSingleValue(SGF, argument);
+  if (!SGF.F.getConventions().useLoweredAddresses()) {
+    auto &context = SGF.getASTContext();
+    auto identifier =
+        stackProtected
+            ? context.getIdentifier("addressOfBorrowOpaque")
+            : context.getIdentifier("unprotectedAddressOfBorrowOpaque");
+    auto builtin = SGF.B.createBuiltin(loc, identifier, rawPointerType,
+                                       substitutions, {borrow.getValue()});
+    return ManagedValue::forUnmanaged(builtin);
+  }
+
   if (!borrow.isPlusZero() || !borrow.getType().isAddress()) {
     SGF.SGM.diagnose(argument->getLoc(), diag::non_borrowed_indirect_addressof);
     return SGF.emitUndef(rawPointerType);

lib/SILOptimizer/Mandatory/AddressLowering.cpp

@@ -1777,7 +1777,7 @@ void CallArgRewriter::rewriteIndirectArgument(Operand *operand) {
   if (apply.getArgumentConvention(*operand).isOwnedConvention()) {
     argBuilder.createTrivialStoreOr(apply.getLoc(), argValue, allocInst,
                                     StoreOwnershipQualifier::Init);
-    apply.insertAfterFullEvaluation([&](SILBuilder &callBuilder) {
+    apply.insertAfterApplication([&](SILBuilder &callBuilder) {
       callBuilder.createDeallocStack(callLoc, allocInst);
     });
     operand->set(allocInst);
@@ -1786,7 +1786,7 @@ void CallArgRewriter::rewriteIndirectArgument(Operand *operand) {
     auto *store =
         argBuilder.emitStoreBorrowOperation(callLoc, borrow, allocInst);
     auto *storeBorrow = dyn_cast<StoreBorrowInst>(store);
-    apply.insertAfterFullEvaluation([&](SILBuilder &callBuilder) {
+    apply.insertAfterApplication([&](SILBuilder &callBuilder) {
       if (storeBorrow) {
         callBuilder.emitEndBorrowOperation(callLoc, storeBorrow);
       }
@@ -2071,7 +2071,7 @@ SILValue ApplyRewriter::materializeIndirectResultAddress(SILValue oldResult,
 
   // Instead of using resultBuilder, insert dealloc immediately after the call
   // for stack discipline across loadable indirect results.
-  apply.insertAfterFullEvaluation([&](SILBuilder &callBuilder) {
+  apply.insertAfterApplication([&](SILBuilder &callBuilder) {
     callBuilder.createDeallocStack(callLoc, allocInst);
   });
 
@@ -2617,13 +2617,29 @@ class UseRewriter : SILInstructionVisitor<UseRewriter> {
     vmi->setOperand(opAddr);
   }
 
+  void visitAddressOfBorrowBuiltinInst(BuiltinInst *bi, bool stackProtected) {
+    SILValue value = bi->getOperand(0);
+    SILValue addr = pass.valueStorageMap.getStorage(value).storageAddress;
+    auto &astCtx = pass.getModule()->getASTContext();
+    SILType rawPointerType = SILType::getRawPointerType(astCtx);
+    SILValue result = builder.createAddressToPointer(
+        bi->getLoc(), addr, rawPointerType, stackProtected);
+    bi->replaceAllUsesWith(result);
+  }
+
   void visitBuiltinInst(BuiltinInst *bi) {
     switch (bi->getBuiltinKind().getValueOr(BuiltinValueKind::None)) {
     case BuiltinValueKind::Copy: {
       SILValue opAddr = addrMat.materializeAddress(use->get());
       bi->setOperand(0, opAddr);
       break;
     }
+    case BuiltinValueKind::AddressOfBorrowOpaque:
+      visitAddressOfBorrowBuiltinInst(bi, /*stackProtected=*/true);
+      break;
+    case BuiltinValueKind::UnprotectedAddressOfBorrowOpaque:
+      visitAddressOfBorrowBuiltinInst(bi, /*stackProtected=*/false);
+      break;
     default:
       bi->dump();
       llvm::report_fatal_error("^^^ Unimplemented builtin opaque value use.");

lib/SILOptimizer/Mandatory/ClosureLifetimeFixup.cpp

@@ -431,7 +431,7 @@ static void insertAfterClosureUser(SILInstruction *closureUser,
   }
   FullApplySite fas = FullApplySite::isa(closureUser);
   assert(fas);
-  fas.insertAfterFullEvaluation(insertAtNonUnreachable);
+  fas.insertAfterApplication(insertAtNonUnreachable);
 }
 
 static SILValue skipConvert(SILValue v) {

lib/SILOptimizer/Transforms/AccessEnforcementReleaseSinking.cpp

@@ -193,6 +193,8 @@ static bool isBarrier(SILInstruction *inst) {
     case BuiltinValueKind::ResumeThrowingContinuationThrowing:
     case BuiltinValueKind::AutoDiffProjectTopLevelSubcontext:
     case BuiltinValueKind::AutoDiffAllocateSubcontext:
+    case BuiltinValueKind::AddressOfBorrowOpaque:
+    case BuiltinValueKind::UnprotectedAddressOfBorrowOpaque:
       return true;
     }
   }

lib/SILOptimizer/Utils/Generics.cpp

@@ -2337,7 +2337,7 @@ replaceWithSpecializedCallee(ApplySite applySite, SILValue callee,
     SILBasicBlock *resultBlock = tai->getNormalBB();
     assert(resultBlock->getSinglePredecessorBlock() == tai->getParent());
     // First insert the cleanups for our arguments int he appropriate spot.
-    FullApplySite(tai).insertAfterFullEvaluation(
+    FullApplySite(tai).insertAfterApplication(
         [&](SILBuilder &argBuilder) {
           cleanupCallArguments(argBuilder, loc, arguments,
                                argsNeedingEndBorrow);
@@ -2363,7 +2363,7 @@ replaceWithSpecializedCallee(ApplySite applySite, SILValue callee,
   }
   case ApplySiteKind::ApplyInst: {
     auto *ai = cast<ApplyInst>(applySite);
-    FullApplySite(ai).insertAfterFullEvaluation(
+    FullApplySite(ai).insertAfterApplication(
         [&](SILBuilder &argBuilder) {
           cleanupCallArguments(argBuilder, loc, arguments,
                                argsNeedingEndBorrow);
@@ -2400,7 +2400,7 @@ replaceWithSpecializedCallee(ApplySite applySite, SILValue callee,
   case ApplySiteKind::BeginApplyInst: {
     auto *bai = cast<BeginApplyInst>(applySite);
     assert(!resultOut);
-    FullApplySite(bai).insertAfterFullEvaluation(
+    FullApplySite(bai).insertAfterApplication(
         [&](SILBuilder &argBuilder) {
           cleanupCallArguments(argBuilder, loc, arguments,
                                argsNeedingEndBorrow);
@@ -2582,7 +2582,7 @@ SILFunction *ReabstractionThunkGenerator::createThunk() {
 
   // Now that we have finished constructing our CFG (note the return above),
   // insert any compensating end borrows that we need.
-  ApplySite.insertAfterFullEvaluation([&](SILBuilder &argBuilder) {
+  ApplySite.insertAfterApplication([&](SILBuilder &argBuilder) {
     cleanupCallArguments(argBuilder, Loc, Arguments, ArgsThatNeedEndBorrow);
   });
 

lib/SILOptimizer/Utils/PartialApplyCombiner.cpp

@@ -179,7 +179,7 @@ void PartialApplyCombiner::processSingleApply(FullApplySite paiAI) {
         auto *ASI = builder.createAllocStack(pai->getLoc(), arg->getType());
         builder.createCopyAddr(pai->getLoc(), arg, ASI, IsTake_t::IsNotTake,
                                IsInitialization_t::IsInitialization);
-        paiAI.insertAfterFullEvaluation([&](SILBuilder &builder) {
+        paiAI.insertAfterApplication([&](SILBuilder &builder) {
           builder.createDeallocStack(destroyloc, ASI);
         });
         arg = ASI;
@@ -207,7 +207,7 @@ void PartialApplyCombiner::processSingleApply(FullApplySite paiAI) {
   // We also need to destroy the partial_apply instruction itself because it is
   // consumed by the apply_instruction.
   if (!pai->hasCalleeGuaranteedContext()) {
-    paiAI.insertAfterFullEvaluation([&](SILBuilder &builder) {
+    paiAI.insertAfterApplication([&](SILBuilder &builder) {
       builder.emitDestroyValueOperation(destroyloc, pai);
     });
   }

test/SILGen/opaque_values_silgen.swift

@@ -499,3 +499,59 @@ func duplicate_with_int4<Value>(condition: Bool, value: Value) -> (Int, Int, Int
   }
 }
 
+// Verify tuple rebuilding.
+// CHECK-LABEL: sil private [ossa] @$s20opaque_values_silgen10duplicate15valuex_xtx_tlFx_xtyXEfU_ : {{.*}} {
+// CHECK:         [[RETVAL:%[^,]+]] = tuple ({{%[^,]+}} : $Value, {{%[^,]+}} : $Value)
+// CHECK:         return [[RETVAL]] : $(Value, Value)
+// CHECK-LABEL: } // end sil function '$s20opaque_values_silgen10duplicate15valuex_xtx_tlFx_xtyXEfU_'
+@_silgen_name("duplicate1")
+func duplicate1<Value>(value: Value) -> (Value, Value) {
+  doit { 
+      (value, value)
+  }
+}
+// CHECK-LABEL: sil private [ossa] @$s20opaque_values_silgen10duplicate25valuex3one_x3twotx_tlFxAD_xAEtyXEfU_ : {{.*}} {
+// CHECK:         [[RETVAL:%[^,]+]] = tuple $(one: Value, two: Value) ({{%[^,]+}}, {{%[^,]+}})   
+// CHECK:         return [[RETVAL]] : $(one: Value, two: Value)           
+// CHECK-LABEL: } // end sil function '$s20opaque_values_silgen10duplicate25valuex3one_x3twotx_tlFxAD_xAEtyXEfU_'
+@_silgen_name("duplicate2")
+func duplicate2<Value>(value: Value) -> (one: Value, two: Value) {
+  doit { 
+      (one: value, two: value)
+  }
+}
+// CHECK-LABEL: sil private [ossa] @$s20opaque_values_silgen19duplicate_with_int15valuex_xSitx_tlFx_xSityXEfU_ : {{.*}} {
+// CHECK:         [[RETVAL:%[^,]+]] = tuple ({{%[^,]+}} : $Value, {{%[^,]+}} : $Value, {{%[^,]+}} : $Int)
+// CHECK:         return [[RETVAL]]
+// CHECK-LABEL: } // end sil function '$s20opaque_values_silgen19duplicate_with_int15valuex_xSitx_tlFx_xSityXEfU_'
+@_silgen_name("duplicate_with_int1")
+func duplicate_with_int1<Value>(value: Value) -> (Value, Value, Int) {
+  doit {
+    (value, value, 42)
+  }
+}
+
+// CHECK-LABEL: sil private [ossa] @$s20opaque_values_silgen19duplicate_with_int25valuex_xt_Sitx_tlFx_xt_SityXEfU_ : {{.*}} {
+// CHECK:         [[INNER:%[^,]+]] = tuple ({{%[^,]+}} : $Value, {{%[^,]+}} : $Value)           
+// CHECK:         [[RETVAL:%[^,]+]] = tuple ([[INNER]] : $(Value, Value), {{%[^,]+}} : $Int)    
+// CHECK:         return [[RETVAL]]
+// CHECK-LABEL: } // end sil function '$s20opaque_values_silgen19duplicate_with_int25valuex_xt_Sitx_tlFx_xt_SityXEfU_'
+@_silgen_name("duplicate_with_int2")
+func duplicate_with_int2<Value>(value: Value) -> ((Value, Value), Int) {
+  doit {
+    ((value, value), 42)
+  }
+}
+// CHECK-LABEL: sil private [ossa] @$s20opaque_values_silgen19duplicate_with_int35valueSi_x_x_x_SitxttSitx_tlFSi_x_x_x_SitxttSityXEfU_ : {{.*}} {
+// CHECK:         [[INNERMOST:%[^,]+]] = tuple ({{%[^,]+}} : $Value, {{%[^,]+}} : $Int)           
+// CHECK:         [[INNERMIDDLE:%[^,]+]] = tuple ({{%[^,]+}} : $Value, [[INNERMOST]] : $(Value, Int), {{%[^,]+}} : $Value) 
+// CHECK:         [[INNERLEAST:%[^,]+]] = tuple ({{%[^,]+}} : $Value, [[INNERMIDDLE]] : $(Value, (Value, Int), Value)) 
+// CHECK:         [[RETVAL:%[^,]+]] = tuple ({{%[^,]+}} : $Int, [[INNERLEAST]] : $(Value, (Value, (Value, Int), Value)), {{%[^,]+}} : $Int) 
+// CHECK:         return [[RETVAL]] : $(Int, (Value, (Value, (Value, Int), Value)), Int) 
+// CHECK-LABEL: } // end sil function '$s20opaque_values_silgen19duplicate_with_int35valueSi_x_x_x_SitxttSitx_tlFSi_x_x_x_SitxttSityXEfU_'
+@_silgen_name("duplicate_with_int3")
+func duplicate_with_int3<Value>(value: Value) -> (Int, (Value, (Value, (Value, Int), Value)), Int) {
+  doit {
+    (42, (value, (value, (value, 43), value)), 44)
+  }
+}

test/SILOptimizer/opaque_values_Onone.swift

@@ -163,3 +163,20 @@ func duplicate_with_int3<Value>(value: Value) -> (Int, (Value, (Value, (Value, I
     (42, (value, (value, (value, 43), value)), 44)
   }
 }
+
+// CHECK-LABEL: sil hidden @get_a_generic_tuple : {{.*}} {
+// CHECK:         [[TUPLE_ADDR:%[^,]+]] = alloc_stack [lexical] $(This, This)
+// CHECK:         [[GIVE_A_GENERIC_TUPLE:%[^,]+]] = function_ref @give_a_generic_tuple
+// CHECK:         [[TUPLE_0_ADDR:%[^,]+]] = tuple_element_addr [[TUPLE_ADDR]] : $*(This, This), 0
+// CHECK:         [[TUPLE_1_ADDR:%[^,]+]] = tuple_element_addr [[TUPLE_ADDR]] : $*(This, This), 1
+// CHECK:         apply [[GIVE_A_GENERIC_TUPLE]]<This>([[TUPLE_0_ADDR]], [[TUPLE_1_ADDR]], {{%[^,]+}})
+// CHECK:         destroy_addr [[TUPLE_ADDR]]
+// CHECK:         dealloc_stack [[TUPLE_ADDR]]
+// CHECK-LABEL: } // end sil function 'get_a_generic_tuple'
+struct This {}
+@_silgen_name("give_a_generic_tuple")
+func give_a_generic_tuple<This>(of ty: This.Type) -> (This, This)
+@_silgen_name("get_a_generic_tuple")
+func get_a_generic_tuple<This>(ty: This.Type) {
+  let p = give_a_generic_tuple(of: ty)
+}

test/SILOptimizer/opaque_values_Onone_stdlib.swift

@@ -31,4 +31,22 @@ public func _copy<T>(_ value: T) -> T {
   #endif
 }
 
+// CHECK-LABEL: sil hidden @getRawPointer : {{.*}} {
+// CHECK:       {{bb[0-9]+}}([[ADDR:%[^,]+]] : $*T):
+// CHECK:         [[PTR:%[^,]+]] = address_to_pointer [stack_protection] [[ADDR]]
+// CHECK:         return [[PTR]]
+// CHECK-LABEL: } // end sil function 'getRawPointer'
+@_silgen_name("getRawPointer")
+func getRawPointer<T>(to value: T) -> Builtin.RawPointer {
+  return Builtin.addressOfBorrow(value)
+}
 
+// CHECK-LABEL: sil hidden @getUnprotectedRawPointer : {{.*}} {
+// CHECK:       {{bb[0-9]+}}([[ADDR:%[^,]+]] : $*T):
+// CHECK:         [[PTR:%[^,]+]] = address_to_pointer [[ADDR]]
+// CHECK:         return [[PTR]]
+// CHECK-LABEL: } // end sil function 'getUnprotectedRawPointer'
+@_silgen_name("getUnprotectedRawPointer")
+func getUnprotectedRawPointer<T>(to value: T) -> Builtin.RawPointer {
+  return Builtin.unprotectedAddressOfBorrow(value)
+}