[AddressLowering] Set the right yield operand.

include/swift/SIL/ApplySite.h

@@ -485,17 +485,21 @@ class ApplySite {
   ///
   /// NOTE: We pass std::next() for begin_apply. If one wishes to insert code
   /// /after/ the end_apply/abort_apply, please use instead
-  /// insertAfterFullEvaluation.
+  /// insertAfterApplication.
   void insertAfterInvocation(function_ref<void(SILBuilder &)> func) const;
 
   /// Pass a builder with insertion points that are guaranteed to be immediately
-  /// after this full apply site has completely finished executing.
+  /// after this apply site has been applied.
+  ///
+  /// For apply and try_apply, that means after the apply.  For partial_apply,
+  /// that means after the partial_apply.  For begin_apply, that means after its
+  /// end_apply and abort_apply instructions.
   ///
   /// This is just like insertAfterInvocation except that if the full apply site
   /// is a begin_apply, we pass the insertion points after the end_apply,
   /// abort_apply rather than an insertion point right after the
   /// begin_apply. For such functionality, please invoke insertAfterInvocation.
-  void insertAfterFullEvaluation(function_ref<void(SILBuilder &)> func) const;
+  void insertAfterApplication(function_ref<void(SILBuilder &)> func) const;
 
   /// Return whether the given apply is of a formally-throwing function
   /// which is statically known not to throw.

lib/SIL/IR/ApplySite.cpp

@@ -20,7 +20,7 @@ void ApplySite::insertAfterInvocation(function_ref<void(SILBuilder &)> func) con
   SILBuilderWithScope::insertAfter(getInstruction(), func);
 }
 
-void ApplySite::insertAfterFullEvaluation(
+void ApplySite::insertAfterApplication(
     function_ref<void(SILBuilder &)> func) const {
   switch (getKind()) {
   case ApplySiteKind::ApplyInst:

lib/SILGen/SILGenEpilog.cpp

@@ -132,7 +132,7 @@ static SILValue buildReturnValue(SILGenFunction &SGF, SILLocation loc,
 
   auto fnConv = SGF.F.getConventions();
   if (!fnConv.useLoweredAddresses()) {
-    // In opaque-values code, nested @out enums are not flattened.  Reconstruct
+    // In opaque-values code, nested @out tuples are not flattened.  Reconstruct
     // nested tuples.
     auto resultType = SGF.F.getLoweredType(SGF.F.mapTypeIntoContext(
         fnConv.getSILResultType(SGF.getTypeExpansionContext())));

lib/SILOptimizer/Mandatory/AddressLowering.cpp

@@ -1778,7 +1778,7 @@ void CallArgRewriter::rewriteIndirectArgument(Operand *operand) {
   if (apply.getArgumentConvention(*operand).isOwnedConvention()) {
     argBuilder.createTrivialStoreOr(apply.getLoc(), argValue, allocInst,
                                     StoreOwnershipQualifier::Init);
-    apply.insertAfterFullEvaluation([&](SILBuilder &callBuilder) {
+    apply.insertAfterApplication([&](SILBuilder &callBuilder) {
       callBuilder.createDeallocStack(callLoc, allocInst);
     });
     operand->set(allocInst);
@@ -1787,7 +1787,7 @@ void CallArgRewriter::rewriteIndirectArgument(Operand *operand) {
     auto *store =
         argBuilder.emitStoreBorrowOperation(callLoc, borrow, allocInst);
     auto *storeBorrow = dyn_cast<StoreBorrowInst>(store);
-    apply.insertAfterFullEvaluation([&](SILBuilder &callBuilder) {
+    apply.insertAfterApplication([&](SILBuilder &callBuilder) {
       if (storeBorrow) {
         callBuilder.emitEndBorrowOperation(callLoc, storeBorrow);
       }
@@ -2072,7 +2072,7 @@ SILValue ApplyRewriter::materializeIndirectResultAddress(SILValue oldResult,
 
   // Instead of using resultBuilder, insert dealloc immediately after the call
   // for stack discipline across loadable indirect results.
-  apply.insertAfterFullEvaluation([&](SILBuilder &callBuilder) {
+  apply.insertAfterApplication([&](SILBuilder &callBuilder) {
     callBuilder.createDeallocStack(callLoc, allocInst);
   });
 
@@ -2610,14 +2610,19 @@ class UseRewriter : SILInstructionVisitor<UseRewriter> {
 
   void visitYieldInst(YieldInst *yield) {
     SILValue addr = addrMat.materializeAddress(use->get());
-    yield->setOperand(0, addr);
+    yield->setOperand(use->getOperandNumber(), addr);
   }
 
   void visitValueMetatypeInst(ValueMetatypeInst *vmi) {
     SILValue opAddr = addrMat.materializeAddress(use->get());
     vmi->setOperand(opAddr);
   }
 
+  void visitExistentialMetatypeInst(ExistentialMetatypeInst *emi) {
+    SILValue opAddr = addrMat.materializeAddress(use->get());
+    emi->setOperand(opAddr);
+  }
+
   void visitAddressOfBorrowBuiltinInst(BuiltinInst *bi, bool stackProtected) {
     SILValue value = bi->getOperand(0);
     SILValue addr = pass.valueStorageMap.getStorage(value).storageAddress;
@@ -2746,6 +2751,21 @@ class UseRewriter : SILInstructionVisitor<UseRewriter> {
 
   void emitExtract(SingleValueInstruction *extractInst);
 
+  void visitSelectEnumInst(SelectEnumInst *sei) {
+    SmallVector<std::pair<EnumElementDecl *, SILValue>> caseValues;
+    for (unsigned index = 0, count = sei->getNumCases(); index < count;
+         ++index) {
+      caseValues.push_back(sei->getCase(index));
+    }
+
+    SILValue opAddr = addrMat.materializeAddress(use->get());
+    SILValue addr =
+        builder.createSelectEnumAddr(sei->getLoc(), opAddr, sei->getType(),
+                                     sei->getDefaultResult(), caseValues);
+    sei->replaceAllUsesWith(addr);
+    pass.deleter.forceDelete(sei);
+  }
+
   // Extract from an opaque struct.
   void visitStructExtractInst(StructExtractInst *extractInst);
 
@@ -2776,7 +2796,26 @@ class UseRewriter : SILInstructionVisitor<UseRewriter> {
         uncheckedCastInst->getLoc(), srcAddr,
         uncheckedCastInst->getType().getAddressType());
 
-    markRewritten(uncheckedCastInst, destAddr);
+    if (uncheckedCastInst->getType().isAddressOnly(*pass.function)) {
+      markRewritten(uncheckedCastInst, destAddr);
+      return;
+    }
+
+    // For loadable cast destination type, replace copies with load copies.
+    if (Operand *use = uncheckedCastInst->getSingleUse()) {
+      if (auto *cvi = dyn_cast<CopyValueInst>(use->getUser())) {
+        auto *load = builder.createLoad(cvi->getLoc(), destAddr,
+                                        LoadOwnershipQualifier::Copy);
+        cvi->replaceAllUsesWith(load);
+        pass.deleter.forceDelete(cvi);
+        return;
+      }
+    }
+    SILValue load =
+        builder.emitLoadBorrowOperation(uncheckedCastInst->getLoc(), destAddr);
+    uncheckedCastInst->replaceAllUsesWith(load);
+    pass.deleter.forceDelete(uncheckedCastInst);
+    emitEndBorrows(load);
   }
 
   void visitUnconditionalCheckedCastInst(

lib/SILOptimizer/Mandatory/ClosureLifetimeFixup.cpp

@@ -431,7 +431,7 @@ static void insertAfterClosureUser(SILInstruction *closureUser,
   }
   FullApplySite fas = FullApplySite::isa(closureUser);
   assert(fas);
-  fas.insertAfterFullEvaluation(insertAtNonUnreachable);
+  fas.insertAfterApplication(insertAtNonUnreachable);
 }
 
 static SILValue skipConvert(SILValue v) {

lib/SILOptimizer/Utils/Generics.cpp

@@ -2337,7 +2337,7 @@ replaceWithSpecializedCallee(ApplySite applySite, SILValue callee,
     SILBasicBlock *resultBlock = tai->getNormalBB();
     assert(resultBlock->getSinglePredecessorBlock() == tai->getParent());
     // First insert the cleanups for our arguments int he appropriate spot.
-    FullApplySite(tai).insertAfterFullEvaluation(
+    FullApplySite(tai).insertAfterApplication(
         [&](SILBuilder &argBuilder) {
           cleanupCallArguments(argBuilder, loc, arguments,
                                argsNeedingEndBorrow);
@@ -2363,7 +2363,7 @@ replaceWithSpecializedCallee(ApplySite applySite, SILValue callee,
   }
   case ApplySiteKind::ApplyInst: {
     auto *ai = cast<ApplyInst>(applySite);
-    FullApplySite(ai).insertAfterFullEvaluation(
+    FullApplySite(ai).insertAfterApplication(
         [&](SILBuilder &argBuilder) {
           cleanupCallArguments(argBuilder, loc, arguments,
                                argsNeedingEndBorrow);
@@ -2400,7 +2400,7 @@ replaceWithSpecializedCallee(ApplySite applySite, SILValue callee,
   case ApplySiteKind::BeginApplyInst: {
     auto *bai = cast<BeginApplyInst>(applySite);
     assert(!resultOut);
-    FullApplySite(bai).insertAfterFullEvaluation(
+    FullApplySite(bai).insertAfterApplication(
         [&](SILBuilder &argBuilder) {
           cleanupCallArguments(argBuilder, loc, arguments,
                                argsNeedingEndBorrow);
@@ -2582,7 +2582,7 @@ SILFunction *ReabstractionThunkGenerator::createThunk() {
 
   // Now that we have finished constructing our CFG (note the return above),
   // insert any compensating end borrows that we need.
-  ApplySite.insertAfterFullEvaluation([&](SILBuilder &argBuilder) {
+  ApplySite.insertAfterApplication([&](SILBuilder &argBuilder) {
     cleanupCallArguments(argBuilder, Loc, Arguments, ArgsThatNeedEndBorrow);
   });
 

lib/SILOptimizer/Utils/PartialApplyCombiner.cpp

@@ -179,7 +179,7 @@ void PartialApplyCombiner::processSingleApply(FullApplySite paiAI) {
         auto *ASI = builder.createAllocStack(pai->getLoc(), arg->getType());
         builder.createCopyAddr(pai->getLoc(), arg, ASI, IsTake_t::IsNotTake,
                                IsInitialization_t::IsInitialization);
-        paiAI.insertAfterFullEvaluation([&](SILBuilder &builder) {
+        paiAI.insertAfterApplication([&](SILBuilder &builder) {
           builder.createDeallocStack(destroyloc, ASI);
         });
         arg = ASI;
@@ -207,7 +207,7 @@ void PartialApplyCombiner::processSingleApply(FullApplySite paiAI) {
   // We also need to destroy the partial_apply instruction itself because it is
   // consumed by the apply_instruction.
   if (!pai->hasCalleeGuaranteedContext()) {
-    paiAI.insertAfterFullEvaluation([&](SILBuilder &builder) {
+    paiAI.insertAfterApplication([&](SILBuilder &builder) {
       builder.emitDestroyValueOperation(destroyloc, pai);
     });
   }

test/SILGen/opaque_values_silgen.swift

@@ -499,3 +499,59 @@ func duplicate_with_int4<Value>(condition: Bool, value: Value) -> (Int, Int, Int
   }
 }
 
+// Verify tuple rebuilding.
+// CHECK-LABEL: sil private [ossa] @$s20opaque_values_silgen10duplicate15valuex_xtx_tlFx_xtyXEfU_ : {{.*}} {
+// CHECK:         [[RETVAL:%[^,]+]] = tuple ({{%[^,]+}} : $Value, {{%[^,]+}} : $Value)
+// CHECK:         return [[RETVAL]] : $(Value, Value)
+// CHECK-LABEL: } // end sil function '$s20opaque_values_silgen10duplicate15valuex_xtx_tlFx_xtyXEfU_'
+@_silgen_name("duplicate1")
+func duplicate1<Value>(value: Value) -> (Value, Value) {
+  doit { 
+      (value, value)
+  }
+}
+// CHECK-LABEL: sil private [ossa] @$s20opaque_values_silgen10duplicate25valuex3one_x3twotx_tlFxAD_xAEtyXEfU_ : {{.*}} {
+// CHECK:         [[RETVAL:%[^,]+]] = tuple $(one: Value, two: Value) ({{%[^,]+}}, {{%[^,]+}})   
+// CHECK:         return [[RETVAL]] : $(one: Value, two: Value)           
+// CHECK-LABEL: } // end sil function '$s20opaque_values_silgen10duplicate25valuex3one_x3twotx_tlFxAD_xAEtyXEfU_'
+@_silgen_name("duplicate2")
+func duplicate2<Value>(value: Value) -> (one: Value, two: Value) {
+  doit { 
+      (one: value, two: value)
+  }
+}
+// CHECK-LABEL: sil private [ossa] @$s20opaque_values_silgen19duplicate_with_int15valuex_xSitx_tlFx_xSityXEfU_ : {{.*}} {
+// CHECK:         [[RETVAL:%[^,]+]] = tuple ({{%[^,]+}} : $Value, {{%[^,]+}} : $Value, {{%[^,]+}} : $Int)
+// CHECK:         return [[RETVAL]]
+// CHECK-LABEL: } // end sil function '$s20opaque_values_silgen19duplicate_with_int15valuex_xSitx_tlFx_xSityXEfU_'
+@_silgen_name("duplicate_with_int1")
+func duplicate_with_int1<Value>(value: Value) -> (Value, Value, Int) {
+  doit {
+    (value, value, 42)
+  }
+}
+
+// CHECK-LABEL: sil private [ossa] @$s20opaque_values_silgen19duplicate_with_int25valuex_xt_Sitx_tlFx_xt_SityXEfU_ : {{.*}} {
+// CHECK:         [[INNER:%[^,]+]] = tuple ({{%[^,]+}} : $Value, {{%[^,]+}} : $Value)           
+// CHECK:         [[RETVAL:%[^,]+]] = tuple ([[INNER]] : $(Value, Value), {{%[^,]+}} : $Int)    
+// CHECK:         return [[RETVAL]]
+// CHECK-LABEL: } // end sil function '$s20opaque_values_silgen19duplicate_with_int25valuex_xt_Sitx_tlFx_xt_SityXEfU_'
+@_silgen_name("duplicate_with_int2")
+func duplicate_with_int2<Value>(value: Value) -> ((Value, Value), Int) {
+  doit {
+    ((value, value), 42)
+  }
+}
+// CHECK-LABEL: sil private [ossa] @$s20opaque_values_silgen19duplicate_with_int35valueSi_x_x_x_SitxttSitx_tlFSi_x_x_x_SitxttSityXEfU_ : {{.*}} {
+// CHECK:         [[INNERMOST:%[^,]+]] = tuple ({{%[^,]+}} : $Value, {{%[^,]+}} : $Int)           
+// CHECK:         [[INNERMIDDLE:%[^,]+]] = tuple ({{%[^,]+}} : $Value, [[INNERMOST]] : $(Value, Int), {{%[^,]+}} : $Value) 
+// CHECK:         [[INNERLEAST:%[^,]+]] = tuple ({{%[^,]+}} : $Value, [[INNERMIDDLE]] : $(Value, (Value, Int), Value)) 
+// CHECK:         [[RETVAL:%[^,]+]] = tuple ({{%[^,]+}} : $Int, [[INNERLEAST]] : $(Value, (Value, (Value, Int), Value)), {{%[^,]+}} : $Int) 
+// CHECK:         return [[RETVAL]] : $(Int, (Value, (Value, (Value, Int), Value)), Int) 
+// CHECK-LABEL: } // end sil function '$s20opaque_values_silgen19duplicate_with_int35valueSi_x_x_x_SitxttSitx_tlFSi_x_x_x_SitxttSityXEfU_'
+@_silgen_name("duplicate_with_int3")
+func duplicate_with_int3<Value>(value: Value) -> (Int, (Value, (Value, (Value, Int), Value)), Int) {
+  doit {
+    (42, (value, (value, (value, 43), value)), 44)
+  }
+}

test/SILOptimizer/address_lowering.sil

@@ -1340,6 +1340,23 @@ exit_error(%47 : @owned $any Error):
   throw %47 : $any Error
 }
 
+// CHECK-LABEL: sil [ossa] @selecteroo : $@convention(thin) <T> () -> () {
+// CHECK:         [[ADDR:%[^,]+]] = alloc_stack $Optional<T>
+// CHECK:         apply undef<T>([[ADDR]]) : $@convention(thin) <τ_0_0> () -> @out Optional<τ_0_0>
+// CHECK:         select_enum_addr [[ADDR]] : $*Optional<T>, case #Optional.some!enumelt: {{%[^,]+}}, default {{%[^,]+}} : $Builtin.Int1
+// CHECK:         dealloc_stack [[ADDR]] : $*Optional<T>
+// CHECK-LABEL: } // end sil function 'selecteroo'
+sil [ossa] @selecteroo : $@convention(thin) <T> () -> () {
+bb0:
+  %11 = apply undef<T>() : $@convention(thin) <τ_0_0> () -> @out Optional<τ_0_0>
+  %12 = integer_literal $Builtin.Int1, -1
+  %13 = integer_literal $Builtin.Int1, 0
+  %14 = select_enum %11 : $Optional<T>, case #Optional.some!enumelt: %12, default %13 : $Builtin.Int1
+  destroy_value %11 : $Optional<T>
+  %retval = tuple ()
+  return %retval : $()
+} // end sil function '$ss17FixedWidthIntegerPsEyxSgSScfC'
+
 sil hidden [ossa] @testBeginApplyDeadYield : $@convention(thin) <T> (@guaranteed TestGeneric<T>) -> () {
 bb0(%0 : @guaranteed $TestGeneric<T>):
   %2 = class_method %0 : $TestGeneric<T>, #TestGeneric.borrowedGeneric!read : <T> (TestGeneric<T>) -> () -> (), $@yield_once @convention(method) <τ_0_0> (@guaranteed TestGeneric<τ_0_0>) -> @yields @in_guaranteed τ_0_0
@@ -1554,6 +1571,53 @@ bb0(%0 : @guaranteed $T, %1 : $@thick U.Type):
   return %6 : $U
 }
 
+// There's only one use and it's a copy, just rewrite it as a load [copy].
+// CHECK-LABEL: sil hidden [ossa] @test_unchecked_bitwise_cast_to_loadable :
+// CHECK:       {{bb[0-9]+}}([[ADDR_IN:%[^,]+]] : $*T):
+// CHECK:         [[STACK:%[^,]+]] = alloc_stack $T
+// CHECK:         copy_addr [[ADDR_IN]] to [init] [[STACK]]
+// CHECK:         [[STACK_AS_CLASS:%[^,]+]] = unchecked_addr_cast [[STACK]] : $*T to $*Klass
+// CHECK:         [[KLASS:%[^,]+]] = load [copy] [[STACK_AS_CLASS]]
+// CHECK:         destroy_addr [[STACK]]
+// CHECK:         dealloc_stack [[STACK]]
+// CHECK:         return [[KLASS]]
+// CHECK-LABEL: } // end sil function 'test_unchecked_bitwise_cast_to_loadable'
+sil hidden [ossa] @test_unchecked_bitwise_cast_to_loadable : $@convention(thin) <T> (@in_guaranteed T) -> @owned Klass {
+bb0(%instance : @guaranteed $T):
+  %copy = copy_value %instance : $T
+  %unowned_klass = unchecked_bitwise_cast %copy : $T to $Klass
+  %klass = copy_value %unowned_klass : $Klass
+  destroy_value %copy : $T
+  return %klass : $Klass
+}
+
+// There is more than one use of the unchecked_addr_cast.  The value should be
+// load_borrow'd and uses of the original should be uses of that load.
+// CHECK-LABEL: sil hidden [ossa] @test_unchecked_bitwise_cast_to_loadable_multiuse : {{.*}} {
+// CHECK:       {{bb[0-9]+}}([[INSTANCE:%[^,]+]] : $*T):
+// CHECK:         [[STACK:%[^,]+]] = alloc_stack $T
+// CHECK:         copy_addr [[INSTANCE]] to [init] [[STACK]]
+// CHECK:         [[STACK_AS_CLASS:%[^,]+]] = unchecked_addr_cast [[STACK]] : $*T to $*Klass
+// CHECK:         [[BORROWED_KLASS:%[^,]+]] = load_borrow [[STACK_AS_CLASS]]
+// CHECK:         [[KLASS_TO_RETURN:%[^,]+]] = copy_value [[BORROWED_KLASS]]
+// CHECK:         [[KLASS_TO_DESTROY:%[^,]+]] = copy_value [[BORROWED_KLASS]]
+// CHECK:         end_borrow [[BORROWED_KLASS]]
+// CHECK:         destroy_value [[KLASS_TO_DESTROY]]
+// CHECK:         destroy_addr [[STACK]]
+// CHECK:         dealloc_stack [[STACK]]
+// CHECK:         return [[KLASS_TO_RETURN]]
+// CHECK-LABEL: } // end sil function 'test_unchecked_bitwise_cast_to_loadable_multiuse'
+sil hidden [ossa] @test_unchecked_bitwise_cast_to_loadable_multiuse : $@convention(thin) <T> (@in_guaranteed T) -> @owned Klass {
+bb0(%instance : @guaranteed $T):
+  %copy = copy_value %instance : $T
+  %unowned_klass = unchecked_bitwise_cast %copy : $T to $Klass
+  %klass = copy_value %unowned_klass : $Klass
+  %klass2 = copy_value %unowned_klass : $Klass
+  destroy_value %klass2 : $Klass
+  destroy_value %copy : $T
+  return %klass : $Klass
+}
+
 // CHECK-LABEL: sil hidden [ossa] @test_unconditional_checked_cast1 : $@convention(thin) <T> (Builtin.Int64) -> @out T {
 // CHECK: bb0(%0 : $*T, %1 : $Builtin.Int64):
 // CHECK:   [[INT:%.*]] = alloc_stack $Builtin.Int64
@@ -1609,3 +1673,31 @@ bb0(%0 : @guaranteed $T):
   return %6 : $U
 }
 
+// CHECK-LABEL: sil [ossa] @yield_two : {{.*}} {
+// CHECK:         tuple_element_addr
+// CHECK:         tuple_element_addr
+// CHECK:         [[KEY:%[^,]+]] = tuple_element_addr {{%[^,]+}} : $*(key: Key, value: Value), 0
+// CHECK:         [[VALUE:%[^,]+]] = tuple_element_addr {{%[^,]+}} : $*(key: Key, value: Value), 1
+// CHECK:         yield ([[KEY]] : $*Key, [[VALUE]] : $*Value)
+// CHECK-LABEL: } // end sil function 'yield_two'
+sil [ossa] @yield_two : $@yield_once @convention(method) <Key, Value> (@in_guaranteed Key, @in_guaranteed Value) -> (@yields @in_guaranteed Key, @yields @in_guaranteed Value) {
+bb0(%0 : @guaranteed $Key, %1 : @guaranteed $Value):
+  %7 = copy_value %0 : $Key
+  %8 = copy_value %1 : $Value
+  %9 = tuple $(key: Key, value: Value) (%7, %8)
+  %10 = begin_borrow %9 : $(key: Key, value: Value)
+  %11 = tuple_extract %10 : $(key: Key, value: Value), 0
+  %12 = tuple_extract %10 : $(key: Key, value: Value), 1
+  yield (%11 : $Key, %12 : $Value), resume bb1, unwind bb2
+
+bb1:
+  end_borrow %10 : $(key: Key, value: Value)
+  destroy_value %9 : $(key: Key, value: Value)
+  %17 = tuple ()
+  return %17 : $()
+
+bb2:
+  end_borrow %10 : $(key: Key, value: Value)
+  destroy_value %9 : $(key: Key, value: Value)
+  unwind
+}