[NFC] Add SILGenCleanup::completeOSSLifetimes

include/swift/AST/SILOptions.h

@@ -173,6 +173,9 @@ class SILOptions {
   /// If set to true, compile with the SIL Opaque Values enabled.
   bool EnableSILOpaqueValues = false;
 
+  /// Require linear OSSA lifetimes after SILGen
+  bool OSSACompleteLifetimes = false;
+
   // The kind of function bodies to skip emitting.
   FunctionBodySkipping SkipFunctionBodies = FunctionBodySkipping::None;
 

include/swift/Option/FrontendOptions.td

@@ -1165,4 +1165,8 @@ def enable_emit_generic_class_ro_t_list :
 def experimental_spi_only_imports :
   Flag<["-"], "experimental-spi-only-imports">,
   HelpText<"Enable use of @_spiOnly imports">;
+
+def enable_ossa_complete_lifetimes :
+  Flag<["-"], "enable-ossa-complete-lifetimes">,
+  HelpText<"Require linear OSSA lifetimes after SILGen">;
 } // end let Flags = [FrontendOption, NoDriverOption, HelpHidden]

include/swift/SIL/LinearLifetimeChecker.h

@@ -57,10 +57,23 @@ class LinearLifetimeChecker {
   friend class SILOwnershipVerifier;
   friend class SILValueOwnershipChecker;
 
-  DeadEndBlocks &deadEndBlocks;
+  // TODO: migrate away from using dead end blocks for OSSA values. end_borrow
+  // or destroy_value should ideally exist on all paths. However, deadEndBlocks
+  // may still be useful for checking memory lifetime for address uses.
+  DeadEndBlocks *deadEndBlocks;
 
 public:
-  LinearLifetimeChecker(DeadEndBlocks &deadEndBlocks)
+  /// \p deadEndBlocks should be provided for lifetimes that do not require
+  /// consuming uses on dead-end paths, which end in an unreachable terminator.
+  /// OSSA values require consumes on all paths, so \p deadEndBlocks are *not*
+  /// required for OSSA lifetimes. Memory lifetimes and access scopes only
+  /// require destroys on non-dead-end paths.
+  ///
+  /// TODO: The verifier currently requires OSSA borrow scopes to end on all
+  /// paths. Owned OSSA lifetimes may still be missing destroys on dead-end
+  /// paths. Once owned values are fully enforced, the same invariant will hold
+  /// for all OSSA values.
+  LinearLifetimeChecker(DeadEndBlocks *deadEndBlocks = nullptr)
       : deadEndBlocks(deadEndBlocks) {}
 
   /// Returns true that \p value forms a linear lifetime with consuming uses \p

include/swift/SIL/OSSALifetimeCompletion.h

@@ -0,0 +1,129 @@
+//===--- OwnershipLifetimeCompletion.h ------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+///
+/// OSSA lifetime completion adds lifetime ending instructions to make
+/// linear lifetimes complete.
+///
+/// Completion is bottom-up recursive over nested borrow scopes. Additionally,
+/// this may be extended to support dependent owned lifetimes in the future to
+/// handle owned non-escaping values.
+///
+/// Lexical lifetimes can only be incomplete as a result of dead-end blocks. In
+/// this case, their lifetime ends immediately before the dead-end block.
+///
+/// Nonlexical lifetimes can be incomplete for any reason. Their lifetime ends
+/// at the liveness boundary.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_SILOPTIMIZER_UTILS_OSSSALIFETIMECOMPLETION_H
+#define SWIFT_SILOPTIMIZER_UTILS_OSSSALIFETIMECOMPLETION_H
+
+#include "swift/SIL/NodeDatastructures.h"
+#include "swift/SIL/OwnershipLiveness.h"
+#include "swift/SIL/SILFunction.h"
+
+namespace swift {
+
+enum class LifetimeCompletion { NoLifetime, AlreadyComplete, WasCompleted };
+
+class OSSALifetimeCompletion {
+  // If domInfo is nullptr, then InteriorLiveness never assumes dominance. As a
+  // result it may report extra unenclosedPhis. In that case, any attempt to
+  // create a new phi would result in an immediately redundant phi.
+  const DominanceInfo *domInfo = nullptr;
+
+  // Cache intructions already handled by the recursive algorithm to avoid
+  // recomputing their lifetimes.
+  ValueSet completedValues;
+
+public:
+  OSSALifetimeCompletion(SILFunction *function, const DominanceInfo *domInfo)
+    : domInfo(domInfo), completedValues(function) {}
+
+  /// Insert a lifetime-ending instruction on every path to complete the OSSA
+  /// lifetime of \p value. Lifetime completion is only relevant for owned
+  /// values or borrow introducers.
+  ///
+  /// Returns true if any new instructions were created to complete the
+  /// lifetime.
+  ///
+  /// TODO: We also need to complete scoped addresses (e.g. store_borrow)!
+  LifetimeCompletion completeOSSALifetime(SILValue value) {
+    if (value->getOwnershipKind() == OwnershipKind::None)
+      return LifetimeCompletion::NoLifetime;
+
+    if (value->getOwnershipKind() != OwnershipKind::Owned) {
+      BorrowedValue borrowedValue(value);
+      if (!borrowedValue)
+        return LifetimeCompletion::NoLifetime;
+
+      if (!borrowedValue.isLocalScope())
+        return LifetimeCompletion::AlreadyComplete;
+    }
+    if (!completedValues.insert(value))
+      return LifetimeCompletion::AlreadyComplete;
+
+    return analyzeAndUpdateLifetime(value)
+      ? LifetimeCompletion::WasCompleted
+      : LifetimeCompletion::AlreadyComplete;
+  }
+
+protected:
+  bool analyzeAndUpdateLifetime(SILValue value);
+};
+
+//===----------------------------------------------------------------------===//
+// UnreachableLifetimeCompletion
+//===----------------------------------------------------------------------===//
+
+/// Fixup OSSA before deleting an unreachable code path.
+///
+/// Only needed when a code path reaches a no-return function, making the
+/// path now partially unreachable. Conditional branch folding requires no fixup
+/// because it causes the entire path to become unreachable.
+class UnreachableLifetimeCompletion {
+  SILFunction *function;
+
+  // If domInfo is nullptr, lifetime completion may attempt to recreate
+  // redundant phis, which should be immediately discarded.
+  const DominanceInfo *domInfo = nullptr;
+
+  BasicBlockSetVector unreachableBlocks;
+  InstructionSet unreachableInsts; // not including those in unreachableBlocks
+  ValueSetVector incompleteValues;
+  bool updatingLifetimes = false;
+
+public:
+  UnreachableLifetimeCompletion(SILFunction *function, DominanceInfo *domInfo)
+    : function(function), unreachableBlocks(function),
+      unreachableInsts(function), incompleteValues(function) {}
+
+  /// Record information about this unreachable instruction and return true if
+  /// ends any simple OSSA lifetimes.
+  ///
+  /// Note: this must be called in forward order so that lifetime completion
+  /// runs from the inside out.
+  void visitUnreachableInst(SILInstruction *instruction);
+
+  void visitUnreachableBlock(SILBasicBlock *block) {
+    unreachableBlocks.insert(block);
+  }
+
+  /// Complete the lifetime of any value defined outside of the unreachable
+  /// region that was previously destroyed in the unreachable region.
+  bool completeLifetimes();
+};
+
+} // namespace swift
+
+#endif

include/swift/SIL/OwnershipLiveness.h

@@ -241,6 +241,11 @@ class InteriorLiveness : public OSSALiveness {
   void compute(const DominanceInfo *domInfo,
                InnerScopeHandlerRef handleInnerScope = InnerScopeHandlerRef());
 
+  /// Compute the boundary from the blocks discovered during liveness analysis.
+  void computeBoundary(PrunedLivenessBoundary &boundary) const {
+    liveness.computeBoundary(boundary);
+  }
+
   AddressUseKind getAddressUseKind() const { return addressUseKind; }
 
   ArrayRef<SILValue> getUnenclosedPhis() const { return unenclosedPhis; }

include/swift/SIL/SILFunction.h

@@ -1403,18 +1403,30 @@ class SILFunction
         decl->getLifetimeAnnotation());
   }
 
-  /// verify - Run the IR verifier to make sure that the SILFunction follows
+  /// verify - Run the SIL verifier to make sure that the SILFunction follows
   /// invariants.
-  void verify(bool SingleFunction = true) const;
+  void verify(bool SingleFunction = true,
+              bool isCompleteOSSA = true,
+              bool checkLinearLifetime = true) const;
+
+  /// Run the SIL verifier without assuming OSSA lifetimes end at dead end
+  /// blocks.
+  void verifyIncompleteOSSA() const {
+    verify(/*SingleFunction=*/true, /*completeOSSALifetimes=*/false);
+  }
 
   /// Verifies the lifetime of memory locations in the function.
   void verifyMemoryLifetime();
 
-  /// Run the SIL ownership verifier to check for ownership invariant failures.
+  /// Run the SIL ownership verifier to check that all values with ownership
+  /// have a linear lifetime. Regular OSSA invariants are checked separately in
+  /// normal SIL verification.
+  ///
+  /// \p deadEndBlocks is nullptr when OSSA lifetimes are complete.
   ///
-  /// NOTE: The ownership verifier is always run when performing normal IR
+  /// NOTE: The ownership verifier is run when performing normal IR
   /// verification, so this verification can be viewed as a subset of
-  /// SILFunction::verify.
+  /// SILFunction::verify(checkLinearLifetimes=true).
   void verifyOwnership(DeadEndBlocks *deadEndBlocks) const;
 
   /// Verify that all non-cond-br critical edges have been split.

include/swift/SIL/SILModule.h

@@ -904,7 +904,17 @@ class SILModule {
 
   /// Run the SIL verifier to make sure that all Functions follow
   /// invariants.
-  void verify() const;
+  void verify(bool isCompleteOSSA = true,
+              bool checkLinearLifetime = true) const;
+
+  /// Run the SIL verifier without assuming OSSA lifetimes end at dead end
+  /// blocks.
+  void verifyIncompleteOSSA() const {
+    verify(/*isCompleteOSSA=*/false);
+  }
+
+  /// Check linear OSSA lifetimes, assuming complete OSSA.
+  void verifyOwnership() const;
 
   /// Check if there are any leaking instructions.
   ///

include/swift/SIL/SILValue.h

@@ -675,6 +675,8 @@ class SILValue {
   }
 
   /// Verify that this SILValue and its uses respects ownership invariants.
+  ///
+  /// \p DEBlocks is nullptr when OSSA lifetimes are complete.
   void verifyOwnership(DeadEndBlocks *DEBlocks) const;
 
   SWIFT_DEBUG_DUMP;

include/swift/SILOptimizer/PassManager/PassPipeline.def

@@ -22,6 +22,7 @@
 #define PASSPIPELINE(NAME, DESCRIPTION)
 #endif
 
+PASSPIPELINE(SILGen, "SILGen Passes")
 PASSPIPELINE(Diagnostic, "Guaranteed Passes")
 PASSPIPELINE(LowerHopToActor, "Lower Hop to Actor")
 PASSPIPELINE(OwnershipEliminator, "Utility pass to just run the ownership eliminator pass")

lib/Frontend/CompilerInvocation.cpp

@@ -1985,6 +1985,8 @@ static bool ParseSILArgs(SILOptions &Opts, ArgList &Args,
     parseExclusivityEnforcementOptions(A, Opts, Diags);
   }
 
+  Opts.OSSACompleteLifetimes |= Args.hasArg(OPT_enable_ossa_complete_lifetimes);
+
   return false;
 }
 

lib/SIL/Utils/CMakeLists.txt

@@ -11,6 +11,7 @@ target_sources(swiftSIL PRIVATE
   MemAccessUtils.cpp
   MemoryLocations.cpp
   OptimizationRemark.cpp
+  OSSALifetimeCompletion.cpp
   OwnershipLiveness.cpp
   OwnershipUtils.cpp
   PrettyStackTrace.cpp