Enable GenericSpecializer for borrow/mutate accessors and fix it for conditionally copyable types

include/swift/SIL/OSSACompleteLifetime.h

@@ -43,11 +43,6 @@ class OSSACompleteLifetime {
   enum HandleTrivialVariable_t { IgnoreTrivialVariable, ExtendTrivialVariable };
 
 private:
-  /// If domInfo is nullptr, then InteriorLiveness never assumes dominance. As a
-  /// result it may report extra unenclosedPhis. In that case, any attempt to
-  /// create a new phi would result in an immediately redundant phi.
-  const DominanceInfo *domInfo = nullptr;
-
   DeadEndBlocks &deadEndBlocks;
 
   /// Cache intructions already handled by the recursive algorithm to avoid
@@ -69,13 +64,11 @@ class OSSACompleteLifetime {
 
 public:
   OSSACompleteLifetime(
-      SILFunction *function, const DominanceInfo *domInfo,
-      DeadEndBlocks &deadEndBlocks,
+      SILFunction *function, DeadEndBlocks &deadEndBlocks,
       HandleTrivialVariable_t handleTrivialVariable = IgnoreTrivialVariable,
-      bool forceLivenessVerification = false,
-      bool nonDestroyingEnd = false)
-      : domInfo(domInfo), deadEndBlocks(deadEndBlocks),
-        completedValues(function), handleTrivialVariable(handleTrivialVariable),
+      bool forceLivenessVerification = false, bool nonDestroyingEnd = false)
+      : deadEndBlocks(deadEndBlocks), completedValues(function),
+        handleTrivialVariable(handleTrivialVariable),
         ForceLivenessVerification(forceLivenessVerification),
         nonDestroyingEnd(nonDestroyingEnd) {}
 
@@ -177,9 +170,6 @@ class OSSACompleteLifetime {
 class UnreachableLifetimeCompletion {
   SILFunction *function;
 
-  // If domInfo is nullptr, lifetime completion may attempt to recreate
-  // redundant phis, which should be immediately discarded.
-  const DominanceInfo *domInfo = nullptr;
   DeadEndBlocks &deadEndBlocks;
 
   BasicBlockSetVector unreachableBlocks;
@@ -188,9 +178,9 @@ class UnreachableLifetimeCompletion {
   bool updatingLifetimes = false;
 
 public:
-  UnreachableLifetimeCompletion(SILFunction *function, DominanceInfo *domInfo,
+  UnreachableLifetimeCompletion(SILFunction *function,
                                 DeadEndBlocks &deadEndBlocks)
-      : function(function), domInfo(domInfo), deadEndBlocks(deadEndBlocks),
+      : function(function), deadEndBlocks(deadEndBlocks),
         unreachableBlocks(function), unreachableInsts(function),
         incompleteValues(function) {}
 

include/swift/SIL/OwnershipLiveness.h

@@ -246,8 +246,7 @@ class InteriorLiveness : public OSSALiveness {
 public:
   InteriorLiveness(SILValue def): OSSALiveness(def) {}
 
-  void compute(const DominanceInfo *domInfo,
-               InnerScopeHandlerRef handleInnerScope = InnerScopeHandlerRef());
+  void compute(InnerScopeHandlerRef handleInnerScope = InnerScopeHandlerRef());
 
   /// Compute the boundary from the blocks discovered during liveness analysis.
   void computeBoundary(PrunedLivenessBoundary &boundary) const {

include/swift/SIL/SILCloner.h

@@ -1377,6 +1377,15 @@ void SILCloner<ImplClass>::visitLoadInst(LoadInst *Inst) {
                                       LoadOwnershipQualifier::Unqualified));
   }
 
+  if (getOpValue(Inst->getOperand())
+          ->getType()
+          .isTrivial(getBuilder().getFunction())) {
+    return recordClonedInstruction(
+        Inst, getBuilder().createLoad(getOpLocation(Inst->getLoc()),
+                                      getOpValue(Inst->getOperand()),
+                                      LoadOwnershipQualifier::Trivial));
+  }
+
   return recordClonedInstruction(
       Inst, getBuilder().createLoad(getOpLocation(Inst->getLoc()),
                                     getOpValue(Inst->getOperand()),
@@ -1394,6 +1403,15 @@ void SILCloner<ImplClass>::visitLoadBorrowInst(LoadBorrowInst *Inst) {
                                       LoadOwnershipQualifier::Unqualified));
   }
 
+  if (getOpValue(Inst->getOperand())
+          ->getType()
+          .isTrivial(getBuilder().getFunction())) {
+    return recordClonedInstruction(
+        Inst, getBuilder().createLoad(getOpLocation(Inst->getLoc()),
+                                      getOpValue(Inst->getOperand()),
+                                      LoadOwnershipQualifier::Trivial));
+  }
+
   recordClonedInstruction(
       Inst, getBuilder().createLoadBorrow(getOpLocation(Inst->getLoc()),
                                           getOpValue(Inst->getOperand())));
@@ -1402,7 +1420,10 @@ void SILCloner<ImplClass>::visitLoadBorrowInst(LoadBorrowInst *Inst) {
 template <typename ImplClass>
 void SILCloner<ImplClass>::visitBeginBorrowInst(BeginBorrowInst *Inst) {
   getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope()));
-  if (!getBuilder().hasOwnership()) {
+  if (!getBuilder().hasOwnership() ||
+      getOpValue(Inst->getOperand())
+          ->getType()
+          .isTrivial(getBuilder().getFunction())) {
     return recordFoldedValue(Inst, getOpValue(Inst->getOperand()));
   }
 
@@ -1416,7 +1437,10 @@ void SILCloner<ImplClass>::visitBeginBorrowInst(BeginBorrowInst *Inst) {
 template <typename ImplClass>
 void SILCloner<ImplClass>::visitBorrowedFromInst(BorrowedFromInst *bfi) {
   getBuilder().setCurrentDebugScope(getOpScope(bfi->getDebugScope()));
-  if (!getBuilder().hasOwnership()) {
+  if (!getBuilder().hasOwnership() ||
+      getOpValue(bfi->getBorrowedValue())
+          ->getType()
+          .isTrivial(getBuilder().getFunction())) {
     return recordFoldedValue(bfi, getOpValue(bfi->getBorrowedValue()));
   }
 
@@ -1456,6 +1480,16 @@ void SILCloner<ImplClass>::visitStoreInst(StoreInst *Inst) {
                                        StoreOwnershipQualifier::Unqualified));
   }
 
+  if (getOpValue(Inst->getSrc())
+          ->getType()
+          .isTrivial(getBuilder().getFunction())) {
+    return recordClonedInstruction(
+        Inst, getBuilder().createStore(getOpLocation(Inst->getLoc()),
+                                       getOpValue(Inst->getSrc()),
+                                       getOpValue(Inst->getDest()),
+                                       StoreOwnershipQualifier::Trivial));
+  }
+
   recordClonedInstruction(
       Inst, getBuilder().createStore(
                 getOpLocation(Inst->getLoc()), getOpValue(Inst->getSrc()),
@@ -1473,6 +1507,16 @@ void SILCloner<ImplClass>::visitStoreBorrowInst(StoreBorrowInst *Inst) {
     return;
   }
 
+  if (getOpValue(Inst->getDest())
+          ->getType()
+          .isTrivial(getBuilder().getFunction())) {
+    getBuilder().createStore(
+        getOpLocation(Inst->getLoc()), getOpValue(Inst->getSrc()),
+        getOpValue(Inst->getDest()), StoreOwnershipQualifier::Trivial);
+    mapValue(Inst, getOpValue(Inst->getDest()));
+    return;
+  }
+
   recordClonedInstruction(
       Inst, getBuilder().createStoreBorrow(getOpLocation(Inst->getLoc()),
                                            getOpValue(Inst->getSrc()),
@@ -1484,7 +1528,10 @@ void SILCloner<ImplClass>::visitEndBorrowInst(EndBorrowInst *Inst) {
   getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope()));
 
   // Do not clone any end_borrow.
-  if (!getBuilder().hasOwnership())
+  if (!getBuilder().hasOwnership() ||
+      getOpValue(Inst->getOperand())
+          ->getType()
+          .isTrivial(getBuilder().getFunction()))
     return;
 
   recordClonedInstruction(
@@ -2314,7 +2361,13 @@ void SILCloner<ImplClass>::visitDestroyValueInst(DestroyValueInst *Inst) {
         return;
       }
     }
-
+
+    if (getOpValue(Inst->getOperand())
+            ->getType()
+            .isTrivial(getBuilder().getFunction())) {
+      return;
+    }
+
     return recordClonedInstruction(
         Inst, getBuilder().createReleaseValue(
                   getOpLocation(Inst->getLoc()), getOpValue(Inst->getOperand()),
@@ -3330,6 +3383,13 @@ template<typename ImplClass>
 void
 SILCloner<ImplClass>::visitDestroyAddrInst(DestroyAddrInst *Inst) {
   getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope()));
+
+  if (getOpValue(Inst->getOperand())
+          ->getType()
+          .isTrivial(getBuilder().getFunction())) {
+    return;
+  }
+
   recordClonedInstruction(
       Inst, getBuilder().createDestroyAddr(getOpLocation(Inst->getLoc()),
                                            getOpValue(Inst->getOperand())));

include/swift/SIL/SILFunction.h

@@ -1564,7 +1564,7 @@ class SILFunction
     return std::find_if(begin(), end(),
       [](const SILBasicBlock &BB) -> bool {
         const TermInst *TI = BB.getTerminator();
-        return isa<ReturnInst>(TI);
+        return isa<ReturnInst>(TI) || isa<ReturnBorrowInst>(TI);
     });
   }
 
@@ -1574,7 +1574,7 @@ class SILFunction
     return std::find_if(begin(), end(),
       [](const SILBasicBlock &BB) -> bool {
         const TermInst *TI = BB.getTerminator();
-        return isa<ReturnInst>(TI);
+        return isa<ReturnInst>(TI) || isa<ReturnBorrowInst>(TI);
     });
   }
 

lib/SIL/Utils/OSSACompleteLifetime.cpp

@@ -169,7 +169,7 @@ static FunctionTest LivenessPartialBoundaryOutsideUsersTest(
     [](auto &function, auto &arguments, auto &test) {
       SILValue value = arguments.takeValue();
       InteriorLiveness liveness(value);
-      liveness.compute(test.getDominanceInfo());
+      liveness.compute();
       visitUsersOutsideLinearLivenessBoundary(
           value, liveness.getLiveness(),
           [](auto *inst) { inst->print(llvm::outs()); });
@@ -583,7 +583,7 @@ bool OSSACompleteLifetime::analyzeAndUpdateLifetime(SILValue value,
                                  nonDestroyingEnd, deadEndBlocks);
   }
   InteriorLiveness liveness(value);
-  liveness.compute(domInfo, handleInnerScope);
+  liveness.compute(handleInnerScope);
   if (VerifyLifetimeCompletion && boundary != Boundary::Availability &&
       liveness.getAddressUseKind() != AddressUseKind::NonEscaping) {
     llvm::errs() << "Incomplete liveness for: " << value;
@@ -619,10 +619,9 @@ static FunctionTest OSSACompleteLifetimeTest(
       auto *deb = test.getDeadEndBlocks();
       llvm::outs() << "OSSA lifetime completion on " << kind
                    << " boundary: " << value;
-      OSSACompleteLifetime completion(&function, /*domInfo*/ nullptr, *deb,
-                                      OSSACompleteLifetime::IgnoreTrivialVariable,
-                                      /*forceLivenessVerification=*/false,
-                                      nonDestroyingEnd);
+      OSSACompleteLifetime completion(
+          &function, *deb, OSSACompleteLifetime::IgnoreTrivialVariable,
+          /*forceLivenessVerification=*/false, nonDestroyingEnd);
       completion.completeOSSALifetime(value, kind);
       function.print(llvm::outs());
     });
@@ -699,7 +698,7 @@ bool UnreachableLifetimeCompletion::completeLifetimes() {
     }
   }
 
-  OSSACompleteLifetime completion(function, domInfo, deadEndBlocks);
+  OSSACompleteLifetime completion(function, deadEndBlocks);
 
   bool changed = false;
   for (auto value : incompleteValues) {

lib/SIL/Utils/OwnershipLiveness.cpp

@@ -115,11 +115,6 @@ struct InteriorLivenessVisitor :
 
   InteriorLiveness &interiorLiveness;
 
-  // If domInfo is nullptr, then InteriorLiveness never assumes dominance. As a
-  // result it may report extra unenclosedPhis. In that case, any attempt to
-  // create a new phi would result in an immediately redundant phi.
-  const DominanceInfo *domInfo = nullptr;
-
   /// handleInnerScopeCallback may add uses to the inner scope, but it may not
   /// modify the use-list containing \p borrowingOperand. This callback can be
   /// used to ensure that the inner scope is complete before visiting its scope
@@ -134,13 +129,11 @@ struct InteriorLivenessVisitor :
   NodeSet visited;
 
   InteriorLivenessVisitor(
-    InteriorLiveness &interiorLiveness,
-    const DominanceInfo *domInfo,
-    InteriorLiveness::InnerScopeHandlerRef handleInnerScope)
-    : interiorLiveness(interiorLiveness),
-      domInfo(domInfo),
-      handleInnerScopeCallback(handleInnerScope),
-      visited(interiorLiveness.ownershipDef->getFunction()) {}
+      InteriorLiveness &interiorLiveness,
+      InteriorLiveness::InnerScopeHandlerRef handleInnerScope)
+      : interiorLiveness(interiorLiveness),
+        handleInnerScopeCallback(handleInnerScope),
+        visited(interiorLiveness.ownershipDef->getFunction()) {}
 
   bool handleUsePoint(Operand *use, UseLifetimeConstraint useConstraint) {
     interiorLiveness.liveness.updateForUse(
@@ -182,11 +175,11 @@ struct InteriorLivenessVisitor :
   }
 };
 
-void InteriorLiveness::compute(const DominanceInfo *domInfo, InnerScopeHandlerRef handleInnerScope) {
+void InteriorLiveness::compute(InnerScopeHandlerRef handleInnerScope) {
   liveness.initializeDef(ownershipDef);
   addressUseKind = AddressUseKind::NonEscaping;
-  InteriorLivenessVisitor(*this, domInfo, handleInnerScope)
-    .visitInteriorUses(ownershipDef);
+  InteriorLivenessVisitor(*this, handleInnerScope)
+      .visitInteriorUses(ownershipDef);
 }
 
 void InteriorLiveness::print(llvm::raw_ostream &OS) const {
@@ -328,12 +321,11 @@ static FunctionTest
                            SILValue value = arguments.takeValue();
                            function.print(llvm::outs());
                            llvm::outs() << "Interior liveness: " << value;
-                           auto *domTree = test.getDominanceInfo();
                            InteriorLiveness liveness(value);
                            auto handleInnerScope = [](SILValue innerBorrow) {
                              llvm::outs() << "Inner scope: " << innerBorrow;
                            };
-                           liveness.compute(domTree, handleInnerScope);
+                           liveness.compute(handleInnerScope);
                            liveness.print(llvm::outs());
 
                            PrunedLivenessBoundary boundary;

lib/SIL/Utils/OwnershipUtils.cpp

@@ -23,6 +23,7 @@
 #include "swift/SIL/SILArgument.h"
 #include "swift/SIL/SILBuilder.h"
 #include "swift/SIL/SILInstruction.h"
+#include "swift/SIL/ScopedAddressUtils.h"
 #include "swift/SIL/Test.h"
 
 using namespace swift;
@@ -1245,6 +1246,11 @@ bool swift::getAllBorrowIntroducingValues(SILValue inputValue,
       continue;
     }
 
+    // If the introducer is a ScopedAddressValue, bailout.
+    if (auto scopedAddress = ScopedAddressValue(value)) {
+      return false;
+    }
+
     // If v produces .none ownership, then we can ignore it. It is important
     // that we put this before checking for guaranteed forwarding instructions,
     // since we want to ignore guaranteed forwarding instructions that in this

lib/SILOptimizer/Mandatory/MoveOnlyAddressCheckerUtils.cpp

@@ -4106,7 +4106,7 @@ bool MoveOnlyAddressChecker::completeLifetimes() {
 
   // Lifetimes must be completed inside out (bottom-up in the CFG).
   PostOrderFunctionInfo *postOrder = poa->get(fn);
-  OSSACompleteLifetime completion(fn, domTree, *deadEndBlocksAnalysis->get(fn));
+  OSSACompleteLifetime completion(fn, *deadEndBlocksAnalysis->get(fn));
   for (auto *block : postOrder->getPostOrder()) {
     for (SILInstruction &inst : reverse(*block)) {
       for (auto result : inst.getResults()) {

lib/SILOptimizer/Mandatory/MoveOnlyChecker.cpp

@@ -123,7 +123,7 @@ void MoveOnlyChecker::checkObjects() {
 void MoveOnlyChecker::completeObjectLifetimes(
     ArrayRef<MarkUnresolvedNonCopyableValueInst *> insts) {
   // TODO: Delete once OSSACompleteLifetime is run as part of SILGenCleanup.
-  OSSACompleteLifetime completion(fn, domTree, *deba->get(fn));
+  OSSACompleteLifetime completion(fn, *deba->get(fn));
 
   // Collect all values derived from each mark_unresolved_non_copyable_value
   // instruction via ownership instructions and phis.

lib/SILOptimizer/Mandatory/PredictableMemOpt.cpp

@@ -1921,8 +1921,6 @@ class OptimizeDeadAlloc {
 
   InstructionDeleter &deleter;
 
-  DominanceInfo *domInfo;
-
   /// A structure that we use to compute our available values.
   AvailableValueDataflowContext DataflowContext;
 
@@ -1935,21 +1933,19 @@ class OptimizeDeadAlloc {
 
   bool isTrivial() const { return MemoryType.isTrivial(getFunction()); }
 
-  OptimizeDeadAlloc(AllocationInst *memory,
-                    SmallVectorImpl<PMOMemoryUse> &uses,
+  OptimizeDeadAlloc(AllocationInst *memory, SmallVectorImpl<PMOMemoryUse> &uses,
                     SmallVectorImpl<SILInstruction *> &releases,
                     DeadEndBlocks &deadEndBlocks, InstructionDeleter &deleter,
                     DominanceInfo *domInfo)
       : Module(memory->getModule()), TheMemory(memory),
         MemoryType(getMemoryType(memory)),
-        NumMemorySubElements(getNumSubElements(
-            MemoryType, *memory->getFunction(),
-            TypeExpansionContext(*memory->getFunction()))),
+        NumMemorySubElements(
+            getNumSubElements(MemoryType, *memory->getFunction(),
+                              TypeExpansionContext(*memory->getFunction()))),
         Uses(uses), Releases(releases), deadEndBlocks(deadEndBlocks),
-        deleter(deleter), domInfo(domInfo),
-        DataflowContext(TheMemory, NumMemorySubElements,
-                        OptimizationMode::ReplaceAlloc, uses, deleter,
-                        deadEndBlocks) {}
+        deleter(deleter), DataflowContext(TheMemory, NumMemorySubElements,
+                                          OptimizationMode::ReplaceAlloc, uses,
+                                          deleter, deadEndBlocks) {}
 
   /// If the allocation is an autogenerated allocation that is only stored to
   /// (after load promotion) then remove it completely.
@@ -2311,8 +2307,7 @@ void OptimizeDeadAlloc::removeDeadAllocation() {
   // post-dominating consuming use sets. This can happen if we have an enum that
   // is known dynamically none along a path. This is dynamically correct, but
   // can not be represented in OSSA so we insert these destroys along said path.
-  OSSACompleteLifetime completion(TheMemory->getFunction(), domInfo,
-                                  deadEndBlocks);
+  OSSACompleteLifetime completion(TheMemory->getFunction(), deadEndBlocks);
 
   while (!valuesNeedingLifetimeCompletion.empty()) {
     auto optV = valuesNeedingLifetimeCompletion.pop_back_val();