trait_selection: Restore type-op solver constraints

compiler/rustc_borrowck/src/handle_placeholders.rs

@@ -246,7 +246,9 @@ pub(crate) fn compute_sccs_applying_placeholder_outlives_constraints<'tcx>(
         mut outlives_constraints,
         universe_causes,
         type_tests,
+        solver_constraints,
     } = constraints;
+    assert!(solver_constraints.is_none(), "solver constraints were not lowered to NLL");
 
     let fr_static = universal_regions.fr_static;
     let compute_sccs =

compiler/rustc_borrowck/src/type_check/constraint_conversion.rs

@@ -67,7 +67,8 @@ impl<'a, 'tcx> ConstraintConversion<'a, 'tcx> {
 
     #[instrument(skip(self), level = "debug")]
     pub(super) fn convert_all(&mut self, query_constraints: &QueryRegionConstraints<'tcx>) {
-        let QueryRegionConstraints { constraints, assumptions } = query_constraints;
+        let QueryRegionConstraints { constraints, assumptions, solver_constraints } =
+            query_constraints;
         let assumptions =
             elaborate::elaborate_outlives_assumptions(self.infcx.tcx, assumptions.iter().copied());
 
@@ -76,6 +77,11 @@ impl<'a, 'tcx> ConstraintConversion<'a, 'tcx> {
                 self.convert(predicate, category, &assumptions);
             });
         }
+
+        if let Some(constraint) = solver_constraints {
+            self.constraints
+                .register_solver_constraint((**constraint).clone().with_spans(self.span));
+        }
     }
 
     /// Given an instance of the closure type, this method instantiates the "extra" requirements

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -17,6 +17,7 @@ use rustc_infer::infer::outlives::env::RegionBoundPairs;
 use rustc_infer::infer::region_constraints::RegionConstraintData;
 use rustc_infer::infer::{
     BoundRegionConversionTime, InferCtxt, NllRegionVariableOrigin, RegionVariableOrigin,
+    SolverRegionConstraint,
 };
 use rustc_infer::traits::{Obligation, ObligationCause, PredicateObligations};
 use rustc_middle::mir::visit::{NonMutatingUseContext, PlaceContext, Visitor};
@@ -112,6 +113,7 @@ pub(crate) fn type_check<'tcx>(
         outlives_constraints: OutlivesConstraintSet::default(),
         type_tests: Vec::default(),
         universe_causes: FxIndexMap::default(),
+        solver_constraints: None,
     };
 
     let CreateResult {
@@ -133,6 +135,9 @@ pub(crate) fn type_check<'tcx>(
             pre_assumptions.is_empty(),
             "there should be no incoming region assumptions = {pre_assumptions:#?}",
         );
+        if let Some(constraint) = infcx.take_solver_region_constraints() {
+            assert!(constraint.is_true(), "incoming solver constraints: {constraint:?}");
+        }
     }
 
     debug!(?normalized_inputs_and_output);
@@ -171,7 +176,8 @@ pub(crate) fn type_check<'tcx>(
 
     let polonius_context = typeck.polonius_context;
 
-    if infcx.tcx.assumptions_on_binders() {
+    if let Some(solver_constraints) = typeck.constraints.solver_constraints.take() {
+        assert!(infcx.tcx.assumptions_on_binders());
         let mut converter = constraint_conversion::ConstraintConversion::new(
             typeck.infcx,
             typeck.universal_regions,
@@ -183,6 +189,7 @@ pub(crate) fn type_check<'tcx>(
             typeck.constraints,
         );
         typeck.infcx.destructure_solver_region_constraints_for_borrowck(
+            *solver_constraints,
             &mut converter,
             typeck.known_type_outlives_obligations,
             typeck.region_bound_pairs,
@@ -291,9 +298,19 @@ pub(crate) struct MirTypeckRegionConstraints<'tcx> {
     pub(crate) universe_causes: FxIndexMap<ty::UniverseIndex, UniverseInfo<'tcx>>,
 
     pub(crate) type_tests: Vec<TypeTest<'tcx>>,
+
+    /// Constraints from completed AoB type operations, lowered once all bounds are available.
+    pub(crate) solver_constraints: Option<Box<SolverRegionConstraint<'tcx>>>,
 }
 
 impl<'tcx> MirTypeckRegionConstraints<'tcx> {
+    pub(crate) fn register_solver_constraint(&mut self, constraint: SolverRegionConstraint<'tcx>) {
+        self.solver_constraints = Some(Box::new(match self.solver_constraints.take() {
+            Some(previous) => SolverRegionConstraint::build_and(*previous, constraint),
+            None => constraint,
+        }));
+    }
+
     /// Creates a `Region` for a given `PlaceholderRegion`, or returns the
     /// region that corresponds to a previously created one.
     pub(crate) fn placeholder_region(

compiler/rustc_infer/src/infer/canonical/query_response.rs

@@ -146,11 +146,13 @@ impl<'tcx> InferCtxt<'tcx> {
         let region_obligations = self.take_registered_region_obligations();
         let region_assumptions = self.take_registered_region_assumptions();
         debug!(?region_obligations);
+        let solver_constraints = self.take_solver_region_constraints();
         let region_constraints = self.with_region_constraints(|region_constraints| {
             make_query_region_constraints(
                 region_obligations,
                 region_constraints,
                 region_assumptions,
+                solver_constraints,
             )
         });
         debug!(?region_constraints);
@@ -195,9 +197,9 @@ impl<'tcx> InferCtxt<'tcx> {
         let InferOk { value: result_args, obligations } =
             self.query_response_instantiation(cause, param_env, original_values, query_response)?;
 
-        for QueryRegionConstraint { constraint, visible_for_leak_check: vis, .. } in
-            &query_response.value.region_constraints.constraints
-        {
+        let QueryRegionConstraints { constraints, assumptions, solver_constraints } =
+            &query_response.value.region_constraints;
+        for QueryRegionConstraint { constraint, visible_for_leak_check: vis, .. } in constraints {
             let constraint = instantiate_value(self.tcx, &result_args, *constraint);
             match constraint {
                 ty::RegionConstraint::Outlives(clause) => {
@@ -209,11 +211,17 @@ impl<'tcx> InferCtxt<'tcx> {
             }
         }
 
-        for assumption in &query_response.value.region_constraints.assumptions {
+        for assumption in assumptions {
             let assumption = instantiate_value(self.tcx, &result_args, *assumption);
             self.register_region_assumption(assumption);
         }
 
+        if let Some(solver_constraints) = solver_constraints {
+            let constraint =
+                instantiate_value(self.tcx, &result_args, (**solver_constraints).clone());
+            self.register_solver_region_constraint(constraint.with_spans(cause.span));
+        }
+
         let user_result: R =
             query_response.instantiate_projected(self.tcx, &result_args, |q_r| q_r.value.clone());
 
@@ -325,27 +333,29 @@ impl<'tcx> InferCtxt<'tcx> {
             }
         }
 
+        let QueryRegionConstraints { constraints, assumptions, solver_constraints } =
+            &query_response.value.region_constraints;
+
         // ...also include the other query region constraints from the query.
-        output_query_region_constraints.constraints.extend(
-            query_response.value.region_constraints.constraints.iter().filter_map(|&r_c| {
-                let r_c = instantiate_value(self.tcx, &result_args, r_c);
+        output_query_region_constraints.constraints.extend(constraints.iter().filter_map(|&r_c| {
+            let r_c = instantiate_value(self.tcx, &result_args, r_c);
 
-                // Screen out `'a: 'a` or `'a == 'a` cases.
-                if r_c.constraint.is_trivial() { None } else { Some(r_c) }
-            }),
-        );
+            // Screen out `'a: 'a` or `'a == 'a` cases.
+            if r_c.constraint.is_trivial() { None } else { Some(r_c) }
+        }));
 
         // FIXME(higher_ranked_auto): Optimize this to instantiate all assumptions
         // at once, rather than calling `instantiate_value` repeatedly which may
         // create more universes.
-        output_query_region_constraints.assumptions.extend(
-            query_response
-                .value
-                .region_constraints
-                .assumptions
-                .iter()
-                .map(|&r_c| instantiate_value(self.tcx, &result_args, r_c)),
-        );
+        output_query_region_constraints
+            .assumptions
+            .extend(assumptions.iter().map(|&r_c| instantiate_value(self.tcx, &result_args, r_c)));
+
+        if let Some(solver_constraints) = solver_constraints {
+            let constraint =
+                instantiate_value(self.tcx, &result_args, (**solver_constraints).clone());
+            output_query_region_constraints.add_solver_constraints(constraint);
+        }
 
         let user_result: R =
             query_response.instantiate_projected(self.tcx, &result_args, |q_r| q_r.value.clone());
@@ -619,6 +629,7 @@ pub fn make_query_region_constraints<'tcx>(
     outlives_obligations: Vec<TypeOutlivesConstraint<'tcx>>,
     region_constraints: &RegionConstraintData<'tcx>,
     assumptions: Vec<ty::ArgOutlivesClause<'tcx>>,
+    solver_constraints: Option<Box<ty::region_constraint::RegionConstraint<TyCtxt<'tcx>>>>,
 ) -> QueryRegionConstraints<'tcx> {
     let RegionConstraintData { constraints, verifys } = region_constraints;
 
@@ -663,5 +674,5 @@ pub fn make_query_region_constraints<'tcx>(
         ))
         .collect();
 
-    QueryRegionConstraints { constraints, assumptions }
+    QueryRegionConstraints { constraints, assumptions, solver_constraints }
 }

compiler/rustc_infer/src/infer/outlives/obligations.rs

@@ -250,11 +250,13 @@ impl<'tcx> InferCtxt<'tcx> {
             region_outlives.freeze(),
             ty::UniverseIndex::ROOT,
         );
-        self.destructure_solver_region_constraints(assumptions, self);
+        let constraint = self.inner.borrow().solver_region_constraint_storage.get_constraint();
+        self.destructure_solver_region_constraints(constraint, assumptions, self);
     }
 
     pub fn destructure_solver_region_constraints_for_borrowck(
         &self,
+        constraint: SolverRegionConstraint<'tcx>,
         // this is always ConstraintConversion but lol
         conversion: impl TypeOutlivesDelegate<'tcx>,
         known_type_outlives: &[PolyTypeOutlivesClause<'tcx>],
@@ -267,19 +269,19 @@ impl<'tcx> InferCtxt<'tcx> {
             region_outlives.maybe_map(|r| Some(Region::new_var(self.tcx, r))).unwrap(),
             ty::UniverseIndex::ROOT,
         );
-        self.destructure_solver_region_constraints(assumptions, conversion);
+        self.destructure_solver_region_constraints(constraint, assumptions, conversion);
     }
 
     #[instrument(level = "debug", skip(self, conversion))]
     pub fn destructure_solver_region_constraints(
         &self,
+        constraint: SolverRegionConstraint<'tcx>,
         assumptions: rustc_type_ir::region_constraint::Assumptions<TyCtxt<'tcx>>,
         mut conversion: impl TypeOutlivesDelegate<'tcx>,
     ) {
         assert!(self.tcx.assumptions_on_binders());
         assert!(self.next_trait_solver());
 
-        let constraint = self.inner.borrow().solver_region_constraint_storage.get_constraint();
         debug!(?constraint);
         let constraint = region_constraint::destructure_type_outlives_constraints_in_root(
             self,

compiler/rustc_infer/src/infer/solver_region_constraints.rs

@@ -2,6 +2,8 @@ use rustc_middle::ty::TyCtxt;
 use rustc_span::Span;
 use tracing::instrument;
 
+use super::InferCtxt;
+
 pub type SolverRegionConstraint<'tcx> =
     rustc_type_ir::region_constraint::RegionConstraint<TyCtxt<'tcx>, Span>;
 
@@ -20,11 +22,32 @@ impl<'tcx> SolverRegionConstraintStorage<'tcx> {
         }
     }
 
+    pub(crate) fn take(&mut self) -> Option<SolverRegionConstraint<'tcx>> {
+        self.0.take()
+    }
+
     #[instrument(level = "debug", skip(self))]
     pub(crate) fn overwrite(&mut self, constraint: SolverRegionConstraint<'tcx>) {
         self.0 = Some(constraint);
     }
 }
 
+impl<'tcx> InferCtxt<'tcx> {
+    /// Drain completed query constraints without allocating an empty constraint.
+    pub fn take_solver_region_constraints(
+        &self,
+    ) -> Option<Box<rustc_type_ir::region_constraint::RegionConstraint<TyCtxt<'tcx>>>> {
+        assert!(!self.in_snapshot(), "cannot take solver region constraints in a snapshot");
+        if !self.tcx.assumptions_on_binders() {
+            return None;
+        }
+        self.inner
+            .borrow_mut()
+            .solver_region_constraint_storage
+            .take()
+            .map(|constraint| Box::new(constraint.without_spans()))
+    }
+}
+
 #[cfg(test)]
 mod tests;

compiler/rustc_infer/src/infer/solver_region_constraints/tests.rs

@@ -1,3 +1,4 @@
+use rustc_middle::infer::canonical::QueryRegionConstraints;
 use rustc_middle::ty::TyCtxt;
 use rustc_span::{BytePos, Span};
 use rustc_type_ir::region_constraint::{And, LeafRegionConstraint, Or};
@@ -16,3 +17,36 @@ fn canonicalization_preserves_only_one_ambiguity() {
     let c = Or::new([And::new([first]), And::new([second])]);
     assert_eq!(c.0.len(), 1);
 }
+
+#[test]
+fn trivial_solver_constraints_keep_query_response_empty() {
+    let mut constraints = QueryRegionConstraints::<'static>::default();
+    assert!(constraints.is_empty());
+    constraints.extend(&QueryRegionConstraints::default());
+    assert!(constraints.solver_constraints.is_none());
+
+    constraints
+        .add_solver_constraints(rustc_type_ir::region_constraint::RegionConstraint::new_true());
+    assert!(constraints.is_empty());
+    constraints.extend(&constraints.clone());
+    assert!(constraints.is_empty());
+}
+
+#[test]
+fn extending_query_response_preserves_solver_constraints() {
+    let mut constraints = QueryRegionConstraints::<'static>::default();
+    constraints
+        .add_solver_constraints(rustc_type_ir::region_constraint::RegionConstraint::new_ambig(()));
+    assert!(!constraints.is_empty());
+
+    let mut output = QueryRegionConstraints::default();
+    output.extend(&constraints);
+    assert!(output.solver_constraints.as_ref().unwrap().is_ambig());
+    output.extend(&QueryRegionConstraints::default());
+    assert!(output.solver_constraints.as_ref().unwrap().is_ambig());
+
+    constraints
+        .add_solver_constraints(rustc_type_ir::region_constraint::RegionConstraint::new_false());
+    output.extend(&constraints);
+    assert!(output.solver_constraints.as_ref().unwrap().is_false());
+}

compiler/rustc_middle/src/infer/canonical.rs

@@ -76,23 +76,52 @@ pub struct QueryResponse<'tcx, R> {
     pub value: R,
 }
 
-#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
+#[derive(Clone, Debug, Default, PartialEq, Hash)]
 #[derive(StableHash, TypeFoldable, TypeVisitable)]
 pub struct QueryRegionConstraints<'tcx> {
     pub constraints: Vec<QueryRegionConstraint<'tcx>>,
     pub assumptions: Vec<ty::ArgOutlivesClause<'tcx>>,
+    /// Unspanned AoB constraints, canonicalized with the rest of the response.
+    /// The caller attaches its own span when consuming them. `None` avoids
+    /// constructing a trivially true constraint when AoB is disabled.
+    pub solver_constraints: Option<Box<ir::region_constraint::RegionConstraint<TyCtxt<'tcx>>>>,
 }
 
-impl QueryRegionConstraints<'_> {
+impl Eq for QueryRegionConstraints<'_> {}
+
+impl<'tcx> QueryRegionConstraints<'tcx> {
     /// Represents an empty (trivially true) set of region constraints.
     ///
     /// FIXME(higher_ranked_auto): This could still just be true if there are only assumptions?
     /// Because I don't expect for us to get cases where an assumption from one query would
     /// discharge a requirement from another query, which is a potential problem if we did throw
     /// away these assumptions because there were no constraints.
     pub fn is_empty(&self) -> bool {
-        let QueryRegionConstraints { constraints, assumptions } = self;
-        constraints.is_empty() && assumptions.is_empty()
+        let QueryRegionConstraints { constraints, assumptions, solver_constraints } = self;
+        constraints.is_empty()
+            && assumptions.is_empty()
+            && solver_constraints.as_ref().is_none_or(|c| c.is_true())
+    }
+
+    pub fn extend(&mut self, other: &Self) {
+        let QueryRegionConstraints { constraints, assumptions, solver_constraints } = other;
+        self.constraints.extend(constraints.iter().cloned());
+        self.assumptions.extend(assumptions.iter().cloned());
+        if let Some(constraint) = solver_constraints {
+            self.add_solver_constraints((**constraint).clone());
+        }
+    }
+
+    pub fn add_solver_constraints(
+        &mut self,
+        constraint: ir::region_constraint::RegionConstraint<TyCtxt<'tcx>>,
+    ) {
+        self.solver_constraints = Some(Box::new(match self.solver_constraints.take() {
+            Some(previous) => {
+                ir::region_constraint::RegionConstraint::build_and(*previous, constraint)
+            }
+            None => constraint,
+        }));
     }
 }
 

compiler/rustc_trait_selection/src/solve/delegate.rs

@@ -380,6 +380,7 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate<
                 region_obligations,
                 region_constraints,
                 region_assumptions,
+                None,
             )
         });