diff --git a/src/action.rs b/src/action.rs index 869510a85..dfacc6cf5 100644 --- a/src/action.rs +++ b/src/action.rs @@ -14,11 +14,11 @@ use crate::{ unblock_graph::BorrowPcgUnblockAction, validity_conditions::ValidityConditions, }, - owned_pcg::{RegainedCapability, RepackOp}, - pcg::capabilities::CapabilityKind, + owned_pcg::RepackOp, + pcg::{CapabilityKind, place_capabilities::PlaceCapabilitiesInterface}, rustc_interface::middle::mir, utils::{ - DebugRepr, HasBorrowCheckerCtxt, PcgNodeComponent, Place, PlaceLike, + DebugRepr, HasBorrowCheckerCtxt, PcgNodeComponent, Place, display::{DisplayOutput, DisplayWithCtxt, OutputMode}, }, }; @@ -116,10 +116,6 @@ impl<'tcx> From>> for PcgActions<'tcx> { } impl<'tcx> PcgActions<'tcx> { - pub(crate) fn extend(&mut self, actions: PcgActions<'tcx>) { - self.actions.extend(actions.actions); - } - /// The subset of actions applied to the Borrow PCG. #[must_use] pub fn borrow_pcg_actions(&self) -> BorrowPcgActions<'tcx> { @@ -157,7 +153,7 @@ impl<'tcx> PcgActions<'tcx> { .iter() .filter_map(|action| match action { PcgAction::Borrow(BorrowPcgAction { - kind: BorrowPcgActionKind::RemoveEdge(edge), + kind: BorrowPcgActionKind::RemoveEdge { edge, .. }, .. }) => Some(BorrowPcgUnblockAction::new(edge.clone())), _ => None, @@ -166,54 +162,167 @@ impl<'tcx> PcgActions<'tcx> { } } -/// A pair of a PCG action and a debug context (indicating the source of the -/// action and possibly its effects). +/// The capability changes of an action: each place gets the capability, or +/// loses its capability (`None`). +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[cfg_attr(feature = "type-export", derive(ts_rs::TS))] +#[cfg_attr(feature = "type-export", ts(export, concrete(P = String)))] +pub struct CapabilityChanges

(Vec<(P, Option)>); + +impl

Default for CapabilityChanges

{ + fn default() -> Self { + Self(vec![]) + } +} + +impl

From)>> for CapabilityChanges

{ + fn from(changes: Vec<(P, Option)>) -> Self { + Self(changes) + } +} + +impl CapabilityChanges

{ + /// Sets the change of `place`, replacing an earlier change of it. + pub(crate) fn set(&mut self, place: P, capability: Option) { + if let Some(change) = self.0.iter_mut().find(|(p, _)| *p == place) { + change.1 = capability; + } else { + self.0.push((place, capability)); + } + } + + pub(crate) fn extend(&mut self, other: &Self) { + for (place, capability) in other.iter() { + self.set(place, capability); + } + } + + pub fn iter(&self) -> impl Iterator)> + '_ { + self.0.iter().copied() + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub(crate) fn apply<'tcx, Ctxt: Copy>( + &self, + capabilities: &mut impl PlaceCapabilitiesInterface<'tcx, CapabilityKind, P>, + ctxt: Ctxt, + ) -> bool { + let mut changed = false; + for (place, capability) in self.iter() { + changed |= match capability { + Some(capability) => capabilities.insert(place, capability, ctxt), + None => capabilities.remove(place, ctxt).is_some(), + }; + } + changed + } +} + +impl> DisplayWithCtxt for CapabilityChanges

{ + fn display_output(&self, ctxt: Ctxt, mode: OutputMode) -> DisplayOutput { + DisplayOutput::join( + self.0.iter().map(|(place, capability)| { + DisplayOutput::Seq(vec![ + place.display_output(ctxt, mode), + ": ".into(), + match capability { + Some(capability) => capability.display_output(ctxt, mode), + None => "None".into(), + }, + ]) + }), + &", ".into(), + ) + } +} + +/// A PCG action, the capability changes it makes, and a debug context +/// (indicating the source of the action and possibly its effects). #[derive(Clone, Debug, PartialEq, Eq, Serialize)] #[cfg_attr(feature = "type-export", derive(ts_rs::TS))] #[cfg_attr(feature = "type-export", ts(export))] -pub struct ActionKindWithDebugInfo> { +pub struct ActionKindWithDebugInfo> { pub(crate) kind: T, + /// The capability changes of the action. Every capability change of an + /// action is listed here. + pub(crate) capabilities: CapabilityChanges

, pub(crate) debug_info: DebugInfo, } -impl ActionKindWithDebugInfo { +impl ActionKindWithDebugInfo { pub fn kind(&self) -> &T { &self.kind } + pub fn capabilities(&self) -> &CapabilityChanges

{ + &self.capabilities + } + pub(crate) fn new(kind: T, debug_info: Option) -> Self { - Self { kind, debug_info } + Self { + kind, + capabilities: CapabilityChanges::default(), + debug_info, + } + } + + pub(crate) fn with_capabilities(mut self, capabilities: CapabilityChanges

) -> Self { + self.capabilities = capabilities; + self } pub(crate) fn debug_line(&self, ctxt: Ctxt) -> Cow<'static, str> where - T: DisplayWithCtxt, + Self: DisplayWithCtxt, { - self.kind.display_output(ctxt, OutputMode::Test).into_text() + self.display_output(ctxt, OutputMode::Test).into_text() } } -impl> DebugRepr for ActionKindWithDebugInfo { - type Repr = ActionKindWithDebugInfo>; +impl, P: DisplayWithCtxt> DebugRepr + for ActionKindWithDebugInfo +{ + type Repr = ActionKindWithDebugInfo>; fn debug_repr(&self, ctxt: Ctxt) -> Self::Repr { ActionKindWithDebugInfo { kind: self.kind.debug_repr(ctxt), + capabilities: CapabilityChanges( + self.capabilities + .0 + .iter() + .map(|(place, capability)| { + ( + place + .display_output(ctxt, OutputMode::Normal) + .into_text() + .to_string(), + *capability, + ) + }) + .collect(), + ), debug_info: self.debug_info.as_ref().map(|ctxt| ctxt.debug_repr(())), } } } -impl> DisplayWithCtxt for ActionKindWithDebugInfo { - fn display_output(&self, ctxt: Ctxt, mode: OutputMode) -> DisplayOutput { - self.kind.display_output(ctxt, mode) - } -} - /// An action applied to the Owned PCG during the PCG analysis /// for which consumers (e.g. Prusti) may wish to perform /// their own effect (e.g. folding a predicate). pub type OwnedPcgAction<'tcx, P = Place<'tcx>> = - ActionKindWithDebugInfo>; + ActionKindWithDebugInfo, P>; + +impl> DisplayWithCtxt + for OwnedPcgAction<'_, P> +{ + fn display_output(&self, ctxt: Ctxt, mode: OutputMode) -> DisplayOutput { + self.kind.display_output(ctxt, mode) + } +} /// An action applied to the Borrow PCG during the PCG analysis /// for which consumers (e.g. Prusti) may wish to perform @@ -223,7 +332,7 @@ pub type BorrowPcgAction< EdgeKind = BorrowPcgEdgeKind<'tcx>, P = Place<'tcx>, VC = ValidityConditions, -> = ActionKindWithDebugInfo>; +> = ActionKindWithDebugInfo, P>; mod private { use serde_derive::Serialize; @@ -255,6 +364,12 @@ impl<'tcx, EdgeKind, P, VC> From> for PcgAction<'tcx, Ed } } +impl<'tcx, EdgeKind, VC> From> for PcgAction<'tcx, EdgeKind, Place<'tcx>, VC> { + fn from(repack: RepackOp<'tcx>) -> Self { + OwnedPcgAction::new(repack, None).into() + } +} + /// An action applied to the PCG during the PCG analysis. pub type PcgAction< 'tcx, @@ -264,8 +379,8 @@ pub type PcgAction< > = private::GenericPcgAction, OwnedPcgAction<'tcx, P>>; pub(crate) type PcgActionDebugRepr = private::GenericPcgAction< - ActionKindWithDebugInfo>, - ActionKindWithDebugInfo, Option>, + ActionKindWithDebugInfo>, + ActionKindWithDebugInfo, String, Option>, >; impl<'a, 'tcx: 'a, Ctxt: HasBorrowCheckerCtxt<'a, 'tcx>> DebugRepr for PcgAction<'tcx> { @@ -279,34 +394,11 @@ impl<'a, 'tcx: 'a, Ctxt: HasBorrowCheckerCtxt<'a, 'tcx>> DebugRepr for Pcg } } -impl<'tcx, P: PcgNodeComponent, VC> PcgAction<'tcx, BorrowPcgEdgeKind<'tcx, P>, P, VC> { - pub(crate) fn restore_capability( - place: P, - capability: CapabilityKind, - debug_context: impl Into, - ctxt: Ctxt, - ) -> Self - where - P: PlaceLike<'tcx, Ctxt>, - { - let debug_context: DisplayOutput = debug_context.into(); - if place.is_owned(ctxt) { - PcgAction::<'tcx, BorrowPcgEdgeKind<'tcx, P>, P, VC>::Owned(OwnedPcgAction { - kind: RepackOp::RegainLoanedCapability(RegainedCapability::new(place, capability)), - debug_info: Some(debug_context), - }) - } else { - BorrowPcgAction::restore_capability(place, capability, debug_context).into() - } - } -} - impl<'tcx, EdgeKind, P: PcgNodeComponent, VC> PcgAction<'tcx, EdgeKind, P, VC> { pub(crate) fn debug_line(&self, ctxt: Ctxt) -> Cow<'static, str> where - BorrowPcgActionKind<'tcx, EdgeKind, P, VC>: DisplayWithCtxt, + BorrowPcgAction<'tcx, EdgeKind, P, VC>: DisplayWithCtxt, OwnedPcgAction<'tcx, P>: DisplayWithCtxt, - P: DisplayWithCtxt, { match self { PcgAction::Borrow(action) => action.debug_line(ctxt), diff --git a/src/borrow_checker/mod.rs b/src/borrow_checker/mod.rs index 18ff82471..219b2cd41 100644 --- a/src/borrow_checker/mod.rs +++ b/src/borrow_checker/mod.rs @@ -137,33 +137,17 @@ impl<'tcx, T: RustBorrowCheckerInterface<'tcx> + DisplayCtxtFor> location: Location, ctxt: CompilerCtxt<'_, 'tcx>, ) -> bool { - let mut conflict = false; - each_borrow_involving_path( - &mut (), - ctxt, - blocked_place, - self.borrow_set(), - |borrow_index| self.borrow_in_scope_at(borrow_index, location), - |_this, borrow_index, _borrow| { - tracing::debug!( - "Checking if {} contains {:?} at {:?}", - blocking_place.display_string(ctxt), - borrow_index, - location - ); - if blocking_place - .regions(ctxt) - .iter() - .any(|region| self.origin_contains_loan_at(*region, borrow_index, location)) - { - conflict = true; - ControlFlow::Break(()) - } else { - ControlFlow::Continue(()) - } - }, - ); - conflict + holds_loan_of(self, blocking_place, blocked_place, false, location, ctxt) + } + + fn blocks_entirely( + &self, + blocking_place: Place<'tcx>, + blocked_place: Place<'tcx>, + location: Location, + ctxt: CompilerCtxt<'_, 'tcx>, + ) -> bool { + holds_loan_of(self, blocking_place, blocked_place, true, location, ctxt) } fn as_dyn(&self) -> &dyn BorrowCheckerInterface<'tcx> { @@ -187,6 +171,47 @@ impl<'tcx, T: RustBorrowCheckerInterface<'tcx> + DisplayCtxtFor> } } +/// Whether a region of `blocking_place` contains, at `location`, a loan in +/// scope that conflicts with `blocked_place`; if `entirely`, a loan of +/// `blocked_place` itself or of a place it is part of. +fn holds_loan_of<'tcx>( + bc: &impl RustBorrowCheckerInterface<'tcx>, + blocking_place: Place<'tcx>, + blocked_place: Place<'tcx>, + entirely: bool, + location: Location, + ctxt: CompilerCtxt<'_, 'tcx>, +) -> bool { + let mut conflict = false; + each_borrow_involving_path( + &mut (), + ctxt, + blocked_place, + bc.borrow_set(), + |borrow_index| bc.borrow_in_scope_at(borrow_index, location), + |_this, borrow_index, borrow| { + tracing::debug!( + "Checking if {} contains {:?} at {:?}", + blocking_place.display_string(ctxt), + borrow_index, + location + ); + if (!entirely || borrow.get_borrowed_place().is_prefix_of(blocked_place)) + && blocking_place + .regions(ctxt) + .iter() + .any(|region| bc.origin_contains_loan_at(*region, borrow_index, location)) + { + conflict = true; + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + }, + ); + conflict +} + pub trait BorrowCheckerInterface<'tcx>: DisplayCtxtFor { /* Main Interface Start */ @@ -211,6 +236,17 @@ pub trait BorrowCheckerInterface<'tcx>: DisplayCtxtFor { ctxt: CompilerCtxt<'_, 'tcx>, ) -> bool; + /// Like [`Self::blocks`], but only for a loan of `blocked_place` itself or + /// of a place it is part of: all of `blocked_place` is lent, not just one + /// of its parts. + fn blocks_entirely( + &self, + blocking_place: Place<'tcx>, + blocked_place: Place<'tcx>, + location: Location, + ctxt: CompilerCtxt<'_, 'tcx>, + ) -> bool; + /// Returns true iff `sup` is required to outlive `sub` at `location`. fn outlives(&self, sup: PcgRegion<'tcx>, sub: PcgRegion<'tcx>, location: Location) -> bool; diff --git a/src/borrow_pcg/action/actions.rs b/src/borrow_pcg/action/actions.rs index c68d10e39..a5ea53c39 100644 --- a/src/borrow_pcg/action/actions.rs +++ b/src/borrow_pcg/action/actions.rs @@ -1,11 +1,9 @@ use derive_more::{Deref, DerefMut}; use crate::{ - HasSettings, Weaken, + Weaken, borrow_pcg::{action::BorrowPcgAction, unblock_graph::BorrowPcgUnblockAction}, - pcg_validity_assert, rustc_interface::data_structures::fx::FxHashSet, - utils::DebugCtxt, }; use super::BorrowPcgActionKind; @@ -38,7 +36,7 @@ impl<'tcx> BorrowPcgActions<'tcx> { self.0 .iter() .filter_map(|action| match action.kind() { - BorrowPcgActionKind::RemoveEdge(edge) => { + BorrowPcgActionKind::RemoveEdge { edge, .. } => { Some(BorrowPcgUnblockAction::new(edge.clone())) } _ => None, @@ -46,27 +44,3 @@ impl<'tcx> BorrowPcgActions<'tcx> { .collect() } } - -impl<'tcx> BorrowPcgActions<'tcx> { - pub(crate) fn new() -> Self { - Self(vec![]) - } - - pub(crate) fn last(&self) -> Option<&BorrowPcgAction<'tcx>> { - self.0.last() - } - - pub(crate) fn push( - &mut self, - action: BorrowPcgAction<'tcx>, - ctxt: impl HasSettings<'_> + DebugCtxt, - ) { - pcg_validity_assert!( - self.last() != Some(&action), - [ctxt], - "Action {:?} to be pushed is the same as the last pushed action, this is probably a bug.", - action - ); - self.0.push(action); - } -} diff --git a/src/borrow_pcg/action/mod.rs b/src/borrow_pcg/action/mod.rs index daecfad06..a0a73cb45 100644 --- a/src/borrow_pcg/action/mod.rs +++ b/src/borrow_pcg/action/mod.rs @@ -6,10 +6,11 @@ use serde_derive::Serialize; use super::borrow_pcg_edge::BorrowPcgEdge; use crate::{ - RestoreCapability, Weaken, + Weaken, action::BorrowPcgAction, borrow_pcg::{ edge::{ + borrow::BorrowEdge, borrow_flow::private::FutureEdgeKind, kind::{BorrowPcgEdgeKind, BorrowPcgEdgeType}, }, @@ -68,63 +69,136 @@ impl ApplyActionResult { } impl<'tcx, EdgeKind, P> BorrowPcgAction<'tcx, EdgeKind, P> { + /// An `AddEdge` that changes no capability. pub(crate) fn add_edge( edge: BorrowPcgEdge<'tcx, EdgeKind>, context: impl Into, ) -> Self { - BorrowPcgAction { - kind: BorrowPcgActionKind::AddEdge { edge }, - debug_info: Some(context.into()), - } + Self::add_edge_setting(edge, vec![], context) + } + + /// An `AddEdge` that sets `capabilities` (`None` removes one). + pub(crate) fn add_edge_setting( + edge: BorrowPcgEdge<'tcx, EdgeKind>, + capabilities: Vec<(P, Option)>, + context: impl Into, + ) -> Self { + BorrowPcgAction::new(BorrowPcgActionKind::AddEdge { edge }, Some(context.into())) + .with_capabilities(capabilities.into()) } pub(crate) fn remove_edge( edge: BorrowPcgEdge<'tcx, EdgeKind>, context: impl Into, ) -> Self { - BorrowPcgAction { - kind: BorrowPcgActionKind::RemoveEdge(edge), - debug_info: Some(context.into()), - } + BorrowPcgAction::new( + BorrowPcgActionKind::RemoveEdge { edge }, + Some(context.into()), + ) + } + + pub(crate) fn lend( + place: P, + edge: BorrowPcgEdge<'tcx, EdgeKind>, + context: impl Into, + ) -> Self { + BorrowPcgAction::new( + BorrowPcgActionKind::Lend { place, edge }, + Some(context.into()), + ) + } + + pub(crate) fn weaken_suspended( + place: P, + from: CapabilityKind, + to: Option, + context: impl Into, + ) -> Self { + BorrowPcgAction::new( + BorrowPcgActionKind::WeakenSuspended { place, from, to }, + Some(context.into()), + ) + } + + pub(crate) fn unpack_suspended( + place: P, + parts: Vec

, + context: impl Into, + ) -> Self { + BorrowPcgAction::new( + BorrowPcgActionKind::UnpackSuspended { place, parts }, + Some(context.into()), + ) + } + + pub(crate) fn pack_suspended(place: P, context: impl Into) -> Self { + BorrowPcgAction::new( + BorrowPcgActionKind::PackSuspended { place }, + Some(context.into()), + ) } } impl<'tcx, P: PcgNodeComponent, VC> BorrowPcgAction<'tcx, BorrowPcgEdgeKind<'tcx, P>, P, VC> { - pub(crate) fn restore_capability( + pub(crate) fn weaken( place: P, - capability: CapabilityKind, - debug_context: impl Into, + from: CapabilityKind, + to: CapabilityKind, + context: impl Into, ) -> Self { - BorrowPcgAction::<'tcx, BorrowPcgEdgeKind<'tcx, P>, P, VC> { - kind: BorrowPcgActionKind::<'tcx, BorrowPcgEdgeKind<'tcx, P>, P, VC>::Restore( - RestoreCapability::new(place, capability), - ), - debug_info: Some(debug_context.into()), - } + BorrowPcgAction::new( + BorrowPcgActionKind::Weaken(Weaken::new(place, from, to)), + Some(context.into()), + ) } - pub(crate) fn weaken( + /// Temporary: a weaken for a `StorageDead`, until + /// is resolved. + pub(crate) fn weaken_for_storage_dead( place: P, from: CapabilityKind, - to: Option, + to: CapabilityKind, context: impl Into, ) -> Self { - BorrowPcgAction::<'tcx, BorrowPcgEdgeKind<'tcx, P>, P, VC> { - kind: BorrowPcgActionKind::<'tcx, BorrowPcgEdgeKind<'tcx, P>, P, VC>::Weaken( - Weaken::new(place, from, to), - ), - debug_info: Some(context.into()), - } + BorrowPcgAction::new( + BorrowPcgActionKind::Weaken(Weaken::new_for_storage_dead(place, from, to)), + Some(context.into()), + ) + } + + pub(crate) fn freeze(place: P, context: impl Into) -> Self { + BorrowPcgAction::new(BorrowPcgActionKind::Freeze { place }, Some(context.into())) + } + + pub(crate) fn unfreeze(place: P, context: impl Into) -> Self { + Self::refreeze(place, vec![], context) + } + + pub(crate) fn refreeze(place: P, refrozen: Vec

, context: impl Into) -> Self { + BorrowPcgAction::new( + BorrowPcgActionKind::Unfreeze { place, refrozen }, + Some(context.into()), + ) + } + + pub(crate) fn activate_two_phase_borrow( + borrow: BorrowEdge<'tcx, P>, + context: impl Into, + ) -> Self { + BorrowPcgAction::new( + BorrowPcgActionKind::ActivateTwoPhaseBorrow { borrow }, + Some(context.into()), + ) } pub(crate) fn remove_lifetime_projection_label( projection: LifetimeProjection<'tcx, MaybeLabelledPlace<'tcx, P>>, context: impl Into, ) -> Self { - BorrowPcgAction::<'tcx, BorrowPcgEdgeKind<'tcx, P>, P, VC> { - kind: BorrowPcgActionKind::<'tcx, BorrowPcgEdgeKind<'tcx, P>, P, VC>::remove_lifetime_projection_label(projection), - debug_info: Some(context.into()), - } + BorrowPcgAction::new( + BorrowPcgActionKind::remove_lifetime_projection_label(projection), + Some(context.into()), + ) } pub(crate) fn label_lifetime_projection( @@ -132,10 +206,10 @@ impl<'tcx, P: PcgNodeComponent, VC> BorrowPcgAction<'tcx, BorrowPcgEdgeKind<'tcx label: Option, context: impl Into, ) -> Self { - BorrowPcgAction::<'tcx, BorrowPcgEdgeKind<'tcx, P>, P, VC> { - kind: BorrowPcgActionKind::<'tcx, BorrowPcgEdgeKind<'tcx, P>, P, VC>::label_lifetime_projection(predicate, label), - debug_info: Some(context.into()), - } + BorrowPcgAction::new( + BorrowPcgActionKind::label_lifetime_projection(predicate, label), + Some(context.into()), + ) } pub(crate) fn label_place_and_update_related_capabilities( @@ -143,12 +217,10 @@ impl<'tcx, P: PcgNodeComponent, VC> BorrowPcgAction<'tcx, BorrowPcgEdgeKind<'tcx location: SnapshotLocation, reason: LabelPlaceReason, ) -> Self { - BorrowPcgAction::<'tcx, BorrowPcgEdgeKind<'tcx, P>, P, VC> { - kind: BorrowPcgActionKind::<'tcx, BorrowPcgEdgeKind<'tcx, P>, P, VC>::LabelPlace( - LabelPlaceAction::new(place, location, reason), - ), - debug_info: None, - } + BorrowPcgAction::new( + BorrowPcgActionKind::LabelPlace(LabelPlaceAction::new(place, location, reason)), + None, + ) } } @@ -159,18 +231,16 @@ pub type MakePlaceOldReason = LabelPlaceReason; pub enum LabelPlaceReason { StorageDead, MoveOut, - /// When joining an owned place where one place has capability Read and the - /// other has capability Write, all borrowed expansions of the place with - /// Read capability should be labelled. - /// - /// In particular, the Read capability place could be a shared reference that is dereferenced. - /// The place will not have any capability after the join, but reborrows derived from the dereference - /// may still have capability. - JoinOwnedReadAndWriteCapabilities, Write, Collapse, /// Label lifetime projections of places that are mutated in a loop. JoinLoop, + /// The value of the place is detached from it: the place and its postfixes + /// lose their capabilities, and the borrows of the value keep the + /// labelled place. This detaches the referent of a shared reference from + /// the place holding the reference, and the value of a place that the + /// joined state does not hold from the place. + Detach, } impl LabelPlaceReason { @@ -188,9 +258,7 @@ impl LabelPlaceReason { ]), LabelPlaceReason::StorageDead | LabelPlaceReason::MoveOut - | LabelPlaceReason::JoinOwnedReadAndWriteCapabilities => { - LabelNodePredicate::PlaceIsPostfixOf(place) - } + | LabelPlaceReason::Detach => LabelNodePredicate::PlaceIsPostfixOf(place), LabelPlaceReason::Write => LabelNodePredicate::And(vec![ LabelNodePredicate::PlaceIsPostfixOf(place), // If the place is e.g *x or x.f, don't label it if it @@ -355,8 +423,51 @@ pub enum BorrowPcgActionKind< VC = ValidityConditions, > { LabelLifetimeProjection(LabelLifetimeProjectionAction<'tcx, P>), + /// The value of the place is given up: its capability goes down to the + /// strictly lower `to`. Weaken(Weaken<'tcx, P>), - Restore(RestoreCapability<'tcx, P>), + /// The place goes from E to R because it is read or borrowed for read. + /// Unlike a [`Weaken`], the E capability is only suspended: it is + /// recorded, and comes back through [`BorrowPcgActionKind::Unfreeze`] of + /// the same place, or is lent by + /// [`BorrowPcgActionKind::ActivateTwoPhaseBorrow`]. Until then the place + /// holds at most R. + /// + /// If the place is unpacked, it has no capability and its descendants + /// hold the E or are frozen; they all become read copies under the + /// frozen place, which records what they hold together. + /// + /// A place with R whose parent has R is a read copy of a frozen + /// ancestor, not frozen itself. + Freeze { + place: P, + }, + /// The freeze of `place` ends: it gets back the capability recorded by + /// [`BorrowPcgActionKind::Freeze`]. The descendants `refrozen` are still + /// borrowed: the freeze moves to them and they stay frozen. + /// + /// If `place` is unpacked, the read unpacking becomes an exclusive one: + /// `place` and the places on the way to `refrozen` keep no capability + /// (`e` for a mutable reference, R for a shared reference, which stays + /// frozen), and the other read copies get what is recorded for them or + /// the place they are copies of (R if they are behind a shared + /// reference). + Unfreeze { + place: P, + refrozen: Vec

, + }, + /// The two-phase borrow is activated: its blocked place, which is frozen, + /// goes from R to lent, i.e. the capability suspended by its freeze is + /// lent to the borrow. A blocked place that is a read copy of a frozen + /// ancestor is refrozen first ([`BorrowPcgActionKind::Unfreeze`] of the + /// ancestor, with the place in `refrozen`), so no ancestor keeps R. + ActivateTwoPhaseBorrow { + borrow: BorrowEdge<'tcx, P>, + }, + /// The current value of the place is labelled: the edges referring to the + /// place (and, depending on the reason, its postfixes) now refer to the + /// labelled place, and the current places lose the capabilities that + /// were part of the labelled value. LabelPlace(LabelPlaceAction<'tcx, P>), /// Remove an edge from the PCG. In terms of the PCG itself, the validity /// conditions associated with the edge are not relevant (there is no @@ -365,10 +476,62 @@ pub enum BorrowPcgActionKind< /// symbolic-execution based Prusti purification already performs some /// filtering on edges based on validity conditions and might want to ignore /// removal actions for edges that it already ignored. - RemoveEdge(BorrowPcgEdge<'tcx, EdgeKind, VC>), + /// + /// The places the edge blocked by that no other edge refers to lose their + /// capabilities, and the places it blocked that are unblocked get theirs + /// back: the edge was an unpacking (its base gets what its expansion + /// holds) or a loan (the lent place gets exactly what was recorded when + /// it was lent). + RemoveEdge { + edge: BorrowPcgEdge<'tcx, EdgeKind, VC>, + }, + /// Add an edge to the PCG. An unpacking (a borrow-PCG expansion or a + /// dereference) moves the capability of its base to its expansion, and a + /// mutable borrow takes the capability of the borrowed place, which is + /// recorded. AddEdge { edge: BorrowPcgEdge<'tcx, EdgeKind, VC>, }, + /// At a join, `place` is blocked by `edge`, which exists only on another + /// incoming path (it blocks `place`, a part of it, or the place it is + /// part of), or is a loan of this path that blocks `place` and keeps it + /// after the join. The capability of `place` is held by the edge (nothing is + /// given up): it is recorded for `place`, a frozen place keeping what its + /// freeze records and lending its read copies with it, and returns + /// through the [`BorrowPcgActionKind::RemoveEdge`] of the edge. + Lend { + place: P, + edge: BorrowPcgEdge<'tcx, EdgeKind, VC>, + }, + /// The capability that the lent or frozen `place` gets back when its loan + /// or freeze ends goes down from `from` to `to`: at a join, to the meet of + /// the incoming paths, or to none if another path does not suspend it; at + /// a loop head, to W if `place` may be uninitialized there. The + /// capability `place` holds does not change. + WeakenSuspended { + place: P, + from: CapabilityKind, + to: Option, + }, + /// The capability suspended for the lent `place` is split into `parts`, + /// one level of its unpacking, as a capability `place` held would be: each + /// part of its value gets the same, the referent of a mutable reference + /// gets E and the reference keeps e. A part that records less keeps that. + /// At a join, the paths then record the same places; when a loan ends, + /// the record goes back along the layout of `place`. + UnpackSuspended { + place: P, + parts: Vec

, + }, + /// The capabilities suspended for the parts of `place`'s value are joined + /// into one for `place`, their meet with what `place` records; a mutable + /// reference recording e whose referent records E records E (the inverse + /// of [`BorrowPcgActionKind::UnpackSuspended`]). At a join, the paths + /// unpacked `place` differently (different variants of an enum); when + /// parts frozen apart are packed, their freezes join in `place`. + PackSuspended { + place: P, + }, } impl<'tcx, P: PcgNodeComponent, VC> BorrowPcgActionKind<'tcx, BorrowPcgEdgeKind<'tcx, P>, P, VC> { @@ -416,11 +579,37 @@ impl<'a, 'tcx: 'a, Ctxt: HasBorrowCheckerCtxt<'a, 'tcx>, EdgeKind: DisplayWithCt action.display_output(ctxt, mode) } BorrowPcgActionKind::Weaken(weaken) => weaken.display_output(ctxt, mode), - BorrowPcgActionKind::Restore(restore_capability) => { - restore_capability.display_output(ctxt, mode) + BorrowPcgActionKind::Freeze { place } => DisplayOutput::join( + vec!["Freeze".into(), place.display_output(ctxt, mode)], + &DisplayOutput::SPACE, + ), + BorrowPcgActionKind::Unfreeze { place, refrozen } if refrozen.is_empty() => { + DisplayOutput::join( + vec!["Unfreeze".into(), place.display_output(ctxt, mode)], + &DisplayOutput::SPACE, + ) } + BorrowPcgActionKind::Unfreeze { place, refrozen } => DisplayOutput::join( + vec![ + "Refreeze".into(), + place.display_output(ctxt, mode), + "to".into(), + DisplayOutput::join( + refrozen.iter().map(|p| p.display_output(ctxt, mode)), + &", ".into(), + ), + ], + &DisplayOutput::SPACE, + ), + BorrowPcgActionKind::ActivateTwoPhaseBorrow { borrow } => DisplayOutput::join( + vec![ + "Activate two-phase borrow".into(), + borrow.display_output(ctxt, mode), + ], + &DisplayOutput::SPACE, + ), BorrowPcgActionKind::LabelPlace(action) => action.display_output(ctxt, mode), - BorrowPcgActionKind::RemoveEdge(edge) => DisplayOutput::join( + BorrowPcgActionKind::RemoveEdge { edge } => DisplayOutput::join( vec!["Remove Edge".into(), edge.display_output(ctxt, mode)], &DisplayOutput::SPACE, ), @@ -428,6 +617,86 @@ impl<'a, 'tcx: 'a, Ctxt: HasBorrowCheckerCtxt<'a, 'tcx>, EdgeKind: DisplayWithCt vec!["Add Edge".into(), edge.display_output(ctxt, mode)], &DisplayOutput::SPACE, ), + BorrowPcgActionKind::Lend { place, edge } => DisplayOutput::join( + vec![ + "Lend".into(), + place.display_output(ctxt, mode), + "to".into(), + edge.display_output(ctxt, mode), + ], + &DisplayOutput::SPACE, + ), + BorrowPcgActionKind::WeakenSuspended { place, from, to } => DisplayOutput::join( + vec![ + "Weaken suspended".into(), + place.display_output(ctxt, mode), + "from".into(), + from.display_output(ctxt, mode), + "to".into(), + match to { + Some(to) => to.display_output(ctxt, mode), + None => "None".into(), + }, + ], + &DisplayOutput::SPACE, + ), + BorrowPcgActionKind::UnpackSuspended { place, parts } => DisplayOutput::join( + vec![ + "Unpack suspended".into(), + place.display_output(ctxt, mode), + "to".into(), + DisplayOutput::join( + parts.iter().map(|p| p.display_output(ctxt, mode)), + &", ".into(), + ), + ], + &DisplayOutput::SPACE, + ), + BorrowPcgActionKind::PackSuspended { place } => DisplayOutput::join( + vec!["Pack suspended".into(), place.display_output(ctxt, mode)], + &DisplayOutput::SPACE, + ), } } } + +/// Shows the capability changes that the kind does not determine on its own: +/// the capabilities an added edge sets, and the ones a removed edge returns. +impl<'a, 'tcx: 'a, Ctxt: HasBorrowCheckerCtxt<'a, 'tcx>, EdgeKind: DisplayWithCtxt> + DisplayWithCtxt for BorrowPcgAction<'tcx, EdgeKind> +{ + fn display_output(&self, ctxt: Ctxt, mode: OutputMode) -> DisplayOutput { + let kind = self.kind.display_output(ctxt, mode); + let gained = self + .capabilities + .iter() + .filter_map(|(place, cap)| Some((place, cap?))) + .collect::>(); + let suffix = match &self.kind { + BorrowPcgActionKind::AddEdge { .. } if !self.capabilities.is_empty() => { + vec![ + "setting".into(), + self.capabilities.display_output(ctxt, mode), + ] + } + BorrowPcgActionKind::RemoveEdge { .. } if !gained.is_empty() => { + vec![DisplayOutput::join( + gained.into_iter().map(|(place, cap)| { + DisplayOutput::join( + vec![ + "returning".into(), + cap.display_output(ctxt, mode), + "to".into(), + place.display_output(ctxt, mode), + ], + &DisplayOutput::SPACE, + ) + }), + &", ".into(), + )] + } + _ => return kind, + }; + DisplayOutput::join(std::iter::once(kind).chain(suffix), &DisplayOutput::SPACE) + } +} diff --git a/src/borrow_pcg/borrow_pcg_expansion.rs b/src/borrow_pcg/borrow_pcg_expansion.rs index 2fa22e125..58ca77945 100644 --- a/src/borrow_pcg/borrow_pcg_expansion.rs +++ b/src/borrow_pcg/borrow_pcg_expansion.rs @@ -82,10 +82,6 @@ impl HasValidityCheck for PlaceExpansion<'_> { } impl PlaceExpansion<'_, D> { - pub(crate) fn is_enum_expansion(&self) -> bool { - matches!(self, PlaceExpansion::Guided(RepackGuide::Downcast(..))) - } - pub(crate) fn is_deref(&self) -> bool { matches!(self, PlaceExpansion::Deref(_)) } diff --git a/src/borrow_pcg/edge/borrow_flow.rs b/src/borrow_pcg/edge/borrow_flow.rs index 856a47e9f..afc6e177d 100644 --- a/src/borrow_pcg/edge/borrow_flow.rs +++ b/src/borrow_pcg/edge/borrow_flow.rs @@ -133,6 +133,11 @@ where if short_matches { changed |= self.target.label_lifetime_projection(label); } + // Labelling one end can make it the other end: a flow from a node to + // itself says nothing. + if self.source == self.target.rebase() { + return LabelLifetimeProjectionResult::ShouldCollapse; + } self.assert_validity(ctxt); changed } diff --git a/src/borrow_pcg/edge/kind.rs b/src/borrow_pcg/edge/kind.rs index e7e27ef5d..8f087cb7b 100644 --- a/src/borrow_pcg/edge/kind.rs +++ b/src/borrow_pcg/edge/kind.rs @@ -26,6 +26,20 @@ pub enum BorrowPcgEdgeKind<'tcx, P = Place<'tcx>> { RawPtrDeref(RawPtrDerefEdge<'tcx, P>), } +impl

BorrowPcgEdgeKind<'_, P> { + /// Whether the edge lends the places it blocks (a borrow, a call, a loop), + /// rather than unpacking them or aliasing them by a raw pointer. + pub(crate) fn is_loan(&self) -> bool { + !matches!( + self, + Self::Deref(_) + | Self::RawPtrDeref(_) + | Self::BorrowPcgExpansion(_) + | Self::Delegation(_) + ) + } +} + #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] pub enum BorrowPcgEdgeType { Borrow, diff --git a/src/borrow_pcg/graph/join.rs b/src/borrow_pcg/graph/join.rs index b074601ce..8b1582792 100644 --- a/src/borrow_pcg/graph/join.rs +++ b/src/borrow_pcg/graph/join.rs @@ -11,7 +11,7 @@ use crate::{ state::BorrowStateRef, validity_conditions::ValidityConditions, }, - r#loop::{PlaceUsage, PlaceUsages}, + r#loop::{LoopId, PlaceUsage, PlaceUsages}, pcg::{ BodyAnalysis, CapabilityKind, PcgNode, PcgNodeLike, PcgRef, PcgRefLike, ctxt::AnalysisCtxt, @@ -21,9 +21,12 @@ use crate::{ }, }, pcg_validity_assert, - rustc_interface::middle::mir::{self, BasicBlock}, + rustc_interface::{ + borrowck::TwoPhaseActivation, + middle::mir::{self, BasicBlock}, + }, utils::{ - DebugImgcat, HasBorrowCheckerCtxt, SnapshotLocation, + DebugImgcat, HasBorrowCheckerCtxt, Place, PlaceLike, SnapshotLocation, data_structures::HashSet, display::DisplayWithCtxt, logging::{self, LogPredicate}, @@ -284,16 +287,18 @@ impl<'tcx> BorrowsGraph<'tcx> { live_loop_places.display_string(ctxt.ctxt) ); - let loop_blocked_places = live_loop_places.usages_where(|p| { - ctxt.ctxt.borrow_checker.is_directly_blocked( - p.place, - mir::Location { - block: args.self_block, - statement_index: 0, - }, - ctxt.ctxt, - ) - }); + let loop_blocked_places = live_loop_places + .usages_where(|p| { + ctxt.ctxt.borrow_checker.is_directly_blocked( + p.place, + mir::Location { + block: args.self_block, + statement_index: 0, + }, + ctxt.ctxt, + ) + }) + .outside_variants(ctxt.ctxt); logging::log!( &LogPredicate::DebugBlock, @@ -360,7 +365,11 @@ impl<'tcx> BorrowsGraph<'tcx> { .loop_analysis .loop_head_of(loop_head) .unwrap(); - let invariant_capabilities = ctxt.body_analysis.loop_invariant_capabilities(loop_id); + let invariant_capabilities = ctxt + .body_analysis + .loop_invariant_capabilities(loop_id) + .outside_variants(ctxt.ctxt) + .consolidate(); logging::log!( &LogPredicate::DebugBlock, @@ -370,17 +379,19 @@ impl<'tcx> BorrowsGraph<'tcx> { ); self.unpack_for_loop_invariant_capabilities( - &loop_blocked_places, - invariant_capabilities, + &invariant_capabilities, validity_conditions, &mut args.reborrow(), ctxt, ); - let expand_places = loop_blocker_places.joined_with(&loop_blocked_places); + // A blocker in a variant keeps the lifetime projection that holds the + // loan; the loop head holds the blocker at the enum. + let expand_places = loop_blocker_places + .outside_variants(ctxt.ctxt) + .joined_with(&loop_blocked_places); self.expand_places_for_abstraction( - &loop_blocked_places, &expand_places, validity_conditions, &mut args.reborrow(), @@ -440,7 +451,6 @@ impl<'tcx> BorrowsGraph<'tcx> { let ConstructAbstractionGraphResult { graph: mut abstraction_graph, to_label, - capability_updates, blocked_root_places, } = self.get_loop_abstraction_graph( &loop_blocked_places, @@ -476,14 +486,6 @@ impl<'tcx> BorrowsGraph<'tcx> { ); } - for (place, cap_option) in capability_updates { - if let Some(cap) = cap_option { - capabilities.insert(place, cap, ctxt); - } else { - capabilities.remove(place, ctxt); - } - } - // For each root place that is actually blocked by the loop abstraction, // downgrade mutable-reference ancestors from E to W. Without this, // `variant: E` and `(*variant).fields: E` can coexist once the @@ -579,6 +581,7 @@ impl<'tcx> BorrowsGraph<'tcx> { capabilities.remove(place, ctxt); } } + self.restrict_to_loans(capabilities, loop_head, loop_id, ctxt); self.render_debug_graph( loop_head, Some(DebugImgcat::JoinLoop), @@ -587,4 +590,95 @@ impl<'tcx> BorrowsGraph<'tcx> { ctxt.ctxt, ); } + + /// Lowers the capabilities of the loop head to what the loans of this + /// graph leave, for each place in the value of a lent place or that a + /// lent place is part of (short of a reference to it). A place that a + /// mutable borrow in scope here covers (it borrows the place or a place + /// it is part of) holds nothing. A place a mutable borrow takes a part of + /// keeps the R of a read unpacking and holds nothing if it holds more. + /// Otherwise, a place holds at most R. Each place records what it held, + /// which comes back once the loan ends in the loop. + fn restrict_to_loans( + &self, + capabilities: &mut PlaceCapabilities<'tcx>, + loop_head: BasicBlock, + loop_id: LoopId, + ctxt: AnalysisCtxt<'_, 'tcx>, + ) { + // A two-phase borrow is mutable once activated: on every path to the + // head, or on the back edge if the loop contains the activation. + // Until then it is reserved, and shared. + let is_active_here = |activation| match activation { + TwoPhaseActivation::NotTwoPhase => true, + TwoPhaseActivation::NotActivated => false, + TwoPhaseActivation::ActivatedAt(activation) => { + ctxt.body() + .basic_blocks + .dominators() + .dominates(activation.block, loop_head) + || ctxt + .body_analysis + .loop_analysis + .in_loop(activation.block, loop_id) + } + }; + let location = mir::Location { + block: loop_head, + statement_index: 0, + }; + let lent = self + .lent_places(ctxt.bc_ctxt()) + .into_iter() + .filter(|lent| !lent.contains_unsafe_deref(ctxt.ctxt)) + .collect::>(); + let restricted = capabilities + .iter() + .filter(|(place, _)| { + !place.contains_unsafe_deref(ctxt.ctxt) + && lent.iter().any(|lent| { + lent.is_prefix_of(*place) + || place.is_prefix_of(*lent) + && !lent.projects_indirection_from(*place, ctxt.ctxt) + }) + }) + .filter_map(|(place, held)| { + let (covered, part_taken) = ctxt + .bc() + .borrows_blocking(place, location, ctxt.bc_ctxt()) + .iter() + .filter(|borrow| { + borrow.kind().mutability().is_mut() + && is_active_here(borrow.activation_location()) + }) + .map(|borrow| Place::from(borrow.borrowed_place())) + .fold((false, false), |(covered, part_taken), borrowed| { + ( + covered || borrowed.is_prefix_of(place), + part_taken + || place.is_strict_prefix_of(borrowed) + && !borrowed.projects_indirection_from(place, ctxt.ctxt), + ) + }); + let lent_with = covered || part_taken && !held.is_read(); + let allowed = (!lent_with).then_some(CapabilityKind::Read); + (lent_with + || matches!( + held, + CapabilityKind::Exclusive | CapabilityKind::ShallowExclusive + )) + .then_some((place, held, allowed)) + }) + .collect::>(); + for (place, held, allowed) in restricted { + if !held.is_read() { + capabilities.suspend(place, held); + } + if let Some(allowed) = allowed { + capabilities.insert(place, allowed, ctxt); + } else { + capabilities.remove(place, ctxt); + } + } + } } diff --git a/src/borrow_pcg/graph/loop_abstraction.rs b/src/borrow_pcg/graph/loop_abstraction.rs index 022a75105..e1d6b2795 100644 --- a/src/borrow_pcg/graph/loop_abstraction.rs +++ b/src/borrow_pcg/graph/loop_abstraction.rs @@ -27,15 +27,16 @@ use crate::{ ActionApplier, HasSnapshotLocation, ObtainType, PlaceObtainer, RenderDebugGraph, expand::PlaceExpander, }, - place_capabilities::{PlaceCapabilities, PlaceCapabilitiesInterface}, + place_capabilities::{ + PlaceCapabilities, PlaceCapabilitiesInterface, PlaceCapabilitiesReader, + }, }, pcg_validity_assert, rustc_interface::middle::mir, utils::{ CompilerCtxt, DebugImgcat, HasBorrowCheckerCtxt, HasCompilerCtxt, Place, SnapshotLocation, - data_structures::{HashMap, HashSet}, + data_structures::HashSet, display::{DisplayOutput, DisplayWithCompilerCtxt, DisplayWithCtxt, OutputMode}, - logging::{self, LogPredicate}, maybe_old::MaybeLabelledPlace, remote::RemotePlace, }, @@ -44,7 +45,6 @@ use crate::{ pub(crate) struct ConstructAbstractionGraphResult<'tcx> { pub(crate) graph: BorrowsGraph<'tcx>, pub(crate) to_label: HashSet>, - pub(crate) capability_updates: HashMap, Option>, /// Local root places that are actually blocked by the loop abstraction. /// Their mutable-reference ancestors must be downgraded from E to W. pub(crate) blocked_root_places: HashSet>, @@ -54,13 +54,11 @@ impl<'tcx> ConstructAbstractionGraphResult<'tcx> { pub(crate) fn new( graph: BorrowsGraph<'tcx>, to_label: HashSet>, - capability_updates: HashMap, Option>, blocked_root_places: HashSet>, ) -> Self { Self { graph, to_label, - capability_updates, blocked_root_places, } } @@ -133,7 +131,6 @@ impl<'tcx> BorrowsGraph<'tcx> { ctxt: AnalysisCtxt<'mir, 'tcx>, ) -> ConstructAbstractionGraphResult<'tcx> { let mut graph = BorrowsGraph::default(); - let mut capability_updates = HashMap::default(); let mut to_label = HashSet::default(); let mut blocked_root_places: HashSet> = HashSet::default(); @@ -157,22 +154,35 @@ impl<'tcx> BorrowsGraph<'tcx> { for blocker in candidate_blockers.iter_places() { for root in root_places { let relevant_root = root.relevant_place_for_blocking(); - if blocker == relevant_root - || ctxt.ctxt.borrow_checker.blocks( + let lends_root = blocker == relevant_root + || ctxt.bc().blocks_entirely( blocker, relevant_root, loop_head_location, ctxt.bc_ctxt(), - ) + ); + if lends_root + || ctxt + .bc() + .blocks(blocker, relevant_root, loop_head_location, ctxt.bc_ctxt()) { tracing::debug!( "{} blocks root {}", blocker.display_string(ctxt), relevant_root.display_string(ctxt) ); - add_block_edges(&mut expander, *root, blocker, ctxt); + if lends_root { + add_block_edges(&mut expander, *root, blocker, ctxt); + } else { + // The blocker borrows only a part of the root, which + // the loop lends apart; the root's lifetimes flow into + // the blocker. + add_rp_block_edges(&mut expander, *root, blocker, ctxt); + } if let MaybeRemoteCurrentPlace::Local(root) = root { - blocked_root_places.insert(*root); + if lends_root { + blocked_root_places.insert(*root); + } for rp in root.lifetime_projections(ctxt) { to_label.insert(LabelNodePredicate::all_non_future( (*root).into(), @@ -204,11 +214,6 @@ impl<'tcx> BorrowsGraph<'tcx> { for blocker in &blockers { add_block_edges(&mut expander, blocked_place.into(), *blocker, ctxt); } - if blocked_place_usage.usage.is_read() { - capability_updates.insert(blocked_place, Some(CapabilityKind::Read)); - } else { - capability_updates.insert(blocked_place, None); - } for rp in blocked_place.lifetime_projections(ctxt) { to_label.insert(LabelNodePredicate::all_non_future( blocked_place.into(), @@ -222,10 +227,7 @@ impl<'tcx> BorrowsGraph<'tcx> { "Abstraction graph after connections expansion", ); - expander.expand_to_places( - loop_blocked_places, - &loop_blocked_places.joined_with(candidate_blockers), - ); + expander.expand_to_places(&loop_blocked_places.joined_with(candidate_blockers)); expander.render_debug_graph( Some(DebugImgcat::JoinLoop), @@ -319,21 +321,7 @@ impl<'tcx> BorrowsGraph<'tcx> { graph.remove(kind); } tracing::debug!("Completed loop abstraction"); - for (place, capability) in &capability_updates { - logging::log!( - &LogPredicate::DebugBlock, - ctxt, - "capability update for {}: {:?}", - place.display_string(ctxt), - capability - ); - } - ConstructAbstractionGraphResult::new( - graph, - to_label, - capability_updates, - blocked_root_places, - ) + ConstructAbstractionGraphResult::new(graph, to_label, blocked_root_places) } pub(crate) fn get_borrow_roots( @@ -404,33 +392,37 @@ impl<'tcx> BorrowsGraph<'tcx> { /// to a place that has already been downgraded for a read. pub(crate) fn unpack_for_loop_invariant_capabilities<'a>( &mut self, - loop_blocked_places: &PlaceUsages<'tcx>, invariant_capabilities: &PlaceUsages<'tcx>, validity_conditions: &ValidityConditions, args: &mut JoinBorrowsArgs<'_, 'a, 'tcx>, ctxt: AnalysisCtxt<'a, 'tcx>, ) { self.obtain_places_at_loop_head( - loop_blocked_places, &invariant_capabilities.loop_head_unpack_order(), validity_conditions, args, ctxt, ); - for (place, capability) in args.capabilities.iter_mut() { - if capability.is_exclusive() - && invariant_capabilities - .iter() - .any(|usage| usage.usage.is_read() && usage.place.is_prefix_of(*place)) - { - *capability = CapabilityKind::Read; - } + // The places the loop only reads are frozen at the loop head. + let frozen = args + .capabilities + .iter() + .filter(|(place, capability)| { + capability.is_exclusive() + && invariant_capabilities + .iter() + .any(|usage| usage.usage.is_read() && usage.place.is_prefix_of(*place)) + }) + .collect::>(); + for (place, capability) in frozen { + args.capabilities.suspend(place, capability); + args.capabilities + .insert(place, CapabilityKind::Read, ctxt.ctxt); } } pub(crate) fn expand_places_for_abstraction<'a>( &mut self, - loop_blocked_places: &PlaceUsages<'tcx>, to_expand: &PlaceUsages<'tcx>, validity_conditions: &ValidityConditions, args: &mut JoinBorrowsArgs<'_, 'a, 'tcx>, @@ -447,18 +439,11 @@ impl<'tcx> BorrowsGraph<'tcx> { to_obtain.retain(|p| !place_usage.place.is_prefix_of(p.place)); to_obtain.push(place_usage); } - self.obtain_places_at_loop_head( - loop_blocked_places, - &to_obtain, - validity_conditions, - args, - ctxt, - ); + self.obtain_places_at_loop_head(&to_obtain, validity_conditions, args, ctxt); } fn obtain_places_at_loop_head<'a>( &mut self, - loop_blocked_places: &PlaceUsages<'tcx>, to_obtain: &[PlaceUsage<'tcx>], validity_conditions: &ValidityConditions, args: &mut JoinBorrowsArgs<'_, 'a, 'tcx>, @@ -480,7 +465,6 @@ impl<'tcx> BorrowsGraph<'tcx> { obtainer.render_debug_graph(Some(DebugImgcat::JoinLoop), "Before obtaining (self)"); for place_usage in to_obtain { let obtain_type = ObtainType::LoopInvariant { - is_blocked: loop_blocked_places.contains(place_usage.place), usage_type: place_usage.usage, }; obtainer.obtain(place_usage.place, obtain_type).unwrap(); @@ -654,13 +638,13 @@ impl<'tcx> AbsExpander<'_, '_, 'tcx> { } } - fn expand_to_places(&mut self, blocked_places: &PlaceUsages<'tcx>, places: &PlaceUsages<'tcx>) { + fn expand_to_places(&mut self, places: &PlaceUsages<'tcx>) { for place in places.iter() { tracing::debug!("loop expanding to {}", place.display_string(self.ctxt)); self.expand_to( place.place, + None, ObtainType::LoopInvariant { - is_blocked: blocked_places.contains(place.place), usage_type: place.usage, }, self.ctxt, @@ -675,7 +659,7 @@ impl<'tcx> ActionApplier<'tcx> for AbsExpander<'_, '_, 'tcx> { tracing::debug!("applying action: {}", action.debug_line(self.ctxt)); match action { PcgAction::Borrow(action) => match action.kind { - BorrowPcgActionKind::AddEdge { edge } => { + BorrowPcgActionKind::AddEdge { edge, .. } => { self.graph.insert(edge, self.ctxt); } BorrowPcgActionKind::LabelLifetimeProjection(action) => { @@ -685,7 +669,10 @@ impl<'tcx> ActionApplier<'tcx> for AbsExpander<'_, '_, 'tcx> { self.ctxt, ); } - BorrowPcgActionKind::RemoveEdge(borrow_pcg_edge) => { + BorrowPcgActionKind::RemoveEdge { + edge: borrow_pcg_edge, + .. + } => { self.graph.remove(borrow_pcg_edge.kind()); } _ => unreachable!(), @@ -710,21 +697,21 @@ impl<'mir, 'tcx> PlaceExpander<'mir, 'tcx> for AbsExpander<'_, 'mir, 'tcx> { true } - fn update_capabilities_for_borrow_expansion( - &mut self, + fn borrow_expansion_capabilities( + &self, _expansion: &borrow_pcg::borrow_pcg_expansion::BorrowPcgExpansion<'tcx>, _block_type: pcg::place_capabilities::BlockType, _ctxt: CompilerCtxt<'_, 'tcx>, - ) -> bool { - true + ) -> Vec<(Place<'tcx>, Option)> { + vec![] } - fn update_capabilities_for_deref( - &mut self, + fn deref_capabilities( + &self, _ref_place: Place<'tcx>, _capability: CapabilityKind, _ctxt: CompilerCtxt<'_, 'tcx>, - ) -> bool { - true + ) -> Vec<(Place<'tcx>, Option)> { + vec![] } fn location(&self) -> mir::Location { diff --git a/src/borrow_pcg/graph/mod.rs b/src/borrow_pcg/graph/mod.rs index 8412aa5a5..ba3e7192c 100644 --- a/src/borrow_pcg/graph/mod.rs +++ b/src/borrow_pcg/graph/mod.rs @@ -35,7 +35,7 @@ use frozen::{CachedLeafEdges, FrozenGraphRef}; use itertools::Itertools; use super::{ - borrow_pcg_edge::{BlockedNode, BorrowPcgEdge, BorrowPcgEdgeLike, BorrowPcgEdgeRef, LocalNode}, + borrow_pcg_edge::{BlockedNode, BorrowPcgEdge, BorrowPcgEdgeRef, LocalNode}, edge::borrow::BorrowEdge, edge_data::EdgeData, validity_conditions::ValidityConditions, @@ -174,6 +174,22 @@ impl<'tcx, EdgeKind, VC> BorrowsGraph<'tcx, EdgeKind, VC> { } impl<'tcx, P: PcgNodeComponent, VC> BorrowsGraph<'tcx, BorrowPcgEdgeKind<'tcx, P>, VC> { + /// The current places that loans block. + pub(crate) fn lent_places(&self, ctxt: Ctxt) -> HashSet

+ where + BorrowPcgEdgeKind<'tcx, P>: EdgeData<'tcx, Ctxt, P>, + { + self.edges + .keys() + .filter(|kind| kind.is_loan()) + .flat_map(|kind| { + kind.blocked_nodes(ctxt) + .filter_map(|node| node.as_current_place()) + .collect::>() + }) + .collect() + } + pub(crate) fn abstraction_edge_kinds<'slf>( &'slf self, ) -> impl Iterator> + 'slf { @@ -381,18 +397,6 @@ impl<'tcx> BorrowsGraph<'tcx> { .map(|(kind, conditions)| BorrowPcgEdge::new(kind, conditions)) } - pub(crate) fn abstraction_edges<'slf>( - &'slf self, - ) -> impl Iterator>> + 'slf { - self.edges().filter_map(|edge| match edge.kind { - BorrowPcgEdgeKind::Abstraction(abstraction) => Some(Conditioned { - conditions: edge.conditions().clone(), - value: abstraction, - }), - _ => None, - }) - } - /// All edges that are not blocked by any other edge. The argument /// `blocking_map` can be provided to use a shared cache for computation /// of blocking calculations. The argument should be used if this function diff --git a/src/borrow_pcg/state/mod.rs b/src/borrow_pcg/state/mod.rs index 0927c101a..28e1b7887 100644 --- a/src/borrow_pcg/state/mod.rs +++ b/src/borrow_pcg/state/mod.rs @@ -6,7 +6,7 @@ use crate::{ HasSettings, borrow_pcg::{ action::ApplyActionResult, - borrow_pcg_edge::{BlockingNode, LocalNode}, + borrow_pcg_edge::BlockingNode, edge_data::{LabelEdgeLifetimeProjections, LabelEdgePlaces, display_node_replacements}, graph::join::JoinBorrowsArgs, region_projection::PcgLifetimeProjectionBase, @@ -14,10 +14,7 @@ use crate::{ JoinValidityConditionsResult, ValidityConditionOps, ValidityConditionsLike, }, }, - pcg::{ - PcgNodeWithPlace, - place_capabilities::{PlaceCapabilities, PlaceCapabilitiesReader}, - }, + pcg::PcgNodeWithPlace, utils::{ DebugCtxt, HasBorrowCheckerCtxt, HasCompilerCtxt, HasLocals, PlaceLike, data_structures::HashSet, @@ -32,11 +29,10 @@ use super::{ validity_conditions::{PathCondition, ValidityConditions}, }; use crate::{ - action::BorrowPcgAction, + action::{BorrowPcgAction, CapabilityChanges}, borrow_pcg::{ action::{BorrowPcgActionKind, LabelPlaceReason}, edge::{ - borrow::BorrowEdge, borrow_flow::{BorrowFlowEdge, BorrowFlowEdgeKind}, kind::BorrowPcgEdgeKind, }, @@ -45,11 +41,14 @@ use crate::{ region_projection::{LifetimeProjection, LifetimeProjectionLabel}, }, pcg::{ - CapabilityKind, PcgNode, ctxt::AnalysisCtxt, place_capabilities::PlaceCapabilitiesInterface, + CapabilityKind, PcgNode, + ctxt::AnalysisCtxt, + meet, + place_capabilities::{PlaceCapabilitiesInterface, PlaceCapabilitiesReader}, }, pcg_validity_assert, rustc_interface::middle::{ - mir::{self, BasicBlock, BorrowKind, Location, MutBorrowKind}, + mir::{self, BasicBlock}, ty, }, utils::{ @@ -90,6 +89,122 @@ fn map_label_predicate<'tcx, P: Copy>( } } +/// The capability changes of ending the freeze of `place` while `refrozen` +/// stay frozen: the read unpacking of `place` becomes an exclusive one, +/// except for `refrozen` and their unpackings. Each leaf gets back the +/// capability suspended for it or for the frozen place it is a read copy of. +fn end_freeze<'tcx, Ctxt: Copy, P: PlaceLike<'tcx, Ctxt>>( + place: P, + refrozen: &[P], + capabilities: &impl PlaceCapabilitiesReader<'tcx, CapabilityKind, P>, + ctxt: Ctxt, +) -> CapabilityChanges

{ + let holders = std::iter::once(place) + .chain( + capabilities + .capabilities_for_strict_postfixes_of(place) + .map(|(p, _)| p), + ) + .collect::>(); + let mut changes = CapabilityChanges::default(); + for &p in &holders { + if refrozen.iter().any(|t| t.is_prefix_of(p)) { + continue; + } + let unpacked = holders.iter().any(|h| p.is_strict_prefix_of(*h)); + changes.set( + p, + exclusively_unpacked_capability( + p, + unpacked, + suspended_for(place, p, capabilities), + ctxt, + ), + ); + } + changes +} + +/// The capability suspended for `p` by the freeze of `frozen`: the one +/// suspended for `p` or for the closest place between `frozen` and `p`. +fn suspended_for<'tcx, Ctxt: Copy, P: PlaceLike<'tcx, Ctxt>>( + frozen: P, + p: P, + capabilities: &impl PlaceCapabilitiesReader<'tcx, CapabilityKind, P>, +) -> Option { + capabilities + .iter_suspended() + .filter(|(q, _)| frozen.is_prefix_of(*q) && q.is_prefix_of(p)) + .reduce(|a, b| if a.0.is_strict_prefix_of(b.0) { b } else { a }) + .map(|(_, capability)| capability) +} + +/// Whether `p` is part of the current value of `place` that labelling it +/// for `reason` labels. +fn is_labelled<'tcx, Ctxt: Copy, P: PlaceLike<'tcx, Ctxt>>( + place: P, + reason: LabelPlaceReason, + p: P, + ctxt: Ctxt, +) -> bool { + match reason { + LabelPlaceReason::Detach => place.is_prefix_of(p), + _ => p.projects_indirection_from(place, ctxt), + } +} + +/// The capability changes of labelling the current value of `place`: the +/// current places that were part of the labelled value lose their +/// capabilities. +pub(crate) fn label_place_changes<'tcx, Ctxt: Copy, P: PlaceLike<'tcx, Ctxt>>( + place: P, + reason: LabelPlaceReason, + capabilities: &impl PlaceCapabilitiesReader<'tcx, CapabilityKind, P>, + ctxt: Ctxt, +) -> CapabilityChanges

{ + capabilities + .iter() + .filter(|(p, _)| is_labelled(place, reason, *p, ctxt)) + .map(|(p, _)| (p, None)) + .collect::>() + .into() +} + +fn is_behind_shared_ref<'tcx, Ctxt: Copy, P: PlaceLike<'tcx, Ctxt>>(place: P, ctxt: Ctxt) -> bool { + place + .iter_projections(ctxt) + .into_iter() + .any(|(base, elem)| { + matches!(elem, mir::ProjectionElem::Deref) + && matches!( + base.rust_ty(ctxt).kind(), + ty::Ref(_, _, ty::Mutability::Not) + ) + }) +} + +/// The capability of `place` in an exclusive unpacking: an `unpacked` place +/// keeps none (`e` for a mutable reference, R for a dereferenced shared +/// reference), and a leaf has `leaf`. Places behind a shared reference keep R. +fn exclusively_unpacked_capability<'tcx, Ctxt: Copy, P: PlaceLike<'tcx, Ctxt>>( + place: P, + unpacked: bool, + leaf: Option, + ctxt: Ctxt, +) -> Option { + if is_behind_shared_ref(place, ctxt) { + return Some(CapabilityKind::Read); + } + if !unpacked { + return leaf; + } + match place.rust_ty(ctxt).kind() { + ty::Ref(_, _, ty::Mutability::Mut) => Some(CapabilityKind::ShallowExclusive), + ty::Ref(_, _, ty::Mutability::Not) => Some(CapabilityKind::Read), + _ => None, + } +} + /// The state of the Borrow PCG, including the Borrow PCG graph and the validity /// conditions associated with the current basic block. #[derive(Clone, Debug, PartialEq, Eq)] @@ -177,28 +292,17 @@ pub(crate) trait BorrowsStateLike<'tcx, EdgeKind = BorrowPcgEdgeKind<'tcx>, VC = } fn graph(&self) -> &BorrowsGraph<'tcx, EdgeKind, VC>; - fn leaf_nodes<'a, Ctxt: HasCompilerCtxt<'a, 'tcx> + DebugCtxt>( - &self, - ctxt: Ctxt, - ) -> Vec> - where - EdgeKind: Eq + std::hash::Hash + EdgeData<'tcx, Ctxt, Place<'tcx>>, - 'tcx: 'a, - { - self.graph().frozen_graph().leaf_nodes(ctxt) - } - - fn label_place_and_update_related_capabilities< + /// Labels the current value of `place` in the graph; the capability + /// changes are those of [`label_place_changes`]. + fn label_place< 'a, P: PlaceLike<'tcx, Ctxt> + DisplayWithCtxt, Ctxt: Copy + HasCompilerCtxt<'a, 'tcx>, - C, >( &mut self, place: P, reason: LabelPlaceReason, labeller: &impl PlaceLabeller<'tcx, Ctxt, P>, - capabilities: &mut impl PlaceCapabilitiesInterface<'tcx, C, P>, ctxt: Ctxt, ) -> ApplyActionResult where @@ -209,12 +313,6 @@ pub(crate) trait BorrowsStateLike<'tcx, EdgeKind = BorrowPcgEdgeKind<'tcx>, VC = { let state = self.as_mut_ref(); let replacements = state.graph.label_place(place, reason, labeller, ctxt); - // If in a join we don't want to change capabilities because this will - // essentially be handled by the join logic. - // See 69_http_header_map.rs - if reason != LabelPlaceReason::JoinOwnedReadAndWriteCapabilities { - capabilities.retain(|p, _| !p.projects_indirection_from(place, ctxt)); - } let display = display_node_replacements(&replacements, ctxt, OutputMode::Normal); ApplyActionResult { changed: true, @@ -237,110 +335,359 @@ pub(crate) trait BorrowsStateLike<'tcx, EdgeKind = BorrowPcgEdgeKind<'tcx>, VC = .label_lifetime_projections(predicate, label, ctxt) } - fn remove<'a, P: Copy + Eq + std::hash::Hash, Ctxt: Copy, C>( - &mut self, - edge: &EdgeKind, - capabilities: &mut impl PlaceCapabilitiesInterface<'tcx, C, P>, - ctxt: Ctxt, - ) -> bool - where - 'tcx: 'a, - EdgeKind: EdgeData<'tcx, Ctxt, P> + Eq + std::hash::Hash, - { - let state = self.as_mut_ref(); - let removed = state.graph.remove(edge).is_some(); - if removed { - for node in edge.blocked_by_nodes(ctxt) { - if !state.graph.contains(node, ctxt) - && let PcgNode::Place(MaybeLabelledPlace::Current(place)) = node - { - let _ = capabilities.remove(place, ctxt); - } - } - } - removed - } - fn apply_action< 'a, Ctxt: DebugCtxt + HasCompilerCtxt<'a, 'tcx> + HasSettings<'a>, P: PlaceLike<'tcx, Ctxt> + DisplayWithCtxt, >( &mut self, - action: BorrowPcgAction<'tcx, EdgeKind, P, VC>, + action: &mut BorrowPcgAction<'tcx, EdgeKind, P, VC>, capabilities: &mut impl PlaceCapabilitiesInterface<'tcx, CapabilityKind, P>, ctxt: Ctxt, ) -> ApplyActionResult where 'tcx: 'a, - VC: ValidityConditionOps, + VC: ValidityConditionOps + Clone, EdgeKind: EdgeData<'tcx, Ctxt, P> + LabelEdgePlaces<'tcx, Ctxt, P> + LabelEdgeLifetimeProjections<'tcx, Ctxt, P> + Eq - + std::hash::Hash, + + std::hash::Hash + + Clone, MaybeRemotePlace<'tcx, P>: DisplayWithCtxt, { - match action.kind { - BorrowPcgActionKind::Restore(restore) => { - let restore_place = restore.place(); - if let Some(cap) = capabilities.get(restore_place, ctxt) { - pcg_validity_assert!(cap < restore.capability()); - } - if !capabilities.insert(restore_place, restore.capability(), ctxt) { - // panic!("Capability should have been updated") + // The changes the kind determines, then the ones given with the action. + let mut changes = CapabilityChanges::default(); + let mut result = ApplyActionResult::changed_no_display(); + match &action.kind { + BorrowPcgActionKind::Weaken(weaken) => { + let current = capabilities.get(weaken.place, ctxt); + pcg_validity_assert!( + current == Some(weaken.from) && weaken.from > weaken.to, + [ctxt], + "Weakening {} with capability {:?} from {:?} to {:?}", + weaken.place.display_string(ctxt), + current, + weaken.from, + weaken.to + ); + changes.set(weaken.place, Some(weaken.to)); + } + BorrowPcgActionKind::Freeze { place } => { + pcg_validity_assert!( + matches!( + capabilities.get(*place, ctxt), + Some(CapabilityKind::Exclusive | CapabilityKind::ShallowExclusive) | None + ), + [ctxt], + "Freezing {} with capability {:?}", + place.display_string(ctxt), + capabilities.get(*place, ctxt) + ); + // An unpacked place suspends what its parts hold together, + // a frozen part what its freeze suspends; what lies behind a + // shared reference is not part of its value. + let packed = capabilities + .capabilities_for_strict_postfixes_of(*place) + .filter(|(p, _)| !is_behind_shared_ref(*p, ctxt)) + .map(|(p, cap)| capabilities.suspended(p).unwrap_or(cap)) + .reduce(meet); + let suspended = + std::iter::once((*place, capabilities.get(*place, ctxt).or(packed))) + .chain( + capabilities + .capabilities_for_strict_postfixes_of(*place) + .map(|(p, cap)| (p, Some(cap))), + ) + .filter_map(|(p, cap)| Some((p, cap.filter(|cap| !cap.is_read())?))) + .collect::>(); + for (p, cap) in suspended { + capabilities.suspend(p, cap); } - if restore.capability() == CapabilityKind::Exclusive { - self.label_lifetime_projections( - &LabelNodePredicate::all_future_postfixes(restore_place), - None, - ctxt, + changes.set(*place, Some(CapabilityKind::Read)); + for (p, cap) in capabilities.capabilities_for_strict_postfixes_of(*place) { + pcg_validity_assert!( + !cap.is_write(), + [ctxt], + "Freezing {} with uninitialised {}", + place.display_string(ctxt), + p.display_string(ctxt) ); + changes.set(p, Some(CapabilityKind::Read)); } - ApplyActionResult::changed_no_display() } - BorrowPcgActionKind::Weaken(weaken) => { - let weaken_place = weaken.place(); + BorrowPcgActionKind::Unfreeze { place, refrozen } => { pcg_validity_assert!( - capabilities.get(weaken_place, ctxt).unwrap() == weaken.from, + capabilities.get(*place, ctxt) == Some(CapabilityKind::Read), [ctxt], - "Weakening from {:?} to {:?} is not valid", - capabilities.get(weaken_place, ctxt).unwrap(), - weaken.from + "Unfreezing {} with capability {:?}", + place.display_string(ctxt), + capabilities.get(*place, ctxt) ); - match weaken.to { - Some(to) => { - capabilities.insert(weaken_place, to, ctxt); - } - None => { - assert!(capabilities.remove(weaken_place, ctxt).is_some()); + changes = end_freeze(*place, refrozen, capabilities, ctxt); + // The freeze moves, with what it suspends, to `refrozen` and + // to the shared references whose referents stay unpacked. + let still_frozen = refrozen + .iter() + .copied() + .chain( + changes + .iter() + .filter(|(p, cap)| { + *cap == Some(CapabilityKind::Read) + && !is_behind_shared_ref(*p, ctxt) + }) + .map(|(p, _)| p), + ) + .collect::>(); + for &t in &still_frozen { + if capabilities.suspended(t).is_none() + && let Some(suspended) = suspended_for(*place, t, capabilities) + { + capabilities.suspend(t, suspended); } } - ApplyActionResult::changed_no_display() + capabilities.retain_suspended(|p, _| { + !place.is_prefix_of(p) || still_frozen.iter().any(|t| t.is_prefix_of(p)) + }); } - BorrowPcgActionKind::LabelPlace(action) => self - .label_place_and_update_related_capabilities( - action.place, - action.reason, - &SetLabel(action.location), - capabilities, - ctxt, - ), - BorrowPcgActionKind::RemoveEdge(edge) => { - ApplyActionResult::from_changed(self.remove(&edge.value, capabilities, ctxt)) + BorrowPcgActionKind::ActivateTwoPhaseBorrow { borrow } => { + let place = borrow.blocked_place.place(); + let frozen = capabilities.get(place, ctxt) == Some(CapabilityKind::Read) + && capabilities.suspended(place).is_some() + && !capabilities + .iter() + .any(|(p, cap)| p.is_strict_prefix_of(place) && cap.is_read()); + pcg_validity_assert!( + frozen, + [ctxt], + "Activating a two-phase borrow of {}, which is not frozen (capability {:?})", + place.display_string(ctxt), + capabilities.get(place, ctxt) + ); + changes.set(place, None); + } + BorrowPcgActionKind::LabelPlace(label) => { + changes = label_place_changes(label.place, label.reason, capabilities, ctxt); + capabilities + .retain_suspended(|p, _| !is_labelled(label.place, label.reason, p, ctxt)); + result = + self.label_place(label.place, label.reason, &SetLabel(label.location), ctxt); + } + BorrowPcgActionKind::RemoveEdge { edge } => { + let graph = self.graph_mut(); + let removed = graph.remove(&edge.value).is_some(); + for node in edge.value.blocked_by_nodes(ctxt) { + if !graph.contains(node, ctxt) + && let PcgNode::Place(MaybeLabelledPlace::Current(place)) = node + { + if capabilities.get(place, ctxt).is_some() { + changes.set(place, None); + } + capabilities.resume(place); + } + } + for (place, cap) in action.capabilities.iter() { + if let (Some(current), Some(cap)) = (capabilities.get(place, ctxt), cap) { + pcg_validity_assert!(current < cap, [ctxt]); + } + } + result = ApplyActionResult::from_changed(removed); } BorrowPcgActionKind::AddEdge { edge } => { - ApplyActionResult::from_changed(self.graph_mut().insert(edge, ctxt)) + // A loan suspends the capabilities it takes; an unpacking + // moves them to its expansion. + for (place, capability) in action.capabilities.iter() { + if capability.is_none() + && let Some(lent) = capabilities.get(place, ctxt) + && !edge.value.blocked_by_nodes(ctxt).any(|node| { + matches!( + node, + PcgNode::Place(MaybeLabelledPlace::Current(p)) + if place.is_strict_prefix_of(p) + ) + }) + { + capabilities.suspend(place, lent); + } + } + result = + ApplyActionResult::from_changed(self.graph_mut().insert(edge.clone(), ctxt)); + } + BorrowPcgActionKind::Lend { place, edge } => { + // What `place` lends is recorded for it: a frozen place lends + // what its freeze suspends, with its read copies; the referent + // of a mutable reference held with E is lent through it, and + // the reference keeps e. + let reference = capabilities + .iter() + .find(|(reference, cap)| { + *cap == CapabilityKind::Exclusive + && reference.is_strict_prefix_of(*place) + && place.projects_indirection_from(*reference, ctxt) + }) + .map(|(reference, _)| reference); + if let Some(reference) = reference { + changes.set(reference, Some(CapabilityKind::ShallowExclusive)); + } + let held = capabilities.get(*place, ctxt); + let suspended = capabilities.suspended(*place); + if held == Some(CapabilityKind::Read) && suspended.is_some() { + for (p, _) in capabilities.capabilities_for_strict_postfixes_of(*place) { + changes.set(p, None); + } + } + if let Some(lent) = suspended + .or(held) + .or(reference.map(|_| CapabilityKind::Exclusive)) + { + capabilities.suspend(*place, lent); + } + self.graph_mut().insert(edge.clone(), ctxt); + changes.set(*place, None); + } + BorrowPcgActionKind::UnpackSuspended { place, parts } => { + let suspended = capabilities.resume(*place); + pcg_validity_assert!( + suspended.is_some(), + [ctxt], + "Unpacking the suspended capability of {}, which has none", + place.display_string(ctxt) + ); + if let Some(suspended) = suspended { + for &part in parts { + let share = if part.projects_indirection_from(*place, ctxt) { + match (place.rust_ty(ctxt).kind(), suspended) { + (ty::Ref(_, _, ty::Mutability::Mut), CapabilityKind::Exclusive) => { + capabilities.suspend(*place, CapabilityKind::ShallowExclusive); + CapabilityKind::Exclusive + } + (ty::Ref(_, _, ty::Mutability::Not), _) => { + capabilities.suspend(*place, suspended); + CapabilityKind::Read + } + _ => { + capabilities.suspend(*place, suspended); + continue; + } + } + } else { + suspended + }; + // A part that records less gets back at most that. + let share = capabilities + .suspended(part) + .map_or(share, |own| meet(own, share)); + capabilities.suspend(part, share); + } + } + } + BorrowPcgActionKind::PackSuspended { place } => { + // The inverse of `UnpackSuspended` at a mutable reference: e + // and an E referent give E. + if let ty::Ref(_, _, ty::Mutability::Mut) = place.rust_ty(ctxt).kind() + && capabilities.suspended(*place) == Some(CapabilityKind::ShallowExclusive) + { + let referents = capabilities + .iter_suspended() + .filter(|(p, _)| { + place.is_strict_prefix_of(*p) + && p.projects_indirection_from(*place, ctxt) + }) + .collect::>(); + if !referents.is_empty() + && referents + .iter() + .all(|(_, cap)| *cap == CapabilityKind::Exclusive) + { + for (referent, _) in referents { + capabilities.resume(referent); + } + capabilities.suspend(*place, CapabilityKind::Exclusive); + } + } + let parts = capabilities + .iter_suspended() + .filter(|(p, _)| { + place.is_strict_prefix_of(*p) && !p.projects_indirection_from(*place, ctxt) + }) + .collect::>(); + for (part, _) in &parts { + capabilities.resume(*part); + } + if let Some(packed) = parts.into_iter().map(|(_, cap)| cap).reduce(meet) { + let packed = capabilities + .resume(*place) + .map_or(packed, |own| meet(own, packed)); + capabilities.suspend(*place, packed); + } + } + BorrowPcgActionKind::WeakenSuspended { place, from, to } => { + let current = capabilities.resume(*place); + pcg_validity_assert!( + current == Some(*from) + && to.is_none_or(|to| to != *from && meet(*from, to) == to), + [ctxt], + "Weakening the suspended {:?} of {} from {:?} to {:?}", + current, + place.display_string(ctxt), + from, + to + ); + if let Some(to) = to { + capabilities.suspend(*place, *to); + } } - BorrowPcgActionKind::LabelLifetimeProjection(action) => { - let predicate = map_label_predicate(action.predicate()); - ApplyActionResult::from_changed(self.label_lifetime_projections( + BorrowPcgActionKind::LabelLifetimeProjection(label) => { + let predicate = map_label_predicate(label.predicate()); + result = ApplyActionResult::from_changed(self.label_lifetime_projections( &predicate, - action.label(), + label.label(), + ctxt, + )); + } + } + changes.extend(&action.capabilities); + result.changed |= if matches!( + action.kind, + BorrowPcgActionKind::Unfreeze { .. } | BorrowPcgActionKind::RemoveEdge { .. } + ) { + self.regain(&changes, capabilities, ctxt) + } else { + changes.apply(capabilities, ctxt) + }; + action.capabilities = changes; + result + } + + /// Gives back the capabilities `changes`, ending the suspension of the + /// places that get one above R (a place that gets R stays frozen); the + /// places that get E back are accessed exclusively again. + fn regain<'a, Ctxt: Copy, P: Copy + Eq + std::hash::Hash>( + &mut self, + changes: &CapabilityChanges

, + capabilities: &mut impl PlaceCapabilitiesInterface<'tcx, CapabilityKind, P>, + ctxt: Ctxt, + ) -> bool + where + 'tcx: 'a, + EdgeKind: LabelEdgeLifetimeProjections<'tcx, Ctxt, P> + Eq + std::hash::Hash, + VC: ValidityConditionOps, + { + let changed = changes.apply(capabilities, ctxt); + for (place, cap) in changes.iter() { + if cap.is_some_and(|cap| !cap.is_read()) { + capabilities.resume(place); + } + if cap == Some(CapabilityKind::Exclusive) { + self.label_lifetime_projections( + &LabelNodePredicate::all_future_postfixes(place), + None, ctxt, - )) + ); } } + changed } fn edges_blocking<'slf, Ctxt: Copy + DebugCtxt>( @@ -447,7 +794,7 @@ impl<'a, 'tcx> BorrowsState<'a, 'tcx, BorrowPcgEdgeKind<'tcx>, ValidityCondition .unwrap(); assert!( self.apply_action( - BorrowPcgAction::add_edge( + &mut BorrowPcgAction::add_edge( BorrowPcgEdge::new( BorrowFlowEdge::new( source_projection, @@ -534,72 +881,6 @@ impl<'a, 'tcx> BorrowsState<'a, 'tcx> { .flat_map(|e| e.blocked_by_nodes(ctxt).collect::>()) .collect() } - - #[allow(clippy::too_many_arguments)] - pub(crate) fn add_borrow + HasSettings<'a>>( - &mut self, - blocked_place: Place<'tcx>, - assigned_place: Place<'tcx>, - kind: BorrowKind, - location: Location, - region: ty::Region<'tcx>, - capabilities: &mut PlaceCapabilities<'tcx>, - ctxt: Ctxt, - ) where - 'tcx: 'a, - { - assert!( - assigned_place.ty(ctxt).ty.ref_mutability().is_some(), - "{:?}:{:?} Assigned place {:?} is not a reference. Ty: {:?}", - ctxt.body().source.def_id(), - location, - assigned_place, - assigned_place.ty(ctxt).ty - ); - let borrow_edge = BorrowEdge::new( - blocked_place.into(), - assigned_place.into(), - kind, - location, - region, - ctxt, - ); - assert!(self.graph.insert( - BorrowPcgEdge::new( - BorrowPcgEdgeKind::Borrow(borrow_edge), - self.validity_conditions.clone(), - ), - ctxt.ctxt() - )); - - if let BorrowKind::Mut { - kind: MutBorrowKind::Default | MutBorrowKind::ClosureCapture, - } = kind - { - let _ = capabilities.remove(blocked_place, ctxt); - } else { - let blocked_place_capability = capabilities.get(blocked_place, ctxt); - match blocked_place_capability { - Some(CapabilityKind::Exclusive) => { - assert!(capabilities.insert(blocked_place, CapabilityKind::Read, ctxt)); - } - Some(CapabilityKind::Read) => { - // Do nothing, this just adds another shared borrow - } - other => { - // Shouldn't be None or Write, due to capability updates - // based on the TripleWalker analysis - pcg_validity_assert!( - false, - "{:?}: Unexpected capability for borrow blocked place {:?}: {:?}", - location, - blocked_place, - other - ); - } - } - } - } } impl<'a, 'tcx, EdgeKind: PartialEq + Eq + std::hash::Hash> BorrowsState<'a, 'tcx, EdgeKind> { diff --git a/src/lib.rs b/src/lib.rs index c9dc9daa0..a2c4e78bf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -82,16 +82,15 @@ use visualization::mir_graph::generate_json_from_mir; /// The result of the PCG analysis. pub type PcgOutput<'a, 'tcx> = results::PcgAnalysisResults<'a, 'tcx>; -/// Instructs that the current capability to the place (first [`CapabilityKind`]) should -/// be weakened to the second given capability. We guarantee that `_.1 > _.2`. -/// If `_.2` is `None`, the capability is removed. +/// The capability of the place goes from `from` down to the strictly lower +/// `to`: the place's value is given up (`to` is W), or part of it. #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Serialize)] #[cfg_attr(feature = "type-export", derive(ts_rs::TS))] #[cfg_attr(feature = "type-export", ts(export, concrete(Place = String)))] -pub struct Weaken<'tcx, Place = utils::Place<'tcx>, ToCap = Option> { +pub struct Weaken<'tcx, Place = utils::Place<'tcx>> { pub(crate) place: Place, pub(crate) from: CapabilityKind, - pub(crate) to: ToCap, + pub(crate) to: CapabilityKind, /// Temporary flag indicating this Weaken was emitted due to a `StorageDead`. /// Required until is resolved. pub(crate) for_storage_dead: bool, @@ -99,10 +98,8 @@ pub struct Weaken<'tcx, Place = utils::Place<'tcx>, ToCap = Option, } -impl<'a, 'tcx: 'a, Ctxt: HasCompilerCtxt<'a, 'tcx>, ToCap: Copy + serde::Serialize> DebugRepr - for Weaken<'tcx, Place<'tcx>, ToCap> -{ - type Repr = Weaken<'static, String, ToCap>; +impl<'a, 'tcx: 'a, Ctxt: HasCompilerCtxt<'a, 'tcx>> DebugRepr for Weaken<'tcx, Place<'tcx>> { + type Repr = Weaken<'static, String>; fn debug_repr(&self, ctxt: Ctxt) -> Self::Repr { Weaken { place: self.place.display_string(ctxt), @@ -114,8 +111,8 @@ impl<'a, 'tcx: 'a, Ctxt: HasCompilerCtxt<'a, 'tcx>, ToCap: Copy + serde::Seriali } } -impl Weaken<'_, Place, ToCap> { - pub(crate) fn new(place: Place, from: CapabilityKind, to: ToCap) -> Self { +impl Weaken<'_, Place> { + pub(crate) fn new(place: Place, from: CapabilityKind, to: CapabilityKind) -> Self { Self { place, from, @@ -127,7 +124,11 @@ impl Weaken<'_, Place, ToCap> { /// Create a Weaken that is due to a `StorageDead`. /// Temporary workaround until is resolved. - pub(crate) fn new_for_storage_dead(place: Place, from: CapabilityKind, to: ToCap) -> Self { + pub(crate) fn new_for_storage_dead( + place: Place, + from: CapabilityKind, + to: CapabilityKind, + ) -> Self { Self { place, from, @@ -148,10 +149,7 @@ impl Weaken<'_, Place, ToCap> { self.place } - pub fn to_cap(&self) -> ToCap - where - ToCap: Copy, - { + pub fn to_cap(&self) -> CapabilityKind { self.to } @@ -164,10 +162,6 @@ impl Weaken<'_, Place, ToCap> { impl<'a, 'tcx: 'a, Ctxt: HasBorrowCheckerCtxt<'a, 'tcx>> DisplayWithCtxt for Weaken<'tcx> { fn display_output(&self, ctxt: Ctxt, mode: OutputMode) -> DisplayOutput { - let to_str = match self.to { - Some(to) => to.display_output(ctxt, mode), - None => "None".into(), - }; DisplayOutput::join( vec![ "Weaken".into(), @@ -175,70 +169,13 @@ impl<'a, 'tcx: 'a, Ctxt: HasBorrowCheckerCtxt<'a, 'tcx>> DisplayWithCtxt f "from".into(), self.from.display_output(ctxt, mode), "to".into(), - to_str, - ], - &DisplayOutput::SPACE, - ) - } -} - -/// Instructs that the capability to the place should be restored to the -/// given capability, e.g. after a borrow expires, the borrowed place should be -/// restored to exclusive capability. -#[derive(Clone, Debug, Eq, PartialEq, Hash)] -pub struct RestoreCapability<'tcx, P = Place<'tcx>> { - place: P, - capability: CapabilityKind, - _marker: PhantomData<&'tcx ()>, -} - -impl<'a, 'tcx: 'a, Ctxt: HasCompilerCtxt<'a, 'tcx>> ToJsonWithCtxt - for RestoreCapability<'tcx> -{ - fn to_json(&self, ctxt: Ctxt) -> serde_json::Value { - json!({ - "place": self.place.to_json(ctxt.ctxt()), - "capability": format!("{:?}", self.capability), - }) - } -} - -impl<'a, 'tcx: 'a, Ctxt: HasBorrowCheckerCtxt<'a, 'tcx>> DisplayWithCtxt - for RestoreCapability<'tcx> -{ - fn display_output(&self, ctxt: Ctxt, mode: OutputMode) -> DisplayOutput { - DisplayOutput::join( - vec![ - "Restore".into(), - self.place.display_output(ctxt, mode), - "to".into(), - self.capability.display_output(ctxt, mode), + self.to.display_output(ctxt, mode), ], &DisplayOutput::SPACE, ) } } -impl RestoreCapability<'_, P> { - pub(crate) fn new(place: P, capability: CapabilityKind) -> Self { - Self { - place, - capability, - _marker: PhantomData, - } - } - - pub fn place(&self) -> P { - self.place - } - - pub fn capability(&self) -> CapabilityKind { - self.capability - } -} - -impl RestoreCapability<'_> {} - impl<'a, 'tcx: 'a, Ctxt: HasCompilerCtxt<'a, 'tcx>> ToJsonWithCtxt for Weaken<'tcx> { fn to_json(&self, ctxt: Ctxt) -> serde_json::Value { json!({ diff --git a/src/loop/mod.rs b/src/loop/mod.rs index 6637e7d10..e4e53c5aa 100644 --- a/src/loop/mod.rs +++ b/src/loop/mod.rs @@ -30,7 +30,7 @@ use crate::{ mir_dataflow::{Forward, JoinSemiLattice, fmt::DebugWithContext}, }, utils::{ - HasCompilerCtxt, Place, + HasCompilerCtxt, Place, PlaceProjectable, data_structures::HashMap, display::{DisplayOutput, DisplayWithCtxt, OutputMode}, visitor::FallableVisitor, @@ -562,6 +562,26 @@ impl<'tcx> PlaceUsages<'tcx> { PlaceUsages(consolidated) } + /// These usages, each place in an enum variant replaced by the enum: a + /// loop head does not know the variant, as not every path into it matched + /// it. + pub(crate) fn outside_variants<'a>(&self, ctxt: impl HasCompilerCtxt<'a, 'tcx>) -> Self + where + 'tcx: 'a, + { + let mut usages = PlaceUsages::default(); + for usage in self.iter() { + let place = usage + .place + .iter_projections(ctxt) + .into_iter() + .find(|(_, elem)| matches!(elem, mir::ProjectionElem::Downcast(..))) + .map_or(usage.place, |(enum_place, _)| enum_place); + let _ = usages.update(place, usage.usage); + } + usages + } + /// The usages of this set in the order in which the corresponding places /// should be unpacked at a loop head: the places requiring write or /// exclusive access first, then the read-only places. diff --git a/src/owned_pcg/impl/join/data.rs b/src/owned_pcg/impl/join/data.rs deleted file mode 100644 index 6d0f13add..000000000 --- a/src/owned_pcg/impl/join/data.rs +++ /dev/null @@ -1,39 +0,0 @@ -use crate::{ - borrow_pcg::state::BorrowsState, pcg::place_capabilities::PlaceCapabilities, - utils::SnapshotLocation, -}; - -pub(crate) struct JoinOwnedData<'a, 'pcg, 'tcx, T> { - pub(crate) owned: T, - pub(crate) borrows: &'pcg mut BorrowsState<'a, 'tcx>, - pub(crate) capabilities: &'pcg mut PlaceCapabilities<'tcx>, - pub(crate) snapshot_location: SnapshotLocation, -} - -impl<'a, 'pcg, 'tcx, T> JoinOwnedData<'a, 'pcg, 'tcx, T> { - pub(crate) fn map_owned<'slf: 'res, 'res, U: 'res>( - &'slf mut self, - f: impl Fn(&'slf mut T) -> U, - ) -> JoinOwnedData<'a, 'res, 'tcx, U> - where - 'pcg: 'res, - { - JoinOwnedData { - owned: f(&mut self.owned), - borrows: self.borrows, - capabilities: self.capabilities, - snapshot_location: self.snapshot_location, - } - } -} - -impl<'a, 'pcg, 'tcx, T> JoinOwnedData<'a, 'pcg, 'tcx, &'pcg mut T> { - pub(crate) fn reborrow<'slf>(&'slf mut self) -> JoinOwnedData<'a, 'slf, 'tcx, &'slf mut T> { - JoinOwnedData { - owned: self.owned, - borrows: self.borrows, - capabilities: self.capabilities, - snapshot_location: self.snapshot_location, - } - } -} diff --git a/src/owned_pcg/impl/join/mod.rs b/src/owned_pcg/impl/join/mod.rs deleted file mode 100644 index af72fe217..000000000 --- a/src/owned_pcg/impl/join/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub(crate) mod data; -pub(super) mod obtain; diff --git a/src/owned_pcg/impl/join/obtain.rs b/src/owned_pcg/impl/join/obtain.rs deleted file mode 100644 index 30bb7427d..000000000 --- a/src/owned_pcg/impl/join/obtain.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::{ - action::PcgAction, - borrow_pcg::state::{BorrowStateMutRef, BorrowsStateLike}, - error::PcgError, - owned_pcg::{LocalExpansions, RepackOp, join::data::JoinOwnedData}, - pcg::{ - CompilerCtxtWithSettings, - obtain::{ActionApplier, HasSnapshotLocation, PlaceCollapser}, - place_capabilities::PlaceCapabilities, - }, - rustc_interface::middle::mir, - utils::{CompilerCtxt, Place, SnapshotLocation, data_structures::HashSet}, -}; - -pub(crate) type JoinCtxt<'a, 'tcx> = CompilerCtxtWithSettings<'a, 'tcx>; - -pub(crate) struct JoinObtainer<'pcg: 'exp, 'exp, 'slf, 'a, 'tcx> { - pub(crate) ctxt: JoinCtxt<'a, 'tcx>, - pub(crate) data: &'slf mut JoinOwnedData<'a, 'pcg, 'tcx, &'exp mut LocalExpansions<'tcx>>, - pub(crate) actions: Vec>, -} - -impl HasSnapshotLocation for JoinObtainer<'_, '_, '_, '_, '_> { - fn prev_snapshot_location(&self) -> SnapshotLocation { - self.data.snapshot_location - } -} - -impl<'tcx> ActionApplier<'tcx> for JoinObtainer<'_, '_, '_, '_, 'tcx> { - fn apply_action(&mut self, action: PcgAction<'tcx>) -> Result<(), PcgError> { - match action { - PcgAction::Borrow(action) => { - self.data - .borrows - .apply_action(action.clone(), self.data.capabilities, self.ctxt); - } - PcgAction::Owned(action) => match action.kind { - RepackOp::Collapse(collapse) => { - self.data.owned.perform_collapse_action( - collapse, - self.data.capabilities, - self.ctxt, - ); - self.actions.push(action.kind); - } - RepackOp::RegainLoanedCapability(regained_capability) => { - self.data.capabilities.regain_loaned_capability( - regained_capability.place, - regained_capability.capability, - self.data.borrows.as_mut_ref(), - self.ctxt, - ); - self.actions.push(action.kind); - } - _ => unreachable!(), - }, - } - Ok(()) - } -} - -impl<'a, 'tcx> PlaceCollapser<'a, 'tcx> for JoinObtainer<'_, '_, '_, 'a, 'tcx> { - fn get_local_expansions(&self, _local: mir::Local) -> &LocalExpansions<'tcx> { - self.data.owned - } - - fn borrows_state(&mut self) -> BorrowStateMutRef<'_, 'tcx> { - self.data.borrows.into() - } - - fn capabilities(&mut self) -> &mut PlaceCapabilities<'tcx> { - self.data.capabilities - } - - /// Owned leaf places that are not borrowed. - fn leaf_places(&self, ctxt: CompilerCtxt<'a, 'tcx>) -> HashSet> { - let owned_places = self.blocked_owned_places(ctxt); - self.data - .owned - .leaf_places(ctxt) - .into_iter() - .map(Into::into) - .filter(|p| !owned_places.contains(p)) - .collect() - } - - fn blocked_owned_places(&self, ctxt: CompilerCtxt<'a, 'tcx>) -> HashSet> { - self.data.borrows.graph().owned_places(ctxt) - } -} diff --git a/src/owned_pcg/impl/join_semi_lattice.rs b/src/owned_pcg/impl/join_semi_lattice.rs index 3ffe4ca38..4ff63349a 100644 --- a/src/owned_pcg/impl/join_semi_lattice.rs +++ b/src/owned_pcg/impl/join_semi_lattice.rs @@ -6,140 +6,28 @@ use crate::{ HasSettings, - borrow_pcg::{ - action::LabelPlaceReason, borrow_pcg_expansion::PlaceExpansion, has_pcs_elem::SetLabel, - state::BorrowsStateLike, - }, - error::PcgError, - owned_pcg::{ - ExpandedPlace, RepackCollapse, RepackExpand, RepackGuide, RepackOp, - join::{data::JoinOwnedData, obtain::JoinCtxt}, - }, + action::CapabilityChanges, + borrow_pcg::borrow_pcg_expansion::PlaceExpansion, + owned_pcg::{RepackCollapse, RepackExpand}, pcg::{CapabilityKind, place_capabilities::PlaceCapabilitiesInterface}, pcg_validity_assert, pcg_validity_expect_some, utils::{ - DebugCtxt, HasCompilerCtxt, Place, - data_structures::{HashMap, HashSet}, + DebugCtxt, HasCompilerCtxt, Place, data_structures::HashSet, display::DisplayWithCompilerCtxt, }, }; -use itertools::Itertools; - -use crate::{ - owned_pcg::LocalExpansions, - pcg::owned_state::{LocalInitState, OwnedPcg}, - rustc_interface::middle::mir, -}; - -impl<'a, 'pcg, 'tcx> JoinOwnedData<'a, 'pcg, 'tcx, &'pcg mut LocalInitState<'tcx>> { - #[tracing::instrument(skip(self, other, ctxt), fields(self.snapshot_location = ?self.snapshot_location, other.snapshot_location = ?other.snapshot_location), level = "warn")] - pub(crate) fn join( - &mut self, - mut other: JoinOwnedData<'a, 'pcg, 'tcx, &'pcg LocalInitState<'tcx>>, - ctxt: JoinCtxt<'a, 'tcx>, - ) -> Result>, PcgError> { - match (&mut *self.owned, &mut other.owned) { - (LocalInitState::Unallocated, LocalInitState::Unallocated) => Ok(vec![]), - ( - LocalInitState::Allocated { - expansions: to_places, - .. - }, - LocalInitState::Allocated { - expansions: from_places, - .. - }, - ) => { - let self_allocated = JoinOwnedData { - owned: to_places, - borrows: self.borrows, - capabilities: self.capabilities, - snapshot_location: self.snapshot_location, - }; - let mut from_places = from_places.clone(); - let other_allocated = JoinOwnedData { - owned: &mut from_places, - borrows: other.borrows, - capabilities: other.capabilities, - snapshot_location: other.snapshot_location, - }; - self_allocated.join(other_allocated, ctxt) - } - (LocalInitState::Allocated { expansions, .. }, LocalInitState::Unallocated) => { - self.borrows.label_place_and_update_related_capabilities( - expansions.local.into(), - LabelPlaceReason::StorageDead, - &SetLabel(self.snapshot_location), - self.capabilities, - ctxt, - ); - let mut repacks = vec![]; - for (place, k) in self.capabilities.owned_capabilities(expansions.local, ctxt) { - if *k > CapabilityKind::Write { - // Temporary: mark as for_storage_dead until - // https://github.com/prusti/pcg/issues/137 is resolved. - repacks.push(RepackOp::weaken_for_storage_dead( - place, - *k, - CapabilityKind::Write, - )); - *k = CapabilityKind::Write; - } - } - repacks.extend(expansions.collapse( - expansions.local.into(), - None, - self.capabilities, - ctxt, - )); - repacks.push(RepackOp::StorageDead(expansions.local)); - *self.owned = LocalInitState::Unallocated; - Ok(repacks) - } - (LocalInitState::Unallocated, LocalInitState::Allocated { expansions, .. }) => { - other.borrows.label_place_and_update_related_capabilities( - expansions.local.into(), - LabelPlaceReason::StorageDead, - &SetLabel(other.snapshot_location), - other.capabilities, - ctxt, - ); - Ok(vec![]) - } - } - } -} -impl<'a, 'pcg, 'tcx> JoinOwnedData<'a, 'pcg, 'tcx, &'pcg mut OwnedPcg<'tcx>> { - pub(crate) fn join( - &mut self, - mut other: JoinOwnedData<'a, 'pcg, 'tcx, &'pcg OwnedPcg<'tcx>>, - ctxt: JoinCtxt<'a, 'tcx>, - ) -> Result>, PcgError> { - let mut actions = vec![]; - for local in 0..self.owned.num_locals() { - let local: mir::Local = local.into(); - let mut owned_local_data = self.map_owned(|owned| &mut owned[local]); - let other_owned_local_data = other.map_owned(|owned| &owned[local]); - actions.extend(owned_local_data.join(other_owned_local_data, ctxt)?); - } - Ok(actions) - } -} +use crate::owned_pcg::LocalExpansions; impl<'tcx> LocalExpansions<'tcx> { - pub(crate) fn expansions_shortest_first(&self) -> impl Iterator> { - self.expansions - .iter() - .sorted_by_key(|ep| ep.place.projection().len()) - } - + /// Performs `expand` and returns its capability changes. pub(crate) fn perform_expand_action<'a, Ctxt: HasCompilerCtxt<'a, 'tcx>>( &mut self, expand: RepackExpand<'tcx>, capabilities: &mut impl PlaceCapabilitiesInterface<'tcx>, ctxt: Ctxt, - ) where + ) -> CapabilityChanges> + where 'tcx: 'a, { let target_places = expand.target_places(ctxt); @@ -161,14 +49,16 @@ impl<'tcx> LocalExpansions<'tcx> { CapabilityKind::Exclusive }) }; + let mut changes = CapabilityChanges::default(); for target_place in target_places { - capabilities.insert(target_place, source_cap, ctxt); - } - if expand.capability.is_read() { - capabilities.insert(expand.from, CapabilityKind::Read, ctxt); - } else { - capabilities.remove(expand.from, ctxt); + changes.set(target_place, Some(source_cap)); } + changes.set( + expand.from, + expand.capability.is_read().then_some(CapabilityKind::Read), + ); + changes.apply(capabilities, ctxt); + changes } pub(crate) fn all_descendants_of<'a>( @@ -201,39 +91,7 @@ impl<'tcx> LocalExpansions<'tcx> { .collect() } - pub(crate) fn collapse_actions_for<'a, Ctxt: HasCompilerCtxt<'a, 'tcx>>( - &self, - place: Place<'tcx>, - capabilities: &impl PlaceCapabilitiesInterface<'tcx>, - ctxt: Ctxt, - ) -> Vec> - where - 'tcx: 'a, - { - let children = self.all_children_of(place, ctxt); - let mut collapses_by_guide: HashMap, CapabilityKind> = - HashMap::default(); - for child in children { - let guide: Option = child.last_projection().unwrap().1.try_into().ok(); - // The place does not carry the label of an index guide; the - // stored expansion does. - let guide = guide.map(|guide| { - self.expansions - .iter() - .filter(|expanded| expanded.place == place) - .find_map(|expanded| expanded.guide().filter(|stored| *stored == guide)) - .unwrap_or(guide) - }); - let child_cap = capabilities.get(child, ctxt).unwrap(); - let entry = collapses_by_guide.entry(guide).or_insert(child_cap); - *entry = entry.minimum(child_cap).unwrap(); - } - collapses_by_guide - .into_iter() - .map(|(guide, cap)| RepackCollapse::new(place, cap, guide)) - .collect() - } - + /// Performs `collapse` and returns its capability changes. pub(crate) fn perform_collapse_action< 'a, Ctxt: HasCompilerCtxt<'a, 'tcx> + DebugCtxt + HasSettings<'a>, @@ -242,15 +100,36 @@ impl<'tcx> LocalExpansions<'tcx> { collapse: RepackCollapse<'tcx>, place_capabilities: &mut impl PlaceCapabilitiesInterface<'tcx>, ctxt: Ctxt, - ) where + ) -> CapabilityChanges> + where 'tcx: 'a, { - let expansion_places = self.all_children_of(collapse.to, ctxt); + // Only the expansion with the collapse's guide is packed: a place + // can be read-unpacked at several indices. + let collapsed = self + .expansions + .iter() + .filter(|ep| ep.place == collapse.to && ep.guide() == collapse.guide) + .cloned() + .collect::>(); + pcg_validity_assert!( + !collapsed.is_empty(), + [ctxt], + "Collapsing {}, which has no expansion with guide {:?}", + collapse.to.display_string(ctxt.ctxt()), + collapse.guide + ); + let expansion_places = collapsed + .iter() + .flat_map(|ep| ep.expansion_places(ctxt).unwrap()) + .collect::>(); + let mut changes = CapabilityChanges::default(); let retained_cap: CapabilityKind = expansion_places .iter() .fold(CapabilityKind::Exclusive, |acc, place| { - let removed_cap = place_capabilities.remove(*place, ctxt); + let removed_cap = place_capabilities.get(*place, ctxt); + changes.set(*place, None); let removed_cap = pcg_validity_expect_some!( removed_cap, fallback: CapabilityKind::Exclusive, @@ -268,7 +147,9 @@ impl<'tcx> LocalExpansions<'tcx> { acc ) }); - self.remove_all_expansions_from(collapse.to, ctxt); - place_capabilities.insert(collapse.to, retained_cap, ctxt); + self.expansions.retain(|ep| !collapsed.contains(ep)); + changes.set(collapse.to, Some(retained_cap)); + changes.apply(place_capabilities, ctxt); + changes } } diff --git a/src/owned_pcg/impl/local/join.rs b/src/owned_pcg/impl/local/join.rs deleted file mode 100644 index 940873f39..000000000 --- a/src/owned_pcg/impl/local/join.rs +++ /dev/null @@ -1,557 +0,0 @@ -use crate::{ - DebugLines, - action::{BorrowPcgAction, PcgAction}, - borrow_pcg::action::LabelPlaceReason, - capability_gte, - error::PcgError, - owned_pcg::{ - ExpandedPlace, LocalExpansions, RegainedCapability, RepackExpand, RepackOp, - join::{ - data::JoinOwnedData, - obtain::{JoinCtxt, JoinObtainer}, - }, - }, - pcg::{ - CapabilityKind, - obtain::{ActionApplier, HasSnapshotLocation, PlaceCollapser}, - place_capabilities::{PlaceCapabilitiesInterface, PlaceCapabilitiesReader}, - }, - pcg_validity_assert, - utils::{ - self, HasBorrowCheckerCtxt, Place, data_structures::HashSet, - display::DisplayWithCompilerCtxt, - }, -}; - -enum JoinDifferentExpansionsResult<'tcx> { - ExpandedForRead(RepackExpand<'tcx>), - ExpandedForNoCapability, - Collapsed(Vec>), -} - -impl<'tcx> JoinDifferentExpansionsResult<'tcx> { - fn actions(self) -> Vec> { - match self { - JoinDifferentExpansionsResult::ExpandedForRead(action) => { - vec![RepackOp::Expand(action)] - } - JoinDifferentExpansionsResult::ExpandedForNoCapability => vec![], - JoinDifferentExpansionsResult::Collapsed(actions) => actions, - } - } -} - -enum JoinExpandedPlaceResult<'tcx> { - JoinedWithSameExpansion(Vec>), - CreatedExpansion(Vec>), - JoinedWithOtherExpansions(JoinDifferentExpansionsResult<'tcx>), - CollapsedOtherExpansion(Vec>), -} - -impl<'tcx> JoinExpandedPlaceResult<'tcx> { - fn actions(self) -> Vec> { - match self { - JoinExpandedPlaceResult::JoinedWithSameExpansion(actions) - | JoinExpandedPlaceResult::CreatedExpansion(actions) - | JoinExpandedPlaceResult::CollapsedOtherExpansion(actions) => actions, - JoinExpandedPlaceResult::JoinedWithOtherExpansions(result) => result.actions(), - } - } - - fn performed_collapse(&self) -> bool { - matches!( - self, - JoinExpandedPlaceResult::JoinedWithOtherExpansions( - JoinDifferentExpansionsResult::Collapsed(_) - ) | JoinExpandedPlaceResult::CollapsedOtherExpansion(_) - ) - } -} - -impl<'pcg, 'a: 'pcg, 'tcx> JoinOwnedData<'a, 'pcg, 'tcx, &'pcg mut LocalExpansions<'tcx>> { - fn join_different_expansions_from_place<'other>( - &mut self, - other: &mut JoinOwnedData<'a, 'pcg, 'tcx, &'other mut LocalExpansions<'tcx>>, - other_expansion: &ExpandedPlace<'tcx>, - ctxt: JoinCtxt<'a, 'tcx>, - ) -> Result, PcgError> - where - 'pcg: 'other, - 'tcx: 'a, - { - let base_place = other_expansion.place; - let self_cap = self.capabilities.get(base_place, ctxt); - let other_cap = other.capabilities.get(base_place, ctxt); - if self_cap == Some(CapabilityKind::Read) && other_cap == Some(CapabilityKind::Read) { - let action = - RepackExpand::new(base_place, other_expansion.guide(), CapabilityKind::Read); - self.owned - .perform_expand_action(action, self.capabilities, ctxt); - Ok(JoinDifferentExpansionsResult::ExpandedForRead(action)) - } else if other - .owned - .contains_descendent_leaf_node_without_capability(base_place, other.capabilities, ctxt) - { - // If this case holds, the leaf node is exclusively borrowed. - // We want to insert this expansion (and all further expansions), without any capabilities - tracing::debug!( - "Other expansions contain a descendent leaf node without capability for place {}", - base_place.display_string(ctxt) - ); - self.owned - .insert_expansion(other_expansion.place, other_expansion.expansion.clone()); - Ok(JoinDifferentExpansionsResult::ExpandedForNoCapability) - } else { - let mut self_obtainer = JoinObtainer { - ctxt, - data: self, - actions: vec![], - }; - - // The capability of the place should be Write because presumably something - // was moved out in either of the expansions. - self_obtainer.collapse_owned_places_and_lifetime_projections_to( - base_place, - CapabilityKind::Write, - format!( - "Join: collapse owned places {}", - other_expansion.place.display_string(ctxt) - ), - ctxt, - )?; - - pcg_validity_assert!( - self_obtainer.data.capabilities.get(base_place, ctxt) - == Some(CapabilityKind::Write), - "Expected capability for {} to be Write", - base_place.display_string(ctxt) - ); - - let mut other_obtainer = JoinObtainer { - ctxt, - data: other, - actions: vec![], - }; - - other_obtainer.collapse_owned_places_and_lifetime_projections_to( - base_place, - CapabilityKind::Write, - format!( - "Join: collapse owned places {}", - other_expansion.place.display_string(ctxt) - ), - ctxt, - )?; - - Ok(JoinDifferentExpansionsResult::Collapsed( - self_obtainer.actions, - )) - } - } - - fn expand_from_place_with_caps( - &mut self, - other: &JoinOwnedData<'a, 'pcg, 'tcx, &'pcg mut LocalExpansions<'tcx>>, - expansion: &ExpandedPlace<'tcx>, - self_cap: CapabilityKind, - other_cap: CapabilityKind, - ctxt: JoinCtxt<'a, 'tcx>, - ) -> Result>, PcgError> { - let place = expansion.place; - let guide = expansion.guide(); - pcg_validity_assert!(!self.owned.contains_expansion_from(place)); - if let Some(expand_cap) = self_cap.minimum(other_cap) { - tracing::debug!( - "Expanding from place {} with cap {:?}", - place.display_string(ctxt), - expand_cap - ); - let expand_action = RepackExpand::new(place, guide, expand_cap); - self.owned - .perform_expand_action(expand_action, self.capabilities, ctxt); - let mut actions = vec![RepackOp::Expand(expand_action)]; - actions.extend(self.join_all_places_in_expansion(other, expansion, ctxt)?); - Ok(actions) - } else if self_cap == CapabilityKind::Read { - pcg_validity_assert!(other_cap == CapabilityKind::Write); - tracing::debug!( - "No join expansion from place {}", - place.display_string(ctxt) - ); - let actions = self.join_leaf_read_and_write_capabilities(other, place, ctxt)?; - pcg_validity_assert!(!actions.is_empty()); - Ok(actions) - } else if self_cap == CapabilityKind::Write && other_cap == CapabilityKind::Read { - self.join_write_leaf_with_read_expansion(other, place, ctxt); - Ok(vec![]) - } else { - Ok(vec![]) - } - } - - /// Joins `place`, a leaf with [`CapabilityKind::Write`] in `self`, with - /// the read-only expansion of `place` in `other`. - /// - /// Neither capability is available on both paths, so the joined place - /// retains the expansion of `other` (whose places may be blocked by - /// borrows from that path) without any capability. - fn join_write_leaf_with_read_expansion( - &mut self, - other: &JoinOwnedData<'a, 'pcg, 'tcx, &'pcg mut LocalExpansions<'tcx>>, - place: Place<'tcx>, - ctxt: JoinCtxt<'a, 'tcx>, - ) { - tracing::debug!( - "Joining write leaf {} with read expansion to no capability", - place.display_string(ctxt) - ); - for expansion in other.owned.expansions() { - if place.is_prefix_of(expansion.place) { - self.owned - .insert_expansion(expansion.place, expansion.expansion.clone()); - } - } - self.capabilities.remove_all_postfixes(place, ctxt); - } - - fn join_all_places_in_expansion( - &mut self, - other: &JoinOwnedData<'a, 'pcg, 'tcx, &'pcg mut LocalExpansions<'tcx>>, - expansion: &ExpandedPlace<'tcx>, - ctxt: JoinCtxt<'a, 'tcx>, - ) -> Result>, PcgError> { - let mut actions = vec![]; - for expansion_place in expansion.expansion_places(ctxt).unwrap() { - actions.extend(self.join_owned_places(other, expansion_place, ctxt)?); - } - Ok(actions) - } - - /// See - fn join_other_expanded_place( - &mut self, - other: &mut JoinOwnedData<'a, 'pcg, 'tcx, &'pcg mut LocalExpansions<'tcx>>, - other_expansion: &ExpandedPlace<'tcx>, - ctxt: JoinCtxt<'a, 'tcx>, - ) -> Result, PcgError> { - tracing::debug!("Joining expansion: {other_expansion:?}"); - let place = other_expansion.place; - let self_expansions = self - .owned - .expansions_from(place) - .cloned() - .collect::>(); - if self_expansions.contains(other_expansion) { - // If some of our expansions have leafs, we want to merge with the other ones - // for example, to propagate read caps from the other borrowed places to our own - Ok(JoinExpandedPlaceResult::JoinedWithSameExpansion( - self.join_all_places_in_expansion(other, other_expansion, ctxt)?, - )) - } else if self_expansions.is_empty() - && let Some(self_cap) = self.capabilities.get(place, ctxt) - { - let other_cap = other.capabilities.get(place, ctxt); - tracing::debug!("Self cap: {self_cap:?}, Other cap: {other_cap:?}"); - if let Some(other_cap) = other_cap { - Ok(JoinExpandedPlaceResult::CreatedExpansion( - self.expand_from_place_with_caps( - other, - other_expansion, - self_cap, - other_cap, - ctxt, - )?, - )) - } else if other_expansion.is_enum_expansion() { - // The other expansion is a downcast to a variant that is - // presumably borrowed or partially-moved (see 206_issue_77.rs). - // It won't survive the join, so collapse it, emitting the - // operations: leaves with capability greater than write are weakened - // first, so that the emitted collape guarantees - // that all packed-up places hold exactly its capability. - let mut ops: Vec> = other - .capabilities - .owned_capabilities(place.local, ctxt) - .filter_map(|(p, k)| { - (place.is_prefix_of(p) && *k > CapabilityKind::Write).then(|| { - let weaken = RepackOp::weaken(p, *k, CapabilityKind::Write); - *k = CapabilityKind::Write; - weaken - }) - }) - .collect(); - ops.extend(other.owned.collapse( - place, - Some(CapabilityKind::Write), - other.capabilities, - ctxt, - )); - Ok(JoinExpandedPlaceResult::CollapsedOtherExpansion(ops)) - } else { - // We might as well just expand our place - let expand_action = RepackExpand::new(place, other_expansion.guide(), self_cap); - self.owned - .perform_expand_action(expand_action, self.capabilities, ctxt); - Ok(JoinExpandedPlaceResult::CreatedExpansion(vec![ - RepackOp::Expand(expand_action), - ])) - } - } else { - Ok(JoinExpandedPlaceResult::JoinedWithOtherExpansions( - self.join_different_expansions_from_place(other, other_expansion, ctxt)?, - )) - } - } - - fn visit_each_other_expansion_iteration( - &mut self, - other: &mut JoinOwnedData<'a, 'pcg, 'tcx, &'pcg mut LocalExpansions<'tcx>>, - ctxt: JoinCtxt<'a, 'tcx>, - ) -> Result>, PcgError> { - let expansions_shortest_first = other - .owned - .expansions_shortest_first() - .cloned() - .collect::>(); - let mut actions: Vec> = vec![]; - let mut collapsed_places: HashSet> = HashSet::default(); - for other_expansion in expansions_shortest_first { - if collapsed_places - .iter() - .any(|p| p.is_prefix_of(other_expansion.place)) - { - continue; - } - let result = self.join_other_expanded_place(other, &other_expansion, ctxt)?; - let performed_collapse = result.performed_collapse(); - if performed_collapse { - collapsed_places.insert(other_expansion.place); - } - actions.extend(result.actions()); - } - Ok(actions) - } - - fn visit_each_other_expansion( - &mut self, - mut other: JoinOwnedData<'a, 'pcg, 'tcx, &'pcg mut LocalExpansions<'tcx>>, - ctxt: JoinCtxt<'a, 'tcx>, - ) -> Result>, PcgError> { - let mut actions = vec![]; - let mut iteration = 0; - loop { - iteration += 1; - tracing::debug!("Iteration {iteration}"); - let iteration_actions = self.visit_each_other_expansion_iteration(&mut other, ctxt)?; - if iteration_actions.is_empty() { - break; - } - actions.extend(iteration_actions); - } - Ok(actions) - } - - fn render_debug_graph<'slf>( - &'slf self, - comment: &str, - ctxt: impl HasBorrowCheckerCtxt<'a, 'tcx>, - ) where - 'tcx: 'slf, - 'tcx: 'a, - { - self.borrows.graph.render_debug_graph( - self.snapshot_location.location().block, - Some(utils::DebugImgcat::JoinOwned), - self.capabilities, - comment, - ctxt.bc_ctxt(), - ); - } - - fn join_leaf_read_and_write_capabilities( - &mut self, - other: &JoinOwnedData<'a, 'pcg, 'tcx, &'pcg mut LocalExpansions<'tcx>>, - place: Place<'tcx>, - ctxt: JoinCtxt<'a, 'tcx>, - ) -> Result>, PcgError> { - pcg_validity_assert!(self.owned.is_leaf_place(place, ctxt)); - pcg_validity_assert!(other.owned.is_leaf_place(place, ctxt)); - pcg_validity_assert!(self.capabilities.get(place, ctxt) == Some(CapabilityKind::Read)); - pcg_validity_assert!(other.capabilities.get(place, ctxt) == Some(CapabilityKind::Write)); - let mut join_obtainer = JoinObtainer { - ctxt, - data: self, - actions: vec![], - }; - let action = BorrowPcgAction::label_place_and_update_related_capabilities( - place, - join_obtainer.prev_snapshot_location(), - LabelPlaceReason::JoinOwnedReadAndWriteCapabilities, - ); - join_obtainer.apply_action(PcgAction::Borrow(action))?; - join_obtainer - .data - .capabilities - .insert(place, CapabilityKind::Write, ctxt); - let mut actions = join_obtainer.actions; - // This side's read-lending has ended and the value it still holds is - // dropped by the join (the other side moved it out): restore the - // place to exclusive and then weaken it, so consumers see a - // capability restoration followed by an ordinary value drop. - actions.push(RepackOp::RegainLoanedCapability(RegainedCapability::new( - place, - CapabilityKind::Exclusive, - ))); - actions.push(RepackOp::weaken( - place, - CapabilityKind::Exclusive, - CapabilityKind::Write, - )); - Ok(actions) - } - - pub(crate) fn join_owned_places( - &mut self, - other: &JoinOwnedData<'a, 'pcg, 'tcx, &'pcg mut LocalExpansions<'tcx>>, - place: Place<'tcx>, - ctxt: JoinCtxt<'a, 'tcx>, - ) -> Result>, PcgError> { - if !self.owned.is_leaf_place(place, ctxt) || !other.owned.is_leaf_place(place, ctxt) { - return Ok(vec![]); - } - let Some(self_cap) = self.capabilities.get(place, ctxt) else { - return Ok(vec![]); - }; - let Some(other_cap) = other.capabilities.get(place, ctxt) else { - return Ok(vec![]); - }; - match (self_cap, other_cap) { - (capability_gte!(Read), CapabilityKind::Read) => { - copy_read_capabilities(other.capabilities, self.capabilities, place, ctxt); - Ok(vec![]) - } - (CapabilityKind::Read, CapabilityKind::Write) => { - self.join_leaf_read_and_write_capabilities(other, place, ctxt) - } - (CapabilityKind::Exclusive, CapabilityKind::Write) => { - for (p, c) in other - .capabilities - .capabilities_for_strict_postfixes_of(place) - { - self.capabilities.insert(p, c, ctxt); - } - Ok(vec![]) - } - (CapabilityKind::ShallowExclusive, CapabilityKind::Write) => { - // A reference lent through on this side but moved out on the - // other: the joined state can only be Write. Preserve the - // predecessor's value with a label before emitting the - // weaken; outstanding borrows still refer to that value. - let mut join_obtainer = JoinObtainer { - ctxt, - data: self, - actions: vec![], - }; - let action = BorrowPcgAction::label_place_and_update_related_capabilities( - place, - join_obtainer.prev_snapshot_location(), - LabelPlaceReason::JoinOwnedReadAndWriteCapabilities, - ); - join_obtainer.apply_action(PcgAction::Borrow(action))?; - join_obtainer - .data - .capabilities - .insert(place, CapabilityKind::Write, ctxt); - let mut actions = join_obtainer.actions; - actions.push(RepackOp::weaken( - place, - CapabilityKind::ShallowExclusive, - CapabilityKind::Write, - )); - Ok(actions) - } - ( - CapabilityKind::Read - | CapabilityKind::Write - | CapabilityKind::Exclusive - | CapabilityKind::ShallowExclusive, - _, - ) => Ok(vec![]), - } - } - - pub(crate) fn join( - mut self, - mut other: JoinOwnedData<'a, 'pcg, 'tcx, &'pcg mut LocalExpansions<'tcx>>, - ctxt: JoinCtxt<'a, 'tcx>, - ) -> Result>, PcgError> { - let mut actions: Vec> = Vec::new(); - if self.owned.has_expansions() || other.owned.has_expansions() { - actions.extend( - other - .reborrow() - .visit_each_other_expansion(self.reborrow(), ctxt)?, - ); - self.render_debug_graph("Self After join_expansions (other)", ctxt); - other.render_debug_graph("Other After join_expansions (other)", ctxt); - actions.extend( - self.reborrow() - .visit_each_other_expansion(other.reborrow(), ctxt)?, - ); - self.render_debug_graph("Self After join_expansions (self)", ctxt); - other.render_debug_graph("Other After join_expansions (self)", ctxt); - } else { - let place = Place::from(self.owned.get_local()); - actions.extend(self.reborrow().join_owned_places(&other, place, ctxt)?); - actions.extend(other.join_owned_places(&self, place, ctxt)?); - } - tracing::debug!( - "Join {} actions:\n\t{}", - Place::from(self.owned.get_local()).display_string(ctxt), - actions.debug_lines(ctxt).join("\n\t") - ); - tracing::debug!( - "Capabilities:\n\t{}", - self.capabilities.debug_lines(ctxt).join("\n\t") - ); - Ok(actions) - } -} - -impl<'tcx> LocalExpansions<'tcx> { - fn contains_descendent_leaf_node_without_capability<'a>( - &self, - place: Place<'tcx>, - place_capabilities: &impl PlaceCapabilitiesInterface<'tcx>, - ctxt: impl HasBorrowCheckerCtxt<'a, 'tcx>, - ) -> bool - where - 'tcx: 'a, - { - self.leaf_places(ctxt).into_iter().any(|p| { - let p_place: Place<'tcx> = p.into(); - if place.is_prefix_of(p_place) && place_capabilities.get(p_place, ctxt).is_none() { - tracing::debug!( - "Place {} is a leaf node without capability", - p.display_string(ctxt) - ); - true - } else { - false - } - }) - } -} - -fn copy_read_capabilities<'a, 'tcx: 'a>( - cap_source: &impl PlaceCapabilitiesInterface<'tcx>, - cap_target: &mut impl PlaceCapabilitiesInterface<'tcx>, - place: Place<'tcx>, - ctxt: impl HasBorrowCheckerCtxt<'a, 'tcx>, -) { - cap_target.insert(place, CapabilityKind::Read, ctxt); - for (p, c) in cap_source.capabilities_for_strict_postfixes_of(place) { - pcg_validity_assert!(c.is_read()); - cap_target.insert(p, c, ctxt); - } -} diff --git a/src/owned_pcg/impl/local/mod.rs b/src/owned_pcg/impl/local/mod.rs index f4ab27b1c..3d964b150 100644 --- a/src/owned_pcg/impl/local/mod.rs +++ b/src/owned_pcg/impl/local/mod.rs @@ -4,25 +4,19 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. -pub(crate) mod join; - use std::fmt::Debug; use crate::{ - HasSettings, borrow_pcg::borrow_pcg_expansion::PlaceExpansion, error::PcgUnsupportedError, owned_pcg::RepackGuide, - pcg::place_capabilities::{ - PlaceCapabilities, PlaceCapabilitiesInterface, PlaceCapabilitiesReader, - }, + pcg::place_capabilities::{PlaceCapabilities, PlaceCapabilitiesReader}, rustc_interface::middle::mir::Local, - utils::{DebugCtxt, HasCompilerCtxt, PlaceLike, SnapshotLocation, data_structures::HashSet}, + utils::{HasCompilerCtxt, PlaceLike, SnapshotLocation, data_structures::HashSet}, }; use itertools::Itertools; use crate::{ - owned_pcg::RepackOp, pcg::CapabilityKind, utils::{CompilerCtxt, OwnedPlace, Place, display::DisplayWithCompilerCtxt}, }; @@ -36,9 +30,6 @@ pub(crate) struct ExpandedPlace<'tcx, P = Place<'tcx>> { } impl ExpandedPlace<'_> { - pub(crate) fn is_enum_expansion(&self) -> bool { - self.expansion.is_enum_expansion() - } pub(crate) fn guide(&self) -> Option { self.expansion.guide() } @@ -109,22 +100,25 @@ impl<'tcx> LocalExpansions<'tcx> { !self.expansions.is_empty() } - pub(crate) fn remove_all_expansions_from<'a>( + pub(crate) fn insert_expansion(&mut self, place: Place<'tcx>, expansion: PlaceExpansion<'tcx>) { + self.expansions.insert(ExpandedPlace { place, expansion }); + } + + /// Removes the expansion to `place` and the expansions of `place` and of + /// the places below it. + pub(crate) fn forget_expansions_to<'a>( &mut self, place: Place<'tcx>, ctxt: impl HasCompilerCtxt<'a, 'tcx>, ) where 'tcx: 'a, { - tracing::debug!( - "Removing all expansions from {}", - place.display_string(ctxt.ctxt()) - ); - self.expansions.retain(|pe| pe.place != place); - } - - pub(crate) fn insert_expansion(&mut self, place: Place<'tcx>, expansion: PlaceExpansion<'tcx>) { - self.expansions.insert(ExpandedPlace { place, expansion }); + self.expansions.retain(|expanded| { + !place.is_prefix_of(expanded.place) + && !expanded + .expansion_places(ctxt) + .is_ok_and(|places| places.contains(&place)) + }); } pub(crate) fn expansions(&self) -> &HashSet> { @@ -150,19 +144,6 @@ impl<'tcx> LocalExpansions<'tcx> { .collect() } - pub(crate) fn is_leaf_place<'a>( - &self, - place: Place<'tcx>, - ctxt: impl HasCompilerCtxt<'a, 'tcx>, - ) -> bool - where - 'tcx: 'a, - { - place - .as_owned_place(ctxt) - .is_some_and(|p| self.leaf_places(ctxt).contains(&p)) - } - pub fn leaf_places<'a>(&self, ctxt: impl HasCompilerCtxt<'a, 'tcx>) -> HashSet> where 'tcx: 'a, @@ -212,10 +193,6 @@ impl<'tcx> LocalExpansions<'tcx> { .any(|e| e.expansion_places(ctxt).unwrap().contains(&place)) } - pub(crate) fn get_local(&self) -> Local { - self.local - } - pub(crate) fn places_to_collapse_for_obtain_of<'a>( &self, place: Place<'tcx>, @@ -239,34 +216,4 @@ impl<'tcx> LocalExpansions<'tcx> { places.push(place); places } - - pub(crate) fn collapse<'a, Ctxt: HasCompilerCtxt<'a, 'tcx> + DebugCtxt + HasSettings<'a>>( - &mut self, - to: Place<'tcx>, - _for_cap: Option, - capabilities: &mut impl PlaceCapabilitiesInterface<'tcx>, - ctxt: Ctxt, - ) -> Vec> - where - 'tcx: 'a, - { - if !self.contains_expansion_from(to) { - return vec![]; - } - let places_to_collapse = self.places_to_collapse_for_obtain_of(to, ctxt); - let ops: Vec> = places_to_collapse - .into_iter() - .flat_map(|place| { - let actions = self.collapse_actions_for(place, capabilities, ctxt); - actions - .into_iter() - .map(|action| { - self.perform_collapse_action(action, capabilities, ctxt); - RepackOp::Collapse(action) - }) - .collect::>() - }) - .collect(); - ops - } } diff --git a/src/owned_pcg/impl/mod.rs b/src/owned_pcg/impl/mod.rs index e56e2d523..f421b2be8 100644 --- a/src/owned_pcg/impl/mod.rs +++ b/src/owned_pcg/impl/mod.rs @@ -4,7 +4,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. -pub(crate) mod join; pub(crate) mod join_semi_lattice; mod local; mod update; diff --git a/src/owned_pcg/impl/update.rs b/src/owned_pcg/impl/update.rs index c24df81ec..f5040af9b 100644 --- a/src/owned_pcg/impl/update.rs +++ b/src/owned_pcg/impl/update.rs @@ -75,7 +75,7 @@ impl<'tcx> OwnedPcg<'tcx> { [ctxt] ); } - PlaceCondition::RemoveCapability(_) => unreachable!(), + PlaceCondition::Freeze(_) => unreachable!(), } } @@ -103,6 +103,21 @@ impl<'tcx> OwnedPcg<'tcx> { place_capabilities.insert(local.into(), CapabilityKind::Write, ctxt); } PlaceCondition::Capability(place, cap) => { + // A lent place that gets a new value (its loan is dead) does + // not get the old one back. A frozen place keeps its freeze, + // which the validity check rejects for a capability above R. + if !cap.is_read() { + let dropped = place_capabilities + .iter_suspended() + .map(|(p, _)| p) + .filter(|p| { + place.is_prefix_of(*p) && place_capabilities.get(*p, ctxt).is_none() + }) + .collect::>(); + for p in dropped { + place_capabilities.resume(p); + } + } place_capabilities.insert(place, cap, ctxt); // It's possible that the place could have been already expanded // exclusively (when it could have originally been expanded for @@ -123,9 +138,8 @@ impl<'tcx> OwnedPcg<'tcx> { PlaceCondition::ExpandTwoPhase(place) => { place_capabilities.insert(place, CapabilityKind::Read, ctxt); } - PlaceCondition::RemoveCapability(place) => { - place_capabilities.remove(place, ctxt); - } + // The place is frozen by `PcgVisitor::ensure_triple`. + PlaceCondition::Freeze(_) => {} } } } diff --git a/src/owned_pcg/results/repacks.rs b/src/owned_pcg/results/repacks.rs index 86d2dcc1b..db7a25621 100644 --- a/src/owned_pcg/results/repacks.rs +++ b/src/owned_pcg/results/repacks.rs @@ -6,10 +6,7 @@ use std::marker::PhantomData; -use crate::{ - Weaken, pcg_validity_assert, - rustc_interface::middle::mir::{self, PlaceElem}, -}; +use crate::rustc_interface::middle::mir::{self, PlaceElem}; use crate::{ pcg::CapabilityKind, @@ -323,19 +320,6 @@ impl<'a, 'tcx: 'a, Ctxt: HasCompilerCtxt<'a, 'tcx>> DebugRepr for RepackEx } impl<'tcx> RepackExpand<'tcx> { - pub(crate) fn new( - from: Place<'tcx>, - guide: Option, - capability: CapabilityKind, - ) -> Self { - Self { - from, - guide, - capability, - _marker: PhantomData, - } - } - #[must_use] pub fn capability(&self) -> CapabilityKind { self.capability @@ -453,19 +437,6 @@ pub enum RepackOp<'tcx, Local = mir::Local, Place = crate::utils::Place<'tcx>, G /// handling blocks, but can also appear in regular code for example in the MIR of /// [this function](https://github.com/dtolnay/syn/blob/3da56a712abf7933b91954dbfb5708b452f88504/src/attr.rs#L623-L628). StorageDead(Local), - /// This Op only appears within a `BasicBlock` and is attached to a - /// [`mir::StatementKind::StorageDead`](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/enum.StatementKind.html#variant.StorageDead) - /// statement. We emit it for any such statement where the local may already be dead. We - /// guarantee to have inserted a [`RepackOp::StorageDead`] before this Op so that one can - /// safely ignore the statement this is attached to. - IgnoreStorageDead(Local), - /// Instructs that the current capability to the place (first [`CapabilityKind`]) should - /// be weakened to the second given capability. We guarantee that `_.1 > _.2`. - /// - /// This Op is used prior to a [`RepackOp::Collapse`] to ensure that all packed up places have - /// the same capability. It can also appear at basic block join points, where one branch has - /// a weaker capability than the other. - Weaken(Weaken<'tcx, Place, CapabilityKind>), /// Instructs that one should unpack `place` with the capability. /// We guarantee that the current state holds exactly the given capability for the given place. /// `guide` denotes e.g. the enum variant to unpack to. One can use @@ -480,47 +451,6 @@ pub enum RepackOp<'tcx, Local = mir::Local, Place = crate::utils::Place<'tcx>, G Collapse(RepackCollapse<'tcx, Place, Guide>), /// TODO DerefShallowInit(Place, Place), - /// This place should have its capability changed from `Lent` (for mutably - /// borrowed places) or `Read` (for shared borrow places), to the given - /// capability, because it is no longer lent out. - RegainLoanedCapability(RegainedCapability), -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] -#[cfg_attr(feature = "type-export", derive(ts_rs::TS))] -#[cfg_attr(feature = "type-export", ts(export))] -pub struct RegainedCapability { - pub(crate) place: Place, - pub(crate) capability: CapabilityKind, -} - -impl RegainedCapability { - pub fn new(place: Place, capability: CapabilityKind) -> Self { - Self { place, capability } - } - - pub fn place(&self) -> Place - where - Place: Copy, - { - self.place - } - - pub fn capability(&self) -> CapabilityKind { - self.capability - } -} - -impl<'a, 'tcx: 'a, Ctxt: HasCompilerCtxt<'a, 'tcx>> DebugRepr - for RegainedCapability> -{ - type Repr = RegainedCapability; - fn debug_repr(&self, ctxt: Ctxt) -> Self::Repr { - RegainedCapability { - place: self.place.display_string(ctxt), - capability: self.capability, - } - } } impl<'a, 'tcx: 'a, Ctxt: HasCompilerCtxt<'a, 'tcx>> DebugRepr for RepackOp<'tcx> { @@ -528,13 +458,6 @@ impl<'a, 'tcx: 'a, Ctxt: HasCompilerCtxt<'a, 'tcx>> DebugRepr for RepackOp fn debug_repr(&self, ctxt: Ctxt) -> Self::Repr { match self { RepackOp::StorageDead(local) => RepackOp::StorageDead(local.display_string(ctxt)), - RepackOp::IgnoreStorageDead(local) => { - RepackOp::IgnoreStorageDead(local.display_string(ctxt)) - } - RepackOp::RegainLoanedCapability(regained_capability) => { - RepackOp::RegainLoanedCapability(regained_capability.debug_repr(ctxt)) - } - RepackOp::Weaken(weaken) => RepackOp::Weaken(weaken.debug_repr(ctxt)), RepackOp::Expand(expand) => RepackOp::Expand(expand.debug_repr(ctxt)), RepackOp::Collapse(collapse) => RepackOp::Collapse(collapse.debug_repr(ctxt)), RepackOp::DerefShallowInit(place, place2) => { @@ -550,13 +473,6 @@ impl> DisplayWithCtxt fn display_output(&self, ctxt: Ctxt, _mode: OutputMode) -> DisplayOutput { DisplayOutput::Text( match self { - RepackOp::RegainLoanedCapability(regained_capability) => { - format!( - "Restore capability {:?} to {}", - regained_capability.capability, - regained_capability.place.display_string(ctxt), - ) - } RepackOp::Expand(expand) => format!( "unpack {} with capability {:?}", expand.from.display_string(ctxt), @@ -570,28 +486,6 @@ impl> DisplayWithCtxt } impl<'tcx> RepackOp<'tcx> { - /// Create a Weaken, checking that it strictly reduces the capability. - pub(crate) fn weaken(place: Place<'tcx>, from: CapabilityKind, to: CapabilityKind) -> Self { - pcg_validity_assert!( - from > to, - "a weaken must strictly reduce the capability ({from:?} -> {to:?})" - ); - Self::Weaken(Weaken::new(place, from, to)) - } - - /// Temporary: create a Weaken for `StorageDead`. - /// Required until is resolved. - pub(crate) fn weaken_for_storage_dead( - place: Place<'tcx>, - from: CapabilityKind, - to: CapabilityKind, - ) -> Self { - pcg_validity_assert!( - from > to, - "a weaken must strictly reduce the capability ({from:?} -> {to:?})" - ); - Self::Weaken(Weaken::new_for_storage_dead(place, from, to)) - } pub(crate) fn expand<'a>( from: Place<'tcx>, guide: Option, @@ -611,11 +505,9 @@ impl<'tcx> RepackOp<'tcx> { #[must_use] pub fn affected_place(&self) -> Place<'tcx> { match *self { - RepackOp::StorageDead(local) | RepackOp::IgnoreStorageDead(local) => local.into(), - RepackOp::Weaken(Weaken { place, .. }) - | RepackOp::Collapse(RepackCollapse { to: place, .. }) + RepackOp::StorageDead(local) => local.into(), + RepackOp::Collapse(RepackCollapse { to: place, .. }) | RepackOp::Expand(RepackExpand { from: place, .. }) - | RepackOp::RegainLoanedCapability(RegainedCapability { place, .. }) | RepackOp::DerefShallowInit(place, _) => place, } } diff --git a/src/pcg/domain/join.rs b/src/pcg/domain/join.rs new file mode 100644 index 000000000..2ee17d797 --- /dev/null +++ b/src/pcg/domain/join.rs @@ -0,0 +1,996 @@ +//! The join of PCG states. The joined state holds the meet of the +//! predecessors' capabilities, and each predecessor reaches it through +//! explicit actions. + +use std::cmp::Reverse; + +use crate::{ + action::{AppliedActions, BorrowPcgAction, OwnedPcgAction, PcgAction}, + borrow_pcg::{ + action::LabelPlaceReason, + borrow_pcg_edge::{BorrowPcgEdge, BorrowPcgEdgeLike}, + edge::kind::BorrowPcgEdgeKind, + edge_data::EdgeData, + graph::BorrowsGraph, + }, + error::PcgError, + r#loop::PlaceUsageType, + owned_pcg::RepackOp, + pcg::{ + CapabilityKind, PcgMutRef, PcgRefLike, + obtain::{HasSnapshotLocation, ObtainType, PlaceObtainer, expand::PlaceExpander}, + owned_state::OwnedPcg, + place_capabilities::{ + PlaceCapabilities, PlaceCapabilitiesInterface, PlaceCapabilitiesReader, + }, + }, + pcg_validity_assert, + rustc_interface::middle::mir::{self, ProjectionElem}, + utils::{ + DataflowCtxt, HasBorrowCheckerCtxt, HasCompilerCtxt, HasPlace, Place, PlaceLike, + SnapshotLocation, + data_structures::HashSet, + display::{DebugLines, DisplayWithCompilerCtxt}, + }, +}; + +use super::{Pcg, is_variant_unpacking}; + +/// The capability of a place in the join of two paths that hold it with `a` +/// and `b`: the lower one. A reference that one path holds frozen (R) and +/// the other with its referent lent (e) keeps its own value: e. A place that +/// one path holds frozen and the other has moved out (W) can only be +/// overwritten: W. Neither holds while a loan keeps the place frozen (see +/// `Pcg::joined_capabilities`). +pub(crate) fn meet(a: CapabilityKind, b: CapabilityKind) -> CapabilityKind { + match (a, b) { + (CapabilityKind::Read, CapabilityKind::ShallowExclusive) + | (CapabilityKind::ShallowExclusive, CapabilityKind::Read) => { + CapabilityKind::ShallowExclusive + } + _ => a.minimum(b).unwrap_or(CapabilityKind::Write), + } +} + +/// The capability `place` has in `capabilities` or, if it has none, gets by +/// unpacking its closest ancestor that has one. Unpacking enters an enum +/// variant only if `enter_variants`: a variant is known only on the paths +/// that matched it. +fn unpacked_capability<'a, 'tcx: 'a>( + capabilities: &PlaceCapabilities<'tcx>, + place: Place<'tcx>, + enter_variants: bool, + ctxt: impl HasCompilerCtxt<'a, 'tcx>, +) -> Option { + unpacked(|p| capabilities.get(p, ctxt), place, enter_variants, ctxt) +} + +/// The capability `place` has in `lookup` or, if it has none, gets by +/// unpacking its closest ancestor that has one; see [`unpacked_capability`]. +pub(crate) fn unpacked<'a, 'tcx: 'a>( + lookup: impl Fn(Place<'tcx>) -> Option, + place: Place<'tcx>, + enter_variants: bool, + ctxt: impl HasCompilerCtxt<'a, 'tcx>, +) -> Option { + let (ancestor, mut capability) = std::iter::successors(Some(place), |p| p.parent_place()) + .find_map(|p| Some((p, lookup(p)?)))?; + for (base, elem) in place.iter_projections_after(ancestor, ctxt)? { + capability = match (elem, capability) { + (ProjectionElem::Downcast(..), _) if !enter_variants => return None, + (ProjectionElem::Deref, _) if base.is_raw_ptr(ctxt) => return None, + (ProjectionElem::Deref, CapabilityKind::Exclusive) if !base.is_shared_ref(ctxt) => { + CapabilityKind::Exclusive + } + (ProjectionElem::Deref, CapabilityKind::Exclusive | CapabilityKind::Read) => { + CapabilityKind::Read + } + (ProjectionElem::Deref, _) => return None, + _ => capability, + }; + } + Some(capability) +} + +/// The capability `place` gets when what lies below it is packed into it: +/// the meet of the capabilities below it, none if a leaf of its layout (an +/// owned expansion or a borrow-PCG expansion) has none, by itself or from a +/// place it is part of (it is borrowed). +fn packed_capability<'a, 'tcx: 'a>( + capabilities: &PlaceCapabilities<'tcx>, + owned: &OwnedPcg<'tcx>, + graph: &BorrowsGraph<'tcx>, + place: Place<'tcx>, + ctxt: impl HasCompilerCtxt<'a, 'tcx>, +) -> Option { + if let Some(capability) = unpacked_capability(capabilities, place, false, ctxt) { + return Some(capability); + } + let owned_leaves = owned + .is_allocated(place.local) + .then(|| owned[place.local].expansions().leaf_places(ctxt)) + .into_iter() + .flatten() + .map(Place::from); + let borrowed_expansions = graph + .edges + .keys() + .filter_map(|kind| match kind { + BorrowPcgEdgeKind::BorrowPcgExpansion(expansion) => Some(expansion), + _ => None, + }) + .collect::>(); + let borrowed_leaves = borrowed_expansions + .iter() + .flat_map(|expansion| { + expansion + .expansion() + .iter() + .filter_map(|node| node.as_current_place()) + .collect::>() + }) + .filter(|part| { + !borrowed_expansions + .iter() + .any(|expansion| expansion.base().as_current_place() == Some(*part)) + }); + if owned_leaves.chain(borrowed_leaves).any(|leaf| { + place.is_prefix_of(leaf) && unpacked_capability(capabilities, leaf, true, ctxt).is_none() + }) { + return None; + } + capabilities + .iter() + .filter(|(p, _)| place.is_strict_prefix_of(*p)) + .map(|(_, capability)| capability) + .reduce(meet) +} + +impl<'a, 'tcx: 'a> Pcg<'a, 'tcx> { + /// Joins `other` into this state, which reaches the meet of both. + pub(crate) fn join_with( + &mut self, + other: &Self, + self_block: mir::BasicBlock, + snapshot: SnapshotLocation, + ctxt: impl DataflowCtxt<'a, 'tcx>, + ) -> Result<(), PcgError> { + let joined = self.joined_capabilities(other, ctxt); + self.reach(&joined, other, self_block, snapshot, ctxt)?; + Ok(()) + } + + /// The capabilities of the join of this state and `other`: the meet of + /// the capabilities each path has or gets by unpacking. Where one path + /// holds the place frozen (R) and the meet is more than R (e or W), there + /// is none while a loan of either path blocks the place or a place it is + /// part of: the loan keeps it frozen. Where the paths unpack different + /// variants of an enum, the join holds the enum packed. + fn joined_capabilities( + &self, + other: &Self, + ctxt: impl HasBorrowCheckerCtxt<'a, 'tcx>, + ) -> PlaceCapabilities<'tcx> { + let places = self + .place_capabilities + .iter() + .chain(other.place_capabilities.iter()) + .map(|(place, _)| place) + .collect::>(); + let lent = self + .borrow + .graph + .lent_places(ctxt.bc_ctxt()) + .into_iter() + .chain(other.borrow.graph.lent_places(ctxt.bc_ctxt())) + .collect::>(); + let mut joined = PlaceCapabilities::default(); + for place in places { + let capability = match ( + unpacked_capability(&self.place_capabilities, place, false, ctxt), + unpacked_capability(&other.place_capabilities, place, false, ctxt), + ) { + (Some(a), Some(b)) => Some(meet(a, b)) + .filter(|capability| { + capability.is_read() + || !(a.is_read() || b.is_read()) + || !lent.iter().any(|lent| lent.is_prefix_of(place)) + }) + .map(|capability| (place, capability)), + (Some(_), None) | (None, Some(_)) => self + .diverging_variant_base(other, place, ctxt) + .and_then(|base| { + Some(( + base, + meet( + packed_capability( + &self.place_capabilities, + &self.owned, + &self.borrow.graph, + base, + ctxt, + )?, + packed_capability( + &other.place_capabilities, + &other.owned, + &other.borrow.graph, + base, + ctxt, + )?, + ), + )) + }), + (None, None) => None, + }; + if let Some((place, capability)) = capability { + joined.insert(place, capability, ctxt); + } + } + let packed = joined + .iter() + .filter(|(_, capability)| { + matches!( + capability, + CapabilityKind::Exclusive | CapabilityKind::Write + ) + }) + .map(|(place, _)| place) + .collect::>(); + joined.retain(|place, _| !packed.iter().any(|p| p.is_strict_prefix_of(place))); + joined + } + + /// The enum place above `place` whose variant only one of the paths has + /// unpacked. + fn diverging_variant_base( + &self, + other: &Self, + place: Place<'tcx>, + ctxt: impl HasCompilerCtxt<'a, 'tcx>, + ) -> Option> { + let knows = |pcg: &Self, variant: Place<'tcx>| { + pcg.place_capabilities + .iter() + .any(|(p, _)| variant.is_prefix_of(p)) + }; + let projections = place + .iter_projections_after(place.local.into(), ctxt)? + .collect::>(); + projections + .iter() + .enumerate() + .find_map(|(i, (base, elem))| { + let variant = projections.get(i + 1).map_or(place, |(p, _)| *p); + (matches!(elem, ProjectionElem::Downcast(..)) + && knows(self, variant) != knows(other, variant)) + .then_some(*base) + }) + } + + /// Gives this state the capabilities `target`, recording the actions. + /// `other` is the state across the join: the places this state holds and + /// `target` does not are lent to the edges of `other` that block them. + pub(crate) fn reach( + &mut self, + target: &PlaceCapabilities<'tcx>, + other: &Self, + self_block: mir::BasicBlock, + snapshot: SnapshotLocation, + ctxt: impl DataflowCtxt<'a, 'tcx>, + ) -> Result>, PcgError> { + let mut applied = AppliedActions::default(); + PlaceObtainer::new( + PcgMutRef::from(&mut *self), + Some(&mut applied), + ctxt, + snapshot.location(), + snapshot, + ) + .reach(target, other, self_block)?; + self.owned.join_capabilities(&other.owned); + pcg_validity_assert!( + self.place_capabilities.map == target.map, + [ctxt], + "the join reaches {:?} instead of {:?}", + self.place_capabilities.debug_lines(ctxt), + target.debug_lines(ctxt) + ); + Ok(applied.actions.into_iter().map(|a| a.action).collect()) + } + + /// Gives this state the owned layout that `other` has below the places + /// neither holds or both hold frozen: the parts that are lent or blocked + /// on the other path (a variant field borrowed while the join holds the + /// enum packed). It changes no capability. + pub(crate) fn join_lent_layout(&mut self, other: &Self, ctxt: impl DataflowCtxt<'a, 'tcx>) { + for local in (0..self.owned.num_locals()).map(mir::Local::from) { + if !self.owned.is_allocated(local) || !other.owned.is_allocated(local) { + continue; + } + let mut expansions = other.owned[local] + .expansions() + .expansions() + .iter() + .cloned() + .collect::>(); + expansions.sort_by_key(|expanded| expanded.place.projection.len()); + for expanded in expansions { + let layout = self.owned[local].expansions(); + if layout.expansions().contains(&expanded) + || !layout.contains_place(expanded.place, ctxt.bc_ctxt()) + { + continue; + } + pcg_validity_assert!( + self.place_capabilities + .get(expanded.place, ctxt) + .is_none_or(CapabilityKind::is_read), + [ctxt], + "the join unpacks {}, which holds more than R", + expanded.place.display_string(ctxt.bc_ctxt()) + ); + self.owned[local] + .expansions_mut() + .insert_expansion(expanded.place, expanded.expansion); + } + } + } + + /// The capabilities that the join of `states` suspends: at the finest + /// places that every state records or covers (a recorded place or one + /// above it, such as the enum whose variants the states unpack + /// differently), the meet of what each state has for them. A reference + /// and its referent are separate places. + pub(crate) fn joined_suspended( + states: &[&Self], + ctxt: impl HasCompilerCtxt<'a, 'tcx>, + ) -> PlaceCapabilities<'tcx> { + let covered = states + .iter() + .flat_map(|state| state.place_capabilities.iter_suspended()) + .flat_map(|(place, _)| std::iter::successors(Some(place), |p| p.parent_place())) + .collect::>() + .into_iter() + .filter_map(|place| { + let capability = states + .iter() + .map(|state| state.covered_suspended(place, ctxt)) + .collect::>>()? + .into_iter() + .reduce(meet)?; + Some((place, capability)) + }) + .collect::>(); + let mut joined = PlaceCapabilities::default(); + for &(place, capability) in &covered { + if !covered.iter().any(|(part, _)| { + place.is_strict_prefix_of(*part) && !part.projects_indirection_from(place, ctxt) + }) { + joined.suspend(place, capability); + } + } + joined + } + + /// The capability suspended for `place`, or for it as part of a lent + /// ancestor, or for its lent parts together when nothing of its value is + /// held. + fn covered_suspended( + &self, + place: Place<'tcx>, + ctxt: impl HasCompilerCtxt<'a, 'tcx>, + ) -> Option { + let in_value = + |p: Place<'tcx>| place.is_prefix_of(p) && !p.projects_indirection_from(place, ctxt); + unpacked(|p| self.place_capabilities.suspended(p), place, false, ctxt).or_else(|| { + if self.place_capabilities.iter().any(|(p, _)| in_value(p)) { + return None; + } + self.place_capabilities + .iter_suspended() + .filter(|(p, _)| in_value(*p)) + .map(|(_, capability)| capability) + .reduce(meet) + }) + } + + /// Gives this state the suspended capabilities `target`, recording the + /// actions. + pub(crate) fn reach_suspended( + &mut self, + target: &PlaceCapabilities<'tcx>, + snapshot: SnapshotLocation, + ctxt: impl DataflowCtxt<'a, 'tcx>, + ) -> Result>, PcgError> { + let mut applied = AppliedActions::default(); + PlaceObtainer::new( + PcgMutRef::from(self), + Some(&mut applied), + ctxt, + snapshot.location(), + snapshot, + ) + .reach_suspended(target)?; + Ok(applied.actions.into_iter().map(|a| a.action).collect()) + } +} + +/// Where the places a path gives up at a join are lent: to the edges of the +/// state across the join that the path did not have before the join. +struct Lending<'r, 'a, 'tcx> { + other: &'r Pcg<'a, 'tcx>, + own_edges: &'r HashSet>, + self_block: mir::BasicBlock, +} + +impl<'a, 'tcx: 'a> Lending<'_, 'a, 'tcx> { + /// The edge that `blocked` is lent to: one that is a loan, not an + /// unpacking (except of an enum variant this path did not match). + fn edge( + &self, + blocked: Place<'tcx>, + ctxt: impl HasBorrowCheckerCtxt<'a, 'tcx>, + ) -> Option> { + self.other + .borrow + .graph() + .edges_blocking(blocked.into(), ctxt.bc_ctxt()) + .filter(|edge| { + (is_variant_unpacking(edge.kind()) || edge.kind().is_loan()) + && !self.own_edges.contains(edge.kind()) + }) + .min_by_key(|edge| format!("{:?}", edge.kind())) + .map(|edge| { + let mut edge = edge.to_owned_edge(); + edge.conditions = edge.conditions.without_branch_choice(self.self_block); + edge + }) + } +} + +impl<'pcg, 'a: 'pcg, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> PlaceObtainer<'pcg, 'a, 'tcx, Ctxt> { + fn reach( + &mut self, + target: &PlaceCapabilities<'tcx>, + other: &Pcg<'a, 'tcx>, + self_block: mir::BasicBlock, + ) -> Result<(), PcgError> { + for local in (0..self.pcg.owned.num_locals()).map(mir::Local::from) { + if self.pcg.owned.is_allocated(local) && !other.owned.is_allocated(local) { + self.storage_dead(local)?; + } + } + let own_edges = self + .pcg + .borrow + .graph + .edges + .keys() + .cloned() + .collect::>(); + let lending = Lending { + other, + own_edges: &own_edges, + self_block, + }; + self.move_capabilities_to(target, &lending)?; + self.unpack_to(target, other)?; + self.move_capabilities_to(target, &lending)?; + self.pack_to(target)?; + self.give_up_places_outside(target, &lending) + } + + /// Gives this state the suspended capabilities `target`: each place of + /// `target` gets its record by unpacking the record of a lent ancestor + /// or packing those of its lent parts, then weakened to the target; the + /// other records are dropped. + pub(crate) fn reach_suspended( + &mut self, + target: &PlaceCapabilities<'tcx>, + ) -> Result<(), PcgError> { + let mut places = target.iter_suspended().collect::>(); + places.sort_by_key(|(place, _)| (place.projection.len(), *place)); + for &(place, _) in &places { + if self.pcg.place_capabilities.suspended(place).is_none() { + match std::iter::successors(place.parent_place(), |p| p.parent_place()) + .find(|p| self.pcg.place_capabilities.suspended(*p).is_some()) + { + Some(ancestor) => self.unpack_suspended_to(ancestor, place)?, + None => self.record_and_apply_action( + BorrowPcgAction::pack_suspended(place, "join").into(), + )?, + } + } + } + for (place, to) in places { + if let Some(from) = self.pcg.place_capabilities.suspended(place) + && meet(from, to) != from + { + self.record_and_apply_action( + BorrowPcgAction::weaken_suspended(place, from, Some(meet(from, to)), "join") + .into(), + )?; + } + } + let mut dropped = self + .pcg + .place_capabilities + .iter_suspended() + .filter(|(place, _)| target.suspended(*place).is_none()) + .collect::>(); + dropped.sort_by_key(|(place, _)| *place); + for (place, from) in dropped { + self.record_and_apply_action( + BorrowPcgAction::weaken_suspended(place, from, None, "join").into(), + )?; + } + Ok(()) + } + + /// Unpacks the capability suspended for `ancestor` level by level down to + /// `place`. + fn unpack_suspended_to( + &mut self, + ancestor: Place<'tcx>, + place: Place<'tcx>, + ) -> Result<(), PcgError> { + for (base, _) in place + .iter_projections_after(ancestor, self.ctxt) + .into_iter() + .flatten() + { + let parts = base.expand_one_level(place, self.ctxt)?.expansion(); + self.record_and_apply_action( + BorrowPcgAction::unpack_suspended(base, parts, "join").into(), + )?; + } + Ok(()) + } + + fn storage_dead(&mut self, local: mir::Local) -> Result<(), PcgError> { + self.record_and_apply_action( + BorrowPcgAction::label_place_and_update_related_capabilities( + local.into(), + self.prev_snapshot_location(), + LabelPlaceReason::StorageDead, + ) + .into(), + )?; + self.obtain(local.into(), ObtainType::ForStorageDead)?; + self.record_and_apply_action( + OwnedPcgAction::new(RepackOp::StorageDead(local), Some("join".into())).into(), + ) + } + + /// Unpacks this state down to the places `target` holds; into a loop + /// head, as the loop invariant unpacks them. + fn unpack_to( + &mut self, + target: &PlaceCapabilities<'tcx>, + other: &Pcg<'a, 'tcx>, + ) -> Result<(), PcgError> { + let mut places = target.iter().map(|(place, _)| place).collect::>(); + places.sort_by_key(|place| place.projection.len()); + for place in places { + if self.pcg.place_capabilities.get(place, self.ctxt).is_none() + && let Some(capability) = + unpacked_capability(self.pcg.place_capabilities, place, true, self.ctxt) + { + let obtain_type = match self.prev_snapshot_location() { + SnapshotLocation::Loop(_) => ObtainType::LoopInvariant { + usage_type: if capability.is_read() { + PlaceUsageType::Read + } else { + PlaceUsageType::Exclusive + }, + }, + _ => ObtainType::Capability(capability), + }; + self.expand_to(place, Some(other.borrow.graph()), obtain_type, self.ctxt)?; + } + } + Ok(()) + } + + /// Moves the capability of each place this state holds to the one it + /// has or gets by unpacking in `target`; a frozen place that `target` + /// unpacks exclusively is unfrozen to be unpacked, one whose parts + /// `target` only reads has its freeze moved to its read copies, and a + /// place this state unpacks exclusively that `target` reads is frozen. + /// Read copies follow the place they are copied from. + fn move_capabilities_to( + &mut self, + target: &PlaceCapabilities<'tcx>, + lending: &Lending<'_, 'a, 'tcx>, + ) -> Result<(), PcgError> { + let mut places = self + .pcg + .place_capabilities + .iter() + .chain(target.iter().filter(|(_, capability)| capability.is_read())) + .map(|(place, _)| place) + .collect::>() + .into_iter() + .collect::>(); + places.sort_by_key(|place| place.projection.len()); + for place in places { + let Some(from) = self.pcg.place_capabilities.get(place, self.ctxt) else { + if target.get(place, self.ctxt) == Some(CapabilityKind::Read) + && self.pcg.place_capabilities.unpacks(place) + { + self.record_and_apply_action(BorrowPcgAction::freeze(place, "join").into())?; + } + continue; + }; + let is_read_copy = from == CapabilityKind::Read + && place.parent_place().is_some_and(|parent| { + self.pcg + .place_capability_equals(parent, CapabilityKind::Read, self.ctxt) + }); + let unpacked = + target.get(place, self.ctxt) != Some(CapabilityKind::Read) && target.unpacks(place); + if unpacked + && from == CapabilityKind::Read + && !is_read_copy + && target + .capabilities_for_strict_postfixes_of(place) + .all(|(_, capability)| capability.is_read()) + { + self.refreeze_to_copies(place)?; + continue; + } + let to = if unpacked { + (from == CapabilityKind::Read).then_some(CapabilityKind::Exclusive) + } else { + unpacked_capability(target, place, true, self.ctxt) + }; + if let Some(to) = to + && to != from + && !is_read_copy + { + self.move_capability(place, from, to, target, lending)?; + } + } + Ok(()) + } + + /// Moves the freeze of `place` to its read copies one level down, if it + /// has been unpacked. + fn refreeze_to_copies(&mut self, place: Place<'tcx>) -> Result<(), PcgError> { + let mut copies = self + .pcg + .place_capabilities + .capabilities_for_strict_postfixes_of(place) + .filter(|(copy, _)| copy.parent_place() == Some(place)) + .map(|(copy, _)| copy) + .collect::>(); + copies.sort(); + if copies.is_empty() { + return Ok(()); + } + self.record_and_apply_action(BorrowPcgAction::refreeze(place, copies, "join").into()) + } + + fn move_capability( + &mut self, + place: Place<'tcx>, + from: CapabilityKind, + to: CapabilityKind, + target: &PlaceCapabilities<'tcx>, + lending: &Lending<'_, 'a, 'tcx>, + ) -> Result<(), PcgError> { + match (from, to) { + (CapabilityKind::ShallowExclusive, CapabilityKind::Exclusive) => { + self.remove_edges_absent_from(place, lending.other) + } + ( + CapabilityKind::Exclusive | CapabilityKind::ShallowExclusive, + CapabilityKind::Read, + ) => self.record_and_apply_action(BorrowPcgAction::freeze(place, "join").into()), + (CapabilityKind::Read | CapabilityKind::ShallowExclusive, CapabilityKind::Write) => { + self.obtain(place, ObtainType::Capability(CapabilityKind::Write))?; + self.weaken_to(place, to) + } + (CapabilityKind::Read, _) => { + self.upgrade_read_to_exclusive(place, place, &|base| target.unpacks(base))?; + self.weaken_to(place, to) + } + // The referent of a mutable reference is lent, through the + // reference, to the edge of `other` that blocks it. + (CapabilityKind::Exclusive, CapabilityKind::ShallowExclusive) + if place.is_mut_ref(self.ctxt) + && let Some(edge) = lending.edge(place.project_deref(self.ctxt), self.ctxt) => + { + self.record_and_apply_action( + BorrowPcgAction::lend(place.project_deref(self.ctxt), edge, "join").into(), + ) + } + ( + CapabilityKind::Exclusive, + CapabilityKind::ShallowExclusive | CapabilityKind::Write, + ) => self.weaken_to(place, to), + _ => { + let ctxt = self.ctxt; + pcg_validity_assert!( + false, + [ctxt], + "{} cannot go from {from:?} to {to:?} at a join", + place.display_string(self.ctxt.bc_ctxt()) + ); + Ok(()) + } + } + } + + /// Removes the edges that `other` does not have and that block `place`, + /// its postfixes or, transitively, what they are lent to, leaves first, + /// regaining the capabilities they block. Into a loop head, its loop + /// abstraction stands for them. + fn remove_edges_absent_from( + &mut self, + place: Place<'tcx>, + other: &Pcg<'a, 'tcx>, + ) -> Result<(), PcgError> { + let ctxt = self.ctxt; + let mut lent = vec![place]; + let mut absent = vec![]; + let mut changed = true; + while changed { + changed = false; + for edge in self.pcg.borrow.graph.edges() { + let kind = edge.kind(); + if absent.contains(kind) + || other.borrow.graph().edges.contains_key(kind) + || !kind.blocked_nodes(ctxt.bc_ctxt()).any(|node| { + node.related_maybe_labelled_place() + .is_some_and(|p| lent.iter().any(|l| l.is_prefix_of(p.place()))) + }) + { + continue; + } + lent.extend( + kind.blocked_by_nodes(ctxt.bc_ctxt()) + .map(|node| node.place()), + ); + absent.push(kind.clone()); + changed = true; + } + } + loop { + let fg = self.pcg.borrow.graph.frozen_graph(); + let Some(edge) = fg + .leaf_edges(ctxt) + .into_iter() + .map(BorrowPcgEdgeLike::to_owned_edge) + .find(|edge| absent.contains(edge.kind())) + else { + return Ok(()); + }; + self.remove_edge_and_perform_associated_state_updates(&edge, "join")?; + } + } + + /// Weakens `place` to `to` if it holds more. + fn weaken_to(&mut self, place: Place<'tcx>, to: CapabilityKind) -> Result<(), PcgError> { + match self.pcg.capability_of(place, self.ctxt) { + Some(from) if from > to => self + .record_and_apply_action(BorrowPcgAction::weaken(place, from, to, "join").into()), + _ => Ok(()), + } + } + + /// Weakens to [`CapabilityKind::Write`] the owned places that are not + /// definitely initialized here, held or suspended, and the local + /// references that are dead here. + pub(super) fn weaken_maybe_uninitialized(&mut self) -> Result<(), PcgError> { + let ctxt = self.ctxt; + let location = self.location(); + let body_analysis = ctxt.body_analysis(); + let suspended = self + .pcg + .place_capabilities + .iter_suspended() + .filter(|(place, capability)| { + *capability > CapabilityKind::Write + && place.as_owned_place(ctxt).is_some() + && !body_analysis.is_definitely_initialized_at(location, *place) + }) + .collect::>(); + for (place, from) in suspended { + self.record_and_apply_action( + BorrowPcgAction::weaken_suspended( + place, + from, + Some(CapabilityKind::Write), + "loop head", + ) + .into(), + )?; + } + let mut places = self + .pcg + .place_capabilities + .iter() + .filter_map(|(place, capability)| { + (capability != CapabilityKind::Write + && (!body_analysis.is_definitely_initialized_at(location, place) + || matches!( + capability, + CapabilityKind::Exclusive | CapabilityKind::ShallowExclusive + ) && place.projection.is_empty() + && !place.lifetime_projections(ctxt).is_empty() + && !body_analysis.is_live_and_initialized_at(location, place))) + .then(|| place.as_owned_place(ctxt)) + .flatten() + }) + .collect::>(); + places.sort_by_key(|owned| owned.place().projection.len()); + for owned in places { + let place = owned.place(); + if self.pcg.place_capabilities.get(place, ctxt).is_some() { + self.obtain(place, ObtainType::Capability(CapabilityKind::Write))?; + self.weaken_to(place, CapabilityKind::Write)?; + self.pcg + .owned + .apply_capability_change(owned, CapabilityKind::Write, ctxt); + } + } + Ok(()) + } + + /// Packs what this state holds below the places `target` holds packed. + fn pack_to(&mut self, target: &PlaceCapabilities<'tcx>) -> Result<(), PcgError> { + let mut places = target.iter().collect::>(); + places.sort_by_key(|(place, _)| Reverse(place.projection.len())); + for (place, capability) in places { + if !self + .pcg + .place_capabilities + .capabilities_for_strict_postfixes_of(place) + .any(|(p, _)| target.get(p, self.ctxt).is_none()) + { + continue; + } + self.pack(place, capability, &|base| target.unpacks(base))?; + } + Ok(()) + } + + /// Packs what this state holds below `place` into `place` at + /// `capability`, keeping the read unpackings of the places `keep` holds. + fn pack( + &mut self, + place: Place<'tcx>, + capability: CapabilityKind, + keep: &impl Fn(Place<'tcx>) -> bool, + ) -> Result<(), PcgError> { + match capability { + CapabilityKind::Read => { + self.detach_borrowed_shared_referents(place, keep)?; + self.pack_read_unpacking(place, keep) + } + CapabilityKind::Exclusive | CapabilityKind::Write => { + self.obtain(place, ObtainType::Capability(capability)) + } + CapabilityKind::ShallowExclusive => Ok(()), + } + } + + /// Gives up the places this state holds and `target` does not: each is + /// lent to the edge of `other` that blocks it, or to the loan of this + /// state that still blocks it, or, where `other` unpacked it further and + /// lent its parts, unpacked to lend those. Otherwise the value of the + /// place on this path is detached from it. + fn give_up_places_outside( + &mut self, + target: &PlaceCapabilities<'tcx>, + lending: &Lending<'_, 'a, 'tcx>, + ) -> Result<(), PcgError> { + let ctxt = self.ctxt; + let other = lending.other; + while let Some((place, capability)) = self + .pcg + .place_capabilities + .iter() + .filter(|(place, _)| target.get(*place, ctxt).is_none()) + .min_by_key(|(place, _)| (place.projection.len(), *place)) + { + let lending_edge = |blocked| lending.edge(blocked, ctxt); + let lent_part = || { + other + .borrow + .graph() + .edges() + .flat_map(|edge| { + edge.kind() + .blocked_nodes(ctxt.bc_ctxt()) + .collect::>() + }) + .filter_map(|node| node.related_maybe_labelled_place()?.as_current_place()) + .filter(|blocked| place.is_strict_prefix_of(*blocked)) + .find_map(|blocked| Some((blocked, lending_edge(blocked)?))) + }; + let place_lent_to = std::iter::successors(Some(place), |p| p.parent_place()) + .find_map(|lent| Some((lent, lending_edge(lent)?))); + // Each part this path holds of a lent place is lent with what it + // holds. + if let Some((_, edge)) = place_lent_to { + self.record_and_apply_action(BorrowPcgAction::lend(place, edge, "join").into())?; + continue; + } + // A place that a loan of this path still blocks, directly or as + // part of a lent place, stays with that loan; a place in a value + // that the join gives up as a whole goes with the loan of this + // path on another part of that value. + let given_up = std::iter::successors(Some(place), |p| p.parent_place()) + .take_while(|value| { + !place.projects_indirection_from(*value, ctxt) + && !target.iter().any(|(held, _)| value.is_prefix_of(held)) + }) + .last() + .unwrap_or(place); + let own_loan = self + .pcg + .borrow + .graph + .edges() + .filter(|edge| edge.kind().is_loan()) + .filter_map(|edge| { + let holds_place = edge + .kind() + .blocked_nodes(ctxt.bc_ctxt()) + .filter_map(|node| node.as_current_place()) + .filter(|blocked| { + given_up.is_prefix_of(*blocked) || blocked.is_prefix_of(place) + }) + .map(|blocked| blocked.is_prefix_of(place)) + .max()?; + Some((!holds_place, edge.to_owned_edge())) + }) + .min_by_key(|(elsewhere, edge)| (*elsewhere, format!("{:?}", edge.kind()))) + .map(|(_, edge)| edge); + if let Some(edge) = own_loan { + self.record_and_apply_action(BorrowPcgAction::lend(place, edge, "join").into())?; + continue; + } + if let Some((part, edge)) = lent_part() { + // This path holds `place` packed, so it did not match an enum + // variant on the way to the part: the place itself is lent. + if part + .iter_projections_after(place, ctxt) + .is_some_and(|mut projections| { + projections.any(|(_, elem)| matches!(elem, ProjectionElem::Downcast(..))) + }) + { + self.record_and_apply_action( + BorrowPcgAction::lend(place, edge, "join").into(), + )?; + continue; + } + self.expand_to( + part, + Some(other.borrow.graph()), + ObtainType::Capability(capability), + ctxt, + )?; + // A frozen place's freeze moves to its read copies, which + // are lent or given up apart. + if capability == CapabilityKind::Read { + self.refreeze_to_copies(place)?; + continue; + } + } + if self.pcg.place_capabilities.get(place, ctxt).is_some() { + self.record_and_apply_action( + BorrowPcgAction::label_place_and_update_related_capabilities( + place, + self.prev_snapshot_location(), + LabelPlaceReason::Detach, + ) + .into(), + )?; + } + } + Ok(()) + } +} diff --git a/src/pcg/domain/mod.rs b/src/pcg/domain/mod.rs index d6105de67..81dcd5d38 100644 --- a/src/pcg/domain/mod.rs +++ b/src/pcg/domain/mod.rs @@ -41,10 +41,12 @@ use crate::{ mod dataflow_stmt_phase; mod eval_stmt_phase; +mod join; mod pcg; pub use dataflow_stmt_phase::*; pub use eval_stmt_phase::*; +pub(crate) use join::meet; pub use pcg::*; use super::PcgEngine; @@ -373,20 +375,49 @@ impl<'a, 'tcx> PendingDataflowState<'a, 'tcx, AnalysisCtxt<'a, 'tcx>> { // or applying the loop abstraction. let mut target = (*first.data.pcg.states.0.post_main).clone(); for other in &rest { - target.join_owned( + target.join_with( &other.data.pcg.states.0.post_main, + first.ctxt.block, SnapshotLocation::BeforeJoin(ctxt.block), - SnapshotLocation::After(other.ctxt.block), ctxt, )?; } // Preserve each predecessor's values before its edge repacks. for incoming in std::iter::once(&mut first).chain(rest.iter_mut()) { - Rc::make_mut(&mut incoming.data.pcg.states.0.post_main).join_owned( + Rc::make_mut(&mut incoming.data.pcg.states.0.post_main).reach( + &target.place_capabilities, &target, + incoming.ctxt.block, + SnapshotLocation::After(incoming.ctxt.block), + ctxt, + )?; + } + + let layouts = std::iter::once(&first) + .chain(rest.iter()) + .map(|incoming| incoming.data.pcg.states.0.post_main.clone()) + .collect::>(); + for incoming in std::iter::once(&mut first).chain(rest.iter_mut()) { + let pcg = Rc::make_mut(&mut incoming.data.pcg.states.0.post_main); + for layout in &layouts { + pcg.join_lent_layout(layout, ctxt); + } + } + + // The joined state gets back what every predecessor suspends, at + // the meet. + let joined = Pcg::joined_suspended( + &std::iter::once(&first) + .chain(rest.iter()) + .map(|incoming| &*incoming.data.pcg.states.0.post_main) + .collect::>(), + ctxt, + ); + for incoming in std::iter::once(&mut first).chain(rest.iter_mut()) { + Rc::make_mut(&mut incoming.data.pcg.states.0.post_main).reach_suspended( + &joined, SnapshotLocation::After(incoming.ctxt.block), - SnapshotLocation::BeforeJoin(ctxt.block), ctxt, )?; } @@ -399,7 +430,7 @@ impl<'a, 'tcx> PendingDataflowState<'a, 'tcx, AnalysisCtxt<'a, 'tcx>> { curr.merge_borrows(&other.data.pcg.states.0.post_main, other.ctxt.block, ctxt); } if !rest.is_empty() || ctxt.body_analysis.is_loop_head(ctxt.block) { - curr.finish_join(ctxt); + curr.finish_join(ctxt)?; } Ok(result) } diff --git a/src/pcg/domain/pcg.rs b/src/pcg/domain/pcg.rs index ba15f09ac..92cec63dc 100644 --- a/src/pcg/domain/pcg.rs +++ b/src/pcg/domain/pcg.rs @@ -2,8 +2,10 @@ use std::borrow::Cow; use crate::{ DebugLines, - action::{AppliedActions, OwnedPcgAction, PcgActions}, + action::{BorrowPcgAction, PcgAction, PcgActions}, borrow_pcg::{ + action::BorrowPcgActionKind, + borrow_pcg_edge::BorrowPcgEdgeLike, borrow_pcg_expansion::BorrowPcgExpansion, edge::{borrow::BorrowEdge, kind::BorrowPcgEdgeKind}, graph::{BorrowsGraph, join::JoinBorrowsArgs}, @@ -11,21 +13,21 @@ use crate::{ }, borrows_imgcat_debug, error::PcgError, - r#loop::PlaceUsageType, - owned_pcg::{RepackOp, join::data::JoinOwnedData}, + owned_pcg::RepackGuide, pcg::{ CapabilityKind, ctxt::{AnalysisCtxt, HasSettings}, - obtain::{ObtainType, PlaceObtainer, expand::PlaceExpander}, + obtain::PlaceObtainer, owned_state::OwnedPcg, place_capabilities::{PlaceCapabilities, PlaceCapabilitiesReader}, triple::Triple, }, + pcg_validity_assert, rustc_interface::middle::mir, utils::{ - CompilerCtxt, DataflowCtxt, DebugImgcat, HasBorrowCheckerCtxt, Place, SnapshotLocation, - data_structures::HashSet, display::DisplayWithCompilerCtxt, maybe_old::MaybeLabelledPlace, - validity::HasValidityCheck, + CompilerCtxt, DataflowCtxt, DebugImgcat, HasBorrowCheckerCtxt, Place, PlaceLike, + SnapshotLocation, data_structures::HashSet, display::DisplayWithCompilerCtxt, + maybe_old::MaybeLabelledPlace, validity::HasValidityCheck, }, }; @@ -254,6 +256,60 @@ impl<'a, 'tcx: 'a, Ctxt: HasSettings<'a> + HasBorrowCheckerCtxt<'a, 'tcx>> HasVa } } + for (place, suspended) in self.place_capabilities.iter_suspended() { + if let Some(cap) = self.place_capabilities.get(place, ctxt) + && !cap.is_read() + { + return Err(format!( + "Place {} suspends {:?} but holds {:?}", + place.display_string(ctxt.bc_ctxt()), + suspended, + cap + )); + } + } + + // A lent place, every place in it, and every place it is part of + // (short of a reference to it) holds at most R: a shared loan keeps + // its value frozen until it ends. + let lent = self.borrow.graph.lent_places(ctxt.bc_ctxt()); + for (place, cap) in self.place_capabilities.iter() { + if !cap.is_read() + && !place.contains_unsafe_deref(ctxt) + && let Some(lent) = lent.iter().find(|lent| { + lent.is_prefix_of(place) + || place.is_prefix_of(**lent) + && !lent.projects_indirection_from(place, ctxt) + && !lent.contains_unsafe_deref(ctxt) + }) + { + return Err(format!( + "Place {} holds {:?} while {} is lent", + place.display_string(ctxt.bc_ctxt()), + cap, + lent.display_string(ctxt.bc_ctxt()) + )); + } + } + + // A frozen place, which holds R below no place holding R, records what + // its freeze suspends. + for (place, cap) in self.place_capabilities.iter() { + if cap.is_read() + && !place.projects_shared_ref(ctxt) + && !place.contains_unsafe_deref(ctxt) + && self.place_capabilities.suspended(place).is_none() + && place + .parent_place() + .is_none_or(|parent| self.place_capabilities.get(parent, ctxt) != Some(cap)) + { + return Err(format!( + "Frozen place {} suspends no capability", + place.display_string(ctxt.bc_ctxt()) + )); + } + } + for edge in self.borrow.graph.edges() { match edge.kind { BorrowPcgEdgeKind::Deref(deref_edge) => { @@ -386,18 +442,9 @@ impl<'a, 'tcx: 'a> Pcg<'a, 'tcx> { &self.owned } - pub(crate) fn join_owned_data( - &mut self, - snapshot_location: SnapshotLocation, - ) -> JoinOwnedData<'a, '_, 'tcx, &mut OwnedPcg<'tcx>> { - JoinOwnedData { - owned: &mut self.owned, - borrows: &mut self.borrow, - capabilities: &mut self.place_capabilities, - snapshot_location, - } - } - + /// The actions that turn this state, at the end of `self_block`, into the + /// entry state `other` of its successor `other_block`. Into a loop head, + /// the values this state gives up are labelled with the loop's snapshot. pub(crate) fn bridge( &self, other: &Self, @@ -405,131 +452,51 @@ impl<'a, 'tcx: 'a> Pcg<'a, 'tcx> { other_block: mir::BasicBlock, ctxt: impl DataflowCtxt<'a, 'tcx>, ) -> Result, PcgError> { - let target = other; - let mut slf = self.clone(); - let repacks = slf.join_owned( - target, - SnapshotLocation::After(self_block), - SnapshotLocation::BeforeJoin(other_block), - ctxt, - )?; - let mut actions = PcgActions::new( - repacks - .into_iter() - .map(|repack| OwnedPcgAction::new(repack, None).into()) - .collect(), - ); - if ctxt.is_loop_head(other_block) { - actions.extend(slf.bridge_loop_expansions(target, self_block, other_block, ctxt)?); - } - Ok(actions) - } - - /// Reconcile owned layouts and capabilities. Repacking can update borrow - /// edges, but the borrowed graphs remain separate until all predecessors - /// have been normalized. - pub(crate) fn join_owned( - &mut self, - target: &Self, - self_snapshot: SnapshotLocation, - other_snapshot: SnapshotLocation, - ctxt: impl DataflowCtxt<'a, 'tcx>, - ) -> Result>, PcgError> { - let mut other = target.clone(); - let mut slf_owned_data = self.join_owned_data(self_snapshot); - let other_owned_data = JoinOwnedData { - owned: &other.owned, - borrows: &mut other.borrow, - capabilities: &mut other.place_capabilities, - snapshot_location: other_snapshot, + let snapshot = if ctxt.is_loop_head(other_block) { + SnapshotLocation::Loop(other_block) + } else { + SnapshotLocation::After(self_block) }; - let mut repacks = - slf_owned_data.join(other_owned_data, ctxt.compiler_ctxt_with_settings())?; - for (place, cap) in self.place_capabilities.iter() { - let Some(_owned) = place.as_owned_place(ctxt) else { - continue; - }; - if let Some(other_cap) = other.place_capabilities.get(place, ctxt) - && cap > other_cap - { - repacks.push(RepackOp::weaken(place, cap, other_cap)); - } - } - self.place_capabilities - .join(&other.place_capabilities, ctxt); - self.owned.join_capabilities(&other.owned); - Ok(repacks) - } - - fn bridge_loop_expansions( - &mut self, - other: &Self, - self_block: mir::BasicBlock, - loop_head: mir::BasicBlock, - ctxt: impl DataflowCtxt<'a, 'tcx>, - ) -> Result, PcgError> { - let mut expansions = Vec::new(); - for edge in other.borrow.graph().edges() { - let conditions = edge.conditions.without_branch_choice(self_block); - // The guide is kept: the target place does not carry the label of - // an index guide. - let (base, target, guide) = match edge.kind() { - BorrowPcgEdgeKind::Deref(deref) => { - (deref.blocked_place(), deref.deref_place(), None) - } - BorrowPcgEdgeKind::BorrowPcgExpansion(BorrowPcgExpansion::Place(expansion)) => { - let Some(target) = expansion.expansion().first() else { - continue; - }; - (expansion.base(), *target, expansion.guide) - } - _ => continue, - }; - if let (Some(base), Some(target)) = (base.as_current_place(), target.as_current_place()) + let mut reached = self.clone(); + let mut actions = + reached.reach(&other.place_capabilities, other, self_block, snapshot, ctxt)?; + actions.extend(reached.reach_suspended(&other.place_capabilities, snapshot, ctxt)?); + pcg_validity_assert!( + reached.place_capabilities.suspended == other.place_capabilities.suspended, + [ctxt], + "the edge into {other_block:?} reaches the suspended capabilities {:?} instead of {:?}", + reached.place_capabilities.debug_lines(ctxt), + other.place_capabilities.debug_lines(ctxt) + ); + // The edges of the loop abstraction of `other` are introduced on the + // edge into the loop. + actions.extend( + other + .borrow + .graph() + .edges() + .filter(|edge| { + matches!(edge.kind(), BorrowPcgEdgeKind::Abstraction(_)) + && !reached.borrow.graph().edges.contains_key(edge.kind()) + }) + .map(|edge| BorrowPcgAction::add_edge(edge.to_owned_edge(), "loop").into()), + ); + // An added edge is the edge of `other`, under its conditions. The + // branch choice being taken is implied. + for action in &mut actions { + if let PcgAction::Borrow(action) = action + && let BorrowPcgActionKind::AddEdge { edge, .. } = &mut action.kind { - expansions.push((base, target, guide, conditions)); + let conditions = other + .borrow + .graph() + .edges + .get(edge.kind()) + .unwrap_or(&edge.conditions); + edge.conditions = conditions.without_branch_choice(self_block); } } - expansions.sort_by_key(|(base, _, _, _)| base.projection.len()); - let mut applied = AppliedActions::default(); - for (base, target, guide, conditions) in expansions { - let usage_type = - if other.place_capabilities.get(base, ctxt) == Some(CapabilityKind::Read) { - PlaceUsageType::Read - } else { - PlaceUsageType::Exclusive - }; - let obtain_type = ObtainType::LoopInvariant { - is_blocked: false, - usage_type, - }; - let expansion = base.expand_one_level(target, ctxt)?; - let mut pcg = PcgMutRef::from(&mut *self); - // The current CFG edge's branch choice has already been removed. - // Keep guards from earlier branches. - pcg.borrow.validity_conditions = &conditions; - let mut obtainer = PlaceObtainer::new( - pcg, - Some(&mut applied), - ctxt, - mir::Location { - block: loop_head, - statement_index: 0, - }, - SnapshotLocation::Loop(loop_head), - ); - if obtainer.expand_place_one_level(base, &expansion, guide, obtain_type, ctxt)? { - obtainer.expand_lifetime_projections_one_level( - base, - &expansion, - obtain_type, - ctxt, - )?; - } - } - Ok(PcgActions::new( - applied.actions.into_iter().map(|a| a.action).collect(), - )) + Ok(PcgActions::new(actions)) } #[tracing::instrument(skip(self, other, ctxt))] @@ -541,13 +508,18 @@ impl<'a, 'tcx: 'a> Pcg<'a, 'tcx> { ) { let mut other_borrows = other.borrow.clone(); other_borrows.add_cfg_edge(other_block, ctxt.block, ctxt); - self.place_capabilities - .join(&other.place_capabilities, ctxt); + pcg_validity_assert!( + self.place_capabilities == other.place_capabilities, + [ctxt], + "predecessors reach different capabilities at the join: {:?} and {:?}", + self.place_capabilities.debug_lines(ctxt), + other.place_capabilities.debug_lines(ctxt) + ); self.owned.join_capabilities(&other.owned); self.borrow.merge(&other_borrows, ctxt); } - pub(crate) fn finish_join(&mut self, ctxt: AnalysisCtxt<'a, 'tcx>) { + pub(crate) fn finish_join(&mut self, ctxt: AnalysisCtxt<'a, 'tcx>) -> Result<(), PcgError> { let borrow_args = JoinBorrowsArgs { self_block: ctxt.block, body_analysis: ctxt.body_analysis, @@ -555,6 +527,23 @@ impl<'a, 'tcx: 'a> Pcg<'a, 'tcx> { owned: &mut self.owned, }; self.borrow.finish_join(borrow_args, ctxt); + if ctxt.body_analysis.is_loop_head(ctxt.block) { + // A back edge may arrive with the places that are not definitely + // initialized at the loop head moved out, and with the references + // that are dead there weakened. + PlaceObtainer::new( + PcgMutRef::from(&mut *self), + None, + ctxt, + mir::Location { + block: ctxt.block, + statement_index: 0, + }, + SnapshotLocation::Loop(ctxt.block), + ) + .weaken_maybe_uninitialized()?; + } + Ok(()) } pub(crate) fn debug_lines + HasSettings<'a>>( @@ -582,3 +571,29 @@ impl<'a, 'tcx: 'a> Pcg<'a, 'tcx> { } } } + +/// The base and the first expanded place of a place unpacking edge. +pub(crate) fn unpacking_places<'tcx>( + kind: &BorrowPcgEdgeKind<'tcx>, +) -> Option<(Place<'tcx>, Place<'tcx>, Option)> { + let (base, target, guide) = match kind { + BorrowPcgEdgeKind::Deref(deref) => (deref.blocked_place(), deref.deref_place(), None), + BorrowPcgEdgeKind::BorrowPcgExpansion(BorrowPcgExpansion::Place(expansion)) => ( + expansion.base(), + *expansion.expansion().first()?, + expansion.guide, + ), + _ => return None, + }; + Some((base.as_current_place()?, target.as_current_place()?, guide)) +} + +/// Whether the edge unpacks an enum to one of its variants, which is known +/// only on the paths that took the corresponding branch. +pub(super) fn is_variant_unpacking(kind: &BorrowPcgEdgeKind<'_>) -> bool { + matches!( + kind, + BorrowPcgEdgeKind::BorrowPcgExpansion(expansion) + if matches!(expansion.guide(), Some(RepackGuide::Downcast(..))) + ) +} diff --git a/src/pcg/obtain/expand.rs b/src/pcg/obtain/expand.rs index ad22f492d..976659129 100644 --- a/src/pcg/obtain/expand.rs +++ b/src/pcg/obtain/expand.rs @@ -25,6 +25,7 @@ use crate::{ RenderDebugGraph, }, place_capabilities::BlockType, + unpacking_places, }, rustc_interface::middle::mir, utils::{ @@ -38,24 +39,34 @@ pub(crate) trait PlaceExpander<'a, 'tcx: 'a>: { fn contains_owned_expansion_to(&self, target: Place<'tcx>) -> bool; - fn update_capabilities_for_borrow_expansion( - &mut self, + /// The capabilities that adding the borrow-PCG expansion `expansion` sets. + fn borrow_expansion_capabilities( + &self, expansion: &BorrowPcgExpansion<'tcx>, block_type: BlockType, ctxt: CompilerCtxt<'_, 'tcx>, - ) -> bool; + ) -> Vec<(Place<'tcx>, Option)>; - fn update_capabilities_for_deref( - &mut self, + /// The capabilities that dereferencing `ref_place` for `capability` sets. + fn deref_capabilities( + &self, ref_place: Place<'tcx>, capability: CapabilityKind, ctxt: CompilerCtxt<'_, 'tcx>, - ) -> bool; + ) -> Vec<(Place<'tcx>, Option)>; + + /// Freezes `base` if it has E and is about to be unpacked for read. + fn freeze_for_read_unpacking(&mut self, _base: Place<'tcx>) -> Result<(), PcgError> { + Ok(()) + } - #[tracing::instrument(skip(self, obtain_type, ctxt))] + /// Expands down to `place`. The guide of an expansion `template` has of the + /// same base is kept: `place` does not carry the label of an index guide. + #[tracing::instrument(skip(self, template, obtain_type, ctxt))] fn expand_to( &mut self, place: Place<'tcx>, + template: Option<&BorrowsGraph<'tcx>>, obtain_type: ObtainType, ctxt: impl HasBorrowCheckerCtxt<'a, 'tcx> + HasSettings<'a>, ) -> Result<(), PcgError> { @@ -63,7 +74,13 @@ pub(crate) trait PlaceExpander<'a, 'tcx: 'a>: let base: Place = base.into(); let base = base.with_inherent_region(ctxt); let expansion = base.expand_one_level(place, ctxt)?; - if self.expand_place_one_level(base, &expansion, None, obtain_type, ctxt)? { + let guide = expansion.guide().and_then(|guide| { + template?.edges().find_map(|edge| { + let (expanded, _, stored) = unpacking_places(edge.kind())?; + stored.filter(|stored| expanded == base && *stored == guide) + }) + }); + if self.expand_place_one_level(base, &expansion, guide, obtain_type, ctxt)? { tracing::debug!( "expand region projections for {} one level", base.display_string(ctxt.ctxt()) @@ -147,14 +164,13 @@ pub(crate) trait PlaceExpander<'a, 'tcx: 'a>: .into(), )?; } else if !base.is_raw_ptr(ctxt) { + let capability = self.capability_for_expand(expansion.base_place(), obtain_type, ctxt); + if capability.is_read() { + self.freeze_for_read_unpacking(expansion.base_place())?; + } self.apply_action( OwnedPcgAction::new( - RepackOp::expand( - expansion.base_place(), - expansion.guide(), - self.capability_for_expand(expansion.base_place(), obtain_type, ctxt), - ctxt, - ), + RepackOp::expand(expansion.base_place(), expansion.guide(), capability, ctxt), Some(format!("Expand owned place one level ({obtain_type:?})").into()), ) .into(), @@ -190,22 +206,18 @@ pub(crate) trait PlaceExpander<'a, 'tcx: 'a>: ) }; let deref = DerefEdge::new(base, blocked_lifetime_projection_label, ctxt); + let capability = obtain_type.capability(base, ctxt); + if capability.is_read() || base.is_shared_ref(ctxt) { + self.freeze_for_read_unpacking(base)?; + } self.render_debug_graph(None, "expand_place_one_level: before apply action"); - let action = BorrowPcgAction::add_edge( + let action = BorrowPcgAction::add_edge_setting( BorrowPcgEdge::new(deref.into(), self.path_conditions()), + self.deref_capabilities(base, capability, ctxt.bc_ctxt()), "expand_place_one_level: add deref edge", ); self.apply_action(action.into())?; self.render_debug_graph(None, "expand_place_one_level: after apply action"); - self.update_capabilities_for_deref( - base, - obtain_type.capability(base, ctxt), - ctxt.bc_ctxt(), - ); - self.render_debug_graph( - None, - "expand_place_one_level: after update_capabilities_for_deref", - ); if deref.blocked_lifetime_projection.label().is_some() { self.apply_action( BorrowPcgAction::label_lifetime_projection( @@ -221,17 +233,19 @@ pub(crate) trait PlaceExpander<'a, 'tcx: 'a>: Ok(true) } else if matches!(expansion.kind, ProjectionKind::DerefRawPtr(_)) { let deref = RawPtrDerefEdge::new(base, ctxt); - let action = BorrowPcgAction::add_edge( + let capability = obtain_type.capability(base, ctxt); + if capability.is_read() { + self.freeze_for_read_unpacking(base)?; + } + // Dereferencing a raw pointer only reads its value: the pointer + // keeps its capability. + let action = BorrowPcgAction::add_edge_setting( BorrowPcgEdge::new(deref.into(), self.path_conditions()), + vec![(base.project_deref(ctxt), Some(capability))], "expand_place_one_level: add rawptr deref edge", ); self.apply_action(action.into())?; self.render_debug_graph(None, "expand_place_one_level: after apply action"); - let _ = self.update_capabilities_for_deref( - base, - obtain_type.capability(base, ctxt), - ctxt.bc_ctxt(), - ); Ok(true) } else if let Some(owned_base) = base.as_owned_place(ctxt) { self.expand_owned_place_one_level(owned_base, expansion, obtain_type, ctxt) @@ -282,23 +296,17 @@ pub(crate) trait PlaceExpander<'a, 'tcx: 'a>: let expansion: BorrowPcgExpansion<'tcx> = BorrowPcgExpansion::new_place_expansion(base, &expanded_place.expansion, ctxt)?; - self.render_debug_graph( - None, - &format!( - "add_borrow_pcg_expansion: before update_capabilities_for_borrow_expansion {}", - expansion.display_string(ctxt.bc_ctxt()) - ), - ); - self.update_capabilities_for_borrow_expansion(&expansion, block_type, ctxt.bc_ctxt()); - self.render_debug_graph( - None, - "add_borrow_pcg_expansion: after update_capabilities_for_borrow_expansion", - ); - let action = BorrowPcgAction::add_edge( + if block_type.blocked_place_maximum_retained_capability() == Some(CapabilityKind::Read) { + self.freeze_for_read_unpacking(base)?; + } + let capabilities = + self.borrow_expansion_capabilities(&expansion, block_type, ctxt.bc_ctxt()); + let action = BorrowPcgAction::add_edge_setting( BorrowPcgEdge::new( BorrowPcgEdgeKind::BorrowPcgExpansion(expansion), self.path_conditions(), ), + capabilities, "add_borrow_pcg_expansion", ); self.apply_action(action.into())?; diff --git a/src/pcg/obtain/mod.rs b/src/pcg/obtain/mod.rs index 6418314dc..6e3508b15 100644 --- a/src/pcg/obtain/mod.rs +++ b/src/pcg/obtain/mod.rs @@ -26,7 +26,6 @@ use crate::{ ctxt::AnalysisCtxt, place_capabilities::{PlaceCapabilities, PlaceCapabilitiesReader}, }, - pcg_validity_assert, rustc_interface::middle::mir, utils::{ CompilerCtxt, DataflowCtxt, DebugCtxt, DebugImgcat, HasBorrowCheckerCtxt, HasCompilerCtxt, @@ -92,10 +91,7 @@ pub(crate) enum ObtainType { ForStorageDead, Capability(CapabilityKind), TwoPhaseExpand, - LoopInvariant { - is_blocked: bool, - usage_type: PlaceUsageType, - }, + LoopInvariant { usage_type: PlaceUsageType }, } impl ObtainType { @@ -137,10 +133,7 @@ impl ObtainType { ObtainType::ForStorageDead => CapabilityKind::Write, ObtainType::Capability(cap) => cap, ObtainType::TwoPhaseExpand => CapabilityKind::Read, - ObtainType::LoopInvariant { - is_blocked: _, - usage_type, - } => { + ObtainType::LoopInvariant { usage_type } => { if usage_type == PlaceUsageType::Read || place.is_shared_ref(ctxt) || place.projects_shared_ref(ctxt) @@ -238,13 +231,7 @@ pub(crate) trait PlaceCollapser<'a, 'tcx: 'a>: { continue; } - let action = PcgAction::restore_capability( - place, - CapabilityKind::Exclusive, - "restore capability to leaf place", - ctxt, - ); - self.apply_action(action)?; + self.apply_action(BorrowPcgAction::unfreeze(place, "unfreeze leaf place").into())?; } Ok(()) } @@ -257,9 +244,23 @@ pub(crate) trait PlaceCollapser<'a, 'tcx: 'a>: context: String, ctxt: impl HasBorrowCheckerCtxt<'a, 'tcx> + HasSettings<'a>, ) -> Result<(), PcgError> { + // A place that holds nothing below one that holds a capability is + // lent, or only the layout of the loans of a place in it (a join + // holds the enum packed while a variant field is borrowed): it is + // left as it is. let to_collapse = self .get_local_expansions(place.local) .places_to_collapse_for_obtain_of(place, ctxt); + let capabilities = self.capabilities(); + let to_collapse = to_collapse + .into_iter() + .filter(|p| { + capabilities.get(*p, ctxt).is_some() + || !std::iter::successors(p.parent_place(), |q| q.parent_place()) + .take_while(|q| place.is_prefix_of(*q)) + .any(|q| capabilities.get(q, ctxt).is_some()) + }) + .collect::>(); // Collapsing a place removes its strict descendants from the owned // PCG, which would leave any borrow-PCG edge blocking such a // descendant referring to a place that is no longer there. Read @@ -288,6 +289,17 @@ pub(crate) trait PlaceCollapser<'a, 'tcx: 'a>: .cloned() .collect::>(); for pe in expansions { + // Parts frozen apart are frozen together in `place`. + if capability.is_read() + && place + .expansion_places(&pe.expansion, ctxt)? + .into_iter() + .any(|part| self.capabilities().suspended(part).is_some()) + { + self.apply_action( + BorrowPcgAction::pack_suspended(place, context.clone()).into(), + )?; + } self.apply_action(PcgAction::Owned(OwnedPcgAction::new( RepackOp::Collapse(RepackCollapse::new(place, capability, pe.guide())), Some(context.clone().into()), @@ -322,8 +334,8 @@ pub(crate) trait PlaceCollapser<'a, 'tcx: 'a>: Ok(()) } - /// Gives `place` read capability, downgrading the capabilities of the - /// places in its expansion to read. + /// Gives `place` read capability by freezing it, which makes the places in + /// its expansion read copies. /// /// This is how read capability is obtained for a place that cannot have /// its expansion collapsed, because one of its descendants is blocked by @@ -333,48 +345,8 @@ pub(crate) trait PlaceCollapser<'a, 'tcx: 'a>: place: Place<'tcx>, ctxt: impl HasBorrowCheckerCtxt<'a, 'tcx> + HasSettings<'a>, ) -> Result<(), PcgError> { - let descendants = self - .get_local_expansions(place.local) - .all_descendants_of(place, ctxt); - for descendant in descendants { - let Some(cap) = self.capabilities().get(descendant, ctxt) else { - continue; - }; - if cap == CapabilityKind::Read { - continue; - } - pcg_validity_assert!( - cap == CapabilityKind::Exclusive, - [ctxt], - "Cannot downgrade capability {:?} of {} to read", - cap, - descendant.display_string(ctxt.ctxt()) - ); - self.apply_action( - BorrowPcgAction::weaken( - descendant, - cap, - Some(CapabilityKind::Read), - "downgrade expansion to read", - ) - .into(), - )?; - } - let place_cap = self.capabilities().get(place, ctxt); - if place_cap != Some(CapabilityKind::Read) { - pcg_validity_assert!( - place_cap.is_none(), - [ctxt], - "Cannot give read capability to {}: it has capability {:?}", - place.display_string(ctxt.ctxt()), - place_cap - ); - self.apply_action(PcgAction::restore_capability( - place, - CapabilityKind::Read, - "downgrade expansion to read", - ctxt, - ))?; + if self.capabilities().get(place, ctxt) != Some(CapabilityKind::Read) { + self.apply_action(BorrowPcgAction::freeze(place, "read an unpacked place").into())?; } Ok(()) } diff --git a/src/pcg/place_capabilities.rs b/src/pcg/place_capabilities.rs index 57ac84e0b..e2eb84043 100644 --- a/src/pcg/place_capabilities.rs +++ b/src/pcg/place_capabilities.rs @@ -1,14 +1,10 @@ -use std::{borrow::Cow, cmp, collections::HashMap, marker::PhantomData}; +use std::{borrow::Cow, collections::HashMap, marker::PhantomData}; use itertools::Itertools; use crate::{ HasSettings, - borrow_pcg::{ - borrow_pcg_expansion::BorrowPcgExpansion, - edge_data::LabelNodePredicate, - state::{BorrowStateMutRef, BorrowsStateLike}, - }, + borrow_pcg::borrow_pcg_expansion::BorrowPcgExpansion, pcg::CapabilityKind, rustc_interface::middle::mir, utils::{ @@ -30,6 +26,12 @@ mod private { fn iter(&self) -> impl Iterator + '_; + /// The capability `place` gets back when the loan or freeze that + /// suspended it ends. + fn suspended(&self, place: P) -> Option; + + fn iter_suspended(&self) -> impl Iterator + '_; + fn capabilities_for_strict_postfixes_of<'slf>( &'slf self, place: P, @@ -42,6 +44,16 @@ mod private { self.iter() .filter(move |&(p, _)| place.is_strict_prefix_of(p)) } + + /// Whether a place below `place` has a capability. + fn unpacks(&self, place: P) -> bool + where + P: PrefixRelation, + { + self.capabilities_for_strict_postfixes_of(place) + .next() + .is_some() + } } pub trait PlaceCapabilitiesInterface<'tcx, C = CapabilityKind, P = Place<'tcx>>: @@ -55,6 +67,15 @@ mod private { fn retain(&mut self, predicate: impl Fn(P, C) -> bool); + /// Records that `place` gets `capability` back when the loan or + /// freeze that takes its capability ends. + fn suspend(&mut self, place: P, capability: C); + + /// Removes the record of `place`, returning the suspended capability. + fn resume(&mut self, place: P) -> Option; + + fn retain_suspended(&mut self, predicate: impl Fn(P, C) -> bool); + fn iter_mut<'slf>(&'slf mut self) -> impl Iterator + 'slf where C: 'slf, @@ -98,6 +119,14 @@ where fn iter(&self) -> impl Iterator + '_ { self.map.iter().map(|(k, v)| (*k, *v)) } + + fn suspended(&self, place: P) -> Option { + self.suspended.get(&place).copied() + } + + fn iter_suspended(&self) -> impl Iterator + '_ { + self.suspended.iter().map(|(k, v)| (*k, *v)) + } } impl<'tcx, C, P> PlaceCapabilitiesInterface<'tcx, C, P> for PlaceCapabilities<'tcx, C, P> @@ -117,6 +146,18 @@ where self.map.retain(|place, cap| predicate(*place, *cap)); } + fn suspend(&mut self, place: P, capability: C) { + self.suspended.insert(place, capability); + } + + fn resume(&mut self, place: P) -> Option { + self.suspended.remove(&place) + } + + fn retain_suspended(&mut self, predicate: impl Fn(P, C) -> bool) { + self.suspended.retain(|place, cap| predicate(*place, *cap)); + } + fn iter_mut<'slf>(&'slf mut self) -> impl Iterator + 'slf where C: 'slf, @@ -133,6 +174,9 @@ where P: Eq + std::hash::Hash, { pub(crate) map: HashMap, + /// The capabilities of the lent and frozen places, which they get back + /// when the loan or freeze ends. + pub(crate) suspended: HashMap, pub(crate) _marker: PhantomData<&'tcx ()>, } @@ -143,78 +187,60 @@ where fn default() -> Self { Self { map: HashMap::default(), + suspended: HashMap::default(), _marker: PhantomData, } } } impl<'a, 'tcx: 'a> PlaceCapabilities<'tcx> { - pub(crate) fn update_for_deref>( - &mut self, + /// The capabilities set by dereferencing `ref_place` for `capability`. + pub(crate) fn deref_capabilities>( ref_place: Place<'tcx>, capability: CapabilityKind, ctxt: Ctxt, - ) -> bool { + ) -> Vec<(Place<'tcx>, Option)> { + let deref_place = ref_place.project_deref(ctxt.bc_ctxt()); if capability.is_read() || ref_place.is_shared_ref(ctxt.bc_ctxt()) { - self.insert(ref_place, CapabilityKind::Read, ctxt); - self.insert( - ref_place.project_deref(ctxt.bc_ctxt()), - CapabilityKind::Read, - ctxt, - ); + vec![ + (ref_place, Some(CapabilityKind::Read)), + (deref_place, Some(CapabilityKind::Read)), + ] } else { // The reference itself is still fully accessible (its pointer // value remains readable and overwritable); only the permission // *through* the dereference is given up. - self.insert(ref_place, CapabilityKind::ShallowExclusive, ctxt); - self.insert( - ref_place.project_deref(ctxt.bc_ctxt()), - CapabilityKind::Exclusive, - ctxt, - ); + vec![ + (ref_place, Some(CapabilityKind::ShallowExclusive)), + (deref_place, Some(CapabilityKind::Exclusive)), + ] } - true } - pub(crate) fn update_for_expansion>( - &mut self, + /// The capabilities set by the borrow-PCG expansion `expansion`. + pub(crate) fn expansion_capabilities>( + &self, expansion: &BorrowPcgExpansion<'tcx>, block_type: BlockType, ctxt: Ctxt, - ) -> bool { + ) -> Vec<(Place<'tcx>, Option)> { // We don't change if only expanding region projections let BorrowPcgExpansion::Place(expansion) = expansion else { - return false; + return vec![]; }; - let mut changed = false; - let base = expansion.base; - let base_capability = self.get(base.place(), ctxt); - let expanded_capability = if let Some(capability) = base_capability { - block_type.expansion_capability(base.place(), capability, ctxt) - } else { - return true; + let base = expansion.base.place(); + let Some(base_capability) = self.get(base, ctxt) else { + return vec![]; }; - - changed |= self.update_capabilities_for_block_of_place(base.place(), block_type, ctxt); - - for p in &expansion.expansion { - changed |= self.insert(p.place(), expanded_capability, ctxt); - } - changed - } - - pub(crate) fn update_capabilities_for_block_of_place>( - &mut self, - blocked_place: Place<'tcx>, - block_type: BlockType, - ctxt: Ctxt, - ) -> bool { - let retained_capability = block_type.blocked_place_maximum_retained_capability(); - if let Some(capability) = retained_capability { - self.insert(blocked_place, capability, ctxt) - } else { - self.remove(blocked_place, ctxt).is_some() - } + let expanded_capability = block_type.expansion_capability(base, base_capability, ctxt); + std::iter::once((base, block_type.blocked_place_maximum_retained_capability())) + .chain( + expansion + .expansion + .iter() + .map(|p| (p.place(), Some(expanded_capability))), + ) + .collect() } } @@ -298,6 +324,13 @@ impl<'a, 'tcx: 'a, Ctxt: HasCompilerCtxt<'a, 'tcx> + HasSettings<'a> + DebugCtxt .map(|(node, capability)| { Cow::Owned(format!("{}: {:?}", node.display_string(ctxt), capability)) }) + .chain(self.iter_suspended().map(|(node, capability)| { + Cow::Owned(format!( + "{}: suspended {:?}", + node.display_string(ctxt), + capability + )) + })) .sorted() .collect() } @@ -346,28 +379,6 @@ impl<'tcx> PlaceCapabilities<'tcx> where Self: PlaceCapabilitiesInterface<'tcx>, { - pub(crate) fn regain_loaned_capability< - 'a, - Ctxt: HasBorrowCheckerCtxt<'a, 'tcx> + HasSettings<'a>, - >( - &mut self, - place: Place<'tcx>, - capability: CapabilityKind, - mut borrows: BorrowStateMutRef<'_, 'tcx>, - ctxt: Ctxt, - ) where - 'tcx: 'a, - { - self.insert((*place).into(), capability, ctxt); - if capability == CapabilityKind::Exclusive { - borrows.label_lifetime_projections( - &LabelNodePredicate::all_future_postfixes(place), - None, - ctxt, - ); - } - } - pub(crate) fn remove_all_for_local( &mut self, local: mir::Local, @@ -377,37 +388,6 @@ where } } -impl PlaceCapabilities<'_, C, P> -where - P: Copy + Eq + std::hash::Hash, -{ - pub(crate) fn join(&mut self, other: &Self, _ctxt: Ctxt) -> bool - where - C: PartialOrd, - { - let mut changed = false; - self.map.retain(|place, _| other.map.contains_key(place)); - for (place, other_capability) in &other.map { - let place = *place; - let other_capability = *other_capability; - if let Some(self_capability) = self.map.get(&place) { - if self_capability.partial_cmp(&other_capability) == Some(cmp::Ordering::Greater) { - changed |= self.map.insert(place, other_capability) != Some(other_capability); - } else if let Some(cmp::Ordering::Less | cmp::Ordering::Equal) = - self_capability.partial_cmp(&other_capability) - { - // self is already minimum, no change needed - } else { - // incomparable - remove - self.map.remove(&place); - changed = true; - } - } - } - changed - } -} - impl<'tcx, C: Copy + PartialEq> PlaceCapabilities<'tcx, C> { pub(crate) fn uniform_capability<'a>( &self, diff --git a/src/pcg/triple.rs b/src/pcg/triple.rs index c0188a1ca..bd5d84008 100644 --- a/src/pcg/triple.rs +++ b/src/pcg/triple.rs @@ -50,8 +50,9 @@ pub(crate) enum PlaceCondition<'tcx> { /// This distinction is relevant for expanding lifetime projections: the lifetime projection base of *x will /// be labelled, similarly to the situation where the borrow was exclusive. ExpandTwoPhase(Place<'tcx>), + /// The place is shared or two-phase borrowed: it is frozen at R. + Freeze(Place<'tcx>), Capability(Place<'tcx>, CapabilityKind), - RemoveCapability(Place<'tcx>), AllocateOrDeallocate(Local), Unalloc(Local), Return, @@ -150,19 +151,19 @@ impl<'tcx> FallableVisitor<'tcx> for TripleWalker<'_, 'tcx> { &Ref(_, kind, place) => match kind { BorrowKind::Shared => Triple::new( PlaceCondition::read(place), - Some(PlaceCondition::read(place)), + Some(PlaceCondition::Freeze(place.into())), ), BorrowKind::Mut { kind: MutBorrowKind::TwoPhaseBorrow, } => Triple::new( PlaceCondition::ExpandTwoPhase(place.into()), - Some(PlaceCondition::read(place)), + Some(PlaceCondition::Freeze(place.into())), ), BorrowKind::Fake(..) => return Ok(()), - BorrowKind::Mut { .. } => Triple::new( - PlaceCondition::exclusive(place, self.ctxt), - Some(PlaceCondition::RemoveCapability(place.into())), - ), + // The borrow edge takes the capability. + BorrowKind::Mut { .. } => { + Triple::new(PlaceCondition::exclusive(place, self.ctxt), None) + } }, &RawPtr(mutbl, place) => { #[rustversion::since(2025-03-02)] diff --git a/src/pcg/visitor/assign.rs b/src/pcg/visitor/assign.rs index 37e5ade2a..53a63893b 100644 --- a/src/pcg/visitor/assign.rs +++ b/src/pcg/visitor/assign.rs @@ -5,6 +5,7 @@ use crate::{ GeneralizedLifetime, borrow_pcg_edge::BorrowPcgEdge, edge::{ + borrow::BorrowEdge, borrow_flow::{ AssignmentData, BorrowFlowEdge, BorrowFlowEdgeKind, CastData, OperandType, }, @@ -19,8 +20,9 @@ use crate::{ obtain::{ActionApplier, HasSnapshotLocation, expand::PlaceExpander}, place_capabilities::PlaceCapabilitiesInterface, }, + pcg_validity_assert, rustc_interface::middle::mir::{self, CastKind, Operand, Rvalue}, - utils::Place, + utils::{Place, display::DisplayWithCompilerCtxt}, }; use crate::utils::{ @@ -196,15 +198,45 @@ impl<'a, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> PcgVisitor<'_, 'a, 'tcx, Ctxt> if matches!(kind, mir::BorrowKind::Fake(_)) { return Ok(()); } - self.pcg.borrow.add_borrow( - blocked_place, - target, + let borrow_edge = BorrowEdge::new( + blocked_place.into(), + target.into(), *kind, self.location(), *borrow_region, - &mut self.pcg.place_capabilities, self.ctxt, ); + // A mutable borrow takes the capability of the borrowed place; + // a shared or two-phase borrow froze it. + let capabilities = if let mir::BorrowKind::Mut { + kind: mir::MutBorrowKind::Default | mir::MutBorrowKind::ClosureCapture, + } = kind + { + vec![(blocked_place, None)] + } else { + let ctxt = self.ctxt; + pcg_validity_assert!( + self.pcg + .place_capability_equals(blocked_place, CapabilityKind::Read, ctxt), + [ctxt], + "{:?}: borrowing {} for read with capability {:?}", + self.location(), + blocked_place.display_string(ctxt), + self.pcg.capability_of(blocked_place, ctxt) + ); + vec![] + }; + self.record_and_apply_action( + BorrowPcgAction::add_edge_setting( + BorrowPcgEdge::new( + borrow_edge.into(), + self.pcg.borrow.validity_conditions.clone(), + ), + capabilities, + "borrow", + ) + .into(), + )?; self.label_lifetime_projections_for_borrow(blocked_place, target, *kind)?; } Rvalue::RawPtr(kind, p) if !kind.is_fake() => { diff --git a/src/pcg/visitor/mod.rs b/src/pcg/visitor/mod.rs index 795d9d9f7..dd2753a6b 100644 --- a/src/pcg/visitor/mod.rs +++ b/src/pcg/visitor/mod.rs @@ -1,10 +1,11 @@ use crate::{ - action::{AppliedActions, BorrowPcgAction, PcgAction}, + action::{AppliedActions, BorrowPcgAction, CapabilityChanges}, borrow_pcg::{ GeneralizedLifetime, action::LabelPlaceReason, borrow_pcg_edge::BorrowPcgEdge, edge::borrow_flow::{BorrowFlowEdge, BorrowFlowEdgeKind}, + edge_data::LabelNodePredicate, region_projection::{LifetimeProjection, PcgLifetimeProjectionLike, PlaceOrConst}, }, owned_pcg::RepackExpand, @@ -13,10 +14,11 @@ use crate::{ ctxt::AnalysisCtxt, obtain::{PlaceCollapser, PlaceObtainer, expand::PlaceExpander}, owned_state::OwnedPcg, - place_capabilities::PlaceCapabilitiesInterface, + place_capabilities::{PlaceCapabilitiesInterface, PlaceCapabilitiesReader}, triple::TripleWalker, visitor::upgrade::AdjustCapabilityReason, }, + pcg_validity_assert, rustc_interface::middle::mir::{self, Location, Operand, Rvalue, Statement, Terminator}, utils::{data_structures::HashSet, display::DisplayWithCompilerCtxt}, }; @@ -113,15 +115,33 @@ impl<'pcg, 'a, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> PcgVisitor<'pcg, 'a, 'tcx .ctxt .bc() .is_directly_blocked(place, self.location(), self.ctxt.bc_ctxt()) + && self + .pcg + .borrow + .graph + .edges_blocking(place.into(), self.ctxt.bc_ctxt()) + .next() + .is_none() && self.pcg.capability_of(place, self.ctxt).is_none() { - let action = PcgAction::restore_capability( - place, - CapabilityKind::Exclusive, - "Leaf future node restore cap", - self.ctxt, + // The edge that blocked the place would have returned its + // capability when it was removed. + let ctxt = self.ctxt; + pcg_validity_assert!( + false, + [ctxt], + "{:?}: {} is unblocked without a capability", + self.location(), + place.display_string(ctxt) ); - self.record_and_apply_action(action)?; + self.record_and_apply_action( + BorrowPcgAction::label_lifetime_projection( + LabelNodePredicate::all_future_postfixes(place), + None, + "Leaf future node: remove future label", + ) + .into(), + )?; } } Ok(()) @@ -376,10 +396,13 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> .ctxt .body_analysis() .is_read_only_in_loop(self.location().block, place) + && !self + .blocked_owned_places(self.ctxt.bc_ctxt()) + .contains(&place) { - self.pcg - .place_capabilities - .insert(place, CapabilityKind::Exclusive, self.ctxt); + self.record_and_apply_action( + BorrowPcgAction::unfreeze(place, "unfreeze collapsed local").into(), + )?; } } Ok(true) @@ -425,11 +448,82 @@ impl<'a, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> PcgVisitor<'_, 'a, 'tcx, Ctxt> borrow.display_string(self.ctxt) ); let blocked_place = borrow.blocked_place.place(); - self.place_obtainer() - .remove_read_permission_upwards_and_label_rps( - blocked_place, + let ctxt = self.ctxt; + if !self + .pcg + .place_capability_equals(blocked_place, CapabilityKind::Read, ctxt) + { + pcg_validity_assert!( + false, + [ctxt], + "Activating a two-phase borrow of {} with capability {:?}", + blocked_place.display_string(ctxt), + self.pcg.capability_of(blocked_place, ctxt) + ); + return Ok(()); + } + let borrow = borrow.clone(); + // If the place is a read copy of a frozen ancestor (the ancestor was + // read during the reservation, or the reservation was made under a + // shared borrow of it), the freeze moves to the place first. + let frozen = self.place_obtainer().frozen_ancestor(blocked_place); + if frozen != blocked_place { + self.place_obtainer().end_freeze( + frozen, + Some(blocked_place), AdjustCapabilityReason::TwoPhaseBorrowActivation, )?; + } + // Reads during the reservation may have unpacked the place at R. + if blocked_place.as_owned_place(self.ctxt).is_some() { + let ctxt = self.ctxt; + self.place_obtainer() + .collapse_owned_places_and_lifetime_projections_to( + blocked_place, + CapabilityKind::Read, + "pack the reads of a two-phase borrowed place".to_owned(), + ctxt, + )?; + } + self.place_obtainer() + .detach_borrowed_shared_referents(blocked_place, &|_| false)?; + self.place_obtainer() + .pack_read_unpacking(blocked_place, &|_| false)?; + // The borrow checker allows the activation, so the loans that keep + // the rest of the read unpacking from being packed are dead (the PCG + // can keep them, e.g. for a raw pointer derived from their value): + // the remaining read copies are detached. + while let Some(read_copy) = self + .pcg + .place_capabilities + .capabilities_for_strict_postfixes_of(blocked_place) + .map(|(place, _)| place) + .min_by_key(|place| place.projection.len()) + { + self.record_and_apply_action( + BorrowPcgAction::label_place_and_update_related_capabilities( + read_copy, + self.prev_snapshot_location(), + LabelPlaceReason::Detach, + ) + .into(), + )?; + } + let ctxt = self.ctxt; + pcg_validity_assert!( + !self.pcg.place_capabilities.unpacks(blocked_place), + [ctxt], + "Activating a two-phase borrow of {} whose read unpacking is still borrowed", + blocked_place.display_string(ctxt) + ); + self.record_and_apply_action( + BorrowPcgAction::activate_two_phase_borrow(borrow, "activate two-phase borrow").into(), + )?; + self.place_obtainer().label_rps_up_to( + blocked_place, + frozen, + AdjustCapabilityReason::TwoPhaseBorrowActivation, + )?; Ok(()) } } @@ -440,10 +534,11 @@ impl<'tcx> OwnedPcg<'tcx> { expand: RepackExpand<'tcx>, capabilities: &mut impl PlaceCapabilitiesInterface<'tcx>, ctxt: Ctxt, - ) where + ) -> CapabilityChanges> + where 'tcx: 'a, { let expansions = self[expand.local()].expansions_mut(); - expansions.perform_expand_action(expand, capabilities, ctxt); + expansions.perform_expand_action(expand, capabilities, ctxt) } } diff --git a/src/pcg/visitor/obtain.rs b/src/pcg/visitor/obtain.rs index 47485f7e5..1e9f23f94 100644 --- a/src/pcg/visitor/obtain.rs +++ b/src/pcg/visitor/obtain.rs @@ -2,11 +2,11 @@ use crate::visualization::stmt_graphs; use crate::{ HasSettings, - action::{AppliedAction, BorrowPcgAction, OwnedPcgAction, PcgAction}, + action::{AppliedAction, BorrowPcgAction, CapabilityChanges, PcgAction}, borrow_pcg::{ self, action::{ApplyActionResult, BorrowPcgActionKind, LabelPlaceReason}, - borrow_pcg_edge::BorrowPcgEdge, + borrow_pcg_edge::{BorrowPcgEdge, BorrowPcgEdgeLike}, borrow_pcg_expansion::{BorrowPcgExpansion, PlaceExpansion}, edge::{ borrow_flow::private::FutureEdgeKind, @@ -19,7 +19,7 @@ use crate::{ }, owned_pcg::{LocalExpansions, RepackOp}, pcg::{ - CapabilityKind, EvalStmtPhase, PcgNode, PcgRef, PcgRefLike, + CapabilityKind, EvalStmtPhase, PcgNode, PcgRef, PcgRefLike, meet, obtain::{ ActionApplier, HasSnapshotLocation, ObtainType, PlaceCollapser, PlaceObtainer, RenderDebugGraph, expand::PlaceExpander, @@ -27,12 +27,12 @@ use crate::{ place_capabilities::{ BlockType, PlaceCapabilities, PlaceCapabilitiesInterface, PlaceCapabilitiesReader, }, + unpacking_places, visitor::upgrade::AdjustCapabilityReason, }, - pcg_validity_assert, rustc_interface::middle::mir, utils::{ - CompilerCtxt, DataflowCtxt, DebugCtxt, HasBorrowCheckerCtxt, HasPlace, PackStrategy, + CompilerCtxt, DataflowCtxt, DebugCtxt, HasBorrowCheckerCtxt, PackStrategy, data_structures::HashSet, display::DisplayWithCompilerCtxt, maybe_old::MaybeLabelledPlace, }, }; @@ -84,7 +84,7 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> PlaceCollapser< .leaf_places(ctxt) .into_iter() .map(Into::into) - .filter(|p| !owned_places.contains(p)) + .filter(|p| !owned_places.iter().any(|blocked| blocked.is_prefix_of(*p))) .collect(); leaf_places.extend( self.pcg @@ -103,56 +103,249 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> PlaceCollapser< } } +/// What comes back after a loan ended; see +/// [`PlaceObtainer::loan_comeback`]. +#[derive(Default)] +struct Comeback<'tcx> { + /// The unpackings of records down to the places that get them back. + unpacks: Vec<(Place<'tcx>, Vec>)>, + /// The places that get back what they record after the unpackings. + receivers: Vec>, + /// The places that come back frozen. + frozen: Vec>, + /// The unpacked places whose unpacking is forgotten. + forgotten: Vec>, +} + impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> PlaceObtainer<'state, 'a, 'tcx, Ctxt> { - fn restore_place(&mut self, place: Place<'tcx>, context: &str) -> Result<(), PcgError> { - // The place to restore could come from a local that was conditionally - // allocated and therefore we can't get back to it, and certainly - // shouldn't give it any capability - // TODO: Perhaps the join should label such places? + /// The capabilities that the base of an unpacking gets back when the + /// unpacking is removed and the base is `unblocked`: `packed`, what its + /// expansion held, lowered by its record. + fn packed_capabilities( + &self, + unblocked: &[Place<'tcx>], + packed: CapabilityKind, + ) -> CapabilityChanges> { + let capabilities = &*self.pcg.place_capabilities; + let mut regained = CapabilityChanges::default(); + for &place in unblocked { + // The place could come from a local that was conditionally + // allocated and therefore we can't get back to it. + if !self.pcg.owned.is_allocated(place.local) || capabilities.unpacks(place) { + continue; + } + if matches!( + capabilities.get(place, self.ctxt), + None | Some(CapabilityKind::ShallowExclusive) + ) { + let suspended = capabilities.suspended(place); + regained.set( + place, + Some(suspended.map_or(packed, |lent| meet(lent, packed))), + ); + } + } + regained + } + + /// What comes back after a loan of the places `blocked` ended. Each + /// record holder in the value of one of them comes back once no loan + /// blocks its value: its record is unpacked along its layout (see + /// [`Self::comeback_layout`]), and the places of the layout that then + /// record something get it back. While only shared borrows block its + /// value, it comes back frozen: R at the holder and its layout, the record + /// kept. Either way, an unpacking of a variant, or of a place unpacked in + /// several ways, that no loan needs is forgotten: not every path made it. + /// While a mutable loan blocks its value, nothing comes back. + fn loan_comeback(&self, blocked: &[Place<'tcx>]) -> Comeback<'tcx> { + let ctxt = self.ctxt; + let capabilities = &*self.pcg.place_capabilities; + let mut comeback = Comeback::default(); + let mut holders = capabilities.iter_suspended().collect::>(); + holders.sort_by_key(|(holder, _)| *holder); + for (holder, record) in holders { + if !self.pcg.owned.is_allocated(holder.local) + || capabilities.get(holder, ctxt).is_some() + || !blocked + .iter() + .any(|b| holder.is_prefix_of(*b) || b.is_prefix_of(holder)) + { + continue; + } + let loans = self + .loans_blocking_value_of(holder, record) + .collect::>(); + let lent = loans + .iter() + .flat_map(|kind| kind.blocked_nodes(ctxt.bc_ctxt())) + .filter_map(|node| node.as_current_place()) + .collect::>(); + let (layout, unlent_unpacked) = self.comeback_layout(holder, record, &lent); + let places = std::iter::once(holder) + .chain(layout.iter().flat_map(|(_, parts)| parts.iter().copied())) + .collect::>(); + match loans.as_slice() { + [] => { + comeback.receivers.extend(places); + comeback.unpacks.extend(layout); + comeback.forgotten.extend(unlent_unpacked); + } + loans + if loans.iter().all(|kind| { + matches!(kind, BorrowPcgEdgeKind::Borrow(borrow) if !borrow.is_mut()) + }) && record != CapabilityKind::Write => + { + comeback.frozen.extend(places); + comeback.forgotten.extend(unlent_unpacked); + } + _ => {} + } + } + comeback + } + + /// The layout below `holder`, which records `record`, top-down: each + /// base with the places it is unpacked to, including the referent of a + /// reference (unless `record` is e: the referent is recorded apart). A + /// base unpacked to an enum variant, or in several ways (by the paths of + /// a join), keeps only the unpackings with a place that is `lent` (the + /// loan's path made them); the places of the others are returned as well. + fn comeback_layout( + &self, + holder: Place<'tcx>, + record: CapabilityKind, + lent: &[Place<'tcx>], + ) -> (Vec<(Place<'tcx>, Vec>)>, Vec>) { + let ctxt = self.ctxt; + let mut layout = vec![]; + let mut unlent_unpacked = vec![]; + let mut bases = vec![holder]; + while let Some(base) = bases.pop() { + let unpackings = self.pcg.owned[base.local] + .expansions() + .expansions_from(base) + .map(|expanded| { + expanded + .expansion_places(ctxt) + .into_iter() + .flatten() + .collect() + }) + .chain(self.pcg.borrow.graph.edges.keys().filter_map(|kind| { + match kind { + BorrowPcgEdgeKind::BorrowPcgExpansion(expansion) + if expansion.base().as_current_place() == Some(base) => + { + Some( + expansion + .expansion() + .iter() + .filter_map(|node| node.as_current_place()) + .collect(), + ) + } + BorrowPcgEdgeKind::Deref(deref) + if deref.blocked_place.as_current_place() == Some(base) + && record != CapabilityKind::ShallowExclusive => + { + Some(deref.deref_place.as_current_place().into_iter().collect()) + } + _ => None, + } + })) + .collect::>>>(); + let enters_variant = |parts: &Vec>| { + parts.iter().any(|part| { + matches!( + part.projection.last(), + Some(mir::ProjectionElem::Downcast(..)) + ) + }) + }; + let followed = match unpackings.as_slice() { + [parts] if !enters_variant(parts) => unpackings, + _ => { + let (lent_unpackings, unlent_unpackings): (Vec<_>, Vec<_>) = + unpackings.into_iter().partition(|parts| { + parts + .iter() + .any(|part| lent.iter().any(|lent| part.is_prefix_of(*lent))) + }); + unlent_unpacked.extend(unlent_unpackings.into_iter().flatten()); + lent_unpackings + } + }; + for parts in followed { + bases.extend(parts.iter().copied()); + layout.push((base, parts)); + } + } + (layout, unlent_unpacked) + } + + /// Whether `place` is part of the value of `holder`, which records + /// `record`: below it, not behind a shared reference or a raw pointer, + /// and, if `record` is e, not behind a reference (its referents are + /// recorded apart). + fn in_value_of(&self, holder: Place<'tcx>, record: CapabilityKind, place: Place<'tcx>) -> bool { + holder.is_prefix_of(place) + && place + .iter_projections_after(holder, self.ctxt) + .into_iter() + .flatten() + .all(|(base, elem)| { + !matches!(elem, mir::ProjectionElem::Deref) + || !(base.is_shared_ref(self.ctxt) + || base.is_raw_ptr(self.ctxt) + || record == CapabilityKind::ShallowExclusive && base.is_ref(self.ctxt)) + }) + } + + /// The loans that block `holder`, which records `record`, or a part of + /// its value. + fn loans_blocking_value_of( + &self, + holder: Place<'tcx>, + record: CapabilityKind, + ) -> impl Iterator> { + let ctxt = self.ctxt; + self.pcg.borrow.graph.edges.keys().filter(move |kind| { + kind.is_loan() + && kind + .blocked_nodes(ctxt.bc_ctxt()) + .filter_map(|node| node.as_current_place()) + .any(|blocked| { + blocked.is_prefix_of(holder) || self.in_value_of(holder, record, blocked) + }) + }) + } + + /// Updates `place` after the edge blocking it was removed: a frozen leaf + /// is unfrozen unless the loop it is in only reads it (the loop holds the + /// freeze) or a loan still blocks it, a place it is part of or a place in + /// its value (a variant field, where a join holds the enum packed), and + /// the placeholder labels of its lifetime projections are removed. + fn place_unblocked(&mut self, place: Place<'tcx>, context: &str) -> Result<(), PcgError> { if !self.pcg.owned.is_allocated(place.local) { return Ok(()); } - let blocked_cap = self.pcg.capability_of(place, self.ctxt); - - // TODO: If the place projects a shared ref, do we even need to restore a capability? - // - // For owned places we consult the initialisation state so we - // don't unsoundly restore to `Exclusive` when the place was - // moved out on some incoming path (#137): the join of the init - // trees would then leave the place as `Uninit`, which maps to - // `Write`. - let restore_cap = if place.place().projects_shared_ref(self.ctxt) - || self + if self + .pcg + .place_capability_equals(place, CapabilityKind::Read, self.ctxt) + && let Some(record) = self.pcg.place_capabilities.suspended(place) + && !self .ctxt .body_analysis() .is_read_only_in_loop(self.location().block, place) + && self.frozen_ancestor(place) == place + && !self.pcg.place_capabilities.unpacks(place) + && self.loans_blocking_value_of(place, record).next().is_none() { - CapabilityKind::Read - } else if let Some(owned) = place.as_owned_place(self.ctxt) - && let Some(init_cap) = self.pcg.owned.owned_capability(owned) - { - init_cap - } else { - CapabilityKind::Exclusive - }; - - // The blocked capability would be None if the place was mutably - // borrowed. The capability would be ShallowExclusive if the place is - // a mutable reference (when dereferencing a mutable ref, the ref - // place retains shallow-exclusive capability). - if matches!( - blocked_cap, - None | Some(CapabilityKind::Write | CapabilityKind::ShallowExclusive) - ) && blocked_cap != Some(restore_cap) - { - self.record_and_apply_action(PcgAction::restore_capability( - place, - restore_cap, - context.to_owned(), - self.ctxt, - ))?; + self.record_and_apply_action( + BorrowPcgAction::unfreeze(place, context.to_owned()).into(), + )?; } for rp in place.lifetime_projections(self.ctxt) { self.record_and_apply_action( @@ -168,44 +361,25 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> } Ok(()) } - fn update_unblocked_node_capabilities_and_remove_placeholder_projections( - &mut self, - edge: &BorrowPcgEdgeKind<'tcx>, - ) -> Result<(), PcgError> { - // Snapshot the borrow-graph places *before* acquiring `fg` so the - // owned HashSet doesn't hold a borrow of the graph across the - // restore_place call below. + /// The places blocked by `edge` that, after its removal, are no longer + /// blocked and are still tracked. + fn places_unblocked_by_removal(&self, edge: &BorrowPcgEdgeKind<'tcx>) -> Vec> { let borrow_places = self.pcg.borrow.graph.places(self.ctxt.bc_ctxt()); - let fg = self.pcg.borrow.graph.frozen_graph(); - let blocked_nodes = edge.blocked_nodes(self.ctxt.bc_ctxt()); - - // After removing an edge, some nodes may become accessible, their capabilities should be restored - let to_restore = blocked_nodes - .into_iter() + edge.blocked_nodes(self.ctxt.bc_ctxt()) .filter(|node| !fg.has_edge_blocking(*node, self.ctxt.bc_ctxt())) - .collect::>(); - - for node in to_restore { - if let Some(place) = node.as_current_place() { - // Only restore if the place is still tracked somewhere. After - // a loop abstraction edge is removed, the blocked place (e.g. - // `(*variant).fields`) may no longer appear in either the owned - // PCG or the borrow graph; restoring a capability there would - // violate the invariant that every capability-holding place must - // be tracked. - if !borrow_places.contains(&place) - && !self.pcg.owned.contains_place(place, self.ctxt.bc_ctxt()) - { - continue; - } - self.restore_place( - place, - "update_unblocked_node_capabilities_and_remove_placeholder_projections", - )?; - } - } - Ok(()) + .filter_map(|node| node.as_current_place()) + // Only places that are still tracked somewhere. After a loop + // abstraction edge is removed, the blocked place (e.g. + // `(*variant).fields`) may no longer appear in either the owned + // PCG or the borrow graph; giving a capability there would + // violate the invariant that every capability-holding place must + // be tracked. + .filter(|place| { + borrow_places.contains(place) + || self.pcg.owned.contains_place(*place, self.ctxt.bc_ctxt()) + }) + .collect() } /// If the following conditions apply: @@ -250,12 +424,26 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> edges: HashSet>>, context: &str, ) -> Result<(), PcgError> { - for edge in edges { + let ref_place = deref_place + .as_current_place() + .map(|deref_place| deref_place.parent_place().unwrap()); + let edges = edges.into_iter().collect::>(); + let mut unblocked = vec![]; + for (i, edge) in edges.iter().enumerate() { let borrow_edge: BorrowPcgEdge<'tcx> = - BorrowPcgEdge::new(edge.value.into(), edge.conditions); - self.apply_action( - BorrowPcgAction::remove_edge(borrow_edge, context.to_owned()).into(), - )?; + BorrowPcgEdge::new(edge.value.into(), edge.conditions.clone()); + // Removing the last edge packs the dereference into the reference. + unblocked = self.remove_edge_regaining(borrow_edge, context, |slf| { + // Perhaps the place isn't yet a leaf(e.g. the ref itself is + // conditionally borrowed): then it stays without its + // capability + ref_place + .filter(|ref_place| { + i + 1 == edges.len() && slf.pcg.is_leaf_place(*ref_place, slf.ctxt) + }) + .into_iter() + .collect() + })?; self.unlabel_blocked_region_projections_if_applicable(&edge.value, context)?; } if let Some(deref_place) = deref_place.as_current_place() { @@ -267,15 +455,9 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> ) .into(), )?; - let ref_place = deref_place.parent_place().unwrap(); - // Perhaps the place isn't yet a leaf(e.g. the ref itself is conditionally borrowed) - // In that case we shouldn't restore its caps - if self.pcg.is_leaf_place(ref_place, self.ctxt) { - self.restore_place( - ref_place, - &format!("{context}: remove_deref_edges_to: restore parent place"), - )?; - } + } + for place in unblocked { + self.place_unblocked(place, &format!("{context}: remove_deref_edges_to"))?; } Ok(()) } @@ -295,10 +477,6 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> context, ); } - self.record_and_apply_action( - BorrowPcgAction::remove_edge(edge.clone(), context.to_owned()).into(), - )?; - // This is true iff the expansion is for a place (not a region projection), and changes // could have been made to the root place via the expansion // We check that the base is place and either: @@ -316,7 +494,14 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> false }; - self.update_unblocked_node_capabilities_and_remove_placeholder_projections(&edge.value)?; + // After removing an edge, some nodes may become accessible: an + // unpacking is packed, or a loan ends. + let unblocked = self.remove_edge_regaining(edge.clone(), context, |slf| { + slf.places_unblocked_by_removal(&edge.value) + })?; + for place in unblocked { + self.place_unblocked(place, context)?; + } match &edge.value { BorrowPcgEdgeKind::Deref(deref) => { @@ -395,7 +580,7 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> /// Called once all of `removed_edges`, which are removed together, are /// gone: weakens the places whose lifetime projections their removal left - /// dead and that still have a capability above [`CapabilityKind::Write`], + /// dead and that still have [`CapabilityKind::Exclusive`], /// e.g. because they were in use when the edge was removed. pub(crate) fn finish_weakening_places_with_dead_lifetime_projections( &mut self, @@ -442,14 +627,12 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> &mut self, place: Place<'tcx>, ) -> Result<(), PcgError> { - if let Some(existing_cap) = self.pcg.capability_of(place, self.ctxt) - && existing_cap > CapabilityKind::Write - { + if self.pcg.capability_of(place, self.ctxt) == Some(CapabilityKind::Exclusive) { self.record_and_apply_action( BorrowPcgAction::weaken( place, - existing_cap, - Some(CapabilityKind::Write), + CapabilityKind::Exclusive, + CapabilityKind::Write, "lifetime projection is dead", ) .into(), @@ -499,44 +682,150 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> Ok(()) } - pub(crate) fn remove_read_permission_downwards( + /// Detaches the borrowed referents of shared references below `place`, + /// except of the references `keep` keeps dereferenced: the referent is + /// labelled, so that its borrows keep the labelled referent, and the + /// dereference edge is removed. The referent of a shared reference does + /// not depend on the place holding the reference. + pub(crate) fn detach_borrowed_shared_referents( &mut self, - upgraded_place: Place<'tcx>, + place: Place<'tcx>, + keep: &impl Fn(Place<'tcx>) -> bool, ) -> Result<(), PcgError> { - let to_remove = self - .pcg - .place_capabilities - .iter() - .filter(|(p, _)| { - p.projection.len() > upgraded_place.projection.len() - && upgraded_place.is_prefix_of(*p) + let ctxt = self.ctxt; + let graph = &self.pcg.borrow.graph; + let is_borrowed = |referent: Place<'tcx>| { + graph.edges().any(|edge| { + !matches!( + edge.kind(), + BorrowPcgEdgeKind::Deref(_) | BorrowPcgEdgeKind::BorrowPcgExpansion(_) + ) && edge.kind().blocked_nodes(ctxt.bc_ctxt()).any(|node| { + node.related_maybe_labelled_place() + .and_then(MaybeLabelledPlace::as_current_place) + .is_some_and(|p| referent.is_prefix_of(p)) + }) + }) + }; + let mut referents = graph + .edges() + .filter_map(|edge| { + let BorrowPcgEdgeKind::Deref(deref) = edge.kind() else { + return None; + }; + let reference = deref.blocked_place().as_current_place()?; + let referent = deref.deref_place().as_current_place()?; + (place.is_prefix_of(reference) + && reference.is_shared_ref(ctxt) + && !keep(reference) + && is_borrowed(referent)) + .then_some(referent) }) .collect::>(); - for (p, cap) in to_remove { + referents.sort_by_key(|referent| referent.projection.len()); + let mut detached: Vec> = vec![]; + for referent in referents { + if detached.iter().any(|d| d.is_prefix_of(referent)) { + continue; + } + detached.push(referent); self.record_and_apply_action( - BorrowPcgAction::weaken(p, cap, None, "Remove read permission downwards").into(), + BorrowPcgAction::label_place_and_update_related_capabilities( + referent, + self.prev_snapshot_location(), + LabelPlaceReason::Detach, + ) + .into(), )?; + let reference = referent.parent_place().unwrap(); + let derefs = self + .pcg + .borrow + .graph + .edges() + .filter_map(|edge| match edge.kind() { + BorrowPcgEdgeKind::Deref(deref) + if deref.blocked_place().as_current_place() == Some(reference) => + { + Some(Conditioned::new(*deref, edge.conditions.clone())) + } + _ => None, + }) + .collect::>(); + let Some(deref) = derefs.iter().next() else { + continue; + }; + let deref_place = deref.value.deref_place(); + self.remove_deref_edges_to(deref_place, derefs, "detach shared referent")?; } Ok(()) } - pub(crate) fn upgrade_read_to_exclusive(&mut self, place: Place<'tcx>) -> Result<(), PcgError> { + /// Packs the read unpacking below `place`, deepest first, except where + /// places in it are borrowed and the unpackings of the places `keep` + /// keeps. + pub(crate) fn pack_read_unpacking( + &mut self, + place: Place<'tcx>, + keep: &impl Fn(Place<'tcx>) -> bool, + ) -> Result<(), PcgError> { + loop { + let fg = self.pcg.borrow.graph.frozen_graph(); + let packable = fg + .leaf_edges(self.ctxt) + .into_iter() + .map(BorrowPcgEdgeLike::to_owned_edge) + .find(|edge| { + unpacking_places(edge.kind()).is_some_and(|(base, _, _)| { + place.is_prefix_of(base) + && !keep(base) + && self.pcg.capability_of(base, self.ctxt) == Some(CapabilityKind::Read) + && edge.kind().blocked_by_nodes(self.ctxt).all(|node| { + node.as_current_place().is_some_and(|p| { + self.pcg.capability_of(p, self.ctxt) + == Some(CapabilityKind::Read) + }) + }) + }) + }); + let Some(edge) = packable else { + return Ok(()); + }; + self.remove_edge_and_perform_associated_state_updates(&edge, "pack read unpacking")?; + } + } + + /// Unfreezes the frozen place `place` or that `place` is a read copy of, + /// packing its read unpacking except the unpackings of the places `keep` + /// keeps, which become exclusive. `toward` (`place` or a place in it) is + /// the place being obtained. + pub(crate) fn upgrade_read_to_exclusive( + &mut self, + place: Place<'tcx>, + toward: Place<'tcx>, + keep: &impl Fn(Place<'tcx>) -> bool, + ) -> Result<(), PcgError> { tracing::debug!( "upgrade_read_to_exclusive: {}", place.display_string(self.ctxt.bc_ctxt()) ); - self.record_and_apply_action( - BorrowPcgAction::restore_capability( - place, - CapabilityKind::Exclusive, - "upgrade_read_to_exclusive", - ) - .into(), - )?; - self.remove_read_permission_downwards(place)?; - if let Some(parent) = place.parent_place() { - self.remove_read_permission_upwards_and_label_rps( - parent, + self.detach_borrowed_shared_referents(place, keep)?; + self.pack_read_unpacking(place, keep)?; + let frozen = self.frozen_ancestor(place); + // Packing the whole read unpacking already unfreezes the frozen place. + if self + .pcg + .place_capability_equals(frozen, CapabilityKind::Read, self.ctxt) + { + self.end_freeze( + frozen, + Some(toward), + AdjustCapabilityReason::UpgradeReadToExclusive, + )?; + } + if frozen != place { + self.label_rps_up_to( + place.parent_place().unwrap(), + frozen, AdjustCapabilityReason::UpgradeReadToExclusive, )?; } @@ -561,7 +850,7 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> loop { if let Some(cap) = self.pcg.capability_of(expand_root, self.ctxt) { if cap.is_read() { - self.upgrade_read_to_exclusive(expand_root)?; + self.upgrade_read_to_exclusive(expand_root, place, &|_| false)?; } return Ok(()); } @@ -580,8 +869,167 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> where Ctxt: HasBorrowCheckerCtxt<'a, 'tcx> + HasSettings<'a>, { + let mut action = action; + let result = self.apply_action_without_recording(&mut action)?; + self.record_applied_action(action, result); + Ok(()) + } + + /// Removes `edge`; the places `unblocked` finds unblocked afterwards get + /// their capability back as part of the removal. Returns those places. + pub(crate) fn remove_edge_regaining( + &mut self, + edge: BorrowPcgEdge<'tcx>, + context: &str, + unblocked: impl FnOnce(&Self) -> Vec>, + ) -> Result>, PcgError> { + let packed = matches!( + edge.kind(), + BorrowPcgEdgeKind::Deref(_) + | BorrowPcgEdgeKind::BorrowPcgExpansion(_) + | BorrowPcgEdgeKind::RawPtrDeref(_) + ) + .then(|| { + edge.kind() + .blocked_by_nodes(self.ctxt.bc_ctxt()) + .filter_map(|node| { + node.as_current_place() + .map(|place| self.pcg.capability_of(place, self.ctxt)) + }) + .collect::>>()? + .into_iter() + .reduce(meet) + }); + let blocked = edge + .kind() + .blocked_nodes(self.ctxt.bc_ctxt()) + .filter_map(|node| node.as_current_place()) + .collect::>(); + // What the parts record (frozen or lent apart) goes to the place they + // pack into. + if packed.is_some() + && edge + .kind() + .blocked_by_nodes(self.ctxt.bc_ctxt()) + .any(|node| { + node.as_current_place() + .is_some_and(|part| self.pcg.place_capabilities.suspended(part).is_some()) + }) + { + for &base in &blocked { + self.record_and_apply_action( + BorrowPcgAction::pack_suspended(base, context.to_owned()).into(), + )?; + } + } + let mut action = BorrowPcgAction::remove_edge(edge, context.to_owned()); + let result = + self.pcg + .borrow + .apply_action(&mut action, self.pcg.place_capabilities, self.ctxt); + let unblocked = unblocked(self); + let regained = match packed { + // A base that only shared borrows still block gets back the R its + // parts held. + Some(Some(CapabilityKind::Read)) => { + let read_bases = blocked + .iter() + .copied() + .filter(|base| { + !unblocked.contains(base) + && self + .pcg + .borrow + .graph + .edges_blocking((*base).into(), self.ctxt.bc_ctxt()) + .all(|edge| { + matches!(edge.kind(), BorrowPcgEdgeKind::Borrow(borrow) if !borrow.is_mut()) + }) + }) + .collect::>(); + self.packed_capabilities( + &unblocked + .iter() + .copied() + .chain(read_bases) + .collect::>(), + CapabilityKind::Read, + ) + } + Some(Some(packed)) => self.packed_capabilities(&unblocked, packed), + Some(None) => CapabilityChanges::default(), + None => { + let comeback = self.loan_comeback(&blocked); + for (place, parts) in comeback.unpacks { + self.record_and_apply_action( + BorrowPcgAction::unpack_suspended(place, parts, context.to_owned()).into(), + )?; + } + for part in comeback.forgotten { + self.forget_unpacked(part)?; + } + let mut regained = CapabilityChanges::default(); + for place in comeback.frozen { + regained.set(place, Some(CapabilityKind::Read)); + } + for place in comeback.receivers { + if let Some(record) = self.pcg.place_capabilities.suspended(place) { + regained.set(place, Some(record)); + } + } + regained + } + }; + self.pcg + .borrow + .regain(®ained, self.pcg.place_capabilities, self.ctxt); + action.capabilities.extend(®ained); + self.record_applied_action(action.into(), result); + Ok(unblocked) + } + + /// Forgets `part`, which holds nothing: the unpacking to it and its + /// layout, owned expansions and unpackings in the borrow PCG, which are + /// removed leaves first. + fn forget_unpacked(&mut self, part: Place<'tcx>) -> Result<(), PcgError> { + let ctxt = self.ctxt; + loop { + let fg = self.pcg.borrow.graph.frozen_graph(); + let Some(edge) = fg + .leaf_edges(ctxt) + .into_iter() + .map(BorrowPcgEdgeLike::to_owned_edge) + .find(|edge| { + matches!( + edge.kind(), + BorrowPcgEdgeKind::BorrowPcgExpansion(_) + | BorrowPcgEdgeKind::Deref(_) + | BorrowPcgEdgeKind::RawPtrDeref(_) + ) && edge.kind().blocked_by_nodes(ctxt.bc_ctxt()).any(|node| { + node.as_current_place() + .is_some_and(|below| part.is_prefix_of(below)) + }) + }) + else { + break; + }; + self.remove_edge_and_perform_associated_state_updates(&edge, "forget unpacking")?; + } + if self.pcg.owned.is_allocated(part.local) { + self.pcg.owned[part.local] + .expansions_mut() + .forget_expansions_to(part, ctxt); + } + Ok(()) + } + + /// Applies `action` and records its capability changes in it. + fn apply_action_without_recording( + &mut self, + action: &mut PcgAction<'tcx>, + ) -> Result { tracing::debug!("Applying Action: {}", action.debug_line(self.ctxt)); - let result = match &action { + Ok(match action { PcgAction::Borrow(action) => { // An expansion guided by the index local outlives the local's // value: its guide refers to the value at the label instead. @@ -600,67 +1048,75 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> .graph .label_index(index, label.location, self.ctxt); } - self.pcg - .borrow - .apply_action(action.clone(), self.pcg.place_capabilities, self.ctxt) - } - PcgAction::Owned(owned_action) => match owned_action.kind { - RepackOp::RegainLoanedCapability(regained_capability) => { - self.pcg.place_capabilities.regain_loaned_capability( - regained_capability.place, - regained_capability.capability, - self.pcg.borrow.as_mut_ref(), - self.ctxt, - ); - ApplyActionResult::changed_no_display() + let result = + self.pcg + .borrow + .apply_action(action, self.pcg.place_capabilities, self.ctxt); + // An owned place weakened to W no longer holds its value. + if let BorrowPcgActionKind::Weaken(weaken) = &action.kind + && weaken.to == CapabilityKind::Write + && let Some(owned) = weaken.place.as_owned_place(self.ctxt) + { + self.pcg + .owned + .apply_capability_change(owned, CapabilityKind::Write, self.ctxt); } - RepackOp::Expand(expand) => { - self.pcg.owned.perform_expand_action( + result + } + PcgAction::Owned(owned_action) => { + owned_action.capabilities = match owned_action.kind { + RepackOp::Expand(expand) => self.pcg.owned.perform_expand_action( expand, self.pcg.place_capabilities, self.ctxt, - ); - ApplyActionResult::changed_no_display() - } - RepackOp::DerefShallowInit(from, to) => { - let target_places = from.expand_one_level(to, self.ctxt)?.expansion(); - let capability_projections = self.pcg.owned[from.local].expansions_mut(); - capability_projections.insert_expansion( - from, - PlaceExpansion::from_places(target_places.clone(), self.ctxt), - ); - for target_place in target_places { - self.pcg.place_capabilities.insert( - target_place, - CapabilityKind::Read, - self.ctxt, + ), + RepackOp::DerefShallowInit(from, to) => { + let target_places = from.expand_one_level(to, self.ctxt)?.expansion(); + let capability_projections = self.pcg.owned[from.local].expansions_mut(); + capability_projections.insert_expansion( + from, + PlaceExpansion::from_places(target_places.clone(), self.ctxt), ); + let changes: CapabilityChanges> = target_places + .into_iter() + .map(|place| (place, Some(CapabilityKind::Read))) + .collect::>() + .into(); + changes.apply(self.pcg.place_capabilities, self.ctxt); + changes } - ApplyActionResult::changed_no_display() - } - RepackOp::Collapse(collapse) => { - let capability_projections = self.pcg.owned[collapse.local()].expansions_mut(); - capability_projections.perform_collapse_action( - collapse, - self.pcg.place_capabilities, - self.ctxt, - ); - ApplyActionResult::changed_no_display() - } - RepackOp::Weaken(weaken) => { - pcg_validity_assert!( + RepackOp::Collapse(collapse) => { + let capability_projections = + self.pcg.owned[collapse.local()].expansions_mut(); + capability_projections.perform_collapse_action( + collapse, + self.pcg.place_capabilities, + self.ctxt, + ) + } + RepackOp::StorageDead(local) => { + let changes: CapabilityChanges> = self + .pcg + .place_capabilities + .iter() + .filter(|(place, _)| place.local == local) + .map(|(place, _)| (place, None)) + .collect::>() + .into(); + changes.apply(self.pcg.place_capabilities, self.ctxt); self.pcg - .place_capability_equals(weaken.place, weaken.from, self.ctxt) - || weaken.place.contains_unsafe_deref(self.ctxt) - ); - self.pcg - .place_capabilities - .insert(weaken.place, weaken.to, self.ctxt); - ApplyActionResult::changed_no_display() - } - _ => unreachable!(), - }, - }; + .place_capabilities + .retain_suspended(|place, _| place.local != local); + self.pcg.owned.deallocate(local); + changes + } + }; + ApplyActionResult::changed_no_display() + } + }) + } + + fn record_applied_action(&mut self, action: PcgAction<'tcx>, result: ApplyActionResult) { #[cfg(feature = "visualization")] let location = self.location(); #[cfg(feature = "visualization")] @@ -683,7 +1139,6 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> } actions.push(AppliedAction::new(action, result)); } - Ok(()) } pub(crate) fn phase(&self) -> Option { match self.prev_snapshot_location { @@ -715,44 +1170,6 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> /// /// This also handles corresponding lifetime projections of the place. #[tracing::instrument(skip(self))] - /// Owned leaves below `place` without capability (e.g. from joining a - /// [`CapabilityKind::Write`] leaf with a read-only expansion) are no longer - /// blocked once `place` has been labelled for a write, so they regain - /// [`CapabilityKind::Write`]. - fn restore_write_to_unblocked_owned_leaves( - &mut self, - place: Place<'tcx>, - ) -> Result<(), PcgError> { - let blocked = self.blocked_owned_places(self.ctxt.bc_ctxt()); - let to_restore = self - .pcg - .owned - .leaf_places(self.ctxt) - .into_iter() - .map(Place::from) - .filter(|p| { - place.is_strict_prefix_of(*p) && self.pcg.capability_of(*p, self.ctxt).is_none() - }) - .collect::>(); - let ctxt = self.ctxt; - for leaf in to_restore { - pcg_validity_assert!( - !blocked.contains(&leaf), - [ctxt], - "{} is blocked after labelling {} for a write", - leaf.display_string(ctxt), - place.display_string(ctxt) - ); - self.record_and_apply_action(PcgAction::restore_capability( - leaf, - CapabilityKind::Write, - "restore write to unblocked owned leaf", - self.ctxt, - ))?; - } - Ok(()) - } - pub(crate) fn obtain( &mut self, place: Place<'tcx>, @@ -798,7 +1215,6 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> ) .into(), )?; - self.restore_write_to_unblocked_owned_leaves(place)?; self.render_debug_graph(None, "after step 1"); } @@ -839,13 +1255,16 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> .collect::>(); for (p, from_cap) in to_weaken { let weaken = if obtain_type == ObtainType::ForStorageDead { - RepackOp::weaken_for_storage_dead(p, from_cap, CapabilityKind::Write) + BorrowPcgAction::weaken_for_storage_dead( + p, + from_cap, + CapabilityKind::Write, + "obtain for storage dead", + ) } else { - RepackOp::weaken(p, from_cap, CapabilityKind::Write) + BorrowPcgAction::weaken(p, from_cap, CapabilityKind::Write, "obtain W") }; - self.record_and_apply_action(PcgAction::Owned(OwnedPcgAction::new( - weaken, None, - )))?; + self.record_and_apply_action(weaken.into())?; } } tracing::debug!( @@ -867,6 +1286,18 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> ); } + // An unpacked borrowed place is read by freezing its unpacking (owned + // places are frozen by the collapse above). + if obtain_cap.is_read() + && place.as_owned_place(self.ctxt).is_none() + && self.pcg.capability_of(place, self.ctxt).is_none() + && self.pcg.place_capabilities.unpacks(place) + { + self.record_and_apply_action( + BorrowPcgAction::freeze(place, "read an unpacked place").into(), + )?; + } + // STEP 3 if !obtain_cap.is_read() { tracing::debug!( @@ -907,7 +1338,7 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> self.render_debug_graph(None, "after step 4"); } - self.expand_to(place, obtain_type, self.ctxt)?; + self.expand_to(place, None, obtain_type, self.ctxt)?; if obtain_type == ObtainType::ForStorageDead && let Some(cap @ (CapabilityKind::Exclusive | CapabilityKind::ShallowExclusive)) = @@ -915,10 +1346,15 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> { // Temporary: mark as for_storage_dead until // https://github.com/prusti/pcg/issues/137 is resolved. - self.record_and_apply_action(PcgAction::Owned(OwnedPcgAction::new( - RepackOp::weaken_for_storage_dead(place, cap, CapabilityKind::Write), - None, - )))?; + self.record_and_apply_action( + BorrowPcgAction::weaken_for_storage_dead( + place, + cap, + CapabilityKind::Write, + "obtain for storage dead", + ) + .into(), + )?; } self.render_debug_graph(None, "after step 5"); @@ -930,10 +1366,25 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> impl<'pcg, 'a: 'pcg, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> PlaceExpander<'a, 'tcx> for PlaceObtainer<'pcg, 'a, 'tcx, Ctxt> { + /// An expansion below a place that holds a capability, whose places hold + /// and record nothing, is only the layout of the loans of a place in it + /// (a join holds the enum packed while a variant field is borrowed). It + /// does not count: unpacking the place gives its parts their share. fn contains_owned_expansion_to(&self, target: Place<'tcx>) -> bool { + let capabilities = &self.pcg.place_capabilities; self.pcg.owned[target.local] .expansions() - .contains_expansion_to(target, self.ctxt) + .expansions() + .iter() + .any(|expanded| { + let places = expanded.expansion_places(self.ctxt).unwrap(); + places.contains(&target) + && (capabilities.get(expanded.place, self.ctxt).is_none() + || places.iter().any(|p| { + capabilities.get(*p, self.ctxt).is_some() + || capabilities.suspended(*p).is_some() + })) + }) } fn borrows_graph(&self) -> &borrow_pcg::graph::BorrowsGraph<'tcx> { @@ -944,30 +1395,38 @@ impl<'pcg, 'a: 'pcg, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> PlaceExpander<'a, ' self.pcg.borrow.validity_conditions.clone() } - fn update_capabilities_for_borrow_expansion( - &mut self, + fn borrow_expansion_capabilities( + &self, expansion: &BorrowPcgExpansion<'tcx>, block_type: BlockType, _ctxt: CompilerCtxt<'_, 'tcx>, - ) -> bool { + ) -> Vec<(Place<'tcx>, Option)> { self.pcg .place_capabilities - .update_for_expansion(expansion, block_type, self.ctxt) + .expansion_capabilities(expansion, block_type, self.ctxt) } fn location(&self) -> mir::Location { self.location } - fn update_capabilities_for_deref( - &mut self, + fn deref_capabilities( + &self, ref_place: Place<'tcx>, capability: CapabilityKind, _ctxt: CompilerCtxt<'_, 'tcx>, - ) -> bool { - self.pcg - .place_capabilities - .update_for_deref(ref_place, capability, self.ctxt) + ) -> Vec<(Place<'tcx>, Option)> { + PlaceCapabilities::deref_capabilities(ref_place, capability, self.ctxt) + } + + fn freeze_for_read_unpacking(&mut self, base: Place<'tcx>) -> Result<(), PcgError> { + if self + .pcg + .place_capability_equals(base, CapabilityKind::Exclusive, self.ctxt) + { + self.record_and_apply_action(BorrowPcgAction::freeze(base, "unpack for read").into())?; + } + Ok(()) } fn capability_for_expand( diff --git a/src/pcg/visitor/pack.rs b/src/pcg/visitor/pack.rs index 8b4f3773e..82ca38083 100644 --- a/src/pcg/visitor/pack.rs +++ b/src/pcg/visitor/pack.rs @@ -130,16 +130,20 @@ impl<'pcg, 'a: 'pcg, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx> + DebugCtxt> let mut iteration = 0; let ctxt = self.ctxt; let block = self.location().block; - self.restore_capability_to_leaf_places( - for_place, - |place| !ctxt.body_analysis().is_read_only_in_loop(block, place), - ctxt, - )?; let mut removed_edges = Vec::new(); loop { iteration += 1; let edges_to_remove = self.identify_leaf_edges_to_remove(for_place); if edges_to_remove.is_empty() { + let lent = self.pcg.borrow.graph.lent_places(ctxt.bc_ctxt()); + self.restore_capability_to_leaf_places( + for_place, + |place| { + !ctxt.body_analysis().is_read_only_in_loop(block, place) + && !lent.iter().any(|lent| lent.is_prefix_of(place)) + }, + ctxt, + )?; break self.finish_weakening_places_with_dead_lifetime_projections(&removed_edges); } for action in edges_to_remove.removal_actions() { @@ -217,7 +221,7 @@ impl<'pcg, 'a: 'pcg, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx> + DebugCtxt> BorrowPcgAction::weaken( rawptr, CapabilityKind::Exclusive, - Some(CapabilityKind::Write), + CapabilityKind::Write, context.clone(), ) .into(), @@ -493,6 +497,20 @@ impl<'pcg, 'a: 'pcg, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx> + DebugCtxt> edges_to_remove.push(&edge, reason); } } + // A raw pointer dereference borrows nothing from the pointer: once the + // pointer is gone, the dereferenced place no longer depends on it. + if ancestor_place.is_none() { + for edge in self.pcg.borrows_graph().edges() { + if let BorrowPcgEdgeKind::RawPtrDeref(deref) = edge.kind() + && deref.blocked_place.is_old() + { + edges_to_remove.push( + &edge.to_owned_edge(), + "Dereferenced raw pointer is dead".into(), + ); + } + } + } edges_to_remove } } diff --git a/src/pcg/visitor/stmt.rs b/src/pcg/visitor/stmt.rs index 1509e57a6..8fb0531d3 100644 --- a/src/pcg/visitor/stmt.rs +++ b/src/pcg/visitor/stmt.rs @@ -66,18 +66,11 @@ impl<'a, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> PcgVisitor<'_, 'a, 'tcx, Ctxt> .graph() .contains(target, self.ctxt.bc_ctxt()) { - // The permission to the target may have been Read originally. - // Now, because it's been made old, the non-old place should be a leaf, - // and its permission should be Exclusive. + // The target may have been frozen originally. Now, because + // its dereference has been made old, the freeze ends. if self.pcg.capability_of(target, self.ctxt) == Some(CapabilityKind::Read) { - self.record_and_apply_action( - BorrowPcgAction::restore_capability( - target, - CapabilityKind::Exclusive, - "Assign: restore capability to exclusive", - ) - .into(), - )?; + self.place_obtainer() + .upgrade_read_to_exclusive(target, target, &|_| false)?; } } @@ -92,7 +85,7 @@ impl<'a, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> PcgVisitor<'_, 'a, 'tcx, Ctxt> BorrowPcgAction::weaken( target, target_cap, - Some(CapabilityKind::Write), + CapabilityKind::Write, "pre_main", ) .into(), diff --git a/src/pcg/visitor/triple.rs b/src/pcg/visitor/triple.rs index 3e1465e30..f20cff7d9 100644 --- a/src/pcg/visitor/triple.rs +++ b/src/pcg/visitor/triple.rs @@ -1,12 +1,15 @@ use crate::{ + action::BorrowPcgAction, borrow_pcg::edge::kind::BorrowPcgEdgeKind, error::PcgError, pcg::{ - PcgRefLike, + CapabilityKind, PcgRefLike, obtain::ObtainType, + place_capabilities::PlaceCapabilitiesReader, triple::{PlaceCondition, Triple}, }, - utils::{DataflowCtxt, Place}, + pcg_validity_assert, + utils::{DataflowCtxt, Place, display::DisplayWithCompilerCtxt}, }; use super::PcgVisitor; @@ -16,8 +19,15 @@ impl<'a, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> PcgVisitor<'_, 'a, 'tcx, Ctxt> pub(crate) fn require_triple(&mut self, triple: Triple<'tcx>) -> Result<(), PcgError> { match triple.pre() { PlaceCondition::ExpandTwoPhase(place) => { - self.place_obtainer() - .obtain(place, ObtainType::TwoPhaseExpand)?; + // The reservation obtains `place` as a mutable borrow does, + // unless `place` is only readable because of a live shared + // borrow of it or of an ancestor. + let obtain_type = if self.closest_capability(place) == Some(CapabilityKind::Read) { + ObtainType::TwoPhaseExpand + } else { + ObtainType::Capability(CapabilityKind::Exclusive) + }; + self.place_obtainer().obtain(place, obtain_type)?; } PlaceCondition::Capability(place, capability) => { self.place_obtainer() @@ -51,14 +61,49 @@ impl<'a, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx>> PcgVisitor<'_, 'a, 'tcx, Ctxt> } } PlaceCondition::Unalloc(_) | PlaceCondition::Return => {} - PlaceCondition::RemoveCapability(_) => unreachable!(), + PlaceCondition::Freeze(_) => unreachable!(), } Ok(()) } + /// The capability of `place` or, if it has none, of its closest ancestor + /// that has one. + fn closest_capability(&self, place: Place<'tcx>) -> Option { + let mut current = place; + loop { + if let Some(cap) = self.pcg.capability_of(current, self.ctxt) { + return Some(cap); + } + current = current.parent_place()?; + } + } + #[tracing::instrument(skip(self, triple))] pub(crate) fn ensure_triple(&mut self, triple: Triple<'tcx>) -> Result<(), PcgError> { self.pcg.ensure_triple(triple, self.ctxt); + if let Some(PlaceCondition::Freeze(place)) = triple.post() { + let cap = self.pcg.capability_of(place, self.ctxt); + let unpacked = cap.is_none() && self.pcg.place_capabilities.unpacks(place); + if matches!( + cap, + Some(CapabilityKind::Exclusive | CapabilityKind::ShallowExclusive) + ) || unpacked + { + self.record_and_apply_action( + BorrowPcgAction::freeze(place, "shared or two-phase borrow").into(), + )?; + } else { + let ctxt = self.ctxt; + pcg_validity_assert!( + cap == Some(CapabilityKind::Read), + [ctxt], + "{:?}: Borrowing {} for read with capability {:?}", + self.location(), + place.display_string(self.ctxt), + cap + ); + } + } Ok(()) } } diff --git a/src/pcg/visitor/upgrade.rs b/src/pcg/visitor/upgrade.rs index b84e50ab5..d3e8abfb8 100644 --- a/src/pcg/visitor/upgrade.rs +++ b/src/pcg/visitor/upgrade.rs @@ -9,179 +9,286 @@ use crate::{ borrow_flow::{BorrowFlowEdge, BorrowFlowEdgeKind, private::FutureEdgeKind}, kind::BorrowPcgEdgeKind, }, - edge_data::LabelNodePredicate, - region_projection::LifetimeProjection, + edge_data::{EdgeData, LabelNodePredicate}, + region_projection::{LifetimeProjection, LifetimeProjectionIdx, PcgRegion, Region}, state::BorrowsStateLike, }, error::PcgError, pcg::{ CapabilityKind, PcgNode, PcgRefLike, - obtain::{HasSnapshotLocation, PlaceObtainer}, - place_capabilities::BlockType, + obtain::{HasSnapshotLocation, ObtainType, PlaceObtainer, expand::PlaceExpander}, + place_capabilities::PlaceCapabilitiesReader, }, + rustc_interface::index::IndexVec, utils::{ - DataflowCtxt, DebugCtxt, Place, data_structures::HashSet, display::DisplayWithCompilerCtxt, + DataflowCtxt, DebugCtxt, Place, PlaceLike, data_structures::HashSet, + display::DisplayWithCompilerCtxt, }, }; impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx> + DebugCtxt> PlaceObtainer<'state, 'a, 'tcx, Ctxt> { - fn weaken_capability_from_read( + /// The frozen place of whose read unpacking `place` is part: the topmost + /// of `place` and its ancestors that hold R without interruption. + pub(crate) fn frozen_ancestor(&self, place: Place<'tcx>) -> Place<'tcx> { + let mut frozen = place; + while let Some(parent) = frozen.parent_place() + && self + .pcg + .place_capability_equals(parent, CapabilityKind::Read, self.ctxt) + { + frozen = parent; + } + frozen + } + + /// The places below `frozen` borrowed by edges other than unpackings. + /// Borrows behind a shared reference do not hold a freeze. + fn borrowed_in(&self, frozen: Place<'tcx>) -> Vec> { + let ctxt = self.ctxt; + self.pcg + .borrow + .graph + .edges + .keys() + .filter(|kind| { + !matches!( + kind, + BorrowPcgEdgeKind::BorrowPcgExpansion(_) | BorrowPcgEdgeKind::Deref(_) + ) + }) + .flat_map(|kind| { + kind.blocked_nodes(ctxt.bc_ctxt()) + .filter_map(|node| node.as_current_place()) + .collect::>() + }) + .filter(|place| frozen.is_strict_prefix_of(*place) && !place.projects_shared_ref(ctxt)) + .collect() + } + + /// Ends the freeze of `frozen`: if parts of its value are still borrowed, + /// the freeze moves to them and the rest, with `toward` (the place being + /// obtained), gets E back. The freeze moves to the topmost read copies + /// that are borrowed or have a borrowed place in their value that their + /// own read unpacking does not reach. + pub(crate) fn end_freeze( &mut self, - place: Place<'tcx>, + frozen: Place<'tcx>, + toward: Option>, reason: AdjustCapabilityReason, ) -> Result<(), PcgError> { - if place.is_mut_ref(self.ctxt) { - // We've reached an indirection (e.g from **s to *s), we - // downgrade the ref from R to e (lent through) - // We need to continue: `s` would previously have capability R, which - // is not compatible with it being mutably borrowed - self.record_and_apply_action( - BorrowPcgAction::weaken( - place, - CapabilityKind::Read, - BlockType::DerefMutRefForExclusive.blocked_place_maximum_retained_capability(), - format!( - "{:?}: remove read permission upwards from base place {} (downgrade R to W for mut ref)", - reason, - place.display_string(self.ctxt), - ) - ) - .into(), - )?; + let in_value = self.borrowed_in(frozen); + if !in_value.is_empty() { + self.unpack_read_toward(frozen, toward, &in_value)?; + } + let copies = self + .pcg + .place_capabilities + .capabilities_for_strict_postfixes_of(frozen) + .map(|(p, _)| p) + .collect::>(); + let borrowed = copies + .iter() + .copied() + .filter(|copy| { + in_value.iter().any(|place| { + copy.is_prefix_of(*place) + && !copies + .iter() + .any(|c| copy.is_strict_prefix_of(*c) && c.is_prefix_of(*place)) + }) + }) + .collect::>(); + let to = borrowed + .iter() + .copied() + .filter(|p| !borrowed.iter().any(|q| q.is_strict_prefix_of(*p))) + .sorted() + .collect::>(); + let action = if to.is_empty() { + BorrowPcgAction::unfreeze(frozen, format!("{reason:?}")) } else { - self.record_and_apply_action( - BorrowPcgAction::weaken( - place, - CapabilityKind::Read, - None, - format!( - "{:?}: remove read permission upwards from base place {}", - reason, - place.display_string(self.ctxt), - ), - ) - .into(), + BorrowPcgAction::refreeze(frozen, to, format!("{reason:?}")) + }; + self.record_and_apply_action(action.into()) + } + + /// Unpacks the read unpacking of `frozen` toward `toward`, a part of its + /// value being obtained, and at each place on the way, one level into + /// every other unpacking of the layout that holds a place of `borrowed`. + /// A join that holds an enum packed keeps the layout of a borrowed + /// variant field but drops its read copies: the freeze moves to the + /// variant without entering it, and the variant being obtained gets E. + /// Obtaining `frozen` itself leaves no part to give E back to. + fn unpack_read_toward( + &mut self, + frozen: Place<'tcx>, + toward: Option>, + borrowed: &[Place<'tcx>], + ) -> Result<(), PcgError> { + let ctxt = self.ctxt; + let Some(toward) = toward.filter(|place| { + frozen.is_strict_prefix_of(*place) && !place.projects_indirection_from(frozen, ctxt) + }) else { + return Ok(()); + }; + self.expand_to( + toward, + None, + ObtainType::Capability(CapabilityKind::Read), + ctxt, + )?; + let path = std::iter::successors(toward.parent_place(), |p| p.parent_place()) + .take_while(|p| frozen.is_prefix_of(*p)) + .collect::>(); + let mut beside = self.pcg.owned[frozen.local] + .expansions() + .expansions() + .iter() + .filter(|expanded| path.contains(&expanded.place)) + .flat_map(|expanded| expanded.expansion_places(ctxt).unwrap()) + .filter(|part| { + !part.is_prefix_of(toward) && borrowed.iter().any(|b| part.is_prefix_of(*b)) + }) + .collect::>(); + beside.sort_by_key(|part| (part.projection.len(), *part)); + for part in beside { + self.expand_to( + part, + None, + ObtainType::Capability(CapabilityKind::Read), + ctxt, )?; } Ok(()) } - // Remove read permission upwards for strict ancestors of `place` Used when - // we want to access `place` exclusively when activating a two-phase borrow - // or when it is a postfix of a place that was downgraded to read - // permission. - pub(crate) fn remove_read_permission_upwards_and_label_rps( + /// Labels the lifetime projections of `place` and its ancestors up to + /// `frozen` for the exclusive access of `place`. + pub(crate) fn label_rps_up_to( &mut self, place: Place<'tcx>, + frozen: Place<'tcx>, reason: AdjustCapabilityReason, ) -> Result<(), PcgError> { let place_regions = place.regions(self.ctxt); - let mut prev = None; let mut current = place; - while self - .pcg - .place_capability_equals(current, CapabilityKind::Read, self.ctxt) - { - self.weaken_capability_from_read(current, reason)?; - let leaf_nodes = self.pcg.borrow.leaf_nodes(self.ctxt.bc_ctxt()); - for place in self.pcg.borrow.graph.places(self.ctxt.bc_ctxt()) { - if prev != Some(place) - && current.is_prefix_exact(place) - && leaf_nodes.contains(&place.into()) - && self - .pcg - .place_capability_equals(place, CapabilityKind::Read, self.ctxt) - && !place.projects_shared_ref(self.ctxt) - { - self.record_and_apply_action( - BorrowPcgAction::restore_capability( - place, - CapabilityKind::Exclusive, - format!( - "{:?}: remove_read_permission_upwards_and_label_rps: restore exclusive cap for leaf place {}", - reason, - place.display_string(self.ctxt) - ), - ) - .into(), - )?; - } + loop { + self.label_rps_for_exclusive_access(current, &place_regions, reason)?; + if current == frozen { + return Ok(()); } - for r in &place_regions { - let Some(current_rp) = LifetimeProjection::new( - current, - GeneralizedLifetime::from(*r), - None, - self.ctxt.ctxt(), - ) else { - // e.g. for ReStatic - continue; - }; - if current.is_ref(self.ctxt) - && !current - .project_deref(self.ctxt) - .regions(self.ctxt) - .iter() - .contains(r) - { - // If this region projection couldn't be modified (because its deref doesn't hold r), we skip - // There is probably a better way to do this - continue; - } - let edges_blocking_current_rp = self - .pcg - .borrow - .edges_blocking(current_rp.into(), self.ctxt.bc_ctxt()) - .collect::>(); - if !edges_blocking_current_rp.is_empty() { - let labelled_rp = current_rp - .with_label(Some(self.prev_snapshot_location().into()), self.ctxt); - let expansion_nodes = edges_blocking_current_rp - .iter() - .filter_map(|e| { - if let BorrowPcgEdgeKind::BorrowPcgExpansion(e) = e.kind() - && let PcgNode::LifetimeProjection(rp) = e.base() - && rp == current_rp.into() - { - Some( - e.expansion() - .iter() - .flat_map(|e| e.try_into_lifetime_projection()) - .filter(|e| e.base.is_current()) - .collect::>(), - ) - } else { - None - } - }) - .flatten() - .collect::>(); + current = current.parent_place().unwrap(); + } + } + + /// Labels the lifetime projections of `current` (for the regions + /// `place_regions` of the place being accessed exclusively) that are + /// blocked by other edges, and adds the future edges for them. + pub(crate) fn label_rps_for_exclusive_access( + &mut self, + current: Place<'tcx>, + place_regions: &IndexVec, PcgRegion<'tcx>>, + reason: AdjustCapabilityReason, + ) -> Result<(), PcgError> { + for r in place_regions { + let Some(current_rp) = LifetimeProjection::new( + current, + GeneralizedLifetime::from(*r), + None, + self.ctxt.ctxt(), + ) else { + // e.g. for ReStatic + continue; + }; + if current.is_ref(self.ctxt) + && !current + .project_deref(self.ctxt) + .regions(self.ctxt) + .iter() + .contains(r) + { + // If this region projection couldn't be modified (because its deref doesn't hold r), we skip + // There is probably a better way to do this + continue; + } + let edges_blocking_current_rp = self + .pcg + .borrow + .edges_blocking(current_rp.into(), self.ctxt.bc_ctxt()) + .collect::>(); + if !edges_blocking_current_rp.is_empty() { + let labelled_rp = + current_rp.with_label(Some(self.prev_snapshot_location().into()), self.ctxt); + let expansion_nodes = edges_blocking_current_rp + .iter() + .filter_map(|e| { + if let BorrowPcgEdgeKind::BorrowPcgExpansion(e) = e.kind() + && let PcgNode::LifetimeProjection(rp) = e.base() + && rp == current_rp.into() + { + Some( + e.expansion() + .iter() + .flat_map(|e| e.try_into_lifetime_projection()) + .filter(|e| e.base.is_current()) + .collect::>(), + ) + } else { + None + } + }) + .flatten() + .collect::>(); + self.record_and_apply_action( + BorrowPcgAction::label_lifetime_projection( + LabelNodePredicate::equals_lifetime_projection(current_rp.into()), + Some(self.prev_snapshot_location().into()), + format!( + "{:?}: remove_read_permission_upwards_and_label_rps: label current lifetime projection {} with previous snapshot location {:?}", + reason, + current_rp.display_string(self.ctxt), + self.prev_snapshot_location() + ), + ) + .into(), + )?; + // If the place is owned, we will never regain access to it, + // therefore future nodes aren't necessary + if current.as_borrowed_place(self.ctxt).is_some() { + let future_rp = current_rp.with_placeholder_label(self.ctxt); self.record_and_apply_action( - BorrowPcgAction::label_lifetime_projection( - LabelNodePredicate::equals_lifetime_projection(current_rp.into()), - Some(self.prev_snapshot_location().into()), - format!( - "{:?}: remove_read_permission_upwards_and_label_rps: label current lifetime projection {} with previous snapshot location {:?}", - reason, - current_rp.display_string(self.ctxt), - self.prev_snapshot_location() + BorrowPcgAction::add_edge( + BorrowPcgEdge::new( + BorrowFlowEdge::new( + labelled_rp.into(), + future_rp.into(), + BorrowFlowEdgeKind::Future(FutureEdgeKind::ToFutureSelf), + ) + .into(), + self.pcg.borrow.validity_conditions.clone(), ), + "remove_read_permission_upwards_and_label_rps", ) .into(), )?; - // If the place is owned, we will never regain access to it, - // therefore future nodes aren't necessary - if current.as_borrowed_place(self.ctxt).is_some() { - let future_rp = current_rp.with_placeholder_label(self.ctxt); + for expansion_node in expansion_nodes { + // If the expansion isn't labelled, it is likely a sibling RO of the place gaining exclusive capability + // so we connect to its current version + // Otherwise if its labelled we connect to its placeholder version + let to_connect = if expansion_node.label().is_none() { + expansion_node.into() + } else { + expansion_node.with_placeholder_label(self.ctxt).into() + }; self.record_and_apply_action( BorrowPcgAction::add_edge( BorrowPcgEdge::new( BorrowFlowEdge::new( - labelled_rp.into(), + to_connect, future_rp.into(), - BorrowFlowEdgeKind::Future(FutureEdgeKind::ToFutureSelf), + BorrowFlowEdgeKind::Future(FutureEdgeKind::FromExpansion), ) .into(), self.pcg.borrow.validity_conditions.clone(), @@ -190,41 +297,9 @@ impl<'state, 'a: 'state, 'tcx: 'a, Ctxt: DataflowCtxt<'a, 'tcx> + DebugCtxt> ) .into(), )?; - for expansion_node in expansion_nodes { - // If the expansion isn't labelled, it is likely a sibling RO of the place gaining exclusive capability - // so we connect to its current version - // Otherwise if its labelled we connect to its placeholder version - let to_connect = if expansion_node.label().is_none() { - expansion_node.into() - } else { - expansion_node.with_placeholder_label(self.ctxt).into() - }; - self.record_and_apply_action( - BorrowPcgAction::add_edge( - BorrowPcgEdge::new( - BorrowFlowEdge::new( - to_connect, - future_rp.into(), - BorrowFlowEdgeKind::Future( - FutureEdgeKind::FromExpansion, - ), - ) - .into(), - self.pcg.borrow.validity_conditions.clone(), - ), - "remove_read_permission_upwards_and_label_rps", - ) - .into(), - )?; - } } } } - let Some(parent) = current.parent_place() else { - break; - }; - prev = Some(current); - current = parent; } Ok(()) } diff --git a/src/results/mod.rs b/src/results/mod.rs index 0310f053f..c91b6542a 100644 --- a/src/results/mod.rs +++ b/src/results/mod.rs @@ -9,11 +9,8 @@ use std::{borrow::Cow, collections::HashMap}; use derive_more::Deref; use crate::{ - action::{AppliedActions, BorrowPcgAction, PcgActions}, - borrow_pcg::{ - borrow_pcg_edge::{BorrowPcgEdge, BorrowPcgEdgeRef}, - region_projection::PlaceOrConst, - }, + action::{AppliedActions, PcgActions}, + borrow_pcg::{borrow_pcg_edge::BorrowPcgEdgeRef, region_projection::PlaceOrConst}, error::PcgError, r#loop::{PlaceUsageType, PlaceUsages}, pcg::{ @@ -37,7 +34,6 @@ use crate::{ }; use crate::{ - borrow_pcg::action::actions::BorrowPcgActions, owned_pcg::RepackOp, utils::{CompilerCtxt, eval_stmt_data::EvalStmtData}, }; @@ -132,24 +128,29 @@ impl<'a, 'tcx: 'a> PcgAnalysisResults<'a, 'tcx> { let from_pcg = &state.data.pcg; let from_post_main = from_pcg.states[EvalStmtPhase::PostMain].clone(); let states = state.data.pcg.states.to_owned(); - let self_abstraction_edges = from_post_main - .borrow - .graph() - .abstraction_edges() - .collect::>(); let ctxt: ResultsCtxt<'a, 'tcx> = self.results_ctxt(); let block = &self.body()[location.block]; + // The imaginary target of a false edge is never taken. + let imaginary_target = match block.terminator().kind { + mir::TerminatorKind::FalseEdge { + real_target, + imaginary_target, + } if imaginary_target != real_target => Some(imaginary_target), + _ => None, + }; let succ_blocks = successor_blocks(block.terminator()) .into_iter() .filter(|succ| { - self.cursor - .analysis() - .0 - .reachable_blocks - .borrow() - .contains(succ.index()) + Some(*succ) != imaginary_target + && self + .cursor + .analysis() + .0 + .reachable_blocks + .borrow() + .contains(succ.index()) }) .collect::>(); let succs = succ_blocks @@ -164,27 +165,7 @@ impl<'a, 'tcx: 'a> PcgAnalysisResults<'a, 'tcx> { .pcg .clone(); - let mut actions = - from_post_main.bridge(&to.entry_state, location.block, succ, ctxt)?; - - let mut borrow_actions = BorrowPcgActions::new(); - for abstraction in to.entry_state.borrow.graph().abstraction_edges() { - if !self_abstraction_edges.contains(&abstraction) { - borrow_actions.push( - BorrowPcgAction::add_edge( - BorrowPcgEdge::new( - abstraction.value.clone().into(), - abstraction.conditions, - ), - "terminator", - ), - self.engine(), - ); - } - } - - actions.extend(borrow_actions.into()); - + let actions = from_post_main.bridge(&to.entry_state, location.block, succ, ctxt)?; Ok(PcgSuccessor::new( succ, actions, diff --git a/src/utils/ctxt.rs b/src/utils/ctxt.rs index a43d9b142..256c045ae 100644 --- a/src/utils/ctxt.rs +++ b/src/utils/ctxt.rs @@ -11,7 +11,7 @@ use crate::{ }, error::{PcgError, PcgUnsupportedError}, owned_pcg::RepackGuide, - pcg::{BodyAnalysis, CompilerCtxtWithSettings, ctxt::AnalysisCtxt}, + pcg::{BodyAnalysis, ctxt::AnalysisCtxt}, pcg_validity_assert, rustc_interface::{ FieldIdx, PlaceTy, RustBitSet, @@ -388,10 +388,6 @@ pub(crate) trait DataflowCtxt<'a, 'tcx: 'a>: fn is_loop_head(self, block: BasicBlock) -> bool { self.body_analysis().is_loop_head(block) } - - fn compiler_ctxt_with_settings(self) -> CompilerCtxtWithSettings<'a, 'tcx> { - CompilerCtxtWithSettings::new(self.bc_ctxt(), self.settings(), self.body_analysis()) - } } pub trait HasBorrowCheckerCtxt<'a, 'tcx, BC = &'a dyn BorrowCheckerInterface<'tcx>>: HasCompilerCtxt<'a, 'tcx> + DebugCtxt diff --git a/test-files/029_prusti_issue_25.rs b/test-files/029_prusti_issue_25.rs index 9ba866e46..bb9a837f5 100644 --- a/test-files/029_prusti_issue_25.rs +++ b/test-files/029_prusti_issue_25.rs @@ -6,8 +6,8 @@ fn f1<'a>(p: &'a mut Point) -> &'a mut u32 { // exclusive (e.g. mutably reborrowed) // PCG: bb0[4] post_operands: (*p).x: R if p.x == 0 { - // PCG: bb1[3] pre_operands: Add Edge {p} -> {*p} under conditions bb0 -> bb1 - // PCG: bb1[3] pre_operands: Add Edge {*p} -> {(*p).x, (*p).y} under conditions bb0 -> bb1 + // PCG: bb1[3] pre_operands: Add Edge {p} -> {*p} under conditions bb0 -> bb1 setting p: e, *p: E + // PCG: bb1[3] pre_operands: Add Edge {*p} -> {(*p).x, (*p).y} under conditions bb0 -> bb1 setting *p: None, (*p).x: E, (*p).y: E &mut p.x } else { &mut p.x diff --git a/test-files/040_object_archive.rs b/test-files/040_object_archive.rs index 8ac46894a..96da325ad 100644 --- a/test-files/040_object_archive.rs +++ b/test-files/040_object_archive.rs @@ -55,6 +55,11 @@ impl<'data, R: ReadRef<'data>> Iterator for ArchiveMemberIterator<'data, R> { type Item = read::Result>; fn next(&mut self) -> Option { + // `index` is frozen by the read of `**index`; when it is dead, its + // freeze ends before its value is given up. + // PCG: bb4[1] pre_operands: Unfreeze index + // PCG: bb4[1] pre_operands: Weaken index from E to W + // ~PCG: bb4[1] pre_operands: Weaken index from R to W match &mut self.members { Members::AixBig { ref mut index } => match **index { [] => None, diff --git a/test-files/046_demo_cond_mut_borrow.rs b/test-files/046_demo_cond_mut_borrow.rs index b832dd723..66c558d43 100644 --- a/test-files/046_demo_cond_mut_borrow.rs +++ b/test-files/046_demo_cond_mut_borrow.rs @@ -8,6 +8,14 @@ fn f(c: bool){ }; *r = 7; x += 1; + // Each branch lends the place borrowed in the other branch to that + // borrow, which gives it back when it expires. + // PCG: bb1 -> bb3: Lend y to borrow: _6 before bb2[3]:PreMain = &mut y under conditions bb0 -> bb2 + // PCG: bb2 -> bb3: Lend x to borrow: r = &mut x under conditions bb0 -> bb1 + // PCG: bb3[3] pre_operands: Remove Edge borrow: r = &mut x under conditions bb0 -> bb1 returning E to x + // `r` is weakened once, when its borrow expires. + // PCG: bb3[3] pre_operands: Weaken r from E to W + // ~PCG: bb3[3] pre_operands: Weaken r from W to W } fn main(){ diff --git a/test-files/091_loop.rs b/test-files/091_loop.rs index f3f098ea6..bdf2d6bb8 100644 --- a/test-files/091_loop.rs +++ b/test-files/091_loop.rs @@ -11,8 +11,10 @@ impl List { let mut i = 0; let mut current = self; while i < n { - // PCG: bb1[0] post_main: borrow: y = &mut x - // PCG: bb1[0] post_main: borrow: z = &mut *y + // `y` is dead at the loop head while `*y` is lent to `z`. + // PCG: bb1[0] post_main: borrow: y loop bb1 = &mut x + // PCG: bb1[0] post_main: borrow: z = &mut *y loop bb1 + // PCG: bb1[0] post_main: y: W // PCG: bb1[0] post_main: Loop(bb1): Remote(_1)↓'?13 -> current↓'?17 current = match current.tail { Some(ref mut tail) => tail, diff --git a/test-files/092_http_path.rs b/test-files/092_http_path.rs index ef8fe3107..e5a31bf08 100644 --- a/test-files/092_http_path.rs +++ b/test-files/092_http_path.rs @@ -11,6 +11,10 @@ const NONE: u16 = u16::MAX; impl PathAndQuery { pub fn path(&self) -> &str { + // The path that read `*self` joins the early return, where `self` is + // dead: `self` is unfrozen and its value given up. + // PCG: bb8 -> bb9: Unfreeze self + // PCG: bb8 -> bb9: Weaken self from E to W let ret = if self.query == NONE { &self.data[..] } else { diff --git a/test-files/106_swap.rs b/test-files/106_swap.rs index b2ea26710..04dba942b 100644 --- a/test-files/106_swap.rs +++ b/test-files/106_swap.rs @@ -7,8 +7,8 @@ fn f<'a>(x: &'a mut S<'a>, y: &'a mut S<'a>){ fn client<'a>(x: &'a mut S<'a>, y: &'a mut S<'a>){ // Activation of two-phase borrow: - // PCG: bb0[5] pre_operands: Weaken *x from R to None - // PCG: bb0[5] pre_operands: Weaken *y from R to None + // PCG: bb0[5] pre_operands: Activate two-phase borrow borrow: _4 = &mut *x + // PCG: bb0[5] pre_operands: Activate two-phase borrow borrow: _5 = &mut *y f(x,y); } diff --git a/test-files/115_prusti_bug.rs b/test-files/115_prusti_bug.rs index b1ba942df..4d59711e2 100644 --- a/test-files/115_prusti_bug.rs +++ b/test-files/115_prusti_bug.rs @@ -4,7 +4,7 @@ struct List { } fn foo<'a: 'a>(list: &'a mut List) { - // PCG: bb0[3] pre_operands: Remove Edge {*list} -> {(*list).head, (*list).tail} + // PCG: bb0[3] pre_operands: Remove Edge {*list} -> {(*list).head, (*list).tail} returning E to *list let mut l = &mut list.tail; } diff --git a/test-files/116_jonas_loop.rs b/test-files/116_jonas_loop.rs index 8035f9d35..43253585c 100644 --- a/test-files/116_jonas_loop.rs +++ b/test-files/116_jonas_loop.rs @@ -5,8 +5,10 @@ struct List { fn foo<'a: 'a>(list: &'a mut List) { let mut c = &mut list.tail; - // PCG: bb1[0] pre_operands: Loop(bb1): (*list).tail -> c↓'?8 - // PCG: bb1[0] post_operands: (*list).head: E + // `list` is dead at the loop head: the loop abstracts its referent as of + // the head and `list` is only writable. + // PCG: bb1[0] pre_operands: Loop(bb1): (*list).tail loop bb1 -> c↓'?8 + // PCG: bb1[0] post_operands: list: W while let Some(c_tail) = c { c = &mut c_tail.tail; } diff --git a/test-files/130_prusti_issue_27.rs b/test-files/130_prusti_issue_27.rs index cf30a493e..a3f1c11b8 100644 --- a/test-files/130_prusti_issue_27.rs +++ b/test-files/130_prusti_issue_27.rs @@ -6,8 +6,8 @@ struct List { fn len(head: &List) -> usize { // For the FakeRead, the expansion should happen in PreOperands - // PCG: bb1[0] pre_main: Add Edge {head} -> {*head} under conditions bb0 -> bb1 - // PCG: bb1[0] pre_main: Add Edge {*head} -> {(*head).next} under conditions bb0 -> bb1 + // PCG: bb1[0] pre_main: Add Edge {head} -> {*head} under conditions bb0 -> bb1 setting head: R, *head: R + // PCG: bb1[0] pre_main: Add Edge {*head} -> {(*head).next} under conditions bb0 -> bb1 setting *head: R, (*head).next: R match head.next { None => 1, Some(box ref tail) => 1, diff --git a/test-files/132_oopsla_loop_fix.rs b/test-files/132_oopsla_loop_fix.rs index 45ad6ea81..ff4407c3f 100644 --- a/test-files/132_oopsla_loop_fix.rs +++ b/test-files/132_oopsla_loop_fix.rs @@ -22,9 +22,11 @@ fn main() { // PCG_LIFETIME_DISPLAY: list2 0 'b let mut list3 = List { head: rv3, tail: None}; // PCG_LIFETIME_DISPLAY: list3 0 'c -// PCG: bb4[0] pre_operands: Loop(bb4): *rv1 -> list1↓'a -// PCG: bb4[0] pre_operands: Loop(bb4): *rv1 -> list2↓'b -// PCG: bb4[0] pre_operands: Loop(bb4): *rv1 -> list3↓'c +// `rv1` is dead at the loop head, so the loop abstracts its referent as of the +// head. +// PCG: bb4[0] pre_operands: Loop(bb4): *rv1 loop bb4 -> list1↓'a +// PCG: bb4[0] pre_operands: Loop(bb4): *rv1 loop bb4 -> list2↓'b +// PCG: bb4[0] pre_operands: Loop(bb4): *rv1 loop bb4 -> list3↓'c // PCG: bb4[0] pre_operands: Loop(bb4): v2 -> list2↓'b // PCG: bb4[0] pre_operands: Loop(bb4): v2 -> list3↓'c // PCG: bb4[0] pre_operands: Loop(bb4): v3 -> list3↓'c diff --git a/test-files/160_hybrid_dfa.rs b/test-files/160_hybrid_dfa.rs index 6a6c9aa4a..59946b8ad 100644 --- a/test-files/160_hybrid_dfa.rs +++ b/test-files/160_hybrid_dfa.rs @@ -50,6 +50,13 @@ impl<'i, 'c> Lazy<'i, 'c> { } fn set_all_transitions(&mut self, from: LazyStateID, to: LazyStateID) { + // The loop borrows `*self` mutably while the iterator borrows the + // classes behind the shared reference `self.dfa`. + // PCG: bb2 -> bb3: Label place *(*self).dfa (Detach) + // PCG: bb2 -> bb3: Remove Edge {(*self).dfa} -> {*(*self).dfa loop bb3} + // PCG: bb2 -> bb3: Remove Edge {*self} -> {(*self).dfa, (*self).cache} + // PCG: bb2 -> bb3: Unfreeze self + // ~PCG: bb2 -> bb3: Weaken *(*self).dfa from R to None for unit in self.dfa.classes.representatives(..) { self.set_transition(from, unit, to); } diff --git a/test-files/191_oopsla_examples.rs b/test-files/191_oopsla_examples.rs index 36547b040..96d40c885 100644 --- a/test-files/191_oopsla_examples.rs +++ b/test-files/191_oopsla_examples.rs @@ -33,7 +33,7 @@ fn dec_max(mut pos: Pos2D, mut z: i32) { &mut pos.y }; let max3 = if z > *max2 { - // PCG: bb4[0] pre_operands: Remove Edge borrow: max2 = &mut pos.x under conditions bb0 -> bb1, bb3 -> bb4 + // PCG: bb4[0] pre_operands: Remove Edge borrow: max2 = &mut pos.x under conditions bb0 -> bb1, bb3 -> bb4 returning E to pos.x // PCG: bb4[0] pre_operands: Remove Edge borrow: max2 = &mut (*_7) before bb2[5]:PreMain under conditions bb0 -> bb2, bb3 -> bb4 // PCG: bb4[2] pre_main: unpack pos with capability E // PCG: bb4[2] pre_main: Weaken pos.x from E to W diff --git a/test-files/192_loop_rb.rs b/test-files/192_loop_rb.rs index 82d0cdcea..6ace3f6bf 100644 --- a/test-files/192_loop_rb.rs +++ b/test-files/192_loop_rb.rs @@ -22,8 +22,15 @@ fn loop_rb(a_in: &mut List, b_in: &mut List) { let b = &mut *b_in; let mut x = None; let mut cont = true; - // The borrow of `b` survives the loop. - // PCG: bb1[0] post_main: borrow: b = &mut *b_in + // The borrow of `b` survives the loop; `b_in` is dead at its head. + // PCG: bb1[0] post_main: borrow: b = &mut *b_in loop bb1 + // PCG: bb1[0] post_main: b_in: W + // At the join after the `if`, each branch unpacks the reference the other + // branch reborrowed from and lends its `tail`, keeping `head`. + // PCG: bb7 -> bb9: Add Edge {b} -> {*b} under conditions bb2 -> bb3, bb6 -> bb7 setting b: e, *b: E + // PCG: bb7 -> bb9: Add Edge {*b} -> {(*b).head, (*b).tail} under conditions bb2 -> bb3 setting *b: None, (*b).head: E, (*b).tail: E + // PCG: bb7 -> bb9: Lend (*b).tail to borrow: _22 before bb8[10]:PreMain = &mut (*b).tail under conditions bb2 -> bb3, bb6 -> bb8 + // PCG: bb8 -> bb9: Lend (*a).tail to borrow: _19 before bb7[4]:PostOperands = &mut (*a).tail under conditions bb2 -> bb3, bb6 -> bb7 while cont { a = a.tail.as_mut().unwrap(); cont = a.len() > 0; diff --git a/test-files/211_cond_drop_join.rs b/test-files/211_cond_drop_join.rs index 5289f67e8..c5acafdea 100644 --- a/test-files/211_cond_drop_join.rs +++ b/test-files/211_cond_drop_join.rs @@ -9,7 +9,638 @@ fn ex1_replace_fst_cond(mut p: Pair, s: String) -> Pair { drop(p.fst); } // PCG: bb4 -> bb5: unpack p with capability E - // PCG: bb4 -> bb5: Weaken(Weaken { place: _1.0, from: E, to: W, for_storage_dead: false, _marker: PhantomData<&()> }) + // PCG: bb4 -> bb5: Weaken p.fst from E to W p.fst = s; p } + +struct Chars<'ch> { + chars: &'ch str, + index: usize, +} + +fn halves(chars: &str) -> Option<&str> { + None +} + +// One branch moves `p` while the other keeps it frozen, with `*p.chars` +// reborrowed into the result. The frozen side gives up the value of `p` +// explicitly; the reborrow refers to the labelled referent. +fn moved_or_frozen<'ch>(p: Chars<'ch>) -> Chars<'ch> { + // PCG: bb5 -> bb6: Label place p (Write) + // PCG: bb5 -> bb6: Unfreeze p + // PCG: bb5 -> bb6: Weaken p from E to W + // PCG: bb6[0] pre_operands: p: W + // PCG: bb6[0] pre_operands: {p.chars after bb5} -> {*p.chars after bb5} under conditions bb1 -> bb3 + match halves(p.chars) { + Some(left) => Chars { chars: left, index: 0 }, + None => p, + } +} + +struct Occupied<'a> { + table: &'a mut Vec, + hash_builder: &'a String, +} + +struct Vacant<'a> { + table: &'a mut Vec, + hash_builder: &'a String, +} + +enum Entry<'a> { + Occupied(Occupied<'a>), + Vacant(Vacant<'a>), +} + +// As above, but the other branch moves the parent `o` as a whole. +fn moved_parent_or_frozen(o: Occupied<'_>, b: bool) -> Entry<'_> { + // PCG: bb2 -> bb3: Label place o.hash_builder (Write) + // PCG: bb2 -> bb3: Unfreeze o.hash_builder + // PCG: bb2 -> bb3: Weaken o.hash_builder from E to W + // PCG: bb3[0] pre_operands: o: W + // PCG: bb3[0] pre_operands: {o.hash_builder after bb2} -> {*o.hash_builder after bb2} under conditions bb0 -> bb2 + if b { + Entry::Occupied(o) + } else { + Entry::Vacant(Vacant { + table: o.table, + hash_builder: o.hash_builder, + }) + } +} + +fn read(_r: &i32) {} + +// One branch reads through `x` (x: R), the other lends its referent (x: e): +// `x` itself is kept by both, so the join holds it with e. +fn read_or_lent(x: &mut i32, c: bool) { + // PCG: bb1 -> bb3: Refreeze x to *x + // PCG: bb3[0] pre_operands: x: e + // ~PCG: bb2 -> bb3: Weaken x from e to W + // PCG: bb4[0] pre_operands: x: E + let r: &i32; + if c { + r = &*x; + } else { + let m = &mut *x; + r = &*m; + } + read(r); + *x = 2; +} + +// One branch moves `x.fst` out, the other lends `x` as a whole: the first +// lends each field with what it holds, so the join records W for `x.fst` and +// E for `x.snd`, and each gets that back. +fn moved_part_or_lent(c: bool, x0: Pair, mut z: Pair) { + // PCG: bb2 -> bb4: Lend x.fst to borrow: _13 before bb3[6]:PreMain = &mut x under conditions bb0 -> bb3 + // PCG: bb2 -> bb4: Lend x.snd to borrow: _13 before bb3[6]:PreMain = &mut x under conditions bb0 -> bb3 + // ~PCG: bb2 -> bb4: Label place x.snd (Detach) + // PCG: bb3 -> bb4: Unpack suspended x to x.fst, x.snd + // PCG: bb3 -> bb4: Weaken suspended x.fst from E to W + // PCG: bb4[0] pre_operands: x.fst: suspended W + // PCG: bb4[0] pre_operands: x.snd: suspended E + // PCG: bb6[1] pre_operands: Remove Edge borrow: _13 before bb3[6]:PreMain = &mut x under conditions bb0 -> bb3 returning W to x.fst, returning E to x.snd + let mut x = x0; + let r: &mut Pair; + if c { + drop(x.fst); + r = &mut z; + } else { + r = &mut x; + } + r.snd = String::new(); +} + +// One branch borrows `x` as a whole, the other only `x.fst`: `x.snd` stays +// read while the borrow of `x` lives. +fn borrowed_whole_or_part(c: bool) { + // ~PCG: bb11[0] pre_operands: Unfreeze x.snd + // PCG: bb11[4] pre_operands: x.snd: R + let mut x = Pair { fst: String::new(), snd: String::new() }; + let r: &String; + if c { + let t = &x; + r = &t.snd; + } else { + r = &x.fst; + x.snd = String::new(); + } + let _ = r.len(); +} + +struct P { f: u32, g: u32 } + +// One branch freezes `x` for `a`, the other lends it to `m`: when `m` ends, +// `x` is still frozen for `a` and gets its E back only from its unfreeze. +fn frozen_or_lent(c: bool, mut x: P, mut y: P) -> u32 { + // PCG: bb3[3] post_main: x: R + // PCG: bb3[3] post_main: x: suspended E + // ~PCG: bb3[3] post_main: x: E + // PCG: bb3[5] pre_operands: Unfreeze x + let a: &u32; + let m: &mut P; + if c { + a = &x.f; + m = &mut y; + } else { + m = &mut x; + a = &y.f; + } + m.g = 1; + let s = *a; + s +} + +// One branch lends `x` whole, the other only `x.f`: when the loan of `x` ends, +// only `x.g` comes back, and `x.f` comes back when its own loan ends. +fn whole_or_part_lent(c: bool, mut x: P, mut z: P, mut w: u32) -> u32 { + // PCG: bb3[3] post_main: x.g: E + // PCG: bb3[3] post_main: x.f: suspended E + // ~PCG: bb3[3] post_main: x: E + // PCG: bb3[5] post_main: x: E + let p: &mut P; + let q: &mut u32; + if c { + p = &mut x; + q = &mut w; + } else { + q = &mut x.f; + p = &mut z; + } + p.g = 1; + x.g = 2; + *q = 3; + w +} + +// One branch lends `x` whole, the other each of its fields: the join records +// the fields, and each comes back when its own loan ends. +fn whole_or_every_part_lent(c: bool, mut x: P, mut z: P, mut w: u32, mut v: u32) -> u32 { + // PCG: bb1 -> bb3: Unpack suspended x to x.f, x.g + // PCG: bb3[3] post_operands: x.f: suspended E + // PCG: bb3[3] post_operands: x.g: suspended E + // PCG: bb3[4] pre_operands: Remove Edge borrow: _18 before bb2[6]:PreMain = &mut x.f under conditions bb0 -> bb2 returning E to x.f + // PCG: bb3[5] pre_operands: Remove Edge borrow: _20 before bb2[13]:PreMain = &mut x.g under conditions bb0 -> bb2 returning E to x.g + let p: &mut P; + let q: &mut u32; + let r: &mut u32; + if c { + p = &mut x; + q = &mut w; + r = &mut v; + } else { + q = &mut x.f; + r = &mut x.g; + p = &mut z; + } + p.g = 1; + *q = 3; + *r = 4; + x.f + w +} + +struct Q { a: u32, b: u32 } +struct N { f: Q, g: u32 } + +// One branch freezes `x` for a borrow of `x.f.a`, the other lends `x` whole: +// when the loan of `x` ends, `x` comes back frozen at every level. +fn nested_frozen_or_lent(c: bool, mut x: N, mut y: N) -> u32 { + // PCG: bb3[3] post_operands: x: R + // PCG: bb3[3] post_operands: x: suspended E + // PCG: bb3[3] post_operands: x.f: R + // PCG: bb3[3] post_operands: x.f.a: R + // PCG: bb3[3] post_operands: x.f.b: R + // PCG: bb3[3] post_operands: x.g: R + // PCG: bb3[3] pre_main: Refreeze x to x.f.a + let a: &u32; + let m: &mut N; + if c { + a = &x.f.a; + m = &mut y; + } else { + m = &mut x; + a = &y.f.a; + } + m.g = 1; + x.g = 5; + x.f.b = 6; + let s = *a; + x.f.a = 7; + s +} + +struct S<'a> { f: &'a mut u32, g: u32 } + +// One branch lends `x` whole, the other the referent of `x.f`: when the loan +// of `x` ends, `x.f` comes back with its referent still lent (e), and the +// referent comes back when its own loan ends. +fn whole_or_referent_lent<'a>(c: bool, mut x: S<'a>, mut z: S<'a>, mut w: u32) -> u32 { + // PCG: bb1 -> bb3: Unpack suspended x to x.f, x.g + // PCG: bb1 -> bb3: Unpack suspended x.f to *x.f + // PCG: bb3[3] post_operands: x.f: e + // PCG: bb3[3] post_operands: x.g: E + // PCG: bb3[3] post_operands: *x.f: suspended E + // PCG: bb3[5] pre_operands: Remove Edge borrow: _14 before bb2[6]:PreMain = &mut *x.f under conditions bb0 -> bb2 returning E to *x.f + let p: &mut S<'a>; + let q: &mut u32; + if c { + p = &mut x; + q = &mut w; + } else { + q = &mut *x.f; + p = &mut z; + } + p.g = 1; + x.g = 2; + *q = 3; + *x.f + w +} + +// As `whole_or_part_lent`, with the lent place behind a reference. +fn borrowed_whole_or_part_lent(c: bool, x: &mut P, mut z: P, mut w: u32) -> u32 { + // PCG: bb1 -> bb3: Unpack suspended *x to (*x).f, (*x).g + // PCG: bb3[3] post_operands: (*x).f: suspended E + // PCG: bb3[3] post_operands: (*x).g: E + // PCG: bb3[5] pre_operands: Remove Edge borrow: _14 before bb2[6]:PreMain = &mut (*x).f under conditions bb0 -> bb2 returning E to (*x).f + let p: &mut P; + let q: &mut u32; + if c { + p = &mut *x; + q = &mut w; + } else { + q = &mut x.f; + p = &mut z; + } + p.g = 1; + x.g = 2; + *q = 3; + x.f + w +} + +enum AB { A(u32, u32), B } + +// One branch lends `x` whole, the other a field of its variant `A`: the join +// records `x` (not every path matched `A`), which comes back whole. +fn whole_or_variant_field_lent(c: bool, mut x: AB, mut z: AB, mut w: u32) -> u32 { + // PCG: bb6 -> bb7: Pack suspended x + // PCG: bb8[5] post_operands: x: suspended E + // PCG: bb8[7] pre_operands: Remove Edge borrow: s before bb6[7]:PreMain = &mut x@A.0 under conditions bb0 -> bb2, bb2 -> bb4 returning E to x + // PCG: bb8[7] post_operands: x: E + let p: &mut AB; + let q: &mut u32; + if c { + p = &mut x; + q = &mut w; + } else { + match x { + AB::A(ref mut s, _) => q = s, + AB::B => q = &mut w, + } + p = &mut z; + } + *p = AB::B; + *q = 3; + match x { AB::A(s, t) => s + t, AB::B => w } +} + +// One branch freezes `x` whole for `a`, the other lends `x.f` to `m`: the +// first moves the freeze to the fields, and when `m` ends `x.f` comes back +// frozen while `a` lives. +fn frozen_whole_or_part_lent(c: bool, mut x: P, mut y: P) -> u32 { + // ~PCG: bb1 -> bb3: Unfreeze x + // PCG: bb1 -> bb3: Lend x.f to borrow: _13 before bb2[6]:PreMain = &mut x.f under conditions bb0 -> bb2 + // PCG: bb3[3] pre_operands: Remove Edge borrow: _13 before bb2[6]:PreMain = &mut x.f under conditions bb0 -> bb2 returning R to x.f + // ~PCG: bb3[3] pre_operands: Unfreeze x.f + // PCG: bb3[3] pre_operands: Pack suspended x + // PCG: bb3[3] post_operands: x: R + // PCG: bb3[3] post_operands: x: suspended E + // PCG: bb3[5] pre_operands: Unfreeze x + let a: &P; + let m: &mut u32; + if c { + a = &x; + m = &mut y.f; + } else { + m = &mut x.f; + a = &y; + } + *m = 1; + let s = a.g; + x.f = 2; + x.g = 3; + s + x.f + x.g +} + +// As `frozen_whole_or_part_lent`, with `x` behind a reference. +fn borrowed_frozen_whole_or_part_lent(c: bool, x: &mut P, y: P, mut w: u32) -> u32 { + // PCG: bb3[0] post_operands: (*x).g: R + // ~PCG: bb3[0] pre_operands: Unfreeze (*x).g + // PCG: bb3[3] pre_operands: Remove Edge borrow: _14 before bb2[6]:PreMain = &mut (*x).f under conditions bb0 -> bb2 returning R to (*x).f + // PCG: bb3[3] pre_operands: Pack suspended *x + // PCG: bb3[3] post_operands: *x: R + // PCG: bb3[3] post_operands: *x: suspended E + // PCG: bb3[5] pre_operands: Unfreeze *x + let a: &P; + let m: &mut u32; + if c { + a = &*x; + m = &mut w; + } else { + m = &mut x.f; + a = &y; + } + *m = 1; + let s = a.g; + x.f = 2; + s + x.f + w +} + +enum AC { A(u32, u32), C(u32) } + +// Each arm lends a field of a different variant: the join records the enum. +fn variant_fields_lent(mut x: AC) -> u32 { + // PCG: bb3 -> bb5: Pack suspended x + // PCG: bb4 -> bb5: Pack suspended x + // PCG: bb5[5] post_operands: x: E + let q: &mut u32 = match x { + AC::A(ref mut s, _) => s, + AC::C(ref mut t) => t, + }; + *q = 3; + match x { + AC::A(s, t) => s + t, + AC::C(t) => t, + } +} + +// One branch freezes `x` whole, the other lends a field of its variant `A`: +// the frozen `x` stays lent to `a`, and comes back frozen while `a` lives. +fn frozen_whole_or_variant_field_lent(c: bool, mut x: AB, z: AB, mut w: u32) -> u32 { + // PCG: bb1 -> bb8: Lend x to borrow: _10 before bb1[6]:PreMain = & x under conditions bb0 -> bb1 + // ~PCG: bb1 -> bb8: Label place x (Detach) + // PCG: bb8[3] pre_operands: Remove Edge borrow: s before bb6[7]:PreMain = &mut x@A.0 under conditions bb0 -> bb2, bb2 -> bb4 returning R to x + // PCG: bb9[1] pre_operands: Unfreeze x + let a: &AB; + let q: &mut u32; + if c { + a = &x; + q = &mut w; + } else { + match x { + AB::A(ref mut s, _) => q = s, + AB::B => q = &mut w, + } + a = &z; + } + *q = 3; + let r = match a { + AB::A(s, _) => *s, + AB::B => 0, + }; + x = AB::B; + r + match x { + AB::A(s, t) => s + t, + AB::B => w, + } +} + +// One arm lends a field of `x`'s variant `A` mutably and the other field +// shared: at the join the shared field stays lent to its own borrow, so when +// the mutable loan ends `x` comes back frozen while the shared one lives. +fn variant_fields_lent_and_read(mut x: AB, mut w: u32) -> u32 { + // PCG: bb4 -> bb5: Lend x@A.1 to borrow: b before bb4[13]:PreMain = & x@A.1 under conditions bb0 -> bb2 + // ~PCG: bb4 -> bb5: Label place x@A.1 (Detach) + // PCG: bb5[2] post_operands: x: R + // PCG: bb5[2] post_operands: x: suspended E + // PCG: bb5[2] post_operands: x@A.0: R + // PCG: bb5[2] post_operands: x@A.1: R + let q: &mut u32; + let r: &u32; + let v = 5; + match x { + AB::A(ref mut a, ref b) => { + q = a; + r = b; + } + AB::B => { + q = &mut w; + r = &v; + } + } + *q = 3; + let t = match x { + AB::A(a, _) => a, + AB::B => 0, + }; + t + *r +} + +// As `variant_fields_lent_and_read`, with `x` behind a reference. +fn borrowed_variant_fields_lent_and_read(x: &mut AB, mut w: u32) -> u32 { + // PCG: bb4 -> bb5: Lend (*x)@A.1 to borrow: b before bb4[13]:PreMain = & (*x)@A.1 under conditions bb0 -> bb2 + // ~PCG: bb5[0] post_operands: *x: R + // PCG: bb5[2] post_operands: *x: R + // PCG: bb5[2] post_operands: *x: suspended E + // PCG: bb5[2] post_operands: (*x)@A.0: R + let q: &mut u32; + let r: &u32; + let v = 5; + match *x { + AB::A(ref mut a, ref b) => { + q = a; + r = b; + } + AB::B => { + q = &mut w; + r = &v; + } + } + *q = 3; + let t = match *x { + AB::A(a, _) => a, + AB::B => 0, + }; + t + *r +} + +// One branch lends `x` whole, the other reads the referent of `x.f`: when the +// loan of `x` ends, `x` comes back frozen down to the referent. +fn whole_lent_or_referent_read<'a>(c: bool, mut x: S<'a>, mut z: S<'a>, w: u32) -> u32 { + // PCG: bb3[3] post_operands: x: R + // PCG: bb3[3] post_operands: x: suspended E + // PCG: bb3[3] post_operands: x.f: R + // PCG: bb3[3] post_operands: *x.f: R + // PCG: bb3[3] pre_main: Refreeze x to *x.f + let p: &mut S<'a>; + let a: &u32; + if c { + p = &mut x; + a = &w; + } else { + a = &*x.f; + p = &mut z; + } + p.g = 1; + x.g = 2; + let s = *a; + *x.f = 3; + s + x.g +} + +// One branch moves `x.fst` out, the other lends `x` whole; `x.snd` is read +// after the loan: it comes back E. +fn moved_part_or_lent_then_read(c: bool, mut x: Pair, mut z: Pair) -> usize { + // PCG: bb3 -> bb4: Lend x.fst to borrow: _8 before bb1[6]:PreMain = &mut x under conditions bb0 -> bb1 + // PCG: bb3 -> bb4: Lend x.snd to borrow: _8 before bb1[6]:PreMain = &mut x under conditions bb0 -> bb1 + // PCG: bb5[0] post_operands: x.fst: W + // PCG: bb5[0] post_operands: x.snd: E + let p: &mut Pair; + if c { + p = &mut x; + } else { + drop(x.fst); + p = &mut z; + } + p.snd.push('a'); + x.fst = String::new(); + x.fst.len() + x.snd.len() +} + +enum AS { A(u32, String), B } + +// One arm lends a field of `x`'s variant `A` and moves the other: the moved +// field goes with the loan, so `x` comes back W. +fn variant_field_lent_other_moved(mut x: AS, mut w: u32) -> u32 { + // PCG: bb6 -> bb7: Lend x@A.1 to borrow: s before bb6[1]:PreMain = &mut x@A.0 under conditions bb0 -> bb2 + // PCG: bb3 -> bb7: Weaken suspended x from E to W + // PCG: bb7[2] post_operands: x: W + let q: &mut u32; + match x { + AS::A(ref mut s, t) => { + q = s; + drop(t); + } + AS::B => q = &mut w, + } + *q = 3; + x = AS::B; + w +} + +// One branch freezes `x` whole, the other lends the referent of `x.f`: `x.f` +// holds nothing at the join (neither its R under the live freeze nor e), and +// `x` comes back E once both loans end. +fn frozen_or_referent_lent<'a>(c: bool, mut x: S<'a>, z: S<'a>, mut w: u32) -> u32 { + // ~PCG: bb3[0] post_operands: x.f: e + // PCG: bb3[3] post_operands: x: R + // PCG: bb3[3] post_operands: x: suspended E + // PCG: bb3[5] pre_operands: Unfreeze x + // PCG: bb3[6] post_main: x.f: E + // PCG: bb3[6] post_main: x.g: E + let a: &S<'a>; + let m: &mut u32; + if c { + a = &x; + m = &mut w; + } else { + m = &mut *x.f; + a = &z; + } + *m = 1; + let s = a.g; + x.g = 2; + *x.f = 3; + s +} + +// The branches unpack `x` at different indices: the join records `x`, which +// comes back whole. +fn array_unpacked_two_ways(c: bool, mut x: [u32; 4], i: usize, w: u32) -> u32 { + // PCG: bb1 -> bb4: Pack suspended x + // PCG: bb3 -> bb4: Pack suspended x + // PCG: bb4[3] post_operands: x: E + let q: &mut u32; + if c { + let [ref mut a, ..] = x; + q = a; + } else { + q = &mut x[i]; + } + *q = 1; + x[0] + w +} + +// As `borrowed_whole_or_part_lent`, with the part's loan ending first: the +// parts' records pack into `*x` when its unpacking goes, and `*x` comes back +// E when the loan of `*x` ends. +fn borrowed_whole_or_part_lent_then_read(c: bool, x: &mut P, mut z: P, mut w: u32) -> u32 { + // PCG: bb3[3] pre_operands: Pack suspended *x + // PCG: bb3[3] post_operands: *x: suspended E + // PCG: bb3[4] pre_operands: Remove Edge borrow: _10 before bb1[6]:PreMain = &mut *x under conditions bb0 -> bb1 returning E to *x + // PCG: bb3[4] post_operands: x: E + let p: &mut P; + let q: &mut u32; + if c { + p = &mut *x; + q = &mut w; + } else { + q = &mut x.f; + p = &mut z; + } + *q = 3; + p.g = 1; + let r: &P = &*x; + r.f + w +} + +// A variant field is borrowed on one path only: the join holds `x` frozen and +// packed, with the layout of the borrowed field below it. Borrowing the other +// variant mutably (allowed: variants are disjoint) moves the freeze to `x@A` +// without entering it, and `x@C` gets E. +fn variant_field_read_other_variant_borrowed(mut x: AC) -> u32 { + // PCG: bb3 -> bb4: Freeze x + // PCG: bb6[1] pre_operands: unpack x@C with capability R + // PCG: bb6[1] pre_operands: Refreeze x to x@A + let y = 0; + let mut a: &u32 = &y; + if let AC::A(ref p, _) = x { + a = p; + } + if let AC::C(ref mut q) = x { + *q = 1; + } + *a +} + +// As above, re-matching the same variant: the freeze moves to `x@A.0` and +// `x@A.1` gets E. +fn variant_field_read_same_variant_borrowed(mut x: AC) -> u32 { + // PCG: bb6[1] pre_operands: unpack x@A with capability R + // PCG: bb6[1] pre_operands: Refreeze x to x@A.0 + let y = 0; + let mut a: &u32 = &y; + if let AC::A(ref p, _) = x { + a = p; + } + if let AC::A(_, ref mut q) = x { + *q = 1; + } + *a +} + +// As above, borrowing `x` whole for read: when that borrow ends `x` stays +// frozen, since the loan of `x@A.0` still lives. +fn variant_field_read_whole_read(x: AC) -> u32 { + // PCG: bb4[5] pre_operands: Remove Edge borrow: r = & x + // ~PCG: bb4[5] pre_operands: Unfreeze x + let y = 0; + let mut a: &u32 = &y; + if let AC::A(ref p, _) = x { + a = p; + } + let r = &x; + let _ = r; + *a +} diff --git a/test-files/214_guide.rs b/test-files/214_guide.rs index b2603fd5d..8c79c00f8 100644 --- a/test-files/214_guide.rs +++ b/test-files/214_guide.rs @@ -2,5 +2,17 @@ fn test() { let mut x = [13, 37, 72]; let y: &mut [i32] = &mut x; y[1] = 3; - // PCG: bb1[1] pre_operands: Remove Edge {*y} -> {(*y)[_5]} (guide=Index(_5)) + // PCG: bb1[1] pre_operands: Remove Edge {*y} -> {(*y)[_5]} (guide=Index(_5)) returning E to *y +} + +// `d` is read-unpacked at two indices while frozen: each collapse packs its +// own index at R, and `d` gets its E back only from its unfreeze. +fn read_at_two_indices(i: usize, j: usize) -> u32 { + // PCG: bb2[8] pre_operands: Unfreeze d + let mut d = [0u32; 4]; + let a = &d[i]; + let b = &d[j]; + let s = *a + *b; + d[0] = s; + d[0] } diff --git a/test-files/218_storagedead_weaken.rs b/test-files/218_storagedead_weaken.rs index 3d0ce4683..3f15b189e 100644 --- a/test-files/218_storagedead_weaken.rs +++ b/test-files/218_storagedead_weaken.rs @@ -2,6 +2,6 @@ fn bar() -> *mut i32 { let mut x = 5; &raw mut x // Before the storagedead, there should be a weaken from E to W - // PCG: bb0[4] pre_main: Weaken(Weaken { place: _1, from: E, to: W, for_storage_dead: true, _marker: PhantomData<&()> }) + // PCG: bb0[4] pre_main: Weaken x from E to W // PCG: bb0[4] pre_main: Label place x (StorageDead) } diff --git a/test-files/219_twophase.rs b/test-files/219_twophase.rs index 8d6b2069d..a67bdf5a6 100644 --- a/test-files/219_twophase.rs +++ b/test-files/219_twophase.rs @@ -13,6 +13,175 @@ impl Tmp { fn foo() { let mut x = Tmp {}; x.write(x.read()); - // PCG: bb1[1] pre_operands: Weaken x from R to None - // `x`'s capability should be removed when the two-phase borrow is activated + // The reservation freezes `x`; the activation lends the frozen capability. + // PCG: bb0[5] post_operands: Freeze x + // PCG: bb1[1] pre_operands: Activate two-phase borrow borrow: _3 = &mut x + // PCG: bb2[0] pre_operands: Remove Edge borrow: _3 before bb1[1]:PostOperands = &mut x returning E to x + // ~PCG: bb1[1] pre_operands: Weaken x from R to None } + +struct V { + n: usize, + m: usize, +} + +impl V { + fn set(&mut self, x: usize) { + self.m = x; + } +} + +struct W { + v: V, + k: usize, +} + +fn field_read(mut v: V) -> V { + v.set(v.n); + // The read unpacks the frozen `v` at R, and is packed before activation. + // PCG: bb0[2] post_operands: Freeze v + // PCG: bb0[4] pre_operands: unpack v with capability R + // PCG: bb0[5] pre_operands: Collapse(RepackCollapse { to: _1, capability: R, guide: None, _marker: PhantomData<&()> }) + // PCG: bb0[5] pre_operands: Activate two-phase borrow borrow: _3 = &mut v + // PCG: bb1[0] pre_operands: Remove Edge borrow: _3 before bb0[5]:PostOperands = &mut v returning E to v + v +} + +fn field_read_borrowed(v: &mut V) { + v.set(v.n); + // PCG: bb0[2] post_operands: Freeze *v + // PCG: bb0[5] pre_operands: Activate two-phase borrow borrow: _3 = &mut *v + // PCG: bb1[0] pre_operands: Remove Edge borrow: _3 before bb0[5]:PostOperands = &mut *v returning E to *v +} + +fn under_mut_ref(w: &mut W) { + w.v.set(w.k); + // The reservation unpacks `*w` exclusively, so `(*w).k` keeps E. + // PCG: bb0[2] post_operands: Freeze (*w).v + // PCG: bb0[2] post_operands: (*w).k: E + // PCG: bb0[5] pre_operands: Activate two-phase borrow borrow: _3 = &mut (*w).v + // PCG: bb1[0] pre_operands: Remove Edge borrow: _3 before bb0[5]:PostOperands = &mut (*w).v returning E to (*w).v +} + +impl W { + fn total(&self) -> usize { + self.k + } +} + +fn under_shared_borrow(mut w: W) -> W { + let r = &w; + w.v.set(r.k); + // The reservation is made under the freeze of `w`, which moves to `w.v` + // once `r` is gone. + // PCG: bb0[1] post_operands: Freeze w + // PCG: bb0[8] pre_operands: Refreeze w to w.v + // PCG: bb0[8] pre_operands: Activate two-phase borrow borrow: _4 = &mut w.v + // ~PCG: bb0[8] pre_operands: Weaken w from R to None + // ~PCG: bb1[0] pre_operands: Unfreeze w.k + w +} + +fn ancestor_read(mut w: W) -> W { + w.v.set(w.total()); + // Reading `w` during the reservation freezes all of it; the freeze moves + // back to `w.v` at the activation. + // PCG: bb0[2] post_operands: Freeze w.v + // PCG: bb0[5] pre_operands: Freeze w + // PCG: bb1[1] pre_operands: Refreeze w to w.v + // PCG: bb1[1] pre_operands: Activate two-phase borrow borrow: _3 = &mut w.v + w +} + +struct Bytes { + pos: usize, +} + +impl Bytes { + fn advance(&mut self, n: usize) { + self.pos += n; + } +} + +impl AsRef for Bytes { + fn as_ref(&self) -> &usize { + &self.pos + } +} + +fn reservation_reads_reference(bytes: &mut Bytes) { + // `bytes.as_ref()` borrows the reference `bytes` itself (through the + // `AsRef` impl for `&mut Bytes`) while `*bytes` is reserved: `bytes`, + // whose referent is unpacked (e), is frozen. + // PCG: bb0[2] post_operands: Freeze *bytes + // PCG: bb0[6] post_operands: Freeze bytes + // PCG: bb1[2] pre_operands: Refreeze bytes to *bytes + // PCG: bb1[2] pre_operands: Activate two-phase borrow borrow: _3 = &mut *bytes + bytes.advance(*bytes.as_ref()); +} + +pub struct Header<'a> { + pub name: &'a str, +} + +pub struct Request<'h, 'b> { + pub method: Option<&'b str>, + pub headers: &'h mut [Header<'b>], +} + +impl<'h, 'b> Request<'h, 'b> { + fn parse_uninit( + &mut self, + buf: &'b [u8], + headers: &'h mut [std::mem::MaybeUninit>], + ) -> Result { + let _ = (buf, headers); + Err(()) + } + + // The loan of `(*self).headers` taken by `mem::take` ends although the raw + // pointer `headers` aliases its result, so activating the two-phase + // borrow of `*self` detaches nothing. + pub fn parse(&mut self, buf: &'b [u8]) -> Result { + // PCG: bb1[0] pre_operands: Remove Edge borrow: _5 = &mut (*self).headers returning E to (*self).headers + // ~PCG: bb1[26] pre_operands: Label place (*self).method (Detach) + // PCG: bb1[26] pre_operands: Activate two-phase borrow borrow: _11 = &mut *self + let headers = std::mem::take(&mut self.headers); + unsafe { + let headers: *mut [Header<'_>] = headers; + let headers = headers as *mut [std::mem::MaybeUninit>]; + match self.parse_uninit(buf, &mut *headers) { + Ok(idx) => Ok(idx), + other => { + self.headers = &mut *(headers as *mut [Header<'_>]); + other + } + } + } + } +} + +struct Tree<'u> { + unit: &'u u32, + depth: u32, +} + +impl Tree<'_> { + fn set(&mut self, depth: u32) { + self.depth = depth; + } + + // `*self` is unpacked when the reservation freezes it: it suspends what + // its fields hold (E), not the R of the referent of `(*self).unit`, which + // is behind a shared reference. The activation lends that E. + fn reset(&mut self) { + // PCG: bb0[6] post_operands: Freeze *self + // PCG: bb0[6] post_operands: *self: suspended E + // PCG: bb0[9] pre_operands: Activate two-phase borrow borrow: _4 = &mut *self + let unit = &*self.unit; + self.depth = 0; + self.set(*unit); + } +} + +fn main() {} diff --git a/test-files/228_issue_133_borrow_expiry.rs b/test-files/228_issue_133_borrow_expiry.rs index 54be88be0..0a4eb0255 100644 --- a/test-files/228_issue_133_borrow_expiry.rs +++ b/test-files/228_issue_133_borrow_expiry.rs @@ -29,9 +29,9 @@ pub fn whole_local_borrow_expiry(mut a: ListNode) { let x = &mut a; use_list_node(x); consume_u(a.value); - // PCG: bb1[0] pre_operands: Restore capability E to a + // PCG: bb1[0] pre_operands: Remove Edge borrow: x = &mut a returning E to a // PCG: bb1[0] post_operands: a: E - // ~PCG: bb1[0] pre_operands: Restore capability W to a + // ~PCG: bb1[0] pre_operands: Remove Edge borrow: x = &mut a returning W to a // ~PCG: bb1[0] post_operands: a: W } @@ -46,13 +46,15 @@ pub fn conditional_reborrow_expiry(mut a: ListNode) { // The borrow of `a` was split into two conditional edge variants (one // per match arm); on expiry it must be removed under the union of their // conditions, not just one branch's. - // PCG: bb6[0] pre_operands: Remove Edge borrow: x = &mut a under conditions bb0 -> { bb3, bb2 } - // PCG: bb6[0] pre_operands: Restore capability E to a + // PCG: bb6[0] pre_operands: Remove Edge borrow: x = &mut a under conditions bb0 -> { bb3, bb2 } returning E to a // PCG: bb6[0] post_operands: a: E - // ~PCG: bb6[0] pre_operands: Restore capability W to a + // ~PCG: bb6[0] pre_operands: Remove Edge borrow: x = &mut a under conditions bb0 -> { bb3, bb2 } returning W to a // ~PCG: bb6[0] post_operands: a: W // ~PCG: bb6[0] pre_operands: Remove Edge borrow: x = &mut a under conditions bb0 -> bb3 // ~PCG: bb6[0] pre_operands: Remove Edge borrow: x = &mut a under conditions bb0 -> bb2 + // At the join, `(*x).value` is lent to the reborrow of `*x` made in the + // other arm. + // PCG: bb4 -> bb5: Lend (*x).value to borrow: x = &mut *x under conditions bb0 -> bb3 } fn main() {} diff --git a/test-files/229_join_variant_collapse.rs b/test-files/229_join_variant_collapse.rs index 029875c87..7541a44a7 100644 --- a/test-files/229_join_variant_collapse.rs +++ b/test-files/229_join_variant_collapse.rs @@ -14,12 +14,28 @@ enum E { // the local; the local stays live so nothing collapses it earlier. fn repro(mut x: E, y: E) { if let E::V(B { f: z, .. }) = x {} - // PCG: bb2 -> bb4: Weaken(Weaken { place: (_1@V).0.1, from: E, to: W, for_storage_dead: false, _marker: PhantomData<&()> }) + // PCG: bb2 -> bb4: Weaken x@V.0.g from E to W // PCG: bb2 -> bb4: Collapse(RepackCollapse { to: (_1@V).0, capability: W, guide: None, _marker: PhantomData<&()> }) // PCG: bb2 -> bb4: Collapse(RepackCollapse { to: (_1@V), capability: W, guide: None, _marker: PhantomData<&()> }) // PCG: bb2 -> bb4: Collapse(RepackCollapse { to: _1, capability: W, guide: Some(Downcast(Some("V"), 0)), _marker: PhantomData<&()> }) - // PCG: bb3 -> bb4: Weaken(Weaken { place: _1, from: E, to: W, for_storage_dead: false, _marker: PhantomData<&()> }) + // PCG: bb3 -> bb4: Weaken x from E to W x = y; } +// The `None` path did not match `Some`: it lends `x` itself to the borrow of +// the `Some` field instead of unpacking `x` to that variant. +fn lent_variant_field(mut x: Option) { + // PCG: bb3 -> bb5: Lend x to borrow: v before bb4[7]:PreMain = &mut x@Some.0 under conditions bb0 -> bb2 + // ~PCG: bb3 -> bb5: unpack x@Some with capability E + let r: &mut i32; + let mut y = 0; + match x { + Some(ref mut v) => r = v, + None => r = &mut y, + } + *r = 1; + x = None; + let _ = x; +} + fn main() {} diff --git a/test-files/230_join_validity_conditions.rs b/test-files/230_join_validity_conditions.rs index 3c08f516d..7789eae3e 100644 --- a/test-files/230_join_validity_conditions.rs +++ b/test-files/230_join_validity_conditions.rs @@ -12,10 +12,10 @@ fn from_nth(r: &mut List, n: usize) -> &mut List { // bb9 joins the `then` path (bb0 -> bb1) with the `else` path (bb0 -> bb2, // bb2 -> bb5, ...). The `then` path says nothing about bb2's branches, so // the expansions performed at bb9 are not conditional on bb2 -> bb5. - // PCG: bb9[0] pre_operands: Add Edge {_4} -> {(*_4)} - // PCG: bb9[2] pre_operands: Add Edge {_3} -> {(*_3)} - // ~PCG: bb9[0] pre_operands: Add Edge {_4} -> {(*_4)} under conditions bb2 -> bb5 - // ~PCG: bb9[2] pre_operands: Add Edge {_3} -> {(*_3)} under conditions bb2 -> bb5 + // PCG: bb9[0] pre_operands: Add Edge {_4} -> {(*_4)} setting _4: e, (*_4): E + // PCG: bb9[2] pre_operands: Add Edge {_3} -> {(*_3)} setting _3: e, (*_3): E + // ~PCG: bb9[0] pre_operands: Add Edge {_4} -> {(*_4)} under conditions bb2 -> bb5 setting _4: e, (*_4): E + // ~PCG: bb9[2] pre_operands: Add Edge {_3} -> {(*_3)} under conditions bb2 -> bb5 setting _3: e, (*_3): E if n == 0 { r } else { @@ -26,4 +26,20 @@ fn from_nth(r: &mut List, n: usize) -> &mut List { } } +// The `None`-like path lends `x` to the borrow made after `bb0` chose the +// other branch; the lent edge does not carry that choice into this path. +fn lend_without_branch_choice(c: bool, d: bool) { + // PCG: bb0 -> bb3: Lend x to borrow: _11 before bb1[7]:PreMain = &mut x + let mut x = (1, 2); + let mut z = (3, 4); + let mut r: &mut (i32, i32) = &mut z; + if c && { + r = &mut x; + d + } { + return; + } + *r = (0, 0); +} + fn main() {} diff --git a/test-files/231_loop_invariant_unpack.rs b/test-files/231_loop_invariant_unpack.rs index 5b6286f53..6535678ca 100644 --- a/test-files/231_loop_invariant_unpack.rs +++ b/test-files/231_loop_invariant_unpack.rs @@ -25,6 +25,10 @@ fn write_field_in_loop(mut pair: Pair) { // A field that is only read in the loop is unpacked for `Read`, while the field // that is written to keeps exclusive capability. fn read_and_write_fields_in_loop(mut pair: Pair, other: &Pair) { + // The back edge freezes the whole of `pair` for the loop head's read + // expansion. + // PCG: bb8 -> bb1: Freeze pair + // ~PCG: bb8 -> bb1: Freeze pair.fst let mut cond = false; while !cond { pair.fst = other.snd.clone(); diff --git a/test-files/232_lazy_pack.rs b/test-files/232_lazy_pack.rs index 4fb87af9b..723fdb67c 100644 --- a/test-files/232_lazy_pack.rs +++ b/test-files/232_lazy_pack.rs @@ -18,7 +18,7 @@ fn read_fields_then_overwrite(p: &mut Point) { // Overwriting `*p` requires a capability that `*p` cannot hold while it is // expanded, so the expansion is packed just in time. // PCG: bb0[8] pre_main: Remove Edge {*p} -> {(*p).x, (*p).y} - // PCG: bb0[8] pre_main: Restore *p to E + // PCG: bb0[8] pre_main: Unfreeze p *p = Point { x: 1, y: 2 }; } diff --git a/test-files/234_read_obtain_blocked_field.rs b/test-files/234_read_obtain_blocked_field.rs index d39d75ba2..6d702b7b3 100644 --- a/test-files/234_read_obtain_blocked_field.rs +++ b/test-files/234_read_obtain_blocked_field.rs @@ -1,6 +1,8 @@ //! Regression test: to obtain read capability for `f`, the PCG must not //! collapse `f`, because `f.x` is blocked by a shared borrow. Instead, the -//! expansion of `f` is downgraded to read capability. +//! unpacked `f` is frozen, which makes its fields read copies. The read +//! unpacking of a place that must become exclusive is packed, detaching the +//! borrowed referents of shared references in it. struct Foo { x: u32, @@ -10,9 +12,12 @@ struct Foo { fn client() { let mut f = Foo { x: 0, y: false }; let x_ref = &f.x; + // Writing `f.y` moves the freeze of `f` to the borrowed `f.x`. + // PCG: bb0[4] pre_operands: Freeze f + // PCG: bb0[6] pre_main: Refreeze f to f.x f.y = false; - // PCG: bb0[8] pre_operands: Weaken f.y from E to R - // PCG: bb0[8] pre_operands: Restore capability R to f + // PCG: bb0[8] pre_operands: Freeze f + // ~PCG: bb0[8] pre_operands: Weaken f.y from E to R // PCG: bb0[8] post_operands: f: R // PCG: bb0[8] post_operands: f.x: R // PCG: bb0[8] post_operands: f.y: R @@ -24,11 +29,12 @@ fn client() { /// The same situation for a place reached through a mutable reference. Here /// the expansion of `*p` lives in the borrow PCG rather than the owned PCG, /// and therefore a different mechanism keeps it in place: the expansion edge -/// is not a leaf while `(*p).x` is blocked, and re-expanding `*p` for read -/// downgrades `(*p).y`. +/// is not a leaf while `(*p).x` is blocked, and `*p` is frozen in place. fn client_reborrow(p: &mut Foo) { let x_ref = &p.x; p.y = false; + // PCG: bb0[3] pre_main: Refreeze p to (*p).x + // PCG: bb0[5] pre_operands: Freeze *p // PCG: bb0[5] post_operands: {*p} -> {(*p).x, (*p).y} // PCG: bb0[5] post_operands: *p: R // PCG: bb0[5] post_operands: (*p).x: R @@ -37,4 +43,112 @@ fn client_reborrow(p: &mut Foo) { let j = *x_ref; } +/// Packing the read of `f.x` must not unfreeze `f` while `f_ref` is live. +fn read_while_shared(mut f: Foo) -> u32 { + let f_ref = &f; + let k = f.x; + let j = f_ref.x; + f.x = 3; + // PCG: bb0[1] post_operands: Freeze f + // PCG: bb0[5] pre_operands: Collapse(RepackCollapse { to: _1, capability: R, guide: None, _marker: PhantomData<&()> }) + // PCG: bb0[5] post_operands: f: R + // ~PCG: bb0[5] pre_operands: Unfreeze f + // PCG: bb0[8] pre_operands: Unfreeze f + k + j +} + +// A borrow of the referent of a shared reference does not depend on the place +// holding the reference: when that place must become exclusive, the referent +// is labelled and detached, and the read unpacking is packed. +struct Dfa { + classes: Vec, + inner: &'static Dfa, +} + +struct Lazy<'a> { + dfa: &'a Dfa, + cache: Vec, +} + +impl<'a> Lazy<'a> { + fn set(&mut self, x: u8) { + self.cache.push(x); + } + + /// The loop head holds `*self` with E while the iterator still borrows + /// `classes`: entering the loop detaches the referent of `self.dfa`. + fn loop_entry(&mut self) { + // PCG: bb3 -> bb4: Label place *(*self).dfa (Detach) + // PCG: bb3 -> bb4: Remove Edge {(*self).dfa} -> {*(*self).dfa loop bb4} + // PCG: bb3 -> bb4: Remove Edge {*self} -> {(*self).dfa, (*self).cache} + // PCG: bb3 -> bb4: Unfreeze self + // ~PCG: bb3 -> bb4: Weaken (*(*self).dfa).classes from R to None + // PCG: bb4[0] pre_operands: Remove Edge {self} -> {*self} returning E to self + // PCG: bb7[0] pre_operands: Remove Edge borrow: _6 before bb0[6]:PostOperands = & (*(*self).dfa).classes loop bb4 under conditions bb6 -> bb7 + for c in self.dfa.classes.iter() { + self.set(*c); + } + } + + /// The two-phase borrow of `*self` is activated while `it` borrows + /// `classes`. + fn activation(&mut self) { + // PCG: bb2[6] pre_operands: Refreeze self to *self + // PCG: bb2[6] pre_operands: Label place *(*self).dfa (Detach) + // PCG: bb2[6] pre_operands: Remove Edge {(*self).dfa} -> {*(*self).dfa before bb2[6]:PreOperands} + // PCG: bb2[6] pre_operands: Remove Edge {*self} -> {(*self).dfa, (*self).cache} + // PCG: bb2[6] pre_operands: Activate two-phase borrow borrow: _7 = &mut *self + // ~PCG: bb2[6] pre_operands: Weaken *(*self).dfa from R to None + // PCG: bb4[0] pre_operands: Remove Edge borrow: _5 before bb0[5]:PostOperands = & (*(*self).dfa).classes before bb2[6]:PreOperands + let mut it = self.dfa.classes.iter(); + self.set(0); + it.next(); + } + + /// `*self` is reborrowed mutably while `c` borrows `classes`. + fn reborrow(&mut self) -> u8 { + // PCG: bb1[4] pre_operands: Label place *(*self).dfa (Detach) + // PCG: bb1[4] pre_operands: Remove Edge {(*self).dfa} -> {*(*self).dfa before bb1[4]:PreOperands} + // PCG: bb1[4] pre_operands: Remove Edge {*self} -> {(*self).dfa, (*self).cache} + // PCG: bb1[4] pre_operands: Unfreeze self + let c = &self.dfa.classes[0]; + let r = &mut *self; + r.cache.push(1); + *c + } + + /// Only the outermost borrowed referent is detached. + fn nested(&mut self) -> u8 { + // PCG: bb1[4] pre_operands: Label place *(*self).dfa (Detach) + // ~PCG: bb1[4] pre_operands: Label place *(*(*self).dfa).inner (Detach) + // PCG: bb1[4] pre_operands: Unfreeze self + let c = &self.dfa.inner.classes[0]; + let r = &mut *self; + r.cache.push(1); + *c + } +} + +/// The upgraded place is the frozen reference itself: packing its read +/// unpacking ends its freeze, once. +fn frozen_root<'a>(mut x: &mut Lazy<'a>) -> u8 { + // PCG: bb1[4] pre_operands: Label place *(*x).dfa (Detach) + // PCG: bb1[4] pre_operands: Remove Edge {x} -> {*x} + // PCG: bb1[4] pre_operands: Unfreeze x + let c = &x.dfa.classes[0]; + let r = &mut x; + r.cache.push(1); + *c +} + +fn frozen_root_in_loop<'a>(mut y: &mut Lazy<'a>) { + // PCG: bb3 -> bb4: Label place *(*y).dfa (Detach) + // PCG: bb3 -> bb4: Remove Edge {y} -> {*y} + // PCG: bb3 -> bb4: Unfreeze y + for c in y.dfa.classes.iter() { + let r = &mut y; + r.cache.push(*c); + } +} + fn main() {} diff --git a/test-files/236_read_loop_capabilities.rs b/test-files/236_read_loop_capabilities.rs index 48a7a7a54..622f90792 100644 --- a/test-files/236_read_loop_capabilities.rs +++ b/test-files/236_read_loop_capabilities.rs @@ -51,4 +51,28 @@ fn nested(mut value: i32, outer: bool, inner: bool) -> i32 { value } +struct Stack { + entries: Vec, + depth: usize, +} + +fn same(a: &u32, b: &u32) -> Result { + Ok(a == b) +} + +impl Stack { + // Lending the referent `*self` to the loop leaves what `self` itself + // suspends unchanged. + fn check(&self, child: u32) -> Result<(), usize> { + // PCG: bb4 -> bb5: Lend *self to Loop(bb5): self -> (*self).depth + // PCG: bb5[0] pre_operands: self: suspended E + for entry in self.entries.iter().rev() { + if same(entry, &child).map_err(|_| self.depth)? { + return Err(self.depth); + } + } + Ok(()) + } +} + fn main() {} diff --git a/test-files/237_loop_invariant_capability_merge.rs b/test-files/237_loop_invariant_capability_merge.rs index 127271039..1c33b5f67 100644 --- a/test-files/237_loop_invariant_capability_merge.rs +++ b/test-files/237_loop_invariant_capability_merge.rs @@ -52,4 +52,58 @@ fn write_only(mut a: H, cont: bool) -> H { a } +fn consume(_s: String) {} + +fn moved_in_loop(stop: bool) { + // `s` is not definitely initialized at the loop head (bb2): the back edge + // arrives with it moved out, so the loop head holds it with W and the + // entry weakens it. + // PCG: bb1 -> bb2: Weaken s from E to W + // PCG: bb2[0] pre_operands: s: W + let mut s = String::new(); + loop { + s = String::from("x"); + consume(s); + if stop { + break; + } + } +} + +// `x` is moved out on the back edge and frozen only on entry: the loop head +// holds it with W. +fn moved_in_loop_frozen_on_entry(c: bool) { + // PCG: bb2 -> bb3: Weaken x from E to W + // PCG: bb3[0] pre_operands: x: W + let mut x = String::new(); + let y = String::new(); + let mut r = &x; + loop { + let _ = r.len(); + r = &y; + x = String::new(); + drop(x); + if c { + break; + } + } +} + +// `r` is moved out on the back edge while its referent is lent on entry. +fn moved_in_loop_lent_on_entry(x: &mut i32, y: &mut i32, c: bool) { + // PCG: bb0 -> bb1: Weaken r from e to W + // PCG: bb1[0] pre_operands: r: W + let mut r = &mut *x; + let mut rr = &mut *r; + loop { + *rr += 1; + rr = &mut *y; + r = &mut *x; + let _r2 = r; + if c { + break; + } + } +} + fn main() {} diff --git a/test-files/238_loop_successor_expansions.rs b/test-files/238_loop_successor_expansions.rs index daf008de5..6a0b1df1d 100644 --- a/test-files/238_loop_successor_expansions.rs +++ b/test-files/238_loop_successor_expansions.rs @@ -17,10 +17,10 @@ struct RefPair<'a> { fn borrowed_field(pair: &mut Pair, run: bool) { // PCG: bb1[0] pre_operands: pair: R - // PCG: bb0 -> bb1: Add Edge {pair} -> {*pair} - // PCG: bb0 -> bb1: Add Edge {*pair} -> {(*pair).first, (*pair).second} - // PCG: bb4 -> bb1: Add Edge {pair} -> {*pair} - // PCG: bb4 -> bb1: Add Edge {*pair} -> {(*pair).first, (*pair).second} + // PCG: bb0 -> bb1: Add Edge {pair} -> {*pair} setting pair: R, *pair: R + // PCG: bb0 -> bb1: Add Edge {*pair} -> {(*pair).first, (*pair).second} setting *pair: R, (*pair).first: R, (*pair).second: R + // PCG: bb4 -> bb1: Add Edge {pair} -> {*pair} setting pair: R, *pair: R + // PCG: bb4 -> bb1: Add Edge {*pair} -> {(*pair).first, (*pair).second} setting *pair: R, (*pair).first: R, (*pair).second: R while run { std::hint::black_box(&pair.first); } @@ -30,10 +30,10 @@ fn conditional_entry(pair: &mut Pair, enter: bool, run: bool) { // The entry actions omit the branch choice being taken. The backedge // retains that outer choice, but does not depend on choices inside the loop. // PCG: bb1[0] pre_operands: pair: R - // PCG: bb0 -> bb1: Add Edge {pair} -> {*pair} - // PCG: bb0 -> bb1: Add Edge {*pair} -> {(*pair).first, (*pair).second} - // PCG: bb4 -> bb1: Add Edge {pair} -> {*pair} under conditions bb0 -> bb1 - // PCG: bb4 -> bb1: Add Edge {*pair} -> {(*pair).first, (*pair).second} under conditions bb0 -> bb1 + // PCG: bb0 -> bb1: Add Edge {pair} -> {*pair} setting pair: R, *pair: R + // PCG: bb0 -> bb1: Add Edge {*pair} -> {(*pair).first, (*pair).second} setting *pair: R, (*pair).first: R, (*pair).second: R + // PCG: bb4 -> bb1: Add Edge {pair} -> {*pair} under conditions bb0 -> bb1 setting pair: R, *pair: R + // PCG: bb4 -> bb1: Add Edge {*pair} -> {(*pair).first, (*pair).second} under conditions bb0 -> bb1 setting *pair: R, (*pair).first: R, (*pair).second: R // Borrow edges created inside the loop retain the outer branch choice. // PCG: bb3[2] pre_operands: {pair} -> {*pair} under conditions bb0 -> bb1, bb2 -> bb3 if enter { @@ -45,12 +45,12 @@ fn conditional_entry(pair: &mut Pair, enter: bool, run: bool) { fn nested_fields(outer: &mut Outer, run: bool) { // PCG: bb1[0] pre_operands: outer: R - // PCG: bb0 -> bb1: Add Edge {outer} -> {*outer} - // PCG: bb0 -> bb1: Add Edge {*outer} -> {(*outer).pair} - // PCG: bb0 -> bb1: Add Edge {(*outer).pair} -> {(*outer).pair.first, (*outer).pair.second} - // PCG: bb4 -> bb1: Add Edge {outer} -> {*outer} - // PCG: bb4 -> bb1: Add Edge {*outer} -> {(*outer).pair} - // PCG: bb4 -> bb1: Add Edge {(*outer).pair} -> {(*outer).pair.first, (*outer).pair.second} + // PCG: bb0 -> bb1: Add Edge {outer} -> {*outer} setting outer: R, *outer: R + // PCG: bb0 -> bb1: Add Edge {*outer} -> {(*outer).pair} setting *outer: R, (*outer).pair: R + // PCG: bb0 -> bb1: Add Edge {(*outer).pair} -> {(*outer).pair.first, (*outer).pair.second} setting (*outer).pair: R, (*outer).pair.first: R, (*outer).pair.second: R + // PCG: bb4 -> bb1: Add Edge {outer} -> {*outer} setting outer: R, *outer: R + // PCG: bb4 -> bb1: Add Edge {*outer} -> {(*outer).pair} setting *outer: R, (*outer).pair: R + // PCG: bb4 -> bb1: Add Edge {(*outer).pair} -> {(*outer).pair.first, (*outer).pair.second} setting (*outer).pair: R, (*outer).pair.first: R, (*outer).pair.second: R while run { std::hint::black_box(&outer.pair.first); } @@ -60,12 +60,12 @@ fn existing_expansions(pair: &mut Pair, run: bool) { // PCG: bb1[0] pre_operands: pair: R // PCG: bb1[0] pre_operands: {pair} -> {*pair} // PCG: bb1[0] pre_operands: {*pair} -> {(*pair).first, (*pair).second} - // ~PCG: bb0 -> bb1: Add Edge {pair} -> {*pair} - // ~PCG: bb0 -> bb1: Add Edge {*pair} -> {(*pair).first, (*pair).second} - // ~PCG: bb5 -> bb1: Add Edge {pair} -> {*pair} - // ~PCG: bb5 -> bb1: Add Edge {*pair} -> {(*pair).first, (*pair).second} - // ~PCG: bb5 -> bb1: Add Edge {pair} -> {*pair} under conditions bb2 -> bb3 - // ~PCG: bb5 -> bb1: Add Edge {*pair} -> {(*pair).first, (*pair).second} under conditions bb2 -> bb3 + // ~PCG: bb0 -> bb1: Add Edge {pair} -> {*pair} setting pair: R, *pair: R + // ~PCG: bb0 -> bb1: Add Edge {*pair} -> {(*pair).first, (*pair).second} setting *pair: R, (*pair).first: R, (*pair).second: R + // ~PCG: bb5 -> bb1: Add Edge {pair} -> {*pair} setting pair: R, *pair: R + // ~PCG: bb5 -> bb1: Add Edge {*pair} -> {(*pair).first, (*pair).second} setting *pair: R, (*pair).first: R, (*pair).second: R + // ~PCG: bb5 -> bb1: Add Edge {pair} -> {*pair} under conditions bb2 -> bb3 setting pair: R, *pair: R + // ~PCG: bb5 -> bb1: Add Edge {*pair} -> {(*pair).first, (*pair).second} under conditions bb2 -> bb3 setting *pair: R, (*pair).first: R, (*pair).second: R let hold = &pair.first; while run { std::hint::black_box(hold); @@ -77,17 +77,58 @@ fn reference_field(pair: &RefPair<'_>, run: bool) { // PCG_LIFETIME_DISPLAY: pair 0 'pair // PCG_LIFETIME_DISPLAY: pair 1 'first // PCG: bb1[0] pre_operands: pair: R - // PCG: bb0 -> bb1: Add Edge {pair} -> {*pair} - // PCG: bb0 -> bb1: Add Edge {*pair} -> {(*pair).first, (*pair).second} + // PCG: bb0 -> bb1: Add Edge {pair} -> {*pair} setting pair: R, *pair: R + // PCG: bb0 -> bb1: Add Edge {*pair} -> {(*pair).first, (*pair).second} setting *pair: R, (*pair).first: R, (*pair).second: R // PCG: bb0 -> bb1: Add Edge {*pair↓'first} -> {(*pair).first↓'first} - // PCG: bb0 -> bb1: Add Edge {(*pair).first} -> {*(*pair).first} - // PCG: bb4 -> bb1: Add Edge {pair} -> {*pair} - // PCG: bb4 -> bb1: Add Edge {*pair} -> {(*pair).first, (*pair).second} + // PCG: bb0 -> bb1: Add Edge {(*pair).first} -> {*(*pair).first} setting (*pair).first: R, *(*pair).first: R + // PCG: bb4 -> bb1: Add Edge {pair} -> {*pair} setting pair: R, *pair: R + // PCG: bb4 -> bb1: Add Edge {*pair} -> {(*pair).first, (*pair).second} setting *pair: R, (*pair).first: R, (*pair).second: R // PCG: bb4 -> bb1: Add Edge {*pair↓'first} -> {(*pair).first↓'first} - // PCG: bb4 -> bb1: Add Edge {(*pair).first} -> {*(*pair).first} + // PCG: bb4 -> bb1: Add Edge {(*pair).first} -> {*(*pair).first} setting (*pair).first: R, *(*pair).first: R while run { std::hint::black_box(&*pair.first); } } +struct File<'data> { + symbols: &'data [u8], +} + +struct Iter<'data, 'file> { + file: &'file File<'data>, + index: usize, +} + +struct Comdat<'data, 'file> { + file: &'file File<'data>, + symbol: &'data u8, +} + +fn parse<'data, 'file>( + file: &'file File<'data>, + symbol: &'data u8, +) -> Option> { + if *symbol > 0 { + Some(Comdat { file, symbol }) + } else { + None + } +} + +impl<'data, 'file> Iter<'data, 'file> { + // The back edge unpacks `(*self).file` with the loop's snapshot, which + // labels its current lifetime projection as the loop's: the flow from the + // loop's value into it collapses. + fn next(&mut self) -> Option> { + loop { + let index = self.index; + let symbol = self.file.symbols.get(index)?; + self.index += 1; + if let Some(comdat) = parse(self.file, symbol) { + return Some(comdat); + } + } + } +} + fn main() {} diff --git a/test-files/239_join_snapshots.rs b/test-files/239_join_snapshots.rs index 64ef0c6ee..f8165dab5 100644 --- a/test-files/239_join_snapshots.rs +++ b/test-files/239_join_snapshots.rs @@ -6,7 +6,7 @@ struct Node { fn snapshot_before_join_weakening(mut node: Node) { // PCG: bb4[4] post_main: x: e - // PCG: bb4 -> bb5: Weaken(Weaken { place: _2, from: e, to: W, for_storage_dead: false, _marker: PhantomData<&()> }) + // PCG: bb4 -> bb5: Weaken x from e to W // PCG: bb5[0] pre_operands: x: W // We use after bb4 instead of before bb5 because Prusti can refer to the version of `x` after bb4: @@ -35,8 +35,8 @@ fn snapshots_from_multiple_predecessors(mut node: BranchNode) { // Each borrowed arm must retain its own snapshot and branch condition. // PCG: bb5[4] post_main: x: e // PCG: bb6[4] post_main: x: e - // PCG: bb5 -> bb7: Weaken(Weaken { place: _2, from: e, to: W, for_storage_dead: false, _marker: PhantomData<&()> }) - // PCG: bb6 -> bb7: Weaken(Weaken { place: _2, from: e, to: W, for_storage_dead: false, _marker: PhantomData<&()> }) + // PCG: bb5 -> bb7: Weaken x from e to W + // PCG: bb6 -> bb7: Weaken x from e to W // PCG: bb7[0] pre_operands: x: W // PCG: bb7[0] pre_operands: {*x after bb5} -> {(*x).next after bb5} under conditions bb0 -> bb3 // PCG: bb7[0] pre_operands: {*x after bb6} -> {(*x).next after bb6} under conditions bb0 -> bb2 @@ -53,4 +53,24 @@ fn snapshots_from_multiple_predecessors(mut node: BranchNode) { x.next = Next::None; } +struct RawCell { + p: *mut i32, + n: i32, +} + +// The dereference of a raw pointer is an unpacking, not a loan: the path +// that read it directly gives up its value. +unsafe fn raw_deref_read_or_frozen(mut s: RawCell, c: bool) -> i32 { + // PCG: bb2 -> bb3: Label place *s.p (Detach) + let v; + if c { + let q = &s; + v = *(*q).p; + } else { + v = *s.p; + } + s.n = v; + s.n +} + fn main() {} diff --git a/test-files/240_join_loop_abstraction.rs b/test-files/240_join_loop_abstraction.rs index b7ffbad4c..f809872f7 100644 --- a/test-files/240_join_loop_abstraction.rs +++ b/test-files/240_join_loop_abstraction.rs @@ -9,9 +9,13 @@ fn consume(mut list: List<'_>) { // state loses the current root and weakens its capabilities. // PCG: bb1[0] pre_operands: list: E // PCG: bb1[0] pre_operands: Loop(bb1): Remote(_1)↓'a -> list↓'a - // Preserve the sibling field when the loop abstracts the path through tail. - // PCG: bb1[0] pre_operands: Loop(bb1): Remote(_1)↓'a -> list.head before bb1[0]:PreOperands↓'a - // PCG: bb1[0] pre_operands: list.tail@Some before bb1[0]:PreOperands↓'a -> list.tail before bb1[0]:PreOperands↓'a + // Preserve the sibling field when the loop abstracts the path through + // tail: it stays connected to the remote projection through `list`. + // PCG: bb1[0] pre_operands: list.head before bb1[0]:PreOperands↓'a -> list↓'a + // PCG: bb1[0] pre_operands: Loop(bb1): Remote(_1)↓'a -> list.tail@Some.0↓'a + // The loop head does not unpack `list.tail` to `Some`: the path that + // leaves the loop did not match it. + // ~PCG: bb1[0] pre_operands: list.tail@Some before bb1[0]:PreOperands↓'a -> list.tail before bb1[0]:PreOperands↓'a while let Some(tail) = list.tail { list = *tail; } @@ -24,13 +28,10 @@ fn consume_after_branch<'a>(mut list: List<'a>, other: List<'a>, choose_other: b // PCG: bb7[0] pre_operands: list: E // PCG: bb7[0] pre_operands: Loop(bb7): Remote(_1)↓'list -> list↓'list // PCG: bb7[0] pre_operands: Loop(bb7): Remote(_2)↓'other -> list↓'list - // PCG: bb7[0] pre_operands: Loop(bb7): Remote(_1)↓'list -> list.head before bb7[0]:PreOperands↓'list - // PCG: bb7[0] pre_operands: Loop(bb7): Remote(_2)↓'other -> list.head before bb7[0]:PreOperands↓'list + // PCG: bb7[0] pre_operands: list.head before bb7[0]:PreOperands↓'list -> list↓'list // Collapsing one child must preserve the path from the remote projections to tail. - // PCG: bb7[0] pre_operands: Loop(bb7): Remote(_1)↓'list -> list.tail@Some.0 before bb7[0]:PreOperands↓'list - // PCG: bb7[0] pre_operands: Loop(bb7): Remote(_2)↓'other -> list.tail@Some.0 before bb7[0]:PreOperands↓'list - // PCG: bb7[0] pre_operands: list.tail@Some.0 before bb7[0]:PreOperands↓'list -> list.tail@Some before bb7[0]:PreOperands↓'list - // PCG: bb7[0] pre_operands: list.tail@Some before bb7[0]:PreOperands↓'list -> list.tail before bb7[0]:PreOperands↓'list + // PCG: bb7[0] pre_operands: Loop(bb7): Remote(_1)↓'list -> list.tail@Some.0↓'list + // PCG: bb7[0] pre_operands: Loop(bb7): Remote(_2)↓'other -> list.tail@Some.0↓'list if choose_other { list = other; } @@ -39,4 +40,428 @@ fn consume_after_branch<'a>(mut list: List<'a>, other: List<'a>, choose_other: b } } +pub struct Node { + prefix: Vec, + children: Vec, + wild: bool, +} + +struct Skipped<'n, 'p> { + node: &'n Node, + path: &'p [u8], +} + +impl Node { + // `self` is dead in the loops; the body weakens it, so the inner loop + // head holds it with W. + pub fn at<'n, 'p>(&'n self, mut path: &'p [u8]) -> Option<&'n Node> { + // PCG: bb1 -> bb2: Weaken self from E to W + // PCG: bb2[0] pre_operands: self: W + let mut node = self; + let mut skipped: Vec> = Vec::new(); + 'backtrack: loop { + 'walk: loop { + if path.len() <= node.prefix.len() { + if *path == *node.prefix { + return Some(node); + } + break 'walk; + } + let previous = path; + path = &path[node.prefix.len()..]; + if node.wild { + skipped.push(Skipped { node, path: previous }); + node = &node.children[0]; + continue 'walk; + } + break 'walk; + } + while let Some(s) = skipped.pop() { + node = s.node; + path = s.path; + continue 'backtrack; + } + return None; + } + } +} + +struct P { + f: i32, + g: i32, +} + +// The loop lends `x` while `a` may still borrow it, so the loop head holds +// `x` frozen although an iteration writes `x.g` after `a` is reassigned. +fn shared_loan_ends_in_loop(n: i32) -> i32 { + // PCG: bb1[0] pre_operands: x.f: R + // PCG: bb1[0] pre_operands: x.f: suspended E + // PCG: bb1[0] pre_operands: x.g: R + // PCG: bb1[0] pre_operands: x.g: suspended E + // ~PCG: bb1[0] pre_operands: x.g: E + // The iteration that reassigns `a` gets `x` back, and the back edge + // freezes it again. + // PCG: bb4[0] pre_operands: Unfreeze x.g + // PCG: bb4[0] pre_operands: x.g: E + // PCG: bb4 -> bb6: Freeze x.g + // Nothing is lost when the loop exits. + // PCG: bb10[0] pre_operands: x: E + let mut x = P { f: 0, g: 0 }; + let z = P { f: 1, g: 2 }; + let mut a = &x; + let mut s = 0; + let mut i = 0; + while i < n { + if i == 3 { + a = &z; + x.g = 1; + } + s += a.g + x.g; + i += 1; + } + s +} + +// One iteration lends `x.f` mutably, another `x` whole: neither part holds a +// capability at the loop head. +fn part_or_whole_lent(n: i32) -> i32 { + // PCG_LIFETIME_DISPLAY: a 0 'a + // PCG: bb0 -> bb1: Lend x to Loop(bb1): x -> a↓'a + // PCG: bb1[0] pre_operands: x.f: suspended E + // PCG: bb1[0] pre_operands: x.g: suspended E + // ~PCG: bb1[0] pre_operands: x.g: E + // PCG: bb5[0] pre_operands: x: E + let mut x = P { f: 0, g: 0 }; + let mut y = 0; + let mut a: &mut i32 = &mut y; + let mut i = 0; + while i < n { + *a += 1; + if i % 2 == 0 { + a = &mut x.f; + } else { + let w = &mut x; + a = &mut w.g; + } + i += 1; + } + *a +} + +fn part_or_whole_lent_through_ref(x: &mut P, n: i32) -> i32 { + // PCG_LIFETIME_DISPLAY: a 0 'a + // PCG: bb0 -> bb1: Lend *x to Loop(bb1): *x -> a↓'a + // PCG: bb1[0] pre_operands: (*x).f: suspended E + // PCG: bb1[0] pre_operands: (*x).g: suspended E + // ~PCG: bb1[0] pre_operands: (*x).g: E + // PCG: bb5[0] pre_operands: x: E + let mut y = 0; + let mut a: &mut i32 = &mut y; + let mut i = 0; + while i < n { + *a += 1; + if i % 2 == 0 { + a = &mut x.f; + } else { + let w = &mut *x; + a = &mut w.g; + } + i += 1; + } + *a +} + +// One iteration borrows `x.f` shared, another `x` mutably: the loop head +// lends `x` as for a mutable loan. +fn shared_or_mutable(n: i32) -> i32 { + // PCG_LIFETIME_DISPLAY: a 0 'a + // PCG: bb0 -> bb1: Lend x to Loop(bb1): x -> a↓'a + // ~PCG: bb0 -> bb1: Weaken suspended x from E to None + // PCG: bb1[0] pre_operands: x: suspended E + // ~PCG: bb1[0] pre_operands: x.f: R + // PCG: bb12[0] pre_operands: x: E + let mut x = P { f: 0, g: 0 }; + let y = 0; + let mut a: &i32 = &y; + let mut i = 0; + let mut s = 0; + while i < n { + s += *a; + if i % 2 == 0 { + a = &x.f; + } else { + let w = &mut x; + w.g += 1; + a = &w.f; + } + i += 1; + } + s +} + +// `x.f` is frozen before the loop by a shared loan that ends inside it. +fn frozen_before_loop(n: i32) -> i32 { + // PCG: bb1[0] pre_operands: x.f: R + // PCG: bb1[0] pre_operands: x.f: suspended E + // PCG: bb1[0] pre_operands: x.g: E + // PCG: bb5[0] pre_operands: Unfreeze x.f + // PCG: bb12[0] pre_operands: x: E + let mut x = P { f: 0, g: 0 }; + let y = 0; + let mut r = &x.f; + let mut s = 0; + let mut i = 0; + while i < n { + s += *r; + if i == 3 { + r = &y; + x.f += 1; + } + x.g += 1; + i += 1; + } + s + x.f +} + +enum E { + A(i32, i32), + B(i32), +} + +// An iteration lends a field of variant `A`. The loop head does not know +// the variant of `x`, so it lends `x` whole. +fn variant_field_lent(n: i32) -> i32 { + // PCG_LIFETIME_DISPLAY: a 0 'a + // PCG: bb0 -> bb1: Lend x to Loop(bb1): x -> a↓'a + // PCG: bb1[0] pre_operands: x: suspended E + // ~PCG: bb1[0] pre_operands: x@A.1: E + // PCG: bb5[0] pre_operands: x: E + let mut x = E::A(0, 0); + let mut y = 0; + let mut a: &mut i32 = &mut y; + let mut i = 0; + while i < n { + *a += 1; + match x { + E::A(ref mut p, _) => a = p, + E::B(_) => a = &mut y, + } + i += 1; + } + *a +} + +pub struct CharIndices<'a> { + bytes: &'a [u8], + at: usize, +} + +impl<'a> Iterator for CharIndices<'a> { + type Item = (usize, usize, char); + + fn next(&mut self) -> Option<(usize, usize, char)> { + let b = *self.bytes.get(self.at)?; + self.at += 1; + Some((self.at - 1, self.at, b as char)) + } +} + +pub struct FieldsWith<'a, F> { + f: F, + bytes: &'a [u8], + chars: CharIndices<'a>, +} + +// The `for` loop borrows only `self.chars`; the lifetime it shares with +// `self.bytes` does not lend `*self`, so the loop can call `self.f`. +impl<'a, F: FnMut(char) -> bool> Iterator for FieldsWith<'a, F> { + type Item = &'a [u8]; + + fn next(&mut self) -> Option<&'a [u8]> { + // PCG_LIFETIME_DISPLAY: iter 0 'iter + // PCG: bb13[0] pre_operands: Loop(bb13): (*self).chars -> iter↓'iter under conditions bb3 -> bb6, bb8 -> bb10 + // ~PCG: bb13[0] pre_operands: Loop(bb13): *self -> iter↓'iter under conditions bb3 -> bb6, bb8 -> bb10 + // PCG: bb13[0] pre_operands: (*self).chars: suspended E + // PCG: bb13[0] pre_operands: (*self).f: E + // PCG: bb13[0] pre_operands: (*self).bytes: E + let (start, mut end); + loop { + match self.chars.next() { + None => return None, + Some((s, e, ch)) => { + if !(self.f)(ch) { + start = s; + end = e; + break; + } + } + } + } + for (_, e, ch) in self.chars.by_ref() { + if (self.f)(ch) { + break; + } + end = e; + } + Some(&self.bytes[start..end]) + } +} + +#[derive(Clone, Copy)] +pub struct Rules { + a: u32, + b: u32, +} + +impl Rules { + fn or(self, other: Rules) -> Rules { + if self.a == 0 { other } else { self } + } +} + +pub struct Attrs { + rules: Rules, + name: u32, +} + +impl Attrs { + fn rules(&self) -> Rules { + self.rules + } + fn rename(&mut self, r: Rules) { + self.name = r.b; + } +} + +pub struct Field<'a> { + ty: &'a u32, + attrs: Attrs, +} + +pub struct Variant<'a> { + original: &'a u32, + attrs: Attrs, + fields: Vec>, +} + +pub enum Data<'a> { + Enum(Vec>), + Struct(u32, Vec>), +} + +// The inner loop lends `(*variant).fields` and reads `(*variant).attrs`: +// `*variant` keeps the R of the read unpacking above the lent field. +fn rename_variants<'a>(data: Data<'a>, attrs: &Attrs) -> Data<'a> { + // PCG: bb16[0] pre_operands: variant: R + // PCG: bb16[0] pre_operands: variant: suspended E + // PCG: bb16[0] pre_operands: *variant: R + // PCG: bb16[0] pre_operands: (*variant).attrs: R + // ~PCG: bb16[0] pre_operands: (*variant).fields: R + let mut data = data; + match &mut data { + Data::Enum(variants) => { + for variant in variants { + variant.attrs.rename(attrs.rules()); + for field in &mut variant.fields { + field.attrs.rename(variant.attrs.rules().or(attrs.rules())); + } + } + } + Data::Struct(_, fields) => { + for field in fields { + field.attrs.rename(attrs.rules()); + } + } + } + data +} + +// The two-phase borrow of `v` for `push` is reserved, not activated, at the +// loop head: `v` stays frozen through the loop until the activation. +fn reserved_across_loop(mut v: Vec) -> Vec { + // PCG: bb1[0] pre_operands: v: R + // PCG: bb1[0] pre_operands: v: suspended E + // PCG: bb8[10] pre_operands: Activate two-phase borrow borrow: _3 = &mut v + v.push({ + let mut s = 0; + let mut i = 0; + while i < v.len() { + s += v[i]; + i += 1; + } + s + }); + v +} + +enum Either<'a, 'b> { + A(&'a u32), + B(&'b mut u32), +} + +// The loop lends `z` to the lifetime of the variant field that borrows it, +// not to the first lifetime of `x`: `z` comes back once that loan ends, +// while `a` keeps the other lifetime alive. +fn variant_blocker_lifetime<'a>(mut z: u32, w: &'a u32, n: u32) -> u32 { + // PCG_LIFETIME_DISPLAY: x 1 'b + // PCG: bb2[0] pre_operands: Loop(bb2): z -> x@B.0↓'b + // PCG: bb2[0] pre_operands: z: suspended E + // PCG: bb17[0] pre_operands: z: E + let mut x: Either<'a, '_> = Either::B(&mut z); + for _ in 0..n { + if let Either::B(ref mut r) = x { + **r += 1; + } + } + let a: &u32 = if let Either::A(a) = x { a } else { w }; + let t = z; + t + *a +} + +// At the loop head `a` may borrow `x` and `b` may borrow `x.f` mutably: +// `x.f` is lent, and `x.g`, which only `a` borrows, is frozen. +fn shared_whole_mutable_part(c: bool) -> u32 { + // PCG: bb1[0] pre_operands: x.f: suspended E + // PCG: bb1[0] pre_operands: x.g: R + // PCG: bb1[0] pre_operands: x.g: suspended E + // PCG: bb9[0] pre_operands: x: E + let mut x = P { f: 0, g: 0 }; + let mut y = 0; + let z = P { f: 0, g: 0 }; + let mut s = 0; + let mut a = &x; + let mut b = &mut y; + loop { + s += a.g + x.g; + *b += 1; + if c { + a = &z; + b = &mut x.f; + } + if s > 10 { + break; + } + } + s +} + +// The loop is on one branch between the reservation of the two-phase borrow +// for `push` and its activation: `*v` stays frozen through it. +fn reserved_across_loop_on_branch(v: &mut Vec, c: bool) { + // PCG: bb2[0] pre_operands: *v: R + // PCG: bb2[0] pre_operands: *v: suspended E + // PCG: bb11[1] pre_operands: Activate two-phase borrow borrow: _4 = &mut *v + v.push(if c { + let mut s = 0; + let mut i = 0; + while i < v.len() { + s += v[i]; + i += 1; + } + s + } else { + 0 + }); +} + fn main() {} diff --git a/test-files/241_loop_cut_outputs.rs b/test-files/241_loop_cut_outputs.rs index 607b5d8c8..38e75b6dd 100644 --- a/test-files/241_loop_cut_outputs.rs +++ b/test-files/241_loop_cut_outputs.rs @@ -9,8 +9,12 @@ fn consume(mut list: List<'_, '_>) { // PCG_LIFETIME_DISPLAY: list 1 'second // Cutting the paths through tail must preserve both sibling fields. // Each field remains connected only to its original remote projection. - // PCG: bb1[0] pre_operands: Loop(bb1): Remote(_1)↓'first -> list.first before bb1[0]:PreOperands↓'first - // PCG: bb1[0] pre_operands: Loop(bb1): Remote(_1)↓'second -> list.second before bb1[0]:PreOperands↓'second + // PCG: bb1[0] pre_operands: Loop(bb1): Remote(_1)↓'first -> list↓'first + // PCG: bb1[0] pre_operands: list.first before bb1[0]:PreOperands↓'first -> list↓'first + // PCG: bb1[0] pre_operands: Loop(bb1): Remote(_1)↓'second -> list↓'second + // PCG: bb1[0] pre_operands: list.second before bb1[0]:PreOperands↓'second -> list↓'second + // ~PCG: bb1[0] pre_operands: Loop(bb1): Remote(_1)↓'first -> list↓'second + // ~PCG: bb1[0] pre_operands: Loop(bb1): Remote(_1)↓'second -> list↓'first // ~PCG: bb1[0] pre_operands: Loop(bb1): Remote(_1)↓'first -> list.second before bb1[0]:PreOperands↓'second // ~PCG: bb1[0] pre_operands: Loop(bb1): Remote(_1)↓'second -> list.first before bb1[0]:PreOperands↓'first while let Some(tail) = list.tail { diff --git a/test-files/243_dead_lifetime_struct_join.rs b/test-files/243_dead_lifetime_struct_join.rs index 6283fe6d3..e74f5ca5a 100644 --- a/test-files/243_dead_lifetime_struct_join.rs +++ b/test-files/243_dead_lifetime_struct_join.rs @@ -21,13 +21,15 @@ fn arg<'a>(t: T<'a>) -> Option<&'a i32> { } fn local<'a>(t: T<'a>) -> Option<&'a i32> { - // `u` is W on the `else` branch and read-expanded on the other, so the - // join leaves it without capability until its StorageDead. + // `u` is W on the `else` branch and read-expanded on the other: the read + // path labels the value of `u` that the result borrows from, unfreezes + // `u` and weakens it to W. // PCG: bb3[0] pre_operands: Weaken u from E to W - // ~PCG: bb4[0] pre_operands: u: W + // PCG: bb2 -> bb4: Label place u (Write) + // PCG: bb2 -> bb4: Unfreeze u + // PCG: bb2 -> bb4: Weaken u from E to W + // PCG: bb4[0] pre_operands: u: W // ~PCG: bb4[0] pre_operands: u: R - // PCG: bb4[1] pre_main: Restore capability W to u.ctx - // PCG: bb4[1] pre_main: Restore capability W to u.valid let u = t; if u.valid { Some(u.ctx.row()) } else { None } } diff --git a/visualization/src/actionFormatting.ts b/visualization/src/actionFormatting.ts index cd5f912dc..eac9ca682 100644 --- a/visualization/src/actionFormatting.ts +++ b/visualization/src/actionFormatting.ts @@ -1,5 +1,6 @@ import { BorrowPcgActionKindDebugRepr, + CapabilityChanges, CapabilityKind, RepackOp, } from "./generated_types"; @@ -17,6 +18,12 @@ export function capabilityLetter(capability: CapabilityKind): string { } } +export function capabilityChangesLine(changes: CapabilityChanges): string { + return changes + .map(([place, cap]) => `${place}: ${cap ? capabilityLetter(cap) : "None"}`) + .join(", "); +} + export function actionLine( action: RepackOp | BorrowPcgActionKindDebugRepr ): string { @@ -26,15 +33,12 @@ export function actionLine( case "Collapse": return `Pack ${action.data.to}`; case "Weaken": - if (typeof action.data === "string") { - return action.data; - } - return `${action.data.place}: ${capabilityLetter(action.data.from)} -> ${action.data.to ? capabilityLetter(action.data.to) : "None"}`; - case "RegainLoanedCapability": - return `Restore capability ${capabilityLetter(action.data.capability)} to ${action.data.place}`; case "AddEdge": case "RemoveEdge": - case "Restore": + case "Freeze": + case "Unfreeze": + case "ActivateTwoPhaseBorrow": + case "Lend": return typeof action.data === "string" ? action.data : JSON.stringify(action.data); case "LabelPlace": return action.data diff --git a/visualization/src/components/PCGNavigator.tsx b/visualization/src/components/PCGNavigator.tsx index 75a8d0708..c3422260f 100644 --- a/visualization/src/components/PCGNavigator.tsx +++ b/visualization/src/components/PCGNavigator.tsx @@ -13,7 +13,7 @@ import { DotFileAtPhase, AppliedAction, } from "../generated_types"; -import { actionLine } from "../actionFormatting"; +import { actionLine, capabilityChangesLine } from "../actionFormatting"; import { useLocalStorageBool, useLocalStorageNumber, @@ -543,7 +543,10 @@ export default function PCGNavigator({ const action = item.phase === "successor" ? item.action : item.action.action; let hoverText = action.data.debug_info || ""; - const itemContent = actionLine(action.data.kind); + let itemContent = actionLine(action.data.kind); + if (action.data.capabilities.length > 0) { + itemContent += ` [${capabilityChangesLine(action.data.capabilities)}]`; + } if (item.phase !== "successor") { if (!hoverText) { hoverText = item.action.result.change_summary; diff --git a/visualization/src/generated_types/ActionKindWithDebugInfo.ts b/visualization/src/generated_types/ActionKindWithDebugInfo.ts index 3d76f5a57..5de50e64a 100644 --- a/visualization/src/generated_types/ActionKindWithDebugInfo.ts +++ b/visualization/src/generated_types/ActionKindWithDebugInfo.ts @@ -1,8 +1,14 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { CapabilityChanges } from "./CapabilityChanges"; import type { DisplayOutput } from "./DisplayOutput"; /** - * A pair of a PCG action and a debug context (indicating the source of the - * action and possibly its effects). + * A PCG action, the capability changes it makes, and a debug context + * (indicating the source of the action and possibly its effects). */ -export type ActionKindWithDebugInfo = { kind: T, debug_info: DebugInfo, }; +export type ActionKindWithDebugInfo = { kind: T, +/** + * The capability changes of the action. Every capability change of an + * action is listed here. + */ +capabilities: CapabilityChanges, debug_info: DebugInfo, }; diff --git a/visualization/src/generated_types/AppliedAction.ts b/visualization/src/generated_types/AppliedAction.ts index 2b648ca66..f3c1e28d3 100644 --- a/visualization/src/generated_types/AppliedAction.ts +++ b/visualization/src/generated_types/AppliedAction.ts @@ -5,4 +5,4 @@ import type { BorrowPcgActionKindDebugRepr } from "./BorrowPcgActionKindDebugRep import type { GenericPcgAction } from "./GenericPcgAction"; import type { RepackOp } from "./RepackOp"; -export type AppliedAction = { action: GenericPcgAction, ActionKindWithDebugInfo>, result: ApplyActionResult, _marker: null, }; +export type AppliedAction = { action: GenericPcgAction, ActionKindWithDebugInfo>, result: ApplyActionResult, _marker: null, }; diff --git a/visualization/src/generated_types/BorrowPcgActionKindDiscriminants.ts b/visualization/src/generated_types/BorrowPcgActionKindDiscriminants.ts index d9e222a58..8f96960ea 100644 --- a/visualization/src/generated_types/BorrowPcgActionKindDiscriminants.ts +++ b/visualization/src/generated_types/BorrowPcgActionKindDiscriminants.ts @@ -3,4 +3,4 @@ /** * Auto-generated discriminant enum variants */ -export type BorrowPcgActionKindDiscriminants = "LabelLifetimeProjection" | "Weaken" | "Restore" | "LabelPlace" | "RemoveEdge" | "AddEdge"; +export type BorrowPcgActionKindDiscriminants = "LabelLifetimeProjection" | "Weaken" | "Freeze" | "Unfreeze" | "ActivateTwoPhaseBorrow" | "LabelPlace" | "RemoveEdge" | "AddEdge" | "Lend" | "WeakenSuspended" | "UnpackSuspended" | "PackSuspended"; diff --git a/visualization/src/generated_types/CapabilityChanges.ts b/visualization/src/generated_types/CapabilityChanges.ts new file mode 100644 index 000000000..aea11aa1c --- /dev/null +++ b/visualization/src/generated_types/CapabilityChanges.ts @@ -0,0 +1,8 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { CapabilityKind } from "./CapabilityKind"; + +/** + * The capability changes of an action: each place gets the capability, or + * loses its capability (`None`). + */ +export type CapabilityChanges = Array<[string, CapabilityKind | null]>; diff --git a/visualization/src/generated_types/PcgSuccessorVisualizationData.ts b/visualization/src/generated_types/PcgSuccessorVisualizationData.ts index 99325f741..4d09dcd35 100644 --- a/visualization/src/generated_types/PcgSuccessorVisualizationData.ts +++ b/visualization/src/generated_types/PcgSuccessorVisualizationData.ts @@ -4,4 +4,4 @@ import type { BorrowPcgActionKindDebugRepr } from "./BorrowPcgActionKindDebugRep import type { GenericPcgAction } from "./GenericPcgAction"; import type { RepackOp } from "./RepackOp"; -export type PcgSuccessorVisualizationData = { actions: Array, ActionKindWithDebugInfo>>, }; +export type PcgSuccessorVisualizationData = { actions: Array, ActionKindWithDebugInfo>>, }; diff --git a/visualization/src/generated_types/RegainedCapability.ts b/visualization/src/generated_types/RegainedCapability.ts deleted file mode 100644 index 5c163447f..000000000 --- a/visualization/src/generated_types/RegainedCapability.ts +++ /dev/null @@ -1,4 +0,0 @@ -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { CapabilityKind } from "./CapabilityKind"; - -export type RegainedCapability = { place: Place, capability: CapabilityKind, }; diff --git a/visualization/src/generated_types/RepackOp.ts b/visualization/src/generated_types/RepackOp.ts index 2030bcd97..0d996252b 100644 --- a/visualization/src/generated_types/RepackOp.ts +++ b/visualization/src/generated_types/RepackOp.ts @@ -1,8 +1,5 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { CapabilityKind } from "./CapabilityKind"; -import type { RegainedCapability } from "./RegainedCapability"; import type { RepackCollapse } from "./RepackCollapse"; import type { RepackExpand } from "./RepackExpand"; -import type { Weaken } from "./Weaken"; -export type RepackOp = { "type": "StorageDead", "data": string } | { "type": "IgnoreStorageDead", "data": string } | { "type": "Weaken", "data": Weaken } | { "type": "Expand", "data": RepackExpand } | { "type": "Collapse", "data": RepackCollapse } | { "type": "DerefShallowInit", "data": [string, string] } | { "type": "RegainLoanedCapability", "data": RegainedCapability }; +export type RepackOp = { "type": "StorageDead", "data": string } | { "type": "Expand", "data": RepackExpand } | { "type": "Collapse", "data": RepackCollapse } | { "type": "DerefShallowInit", "data": [string, string] }; diff --git a/visualization/src/generated_types/Weaken.ts b/visualization/src/generated_types/Weaken.ts index c48a4b280..2e118ad7c 100644 --- a/visualization/src/generated_types/Weaken.ts +++ b/visualization/src/generated_types/Weaken.ts @@ -2,11 +2,10 @@ import type { CapabilityKind } from "./CapabilityKind"; /** - * Instructs that the current capability to the place (first [`CapabilityKind`]) should - * be weakened to the second given capability. We guarantee that `_.1 > _.2`. - * If `_.2` is `None`, the capability is removed. + * The capability of the place goes from `from` down to the strictly lower + * `to`: the place's value is given up (`to` is W), or part of it. */ -export type Weaken = { place: string, from: CapabilityKind, to: ToCap, +export type Weaken = { place: string, from: CapabilityKind, to: CapabilityKind, /** * Temporary flag indicating this Weaken was emitted due to a `StorageDead`. * Required until is resolved. diff --git a/visualization/src/generated_types/index.ts b/visualization/src/generated_types/index.ts index 946ea8da4..375eb2854 100644 --- a/visualization/src/generated_types/index.ts +++ b/visualization/src/generated_types/index.ts @@ -7,6 +7,7 @@ export type { BasicBlock } from "./BasicBlock"; export type { BorrowPcgActionKindDebugRepr } from "./BorrowPcgActionKindDebugRepr"; export type { BorrowPcgActionKindDiscriminants } from "./BorrowPcgActionKindDiscriminants"; export type { BranchChoicesDebugRepr } from "./BranchChoicesDebugRepr"; +export type { CapabilityChanges } from "./CapabilityChanges"; export type { CapabilityKind } from "./CapabilityKind"; export type { Conditioned } from "./Conditioned"; export type { DataflowStmtPhase } from "./DataflowStmtPhase"; @@ -30,7 +31,6 @@ export type { PcgLoopDebugData } from "./PcgLoopDebugData"; export type { PcgStmtVisualizationData } from "./PcgStmtVisualizationData"; export type { PcgSuccessorVisualizationData } from "./PcgSuccessorVisualizationData"; export type { PcgVisualizationData } from "./PcgVisualizationData"; -export type { RegainedCapability } from "./RegainedCapability"; export type { RepackCollapse } from "./RepackCollapse"; export type { RepackExpand } from "./RepackExpand"; export type { RepackOp } from "./RepackOp"; diff --git a/visualization/src/types.ts b/visualization/src/types.ts index 3e456e484..a5d6bb5b7 100644 --- a/visualization/src/types.ts +++ b/visualization/src/types.ts @@ -64,8 +64,8 @@ export type ReactFlowNodeData = BasicBlockData & { }; export type PcgAction = GenericPcgAction< - ActionKindWithDebugInfo, - ActionKindWithDebugInfo + ActionKindWithDebugInfo, + ActionKindWithDebugInfo >; export type PcgActions = PcgAction[];