diff --git a/differential-dataflow/examples/columnar/main.rs b/differential-dataflow/examples/columnar/main.rs index 692f1fd27..d8775f44e 100644 --- a/differential-dataflow/examples/columnar/main.rs +++ b/differential-dataflow/examples/columnar/main.rs @@ -90,6 +90,7 @@ fn main() { /// Push on UpdatesTyped for the reduce builder path. mod reachability { + use timely::container::PushInto; use timely::order::Product; use differential_dataflow::Collection; use differential_dataflow::AsCollection; @@ -149,7 +150,7 @@ mod reachability { use differential_dataflow::difference::Multiply; let dst: Node = *dst; let diff: Diff = d1.clone().multiply(d2); - session.give::<(Node, (), IterTime, Diff)>((dst, (), time.clone(), diff)); + session.push_into((dst, (), time.clone(), diff)); }, ).as_collection(); diff --git a/differential-dataflow/src/operators/arrange/arrangement.rs b/differential-dataflow/src/operators/arrange/arrangement.rs index bb7f74f6f..359ad8945 100644 --- a/differential-dataflow/src/operators/arrange/arrangement.rs +++ b/differential-dataflow/src/operators/arrange/arrangement.rs @@ -251,7 +251,7 @@ impl<'scope, Tr1: TraceReader+'static> Arranged<'scope, Tr1> { other, move |k, v1, v2, t, d1, d2, c| { for datum in result(k, v1, v2, t, d1, d2) { - c.give(datum); + c.push_into(datum); } } ) diff --git a/differential-dataflow/src/operators/join.rs b/differential-dataflow/src/operators/join.rs index 72b2085e9..3201c3176 100644 --- a/differential-dataflow/src/operators/join.rs +++ b/differential-dataflow/src/operators/join.rs @@ -4,71 +4,34 @@ //! the multiplication distributes over addition. That is, we will repeatedly evaluate (a + b) * c as (a * c) //! + (b * c), and if this is not equal to the former term, little is known about the actual output. use std::cmp::Ordering; +use std::collections::VecDeque; -use timely::{Accountable, ContainerBuilder}; -use timely::container::PushInto; +use timely::{Container, ContainerBuilder}; +use timely::container::NoopBuilder; use timely::order::PartialOrder; use timely::progress::Timestamp; use timely::dataflow::Stream; -use timely::dataflow::operators::generic::{Operator, OutputBuilderSession, Session}; +use timely::dataflow::operators::generic::{Operator, OutputBuilderSession}; use timely::dataflow::channels::pact::Pipeline; use timely::dataflow::operators::Capability; use crate::lattice::Lattice; use crate::operators::arrange::Arranged; use crate::trace::{BatchCursor, BatchDiff, BatchKey, BatchReader, BatchVal, Cursor, Navigable, TraceReader}; -use crate::trace::cursor::{cursor_list, CursorList}; +use crate::trace::cursor::cursor_list; use crate::operators::ValueHistory; -/// The session passed to join closures. -pub type JoinSession<'a, 'b, T, CB, CT> = Session<'a, 'b, T, EffortBuilder, CT>; - -/// A container builder that tracks the length of outputs to estimate the effort of join closures. -#[derive(Default, Debug)] -pub struct EffortBuilder(pub std::cell::Cell, pub CB); - -impl timely::container::ContainerBuilder for EffortBuilder { - type Container = CB::Container; - - #[inline] - fn extract(&mut self) -> Option<&mut Self::Container> { - let extracted = self.1.extract(); - self.0.replace(self.0.take() + extracted.as_ref().map_or(0, |e| e.record_count() as usize)); - extracted - } - - #[inline] - fn finish(&mut self) -> Option<&mut Self::Container> { - let finished = self.1.finish(); - self.0.replace(self.0.take() + finished.as_ref().map_or(0, |e| e.record_count() as usize)); - finished - } -} - -impl, D> PushInto for EffortBuilder { - #[inline] - fn push_into(&mut self, item: D) { - self.1.push_into(item); - } -} - /// A type that can manage the joining of lists of batches. -pub(crate) trait JoinTactic, CB: ContainerBuilder> { - /// Prepare for work the join of two lists of corresponding batches, against a sufficient capability. - /// - /// `fresh` names which input contributed the freshly-arrived batch; its times all lie at or beyond - /// the capability, so a tactic need not advance that side by the capability's meet. - fn defer(&mut self, input0: Vec, input1: Vec, fresh: Fresh, capability: Capability); - /// Perform an amount of work that just barely exceeds `fuel`, which is decremented. +/// +/// The trait is parameterized by the output container `C`, not by the builder that assembles it: a tactic +/// yields finished containers, and how it produces them (pushing records into a [`ContainerBuilder`], or +/// otherwise) is its own concern. +pub(crate) trait JoinTactic, C> { + /// Prepare the join of two lists of batches into an iterator of output containers. /// - /// **Fuel protocol.** On return, `fuel` is non-negative *iff* all outstanding work is exhausted, - /// and left negative exactly when work remains. The driver ([`join_with_tactic`]) reschedules the - /// operator iff `fuel < 0`, so returning non-negative with work still queued silently drops it - /// (and returning negative with nothing to do spins). The protocol is not driver-checkable — the - /// work-remaining state is tactic-internal — so derive the sign *from* that state rather than - /// tracking it separately, and it holds by construction (as the in-tree tactic does, setting - /// `fuel` from whether its queues are empty). - fn work(&mut self, fuel: &mut isize, output: &mut OutputBuilderSession>); + /// The supplied `fresh` and `meet` indicate respectively which input is "novel", and should drive the + /// join, as well as a lower bound on that input's times, so that the other input can be loaded compacted. + fn prep(&mut self, input0: Vec, input1: Vec, fresh: Fresh, meet: B0::Time) -> Box>; } /// Which input contributed the freshly-arrived batch of a deferred join unit. @@ -102,10 +65,10 @@ where Tr2: TraceReader+'static, BatchCursor: Cursor