Safe Haskell	None
Language	Haskell2010

DataFrame.Lazy.Internal.Executor

Description

Pull-based (iterator) execution engine.

Each operator returns a Stream — an IO action that produces the next DataFrame batch on each call and returns Nothing when exhausted. Blocking operators (Sort, HashJoin) materialise their input before producing output. HashAggregate uses streaming partial aggregation when all aggregate expressions support it.

Synopsis

data ExecutorConfig = ExecutorConfig {
- memoryBudgetBytes :: !Int
- spillDirectory :: FilePath
- defaultBatchSize :: !Int
}
defaultExecutorConfig :: ExecutorConfig
execute :: PhysicalPlan -> ExecutorConfig -> IO DataFrame
foldBatches :: (b -> DataFrame -> IO b) -> b -> PhysicalPlan -> ExecutorConfig -> IO b

Documentation

data ExecutorConfig Source #

Constructors

ExecutorConfig
Fields memoryBudgetBytes :: !Int Per-node spill threshold (currently informational; not enforced yet). spillDirectory :: FilePath defaultBatchSize :: !Int

defaultExecutorConfig :: ExecutorConfig Source #

execute :: PhysicalPlan -> ExecutorConfig -> IO DataFrame Source #

Execute a physical plan, returning the complete result as a single DataFrame.

foldBatches :: (b -> DataFrame -> IO b) -> b -> PhysicalPlan -> ExecutorConfig -> IO b Source #

Fold a function over every batch produced by a physical plan. The fold is strict in the accumulator; each batch is discarded after folding.