Copyright	(c) 2024 - 2026 Michael Chavinda
License	MIT
Stability	experimental
Safe Haskell	None
Language	Haskell2010

DataFrame.Typed.Lazy

Contents

Core type
Data sources
Schema-preserving operations
Schema-modifying operations
Aggregation
Joins
Sort
Execution
Re-exports for pipeline construction

Description

Type-safe lazy query pipelines: compile-time schema tracking (TypedDataFrame) with the deferred execution of LazyDataFrame. Queries build a phantom-typed logical plan; execution is deferred until run.

{-# LANGUAGE DataKinds, TypeApplications, TypeOperators #-}
import qualified DataFrame.Typed.Lazy as TL
import DataFrame.Typed (Column)

type Schema = '[ '("id", Int), '("name", Text), '("score", Double)]

main = do
    let query = TL.scanCsv @Schema "data.csv"
              & TL.filter (TL.col @"score" TL..>. TL.lit 0.5)
              & TL.select @'["id", "name"]
    df <- TL.run query   -- TypedDataFrame '[ '("id", Int), '("name", Text)]
    print df

Synopsis

data TypedLazyDataFrame (cols :: [(Symbol, Type)])
scanCsv :: forall (cols :: [(Symbol, Type)]). Schema -> Text -> TypedLazyDataFrame cols
scanSeparated :: forall (cols :: [(Symbol, Type)]). Char -> Schema -> Text -> TypedLazyDataFrame cols
scanParquet :: forall (cols :: [(Symbol, Type)]). Schema -> Text -> TypedLazyDataFrame cols
fromDataFrame :: forall (cols :: [(Symbol, Type)]). TypedDataFrame cols -> TypedLazyDataFrame cols
fromTypedDataFrame :: forall (cols :: [(Symbol, Type)]). TypedDataFrame cols -> TypedLazyDataFrame cols
filter :: forall (cols :: [(Symbol, Type)]). TExpr cols Bool -> TypedLazyDataFrame cols -> TypedLazyDataFrame cols
take :: forall (cols :: [(Symbol, Type)]). Int -> TypedLazyDataFrame cols -> TypedLazyDataFrame cols
derive :: forall (name :: Symbol) a (cols :: [(Symbol, Type)]). (KnownSymbol name, Columnable a, AssertAbsent name cols) => TExpr cols a -> TypedLazyDataFrame cols -> TypedLazyDataFrame (Snoc cols '(name, a))
select :: forall (names :: [Symbol]) (cols :: [(Symbol, Type)]). (AllKnownSymbol names, AssertAllPresent names cols) => TypedLazyDataFrame cols -> TypedLazyDataFrame (SubsetSchema names cols)
groupBy :: forall (keys :: [Symbol]) (cols :: [(Symbol, Type)]). (AllKnownSymbol keys, AssertAllPresent keys cols) => TypedLazyDataFrame cols -> TypedLazyGrouped keys cols
aggregate :: forall (keys :: [Symbol]) (cols :: [(Symbol, Type)]) (aggs :: [(Symbol, Type)]). TAgg keys cols aggs -> TypedLazyGrouped keys cols -> TypedLazyDataFrame (Append (GroupKeyColumns keys cols) (Reverse aggs))
innerJoin :: forall (key :: Symbol) (left :: [(Symbol, Type)]) (right :: [(Symbol, Type)]). (KnownSymbol key, AssertAllPresent '[key] left, AssertAllPresent '[key] right, AssertKeyTypesMatch '[key] left right) => TypedLazyDataFrame left -> TypedLazyDataFrame right -> TypedLazyDataFrame (InnerJoinSchema '[key] left right)
leftJoin :: forall (key :: Symbol) (left :: [(Symbol, Type)]) (right :: [(Symbol, Type)]). (KnownSymbol key, AssertAllPresent '[key] left, AssertAllPresent '[key] right, AssertKeyTypesMatch '[key] left right) => TypedLazyDataFrame left -> TypedLazyDataFrame right -> TypedLazyDataFrame (LeftJoinSchema '[key] left right)
rightJoin :: forall (key :: Symbol) (left :: [(Symbol, Type)]) (right :: [(Symbol, Type)]). (KnownSymbol key, AssertAllPresent '[key] left, AssertAllPresent '[key] right, AssertKeyTypesMatch '[key] left right) => TypedLazyDataFrame left -> TypedLazyDataFrame right -> TypedLazyDataFrame (RightJoinSchema '[key] left right)
fullOuterJoin :: forall (key :: Symbol) (left :: [(Symbol, Type)]) (right :: [(Symbol, Type)]). (KnownSymbol key, AssertAllPresent '[key] left, AssertAllPresent '[key] right, AssertKeyTypesMatch '[key] left right) => TypedLazyDataFrame left -> TypedLazyDataFrame right -> TypedLazyDataFrame (FullOuterJoinSchema '[key] left right)
sortBy :: forall (cols :: [(Symbol, Type)]). [(Text, SortOrder)] -> TypedLazyDataFrame cols -> TypedLazyDataFrame cols
run :: forall (cols :: [(Symbol, Type)]). KnownSchema cols => TypedLazyDataFrame cols -> IO (TypedDataFrame cols)
module DataFrame.Typed.Expr
module DataFrame.Typed.Types
data SortOrder
- = Ascending
- | Descending

Core type

data TypedLazyDataFrame (cols :: [(Symbol, Type)]) Source #

A lazy query with compile-time schema tracking.

Instances

Instances details

Show (TypedLazyDataFrame cols) Source #
Instance details Defined in DataFrame.Typed.Lazy Methods showsPrec :: Int -> TypedLazyDataFrame cols -> ShowS # show :: TypedLazyDataFrame cols -> String # showList :: [TypedLazyDataFrame cols] -> ShowS #

Data sources

scanCsv :: forall (cols :: [(Symbol, Type)]). Schema -> Text -> TypedLazyDataFrame cols Source #

Scan a CSV file with a given schema.

scanSeparated :: forall (cols :: [(Symbol, Type)]). Char -> Schema -> Text -> TypedLazyDataFrame cols Source #

Scan a character-separated file with a given schema.

scanParquet :: forall (cols :: [(Symbol, Type)]). Schema -> Text -> TypedLazyDataFrame cols Source #

Scan a Parquet file, directory, or glob pattern with a given schema.

fromDataFrame :: forall (cols :: [(Symbol, Type)]). TypedDataFrame cols -> TypedLazyDataFrame cols Source #

Lift an already-loaded eager TypedDataFrame into a lazy plan.

fromTypedDataFrame :: forall (cols :: [(Symbol, Type)]). TypedDataFrame cols -> TypedLazyDataFrame cols Source #

Synonym for fromDataFrame.

Schema-preserving operations

filter :: forall (cols :: [(Symbol, Type)]). TExpr cols Bool -> TypedLazyDataFrame cols -> TypedLazyDataFrame cols Source #

Keep rows that satisfy the predicate.

take :: forall (cols :: [(Symbol, Type)]). Int -> TypedLazyDataFrame cols -> TypedLazyDataFrame cols Source #

Retain at most n rows.

Schema-modifying operations

derive :: forall (name :: Symbol) a (cols :: [(Symbol, Type)]). (KnownSymbol name, Columnable a, AssertAbsent name cols) => TExpr cols a -> TypedLazyDataFrame cols -> TypedLazyDataFrame (Snoc cols '(name, a)) Source #

Add a computed column.

select :: forall (names :: [Symbol]) (cols :: [(Symbol, Type)]). (AllKnownSymbol names, AssertAllPresent names cols) => TypedLazyDataFrame cols -> TypedLazyDataFrame (SubsetSchema names cols) Source #

Retain only the listed columns.

Aggregation

groupBy :: forall (keys :: [Symbol]) (cols :: [(Symbol, Type)]). (AllKnownSymbol keys, AssertAllPresent keys cols) => TypedLazyDataFrame cols -> TypedLazyGrouped keys cols Source #

Group by key columns.

aggregate :: forall (keys :: [Symbol]) (cols :: [(Symbol, Type)]) (aggs :: [(Symbol, Type)]). TAgg keys cols aggs -> TypedLazyGrouped keys cols -> TypedLazyDataFrame (Append (GroupKeyColumns keys cols) (Reverse aggs)) Source #

Aggregate a grouped lazy query.

Joins

innerJoin :: forall (key :: Symbol) (left :: [(Symbol, Type)]) (right :: [(Symbol, Type)]). (KnownSymbol key, AssertAllPresent '[key] left, AssertAllPresent '[key] right, AssertKeyTypesMatch '[key] left right) => TypedLazyDataFrame left -> TypedLazyDataFrame right -> TypedLazyDataFrame (InnerJoinSchema '[key] left right) Source #

Typed inner join on a single key column present in both schemas.

leftJoin :: forall (key :: Symbol) (left :: [(Symbol, Type)]) (right :: [(Symbol, Type)]). (KnownSymbol key, AssertAllPresent '[key] left, AssertAllPresent '[key] right, AssertKeyTypesMatch '[key] left right) => TypedLazyDataFrame left -> TypedLazyDataFrame right -> TypedLazyDataFrame (LeftJoinSchema '[key] left right) Source #

Typed left join. The right table's non-key columns become Maybe.

rightJoin :: forall (key :: Symbol) (left :: [(Symbol, Type)]) (right :: [(Symbol, Type)]). (KnownSymbol key, AssertAllPresent '[key] left, AssertAllPresent '[key] right, AssertKeyTypesMatch '[key] left right) => TypedLazyDataFrame left -> TypedLazyDataFrame right -> TypedLazyDataFrame (RightJoinSchema '[key] left right) Source #

Typed right join. The left table's non-key columns become Maybe.

fullOuterJoin :: forall (key :: Symbol) (left :: [(Symbol, Type)]) (right :: [(Symbol, Type)]). (KnownSymbol key, AssertAllPresent '[key] left, AssertAllPresent '[key] right, AssertKeyTypesMatch '[key] left right) => TypedLazyDataFrame left -> TypedLazyDataFrame right -> TypedLazyDataFrame (FullOuterJoinSchema '[key] left right) Source #

Typed full outer join. Non-key columns from both tables become Maybe.

Sort

sortBy :: forall (cols :: [(Symbol, Type)]). [(Text, SortOrder)] -> TypedLazyDataFrame cols -> TypedLazyDataFrame cols Source #

Sort the result by column name and direction.

Execution

run :: forall (cols :: [(Symbol, Type)]). KnownSchema cols => TypedLazyDataFrame cols -> IO (TypedDataFrame cols) Source #

Execute the lazy query and return a typed DataFrame.

Re-exports for pipeline construction

module DataFrame.Typed.Expr

module DataFrame.Typed.Types

data SortOrder Source #

Sort direction used in Sort nodes and the public API.

Constructors

Ascending
Descending

Instances

Instances details

Show SortOrder Source #
Instance details Defined in DataFrame.Lazy.Internal.LogicalPlan Methods showsPrec :: Int -> SortOrder -> ShowS # show :: SortOrder -> String # showList :: [SortOrder] -> ShowS #
Eq SortOrder Source #
Instance details Defined in DataFrame.Lazy.Internal.LogicalPlan Methods (==) :: SortOrder -> SortOrder -> Bool # (/=) :: SortOrder -> SortOrder -> Bool #
Ord SortOrder Source #
Instance details Defined in DataFrame.Lazy.Internal.LogicalPlan Methods compare :: SortOrder -> SortOrder -> Ordering # (<) :: SortOrder -> SortOrder -> Bool # (<=) :: SortOrder -> SortOrder -> Bool # (>) :: SortOrder -> SortOrder -> Bool # (>=) :: SortOrder -> SortOrder -> Bool # max :: SortOrder -> SortOrder -> SortOrder # min :: SortOrder -> SortOrder -> SortOrder #