| Safe Haskell | None |
|---|---|
| Language | Haskell2010 |
DataFrame.IO.Parquet.Utils
Synopsis
- data ColumnDescription = ColumnDescription {
- colElementType :: !(Maybe ThriftType)
- maxDefinitionLevel :: !Int32
- maxRepetitionLevel :: !Int32
- colLogicalType :: !(Maybe LogicalType)
- colConvertedType :: !(Maybe ConvertedType)
- typeLength :: !(Maybe Int32)
- generateColumnDescriptions :: [SchemaElement] -> [ColumnDescription]
- getColumnNames :: [SchemaElement] -> [Text]
- foldNonNullable :: (MonadIO m, Columnable a) => Int -> PageFold m Vector a -> m Column
- foldNonNullableUnboxed :: (MonadIO m, Columnable a, Unbox a) => Int -> PageFold m Vector a -> m Column
- foldNullable :: (MonadIO m, Columnable a) => Int -> Int -> PageFold m Vector a -> m Column
- foldNullableUnboxed :: (MonadIO m, Columnable a, Unbox a) => Int -> Int -> PageFold m Vector a -> m Column
- foldRepeated :: (MonadIO m, Columnable a, Columnable (Maybe [Maybe a]), Columnable (Maybe [Maybe [Maybe a]]), Columnable (Maybe [Maybe [Maybe [Maybe a]]])) => Int -> Int -> PageFold m Vector a -> m Column
- foldRepeatedUnboxed :: (MonadIO m, Columnable a, Unbox a, Columnable (Maybe [Maybe a]), Columnable (Maybe [Maybe [Maybe a]]), Columnable (Maybe [Maybe [Maybe [Maybe a]]])) => Int -> Int -> PageFold m Vector a -> m Column
Documentation
data ColumnDescription Source #
Constructors
| ColumnDescription | |
Fields
| |
Instances
| Show ColumnDescription Source # | |
Defined in DataFrame.IO.Parquet.Utils Methods showsPrec :: Int -> ColumnDescription -> ShowS # show :: ColumnDescription -> String # showList :: [ColumnDescription] -> ShowS # | |
| Eq ColumnDescription Source # | |
Defined in DataFrame.IO.Parquet.Utils Methods (==) :: ColumnDescription -> ColumnDescription -> Bool # (/=) :: ColumnDescription -> ColumnDescription -> Bool # | |
getColumnNames :: [SchemaElement] -> [Text] Source #
foldNonNullable :: (MonadIO m, Columnable a) => Int -> PageFold m Vector a -> m Column Source #
Fold a column's value pages into a non-nullable Column.
Pre-allocates a mutable vector of totalRows and fills it page-by-page via a
single streaming left fold (PageFold), avoiding any intermediate list or
concatenation allocation. Only one page's values are live at a time.
foldNonNullableUnboxed :: (MonadIO m, Columnable a, Unbox a) => Int -> PageFold m Vector a -> m Column Source #
foldNullable :: (MonadIO m, Columnable a) => Int -> Int -> PageFold m Vector a -> m Column Source #
Fold a column's (values, def-levels) pages into a nullable Column.
Pre-allocates the output buffer and a valid-mask vector of totalRows, then
scatters values inline during a single streaming left fold (PageFold).
A hasNull flag is accumulated during the scatter so the
buildBitmapFromValid call is skipped entirely when all values are present.
foldNullableUnboxed :: (MonadIO m, Columnable a, Unbox a) => Int -> Int -> PageFold m Vector a -> m Column Source #
foldRepeated :: (MonadIO m, Columnable a, Columnable (Maybe [Maybe a]), Columnable (Maybe [Maybe [Maybe a]]), Columnable (Maybe [Maybe [Maybe [Maybe a]]])) => Int -> Int -> PageFold m Vector a -> m Column Source #
Fold a stream of (values, def-levels, rep-levels) triples into a
repeated (list) Column using Dremel-style level stitching.
The stitching function is selected by maxRep:
maxRep == 1→stitchList→[Maybe [Maybe a]]maxRep == 2→stitchList2→[Maybe [Maybe [Maybe a]]]maxRep >= 3→stitchList3→[Maybe [Maybe [Maybe [Maybe a]]]]
Threshold formula: defT_r = maxDef - 2 * (maxRep - r).
foldRepeatedUnboxed :: (MonadIO m, Columnable a, Unbox a, Columnable (Maybe [Maybe a]), Columnable (Maybe [Maybe [Maybe a]]), Columnable (Maybe [Maybe [Maybe [Maybe a]]])) => Int -> Int -> PageFold m Vector a -> m Column Source #