Safe Haskell	None
Language	Haskell2010

DataFrame.Internal.ColumnBuilder

Description

Mutable, growable column builders for high-throughput ingest. No per-append IORef traffic: hot counters live in an unboxed vector, payloads double on demand, and validity is only materialized once a null is seen.

Synopsis

class ColumnBuilder (b :: Type -> Type) where
- appendNull :: b s -> ST s ()
- builderLength :: b s -> ST s Int
- freezeBuilder :: b s -> ST s Column
data NumBuilder a s
type IntBuilder = NumBuilder Int
type DoubleBuilder = NumBuilder Double
data TextBuilder s
data TextChunk = TextChunk {
- tcBytes :: !Array
- tcUsed :: !Int
- tcOffsets :: !(Vector Int)
- tcBitmap :: !(Maybe Bitmap)
}
newIntBuilder :: Int -> ST s (IntBuilder s)
newDoubleBuilder :: Int -> ST s (DoubleBuilder s)
newNumBuilder :: Unbox a => a -> Int -> ST s (NumBuilder a s)
newTextBuilder :: Int -> Int -> ST s (TextBuilder s)
appendInt :: IntBuilder s -> Int -> ST s ()
appendDouble :: DoubleBuilder s -> Double -> ST s ()
appendNum :: Unbox a => NumBuilder a s -> a -> ST s ()
appendText :: TextBuilder s -> Text -> ST s ()
appendTextSlice :: TextBuilder s -> Array -> Int -> Int -> ST s ()
appendTextSliceFromPtr :: TextBuilder s -> Ptr Word8 -> Int -> ST s ()
freezeTextChunk :: TextBuilder s -> ST s TextChunk
mergeColumns :: [Column] -> Column
mergeTextChunks :: [TextChunk] -> Column

Documentation

class ColumnBuilder (b :: Type -> Type) where Source #

Operations shared by all column builders. A builder must not be used again after freezeBuilder (its storage is frozen in place, not copied).

Methods

appendNull :: b s -> ST s () Source #

Append a null row (sentinel payload + invalid bit).

builderLength :: b s -> ST s Int Source #

Rows appended so far.

freezeBuilder :: b s -> ST s Column Source #

Freeze into a fully-forced Column; bitmap only when a null was seen.

Instances

Instances details

ColumnBuilder TextBuilder Source #
Instance details Defined in DataFrame.Internal.ColumnBuilder Methods appendNull :: TextBuilder s -> ST s () Source # builderLength :: TextBuilder s -> ST s Int Source # freezeBuilder :: TextBuilder s -> ST s Column Source #
(Columnable a, Unbox a) => ColumnBuilder (NumBuilder a) Source #
Instance details Defined in DataFrame.Internal.ColumnBuilder Methods appendNull :: NumBuilder a s -> ST s () Source # builderLength :: NumBuilder a s -> ST s Int Source # freezeBuilder :: NumBuilder a s -> ST s Column Source #

data NumBuilder a s Source #

Builder for unboxed numeric payloads (Int, Double, ...). nbNull is the sentinel written into null slots (protected by the bitmap).

Instances

Instances details

(Columnable a, Unbox a) => ColumnBuilder (NumBuilder a) Source #
Instance details Defined in DataFrame.Internal.ColumnBuilder Methods appendNull :: NumBuilder a s -> ST s () Source # builderLength :: NumBuilder a s -> ST s Int Source # freezeBuilder :: NumBuilder a s -> ST s Column Source #

type IntBuilder = NumBuilder Int Source #

type DoubleBuilder = NumBuilder Double Source #

data TextBuilder s Source #

Builder for Text columns. All field bytes go into one exponentially grown byte array; rows are recorded as offsets, so an append is a memcpy and freezing slices Text values off the shared array without copying.

Instances

Instances details

ColumnBuilder TextBuilder Source #
Instance details Defined in DataFrame.Internal.ColumnBuilder Methods appendNull :: TextBuilder s -> ST s () Source # builderLength :: TextBuilder s -> ST s Int Source # freezeBuilder :: TextBuilder s -> ST s Column Source #

data TextChunk Source #

A frozen text-builder chunk: raw UTF-8 bytes plus row offsets (row i spans bytes [offsets!i, offsets!(i+1))) and an optional validity bitmap. Text values are only created when chunks merge into a Column.

Constructors

TextChunk
Fields tcBytes :: !Array tcUsed :: !Int tcOffsets :: !(Vector Int) tcBitmap :: !(Maybe Bitmap)

newIntBuilder :: Int -> ST s (IntBuilder s) Source #

newDoubleBuilder :: Int -> ST s (DoubleBuilder s) Source #

newNumBuilder :: Unbox a => a -> Int -> ST s (NumBuilder a s) Source #

New numeric builder with a row-capacity hint and a null sentinel.

newTextBuilder :: Int -> Int -> ST s (TextBuilder s) Source #

New text builder with row-count and total-byte capacity hints.

appendInt :: IntBuilder s -> Int -> ST s () Source #

appendDouble :: DoubleBuilder s -> Double -> ST s () Source #

appendNum :: Unbox a => NumBuilder a s -> a -> ST s () Source #

appendText :: TextBuilder s -> Text -> ST s () Source #

Append an already-decoded Text (its bytes are UTF-8 already).

appendTextSlice :: TextBuilder s -> Array -> Int -> Int -> ST s () Source #

Append len raw bytes at off in src as one field (one memcpy).

appendTextSliceFromPtr :: TextBuilder s -> Ptr Word8 -> Int -> ST s () Source #

appendTextSlice from foreign memory (e.g. an mmapped file buffer).

freezeTextChunk :: TextBuilder s -> ST s TextChunk Source #

Freeze a TextBuilder into a raw TextChunk for byte-level merging (mergeTextChunks): no Text values are created until chunks merge.

mergeColumns :: [Column] -> Column Source #

Merge per-chunk columns into one column: one allocation + memcpy per payload, with bitmaps spliced across non-byte-aligned chunk boundaries. All chunks must have the same element type.

mergeTextChunks :: [TextChunk] -> Column Source #

Merge text chunks into one packed-text Column: one byte-array copy per chunk, one offset rebase, then wrap the shared buffer + offsets as PackedText (no per-row header, decode deferred).