{-# LANGUAGE BangPatterns #-}
{-# LANGUAGE FlexibleContexts #-}
{-# LANGUAGE MonoLocalBinds #-}
{-# LANGUAGE OverloadedStrings #-}

module DataFrame.IO.Parquet.Dictionary where

import Control.Monad
import Data.Bits
import qualified Data.ByteString as BS
import Data.IORef
import Data.Int
import Data.Maybe
import qualified Data.Text as T
import Data.Text.Encoding
import Data.Time
import qualified Data.Vector as V
import qualified Data.Vector.Mutable as VM
import qualified Data.Vector.Unboxed as VU
import DataFrame.IO.Parquet.Binary
import DataFrame.IO.Parquet.Encoding
import DataFrame.IO.Parquet.Levels
import DataFrame.IO.Parquet.Time
import DataFrame.IO.Parquet.Types
import qualified DataFrame.Internal.Column as DI
import GHC.Float

dictCardinality :: DictVals -> Int
dictCardinality :: DictVals -> Int
dictCardinality (DBool Vector Bool
ds) = Vector Bool -> Int
forall a. Vector a -> Int
V.length Vector Bool
ds
dictCardinality (DInt32 Vector Int32
ds) = Vector Int32 -> Int
forall a. Vector a -> Int
V.length Vector Int32
ds
dictCardinality (DInt64 Vector Int64
ds) = Vector Int64 -> Int
forall a. Vector a -> Int
V.length Vector Int64
ds
dictCardinality (DInt96 Vector UTCTime
ds) = Vector UTCTime -> Int
forall a. Vector a -> Int
V.length Vector UTCTime
ds
dictCardinality (DFloat Vector Float
ds) = Vector Float -> Int
forall a. Vector a -> Int
V.length Vector Float
ds
dictCardinality (DDouble Vector Double
ds) = Vector Double -> Int
forall a. Vector a -> Int
V.length Vector Double
ds
dictCardinality (DText Vector Text
ds) = Vector Text -> Int
forall a. Vector a -> Int
V.length Vector Text
ds

readDictVals :: ParquetType -> BS.ByteString -> Maybe Int32 -> DictVals
readDictVals :: ParquetType -> ByteString -> Maybe Int32 -> DictVals
readDictVals ParquetType
PBOOLEAN ByteString
bs (Just Int32
count) = Vector Bool -> DictVals
DBool ([Bool] -> Vector Bool
forall a. [a] -> Vector a
V.fromList (Int -> [Bool] -> [Bool]
forall a. Int -> [a] -> [a]
take (Int32 -> Int
forall a b. (Integral a, Num b) => a -> b
fromIntegral Int32
count) ([Bool] -> [Bool]) -> [Bool] -> [Bool]
forall a b. (a -> b) -> a -> b
$ ByteString -> [Bool]
readPageBool ByteString
bs))
readDictVals ParquetType
PINT32 ByteString
bs Maybe Int32
_ = Vector Int32 -> DictVals
DInt32 ([Int32] -> Vector Int32
forall a. [a] -> Vector a
V.fromList (ByteString -> [Int32]
readPageInt32 ByteString
bs))
readDictVals ParquetType
PINT64 ByteString
bs Maybe Int32
_ = Vector Int64 -> DictVals
DInt64 ([Int64] -> Vector Int64
forall a. [a] -> Vector a
V.fromList (ByteString -> [Int64]
readPageInt64 ByteString
bs))
readDictVals ParquetType
PINT96 ByteString
bs Maybe Int32
_ = Vector UTCTime -> DictVals
DInt96 ([UTCTime] -> Vector UTCTime
forall a. [a] -> Vector a
V.fromList (ByteString -> [UTCTime]
readPageInt96Times ByteString
bs))
readDictVals ParquetType
PFLOAT ByteString
bs Maybe Int32
_ = Vector Float -> DictVals
DFloat ([Float] -> Vector Float
forall a. [a] -> Vector a
V.fromList (ByteString -> [Float]
readPageFloat ByteString
bs))
readDictVals ParquetType
PDOUBLE ByteString
bs Maybe Int32
_ = Vector Double -> DictVals
DDouble ([Double] -> Vector Double
forall a. [a] -> Vector a
V.fromList (ByteString -> [Double]
readPageWord64 ByteString
bs))
readDictVals ParquetType
PBYTE_ARRAY ByteString
bs Maybe Int32
_ = Vector Text -> DictVals
DText ([Text] -> Vector Text
forall a. [a] -> Vector a
V.fromList (ByteString -> [Text]
readPageBytes ByteString
bs))
readDictVals ParquetType
PFIXED_LEN_BYTE_ARRAY ByteString
bs (Just Int32
len) = Vector Text -> DictVals
DText ([Text] -> Vector Text
forall a. [a] -> Vector a
V.fromList (ByteString -> Int -> [Text]
readPageFixedBytes ByteString
bs (Int32 -> Int
forall a b. (Integral a, Num b) => a -> b
fromIntegral Int32
len)))
readDictVals ParquetType
t ByteString
_ Maybe Int32
_ = [Char] -> DictVals
forall a. HasCallStack => [Char] -> a
error ([Char] -> DictVals) -> [Char] -> DictVals
forall a b. (a -> b) -> a -> b
$ [Char]
"Unsupported dictionary type: " [Char] -> [Char] -> [Char]
forall a. [a] -> [a] -> [a]
++ ParquetType -> [Char]
forall a. Show a => a -> [Char]
show ParquetType
t

readPageInt32 :: BS.ByteString -> [Int32]
readPageInt32 :: ByteString -> [Int32]
readPageInt32 ByteString
xs
    | ByteString -> Bool
BS.null ByteString
xs = []
    | Bool
otherwise = ByteString -> Int32
littleEndianInt32 (Int -> ByteString -> ByteString
BS.take Int
4 ByteString
xs) Int32 -> [Int32] -> [Int32]
forall a. a -> [a] -> [a]
: ByteString -> [Int32]
readPageInt32 (Int -> ByteString -> ByteString
BS.drop Int
4 ByteString
xs)

readPageWord64 :: BS.ByteString -> [Double]
readPageWord64 :: ByteString -> [Double]
readPageWord64 ByteString
xs
    | ByteString -> Bool
BS.null ByteString
xs = []
    | Bool
otherwise =
        Word64 -> Double
castWord64ToDouble (ByteString -> Word64
littleEndianWord64 (Int -> ByteString -> ByteString
BS.take Int
8 ByteString
xs))
            Double -> [Double] -> [Double]
forall a. a -> [a] -> [a]
: ByteString -> [Double]
readPageWord64 (Int -> ByteString -> ByteString
BS.drop Int
8 ByteString
xs)

readPageBytes :: BS.ByteString -> [T.Text]
readPageBytes :: ByteString -> [Text]
readPageBytes ByteString
xs
    | ByteString -> Bool
BS.null ByteString
xs = []
    | Bool
otherwise =
        let lenBytes :: Int
lenBytes = Int32 -> Int
forall a b. (Integral a, Num b) => a -> b
fromIntegral (ByteString -> Int32
littleEndianInt32 (ByteString -> Int32) -> ByteString -> Int32
forall a b. (a -> b) -> a -> b
$ Int -> ByteString -> ByteString
BS.take Int
4 ByteString
xs)
            totalBytesRead :: Int
totalBytesRead = Int
lenBytes Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
4
         in ByteString -> Text
decodeUtf8Lenient (Int -> ByteString -> ByteString
BS.take Int
lenBytes (Int -> ByteString -> ByteString
BS.drop Int
4 ByteString
xs))
                Text -> [Text] -> [Text]
forall a. a -> [a] -> [a]
: ByteString -> [Text]
readPageBytes (Int -> ByteString -> ByteString
BS.drop Int
totalBytesRead ByteString
xs)

readPageBool :: BS.ByteString -> [Bool]
readPageBool :: ByteString -> [Bool]
readPageBool ByteString
bs =
    (Word8 -> [Bool]) -> [Word8] -> [Bool]
forall (t :: * -> *) a b. Foldable t => (a -> [b]) -> t a -> [b]
concatMap (\Word8
b -> (Int -> Bool) -> [Int] -> [Bool]
forall a b. (a -> b) -> [a] -> [b]
map (\Int
i -> (Word8
b Word8 -> Int -> Word8
forall a. Bits a => a -> Int -> a
`shiftR` Int
i) Word8 -> Word8 -> Word8
forall a. Bits a => a -> a -> a
.&. Word8
1 Word8 -> Word8 -> Bool
forall a. Eq a => a -> a -> Bool
== Word8
1) [Int
0 .. Int
7]) (ByteString -> [Word8]
BS.unpack ByteString
bs)

readPageInt64 :: BS.ByteString -> [Int64]
readPageInt64 :: ByteString -> [Int64]
readPageInt64 ByteString
xs
    | ByteString -> Bool
BS.null ByteString
xs = []
    | Bool
otherwise =
        Word64 -> Int64
forall a b. (Integral a, Num b) => a -> b
fromIntegral (ByteString -> Word64
littleEndianWord64 (Int -> ByteString -> ByteString
BS.take Int
8 ByteString
xs)) Int64 -> [Int64] -> [Int64]
forall a. a -> [a] -> [a]
: ByteString -> [Int64]
readPageInt64 (Int -> ByteString -> ByteString
BS.drop Int
8 ByteString
xs)

readPageFloat :: BS.ByteString -> [Float]
readPageFloat :: ByteString -> [Float]
readPageFloat ByteString
xs
    | ByteString -> Bool
BS.null ByteString
xs = []
    | Bool
otherwise =
        Word32 -> Float
castWord32ToFloat (ByteString -> Word32
littleEndianWord32 (Int -> ByteString -> ByteString
BS.take Int
4 ByteString
xs))
            Float -> [Float] -> [Float]
forall a. a -> [a] -> [a]
: ByteString -> [Float]
readPageFloat (Int -> ByteString -> ByteString
BS.drop Int
4 ByteString
xs)

readNInt96Times :: Int -> BS.ByteString -> ([UTCTime], BS.ByteString)
readNInt96Times :: Int -> ByteString -> ([UTCTime], ByteString)
readNInt96Times Int
0 ByteString
bs = ([], ByteString
bs)
readNInt96Times Int
k ByteString
bs =
    let timestamp96 :: ByteString
timestamp96 = Int -> ByteString -> ByteString
BS.take Int
12 ByteString
bs
        utcTime :: UTCTime
utcTime = ByteString -> UTCTime
int96ToUTCTime ByteString
timestamp96
        bs' :: ByteString
bs' = Int -> ByteString -> ByteString
BS.drop Int
12 ByteString
bs
        ([UTCTime]
times, ByteString
rest) = Int -> ByteString -> ([UTCTime], ByteString)
readNInt96Times (Int
k Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
1) ByteString
bs'
     in (UTCTime
utcTime UTCTime -> [UTCTime] -> [UTCTime]
forall a. a -> [a] -> [a]
: [UTCTime]
times, ByteString
rest)

readPageInt96Times :: BS.ByteString -> [UTCTime]
readPageInt96Times :: ByteString -> [UTCTime]
readPageInt96Times ByteString
bs
    | ByteString -> Bool
BS.null ByteString
bs = []
    | Bool
otherwise =
        let ([UTCTime]
times, ByteString
_) = Int -> ByteString -> ([UTCTime], ByteString)
readNInt96Times (ByteString -> Int
BS.length ByteString
bs Int -> Int -> Int
forall a. Integral a => a -> a -> a
`div` Int
12) ByteString
bs
         in [UTCTime]
times

readPageFixedBytes :: BS.ByteString -> Int -> [T.Text]
readPageFixedBytes :: ByteString -> Int -> [Text]
readPageFixedBytes ByteString
xs Int
len
    | ByteString -> Bool
BS.null ByteString
xs = []
    | Bool
otherwise =
        ByteString -> Text
decodeUtf8Lenient (Int -> ByteString -> ByteString
BS.take Int
len ByteString
xs) Text -> [Text] -> [Text]
forall a. a -> [a] -> [a]
: ByteString -> Int -> [Text]
readPageFixedBytes (Int -> ByteString -> ByteString
BS.drop Int
len ByteString
xs) Int
len

{- | Dispatch to the right multi-level list stitching function.
For maxRep=1 uses stitchList; for 2/3 uses stitchList2/3 with computed thresholds.
Threshold formula: defT_r = maxDef - 2*(maxRep - r).
-}
stitchForRepBool :: Int -> Int -> [Int] -> [Int] -> [Bool] -> DI.Column
stitchForRepBool :: Int -> Int -> [Int] -> [Int] -> [Bool] -> Column
stitchForRepBool Int
maxRep Int
maxDef [Int]
rep [Int]
def [Bool]
vals = case Int
maxRep of
    Int
2 -> [Maybe [Maybe [Maybe Bool]]] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList (Int
-> Int -> [Int] -> [Int] -> [Bool] -> [Maybe [Maybe [Maybe Bool]]]
forall a.
Int -> Int -> [Int] -> [Int] -> [a] -> [Maybe [Maybe [Maybe a]]]
stitchList2 (Int
maxDef Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2) Int
maxDef [Int]
rep [Int]
def [Bool]
vals)
    Int
3 -> [Maybe [Maybe [Maybe [Maybe Bool]]]] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList (Int
-> Int
-> Int
-> [Int]
-> [Int]
-> [Bool]
-> [Maybe [Maybe [Maybe [Maybe Bool]]]]
forall a.
Int
-> Int
-> Int
-> [Int]
-> [Int]
-> [a]
-> [Maybe [Maybe [Maybe [Maybe a]]]]
stitchList3 (Int
maxDef Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
4) (Int
maxDef Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2) Int
maxDef [Int]
rep [Int]
def [Bool]
vals)
    Int
_ -> [Maybe [Maybe Bool]] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList (Int -> [Int] -> [Int] -> [Bool] -> [Maybe [Maybe Bool]]
forall a. Int -> [Int] -> [Int] -> [a] -> [Maybe [Maybe a]]
stitchList Int
maxDef [Int]
rep [Int]
def [Bool]
vals)

stitchForRepInt32 :: Int -> Int -> [Int] -> [Int] -> [Int32] -> DI.Column
stitchForRepInt32 :: Int -> Int -> [Int] -> [Int] -> [Int32] -> Column
stitchForRepInt32 Int
maxRep Int
maxDef [Int]
rep [Int]
def [Int32]
vals = case Int
maxRep of
    Int
2 -> [Maybe [Maybe [Maybe Int32]]] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList (Int
-> Int
-> [Int]
-> [Int]
-> [Int32]
-> [Maybe [Maybe [Maybe Int32]]]
forall a.
Int -> Int -> [Int] -> [Int] -> [a] -> [Maybe [Maybe [Maybe a]]]
stitchList2 (Int
maxDef Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2) Int
maxDef [Int]
rep [Int]
def [Int32]
vals)
    Int
3 -> [Maybe [Maybe [Maybe [Maybe Int32]]]] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList (Int
-> Int
-> Int
-> [Int]
-> [Int]
-> [Int32]
-> [Maybe [Maybe [Maybe [Maybe Int32]]]]
forall a.
Int
-> Int
-> Int
-> [Int]
-> [Int]
-> [a]
-> [Maybe [Maybe [Maybe [Maybe a]]]]
stitchList3 (Int
maxDef Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
4) (Int
maxDef Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2) Int
maxDef [Int]
rep [Int]
def [Int32]
vals)
    Int
_ -> [Maybe [Maybe Int32]] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList (Int -> [Int] -> [Int] -> [Int32] -> [Maybe [Maybe Int32]]
forall a. Int -> [Int] -> [Int] -> [a] -> [Maybe [Maybe a]]
stitchList Int
maxDef [Int]
rep [Int]
def [Int32]
vals)

stitchForRepInt64 :: Int -> Int -> [Int] -> [Int] -> [Int64] -> DI.Column
stitchForRepInt64 :: Int -> Int -> [Int] -> [Int] -> [Int64] -> Column
stitchForRepInt64 Int
maxRep Int
maxDef [Int]
rep [Int]
def [Int64]
vals = case Int
maxRep of
    Int
2 -> [Maybe [Maybe [Maybe Int64]]] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList (Int
-> Int
-> [Int]
-> [Int]
-> [Int64]
-> [Maybe [Maybe [Maybe Int64]]]
forall a.
Int -> Int -> [Int] -> [Int] -> [a] -> [Maybe [Maybe [Maybe a]]]
stitchList2 (Int
maxDef Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2) Int
maxDef [Int]
rep [Int]
def [Int64]
vals)
    Int
3 -> [Maybe [Maybe [Maybe [Maybe Int64]]]] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList (Int
-> Int
-> Int
-> [Int]
-> [Int]
-> [Int64]
-> [Maybe [Maybe [Maybe [Maybe Int64]]]]
forall a.
Int
-> Int
-> Int
-> [Int]
-> [Int]
-> [a]
-> [Maybe [Maybe [Maybe [Maybe a]]]]
stitchList3 (Int
maxDef Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
4) (Int
maxDef Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2) Int
maxDef [Int]
rep [Int]
def [Int64]
vals)
    Int
_ -> [Maybe [Maybe Int64]] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList (Int -> [Int] -> [Int] -> [Int64] -> [Maybe [Maybe Int64]]
forall a. Int -> [Int] -> [Int] -> [a] -> [Maybe [Maybe a]]
stitchList Int
maxDef [Int]
rep [Int]
def [Int64]
vals)

stitchForRepUTCTime :: Int -> Int -> [Int] -> [Int] -> [UTCTime] -> DI.Column
stitchForRepUTCTime :: Int -> Int -> [Int] -> [Int] -> [UTCTime] -> Column
stitchForRepUTCTime Int
maxRep Int
maxDef [Int]
rep [Int]
def [UTCTime]
vals = case Int
maxRep of
    Int
2 -> [Maybe [Maybe [Maybe UTCTime]]] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList (Int
-> Int
-> [Int]
-> [Int]
-> [UTCTime]
-> [Maybe [Maybe [Maybe UTCTime]]]
forall a.
Int -> Int -> [Int] -> [Int] -> [a] -> [Maybe [Maybe [Maybe a]]]
stitchList2 (Int
maxDef Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2) Int
maxDef [Int]
rep [Int]
def [UTCTime]
vals)
    Int
3 -> [Maybe [Maybe [Maybe [Maybe UTCTime]]]] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList (Int
-> Int
-> Int
-> [Int]
-> [Int]
-> [UTCTime]
-> [Maybe [Maybe [Maybe [Maybe UTCTime]]]]
forall a.
Int
-> Int
-> Int
-> [Int]
-> [Int]
-> [a]
-> [Maybe [Maybe [Maybe [Maybe a]]]]
stitchList3 (Int
maxDef Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
4) (Int
maxDef Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2) Int
maxDef [Int]
rep [Int]
def [UTCTime]
vals)
    Int
_ -> [Maybe [Maybe UTCTime]] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList (Int -> [Int] -> [Int] -> [UTCTime] -> [Maybe [Maybe UTCTime]]
forall a. Int -> [Int] -> [Int] -> [a] -> [Maybe [Maybe a]]
stitchList Int
maxDef [Int]
rep [Int]
def [UTCTime]
vals)

stitchForRepFloat :: Int -> Int -> [Int] -> [Int] -> [Float] -> DI.Column
stitchForRepFloat :: Int -> Int -> [Int] -> [Int] -> [Float] -> Column
stitchForRepFloat Int
maxRep Int
maxDef [Int]
rep [Int]
def [Float]
vals = case Int
maxRep of
    Int
2 -> [Maybe [Maybe [Maybe Float]]] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList (Int
-> Int
-> [Int]
-> [Int]
-> [Float]
-> [Maybe [Maybe [Maybe Float]]]
forall a.
Int -> Int -> [Int] -> [Int] -> [a] -> [Maybe [Maybe [Maybe a]]]
stitchList2 (Int
maxDef Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2) Int
maxDef [Int]
rep [Int]
def [Float]
vals)
    Int
3 -> [Maybe [Maybe [Maybe [Maybe Float]]]] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList (Int
-> Int
-> Int
-> [Int]
-> [Int]
-> [Float]
-> [Maybe [Maybe [Maybe [Maybe Float]]]]
forall a.
Int
-> Int
-> Int
-> [Int]
-> [Int]
-> [a]
-> [Maybe [Maybe [Maybe [Maybe a]]]]
stitchList3 (Int
maxDef Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
4) (Int
maxDef Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2) Int
maxDef [Int]
rep [Int]
def [Float]
vals)
    Int
_ -> [Maybe [Maybe Float]] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList (Int -> [Int] -> [Int] -> [Float] -> [Maybe [Maybe Float]]
forall a. Int -> [Int] -> [Int] -> [a] -> [Maybe [Maybe a]]
stitchList Int
maxDef [Int]
rep [Int]
def [Float]
vals)

stitchForRepDouble :: Int -> Int -> [Int] -> [Int] -> [Double] -> DI.Column
stitchForRepDouble :: Int -> Int -> [Int] -> [Int] -> [Double] -> Column
stitchForRepDouble Int
maxRep Int
maxDef [Int]
rep [Int]
def [Double]
vals = case Int
maxRep of
    Int
2 -> [Maybe [Maybe [Maybe Double]]] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList (Int
-> Int
-> [Int]
-> [Int]
-> [Double]
-> [Maybe [Maybe [Maybe Double]]]
forall a.
Int -> Int -> [Int] -> [Int] -> [a] -> [Maybe [Maybe [Maybe a]]]
stitchList2 (Int
maxDef Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2) Int
maxDef [Int]
rep [Int]
def [Double]
vals)
    Int
3 -> [Maybe [Maybe [Maybe [Maybe Double]]]] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList (Int
-> Int
-> Int
-> [Int]
-> [Int]
-> [Double]
-> [Maybe [Maybe [Maybe [Maybe Double]]]]
forall a.
Int
-> Int
-> Int
-> [Int]
-> [Int]
-> [a]
-> [Maybe [Maybe [Maybe [Maybe a]]]]
stitchList3 (Int
maxDef Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
4) (Int
maxDef Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2) Int
maxDef [Int]
rep [Int]
def [Double]
vals)
    Int
_ -> [Maybe [Maybe Double]] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList (Int -> [Int] -> [Int] -> [Double] -> [Maybe [Maybe Double]]
forall a. Int -> [Int] -> [Int] -> [a] -> [Maybe [Maybe a]]
stitchList Int
maxDef [Int]
rep [Int]
def [Double]
vals)

stitchForRepText :: Int -> Int -> [Int] -> [Int] -> [T.Text] -> DI.Column
stitchForRepText :: Int -> Int -> [Int] -> [Int] -> [Text] -> Column
stitchForRepText Int
maxRep Int
maxDef [Int]
rep [Int]
def [Text]
vals = case Int
maxRep of
    Int
2 -> [Maybe [Maybe [Maybe Text]]] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList (Int
-> Int -> [Int] -> [Int] -> [Text] -> [Maybe [Maybe [Maybe Text]]]
forall a.
Int -> Int -> [Int] -> [Int] -> [a] -> [Maybe [Maybe [Maybe a]]]
stitchList2 (Int
maxDef Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2) Int
maxDef [Int]
rep [Int]
def [Text]
vals)
    Int
3 -> [Maybe [Maybe [Maybe [Maybe Text]]]] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList (Int
-> Int
-> Int
-> [Int]
-> [Int]
-> [Text]
-> [Maybe [Maybe [Maybe [Maybe Text]]]]
forall a.
Int
-> Int
-> Int
-> [Int]
-> [Int]
-> [a]
-> [Maybe [Maybe [Maybe [Maybe a]]]]
stitchList3 (Int
maxDef Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
4) (Int
maxDef Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
2) Int
maxDef [Int]
rep [Int]
def [Text]
vals)
    Int
_ -> [Maybe [Maybe Text]] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList (Int -> [Int] -> [Int] -> [Text] -> [Maybe [Maybe Text]]
forall a. Int -> [Int] -> [Int] -> [a] -> [Maybe [Maybe a]]
stitchList Int
maxDef [Int]
rep [Int]
def [Text]
vals)

{- | Build a Column from a dictionary + index vector + def levels in a single
mutable-vector pass, avoiding the intermediate [a] and [Maybe a] lists.
For maxRep > 0 (list columns) the caller must use the rep-stitching path instead.
-}
applyDictToColumn ::
    (DI.Columnable a, DI.Columnable (Maybe a)) =>
    V.Vector a ->
    VU.Vector Int ->
    Int -> -- maxDef
    [Int] -> -- defLvls
    IO DI.Column
applyDictToColumn :: forall a.
(Columnable a, Columnable (Maybe a)) =>
Vector a -> Vector Int -> Int -> [Int] -> IO Column
applyDictToColumn Vector a
dict Vector Int
idxs Int
maxDef [Int]
defLvls
    | Int
maxDef Int -> Int -> Bool
forall a. Eq a => a -> a -> Bool
== Int
0 = do
        -- All rows are required; no nullability to check.
        let n :: Int
n = Vector Int -> Int
forall a. Unbox a => Vector a -> Int
VU.length Vector Int
idxs
        Column -> IO Column
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Column -> IO Column) -> Column -> IO Column
forall a b. (a -> b) -> a -> b
$ Vector a -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
Vector a -> Column
DI.fromVector (Int -> (Int -> a) -> Vector a
forall a. Int -> (Int -> a) -> Vector a
V.generate Int
n (\Int
i -> Vector a
dict Vector a -> Int -> a
forall a. Vector a -> Int -> a
V.! (Vector Int
idxs Vector Int -> Int -> Int
forall a. Unbox a => Vector a -> Int -> a
VU.! Int
i)))
    | Bool
otherwise = do
        let n :: Int
n = [Int] -> Int
forall a. [a] -> Int
forall (t :: * -> *) a. Foldable t => t a -> Int
length [Int]
defLvls
        MVector RealWorld (Maybe a)
mv <- Int -> IO (MVector (PrimState IO) (Maybe a))
forall (m :: * -> *) a.
PrimMonad m =>
Int -> m (MVector (PrimState m) a)
VM.new Int
n
        IORef Bool
hasNullRef <- Bool -> IO (IORef Bool)
forall a. a -> IO (IORef a)
newIORef Bool
False
        let go :: Int -> Int -> [Int] -> IO ()
go Int
_ Int
_ [] = () -> IO ()
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure ()
            go !Int
i !Int
j (Int
d : [Int]
ds)
                | Int
d Int -> Int -> Bool
forall a. Eq a => a -> a -> Bool
== Int
maxDef = do
                    MVector (PrimState IO) (Maybe a) -> Int -> Maybe a -> IO ()
forall (m :: * -> *) a.
PrimMonad m =>
MVector (PrimState m) a -> Int -> a -> m ()
VM.write MVector RealWorld (Maybe a)
MVector (PrimState IO) (Maybe a)
mv Int
i (a -> Maybe a
forall a. a -> Maybe a
Just (Vector a
dict Vector a -> Int -> a
forall a. Vector a -> Int -> a
V.! (Vector Int
idxs Vector Int -> Int -> Int
forall a. Unbox a => Vector a -> Int -> a
VU.! Int
j)))
                    Int -> Int -> [Int] -> IO ()
go (Int
i Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1) (Int
j Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1) [Int]
ds
                | Bool
otherwise = do
                    IORef Bool -> Bool -> IO ()
forall a. IORef a -> a -> IO ()
writeIORef IORef Bool
hasNullRef Bool
True
                    MVector (PrimState IO) (Maybe a) -> Int -> Maybe a -> IO ()
forall (m :: * -> *) a.
PrimMonad m =>
MVector (PrimState m) a -> Int -> a -> m ()
VM.write MVector RealWorld (Maybe a)
MVector (PrimState IO) (Maybe a)
mv Int
i Maybe a
forall a. Maybe a
Nothing
                    Int -> Int -> [Int] -> IO ()
go (Int
i Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1) Int
j [Int]
ds
        Int -> Int -> [Int] -> IO ()
go Int
0 Int
0 [Int]
defLvls
        Vector (Maybe a)
vec <- MVector (PrimState IO) (Maybe a) -> IO (Vector (Maybe a))
forall (m :: * -> *) a.
PrimMonad m =>
MVector (PrimState m) a -> m (Vector a)
V.freeze MVector RealWorld (Maybe a)
MVector (PrimState IO) (Maybe a)
mv
        Bool
hasNull <- IORef Bool -> IO Bool
forall a. IORef a -> IO a
readIORef IORef Bool
hasNullRef
        Column -> IO Column
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Column -> IO Column) -> Column -> IO Column
forall a b. (a -> b) -> a -> b
$
            if Bool
hasNull
                then Vector (Maybe a) -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
Vector a -> Column
DI.fromVector Vector (Maybe a)
vec -- VB.Vector (Maybe a) → OptionalColumn
                else Vector a -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
Vector a -> Column
DI.fromVector ((Maybe a -> a) -> Vector (Maybe a) -> Vector a
forall a b. (a -> b) -> Vector a -> Vector b
V.map Maybe a -> a
forall a. HasCallStack => Maybe a -> a
fromJust Vector (Maybe a)
vec) -- VB.Vector a → BoxedColumn/UnboxedColumn

decodeDictV1 ::
    Maybe DictVals ->
    Int ->
    Int ->
    [Int] ->
    [Int] ->
    Int ->
    BS.ByteString ->
    IO DI.Column
decodeDictV1 :: Maybe DictVals
-> Int -> Int -> [Int] -> [Int] -> Int -> ByteString -> IO Column
decodeDictV1 Maybe DictVals
dictValsM Int
maxDef Int
maxRep [Int]
repLvls [Int]
defLvls Int
nPresent ByteString
bytes =
    case Maybe DictVals
dictValsM of
        Maybe DictVals
Nothing -> [Char] -> IO Column
forall a. HasCallStack => [Char] -> a
error [Char]
"Dictionary-encoded page but dictionary is missing"
        Just DictVals
dictVals ->
            let (Vector Int
idxs, ByteString
_rest) = Int -> Int -> ByteString -> (Vector Int, ByteString)
decodeDictIndicesV1 Int
nPresent (DictVals -> Int
dictCardinality DictVals
dictVals) ByteString
bytes
             in do
                    Bool -> IO () -> IO ()
forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
when (Vector Int -> Int
forall a. Unbox a => Vector a -> Int
VU.length Vector Int
idxs Int -> Int -> Bool
forall a. Eq a => a -> a -> Bool
/= Int
nPresent) (IO () -> IO ()) -> IO () -> IO ()
forall a b. (a -> b) -> a -> b
$
                        [Char] -> IO ()
forall a. HasCallStack => [Char] -> a
error ([Char] -> IO ()) -> [Char] -> IO ()
forall a b. (a -> b) -> a -> b
$
                            [Char]
"dict index count mismatch: got "
                                [Char] -> [Char] -> [Char]
forall a. [a] -> [a] -> [a]
++ Int -> [Char]
forall a. Show a => a -> [Char]
show (Vector Int -> Int
forall a. Unbox a => Vector a -> Int
VU.length Vector Int
idxs)
                                [Char] -> [Char] -> [Char]
forall a. [a] -> [a] -> [a]
++ [Char]
", expected "
                                [Char] -> [Char] -> [Char]
forall a. [a] -> [a] -> [a]
++ Int -> [Char]
forall a. Show a => a -> [Char]
show Int
nPresent
                    if Int
maxRep Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
> Int
0
                        then do
                            case DictVals
dictVals of
                                DBool Vector Bool
ds ->
                                    Column -> IO Column
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Column -> IO Column) -> Column -> IO Column
forall a b. (a -> b) -> a -> b
$
                                        Int -> Int -> [Int] -> [Int] -> [Bool] -> Column
stitchForRepBool Int
maxRep Int
maxDef [Int]
repLvls [Int]
defLvls ((Int -> Bool) -> [Int] -> [Bool]
forall a b. (a -> b) -> [a] -> [b]
map (Vector Bool
ds Vector Bool -> Int -> Bool
forall a. Vector a -> Int -> a
V.!) (Vector Int -> [Int]
forall a. Unbox a => Vector a -> [a]
VU.toList Vector Int
idxs))
                                DInt32 Vector Int32
ds ->
                                    Column -> IO Column
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Column -> IO Column) -> Column -> IO Column
forall a b. (a -> b) -> a -> b
$
                                        Int -> Int -> [Int] -> [Int] -> [Int32] -> Column
stitchForRepInt32 Int
maxRep Int
maxDef [Int]
repLvls [Int]
defLvls ((Int -> Int32) -> [Int] -> [Int32]
forall a b. (a -> b) -> [a] -> [b]
map (Vector Int32
ds Vector Int32 -> Int -> Int32
forall a. Vector a -> Int -> a
V.!) (Vector Int -> [Int]
forall a. Unbox a => Vector a -> [a]
VU.toList Vector Int
idxs))
                                DInt64 Vector Int64
ds ->
                                    Column -> IO Column
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Column -> IO Column) -> Column -> IO Column
forall a b. (a -> b) -> a -> b
$
                                        Int -> Int -> [Int] -> [Int] -> [Int64] -> Column
stitchForRepInt64 Int
maxRep Int
maxDef [Int]
repLvls [Int]
defLvls ((Int -> Int64) -> [Int] -> [Int64]
forall a b. (a -> b) -> [a] -> [b]
map (Vector Int64
ds Vector Int64 -> Int -> Int64
forall a. Vector a -> Int -> a
V.!) (Vector Int -> [Int]
forall a. Unbox a => Vector a -> [a]
VU.toList Vector Int
idxs))
                                DInt96 Vector UTCTime
ds ->
                                    Column -> IO Column
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Column -> IO Column) -> Column -> IO Column
forall a b. (a -> b) -> a -> b
$
                                        Int -> Int -> [Int] -> [Int] -> [UTCTime] -> Column
stitchForRepUTCTime
                                            Int
maxRep
                                            Int
maxDef
                                            [Int]
repLvls
                                            [Int]
defLvls
                                            ((Int -> UTCTime) -> [Int] -> [UTCTime]
forall a b. (a -> b) -> [a] -> [b]
map (Vector UTCTime
ds Vector UTCTime -> Int -> UTCTime
forall a. Vector a -> Int -> a
V.!) (Vector Int -> [Int]
forall a. Unbox a => Vector a -> [a]
VU.toList Vector Int
idxs))
                                DFloat Vector Float
ds ->
                                    Column -> IO Column
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Column -> IO Column) -> Column -> IO Column
forall a b. (a -> b) -> a -> b
$
                                        Int -> Int -> [Int] -> [Int] -> [Float] -> Column
stitchForRepFloat Int
maxRep Int
maxDef [Int]
repLvls [Int]
defLvls ((Int -> Float) -> [Int] -> [Float]
forall a b. (a -> b) -> [a] -> [b]
map (Vector Float
ds Vector Float -> Int -> Float
forall a. Vector a -> Int -> a
V.!) (Vector Int -> [Int]
forall a. Unbox a => Vector a -> [a]
VU.toList Vector Int
idxs))
                                DDouble Vector Double
ds ->
                                    Column -> IO Column
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Column -> IO Column) -> Column -> IO Column
forall a b. (a -> b) -> a -> b
$
                                        Int -> Int -> [Int] -> [Int] -> [Double] -> Column
stitchForRepDouble Int
maxRep Int
maxDef [Int]
repLvls [Int]
defLvls ((Int -> Double) -> [Int] -> [Double]
forall a b. (a -> b) -> [a] -> [b]
map (Vector Double
ds Vector Double -> Int -> Double
forall a. Vector a -> Int -> a
V.!) (Vector Int -> [Int]
forall a. Unbox a => Vector a -> [a]
VU.toList Vector Int
idxs))
                                DText Vector Text
ds ->
                                    Column -> IO Column
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Column -> IO Column) -> Column -> IO Column
forall a b. (a -> b) -> a -> b
$
                                        Int -> Int -> [Int] -> [Int] -> [Text] -> Column
stitchForRepText Int
maxRep Int
maxDef [Int]
repLvls [Int]
defLvls ((Int -> Text) -> [Int] -> [Text]
forall a b. (a -> b) -> [a] -> [b]
map (Vector Text
ds Vector Text -> Int -> Text
forall a. Vector a -> Int -> a
V.!) (Vector Int -> [Int]
forall a. Unbox a => Vector a -> [a]
VU.toList Vector Int
idxs))
                        else case DictVals
dictVals of
                            -- Fast path: unboxable types, no nulls — one allocation via VU.map
                            DInt32 Vector Int32
ds | Int
maxDef Int -> Int -> Bool
forall a. Eq a => a -> a -> Bool
== Int
0 -> Column -> IO Column
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Column -> IO Column) -> Column -> IO Column
forall a b. (a -> b) -> a -> b
$ Vector Int32 -> Column
forall a. (Columnable a, Unbox a) => Vector a -> Column
DI.fromUnboxedVector ((Int -> Int32) -> Vector Int -> Vector Int32
forall a b. (Unbox a, Unbox b) => (a -> b) -> Vector a -> Vector b
VU.map (Vector Int32
ds Vector Int32 -> Int -> Int32
forall a. Vector a -> Int -> a
V.!) Vector Int
idxs)
                            DInt64 Vector Int64
ds | Int
maxDef Int -> Int -> Bool
forall a. Eq a => a -> a -> Bool
== Int
0 -> Column -> IO Column
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Column -> IO Column) -> Column -> IO Column
forall a b. (a -> b) -> a -> b
$ Vector Int64 -> Column
forall a. (Columnable a, Unbox a) => Vector a -> Column
DI.fromUnboxedVector ((Int -> Int64) -> Vector Int -> Vector Int64
forall a b. (Unbox a, Unbox b) => (a -> b) -> Vector a -> Vector b
VU.map (Vector Int64
ds Vector Int64 -> Int -> Int64
forall a. Vector a -> Int -> a
V.!) Vector Int
idxs)
                            DFloat Vector Float
ds | Int
maxDef Int -> Int -> Bool
forall a. Eq a => a -> a -> Bool
== Int
0 -> Column -> IO Column
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Column -> IO Column) -> Column -> IO Column
forall a b. (a -> b) -> a -> b
$ Vector Float -> Column
forall a. (Columnable a, Unbox a) => Vector a -> Column
DI.fromUnboxedVector ((Int -> Float) -> Vector Int -> Vector Float
forall a b. (Unbox a, Unbox b) => (a -> b) -> Vector a -> Vector b
VU.map (Vector Float
ds Vector Float -> Int -> Float
forall a. Vector a -> Int -> a
V.!) Vector Int
idxs)
                            DDouble Vector Double
ds | Int
maxDef Int -> Int -> Bool
forall a. Eq a => a -> a -> Bool
== Int
0 -> Column -> IO Column
forall a. a -> IO a
forall (f :: * -> *) a. Applicative f => a -> f a
pure (Column -> IO Column) -> Column -> IO Column
forall a b. (a -> b) -> a -> b
$ Vector Double -> Column
forall a. (Columnable a, Unbox a) => Vector a -> Column
DI.fromUnboxedVector ((Int -> Double) -> Vector Int -> Vector Double
forall a b. (Unbox a, Unbox b) => (a -> b) -> Vector a -> Vector b
VU.map (Vector Double
ds Vector Double -> Int -> Double
forall a. Vector a -> Int -> a
V.!) Vector Int
idxs)
                            DBool Vector Bool
ds -> Vector Bool -> Vector Int -> Int -> [Int] -> IO Column
forall a.
(Columnable a, Columnable (Maybe a)) =>
Vector a -> Vector Int -> Int -> [Int] -> IO Column
applyDictToColumn Vector Bool
ds Vector Int
idxs Int
maxDef [Int]
defLvls
                            DInt32 Vector Int32
ds -> Vector Int32 -> Vector Int -> Int -> [Int] -> IO Column
forall a.
(Columnable a, Columnable (Maybe a)) =>
Vector a -> Vector Int -> Int -> [Int] -> IO Column
applyDictToColumn Vector Int32
ds Vector Int
idxs Int
maxDef [Int]
defLvls
                            DInt64 Vector Int64
ds -> Vector Int64 -> Vector Int -> Int -> [Int] -> IO Column
forall a.
(Columnable a, Columnable (Maybe a)) =>
Vector a -> Vector Int -> Int -> [Int] -> IO Column
applyDictToColumn Vector Int64
ds Vector Int
idxs Int
maxDef [Int]
defLvls
                            DInt96 Vector UTCTime
ds -> Vector UTCTime -> Vector Int -> Int -> [Int] -> IO Column
forall a.
(Columnable a, Columnable (Maybe a)) =>
Vector a -> Vector Int -> Int -> [Int] -> IO Column
applyDictToColumn Vector UTCTime
ds Vector Int
idxs Int
maxDef [Int]
defLvls
                            DFloat Vector Float
ds -> Vector Float -> Vector Int -> Int -> [Int] -> IO Column
forall a.
(Columnable a, Columnable (Maybe a)) =>
Vector a -> Vector Int -> Int -> [Int] -> IO Column
applyDictToColumn Vector Float
ds Vector Int
idxs Int
maxDef [Int]
defLvls
                            DDouble Vector Double
ds -> Vector Double -> Vector Int -> Int -> [Int] -> IO Column
forall a.
(Columnable a, Columnable (Maybe a)) =>
Vector a -> Vector Int -> Int -> [Int] -> IO Column
applyDictToColumn Vector Double
ds Vector Int
idxs Int
maxDef [Int]
defLvls
                            DText Vector Text
ds -> Vector Text -> Vector Int -> Int -> [Int] -> IO Column
forall a.
(Columnable a, Columnable (Maybe a)) =>
Vector a -> Vector Int -> Int -> [Int] -> IO Column
applyDictToColumn Vector Text
ds Vector Int
idxs Int
maxDef [Int]
defLvls

toMaybeInt32 :: Int -> [Int] -> [Int32] -> DI.Column
toMaybeInt32 :: Int -> [Int] -> [Int32] -> Column
toMaybeInt32 Int
maxDef [Int]
def [Int32]
xs =
    let filled :: [Maybe Int32]
filled = Int -> [Int] -> [Int32] -> [Maybe Int32]
forall a. Int -> [Int] -> [a] -> [Maybe a]
stitchNullable Int
maxDef [Int]
def [Int32]
xs
     in if (Maybe Int32 -> Bool) -> [Maybe Int32] -> Bool
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Bool
all Maybe Int32 -> Bool
forall a. Maybe a -> Bool
isJust [Maybe Int32]
filled
            then [Int32] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList ((Maybe Int32 -> Int32) -> [Maybe Int32] -> [Int32]
forall a b. (a -> b) -> [a] -> [b]
map (Int32 -> Maybe Int32 -> Int32
forall a. a -> Maybe a -> a
fromMaybe Int32
0) [Maybe Int32]
filled)
            else [Maybe Int32] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList [Maybe Int32]
filled

toMaybeDouble :: Int -> [Int] -> [Double] -> DI.Column
toMaybeDouble :: Int -> [Int] -> [Double] -> Column
toMaybeDouble Int
maxDef [Int]
def [Double]
xs =
    let filled :: [Maybe Double]
filled = Int -> [Int] -> [Double] -> [Maybe Double]
forall a. Int -> [Int] -> [a] -> [Maybe a]
stitchNullable Int
maxDef [Int]
def [Double]
xs
     in if (Maybe Double -> Bool) -> [Maybe Double] -> Bool
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Bool
all Maybe Double -> Bool
forall a. Maybe a -> Bool
isJust [Maybe Double]
filled
            then [Double] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList ((Maybe Double -> Double) -> [Maybe Double] -> [Double]
forall a b. (a -> b) -> [a] -> [b]
map (Double -> Maybe Double -> Double
forall a. a -> Maybe a -> a
fromMaybe Double
0) [Maybe Double]
filled)
            else [Maybe Double] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList [Maybe Double]
filled

toMaybeText :: Int -> [Int] -> [T.Text] -> DI.Column
toMaybeText :: Int -> [Int] -> [Text] -> Column
toMaybeText Int
maxDef [Int]
def [Text]
xs =
    let filled :: [Maybe Text]
filled = Int -> [Int] -> [Text] -> [Maybe Text]
forall a. Int -> [Int] -> [a] -> [Maybe a]
stitchNullable Int
maxDef [Int]
def [Text]
xs
     in if (Maybe Text -> Bool) -> [Maybe Text] -> Bool
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Bool
all Maybe Text -> Bool
forall a. Maybe a -> Bool
isJust [Maybe Text]
filled
            then [Text] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList ((Maybe Text -> Text) -> [Maybe Text] -> [Text]
forall a b. (a -> b) -> [a] -> [b]
map (Text -> Maybe Text -> Text
forall a. a -> Maybe a -> a
fromMaybe Text
"") [Maybe Text]
filled)
            else [Maybe Text] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList [Maybe Text]
filled

toMaybeBool :: Int -> [Int] -> [Bool] -> DI.Column
toMaybeBool :: Int -> [Int] -> [Bool] -> Column
toMaybeBool Int
maxDef [Int]
def [Bool]
xs =
    let filled :: [Maybe Bool]
filled = Int -> [Int] -> [Bool] -> [Maybe Bool]
forall a. Int -> [Int] -> [a] -> [Maybe a]
stitchNullable Int
maxDef [Int]
def [Bool]
xs
     in if (Maybe Bool -> Bool) -> [Maybe Bool] -> Bool
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Bool
all Maybe Bool -> Bool
forall a. Maybe a -> Bool
isJust [Maybe Bool]
filled
            then [Bool] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList ((Maybe Bool -> Bool) -> [Maybe Bool] -> [Bool]
forall a b. (a -> b) -> [a] -> [b]
map (Bool -> Maybe Bool -> Bool
forall a. a -> Maybe a -> a
fromMaybe Bool
False) [Maybe Bool]
filled)
            else [Maybe Bool] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList [Maybe Bool]
filled

toMaybeInt64 :: Int -> [Int] -> [Int64] -> DI.Column
toMaybeInt64 :: Int -> [Int] -> [Int64] -> Column
toMaybeInt64 Int
maxDef [Int]
def [Int64]
xs =
    let filled :: [Maybe Int64]
filled = Int -> [Int] -> [Int64] -> [Maybe Int64]
forall a. Int -> [Int] -> [a] -> [Maybe a]
stitchNullable Int
maxDef [Int]
def [Int64]
xs
     in if (Maybe Int64 -> Bool) -> [Maybe Int64] -> Bool
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Bool
all Maybe Int64 -> Bool
forall a. Maybe a -> Bool
isJust [Maybe Int64]
filled
            then [Int64] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList ((Maybe Int64 -> Int64) -> [Maybe Int64] -> [Int64]
forall a b. (a -> b) -> [a] -> [b]
map (Int64 -> Maybe Int64 -> Int64
forall a. a -> Maybe a -> a
fromMaybe Int64
0) [Maybe Int64]
filled)
            else [Maybe Int64] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList [Maybe Int64]
filled

toMaybeFloat :: Int -> [Int] -> [Float] -> DI.Column
toMaybeFloat :: Int -> [Int] -> [Float] -> Column
toMaybeFloat Int
maxDef [Int]
def [Float]
xs =
    let filled :: [Maybe Float]
filled = Int -> [Int] -> [Float] -> [Maybe Float]
forall a. Int -> [Int] -> [a] -> [Maybe a]
stitchNullable Int
maxDef [Int]
def [Float]
xs
     in if (Maybe Float -> Bool) -> [Maybe Float] -> Bool
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Bool
all Maybe Float -> Bool
forall a. Maybe a -> Bool
isJust [Maybe Float]
filled
            then [Float] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList ((Maybe Float -> Float) -> [Maybe Float] -> [Float]
forall a b. (a -> b) -> [a] -> [b]
map (Float -> Maybe Float -> Float
forall a. a -> Maybe a -> a
fromMaybe Float
0.0) [Maybe Float]
filled)
            else [Maybe Float] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList [Maybe Float]
filled

toMaybeUTCTime :: Int -> [Int] -> [UTCTime] -> DI.Column
toMaybeUTCTime :: Int -> [Int] -> [UTCTime] -> Column
toMaybeUTCTime Int
maxDef [Int]
def [UTCTime]
times =
    let filled :: [Maybe UTCTime]
filled = Int -> [Int] -> [UTCTime] -> [Maybe UTCTime]
forall a. Int -> [Int] -> [a] -> [Maybe a]
stitchNullable Int
maxDef [Int]
def [UTCTime]
times
        defaultTime :: UTCTime
defaultTime = Day -> DiffTime -> UTCTime
UTCTime (Year -> Int -> Int -> Day
fromGregorian Year
1970 Int
1 Int
1) (Year -> DiffTime
secondsToDiffTime Year
0)
     in if (Maybe UTCTime -> Bool) -> [Maybe UTCTime] -> Bool
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Bool
all Maybe UTCTime -> Bool
forall a. Maybe a -> Bool
isJust [Maybe UTCTime]
filled
            then [UTCTime] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList ((Maybe UTCTime -> UTCTime) -> [Maybe UTCTime] -> [UTCTime]
forall a b. (a -> b) -> [a] -> [b]
map (UTCTime -> Maybe UTCTime -> UTCTime
forall a. a -> Maybe a -> a
fromMaybe UTCTime
defaultTime) [Maybe UTCTime]
filled)
            else [Maybe UTCTime] -> Column
forall a.
(Columnable a, ColumnifyRep (KindOf a) a) =>
[a] -> Column
DI.fromList [Maybe UTCTime]
filled