{-# LANGUAGE BangPatterns #-}
{-# LANGUAGE ScopedTypeVariables #-}
module DataFrame.Operations.Inference (
DateFormat,
ParsingAssumption (..),
makeParsingAssumption,
makeParsingAssumptionBytes,
byteStringDateParser,
parseTimeOpt,
promoteIntColumn,
promoteIntColumnIndexed,
readIntStrict,
) where
import qualified Data.ByteString as BS
import qualified Data.Text as T
import qualified Data.Text.Encoding as TE
import qualified Data.Vector as V
import qualified Data.Vector.Unboxed.Mutable as VUM
import Control.Monad.ST (runST)
import Data.Maybe (isJust)
import Data.Time (Day, defaultTimeLocale, parseTimeM)
import DataFrame.Internal.Column (Column (..), finalizeParseResult)
import DataFrame.Internal.Parsing (readByteStringDate, readInt)
import DataFrame.Internal.Parsing.Fast (
parseBoolField,
parseDateField,
parseDoubleField,
parseIntField,
)
type DateFormat = String
data ParsingAssumption
= BoolAssumption
| IntAssumption
| DoubleAssumption
| DateAssumption
| NoAssumption
| TextAssumption
deriving (ParsingAssumption -> ParsingAssumption -> Bool
(ParsingAssumption -> ParsingAssumption -> Bool)
-> (ParsingAssumption -> ParsingAssumption -> Bool)
-> Eq ParsingAssumption
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: ParsingAssumption -> ParsingAssumption -> Bool
== :: ParsingAssumption -> ParsingAssumption -> Bool
$c/= :: ParsingAssumption -> ParsingAssumption -> Bool
/= :: ParsingAssumption -> ParsingAssumption -> Bool
Eq, Int -> ParsingAssumption -> ShowS
[ParsingAssumption] -> ShowS
ParsingAssumption -> String
(Int -> ParsingAssumption -> ShowS)
-> (ParsingAssumption -> String)
-> ([ParsingAssumption] -> ShowS)
-> Show ParsingAssumption
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: Int -> ParsingAssumption -> ShowS
showsPrec :: Int -> ParsingAssumption -> ShowS
$cshow :: ParsingAssumption -> String
show :: ParsingAssumption -> String
$cshowList :: [ParsingAssumption] -> ShowS
showList :: [ParsingAssumption] -> ShowS
Show)
parseTimeOpt :: DateFormat -> T.Text -> Maybe Day
parseTimeOpt :: String -> Text -> Maybe Day
parseTimeOpt String
dateFormat Text
s =
Bool -> TimeLocale -> String -> String -> Maybe Day
forall (m :: * -> *) t.
(MonadFail m, ParseTime t) =>
Bool -> TimeLocale -> String -> String -> m t
parseTimeM
Bool
True
TimeLocale
defaultTimeLocale
String
dateFormat
(Text -> String
T.unpack Text
s)
byteStringDateParser :: DateFormat -> BS.ByteString -> Maybe Day
byteStringDateParser :: String -> ByteString -> Maybe Day
byteStringDateParser String
"%Y-%m-%d" = ByteString -> Maybe Day
parseDateField
byteStringDateParser String
fmt = String -> ByteString -> Maybe Day
readByteStringDate String
fmt
{-# INLINE byteStringDateParser #-}
readIntStrict :: T.Text -> Maybe Int
readIntStrict :: Text -> Maybe Int
readIntStrict Text
t
| Text -> Int
T.length Text
t Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
<= Int
18 = HasCallStack => Text -> Maybe Int
Text -> Maybe Int
readInt Text
t
| Bool
otherwise = ByteString -> Maybe Int
parseIntField (Text -> ByteString
TE.encodeUtf8 Text
t)
{-# INLINE readIntStrict #-}
pickAssumption ::
Bool -> Bool -> Bool -> Bool -> Bool -> ParsingAssumption
pickAssumption :: Bool -> Bool -> Bool -> Bool -> Bool -> ParsingAssumption
pickAssumption Bool
seen Bool
b Bool
i Bool
d Bool
dt
| Bool -> Bool
not Bool
seen = ParsingAssumption
NoAssumption
| Bool
b = ParsingAssumption
BoolAssumption
| Bool
i Bool -> Bool -> Bool
&& Bool
d = ParsingAssumption
IntAssumption
| Bool
d = ParsingAssumption
DoubleAssumption
| Bool
dt = ParsingAssumption
DateAssumption
| Bool
otherwise = ParsingAssumption
TextAssumption
makeParsingAssumption ::
DateFormat -> V.Vector (Maybe T.Text) -> ParsingAssumption
makeParsingAssumption :: String -> Vector (Maybe Text) -> ParsingAssumption
makeParsingAssumption String
dfmt Vector (Maybe Text)
cells = Int -> Bool -> Bool -> Bool -> Bool -> Bool -> ParsingAssumption
go Int
0 Bool
False Bool
True Bool
True Bool
True Bool
True
where
n :: Int
n = Vector (Maybe Text) -> Int
forall a. Vector a -> Int
V.length Vector (Maybe Text)
cells
go :: Int -> Bool -> Bool -> Bool -> Bool -> Bool -> ParsingAssumption
go !Int
i !Bool
seen !Bool
b !Bool
int !Bool
d !Bool
dt
| Int
i Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
>= Int
n Bool -> Bool -> Bool
|| (Bool
seen Bool -> Bool -> Bool
&& Bool -> Bool
not (Bool
b Bool -> Bool -> Bool
|| Bool
int Bool -> Bool -> Bool
|| Bool
d Bool -> Bool -> Bool
|| Bool
dt)) =
Bool -> Bool -> Bool -> Bool -> Bool -> ParsingAssumption
pickAssumption Bool
seen Bool
b Bool
int Bool
d Bool
dt
| Bool
otherwise = case Vector (Maybe Text) -> Int -> Maybe Text
forall a. Vector a -> Int -> a
V.unsafeIndex Vector (Maybe Text)
cells Int
i of
Maybe Text
Nothing -> Int -> Bool -> Bool -> Bool -> Bool -> Bool -> ParsingAssumption
go (Int
i Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1) Bool
seen Bool
b Bool
int Bool
d Bool
dt
Just Text
t ->
let bs :: ByteString
bs = Text -> ByteString
TE.encodeUtf8 Text
t
in Int -> Bool -> Bool -> Bool -> Bool -> Bool -> ParsingAssumption
go
(Int
i Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1)
Bool
True
(Bool
b Bool -> Bool -> Bool
&& Maybe Bool -> Bool
forall a. Maybe a -> Bool
isJust (ByteString -> Maybe Bool
parseBoolField ByteString
bs))
(Bool
int Bool -> Bool -> Bool
&& Maybe Int -> Bool
forall a. Maybe a -> Bool
isJust (ByteString -> Maybe Int
parseIntField ByteString
bs))
(Bool
d Bool -> Bool -> Bool
&& Maybe Double -> Bool
forall a. Maybe a -> Bool
isJust (ByteString -> Maybe Double
parseDoubleField ByteString
bs))
(Bool
dt Bool -> Bool -> Bool
&& Maybe Day -> Bool
forall a. Maybe a -> Bool
isJust (String -> Text -> Maybe Day
parseTimeOpt String
dfmt Text
t))
makeParsingAssumptionBytes ::
DateFormat -> V.Vector (Maybe BS.ByteString) -> ParsingAssumption
makeParsingAssumptionBytes :: String -> Vector (Maybe ByteString) -> ParsingAssumption
makeParsingAssumptionBytes String
dfmt Vector (Maybe ByteString)
cells = Int -> Bool -> Bool -> Bool -> Bool -> Bool -> ParsingAssumption
go Int
0 Bool
False Bool
True Bool
True Bool
True Bool
True
where
n :: Int
n = Vector (Maybe ByteString) -> Int
forall a. Vector a -> Int
V.length Vector (Maybe ByteString)
cells
dateP :: ByteString -> Maybe Day
dateP = String -> ByteString -> Maybe Day
byteStringDateParser String
dfmt
go :: Int -> Bool -> Bool -> Bool -> Bool -> Bool -> ParsingAssumption
go !Int
i !Bool
seen !Bool
b !Bool
int !Bool
d !Bool
dt
| Int
i Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
>= Int
n Bool -> Bool -> Bool
|| (Bool
seen Bool -> Bool -> Bool
&& Bool -> Bool
not (Bool
b Bool -> Bool -> Bool
|| Bool
int Bool -> Bool -> Bool
|| Bool
d Bool -> Bool -> Bool
|| Bool
dt)) =
Bool -> Bool -> Bool -> Bool -> Bool -> ParsingAssumption
pickAssumption Bool
seen Bool
b Bool
int Bool
d Bool
dt
| Bool
otherwise = case Vector (Maybe ByteString) -> Int -> Maybe ByteString
forall a. Vector a -> Int -> a
V.unsafeIndex Vector (Maybe ByteString)
cells Int
i of
Maybe ByteString
Nothing -> Int -> Bool -> Bool -> Bool -> Bool -> Bool -> ParsingAssumption
go (Int
i Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1) Bool
seen Bool
b Bool
int Bool
d Bool
dt
Just ByteString
bs ->
Int -> Bool -> Bool -> Bool -> Bool -> Bool -> ParsingAssumption
go
(Int
i Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1)
Bool
True
(Bool
b Bool -> Bool -> Bool
&& Maybe Bool -> Bool
forall a. Maybe a -> Bool
isJust (ByteString -> Maybe Bool
parseBoolField ByteString
bs))
(Bool
int Bool -> Bool -> Bool
&& Maybe Int -> Bool
forall a. Maybe a -> Bool
isJust (ByteString -> Maybe Int
parseIntField ByteString
bs))
(Bool
d Bool -> Bool -> Bool
&& Maybe Double -> Bool
forall a. Maybe a -> Bool
isJust (ByteString -> Maybe Double
parseDoubleField ByteString
bs))
(Bool
dt Bool -> Bool -> Bool
&& Maybe Day -> Bool
forall a. Maybe a -> Bool
isJust (ByteString -> Maybe Day
dateP ByteString
bs))
promoteIntColumn ::
forall src.
(Int -> src -> Bool) ->
(src -> Maybe Int) ->
(src -> Maybe Double) ->
V.Vector src ->
Maybe Column
promoteIntColumn :: forall src.
(Int -> src -> Bool)
-> (src -> Maybe Int)
-> (src -> Maybe Double)
-> Vector src
-> Maybe Column
promoteIntColumn Int -> src -> Bool
isNullAt src -> Maybe Int
parseI src -> Maybe Double
parseD Vector src
vec =
Int
-> (Int -> Bool)
-> (Int -> Maybe Int)
-> (Int -> Maybe Double)
-> Maybe Column
promoteIntColumnIndexed
(Vector src -> Int
forall a. Vector a -> Int
V.length Vector src
vec)
(\Int
i -> Int -> src -> Bool
isNullAt Int
i (Vector src -> Int -> src
forall a. Vector a -> Int -> a
V.unsafeIndex Vector src
vec Int
i))
(src -> Maybe Int
parseI (src -> Maybe Int) -> (Int -> src) -> Int -> Maybe Int
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Vector src -> Int -> src
forall a. Vector a -> Int -> a
V.unsafeIndex Vector src
vec)
(src -> Maybe Double
parseD (src -> Maybe Double) -> (Int -> src) -> Int -> Maybe Double
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Vector src -> Int -> src
forall a. Vector a -> Int -> a
V.unsafeIndex Vector src
vec)
{-# INLINE promoteIntColumn #-}
promoteIntColumnIndexed ::
Int ->
(Int -> Bool) ->
(Int -> Maybe Int) ->
(Int -> Maybe Double) ->
Maybe Column
promoteIntColumnIndexed :: Int
-> (Int -> Bool)
-> (Int -> Maybe Int)
-> (Int -> Maybe Double)
-> Maybe Column
promoteIntColumnIndexed Int
n Int -> Bool
isNullAt Int -> Maybe Int
parseI Int -> Maybe Double
parseD = (forall s. ST s (Maybe Column)) -> Maybe Column
forall a. (forall s. ST s a) -> a
runST ((forall s. ST s (Maybe Column)) -> Maybe Column)
-> (forall s. ST s (Maybe Column)) -> Maybe Column
forall a b. (a -> b) -> a -> b
$ do
MVector s Int
values <- Int -> ST s (MVector (PrimState (ST s)) Int)
forall (m :: * -> *) a.
(PrimMonad m, Unbox a) =>
Int -> m (MVector (PrimState m) a)
VUM.unsafeNew Int
n
STVector s Word8
vmask <- Int -> ST s (MVector (PrimState (ST s)) Word8)
forall (m :: * -> *) a.
(PrimMonad m, Unbox a) =>
Int -> m (MVector (PrimState m) a)
VUM.unsafeNew Int
n
let intLoop :: Int -> Bool -> ST s (Maybe Column)
intLoop !Int
i !Bool
anyNull
| Int
i Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
>= Int
n =
((Maybe Bitmap, Vector Int) -> Column)
-> Maybe (Maybe Bitmap, Vector Int) -> Maybe Column
forall a b. (a -> b) -> Maybe a -> Maybe b
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
fmap ((Maybe Bitmap -> Vector Int -> Column)
-> (Maybe Bitmap, Vector Int) -> Column
forall a b c. (a -> b -> c) -> (a, b) -> c
uncurry Maybe Bitmap -> Vector Int -> Column
forall a.
(Columnable a, Unbox a) =>
Maybe Bitmap -> Vector a -> Column
UnboxedColumn)
(Maybe (Maybe Bitmap, Vector Int) -> Maybe Column)
-> ST s (Maybe (Maybe Bitmap, Vector Int)) -> ST s (Maybe Column)
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> MVector s Int
-> STVector s Word8
-> Bool
-> ST s (Maybe (Maybe Bitmap, Vector Int))
forall a s.
Unbox a =>
STVector s a
-> STVector s Word8
-> Bool
-> ST s (Maybe (Maybe Bitmap, Vector a))
finalizeParseResult MVector s Int
values STVector s Word8
vmask Bool
anyNull
| Int -> Bool
isNullAt Int
i = do
MVector (PrimState (ST s)) Word8 -> Int -> Word8 -> ST s ()
forall (m :: * -> *) a.
(PrimMonad m, Unbox a) =>
MVector (PrimState m) a -> Int -> a -> m ()
VUM.unsafeWrite STVector s Word8
MVector (PrimState (ST s)) Word8
vmask Int
i Word8
0
MVector (PrimState (ST s)) Int -> Int -> Int -> ST s ()
forall (m :: * -> *) a.
(PrimMonad m, Unbox a) =>
MVector (PrimState m) a -> Int -> a -> m ()
VUM.unsafeWrite MVector s Int
MVector (PrimState (ST s)) Int
values Int
i Int
0
Int -> Bool -> ST s (Maybe Column)
intLoop (Int
i Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1) Bool
True
| Bool
otherwise = case Int -> Maybe Int
parseI Int
i of
Just Int
v -> do
MVector (PrimState (ST s)) Word8 -> Int -> Word8 -> ST s ()
forall (m :: * -> *) a.
(PrimMonad m, Unbox a) =>
MVector (PrimState m) a -> Int -> a -> m ()
VUM.unsafeWrite STVector s Word8
MVector (PrimState (ST s)) Word8
vmask Int
i Word8
1
MVector (PrimState (ST s)) Int -> Int -> Int -> ST s ()
forall (m :: * -> *) a.
(PrimMonad m, Unbox a) =>
MVector (PrimState m) a -> Int -> a -> m ()
VUM.unsafeWrite MVector s Int
MVector (PrimState (ST s)) Int
values Int
i Int
v
Int -> Bool -> ST s (Maybe Column)
intLoop (Int
i Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1) Bool
anyNull
Maybe Int
Nothing -> case Int -> Maybe Double
parseD Int
i of
Just Double
dv -> Int -> Bool -> Double -> ST s (Maybe Column)
promote Int
i Bool
anyNull Double
dv
Maybe Double
Nothing -> Maybe Column -> ST s (Maybe Column)
forall a. a -> ST s a
forall (m :: * -> *) a. Monad m => a -> m a
return Maybe Column
forall a. Maybe a
Nothing
promote :: Int -> Bool -> Double -> ST s (Maybe Column)
promote !Int
i !Bool
anyNull !Double
dv = do
MVector s Double
dvalues <- Int -> ST s (MVector (PrimState (ST s)) Double)
forall (m :: * -> *) a.
(PrimMonad m, Unbox a) =>
Int -> m (MVector (PrimState m) a)
VUM.unsafeNew Int
n
let copy :: Int -> m ()
copy !Int
j
| Int
j Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
>= Int
i = () -> m ()
forall a. a -> m a
forall (m :: * -> *) a. Monad m => a -> m a
return ()
| Bool
otherwise = do
Int
x <- MVector (PrimState m) Int -> Int -> m Int
forall (m :: * -> *) a.
(PrimMonad m, Unbox a) =>
MVector (PrimState m) a -> Int -> m a
VUM.unsafeRead MVector s Int
MVector (PrimState m) Int
values Int
j
MVector (PrimState m) Double -> Int -> Double -> m ()
forall (m :: * -> *) a.
(PrimMonad m, Unbox a) =>
MVector (PrimState m) a -> Int -> a -> m ()
VUM.unsafeWrite MVector s Double
MVector (PrimState m) Double
dvalues Int
j (Int -> Double
forall a b. (Integral a, Num b) => a -> b
fromIntegral Int
x)
Int -> m ()
copy (Int
j Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1)
Int -> ST s ()
forall {m :: * -> *}. (PrimState m ~ s, PrimMonad m) => Int -> m ()
copy Int
0
MVector (PrimState (ST s)) Word8 -> Int -> Word8 -> ST s ()
forall (m :: * -> *) a.
(PrimMonad m, Unbox a) =>
MVector (PrimState m) a -> Int -> a -> m ()
VUM.unsafeWrite STVector s Word8
MVector (PrimState (ST s)) Word8
vmask Int
i Word8
1
MVector (PrimState (ST s)) Double -> Int -> Double -> ST s ()
forall (m :: * -> *) a.
(PrimMonad m, Unbox a) =>
MVector (PrimState m) a -> Int -> a -> m ()
VUM.unsafeWrite MVector s Double
MVector (PrimState (ST s)) Double
dvalues Int
i Double
dv
MVector s Double -> Int -> Bool -> ST s (Maybe Column)
dblLoop MVector s Double
dvalues (Int
i Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1) Bool
anyNull
dblLoop :: MVector s Double -> Int -> Bool -> ST s (Maybe Column)
dblLoop MVector s Double
dvalues !Int
i !Bool
anyNull
| Int
i Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
>= Int
n =
((Maybe Bitmap, Vector Double) -> Column)
-> Maybe (Maybe Bitmap, Vector Double) -> Maybe Column
forall a b. (a -> b) -> Maybe a -> Maybe b
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
fmap ((Maybe Bitmap -> Vector Double -> Column)
-> (Maybe Bitmap, Vector Double) -> Column
forall a b c. (a -> b -> c) -> (a, b) -> c
uncurry Maybe Bitmap -> Vector Double -> Column
forall a.
(Columnable a, Unbox a) =>
Maybe Bitmap -> Vector a -> Column
UnboxedColumn)
(Maybe (Maybe Bitmap, Vector Double) -> Maybe Column)
-> ST s (Maybe (Maybe Bitmap, Vector Double))
-> ST s (Maybe Column)
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> MVector s Double
-> STVector s Word8
-> Bool
-> ST s (Maybe (Maybe Bitmap, Vector Double))
forall a s.
Unbox a =>
STVector s a
-> STVector s Word8
-> Bool
-> ST s (Maybe (Maybe Bitmap, Vector a))
finalizeParseResult MVector s Double
dvalues STVector s Word8
vmask Bool
anyNull
| Int -> Bool
isNullAt Int
i = do
MVector (PrimState (ST s)) Word8 -> Int -> Word8 -> ST s ()
forall (m :: * -> *) a.
(PrimMonad m, Unbox a) =>
MVector (PrimState m) a -> Int -> a -> m ()
VUM.unsafeWrite STVector s Word8
MVector (PrimState (ST s)) Word8
vmask Int
i Word8
0
MVector (PrimState (ST s)) Double -> Int -> Double -> ST s ()
forall (m :: * -> *) a.
(PrimMonad m, Unbox a) =>
MVector (PrimState m) a -> Int -> a -> m ()
VUM.unsafeWrite MVector s Double
MVector (PrimState (ST s)) Double
dvalues Int
i Double
0
MVector s Double -> Int -> Bool -> ST s (Maybe Column)
dblLoop MVector s Double
dvalues (Int
i Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1) Bool
True
| Bool
otherwise = case Int -> Maybe Double
parseD Int
i of
Just Double
dv -> do
MVector (PrimState (ST s)) Word8 -> Int -> Word8 -> ST s ()
forall (m :: * -> *) a.
(PrimMonad m, Unbox a) =>
MVector (PrimState m) a -> Int -> a -> m ()
VUM.unsafeWrite STVector s Word8
MVector (PrimState (ST s)) Word8
vmask Int
i Word8
1
MVector (PrimState (ST s)) Double -> Int -> Double -> ST s ()
forall (m :: * -> *) a.
(PrimMonad m, Unbox a) =>
MVector (PrimState m) a -> Int -> a -> m ()
VUM.unsafeWrite MVector s Double
MVector (PrimState (ST s)) Double
dvalues Int
i Double
dv
MVector s Double -> Int -> Bool -> ST s (Maybe Column)
dblLoop MVector s Double
dvalues (Int
i Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1) Bool
anyNull
Maybe Double
Nothing -> Maybe Column -> ST s (Maybe Column)
forall a. a -> ST s a
forall (m :: * -> *) a. Monad m => a -> m a
return Maybe Column
forall a. Maybe a
Nothing
Int -> Bool -> ST s (Maybe Column)
intLoop Int
0 Bool
False
{-# INLINE promoteIntColumnIndexed #-}