Safe Haskell	None
Language	Haskell2010

DataFrame.IO.Parquet.Thrift

Synopsis

data SchemaElement = SchemaElement {
- elementName :: Text
- elementType :: TType
- typeLength :: Int32
- numChildren :: Int32
- fieldId :: Int32
- repetitionType :: RepetitionType
- convertedType :: Int32
- scale :: Int32
- precision :: Int32
- logicalType :: LogicalType
}
createParquetSchema :: DataFrame -> [SchemaElement]
data KeyValue = KeyValue {
- key :: String
- value :: String
}
data FileMetadata = FileMetaData {
- version :: Int32
- schema :: [SchemaElement]
- numRows :: Integer
- rowGroups :: [RowGroup]
- keyValueMetadata :: [KeyValue]
- createdBy :: Maybe String
- columnOrders :: [ColumnOrder]
- encryptionAlgorithm :: EncryptionAlgorithm
- footerSigningKeyMetadata :: ByteString
}
data TType
- = STOP
- | BOOL
- | BYTE
- | I16
- | I32
- | I64
- | I96
- | FLOAT
- | DOUBLE
- | STRING
- | LIST
- | SET
- | MAP
- | STRUCT
- | UUID
haskellToTType :: Typeable a => TType
defaultMetadata :: FileMetadata
data ColumnMetaData = ColumnMetaData {
- columnType :: ParquetType
- columnEncodings :: [ParquetEncoding]
- columnPathInSchema :: [String]
- columnCodec :: CompressionCodec
- columnNumValues :: Int64
- columnTotalUncompressedSize :: Int64
- columnTotalCompressedSize :: Int64
- columnKeyValueMetadata :: [KeyValue]
- columnDataPageOffset :: Int64
- columnIndexPageOffset :: Int64
- columnDictionaryPageOffset :: Int64
- columnStatistics :: ColumnStatistics
- columnEncodingStats :: [PageEncodingStats]
- bloomFilterOffset :: Int64
- bloomFilterLength :: Int32
- columnSizeStatistics :: SizeStatistics
- columnGeospatialStatistics :: GeospatialStatistics
}
data ColumnChunk = ColumnChunk {
- columnChunkFilePath :: String
- columnChunkMetadataFileOffset :: Int64
- columnMetaData :: ColumnMetaData
- columnChunkOffsetIndexOffset :: Int64
- columnChunkOffsetIndexLength :: Int32
- columnChunkColumnIndexOffset :: Int64
- columnChunkColumnIndexLength :: Int32
- cryptoMetadata :: ColumnCryptoMetadata
- encryptedColumnMetadata :: ByteString
}
data RowGroup = RowGroup {
- rowGroupColumns :: [ColumnChunk]
- totalByteSize :: Int64
- rowGroupNumRows :: Int64
- rowGroupSortingColumns :: [SortingColumn]
- fileOffset :: Int64
- totalCompressedSize :: Int64
- ordinal :: Int16
}
defaultSchemaElement :: SchemaElement
emptyColumnMetadata :: ColumnMetaData
emptyColumnChunk :: ColumnChunk
emptyKeyValue :: KeyValue
emptyRowGroup :: RowGroup
compactBooleanTrue :: Word8
compactI32 :: Word8
compactI64 :: Word8
compactDouble :: Word8
compactBinary :: Word8
compactList :: Word8
compactStruct :: Word8
toTType :: Word8 -> TType
readField :: ByteString -> IORef Int -> Int16 -> IO (Maybe (TType, Int16))
skipToStructEnd :: ByteString -> IORef Int -> IO ()
skipFieldData :: TType -> ByteString -> IORef Int -> IO ()
skipList :: ByteString -> IORef Int -> IO ()
readMetadataByHandleMetaSize :: FileBufferedOrSeekable -> Int -> IO FileMetadata
readMetadata :: ByteString -> Int -> IO FileMetadata
readFileMetaData :: FileMetadata -> ByteString -> IORef Int -> Int16 -> IO FileMetadata
readSchemaElement :: SchemaElement -> ByteString -> IORef Int -> Int16 -> IO SchemaElement
readRowGroup :: RowGroup -> ByteString -> IORef Int -> Int16 -> IO RowGroup
readColumnChunk :: ColumnChunk -> ByteString -> IORef Int -> Int16 -> IO ColumnChunk
readColumnMetadata :: ColumnMetaData -> ByteString -> IORef Int -> Int16 -> IO ColumnMetaData
readEncryptionAlgorithm :: ByteString -> IORef Int -> Int16 -> IO EncryptionAlgorithm
readColumnOrder :: ByteString -> IORef Int -> Int16 -> IO ColumnOrder
readAesGcmCtrV1 :: EncryptionAlgorithm -> ByteString -> IORef Int -> Int16 -> IO EncryptionAlgorithm
readAesGcmV1 :: EncryptionAlgorithm -> ByteString -> IORef Int -> Int16 -> IO EncryptionAlgorithm
readTypeOrder :: ByteString -> IORef Int -> Int16 -> IO ColumnOrder
readKeyValue :: KeyValue -> ByteString -> IORef Int -> Int16 -> IO KeyValue
readPageEncodingStats :: PageEncodingStats -> ByteString -> IORef Int -> Int16 -> IO PageEncodingStats
readParquetEncoding :: ByteString -> IORef Int -> Int16 -> IO ParquetEncoding
readStatistics :: ColumnStatistics -> ByteString -> IORef Int -> Int16 -> IO ColumnStatistics
readSizeStatistics :: SizeStatistics -> ByteString -> IORef Int -> Int16 -> IO SizeStatistics
footerSize :: Int
toIntegralType :: Int32 -> TType
readLogicalType :: LogicalType -> ByteString -> IORef Int -> Int16 -> IO LogicalType
readIntType :: Int8 -> Bool -> ByteString -> IORef Int -> Int16 -> IO LogicalType
readDecimalType :: Int32 -> Int32 -> ByteString -> IORef Int -> Int16 -> IO LogicalType
readTimeType :: Bool -> TimeUnit -> ByteString -> IORef Int -> Int16 -> IO LogicalType
readTimestampType :: Bool -> TimeUnit -> ByteString -> IORef Int -> Int16 -> IO LogicalType
readUnit :: TimeUnit -> ByteString -> IORef Int -> Int16 -> IO TimeUnit

Documentation

data SchemaElement Source #

Constructors

SchemaElement
Fields elementName :: Text elementType :: TType typeLength :: Int32 numChildren :: Int32 fieldId :: Int32 repetitionType :: RepetitionType convertedType :: Int32 scale :: Int32 precision :: Int32 logicalType :: LogicalType

Instances

Instances details

Show SchemaElement Source #
Instance details Defined in DataFrame.IO.Parquet.Thrift Methods showsPrec :: Int -> SchemaElement -> ShowS # show :: SchemaElement -> String # showList :: [SchemaElement] -> ShowS #
Eq SchemaElement Source #
Instance details Defined in DataFrame.IO.Parquet.Thrift Methods (==) :: SchemaElement -> SchemaElement -> Bool # (/=) :: SchemaElement -> SchemaElement -> Bool #

createParquetSchema :: DataFrame -> [SchemaElement] Source #

data KeyValue Source #

Constructors

KeyValue
Fields key :: String value :: String

Instances

Instances details

Show KeyValue Source #
Instance details Defined in DataFrame.IO.Parquet.Thrift Methods showsPrec :: Int -> KeyValue -> ShowS # show :: KeyValue -> String # showList :: [KeyValue] -> ShowS #
Eq KeyValue Source #
Instance details Defined in DataFrame.IO.Parquet.Thrift Methods (==) :: KeyValue -> KeyValue -> Bool # (/=) :: KeyValue -> KeyValue -> Bool #

data FileMetadata Source #

Constructors

FileMetaData
Fields version :: Int32 schema :: [SchemaElement] numRows :: Integer rowGroups :: [RowGroup] keyValueMetadata :: [KeyValue] createdBy :: Maybe String columnOrders :: [ColumnOrder] encryptionAlgorithm :: EncryptionAlgorithm footerSigningKeyMetadata :: ByteString

Instances

Instances details

Show FileMetadata Source #
Instance details Defined in DataFrame.IO.Parquet.Thrift Methods showsPrec :: Int -> FileMetadata -> ShowS # show :: FileMetadata -> String # showList :: [FileMetadata] -> ShowS #
Eq FileMetadata Source #
Instance details Defined in DataFrame.IO.Parquet.Thrift Methods (==) :: FileMetadata -> FileMetadata -> Bool # (/=) :: FileMetadata -> FileMetadata -> Bool #

data TType Source #

Constructors

STOP
BOOL
BYTE
I16
I32
I64
I96
FLOAT
DOUBLE
STRING
LIST
SET
MAP
STRUCT
UUID

Instances

Instances details

Show TType Source #
Instance details Defined in DataFrame.IO.Parquet.Thrift Methods showsPrec :: Int -> TType -> ShowS # show :: TType -> String # showList :: [TType] -> ShowS #
Eq TType Source #
Instance details Defined in DataFrame.IO.Parquet.Thrift Methods (==) :: TType -> TType -> Bool # (/=) :: TType -> TType -> Bool #

haskellToTType :: Typeable a => TType Source #

defaultMetadata :: FileMetadata Source #

data ColumnMetaData Source #

Constructors

ColumnMetaData

Instances

Instances details

Show ColumnMetaData Source #
Instance details Defined in DataFrame.IO.Parquet.Thrift Methods showsPrec :: Int -> ColumnMetaData -> ShowS # show :: ColumnMetaData -> String # showList :: [ColumnMetaData] -> ShowS #
Eq ColumnMetaData Source #
Instance details Defined in DataFrame.IO.Parquet.Thrift Methods (==) :: ColumnMetaData -> ColumnMetaData -> Bool # (/=) :: ColumnMetaData -> ColumnMetaData -> Bool #

data ColumnChunk Source #

Constructors

ColumnChunk
Fields columnChunkFilePath :: String columnChunkMetadataFileOffset :: Int64 columnMetaData :: ColumnMetaData columnChunkOffsetIndexOffset :: Int64 columnChunkOffsetIndexLength :: Int32 columnChunkColumnIndexOffset :: Int64 columnChunkColumnIndexLength :: Int32 cryptoMetadata :: ColumnCryptoMetadata encryptedColumnMetadata :: ByteString

Instances

Instances details

Show ColumnChunk Source #
Instance details Defined in DataFrame.IO.Parquet.Thrift Methods showsPrec :: Int -> ColumnChunk -> ShowS # show :: ColumnChunk -> String # showList :: [ColumnChunk] -> ShowS #
Eq ColumnChunk Source #
Instance details Defined in DataFrame.IO.Parquet.Thrift Methods (==) :: ColumnChunk -> ColumnChunk -> Bool # (/=) :: ColumnChunk -> ColumnChunk -> Bool #

data RowGroup Source #

Constructors

RowGroup
Fields rowGroupColumns :: [ColumnChunk] totalByteSize :: Int64 rowGroupNumRows :: Int64 rowGroupSortingColumns :: [SortingColumn] fileOffset :: Int64 totalCompressedSize :: Int64 ordinal :: Int16

Instances

Instances details

Show RowGroup Source #
Instance details Defined in DataFrame.IO.Parquet.Thrift Methods showsPrec :: Int -> RowGroup -> ShowS # show :: RowGroup -> String # showList :: [RowGroup] -> ShowS #
Eq RowGroup Source #
Instance details Defined in DataFrame.IO.Parquet.Thrift Methods (==) :: RowGroup -> RowGroup -> Bool # (/=) :: RowGroup -> RowGroup -> Bool #