Safe Haskell	None
Language	Haskell2010

Parquet.Stream.Reader

Synopsis

data ColumnValue = ColumnValue {
- _cvRepetitionLevel :: Word32
- _cvDefinitionLevel :: Word32
- _cvMaxDefinitionLevel :: Word32
- _cvValue :: Value
}
replicateMSized :: Monad m => Int -> m (Int64, result) -> m (Int64, [result])
forSized :: Monad m => [a] -> (a -> m (Int64, result)) -> m (Int64, [result])
traverseSized :: Monad m => (a -> m (Int64, result)) -> [a] -> m (Int64, [result])
maxLevelToBitWidth :: Word8 -> BitWidth
dataPageReader :: forall m. (PR m, MonadReader PageCtx m) => DataPageHeader -> Maybe [Value] -> ConduitT ByteString ColumnValue m ()
data Value
- = ValueInt64 Int64
- | ValueByteString ByteString
- | Null
decodeValue :: (PR m, MonadReader PageCtx m) => ConduitT ByteString o m (Int64, Value)
dictPageReader :: (MonadReader PageCtx m, PR m) => DictionaryPageHeader -> ConduitT ByteString o m (Int64, [Value])
data PageCtx = PageCtx {
- _pcSchema :: Map Text SchemaElement
- _pcPath :: NonEmpty Text
- _pcColumnTy :: Type
}
getLastSchemaElement :: (MonadError Text m, MonadReader PageCtx m) => m SchemaElement
readDefinitionLevel :: (PR m, MonadReader PageCtx m) => Encoding -> BitWidth -> Int32 -> ConduitT ByteString a m (Int64, [Word32])
readRepetitionLevel :: (MonadThrow m, MonadError Text m, MonadReader PageCtx m, MonadLogger m) => Encoding -> BitWidth -> Int32 -> ConduitT ByteString a m (Int64, [Word32])
sizedGet :: Get result -> Get (Int64, result)
decodeLevel :: (MonadThrow m, MonadError Text m, MonadLogger m, MonadReader PageCtx m) => Encoding -> BitWidth -> Int32 -> ConduitT ByteString a m (Int64, [Word32])
calcMaxEncodingLevels :: (MonadReader PageCtx m, MonadError Text m) => m (Word8, Word8)
getRepType :: MonadError Text m => SchemaElement -> m FieldRepetitionType
validateCompression :: MonadError Text m => ColumnMetaData -> m ()
readColumnChunk :: PR m => Map Text SchemaElement -> ColumnChunk -> ConduitT ByteString ColumnValue m ()
readPage :: (MonadReader PageCtx m, PR m) => Int64 -> Maybe [Value] -> ConduitT ByteString ColumnValue m ()
failOnError :: Show err => IO (Either err b) -> IO b
decodeConduit :: forall a size m o. (MonadError Text m, MonadIO m, Integral size, Pinchable a) => size -> ConduitT ByteString o m (Int, a)

Documentation

data ColumnValue Source #

Constructors

ColumnValue
Fields _cvRepetitionLevel :: Word32 _cvDefinitionLevel :: Word32 _cvMaxDefinitionLevel :: Word32 _cvValue :: Value

Instances

Eq ColumnValue Source #
Instance details Defined in Parquet.Stream.Reader Methods (==) :: ColumnValue -> ColumnValue -> Bool # (/=) :: ColumnValue -> ColumnValue -> Bool #
Show ColumnValue Source #
Instance details Defined in Parquet.Stream.Reader Methods showsPrec :: Int -> ColumnValue -> ShowS # show :: ColumnValue -> String # showList :: [ColumnValue] -> ShowS #

replicateMSized :: Monad m => Int -> m (Int64, result) -> m (Int64, [result]) Source #

TODO: This is so unoptimized that my eyes bleed.

forSized :: Monad m => [a] -> (a -> m (Int64, result)) -> m (Int64, [result]) Source #

traverseSized :: Monad m => (a -> m (Int64, result)) -> [a] -> m (Int64, [result]) Source #

maxLevelToBitWidth :: Word8 -> BitWidth Source #

dataPageReader :: forall m. (PR m, MonadReader PageCtx m) => DataPageHeader -> Maybe [Value] -> ConduitT ByteString ColumnValue m () Source #

data Value Source #

Constructors

ValueInt64 Int64
ValueByteString ByteString
Null

Instances

Eq Value Source #
Instance details Defined in Parquet.Stream.Reader Methods (==) :: Value -> Value -> Bool # (/=) :: Value -> Value -> Bool #
Show Value Source #
Instance details Defined in Parquet.Stream.Reader Methods showsPrec :: Int -> Value -> ShowS # show :: Value -> String # showList :: [Value] -> ShowS #

decodeValue :: (PR m, MonadReader PageCtx m) => ConduitT ByteString o m (Int64, Value) Source #

dictPageReader :: (MonadReader PageCtx m, PR m) => DictionaryPageHeader -> ConduitT ByteString o m (Int64, [Value]) Source #

data PageCtx Source #

Constructors

PageCtx
Fields _pcSchema :: Map Text SchemaElement _pcPath :: NonEmpty Text _pcColumnTy :: Type

Instances

Eq PageCtx Source #
Instance details Defined in Parquet.Stream.Reader Methods (==) :: PageCtx -> PageCtx -> Bool # (/=) :: PageCtx -> PageCtx -> Bool #
Show PageCtx Source #
Instance details Defined in Parquet.Stream.Reader Methods showsPrec :: Int -> PageCtx -> ShowS # show :: PageCtx -> String # showList :: [PageCtx] -> ShowS #

getLastSchemaElement :: (MonadError Text m, MonadReader PageCtx m) => m SchemaElement Source #

readDefinitionLevel :: (PR m, MonadReader PageCtx m) => Encoding -> BitWidth -> Int32 -> ConduitT ByteString a m (Int64, [Word32]) Source #

readRepetitionLevel :: (MonadThrow m, MonadError Text m, MonadReader PageCtx m, MonadLogger m) => Encoding -> BitWidth -> Int32 -> ConduitT ByteString a m (Int64, [Word32]) Source #

sizedGet :: Get result -> Get (Int64, result) Source #

decodeLevel :: (MonadThrow m, MonadError Text m, MonadLogger m, MonadReader PageCtx m) => Encoding -> BitWidth -> Int32 -> ConduitT ByteString a m (Int64, [Word32]) Source #

calcMaxEncodingLevels :: (MonadReader PageCtx m, MonadError Text m) => m (Word8, Word8) Source #

Algorithm: https://blog.twitter.com/engineering/en_us/a/2013/dremel-made-simple-with-parquet.html

getRepType :: MonadError Text m => SchemaElement -> m FieldRepetitionType Source #

validateCompression :: MonadError Text m => ColumnMetaData -> m () Source #

readColumnChunk :: PR m => Map Text SchemaElement -> ColumnChunk -> ConduitT ByteString ColumnValue m () Source #

readPage :: (MonadReader PageCtx m, PR m) => Int64 -> Maybe [Value] -> ConduitT ByteString ColumnValue m () Source #

failOnError :: Show err => IO (Either err b) -> IO b Source #

decodeConduit :: forall a size m o. (MonadError Text m, MonadIO m, Integral size, Pinchable a) => size -> ConduitT ByteString o m (Int, a) Source #