Safe Haskell | Safe-Inferred |
---|---|
Language | Haskell2010 |
Synopsis
- isReadOnlyEnvironment :: forall emode. Mode emode => Bool
- data Limits = Limits {
- mapSize :: !Int
- maxDatabases :: !Int
- maxReaders :: !Int
- defaultLimits :: Limits
- openEnvironment :: forall emode. Mode emode => FilePath -> Limits -> IO (Environment emode)
- closeEnvironment :: forall emode. Mode emode => Environment emode -> IO ()
- getDatabase :: forall emode. Mode emode => Environment emode -> Maybe String -> IO (Database emode)
- closeDatabase :: forall emode. Mode emode => Database emode -> IO ()
- data MaybeTxn tmode a where
- data EitherTxn tmode a b where
- newtype UseUnsafeFFI = UseUnsafeFFI Bool
- readLMDB :: forall m emode tmode. (MonadIO m, Mode emode, Mode tmode, SubMode emode tmode) => Unfold m (ReadOptions, Database emode, EitherTxn tmode (Maybe ChunkSize) (Transaction tmode emode, Cursor)) (ByteString, ByteString)
- readLMDB' :: forall m emode tmode. (MonadIO m, Mode emode, Mode tmode, SubMode emode tmode) => Unfold m (ReadOptions, UseUnsafeFFI, Database emode, EitherTxn tmode (Maybe ChunkSize) (Transaction tmode emode, Cursor)) (ByteString, ByteString)
- unsafeReadLMDB :: forall m k v emode tmode. (MonadIO m, Mode emode, Mode tmode, SubMode emode tmode) => Unfold m (ReadOptions, Database emode, EitherTxn tmode (Maybe ChunkSize) (Transaction tmode emode, Cursor), CStringLen -> IO k, CStringLen -> IO v) (k, v)
- unsafeReadLMDB' :: forall m k v emode tmode. (MonadIO m, Mode emode, Mode tmode, SubMode emode tmode) => Unfold m (ReadOptions, UseUnsafeFFI, Database emode, EitherTxn tmode (Maybe ChunkSize) (Transaction tmode emode, Cursor), CStringLen -> IO k, CStringLen -> IO v) (k, v)
- data ReadLMDBFixed_ emode k v = ReadLMDBFixed_ {
- r_db :: Database emode
- r_kmap :: CStringLen -> IO k
- r_vmap :: CStringLen -> IO v
- r_newtxncurs :: Ptr MDB_env -> MDB_dbi_t -> TMVarS NumReaders -> IO (Ptr MDB_cursor, IOFinalizer)
- r_cursor_get :: Ptr MDB_cursor -> Ptr MDB_val -> Ptr MDB_val -> MDB_cursor_op_t -> IO CInt
- r_nextPrevOp :: MDB_cursor_op_t
- r_nextChunkOp :: ByteString -> ReadStart
- r_chunkSzInc :: Int
- r_chunkSzMax :: Int
- r_pk :: Ptr MDB_val
- r_pv :: Ptr MDB_val
- r_pref :: IOFinalizer
- getLMDB :: forall emode tmode. (Mode emode, Mode tmode, SubMode emode tmode) => Database emode -> MaybeTxn tmode (Transaction tmode emode) -> ByteString -> IO (Maybe ByteString)
- data Transaction tmode emode = Transaction !(Environment emode) !(Ptr MDB_txn)
- beginReadOnlyTransaction :: forall emode. Mode emode => Environment emode -> IO (Transaction ReadOnly emode)
- abortReadOnlyTransaction :: forall emode. Mode emode => Transaction ReadOnly emode -> IO ()
- withReadOnlyTransaction :: forall m a emode. (Mode emode, MonadBaseControl IO m, MonadIO m) => Environment emode -> (Transaction ReadOnly emode -> m a) -> m a
- newtype Cursor = Cursor (Ptr MDB_cursor)
- openCursor :: forall emode tmode. (Mode emode, Mode tmode, SubMode emode tmode) => Transaction tmode emode -> Database emode -> IO Cursor
- closeCursor :: Cursor -> IO ()
- withCursor :: forall m a emode tmode. (MonadBaseControl IO m, MonadIO m, Mode emode, Mode tmode, SubMode emode tmode) => Transaction tmode emode -> Database emode -> (Cursor -> m a) -> m a
- data ReadOptions = ReadOptions {}
- defaultReadOptions :: ReadOptions
- data ReadDirection
- data ReadStart
- = ReadBeg
- | ReadEnd
- | ReadGE !ByteString
- | ReadGT !ByteString
- | ReadLE !ByteString
- | ReadLT !ByteString
- beginReadWriteTransaction :: Environment ReadWrite -> IO (Transaction ReadWrite ReadWrite)
- abortReadWriteTransaction :: Transaction ReadWrite ReadWrite -> IO ()
- commitReadWriteTransaction :: Transaction ReadWrite ReadWrite -> IO ()
- withReadWriteTransaction :: forall m a. (MonadBaseControl IO m, MonadIO m) => Environment ReadWrite -> (Transaction ReadWrite ReadWrite -> m a) -> m a
- data OverwriteOptions m a where
- OverwriteAllow :: OverwriteOptions m ()
- OverwriteDisallow :: Either (WriteAccum m a) (WriteAccumWithOld m a) -> OverwriteOptions m a
- OverwriteAppend :: WriteAccum m a -> OverwriteOptions m a
- type WriteAccum m a = Fold m (ByteString, ByteString) a
- type WriteAccumWithOld m a = Fold m (ByteString, ByteString, ByteString) a
- newtype WriteOptions m a = WriteOptions {}
- type ShowKey = ByteString -> String
- type ShowValue = ByteString -> String
- writeAccumThrow :: Monad m => Maybe (ShowKey, ShowValue) -> WriteAccum m ()
- writeAccumThrowAllowSameValue :: Monad m => Maybe (ShowKey, ShowValue) -> WriteAccumWithOld m ()
- writeAccumIgnore :: Monad m => WriteAccum m ()
- writeAccumStop :: Monad m => WriteAccum m ()
- defaultWriteOptions :: WriteOptions m ()
- data ChunkSize
- = ChunkNumPairs !Int
- | ChunkBytes !Int
- chunkPairsFold :: forall m a. Monad m => ChunkSize -> Fold m (Seq (ByteString, ByteString)) a -> Fold m (ByteString, ByteString) a
- chunkPairs :: Monad m => ChunkSize -> Stream m (ByteString, ByteString) -> Stream m (Seq (ByteString, ByteString))
- writeLMDBChunk :: forall m a. (MonadBaseControl IO m, MonadIO m, MonadCatch m) => WriteOptions m a -> Database ReadWrite -> Seq (ByteString, ByteString) -> m a
- writeLMDBChunk' :: forall m a. (MonadBaseControl IO m, MonadIO m, MonadCatch m) => UseUnsafeFFI -> WriteOptions m a -> Database ReadWrite -> Seq (ByteString, ByteString) -> m a
- writeLMDB :: forall m a. (MonadIO m, MonadCatch m, MonadThrow m) => WriteOptions m a -> Database ReadWrite -> Transaction ReadWrite ReadWrite -> Fold m (ByteString, ByteString) a
- writeLMDB' :: forall m a. (MonadIO m, MonadCatch m, MonadThrow m) => UseUnsafeFFI -> WriteOptions m a -> Database ReadWrite -> Transaction ReadWrite ReadWrite -> Fold m (ByteString, ByteString) a
- waitReaders :: Mode emode => Environment emode -> IO ()
- clearDatabase :: Database ReadWrite -> IO ()
- deleteLMDB :: DeleteOptions -> Database emode -> Transaction ReadWrite emode -> ByteString -> IO ()
- newtype DeleteOptions = DeleteOptions {}
- defaultDeleteOptions :: DeleteOptions
- kibibyte :: Num a => a
- mebibyte :: Num a => a
- gibibyte :: Num a => a
- tebibyte :: Num a => a
- class Mode a where
- isReadOnlyMode :: a -> Bool
- data ReadWrite
- data ReadOnly
- type family SubMode emode tmode where ...
- data Environment emode = Environment !(Ptr MDB_env) !(TMVarS NumReaders, WriteLock, WriteThread, CloseDbLock)
- newtype TMVarS a = TMVarS (TMVar a)
- newTMVarSIO :: a -> IO (TMVarS a)
- takeTMVarS :: TMVarS a -> STM a
- putTMVarS :: TMVarS a -> a -> STM ()
- newtype NumReaders = NumReaders Int
- newtype WriteCounter = WriterCounter Int
- newtype WriteThread = WriteThread (MVar ThreadId)
- newtype WriteLock = WriteLock (MVar ())
- newtype CloseDbLock = CloseDbLock (MVar ())
- data Database emode = Database !(Environment emode) !MDB_dbi_t
- getNamedDb :: forall emode. Mode emode => Environment emode -> String -> IO (Maybe MDB_dbi_t)
- detectUserErrors :: Bool -> MVar ThreadId -> (String -> IO ()) -> IO ()
- liftedBracket2 :: MonadBaseControl IO m => m a -> (a -> m b) -> (a -> m b) -> (a -> m c) -> m c
- bracket2 :: IO a -> (a -> IO b) -> (a -> IO b) -> (a -> IO c) -> IO c
Documentation
isReadOnlyEnvironment :: forall emode. Mode emode => Bool Source #
LMDB environments have various limits on the size and number of databases and concurrent readers.
Limits | |
|
defaultLimits :: Limits Source #
The default limits are 1 MiB map size, 0 named databases (see Databases), and
126 concurrent readers. These can be adjusted freely, and in particular the mapSize
may be set
very large (limited only by available address space). However, LMDB is not optimized for a large
number of named databases so maxDatabases
should be kept to a minimum.
The default mapSize
is intentionally small, and should be changed to something appropriate for
your application. It ought to be a multiple of the OS page size, and should be chosen as large as
possible to accommodate future growth of the database(s). Once set for an environment, this limit
cannot be reduced to a value smaller than the space already consumed by the environment; however,
it can later be increased.
If you are going to use any named databases then you will need to change maxDatabases
to the
number of named databases you plan to use. However, you do not need to change this field if you
are only going to use the single main (unnamed) database.
openEnvironment :: forall emode. Mode emode => FilePath -> Limits -> IO (Environment emode) Source #
Open an LMDB environment in either ReadWrite
or ReadOnly
mode. The FilePath
argument may
be either a directory or a regular file, but it must already exist; when creating a new
environment, one should create an empty file or directory beforehand. If a regular file, an
additional file with "-lock" appended to the name is automatically created for the reader lock
table.
Note that an environment must have been opened in ReadWrite
mode at least once before it can be
opened in ReadOnly
mode.
An environment opened in ReadOnly
mode may still modify the reader lock table (except when the
filesystem is read-only, in which case no locks are used).
To satisfy certain low-level LMDB requirements, please do not have opened the same environment
(i.e., the same FilePath
) more than once in the same process at the same time. Furthermore,
please use the environment in the process that opened it (not after forking a new process).
closeEnvironment :: forall emode. Mode emode => Environment emode -> IO () Source #
Closes the given environment.
If you have merely a few dozen environments at most, there should be no need for this. (It is a common practice with LMDB to create one’s environments once and reuse them for the remainder of the program’s execution.)
To satisfy certain low-level LMDB requirements:
- Before calling this function, please call
closeDatabase
on all databases in the environment. - Before calling this function, close all cursors and commit/abort all transactions on the
environment. To make sure this requirement is satisified for read-only transactions, either (a)
call
waitReaders
or (b) pass precreated cursors/transactions toreadLMDB
andunsafeReadLMDB
. - After calling this function, do not use the environment or any related databases, transactions, and cursors.
getDatabase :: forall emode. Mode emode => Environment emode -> Maybe String -> IO (Database emode) Source #
Gets a database with the given name.
If only one database is desired within the environment, the name can be Nothing
(known as the
“unnamed database”).
If one or more named databases (a database with a Just
name) are desired, the maxDatabases
of
the environment’s limits should have been adjusted accordingly. The unnamed database will in this
case contain the names of the named databases as keys, which one is allowed to read but not
write.
Warning: When getting a named database for the first time (i.e., creating it), one must do so
in the ReadWrite
environment mode. (This restriction does not apply for the unnamed database.)
In this case, this function spawns a bound thread and creates a temporary read-write transaction
under the hood; see Transactions.
closeDatabase :: forall emode. Mode emode => Database emode -> IO () Source #
Closes the given database.
If you have merely a few dozen databases at most, there should be no need for this. (It is a common practice with LMDB to create one’s databases once and reuse them for the remainder of the program’s execution.)
To satisfy certain low-level LMDB requirements:
- Before calling this function, please make sure all read-write transactions that have modified the database have already been committed or aborted.
- After calling this function, do not use the database or any of its cursors again. To make sure
this requirement is satisfied for cursors on read-only transactions, either (a) call
waitReaders
or (b) pass precreated cursors/transactions toreadLMDB
andunsafeReadLMDB
.
data MaybeTxn tmode a where Source #
A type for an optional thing where we want to fix the transaction mode to ReadOnly
in the
nothing case. (Maybe
isn’t powerful enough for this.)
data EitherTxn tmode a b where Source #
A type for an Either
-like choice where we want to fix the transaction mode to ReadOnly
in
the Left
case. (Either
isn’t powerful enough for this.)
newtype UseUnsafeFFI Source #
Use unsafe
FFI calls under the hood. This can increase iteration speed, but one should
bear in mind that unsafe
FFI calls, since they block all other threads, can have an adverse
impact on the performance of the rest of the program.
Internal.
Instances
Show UseUnsafeFFI Source # | |
Defined in Streamly.External.LMDB.Internal showsPrec :: Int -> UseUnsafeFFI -> ShowS # show :: UseUnsafeFFI -> String # showList :: [UseUnsafeFFI] -> ShowS # |
readLMDB :: forall m emode tmode. (MonadIO m, Mode emode, Mode tmode, SubMode emode tmode) => Unfold m (ReadOptions, Database emode, EitherTxn tmode (Maybe ChunkSize) (Transaction tmode emode, Cursor)) (ByteString, ByteString) Source #
Creates an unfold with which we can stream key-value pairs from the given database.
If an existing transaction and cursor are not provided, there are two possibilities: (a) If a chunk size is not provided, a read-only transaction and cursor are automatically created for the entire duration of the unfold. (b) Otherwise, new transactions and cursors are automatically created according to the desired chunk size. In this case, each transaction (apart from the first one) starts as expected at the key next to (i.e., the largest/smallest key less/greater than) the previously encountered key.
If you want to iterate through a large database while avoiding a long-lived transaction (see
Transactions), it is your responsibility to either chunk up your usage of
readLMDB
(with which readStart
can help) or specify a chunk size as described above.
Runtime consideration: If you call readLMDB
very frequently without a precreated transaction
and cursor, you might find upon profiling that a significant time is being spent at
mdb_txn_begin
, or find yourself having to increase maxReaders
in the environment’s limits
because the transactions and cursors are not being garbage collected fast enough. In this case,
please consider precreating a transaction and cursor.
If you don’t want the overhead of intermediate ByteString
s (on your way to your eventual data
structures), use unsafeReadLMDB
instead.
readLMDB' :: forall m emode tmode. (MonadIO m, Mode emode, Mode tmode, SubMode emode tmode) => Unfold m (ReadOptions, UseUnsafeFFI, Database emode, EitherTxn tmode (Maybe ChunkSize) (Transaction tmode emode, Cursor)) (ByteString, ByteString) Source #
Similar to readLMDB
, except that it has an extra UseUnsafeFFI
parameter.
Internal.
unsafeReadLMDB :: forall m k v emode tmode. (MonadIO m, Mode emode, Mode tmode, SubMode emode tmode) => Unfold m (ReadOptions, Database emode, EitherTxn tmode (Maybe ChunkSize) (Transaction tmode emode, Cursor), CStringLen -> IO k, CStringLen -> IO v) (k, v) Source #
Similar to readLMDB
, except that the keys and values are not automatically converted into
Haskell ByteString
s.
To ensure safety, please make sure that the memory pointed to by the CStringLen
for each
key/value mapping function call is (a) only read (and not written to); and (b) not used after the
mapping function has returned. One way to transform the CStringLen
s to your desired data
structures is to use unsafePackCStringLen
.
unsafeReadLMDB' :: forall m k v emode tmode. (MonadIO m, Mode emode, Mode tmode, SubMode emode tmode) => Unfold m (ReadOptions, UseUnsafeFFI, Database emode, EitherTxn tmode (Maybe ChunkSize) (Transaction tmode emode, Cursor), CStringLen -> IO k, CStringLen -> IO v) (k, v) Source #
Similar to unsafeReadLMDB
, except that it has an extra UseUnsafeFFI
parameter.
Internal.
data ReadLMDBFixed_ emode k v Source #
State that stays fixed in readLMDB
iterations.
Internal.
ReadLMDBFixed_ | |
|
getLMDB :: forall emode tmode. (Mode emode, Mode tmode, SubMode emode tmode) => Database emode -> MaybeTxn tmode (Transaction tmode emode) -> ByteString -> IO (Maybe ByteString) Source #
Looks up the value for the given key in the given database.
If an existing transaction is not provided, a read-only transaction is automatically created internally.
Runtime consideration: If you call getLMDB
very frequently without a precreated transaction,
you might find upon profiling that a significant time is being spent at mdb_txn_begin
, or find
yourself having to increase maxReaders
in the environment’s limits because the transactions are
not being garbage collected fast enough. In this case, please consider precreating a transaction.
data Transaction tmode emode Source #
A read-only (tmode
: ReadOnly
) or read-write (tmode
: ReadWrite
) transaction.
emode
: the environment’s mode. Note: ReadOnly
environments can only have ReadOnly
transactions; we enforce this at the type level.
Transaction !(Environment emode) !(Ptr MDB_txn) |
beginReadOnlyTransaction :: forall emode. Mode emode => Environment emode -> IO (Transaction ReadOnly emode) Source #
Begins an LMDB read-only transaction on the given environment.
For read-only transactions returned from this function, it is your responsibility to (a) make
sure the transaction only gets used by a single readLMDB
, unsafeReadLMDB
, or getLMDB
at the
same time, (b) use the transaction only on databases in the environment on which the transaction
was begun, (c) make sure that those databases were already obtained before the transaction was
begun, (d) dispose of the transaction with abortReadOnlyTransaction
, and (e) be aware of the
caveats regarding long-lived transactions; see Transactions.
To easily manage a read-only transaction’s lifecycle, we suggest using withReadOnlyTransaction
.
abortReadOnlyTransaction :: forall emode. Mode emode => Transaction ReadOnly emode -> IO () Source #
Disposes of a read-only transaction created with beginReadOnlyTransaction
.
It is your responsibility to not use the transaction or any of its cursors afterwards.
withReadOnlyTransaction :: forall m a emode. (Mode emode, MonadBaseControl IO m, MonadIO m) => Environment emode -> (Transaction ReadOnly emode -> m a) -> m a Source #
Creates a temporary read-only transaction on which the provided action is performed, after which the transaction gets aborted. The transaction also gets aborted upon exceptions.
You have the same responsibilities as documented for beginReadOnlyTransaction
(apart from the
transaction disposal).
openCursor :: forall emode tmode. (Mode emode, Mode tmode, SubMode emode tmode) => Transaction tmode emode -> Database emode -> IO Cursor Source #
Opens a cursor for use with readLMDB
or unsafeReadLMDB
. It is your responsibility to (a)
make sure the cursor only gets used by a single readLMDB
or unsafeReadLMDB
at the same time,
(b) make sure the provided database is within the environment on which the provided transaction
was begun, and (c) dispose of the cursor with closeCursor
.
To easily manage a cursor’s lifecycle, we suggest using withCursor
.
closeCursor :: Cursor -> IO () Source #
Disposes of a cursor created with openCursor
.
withCursor :: forall m a emode tmode. (MonadBaseControl IO m, MonadIO m, Mode emode, Mode tmode, SubMode emode tmode) => Transaction tmode emode -> Database emode -> (Cursor -> m a) -> m a Source #
Creates a temporary cursor on which the provided action is performed, after which the cursor gets closed. The cursor also gets closed upon exceptions.
You have the same responsibilities as documented for openCursor
(apart from the cursor
disposal).
data ReadOptions Source #
Instances
Show ReadOptions Source # | |
Defined in Streamly.External.LMDB.Internal showsPrec :: Int -> ReadOptions -> ShowS # show :: ReadOptions -> String # showList :: [ReadOptions] -> ShowS # |
defaultReadOptions :: ReadOptions Source #
By default, we start reading from the beginning of the database (i.e., from the smallest key) and iterate in forward direction.
data ReadDirection Source #
Direction of key iteration.
Instances
Show ReadDirection Source # | |
Defined in Streamly.External.LMDB.Internal showsPrec :: Int -> ReadDirection -> ShowS # show :: ReadDirection -> String # showList :: [ReadDirection] -> ShowS # |
The key from which an iteration should start.
ReadBeg | Start from the smallest key. |
ReadEnd | Start from the largest key. |
ReadGE !ByteString | Start from the smallest key that is greater than or equal to the given key. |
ReadGT !ByteString | Start from the smallest key that is greater than the given key. |
ReadLE !ByteString | Start from the largest key that is less than or equal to the given key. |
ReadLT !ByteString | Start from the largest key that is less than the given key. |
beginReadWriteTransaction :: Environment ReadWrite -> IO (Transaction ReadWrite ReadWrite) Source #
Begins an LMDB read-write transaction on the given environment.
Unlike read-only transactions, a given read-write transaction is not allowed to stray from the OS
thread on which it was begun, and it is your responsibility to make sure of this. You can achieve
this with, e.g., runInBoundThread
.
Additionally, for read-write transactions returned from this function, it is your responsibility
to (a) use the transaction only on databases in the environment on which the transaction was
begun, (b) make sure that those databases were already obtained before the transaction was begun,
(c) commit/abort the transaction with commitReadWriteTransaction
/abortReadWriteTransaction
,
and (d) be aware of the caveats regarding long-lived transactions; see
Transactions.
To easily manage a read-write transaction’s lifecycle, we suggest using
withReadWriteTransaction
.
abortReadWriteTransaction :: Transaction ReadWrite ReadWrite -> IO () Source #
Aborts a read-write transaction created with beginReadWriteTransaction
.
It is your responsibility to not use the transaction afterwards.
commitReadWriteTransaction :: Transaction ReadWrite ReadWrite -> IO () Source #
Commits a read-write transaction created with beginReadWriteTransaction
.
It is your responsibility to not use the transaction afterwards.
withReadWriteTransaction :: forall m a. (MonadBaseControl IO m, MonadIO m) => Environment ReadWrite -> (Transaction ReadWrite ReadWrite -> m a) -> m a Source #
Spawns a new bound thread and creates a temporary read-write transaction on which the provided action is performed, after which the transaction gets committed. The transaction gets aborted upon exceptions.
You have the same responsibilities as documented for beginReadWriteTransaction
(apart from
running it on a bound thread and committing/aborting it).
data OverwriteOptions m a where Source #
OverwriteAllow
: When a key reoccurs, overwrite the value.OverwriteDisallow
: When a key reoccurs, don’t overwrite and hand the maladaptive key-value pair to the accumulator.OverwriteAppend
: Assume the input data is already increasing, which allows the use ofMDB_APPEND
under the hood and substantially improves write performance. Hand arriving key-value pairs in a maladaptive order to the accumulator.
OverwriteAllow :: OverwriteOptions m () | |
OverwriteDisallow :: Either (WriteAccum m a) (WriteAccumWithOld m a) -> OverwriteOptions m a | |
OverwriteAppend :: WriteAccum m a -> OverwriteOptions m a |
type WriteAccum m a = Fold m (ByteString, ByteString) a Source #
A fold for (key, new value)
.
type WriteAccumWithOld m a = Fold m (ByteString, ByteString, ByteString) a Source #
A fold for (key, new value, old value)
. This has the overhead of getting the old value.
newtype WriteOptions m a Source #
type ShowKey = ByteString -> String Source #
A function that shows a database key.
type ShowValue = ByteString -> String Source #
A function that shows a database value.
writeAccumThrow :: Monad m => Maybe (ShowKey, ShowValue) -> WriteAccum m () Source #
Throws upon the first maladaptive key. If desired, shows the maladaptive key-value pair in the exception.
writeAccumThrowAllowSameValue :: Monad m => Maybe (ShowKey, ShowValue) -> WriteAccumWithOld m () Source #
Throws upon the first maladaptive key where the old value differs from the new value. If desired, shows the maladaptive key-value pair with the old value in the exception.
writeAccumIgnore :: Monad m => WriteAccum m () Source #
Silently ignores maladaptive keys.
writeAccumStop :: Monad m => WriteAccum m () Source #
Gracefully stops upon the first maladaptive key.
defaultWriteOptions :: WriteOptions m () Source #
By default, we allow overwriting.
A chunk size.
ChunkNumPairs !Int | Chunk up key-value pairs by number of pairs. The final chunk can have a fewer number of pairs. |
ChunkBytes !Int | Chunk up key-value pairs by number of bytes. As soon as the byte count for the keys and values is reached, a new chunk is created (such that each chunk has at least one key-value pair and can end up with more than the desired number of bytes). The final chunk can have less than the desired number of bytes. |
chunkPairsFold :: forall m a. Monad m => ChunkSize -> Fold m (Seq (ByteString, ByteString)) a -> Fold m (ByteString, ByteString) a Source #
Chunks up the incoming stream of key-value pairs using the desired chunk size. One can try,
e.g., ChunkBytes mebibyte
(1 MiB chunks) and benchmark from there.
The chunks are processed using the desired fold.
chunkPairs :: Monad m => ChunkSize -> Stream m (ByteString, ByteString) -> Stream m (Seq (ByteString, ByteString)) Source #
Chunks up the incoming stream of key-value pairs using the desired chunk size. One can try,
e.g., ChunkBytes mebibyte
(1 MiB chunks) and benchmark from there.
writeLMDBChunk :: forall m a. (MonadBaseControl IO m, MonadIO m, MonadCatch m) => WriteOptions m a -> Database ReadWrite -> Seq (ByteString, ByteString) -> m a Source #
Writes a chunk of key-value pairs to the given database. Under the hood, it uses writeLMDB
surrounded with a withReadWriteTransaction
.
writeLMDBChunk' :: forall m a. (MonadBaseControl IO m, MonadIO m, MonadCatch m) => UseUnsafeFFI -> WriteOptions m a -> Database ReadWrite -> Seq (ByteString, ByteString) -> m a Source #
Similar to writeLMDBChunk
, except that it has an extra UseUnsafeFFI
parameter.
Internal.
writeLMDB :: forall m a. (MonadIO m, MonadCatch m, MonadThrow m) => WriteOptions m a -> Database ReadWrite -> Transaction ReadWrite ReadWrite -> Fold m (ByteString, ByteString) a Source #
Creates a fold that writes a stream of key-value pairs to the provided database using the provided transaction.
If you have a long stream of key-value pairs that you want to write to an LMDB database while avoiding a long-lived transaction (see Transactions), you can use the functions for chunked writing.
writeLMDB' :: forall m a. (MonadIO m, MonadCatch m, MonadThrow m) => UseUnsafeFFI -> WriteOptions m a -> Database ReadWrite -> Transaction ReadWrite ReadWrite -> Fold m (ByteString, ByteString) a Source #
Similar to writeLMDB
, except that it has an extra UseUnsafeFFI
parameter.
Internal.
waitReaders :: Mode emode => Environment emode -> IO () Source #
Waits for active read-only transactions on the given environment to finish. Note: This triggers garbage collection.
clearDatabase :: Database ReadWrite -> IO () Source #
Clears, i.e., removes all key-value pairs from, the given database.
Warning: Under the hood, this function spawns a bound thread and creates a potentially long-lived read-write transaction; see Transactions.
deleteLMDB :: DeleteOptions -> Database emode -> Transaction ReadWrite emode -> ByteString -> IO () Source #
Deletes the given key from the given database using the given transaction.
newtype DeleteOptions Source #
DeleteOptions | |
|
Instances
Show DeleteOptions Source # | |
Defined in Streamly.External.LMDB.Internal showsPrec :: Int -> DeleteOptions -> ShowS # show :: DeleteOptions -> String # showList :: [DeleteOptions] -> ShowS # |
defaultDeleteOptions :: DeleteOptions Source #
By default, we do not assume the key being deleted already exists in the database.
isReadOnlyMode :: a -> Bool Source #
Instances
Mode ReadOnly Source # | |
Defined in Streamly.External.LMDB.Internal isReadOnlyMode :: ReadOnly -> Bool Source # | |
Mode ReadWrite Source # | |
Defined in Streamly.External.LMDB.Internal isReadOnlyMode :: ReadWrite -> Bool Source # |
Instances
Mode ReadWrite Source # | |
Defined in Streamly.External.LMDB.Internal isReadOnlyMode :: ReadWrite -> Bool Source # |
Instances
Mode ReadOnly Source # | |
Defined in Streamly.External.LMDB.Internal isReadOnlyMode :: ReadOnly -> Bool Source # |
type family SubMode emode tmode where ... Source #
Enforces at the type level that ReadWrite
environments support both ReadWrite
and
ReadOnly
transactions, but ReadOnly
environments support only ReadOnly
transactions.
data Environment emode Source #
newTMVarSIO :: a -> IO (TMVarS a) Source #
takeTMVarS :: TMVarS a -> STM a Source #
newtype NumReaders Source #
Instances
Num NumReaders Source # | |
Defined in Streamly.External.LMDB.Internal (+) :: NumReaders -> NumReaders -> NumReaders # (-) :: NumReaders -> NumReaders -> NumReaders # (*) :: NumReaders -> NumReaders -> NumReaders # negate :: NumReaders -> NumReaders # abs :: NumReaders -> NumReaders # signum :: NumReaders -> NumReaders # fromInteger :: Integer -> NumReaders # | |
Eq NumReaders Source # | |
Defined in Streamly.External.LMDB.Internal (==) :: NumReaders -> NumReaders -> Bool # (/=) :: NumReaders -> NumReaders -> Bool # | |
Ord NumReaders Source # | |
Defined in Streamly.External.LMDB.Internal compare :: NumReaders -> NumReaders -> Ordering # (<) :: NumReaders -> NumReaders -> Bool # (<=) :: NumReaders -> NumReaders -> Bool # (>) :: NumReaders -> NumReaders -> Bool # (>=) :: NumReaders -> NumReaders -> Bool # max :: NumReaders -> NumReaders -> NumReaders # min :: NumReaders -> NumReaders -> NumReaders # |
newtype WriteCounter Source #
Instances
newtype WriteThread Source #
Keeps track of the ThreadId
of the current read-write transaction.
newtype CloseDbLock Source #
CloseDbLock (MVar ()) |
getNamedDb :: forall emode. Mode emode => Environment emode -> String -> IO (Maybe MDB_dbi_t) Source #
Utility function for getting a named database with a read-only transaction, returning Nothing
if it was not found.
Internal.
detectUserErrors :: Bool -> MVar ThreadId -> (String -> IO ()) -> IO () Source #
A utility function for detecting a few user errors.
Internal.
liftedBracket2 :: MonadBaseControl IO m => m a -> (a -> m b) -> (a -> m b) -> (a -> m c) -> m c Source #
liftedBracket2 acquire failure success thing
Same as Control.Exception.Lifted.bracket
(from lifted-base
) except it distinguishes between
failure and success.
Notes:
- When
acquire
,success
, orfailure
throw exceptions, any monadic side effects inm
will be discarded. - When
thing
throws an exception, any monadic side effects inm
produced bything
will be discarded, but the side effects ofacquire
and (non-excepting)failure
will be retained. - When (following a
thing
success)success
throws an exception, any monadic side effects inm
produced bysuccess
will be discarded, but the side effects ofacquire
,thing
, and (non-excepting)failure
will be retained.
Internal.