#include "inline.hs"

-- |
-- Module      : Streamly.Internal.FileSystem.File
-- Copyright   : (c) 2019 Composewell Technologies
--
-- License     : BSD3
-- Maintainer  : streamly@composewell.com
-- Portability : GHC
--
-- Read and write streams and arrays to and from files specified by their paths
-- in the file system. Unlike the handle based APIs which can have a read/write
-- session consisting of multiple reads and writes to the handle, these APIs
-- are one shot read or write APIs. These APIs open the file handle, perform
-- the requested operation and close the handle. Thease are safer compared to
-- the handle based APIs as there is no possibility of a file descriptor
-- leakage.
--
-- > import qualified Streamly.Internal.FileSystem.File as File
--

module Streamly.Internal.FileSystem.File
    (
    -- * Streaming IO
    -- | Stream data to or from a file or device sequentially.  When reading,
    -- the stream is lazy and generated on-demand as the consumer consumes it.
    -- Read IO requests to the IO device are performed in chunks limited to a
    -- maximum size of 32KiB, this is referred to as @defaultChunkSize@ in the
    -- documentation. One IO request may or may not read the full
    -- chunk. If the whole stream is not consumed, it is possible that we may
    -- read slightly more from the IO device than what the consumer needed.
    -- Unless specified otherwise in the API, writes are collected into chunks
    -- of @defaultChunkSize@ before they are written to the IO device.

    -- Streaming APIs work for all kind of devices, seekable or non-seekable;
    -- including disks, files, memory devices, terminals, pipes, sockets and
    -- fifos. While random access APIs work only for files or devices that have
    -- random access or seek capability for example disks, memory devices.
    -- Devices like terminals, pipes, sockets and fifos do not have random
    -- access capability.

    -- ** File IO Using Handle
      withFile

    -- ** Streams
    , read
    , readChunksWith
    , readChunks

    -- ** Unfolds
    , readerWith
    , reader
    -- , readShared
    -- , readUtf8
    -- , readLines
    -- , readFrames
    , chunkReaderWith
    , chunkReaderFromToWith
    , chunkReader

    -- ** Write To File
    , putChunk -- writeChunk?

    -- ** Folds
    , write
    -- , writeUtf8
    -- , writeUtf8ByLines
    -- , writeByFrames
    , writeWith
    , writeChunks

    -- ** Writing Streams
    , fromBytes -- putBytes?
    , fromBytesWith
    , fromChunks

    -- ** Append To File
    , writeAppend
    , writeAppendWith
    -- , appendShared
    , writeAppendArray
    , writeAppendChunks

    -- * Deprecated
    , readWithBufferOf
    , readChunksWithBufferOf
    , readChunksFromToWith
    , toBytes
    , toChunks
    , toChunksWithBufferOf
    , writeWithBufferOf
    , fromBytesWithBufferOf
    )
where

import Control.Monad.Catch (MonadCatch)
import Control.Monad.IO.Class (MonadIO(..))
import Data.Word (Word8)
import System.IO (Handle, openFile, IOMode(..), hClose)
import Prelude hiding (read)

import qualified Control.Monad.Catch as MC
import qualified System.IO as SIO

import Streamly.Data.Fold (groupsOf, drain)
import Streamly.Internal.Data.Array.Type (Array(..))
import Streamly.Internal.Data.Fold.Type (Fold(..))
import Streamly.Data.Stream (Stream)
import Streamly.Internal.Data.Unfold.Type (Unfold(..))
-- import Streamly.String (encodeUtf8, decodeUtf8, foldLines)
import Streamly.Internal.System.IO (defaultChunkSize)

import qualified Streamly.Internal.Data.Array as A
import qualified Streamly.Data.Stream as S
import qualified Streamly.Data.Unfold as UF
import qualified Streamly.Internal.Data.Array.Type as IA (pinnedChunksOf)
import qualified Streamly.Internal.Data.Unfold as UF (bracketIO)
import qualified Streamly.Internal.Data.Fold.Type as FL
    (Step(..), snoc, reduce)
import qualified Streamly.Internal.FileSystem.Handle as FH

-------------------------------------------------------------------------------
-- References
-------------------------------------------------------------------------------
--
-- The following references may be useful to build an understanding about the
-- file API design:
--
-- http://www.linux-mag.com/id/308/ for blocking/non-blocking IO on linux.
-- https://lwn.net/Articles/612483/ Non-blocking buffered file read operations
-- https://en.wikipedia.org/wiki/C_file_input/output for C APIs.
-- https://docs.oracle.com/javase/tutorial/essential/io/file.html for Java API.
-- https://www.w3.org/TR/FileAPI/ for http file API.

-------------------------------------------------------------------------------
-- Safe file reading
-------------------------------------------------------------------------------

-- | @'withFile' name mode act@ opens a file using 'openFile' and passes
-- the resulting handle to the computation @act@.  The handle will be
-- closed on exit from 'withFile', whether by normal termination or by
-- raising an exception.  If closing the handle raises an exception, then
-- this exception will be raised by 'withFile' rather than any exception
-- raised by 'act'.
--
-- /Pre-release/
--
{-# INLINE withFile #-}
withFile :: (MonadIO m, MonadCatch m)
    => FilePath -> IOMode -> (Handle -> Stream m a) -> Stream m a
withFile :: forall (m :: * -> *) a.
(MonadIO m, MonadCatch m) =>
FilePath -> IOMode -> (Handle -> Stream m a) -> Stream m a
withFile FilePath
file IOMode
mode = IO Handle
-> (Handle -> IO ()) -> (Handle -> Stream m a) -> Stream m a
forall (m :: * -> *) b c a.
(MonadIO m, MonadCatch m) =>
IO b -> (b -> IO c) -> (b -> Stream m a) -> Stream m a
S.bracketIO (FilePath -> IOMode -> IO Handle
openFile FilePath
file IOMode
mode) Handle -> IO ()
hClose

-- | Transform an 'Unfold' from a 'Handle' to an unfold from a 'FilePath'.  The
-- resulting unfold opens a handle in 'ReadMode', uses it using the supplied
-- unfold and then makes sure that the handle is closed on normal termination
-- or in case of an exception.  If closing the handle raises an exception, then
-- this exception will be raised by 'usingFile'.
--
-- /Pre-release/
--
{-# INLINE usingFile #-}
usingFile :: (MonadIO m, MonadCatch m)
    => Unfold m Handle a -> Unfold m FilePath a
usingFile :: forall (m :: * -> *) a.
(MonadIO m, MonadCatch m) =>
Unfold m Handle a -> Unfold m FilePath a
usingFile = (FilePath -> IO Handle)
-> (Handle -> IO ()) -> Unfold m Handle a -> Unfold m FilePath a
forall (m :: * -> *) a c d b.
(MonadIO m, MonadCatch m) =>
(a -> IO c) -> (c -> IO d) -> Unfold m c b -> Unfold m a b
UF.bracketIO (FilePath -> IOMode -> IO Handle
`openFile` IOMode
ReadMode) Handle -> IO ()
hClose

{-# INLINE usingFile2 #-}
usingFile2 :: (MonadIO m, MonadCatch m)
    => Unfold m (x, Handle) a -> Unfold m (x, FilePath) a
usingFile2 :: forall (m :: * -> *) x a.
(MonadIO m, MonadCatch m) =>
Unfold m (x, Handle) a -> Unfold m (x, FilePath) a
usingFile2 = ((x, FilePath) -> IO (x, Handle))
-> ((x, Handle) -> IO ())
-> Unfold m (x, Handle) a
-> Unfold m (x, FilePath) a
forall (m :: * -> *) a c d b.
(MonadIO m, MonadCatch m) =>
(a -> IO c) -> (c -> IO d) -> Unfold m c b -> Unfold m a b
UF.bracketIO (x, FilePath) -> IO (x, Handle)
forall {a}. (a, FilePath) -> IO (a, Handle)
before (x, Handle) -> IO ()
forall {a}. (a, Handle) -> IO ()
after

    where

    before :: (a, FilePath) -> IO (a, Handle)
before (a
x, FilePath
file) =  do
        Handle
h <- FilePath -> IOMode -> IO Handle
openFile FilePath
file IOMode
ReadMode
        (a, Handle) -> IO (a, Handle)
forall (m :: * -> *) a. Monad m => a -> m a
return (a
x, Handle
h)

    after :: (a, Handle) -> IO ()
after (a
_, Handle
h) = Handle -> IO ()
hClose Handle
h

{-# INLINE usingFile3 #-}
usingFile3 :: (MonadIO m, MonadCatch m)
    => Unfold m (x, y, z, Handle) a -> Unfold m (x, y, z, FilePath) a
usingFile3 :: forall (m :: * -> *) x y z a.
(MonadIO m, MonadCatch m) =>
Unfold m (x, y, z, Handle) a -> Unfold m (x, y, z, FilePath) a
usingFile3 = ((x, y, z, FilePath) -> IO (x, y, z, Handle))
-> ((x, y, z, Handle) -> IO ())
-> Unfold m (x, y, z, Handle) a
-> Unfold m (x, y, z, FilePath) a
forall (m :: * -> *) a c d b.
(MonadIO m, MonadCatch m) =>
(a -> IO c) -> (c -> IO d) -> Unfold m c b -> Unfold m a b
UF.bracketIO (x, y, z, FilePath) -> IO (x, y, z, Handle)
forall {a} {b} {c}. (a, b, c, FilePath) -> IO (a, b, c, Handle)
before (x, y, z, Handle) -> IO ()
forall {a} {b} {c}. (a, b, c, Handle) -> IO ()
after

    where

    before :: (a, b, c, FilePath) -> IO (a, b, c, Handle)
before (a
x, b
y, c
z, FilePath
file) =  do
        Handle
h <- FilePath -> IOMode -> IO Handle
openFile FilePath
file IOMode
ReadMode
        (a, b, c, Handle) -> IO (a, b, c, Handle)
forall (m :: * -> *) a. Monad m => a -> m a
return (a
x, b
y, c
z, Handle
h)

    after :: (a, b, c, Handle) -> IO ()
after (a
_, b
_, c
_, Handle
h) = Handle -> IO ()
hClose Handle
h

-------------------------------------------------------------------------------
-- Array IO (Input)
-------------------------------------------------------------------------------

-- TODO readArrayOf

-------------------------------------------------------------------------------
-- Array IO (output)
-------------------------------------------------------------------------------

-- | Write an array to a file. Overwrites the file if it exists.
--
-- /Pre-release/
--
{-# INLINABLE putChunk #-}
putChunk :: FilePath -> Array a -> IO ()
putChunk :: forall a. FilePath -> Array a -> IO ()
putChunk FilePath
file Array a
arr = FilePath -> IOMode -> (Handle -> IO ()) -> IO ()
forall r. FilePath -> IOMode -> (Handle -> IO r) -> IO r
SIO.withFile FilePath
file IOMode
WriteMode (Handle -> Array a -> IO ()
forall (m :: * -> *) a. MonadIO m => Handle -> Array a -> m ()
`FH.putChunk` Array a
arr)

-- | append an array to a file.
--
-- /Pre-release/
--
{-# INLINABLE writeAppendArray #-}
writeAppendArray :: FilePath -> Array a -> IO ()
writeAppendArray :: forall a. FilePath -> Array a -> IO ()
writeAppendArray FilePath
file Array a
arr = FilePath -> IOMode -> (Handle -> IO ()) -> IO ()
forall r. FilePath -> IOMode -> (Handle -> IO r) -> IO r
SIO.withFile FilePath
file IOMode
AppendMode (Handle -> Array a -> IO ()
forall (m :: * -> *) a. MonadIO m => Handle -> Array a -> m ()
`FH.putChunk` Array a
arr)

-------------------------------------------------------------------------------
-- Stream of Arrays IO
-------------------------------------------------------------------------------

-- | @readChunksWith size file@ reads a stream of arrays from file @file@.
-- The maximum size of a single array is specified by @size@. The actual size
-- read may be less than or equal to @size@.
--
-- /Pre-release/
--
{-# INLINE readChunksWith #-}
readChunksWith :: (MonadIO m, MonadCatch m)
    => Int -> FilePath -> Stream m (Array Word8)
readChunksWith :: forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Int -> FilePath -> Stream m (Array Word8)
readChunksWith Int
size FilePath
file =
    FilePath
-> IOMode
-> (Handle -> Stream m (Array Word8))
-> Stream m (Array Word8)
forall (m :: * -> *) a.
(MonadIO m, MonadCatch m) =>
FilePath -> IOMode -> (Handle -> Stream m a) -> Stream m a
withFile FilePath
file IOMode
ReadMode (Int -> Handle -> Stream m (Array Word8)
forall (m :: * -> *).
MonadIO m =>
Int -> Handle -> Stream m (Array Word8)
FH.readChunksWith Int
size)

{-# DEPRECATED toChunksWithBufferOf "Please use 'readChunksWith' instead" #-}
{-# INLINE toChunksWithBufferOf #-}
toChunksWithBufferOf :: (MonadIO m, MonadCatch m)
    => Int -> FilePath -> Stream m (Array Word8)
toChunksWithBufferOf :: forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Int -> FilePath -> Stream m (Array Word8)
toChunksWithBufferOf = Int -> FilePath -> Stream m (Array Word8)
forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Int -> FilePath -> Stream m (Array Word8)
readChunksWith

-- XXX read 'Array a' instead of Word8
--
-- | @readChunks file@ reads a stream of arrays from file @file@.
-- The maximum size of a single array is limited to @defaultChunkSize@. The
-- actual size read may be less than @defaultChunkSize@.
--
-- > readChunks = readChunksWith defaultChunkSize
--
-- /Pre-release/
--
{-# INLINE readChunks #-}
readChunks :: (MonadIO m, MonadCatch m)
    => FilePath -> Stream m (Array Word8)
readChunks :: forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
FilePath -> Stream m (Array Word8)
readChunks = Int -> FilePath -> Stream m (Array Word8)
forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Int -> FilePath -> Stream m (Array Word8)
readChunksWith Int
defaultChunkSize

{-# DEPRECATED toChunks "Please use 'readChunks' instead" #-}
{-# INLINE toChunks #-}
toChunks :: (MonadIO m, MonadCatch m) => FilePath -> Stream m (Array Word8)
toChunks :: forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
FilePath -> Stream m (Array Word8)
toChunks = FilePath -> Stream m (Array Word8)
forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
FilePath -> Stream m (Array Word8)
readChunks

-------------------------------------------------------------------------------
-- Read File to Stream
-------------------------------------------------------------------------------

-- TODO for concurrent streams implement readahead IO. We can send multiple
-- read requests at the same time. For serial case we can use async IO. We can
-- also control the read throughput in mbps or IOPS.

-- | Unfold the tuple @(bufsize, filepath)@ into a stream of 'Word8' arrays.
-- Read requests to the IO device are performed using a buffer of size
-- @bufsize@. The size of an array in the resulting stream is always less than
-- or equal to @bufsize@.
--
-- /Pre-release/
--
{-# INLINE chunkReaderWith #-}
chunkReaderWith :: (MonadIO m, MonadCatch m)
    => Unfold m (Int, FilePath) (Array Word8)
chunkReaderWith :: forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Unfold m (Int, FilePath) (Array Word8)
chunkReaderWith = Unfold m (Int, Handle) (Array Word8)
-> Unfold m (Int, FilePath) (Array Word8)
forall (m :: * -> *) x a.
(MonadIO m, MonadCatch m) =>
Unfold m (x, Handle) a -> Unfold m (x, FilePath) a
usingFile2 Unfold m (Int, Handle) (Array Word8)
forall (m :: * -> *).
MonadIO m =>
Unfold m (Int, Handle) (Array Word8)
FH.chunkReaderWith

{-# DEPRECATED readChunksWithBufferOf
    "Please use 'chunkReaderWith' instead" #-}
{-# INLINE readChunksWithBufferOf #-}
readChunksWithBufferOf :: (MonadIO m, MonadCatch m)
    => Unfold m (Int, FilePath) (Array Word8)
readChunksWithBufferOf :: forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Unfold m (Int, FilePath) (Array Word8)
readChunksWithBufferOf = Unfold m (Int, FilePath) (Array Word8)
forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Unfold m (Int, FilePath) (Array Word8)
chunkReaderWith

-- | Unfold the tuple @(from, to, bufsize, filepath)@ into a stream
-- of 'Word8' arrays.
-- Read requests to the IO device are performed using a buffer of size
-- @bufsize@ starting from absolute offset of @from@ till the absolute
-- position of @to@. The size of an array in the resulting stream is always
-- less than or equal to @bufsize@.
--
-- /Pre-release/
{-# INLINE chunkReaderFromToWith #-}
chunkReaderFromToWith :: (MonadIO m, MonadCatch m) =>
    Unfold m (Int, Int, Int, FilePath) (Array Word8)
chunkReaderFromToWith :: forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Unfold m (Int, Int, Int, FilePath) (Array Word8)
chunkReaderFromToWith = Unfold m (Int, Int, Int, Handle) (Array Word8)
-> Unfold m (Int, Int, Int, FilePath) (Array Word8)
forall (m :: * -> *) x y z a.
(MonadIO m, MonadCatch m) =>
Unfold m (x, y, z, Handle) a -> Unfold m (x, y, z, FilePath) a
usingFile3 Unfold m (Int, Int, Int, Handle) (Array Word8)
forall (m :: * -> *).
MonadIO m =>
Unfold m (Int, Int, Int, Handle) (Array Word8)
FH.chunkReaderFromToWith

{-# DEPRECATED readChunksFromToWith
    "Please use 'chunkReaderFromToWith' instead" #-}
{-# INLINE readChunksFromToWith #-}
readChunksFromToWith :: (MonadIO m, MonadCatch m) =>
    Unfold m (Int, Int, Int, FilePath) (Array Word8)
readChunksFromToWith :: forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Unfold m (Int, Int, Int, FilePath) (Array Word8)
readChunksFromToWith = Unfold m (Int, Int, Int, FilePath) (Array Word8)
forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Unfold m (Int, Int, Int, FilePath) (Array Word8)
chunkReaderFromToWith

-- | Unfolds a 'FilePath' into a stream of 'Word8' arrays. Requests to the IO
-- device are performed using a buffer of size
-- 'Streamly.Internal.Data.Array.Type.defaultChunkSize'. The
-- size of arrays in the resulting stream are therefore less than or equal to
-- 'Streamly.Internal.Data.Array.Type.defaultChunkSize'.
--
-- /Pre-release/
{-# INLINE chunkReader #-}
chunkReader :: (MonadIO m, MonadCatch m) => Unfold m FilePath (Array Word8)
chunkReader :: forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Unfold m FilePath (Array Word8)
chunkReader = Unfold m Handle (Array Word8) -> Unfold m FilePath (Array Word8)
forall (m :: * -> *) a.
(MonadIO m, MonadCatch m) =>
Unfold m Handle a -> Unfold m FilePath a
usingFile Unfold m Handle (Array Word8)
forall (m :: * -> *). MonadIO m => Unfold m Handle (Array Word8)
FH.chunkReader

-- | Unfolds the tuple @(bufsize, filepath)@ into a byte stream, read requests
-- to the IO device are performed using buffers of @bufsize@.
--
-- /Pre-release/
{-# INLINE readerWith #-}
readerWith :: (MonadIO m, MonadCatch m) => Unfold m (Int, FilePath) Word8
readerWith :: forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Unfold m (Int, FilePath) Word8
readerWith = Unfold m (Int, Handle) Word8 -> Unfold m (Int, FilePath) Word8
forall (m :: * -> *) x a.
(MonadIO m, MonadCatch m) =>
Unfold m (x, Handle) a -> Unfold m (x, FilePath) a
usingFile2 Unfold m (Int, Handle) Word8
forall (m :: * -> *). MonadIO m => Unfold m (Int, Handle) Word8
FH.readerWith

{-# DEPRECATED readWithBufferOf "Please use 'readerWith' instead" #-}
{-# INLINE readWithBufferOf #-}
readWithBufferOf :: (MonadIO m, MonadCatch m) =>
    Unfold m (Int, FilePath) Word8
readWithBufferOf :: forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Unfold m (Int, FilePath) Word8
readWithBufferOf = Unfold m (Int, FilePath) Word8
forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Unfold m (Int, FilePath) Word8
readerWith

-- | Unfolds a file path into a byte stream. IO requests to the device are
-- performed in sizes of
-- 'Streamly.Internal.Data.Array.Type.defaultChunkSize'.
--
-- /Pre-release/
{-# INLINE reader #-}
reader :: (MonadIO m, MonadCatch m) => Unfold m FilePath Word8
reader :: forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Unfold m FilePath Word8
reader = Unfold m (Array Word8) Word8
-> Unfold m FilePath (Array Word8) -> Unfold m FilePath Word8
forall (m :: * -> *) b c a.
Monad m =>
Unfold m b c -> Unfold m a b -> Unfold m a c
UF.many Unfold m (Array Word8) Word8
forall (m :: * -> *) a. (Monad m, Unbox a) => Unfold m (Array a) a
A.reader (Unfold m Handle (Array Word8) -> Unfold m FilePath (Array Word8)
forall (m :: * -> *) a.
(MonadIO m, MonadCatch m) =>
Unfold m Handle a -> Unfold m FilePath a
usingFile Unfold m Handle (Array Word8)
forall (m :: * -> *). MonadIO m => Unfold m Handle (Array Word8)
FH.chunkReader)

-- | Generate a stream of bytes from a file specified by path. The stream ends
-- when EOF is encountered. File is locked using multiple reader and single
-- writer locking mode.
--
-- /Pre-release/
--
{-# INLINE read #-}
read :: (MonadIO m, MonadCatch m) => FilePath -> Stream m Word8
read :: forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
FilePath -> Stream m Word8
read FilePath
file = Stream m (Array Word8) -> Stream m Word8
forall (m :: * -> *) a.
(Monad m, Unbox a) =>
Stream m (Array a) -> Stream m a
A.concat (Stream m (Array Word8) -> Stream m Word8)
-> Stream m (Array Word8) -> Stream m Word8
forall a b. (a -> b) -> a -> b
$ FilePath
-> IOMode
-> (Handle -> Stream m (Array Word8))
-> Stream m (Array Word8)
forall (m :: * -> *) a.
(MonadIO m, MonadCatch m) =>
FilePath -> IOMode -> (Handle -> Stream m a) -> Stream m a
withFile FilePath
file IOMode
ReadMode Handle -> Stream m (Array Word8)
forall (m :: * -> *). MonadIO m => Handle -> Stream m (Array Word8)
FH.readChunks

{-# DEPRECATED toBytes "Please use 'read' instead"  #-}
{-# INLINE toBytes #-}
toBytes :: (MonadIO m, MonadCatch m) => FilePath -> Stream m Word8
toBytes :: forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
FilePath -> Stream m Word8
toBytes = FilePath -> Stream m Word8
forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
FilePath -> Stream m Word8
read

{-
-- | Generate a stream of elements of the given type from a file 'Handle'. The
-- stream ends when EOF is encountered. File is not locked for exclusive reads,
-- writers can keep writing to the file.
--
-- @since 0.7.0
{-# INLINE readShared #-}
readShared :: MonadIO m => Handle -> Stream m Word8
readShared = undefined
-}

-------------------------------------------------------------------------------
-- Writing
-------------------------------------------------------------------------------

{-# INLINE fromChunksMode #-}
fromChunksMode :: (MonadIO m, MonadCatch m)
    => IOMode -> FilePath -> Stream m (Array a) -> m ()
fromChunksMode :: forall (m :: * -> *) a.
(MonadIO m, MonadCatch m) =>
IOMode -> FilePath -> Stream m (Array a) -> m ()
fromChunksMode IOMode
mode FilePath
file Stream m (Array a)
xs = Fold m () () -> Stream m () -> m ()
forall (m :: * -> *) a b.
Monad m =>
Fold m a b -> Stream m a -> m b
S.fold Fold m () ()
forall (m :: * -> *) a. Monad m => Fold m a ()
drain (Stream m () -> m ()) -> Stream m () -> m ()
forall a b. (a -> b) -> a -> b
$
    FilePath -> IOMode -> (Handle -> Stream m ()) -> Stream m ()
forall (m :: * -> *) a.
(MonadIO m, MonadCatch m) =>
FilePath -> IOMode -> (Handle -> Stream m a) -> Stream m a
withFile FilePath
file IOMode
mode (\Handle
h -> (Array a -> m ()) -> Stream m (Array a) -> Stream m ()
forall (m :: * -> *) a b.
Monad m =>
(a -> m b) -> Stream m a -> Stream m b
S.mapM (Handle -> Array a -> m ()
forall (m :: * -> *) a. MonadIO m => Handle -> Array a -> m ()
FH.putChunk Handle
h) Stream m (Array a)
xs)

-- | Write a stream of arrays to a file. Overwrites the file if it exists.
--
-- /Pre-release/
--
{-# INLINE fromChunks #-}
fromChunks :: (MonadIO m, MonadCatch m)
    => FilePath -> Stream m (Array a) -> m ()
fromChunks :: forall (m :: * -> *) a.
(MonadIO m, MonadCatch m) =>
FilePath -> Stream m (Array a) -> m ()
fromChunks = IOMode -> FilePath -> Stream m (Array a) -> m ()
forall (m :: * -> *) a.
(MonadIO m, MonadCatch m) =>
IOMode -> FilePath -> Stream m (Array a) -> m ()
fromChunksMode IOMode
WriteMode

-- GHC buffer size dEFAULT_FD_BUFFER_SIZE=8192 bytes.
--
-- XXX test this
-- Note that if you use a chunk size less than 8K (GHC's default buffer
-- size) then you are advised to use 'NOBuffering' mode on the 'Handle' in case you
-- do not want buffering to occur at GHC level as well. Same thing applies to
-- writes as well.

-- | Like 'write' but provides control over the write buffer. Output will
-- be written to the IO device as soon as we collect the specified number of
-- input elements.
--
-- /Pre-release/
--
{-# INLINE fromBytesWith #-}
fromBytesWith :: (MonadIO m, MonadCatch m)
    => Int -> FilePath -> Stream m Word8 -> m ()
fromBytesWith :: forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Int -> FilePath -> Stream m Word8 -> m ()
fromBytesWith Int
n FilePath
file Stream m Word8
xs = FilePath -> Stream m (Array Word8) -> m ()
forall (m :: * -> *) a.
(MonadIO m, MonadCatch m) =>
FilePath -> Stream m (Array a) -> m ()
fromChunks FilePath
file (Stream m (Array Word8) -> m ()) -> Stream m (Array Word8) -> m ()
forall a b. (a -> b) -> a -> b
$ Int -> Stream m Word8 -> Stream m (Array Word8)
forall (m :: * -> *) a.
(MonadIO m, Unbox a) =>
Int -> Stream m a -> Stream m (Array a)
IA.pinnedChunksOf Int
n Stream m Word8
xs

{-# DEPRECATED fromBytesWithBufferOf "Please use 'fromBytesWith' instead"  #-}
{-# INLINE fromBytesWithBufferOf #-}
fromBytesWithBufferOf :: (MonadIO m, MonadCatch m)
    => Int -> FilePath -> Stream m Word8 -> m ()
fromBytesWithBufferOf :: forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Int -> FilePath -> Stream m Word8 -> m ()
fromBytesWithBufferOf = Int -> FilePath -> Stream m Word8 -> m ()
forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Int -> FilePath -> Stream m Word8 -> m ()
fromBytesWith

-- > write = 'writeWith' defaultChunkSize
--
-- | Write a byte stream to a file. Combines the bytes in chunks of size
-- up to 'A.defaultChunkSize' before writing. If the file exists it is
-- truncated to zero size before writing. If the file does not exist it is
-- created. File is locked using single writer locking mode.
--
-- /Pre-release/
{-# INLINE fromBytes #-}
fromBytes :: (MonadIO m, MonadCatch m) => FilePath -> Stream m Word8 -> m ()
fromBytes :: forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
FilePath -> Stream m Word8 -> m ()
fromBytes = Int -> FilePath -> Stream m Word8 -> m ()
forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Int -> FilePath -> Stream m Word8 -> m ()
fromBytesWith Int
defaultChunkSize

{-
{-# INLINE write #-}
write :: (MonadIO m, Storable a) => Handle -> Stream m a -> m ()
write = toHandleWith A.defaultChunkSize
-}

-- | Write a stream of chunks to a handle. Each chunk in the stream is written
-- to the device as a separate IO request.
--
-- /Pre-release/
{-# INLINE writeChunks #-}
writeChunks :: (MonadIO m, MonadCatch m)
    => FilePath -> Fold m (Array a) ()
writeChunks :: forall (m :: * -> *) a.
(MonadIO m, MonadCatch m) =>
FilePath -> Fold m (Array a) ()
writeChunks FilePath
path = ((Fold m (Array a) (), Handle)
 -> Array a -> m (Step (Fold m (Array a) (), Handle) ()))
-> m (Step (Fold m (Array a) (), Handle) ())
-> ((Fold m (Array a) (), Handle) -> m ())
-> ((Fold m (Array a) (), Handle) -> m ())
-> Fold m (Array a) ()
forall (m :: * -> *) a b s.
(s -> a -> m (Step s b))
-> m (Step s b) -> (s -> m b) -> (s -> m b) -> Fold m a b
Fold (Fold m (Array a) (), Handle)
-> Array a -> m (Step (Fold m (Array a) (), Handle) ())
forall {m :: * -> *} {a} {b} {b}.
(MonadCatch m, MonadIO m) =>
(Fold m a b, Handle) -> a -> m (Step (Fold m a b, Handle) b)
step m (Step (Fold m (Array a) (), Handle) ())
forall {a} {b}. m (Step (Fold m (Array a) (), Handle) b)
initial (Fold m (Array a) (), Handle) -> m ()
forall {m :: * -> *} {p}. Monad m => p -> m ()
extract (Fold m (Array a) (), Handle) -> m ()
forall {m :: * -> *} {a}.
MonadIO m =>
(Fold m a (), Handle) -> m ()
final
    where
    initial :: m (Step (Fold m (Array a) (), Handle) b)
initial = do
        Handle
h <- IO Handle -> m Handle
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (FilePath -> IOMode -> IO Handle
openFile FilePath
path IOMode
WriteMode)
        Fold m (Array a) ()
fld <- Fold m (Array a) () -> m (Fold m (Array a) ())
forall (m :: * -> *) a b. Monad m => Fold m a b -> m (Fold m a b)
FL.reduce (Handle -> Fold m (Array a) ()
forall (m :: * -> *) a. MonadIO m => Handle -> Fold m (Array a) ()
FH.writeChunks Handle
h)
                m (Fold m (Array a) ()) -> m () -> m (Fold m (Array a) ())
forall (m :: * -> *) a b. MonadCatch m => m a -> m b -> m a
`MC.onException` IO () -> m ()
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (Handle -> IO ()
hClose Handle
h)
        Step (Fold m (Array a) (), Handle) b
-> m (Step (Fold m (Array a) (), Handle) b)
forall (m :: * -> *) a. Monad m => a -> m a
return (Step (Fold m (Array a) (), Handle) b
 -> m (Step (Fold m (Array a) (), Handle) b))
-> Step (Fold m (Array a) (), Handle) b
-> m (Step (Fold m (Array a) (), Handle) b)
forall a b. (a -> b) -> a -> b
$ (Fold m (Array a) (), Handle)
-> Step (Fold m (Array a) (), Handle) b
forall s b. s -> Step s b
FL.Partial (Fold m (Array a) ()
fld, Handle
h)
    step :: (Fold m a b, Handle) -> a -> m (Step (Fold m a b, Handle) b)
step (Fold m a b
fld, Handle
h) a
x = do
        Fold m a b
r <- Fold m a b -> a -> m (Fold m a b)
forall (m :: * -> *) a b.
Monad m =>
Fold m a b -> a -> m (Fold m a b)
FL.snoc Fold m a b
fld a
x m (Fold m a b) -> m () -> m (Fold m a b)
forall (m :: * -> *) a b. MonadCatch m => m a -> m b -> m a
`MC.onException` IO () -> m ()
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (Handle -> IO ()
hClose Handle
h)
        Step (Fold m a b, Handle) b -> m (Step (Fold m a b, Handle) b)
forall (m :: * -> *) a. Monad m => a -> m a
return (Step (Fold m a b, Handle) b -> m (Step (Fold m a b, Handle) b))
-> Step (Fold m a b, Handle) b -> m (Step (Fold m a b, Handle) b)
forall a b. (a -> b) -> a -> b
$ (Fold m a b, Handle) -> Step (Fold m a b, Handle) b
forall s b. s -> Step s b
FL.Partial (Fold m a b
r, Handle
h)

    extract :: p -> m ()
extract p
_ = () -> m ()
forall (m :: * -> *) a. Monad m => a -> m a
return ()

    final :: (Fold m a (), Handle) -> m ()
final (Fold s -> a -> m (Step s ())
_ m (Step s ())
initial1 s -> m ()
_ s -> m ()
final1, Handle
h) = do
        IO () -> m ()
forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO (IO () -> m ()) -> IO () -> m ()
forall a b. (a -> b) -> a -> b
$ Handle -> IO ()
hClose Handle
h
        Step s ()
res <- m (Step s ())
initial1
        case Step s ()
res of
            FL.Partial s
fs -> s -> m ()
final1 s
fs
            FL.Done () -> () -> m ()
forall (m :: * -> *) a. Monad m => a -> m a
return ()

-- | @writeWith chunkSize handle@ writes the input stream to @handle@.
-- Bytes in the input stream are collected into a buffer until we have a chunk
-- of size @chunkSize@ and then written to the IO device.
--
-- /Pre-release/
{-# INLINE writeWith #-}
writeWith :: (MonadIO m, MonadCatch m)
    => Int -> FilePath -> Fold m Word8 ()
writeWith :: forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Int -> FilePath -> Fold m Word8 ()
writeWith Int
n FilePath
path =
    Int
-> Fold m Word8 (Array Word8)
-> Fold m (Array Word8) ()
-> Fold m Word8 ()
forall (m :: * -> *) a b c.
Monad m =>
Int -> Fold m a b -> Fold m b c -> Fold m a c
groupsOf Int
n (Int -> Fold m Word8 (Array Word8)
forall (m :: * -> *) a.
(MonadIO m, Unbox a) =>
Int -> Fold m a (Array a)
A.unsafePinnedCreateOf Int
n) (FilePath -> Fold m (Array Word8) ()
forall (m :: * -> *) a.
(MonadIO m, MonadCatch m) =>
FilePath -> Fold m (Array a) ()
writeChunks FilePath
path)

{-# DEPRECATED writeWithBufferOf "Please use 'writeWith' instead"  #-}
{-# INLINE writeWithBufferOf #-}
writeWithBufferOf :: (MonadIO m, MonadCatch m)
    => Int -> FilePath -> Fold m Word8 ()
writeWithBufferOf :: forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Int -> FilePath -> Fold m Word8 ()
writeWithBufferOf = Int -> FilePath -> Fold m Word8 ()
forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Int -> FilePath -> Fold m Word8 ()
writeWith

-- > write = 'writeWith' A.defaultChunkSize
--
-- | Write a byte stream to a file. Accumulates the input in chunks of up to
-- 'Streamly.Internal.Data.Array.Type.defaultChunkSize' before writing to
-- the IO device.
--
-- /Pre-release/
--
{-# INLINE write #-}
write :: (MonadIO m, MonadCatch m) => FilePath -> Fold m Word8 ()
write :: forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
FilePath -> Fold m Word8 ()
write = Int -> FilePath -> Fold m Word8 ()
forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Int -> FilePath -> Fold m Word8 ()
writeWith Int
defaultChunkSize

-- | Append a stream of arrays to a file.
--
-- /Pre-release/
--
{-# INLINE writeAppendChunks #-}
writeAppendChunks :: (MonadIO m, MonadCatch m)
    => FilePath -> Stream m (Array a) -> m ()
writeAppendChunks :: forall (m :: * -> *) a.
(MonadIO m, MonadCatch m) =>
FilePath -> Stream m (Array a) -> m ()
writeAppendChunks = IOMode -> FilePath -> Stream m (Array a) -> m ()
forall (m :: * -> *) a.
(MonadIO m, MonadCatch m) =>
IOMode -> FilePath -> Stream m (Array a) -> m ()
fromChunksMode IOMode
AppendMode

-- | Like 'append' but provides control over the write buffer. Output will
-- be written to the IO device as soon as we collect the specified number of
-- input elements.
--
-- /Pre-release/
--
{-# INLINE writeAppendWith #-}
writeAppendWith :: (MonadIO m, MonadCatch m)
    => Int -> FilePath -> Stream m Word8 -> m ()
writeAppendWith :: forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Int -> FilePath -> Stream m Word8 -> m ()
writeAppendWith Int
n FilePath
file Stream m Word8
xs =
    FilePath -> Stream m (Array Word8) -> m ()
forall (m :: * -> *) a.
(MonadIO m, MonadCatch m) =>
FilePath -> Stream m (Array a) -> m ()
writeAppendChunks FilePath
file (Stream m (Array Word8) -> m ()) -> Stream m (Array Word8) -> m ()
forall a b. (a -> b) -> a -> b
$ Int -> Stream m Word8 -> Stream m (Array Word8)
forall (m :: * -> *) a.
(MonadIO m, Unbox a) =>
Int -> Stream m a -> Stream m (Array a)
IA.pinnedChunksOf Int
n Stream m Word8
xs

-- | Append a byte stream to a file. Combines the bytes in chunks of size up to
-- 'A.defaultChunkSize' before writing.  If the file exists then the new data
-- is appended to the file.  If the file does not exist it is created. File is
-- locked using single writer locking mode.
--
-- /Pre-release/
--
{-# INLINE writeAppend #-}
writeAppend :: (MonadIO m, MonadCatch m) => FilePath -> Stream m Word8 -> m ()
writeAppend :: forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
FilePath -> Stream m Word8 -> m ()
writeAppend = Int -> FilePath -> Stream m Word8 -> m ()
forall (m :: * -> *).
(MonadIO m, MonadCatch m) =>
Int -> FilePath -> Stream m Word8 -> m ()
writeAppendWith Int
defaultChunkSize

{-
-- | Like 'append' but the file is not locked for exclusive writes.
--
-- @since 0.7.0
{-# INLINE appendShared #-}
appendShared :: MonadIO m => Handle -> Stream m Word8 -> m ()
appendShared = undefined
-}

-------------------------------------------------------------------------------
-- IO with encoding/decoding Unicode characters
-------------------------------------------------------------------------------

{-
-- |
-- > readUtf8 = decodeUtf8 . read
--
-- Read a UTF8 encoded stream of unicode characters from a file handle.
--
-- @since 0.7.0
{-# INLINE readUtf8 #-}
readUtf8 :: MonadIO m => Handle -> Stream m Char
readUtf8 = decodeUtf8 . read

-- |
-- > writeUtf8 h s = write h $ encodeUtf8 s
--
-- Encode a stream of unicode characters to UTF8 and write it to the given file
-- handle. Default block buffering applies to the writes.
--
-- @since 0.7.0
{-# INLINE writeUtf8 #-}
writeUtf8 :: MonadIO m => Handle -> Stream m Char -> m ()
writeUtf8 h s = write h $ encodeUtf8 s

-- | Write a stream of unicode characters after encoding to UTF-8 in chunks
-- separated by a linefeed character @'\n'@. If the size of the buffer exceeds
-- @defaultChunkSize@ and a linefeed is not yet found, the buffer is written
-- anyway.  This is similar to writing to a 'Handle' with the 'LineBuffering'
-- option.
--
-- @since 0.7.0
{-# INLINE writeUtf8ByLines #-}
writeUtf8ByLines :: MonadIO m => Handle -> Stream m Char -> m ()
writeUtf8ByLines = undefined

-- | Read UTF-8 lines from a file handle and apply the specified fold to each
-- line. This is similar to reading a 'Handle' with the 'LineBuffering' option.
--
-- @since 0.7.0
{-# INLINE readLines #-}
readLines :: MonadIO m => Handle -> Fold m Char b -> Stream m b
readLines h f = foldLines (readUtf8 h) f

-------------------------------------------------------------------------------
-- Framing on a sequence
-------------------------------------------------------------------------------

-- | Read a stream from a file handle and split it into frames delimited by
-- the specified sequence of elements. The supplied fold is applied on each
-- frame.
--
-- @since 0.7.0
{-# INLINE readFrames #-}
readFrames :: (MonadIO m, Storable a)
    => Array a -> Handle -> Fold m a b -> Stream m b
readFrames = undefined -- foldFrames . read

-- | Write a stream to the given file handle buffering up to frames separated
-- by the given sequence or up to a maximum of @defaultChunkSize@.
--
-- @since 0.7.0
{-# INLINE writeByFrames #-}
writeByFrames :: (MonadIO m, Storable a)
    => Array a -> Handle -> Stream m a -> m ()
writeByFrames = undefined

-------------------------------------------------------------------------------
-- Random Access IO (Seek)
-------------------------------------------------------------------------------

-- XXX handles could be shared, so we may not want to use the handle state at
-- all for these APIs. we can use pread and pwrite instead. On windows we will
-- need to use readFile/writeFile with an offset argument.

-------------------------------------------------------------------------------

-- | Read the element at the given index treating the file as an array.
--
-- @since 0.7.0
{-# INLINE readIndex #-}
readIndex :: Storable a => Handle -> Int -> Maybe a
readIndex arr i = undefined

-- NOTE: To represent a range to read we have chosen (start, size) instead of
-- (start, end). This helps in removing the ambiguity of whether "end" is
-- included in the range or not.
--
-- We could avoid specifying the range to be read and instead use "take size"
-- on the stream, but it may end up reading more and then consume it partially.

-- | @readSliceWith chunkSize handle pos len@ reads up to @len@ bytes
-- from @handle@ starting at the offset @pos@ from the beginning of the file.
--
-- Reads are performed in chunks of size @chunkSize@.  For block devices, to
-- avoid reading partial blocks @chunkSize@ must align with the block size of
-- the underlying device. If the underlying block size is unknown, it is a good
-- idea to keep it a multiple 4KiB. This API ensures that the start of each
-- chunk is aligned with @chunkSize@ from second chunk onwards.
--
{-# INLINE readSliceWith #-}
readSliceWith :: (MonadIO m, Storable a)
    => Int -> Handle -> Int -> Int -> Stream m a
readSliceWith chunkSize h pos len = undefined

-- | @readSlice h i count@ streams a slice from the file handle @h@ starting
-- at index @i@ and reading up to @count@ elements in the forward direction
-- ending at the index @i + count - 1@.
--
-- @since 0.7.0
{-# INLINE readSlice #-}
readSlice :: (MonadIO m, Storable a)
    => Handle -> Int -> Int -> Stream m a
readSlice = readSliceWith defaultChunkSize

-- | @readSliceRev h i count@ streams a slice from the file handle @h@ starting
-- at index @i@ and reading up to @count@ elements in the reverse direction
-- ending at the index @i - count + 1@.
--
-- @since 0.7.0
{-# INLINE readSliceRev #-}
readSliceRev :: (MonadIO m, Storable a)
    => Handle -> Int -> Int -> Stream m a
readSliceRev h i count = undefined

-- | Write the given element at the given index in the file.
--
-- @since 0.7.0
{-# INLINE writeIndex #-}
writeIndex :: (MonadIO m, Storable a) => Handle -> Int -> a -> m ()
writeIndex h i a = undefined

-- | @writeSlice h i count stream@ writes a stream to the file handle @h@
-- starting at index @i@ and writing up to @count@ elements in the forward
-- direction ending at the index @i + count - 1@.
--
-- @since 0.7.0
{-# INLINE writeSlice #-}
writeSlice :: (Monad m, Storable a)
    => Handle -> Int -> Int -> Stream m a -> m ()
writeSlice h i len s = undefined

-- | @writeSliceRev h i count stream@ writes a stream to the file handle @h@
-- starting at index @i@ and writing up to @count@ elements in the reverse
-- direction ending at the index @i - count + 1@.
--
-- @since 0.7.0
{-# INLINE writeSliceRev #-}
writeSliceRev :: (Monad m, Storable a)
    => Handle -> Int -> Int -> Stream m a -> m ()
writeSliceRev arr i len s = undefined
-}