Copyright	(c) Adam Conner-Sax 2019
License	BSD-3-Clause
Maintainer	adam_conner_sax@yahoo.com
Stability	experimental
Safe Haskell	None
Language	Haskell2010

Control.MapReduce.Engines.ParallelList

Contents

Parallel map/reduce fold builder
re-exports

Description

Some basic attempts at parallel map-reduce folds with lists as intermediate type. There are multiple places we can do things in parallel.

We can map (unpack/assign) in parallel.
We can reduce in parallel.
We can fold our intermediate monoid (in the gatherer) in parallel
We can fold our result monoid in parallel

NB: This does not seem to be faster--and is often slower!--than the serial engine. I leave it here as a starting point for improvement.

Synopsis

parallelListEngine :: forall g y k c x d. (NFData k, NFData c, NFData d, Foldable g, Functor g) => Int -> ([(k, c)] -> [(k, g c)]) -> MapReduceFold y k c [] x d
class NFData a

Parallel map/reduce fold builder

parallelListEngine :: forall g y k c x d. (NFData k, NFData c, NFData d, Foldable g, Functor g) => Int -> ([(k, c)] -> [(k, g c)]) -> MapReduceFold y k c [] x d Source #

Parallel map-reduce-fold list engine. Uses the given parameters to use multiple sparks when mapping and reducing. Chunks the input to numThreads chunks and sparks each chunk for mapping, merges the results, groups, then uses the same chunking and merging to do the reductions. grouping could also be parallel but that is under the control of the given function.

re-exports

class NFData a #

A class of types that can be fully evaluated.

Since: deepseq-1.1.0.0

Instances

NFData Bool
Instance details Defined in Control.DeepSeq Methods rnf :: Bool -> () #
NFData Char
Instance details Defined in Control.DeepSeq Methods rnf :: Char -> () #
NFData Double
Instance details Defined in Control.DeepSeq Methods rnf :: Double -> () #
NFData Float
Instance details Defined in Control.DeepSeq Methods rnf :: Float -> () #
NFData Int
Instance details Defined in Control.DeepSeq Methods rnf :: Int -> () #
NFData Int8
Instance details Defined in Control.DeepSeq Methods rnf :: Int8 -> () #
NFData Int16
Instance details Defined in Control.DeepSeq Methods rnf :: Int16 -> () #
NFData Int32
Instance details Defined in Control.DeepSeq Methods rnf :: Int32 -> () #
NFData Int64
Instance details Defined in Control.DeepSeq Methods rnf :: Int64 -> () #
NFData Integer
Instance details Defined in Control.DeepSeq Methods rnf :: Integer -> () #
NFData Natural	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: Natural -> () #
NFData Ordering
Instance details Defined in Control.DeepSeq Methods rnf :: Ordering -> () #
NFData Word
Instance details Defined in Control.DeepSeq Methods rnf :: Word -> () #
NFData Word8
Instance details Defined in Control.DeepSeq Methods rnf :: Word8 -> () #
NFData Word16
Instance details Defined in Control.DeepSeq Methods rnf :: Word16 -> () #
NFData Word32
Instance details Defined in Control.DeepSeq Methods rnf :: Word32 -> () #
NFData Word64
Instance details Defined in Control.DeepSeq Methods rnf :: Word64 -> () #
NFData CallStack	Since: deepseq-1.4.2.0
Instance details Defined in Control.DeepSeq Methods rnf :: CallStack -> () #
NFData ()
Instance details Defined in Control.DeepSeq Methods rnf :: () -> () #
NFData TyCon	NOTE: Prior to `deepseq-1.4.4.0` this instance was only defined for `base-4.8.0.0` and later. Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: TyCon -> () #
NFData Void	Defined as `rnf = absurd`. Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: Void -> () #
NFData Unique	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: Unique -> () #
NFData Version	Since: deepseq-1.3.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: Version -> () #
NFData ThreadId	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: ThreadId -> () #
NFData ExitCode	Since: deepseq-1.4.2.0
Instance details Defined in Control.DeepSeq Methods rnf :: ExitCode -> () #
NFData MaskingState	Since: deepseq-1.4.4.0
Instance details Defined in Control.DeepSeq Methods rnf :: MaskingState -> () #
NFData TypeRep	NOTE: Prior to `deepseq-1.4.4.0` this instance was only defined for `base-4.8.0.0` and later. Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: TypeRep -> () #
NFData All	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: All -> () #
NFData Any	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: Any -> () #
NFData CChar	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CChar -> () #
NFData CSChar	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CSChar -> () #
NFData CUChar	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CUChar -> () #
NFData CShort	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CShort -> () #
NFData CUShort	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CUShort -> () #
NFData CInt	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CInt -> () #
NFData CUInt	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CUInt -> () #
NFData CLong	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CLong -> () #
NFData CULong	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CULong -> () #
NFData CLLong	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CLLong -> () #
NFData CULLong	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CULLong -> () #
NFData CBool	Since: deepseq-1.4.3.0
Instance details Defined in Control.DeepSeq Methods rnf :: CBool -> () #
NFData CFloat	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CFloat -> () #
NFData CDouble	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CDouble -> () #
NFData CPtrdiff	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CPtrdiff -> () #
NFData CSize	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CSize -> () #
NFData CWchar	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CWchar -> () #
NFData CSigAtomic	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CSigAtomic -> () #
NFData CClock	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CClock -> () #
NFData CTime	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CTime -> () #
NFData CUSeconds	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CUSeconds -> () #
NFData CSUSeconds	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CSUSeconds -> () #
NFData CFile	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CFile -> () #
NFData CFpos	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CFpos -> () #
NFData CJmpBuf	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CJmpBuf -> () #
NFData CIntPtr	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CIntPtr -> () #
NFData CUIntPtr	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CUIntPtr -> () #
NFData CIntMax	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CIntMax -> () #
NFData CUIntMax	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: CUIntMax -> () #
NFData Fingerprint	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: Fingerprint -> () #
NFData SrcLoc	Since: deepseq-1.4.2.0
Instance details Defined in Control.DeepSeq Methods rnf :: SrcLoc -> () #
NFData IntSet
Instance details Defined in Data.IntSet.Internal Methods rnf :: IntSet -> () #
NFData Doc
Instance details Defined in Text.PrettyPrint.HughesPJ Methods rnf :: Doc -> () #
NFData TextDetails
Instance details Defined in Text.PrettyPrint.Annotated.HughesPJ Methods rnf :: TextDetails -> () #
NFData ByteArray
Instance details Defined in Data.Primitive.ByteArray Methods rnf :: ByteArray -> () #
NFData StdGen
Instance details Defined in System.Random.Internal Methods rnf :: StdGen -> () #
NFData a => NFData [a]
Instance details Defined in Control.DeepSeq Methods rnf :: [a] -> () #
NFData a => NFData (Maybe a)
Instance details Defined in Control.DeepSeq Methods rnf :: Maybe a -> () #
NFData a => NFData (Ratio a)
Instance details Defined in Control.DeepSeq Methods rnf :: Ratio a -> () #
NFData (Ptr a)	Since: deepseq-1.4.2.0
Instance details Defined in Control.DeepSeq Methods rnf :: Ptr a -> () #
NFData (FunPtr a)	Since: deepseq-1.4.2.0
Instance details Defined in Control.DeepSeq Methods rnf :: FunPtr a -> () #
NFData (IORef a)	NOTE: Only strict in the reference and not the referenced value. Since: deepseq-1.4.2.0
Instance details Defined in Control.DeepSeq Methods rnf :: IORef a -> () #
NFData (MutableByteArray s)
Instance details Defined in Data.Primitive.ByteArray Methods rnf :: MutableByteArray s -> () #
NFData a => NFData (Complex a)
Instance details Defined in Control.DeepSeq Methods rnf :: Complex a -> () #
NFData (Fixed a)	Since: deepseq-1.3.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: Fixed a -> () #
NFData a => NFData (Min a)	Since: deepseq-1.4.2.0
Instance details Defined in Control.DeepSeq Methods rnf :: Min a -> () #
NFData a => NFData (Max a)	Since: deepseq-1.4.2.0
Instance details Defined in Control.DeepSeq Methods rnf :: Max a -> () #
NFData a => NFData (First a)	Since: deepseq-1.4.2.0
Instance details Defined in Control.DeepSeq Methods rnf :: First a -> () #
NFData a => NFData (Last a)	Since: deepseq-1.4.2.0
Instance details Defined in Control.DeepSeq Methods rnf :: Last a -> () #
NFData m => NFData (WrappedMonoid m)	Since: deepseq-1.4.2.0
Instance details Defined in Control.DeepSeq Methods rnf :: WrappedMonoid m -> () #
NFData a => NFData (Option a)	Since: deepseq-1.4.2.0
Instance details Defined in Control.DeepSeq Methods rnf :: Option a -> () #
NFData (StableName a)	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: StableName a -> () #
NFData a => NFData (ZipList a)	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: ZipList a -> () #
NFData a => NFData (Identity a)	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: Identity a -> () #
NFData a => NFData (First a)	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: First a -> () #
NFData a => NFData (Last a)	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: Last a -> () #
NFData a => NFData (Dual a)	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: Dual a -> () #
NFData a => NFData (Sum a)	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: Sum a -> () #
NFData a => NFData (Product a)	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: Product a -> () #
NFData a => NFData (Down a)	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: Down a -> () #
NFData (MVar a)	NOTE: Only strict in the reference and not the referenced value. Since: deepseq-1.4.2.0
Instance details Defined in Control.DeepSeq Methods rnf :: MVar a -> () #
NFData a => NFData (NonEmpty a)	Since: deepseq-1.4.2.0
Instance details Defined in Control.DeepSeq Methods rnf :: NonEmpty a -> () #
NFData a => NFData (IntMap a)
Instance details Defined in Data.IntMap.Internal Methods rnf :: IntMap a -> () #
NFData a => NFData (Tree a)
Instance details Defined in Data.Tree Methods rnf :: Tree a -> () #
NFData a => NFData (Seq a)
Instance details Defined in Data.Sequence.Internal Methods rnf :: Seq a -> () #
NFData a => NFData (FingerTree a)
Instance details Defined in Data.Sequence.Internal Methods rnf :: FingerTree a -> () #
NFData a => NFData (Digit a)
Instance details Defined in Data.Sequence.Internal Methods rnf :: Digit a -> () #
NFData a => NFData (Node a)
Instance details Defined in Data.Sequence.Internal Methods rnf :: Node a -> () #
NFData a => NFData (Elem a)
Instance details Defined in Data.Sequence.Internal Methods rnf :: Elem a -> () #
NFData a => NFData (Set a)
Instance details Defined in Data.Set.Internal Methods rnf :: Set a -> () #
NFData a => NFData (SmallArray a)
Instance details Defined in Data.Discrimination.Internal.SmallArray Methods rnf :: SmallArray a -> () #
NFData a => NFData (Hashed a)
Instance details Defined in Data.Hashable.Class Methods rnf :: Hashed a -> () #
NFData a => NFData (Doc a)
Instance details Defined in Text.PrettyPrint.Annotated.HughesPJ Methods rnf :: Doc a -> () #
NFData a => NFData (AnnotDetails a)
Instance details Defined in Text.PrettyPrint.Annotated.HughesPJ Methods rnf :: AnnotDetails a -> () #
NFData (PrimArray a)
Instance details Defined in Data.Primitive.PrimArray Methods rnf :: PrimArray a -> () #
NFData a => NFData (SmallArray a)
Instance details Defined in Data.Primitive.SmallArray Methods rnf :: SmallArray a -> () #
NFData a => NFData (Array a)
Instance details Defined in Data.Primitive.Array Methods rnf :: Array a -> () #
NFData g => NFData (AtomicGen g)
Instance details Defined in System.Random.Stateful Methods rnf :: AtomicGen g -> () #
NFData g => NFData (IOGen g)
Instance details Defined in System.Random.Stateful Methods rnf :: IOGen g -> () #
NFData g => NFData (STGen g)
Instance details Defined in System.Random.Stateful Methods rnf :: STGen g -> () #
NFData g => NFData (StateGen g)
Instance details Defined in System.Random.Internal Methods rnf :: StateGen g -> () #
(Storable a, NFData a) => NFData (Array a)
Instance details Defined in Streamly.Internal.Memory.Array.Types Methods rnf :: Array a -> () #
NFData a => NFData (HashSet a)
Instance details Defined in Data.HashSet.Internal Methods rnf :: HashSet a -> () #
NFData (Vector a)
Instance details Defined in Data.Vector.Unboxed.Base Methods rnf :: Vector a -> () #
NFData (Vector a)
Instance details Defined in Data.Vector.Primitive Methods rnf :: Vector a -> () #
NFData a => NFData (Vector a)
Instance details Defined in Data.Vector Methods rnf :: Vector a -> () #
NFData (a -> b)	This instance is for convenience and consistency with `seq`. This assumes that WHNF is equivalent to NF for functions. Since: deepseq-1.3.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: (a -> b) -> () #
(NFData a, NFData b) => NFData (Either a b)
Instance details Defined in Control.DeepSeq Methods rnf :: Either a b -> () #
(NFData a, NFData b) => NFData (a, b)
Instance details Defined in Control.DeepSeq Methods rnf :: (a, b) -> () #
(NFData a, NFData b) => NFData (Array a b)
Instance details Defined in Control.DeepSeq Methods rnf :: Array a b -> () #
(NFData a, NFData b) => NFData (Arg a b)	Since: deepseq-1.4.2.0
Instance details Defined in Control.DeepSeq Methods rnf :: Arg a b -> () #
NFData (Proxy a)	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: Proxy a -> () #
NFData (STRef s a)	NOTE: Only strict in the reference and not the referenced value. Since: deepseq-1.4.2.0
Instance details Defined in Control.DeepSeq Methods rnf :: STRef s a -> () #
(NFData k, NFData a) => NFData (Map k a)
Instance details Defined in Data.Map.Internal Methods rnf :: Map k a -> () #
(NFData k, NFData v) => NFData (HashMap k v)
Instance details Defined in Data.HashMap.Internal Methods rnf :: HashMap k v -> () #
NFData (MutablePrimArray s a)
Instance details Defined in Data.Primitive.PrimArray Methods rnf :: MutablePrimArray s a -> () #
NFData a => NFData (ZipSerialM Identity a)
Instance details Defined in Streamly.Streams.Zip Methods rnf :: ZipSerialM Identity a -> () #
NFData a => NFData (SerialT Identity a)
Instance details Defined in Streamly.Streams.Serial Methods rnf :: SerialT Identity a -> () #
NFData a => NFData (WSerialT Identity a)
Instance details Defined in Streamly.Streams.Serial Methods rnf :: WSerialT Identity a -> () #
(NFData k, NFData v) => NFData (Leaf k v)
Instance details Defined in Data.HashMap.Internal Methods rnf :: Leaf k v -> () #
NFData (MVector s a)
Instance details Defined in Data.Vector.Unboxed.Base Methods rnf :: MVector s a -> () #
NFData (MVector s a)
Instance details Defined in Data.Vector.Primitive.Mutable Methods rnf :: MVector s a -> () #
(NFData a1, NFData a2, NFData a3) => NFData (a1, a2, a3)
Instance details Defined in Control.DeepSeq Methods rnf :: (a1, a2, a3) -> () #
NFData a => NFData (Const a b)	Since: deepseq-1.4.0.0
Instance details Defined in Control.DeepSeq Methods rnf :: Const a b -> () #
NFData (a :~: b)	Since: deepseq-1.4.3.0
Instance details Defined in Control.DeepSeq Methods rnf :: (a :~: b) -> () #
NFData b => NFData (Tagged s b)
Instance details Defined in Data.Tagged Methods rnf :: Tagged s b -> () #
(NFData a1, NFData a2, NFData a3, NFData a4) => NFData (a1, a2, a3, a4)
Instance details Defined in Control.DeepSeq Methods rnf :: (a1, a2, a3, a4) -> () #
(NFData1 f, NFData1 g, NFData a) => NFData (Product f g a)	Since: deepseq-1.4.3.0
Instance details Defined in Control.DeepSeq Methods rnf :: Product f g a -> () #
(NFData1 f, NFData1 g, NFData a) => NFData (Sum f g a)	Since: deepseq-1.4.3.0
Instance details Defined in Control.DeepSeq Methods rnf :: Sum f g a -> () #
NFData (a :~~: b)	Since: deepseq-1.4.3.0
Instance details Defined in Control.DeepSeq Methods rnf :: (a :~~: b) -> () #
(NFData a1, NFData a2, NFData a3, NFData a4, NFData a5) => NFData (a1, a2, a3, a4, a5)
Instance details Defined in Control.DeepSeq Methods rnf :: (a1, a2, a3, a4, a5) -> () #
(NFData1 f, NFData1 g, NFData a) => NFData (Compose f g a)	Since: deepseq-1.4.3.0
Instance details Defined in Control.DeepSeq Methods rnf :: Compose f g a -> () #
(NFData a1, NFData a2, NFData a3, NFData a4, NFData a5, NFData a6) => NFData (a1, a2, a3, a4, a5, a6)
Instance details Defined in Control.DeepSeq Methods rnf :: (a1, a2, a3, a4, a5, a6) -> () #
(NFData a1, NFData a2, NFData a3, NFData a4, NFData a5, NFData a6, NFData a7) => NFData (a1, a2, a3, a4, a5, a6, a7)
Instance details Defined in Control.DeepSeq Methods rnf :: (a1, a2, a3, a4, a5, a6, a7) -> () #
(NFData a1, NFData a2, NFData a3, NFData a4, NFData a5, NFData a6, NFData a7, NFData a8) => NFData (a1, a2, a3, a4, a5, a6, a7, a8)
Instance details Defined in Control.DeepSeq Methods rnf :: (a1, a2, a3, a4, a5, a6, a7, a8) -> () #
(NFData a1, NFData a2, NFData a3, NFData a4, NFData a5, NFData a6, NFData a7, NFData a8, NFData a9) => NFData (a1, a2, a3, a4, a5, a6, a7, a8, a9)
Instance details Defined in Control.DeepSeq Methods rnf :: (a1, a2, a3, a4, a5, a6, a7, a8, a9) -> () #