transient-universe-0.3.5: Remote execution and map-reduce: distributed computing for Transient

Safe HaskellNone
LanguageHaskell2010

Transient.MapReduce

Contents

Synopsis

Documentation

data DDS a Source #

Constructors

Loggable a => DDS (Cloud (PartRef a)) 

data PartRef a Source #

Constructors

Ref Node Path Save 

type Save = Bool Source #

class (Foldable c, Typeable c, Typeable a, Monoid (c a), Loggable (c a)) => Distributable c a where Source #

Minimal complete definition

singleton, splitAt, fromList

Methods

singleton :: a -> c a Source #

splitAt :: Int -> c a -> (c a, c a) Source #

fromList :: [a] -> c a Source #

Instances

mapKeyB :: (Loggable a, Loggable b, Loggable k, Ord k) => (a -> (k, b)) -> DDS (Vector a) -> DDS (Map k (Vector b)) Source #

perform a map and partition the result with different keys using boxed vectors The final result will be used by reduce.

mapKeyU :: (Loggable a, Unbox a, Loggable b, Unbox b, Loggable k, Ord k) => (a -> (k, b)) -> DDS (Vector a) -> DDS (Map k (Vector b)) Source #

perform a map and partition the result with different keys using unboxed vectors The final result will be used by reduce.

mapKey :: (Distributable vector a, Distributable vector b, Loggable k, Ord k) => (a -> (k, b)) -> DDS (vector a) -> DDS (Map k (vector b)) Source #

perform a map and partition the result with different keys. The final result will be used by reduce.

reduce :: (Hashable k, Ord k, Distributable vector a, Loggable k, Loggable a) => (a -> a -> a) -> DDS (Map k (vector a)) -> Cloud (Map k a) Source #

parallelize :: Alternative f => (a1 -> f a) -> [a1] -> f a Source #

mparallelize :: (Monoid a, Typeable * a, Show a, Read a) => (a1 -> Cloud a) -> [a1] -> Cloud a Source #

runAtP :: Loggable a => Node -> (Path -> IO a) -> Path -> Cloud a Source #

search :: [Char] -> a Source #

distribute :: (Loggable a, Distributable vector a) => vector a -> DDS (vector a) Source #

distribute a vector of values among many nodes. If the vector is static and sharable, better use the get* primitives since each node will load the data independently.

distribute' :: (Distributable vector a, Show a, Read a) => vector a -> Cloud (PartRef (vector a)) Source #

distribute'' :: (Loggable a, Distributable vector a) => [vector a] -> [Node] -> Cloud (PartRef (vector a)) Source #

getText :: (Loggable a, Distributable vector a) => (String -> [a]) -> String -> DDS (vector a) Source #

input data from a text that must be static and shared by all the nodes. The function parameter partition the text in words

textUrl :: String -> DDS (Vector Text) Source #

get the worlds of an URL

getUrl :: (Loggable a, Distributable vector a) => (String -> [a]) -> String -> DDS (vector a) Source #

generate a DDS from the content of a URL. The first parameter is a function that divide the text in words

textFile :: String -> DDS (Vector Text) Source #

get the words of a file

getFile :: (Loggable a, Distributable vector a) => (String -> [a]) -> String -> DDS (vector a) Source #

generate a DDS from a file. All the nodes must access the file with the same path the first parameter is the parser that generates elements from the content

streamDDS :: (Loggable a, Distributable vector a) => Integer -> IO (StreamData a) -> DDS (vector a) Source #

produce a stream of DDS's that can be map-reduced. Similar to spark streams. each interval of time,a new DDS is produced.(to be tested)

Orphan instances

Foldable Vector Source # 

Methods

fold :: Monoid m => Vector m -> m #

foldMap :: Monoid m => (a -> m) -> Vector a -> m #

foldr :: (a -> b -> b) -> b -> Vector a -> b #

foldr' :: (a -> b -> b) -> b -> Vector a -> b #

foldl :: (b -> a -> b) -> b -> Vector a -> b #

foldl' :: (b -> a -> b) -> b -> Vector a -> b #

foldr1 :: (a -> a -> a) -> Vector a -> a #

foldl1 :: (a -> a -> a) -> Vector a -> a #

toList :: Vector a -> [a] #

null :: Vector a -> Bool #

length :: Vector a -> Int #

elem :: Eq a => a -> Vector a -> Bool #

maximum :: Ord a => Vector a -> a #

minimum :: Ord a => Vector a -> a #

sum :: Num a => Vector a -> a #

product :: Num a => Vector a -> a #