Safe Haskell	None
Language	Haskell2010

Development.Benchmark.Rules

Description

This module provides a bunch of Shake rules to build multiple revisions of a project and analyse their performance.

It assumes a project bench suite composed of examples that runs a fixed set of experiments on every example

Your code must implement all of the GetFoo oracles and the IsExample class, instantiate the Shake rules, and probably want a set of targets.

The results of the benchmarks and the analysis are recorded in the file system, using the following structure:

build-folder ├── binaries │ └── git-reference │ ├── ghc.path - path to ghc used to build the executable │ └── executable - binary for this version │ └── commitid - Git commit id for this reference ├─ example │ ├── results.csv - aggregated results for all the versions │ └── git-reference │ ├── experiment.gcStats.log - RTS -s output │ ├── experiment.csv - stats for the experiment │ ├── experiment.svg - Graph of bytes over elapsed time │ ├── experiment.diff.svg - idem, including the previous version │ ├── experiment.heap.svg - Heap profile │ ├── experiment.log - bench stdout │ └── results.csv - results of all the experiments for the example ├── results.csv - aggregated results of all the experiments and versions └── experiment.svg - graph of bytes over elapsed time, for all the included versions

For diff graphs, the "previous version" is the preceding entry in the list of versions in the config file. A possible improvement is to obtain this info via `git rev-list`.

Synopsis

buildRules :: FilePattern -> MkBuildRules BuildSystem -> Rules ()
data MkBuildRules buildSystem = MkBuildRules {
- findGhc :: buildSystem -> FilePath -> IO FilePath
- executableName :: String
- projectDepends :: Action ()
- buildProject :: buildSystem -> [CmdOption] -> OutputFolder -> Action ()
}
benchRules :: RuleResultForExample example => FilePattern -> MkBenchRules BuildSystem example -> Rules ()
data MkBenchRules buildSystem example = forall setup. MkBenchRules {
- setupProject :: Action setup
- benchProject :: setup -> buildSystem -> [CmdOption] -> BenchProject example -> Action ()
- warmupProject :: buildSystem -> FilePath -> [CmdOption] -> example -> Action ()
- executableName :: String
}
data BenchProject example = BenchProject {
- outcsv :: FilePath
- exePath :: FilePath
- exeExtraArgs :: [String]
- example :: example
- experiment :: Escaped String
}
data ProfilingMode
- = NoProfiling
- | CheapHeapProfiling Seconds
csvRules :: forall example. RuleResultForExample example => FilePattern -> Rules ()
svgRules :: FilePattern -> Rules ()
heapProfileRules :: FilePattern -> Rules ()
phonyRules :: (Traversable t, IsExample e) => String -> String -> ProfilingMode -> FilePath -> t e -> Rules ()
allTargetsForExample :: IsExample e => ProfilingMode -> FilePath -> e -> Action [FilePath]
newtype GetExample = GetExample String
newtype GetExamples = GetExamples ()
class (Binary e, Eq e, Hashable e, NFData e, Show e, Typeable e) => IsExample e where
- getExampleName :: e -> String
type RuleResultForExample e = (RuleResult GetExample ~ Maybe e, RuleResult GetExamples ~ [e], IsExample e)
newtype GetExperiments = GetExperiments ()
newtype GetVersions = GetVersions ()
newtype GetCommitId = GetCommitId String
newtype GetBuildSystem = GetBuildSystem ()
data BuildSystem
- = Cabal
- | Stack
findGhcForBuildSystem :: BuildSystem -> FilePath -> IO FilePath
newtype Escaped a = Escaped {
- escaped :: a
}
newtype Unescaped a = Unescaped {
- unescaped :: a
}
escapeExperiment :: Unescaped String -> Escaped String
unescapeExperiment :: Escaped String -> Unescaped String
data GitCommit

Documentation

buildRules :: FilePattern -> MkBuildRules BuildSystem -> Rules () Source #

Rules that drive a build system to build various revisions of a project

data MkBuildRules buildSystem Source #

Constructors

MkBuildRules

Fields

findGhc :: buildSystem -> FilePath -> IO FilePath
Return the path to the GHC executable to use for the project found in the cwd
executableName :: String
Name of the binary produced by $sel:buildProject:MkBuildRules
projectDepends :: Action ()
An action that captures the source dependencies, used for the HEAD build
buildProject :: buildSystem -> [CmdOption] -> OutputFolder -> Action ()
Build the project found in the cwd and save the build artifacts in the output folder

benchRules :: RuleResultForExample example => FilePattern -> MkBenchRules BuildSystem example -> Rules () Source #

data MkBenchRules buildSystem example Source #

Constructors

forall setup. MkBenchRules

Fields

setupProject :: Action setup
Workaround for Shake not allowing to call askOracle from 'benchProject
benchProject :: setup -> buildSystem -> [CmdOption] -> BenchProject example -> Action ()
An action that invokes the executable to run the benchmark
warmupProject :: buildSystem -> FilePath -> [CmdOption] -> example -> Action ()
An action that performs any necessary warmup. Will only be invoked once
executableName :: String
Name of the executable to benchmark. Should match the one used to MkBuildRules

data BenchProject example Source #

Constructors

BenchProject
Fields outcsv :: FilePath where to save the CSV output exePath :: FilePath where to find the executable for benchmarking exeExtraArgs :: [String] extra args for the executable example :: example example to benchmark experiment :: Escaped String experiment to run

data ProfilingMode Source #

Constructors

NoProfiling
CheapHeapProfiling Seconds

Instances

Instances details

Eq ProfilingMode Source #
Instance details Defined in Development.Benchmark.Rules Methods (==) :: ProfilingMode -> ProfilingMode -> Bool # (/=) :: ProfilingMode -> ProfilingMode -> Bool #

csvRules :: forall example. RuleResultForExample example => FilePattern -> Rules () Source #

Rules to aggregate the CSV output of individual experiments

svgRules :: FilePattern -> Rules () Source #

Rules to produce charts for the GC stats

heapProfileRules :: FilePattern -> Rules () Source #

phonyRules Source #

Arguments

:: (Traversable t, IsExample e)
=> String	prefix
-> String	Executable name
-> ProfilingMode
-> FilePath
-> t e
-> Rules ()

Generate a set of phony rules: * prefixall * prefix example for each example

allTargetsForExample :: IsExample e => ProfilingMode -> FilePath -> e -> Action [FilePath] Source #

newtype GetExample Source #

Constructors

GetExample String

Instances

Instances details

Eq GetExample Source #
Instance details Defined in Development.Benchmark.Rules Methods (==) :: GetExample -> GetExample -> Bool # (/=) :: GetExample -> GetExample -> Bool #
Show GetExample Source #
Instance details Defined in Development.Benchmark.Rules Methods showsPrec :: Int -> GetExample -> ShowS # show :: GetExample -> String # showList :: [GetExample] -> ShowS #
Hashable GetExample Source #
Instance details Defined in Development.Benchmark.Rules Methods hashWithSalt :: Int -> GetExample -> Int # hash :: GetExample -> Int #
Binary GetExample Source #
Instance details Defined in Development.Benchmark.Rules Methods put :: GetExample -> Put # get :: Get GetExample # putList :: [GetExample] -> Put #
NFData GetExample Source #
Instance details Defined in Development.Benchmark.Rules Methods rnf :: GetExample -> () #

newtype GetExamples Source #

Constructors

GetExamples ()

Instances

Instances details

Eq GetExamples Source #
Instance details Defined in Development.Benchmark.Rules Methods (==) :: GetExamples -> GetExamples -> Bool # (/=) :: GetExamples -> GetExamples -> Bool #
Show GetExamples Source #
Instance details Defined in Development.Benchmark.Rules Methods showsPrec :: Int -> GetExamples -> ShowS # show :: GetExamples -> String # showList :: [GetExamples] -> ShowS #
Hashable GetExamples Source #
Instance details Defined in Development.Benchmark.Rules Methods hashWithSalt :: Int -> GetExamples -> Int # hash :: GetExamples -> Int #
Binary GetExamples Source #
Instance details Defined in Development.Benchmark.Rules Methods put :: GetExamples -> Put # get :: Get GetExamples # putList :: [GetExamples] -> Put #
NFData GetExamples Source #
Instance details Defined in Development.Benchmark.Rules Methods rnf :: GetExamples -> () #

class (Binary e, Eq e, Hashable e, NFData e, Show e, Typeable e) => IsExample e where Source #

Knowledge needed to run an example

Methods

getExampleName :: e -> String Source #

type RuleResultForExample e = (RuleResult GetExample ~ Maybe e, RuleResult GetExamples ~ [e], IsExample e) Source #

newtype GetExperiments Source #

Constructors

GetExperiments ()

Instances

Instances details

Eq GetExperiments Source #
Instance details Defined in Development.Benchmark.Rules Methods (==) :: GetExperiments -> GetExperiments -> Bool # (/=) :: GetExperiments -> GetExperiments -> Bool #
Show GetExperiments Source #
Instance details Defined in Development.Benchmark.Rules Methods showsPrec :: Int -> GetExperiments -> ShowS # show :: GetExperiments -> String # showList :: [GetExperiments] -> ShowS #
Hashable GetExperiments Source #
Instance details Defined in Development.Benchmark.Rules Methods hashWithSalt :: Int -> GetExperiments -> Int # hash :: GetExperiments -> Int #
Binary GetExperiments Source #
Instance details Defined in Development.Benchmark.Rules Methods put :: GetExperiments -> Put # get :: Get GetExperiments # putList :: [GetExperiments] -> Put #
NFData GetExperiments Source #
Instance details Defined in Development.Benchmark.Rules Methods rnf :: GetExperiments -> () #
type RuleResult GetExperiments Source #
Instance details Defined in Development.Benchmark.Rules type RuleResult GetExperiments = [Unescaped String]

newtype GetVersions Source #

Constructors

GetVersions ()

Instances

Instances details

Eq GetVersions Source #
Instance details Defined in Development.Benchmark.Rules Methods (==) :: GetVersions -> GetVersions -> Bool # (/=) :: GetVersions -> GetVersions -> Bool #
Show GetVersions Source #
Instance details Defined in Development.Benchmark.Rules Methods showsPrec :: Int -> GetVersions -> ShowS # show :: GetVersions -> String # showList :: [GetVersions] -> ShowS #
Hashable GetVersions Source #
Instance details Defined in Development.Benchmark.Rules Methods hashWithSalt :: Int -> GetVersions -> Int # hash :: GetVersions -> Int #
Binary GetVersions Source #
Instance details Defined in Development.Benchmark.Rules Methods put :: GetVersions -> Put # get :: Get GetVersions # putList :: [GetVersions] -> Put #
NFData GetVersions Source #
Instance details Defined in Development.Benchmark.Rules Methods rnf :: GetVersions -> () #
type RuleResult GetVersions Source #
Instance details Defined in Development.Benchmark.Rules type RuleResult GetVersions = [GitCommit]

newtype GetCommitId Source #

Constructors

GetCommitId String

Instances

Instances details

Eq GetCommitId Source #
Instance details Defined in Development.Benchmark.Rules Methods (==) :: GetCommitId -> GetCommitId -> Bool # (/=) :: GetCommitId -> GetCommitId -> Bool #
Show GetCommitId Source #
Instance details Defined in Development.Benchmark.Rules Methods showsPrec :: Int -> GetCommitId -> ShowS # show :: GetCommitId -> String # showList :: [GetCommitId] -> ShowS #
Hashable GetCommitId Source #
Instance details Defined in Development.Benchmark.Rules Methods hashWithSalt :: Int -> GetCommitId -> Int # hash :: GetCommitId -> Int #
Binary GetCommitId Source #
Instance details Defined in Development.Benchmark.Rules Methods put :: GetCommitId -> Put # get :: Get GetCommitId # putList :: [GetCommitId] -> Put #
NFData GetCommitId Source #
Instance details Defined in Development.Benchmark.Rules Methods rnf :: GetCommitId -> () #
type RuleResult GetCommitId Source #
Instance details Defined in Development.Benchmark.Rules type RuleResult GetCommitId = String

newtype GetBuildSystem Source #

Constructors

GetBuildSystem ()

Instances

Instances details

Eq GetBuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods (==) :: GetBuildSystem -> GetBuildSystem -> Bool # (/=) :: GetBuildSystem -> GetBuildSystem -> Bool #
Show GetBuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods showsPrec :: Int -> GetBuildSystem -> ShowS # show :: GetBuildSystem -> String # showList :: [GetBuildSystem] -> ShowS #
Hashable GetBuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods hashWithSalt :: Int -> GetBuildSystem -> Int # hash :: GetBuildSystem -> Int #
Binary GetBuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods put :: GetBuildSystem -> Put # get :: Get GetBuildSystem # putList :: [GetBuildSystem] -> Put #
NFData GetBuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods rnf :: GetBuildSystem -> () #
type RuleResult GetBuildSystem Source #
Instance details Defined in Development.Benchmark.Rules type RuleResult GetBuildSystem = BuildSystem

data BuildSystem Source #

Default build system that handles Cabal and Stack

Constructors

Cabal
Stack

Instances

Instances details

Eq BuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods (==) :: BuildSystem -> BuildSystem -> Bool # (/=) :: BuildSystem -> BuildSystem -> Bool #
Read BuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods readsPrec :: Int -> ReadS BuildSystem # readList :: ReadS [BuildSystem] # readPrec :: ReadPrec BuildSystem # readListPrec :: ReadPrec [BuildSystem] #
Show BuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods showsPrec :: Int -> BuildSystem -> ShowS # show :: BuildSystem -> String # showList :: [BuildSystem] -> ShowS #
Generic BuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Associated Types type Rep BuildSystem :: Type -> Type # Methods from :: BuildSystem -> Rep BuildSystem x # to :: Rep BuildSystem x -> BuildSystem #
Hashable BuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods hashWithSalt :: Int -> BuildSystem -> Int # hash :: BuildSystem -> Int #
ToJSON BuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods toJSON :: BuildSystem -> Value # toEncoding :: BuildSystem -> Encoding # toJSONList :: [BuildSystem] -> Value # toEncodingList :: [BuildSystem] -> Encoding #
FromJSON BuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods parseJSON :: Value -> Parser BuildSystem # parseJSONList :: Value -> Parser [BuildSystem] #
Binary BuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods put :: BuildSystem -> Put # get :: Get BuildSystem # putList :: [BuildSystem] -> Put #
NFData BuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods rnf :: BuildSystem -> () #
type Rep BuildSystem Source #
Instance details Defined in Development.Benchmark.Rules type Rep BuildSystem = D1 ('MetaData "BuildSystem" "Development.Benchmark.Rules" "shake-bench-0.1.0.3-9iZtOhtMvLmBAUDRjvlAYZ" 'False) (C1 ('MetaCons "Cabal" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "Stack" 'PrefixI 'False) (U1 :: Type -> Type))

findGhcForBuildSystem :: BuildSystem -> FilePath -> IO FilePath Source #

newtype Escaped a Source #

Constructors

Escaped
Fields escaped :: a

newtype Unescaped a Source #

Constructors

Unescaped
Fields unescaped :: a

Instances

Instances details

Eq a => Eq (Unescaped a) Source #
Instance details Defined in Development.Benchmark.Rules Methods (==) :: Unescaped a -> Unescaped a -> Bool # (/=) :: Unescaped a -> Unescaped a -> Bool #
Show a => Show (Unescaped a) Source #
Instance details Defined in Development.Benchmark.Rules Methods showsPrec :: Int -> Unescaped a -> ShowS # show :: Unescaped a -> String # showList :: [Unescaped a] -> ShowS #
Hashable a => Hashable (Unescaped a) Source #
Instance details Defined in Development.Benchmark.Rules Methods hashWithSalt :: Int -> Unescaped a -> Int # hash :: Unescaped a -> Int #
ToJSON a => ToJSON (Unescaped a) Source #
Instance details Defined in Development.Benchmark.Rules Methods toJSON :: Unescaped a -> Value # toEncoding :: Unescaped a -> Encoding # toJSONList :: [Unescaped a] -> Value # toEncodingList :: [Unescaped a] -> Encoding #
FromJSON a => FromJSON (Unescaped a) Source #
Instance details Defined in Development.Benchmark.Rules Methods parseJSON :: Value -> Parser (Unescaped a) # parseJSONList :: Value -> Parser [Unescaped a] #
Binary a => Binary (Unescaped a) Source #
Instance details Defined in Development.Benchmark.Rules Methods put :: Unescaped a -> Put # get :: Get (Unescaped a) # putList :: [Unescaped a] -> Put #
NFData a => NFData (Unescaped a) Source #
Instance details Defined in Development.Benchmark.Rules Methods rnf :: Unescaped a -> () #

escapeExperiment :: Unescaped String -> Escaped String Source #

unescapeExperiment :: Escaped String -> Unescaped String Source #

data GitCommit Source #

Instances

Instances details

Eq GitCommit Source #
Instance details Defined in Development.Benchmark.Rules Methods (==) :: GitCommit -> GitCommit -> Bool # (/=) :: GitCommit -> GitCommit -> Bool #
Show GitCommit Source #
Instance details Defined in Development.Benchmark.Rules Methods showsPrec :: Int -> GitCommit -> ShowS # show :: GitCommit -> String # showList :: [GitCommit] -> ShowS #
Generic GitCommit Source #
Instance details Defined in Development.Benchmark.Rules Associated Types type Rep GitCommit :: Type -> Type # Methods from :: GitCommit -> Rep GitCommit x # to :: Rep GitCommit x -> GitCommit #
Hashable GitCommit Source #
Instance details Defined in Development.Benchmark.Rules Methods hashWithSalt :: Int -> GitCommit -> Int # hash :: GitCommit -> Int #
ToJSON GitCommit Source #
Instance details Defined in Development.Benchmark.Rules Methods toJSON :: GitCommit -> Value # toEncoding :: GitCommit -> Encoding # toJSONList :: [GitCommit] -> Value # toEncodingList :: [GitCommit] -> Encoding #
FromJSON GitCommit Source #
Instance details Defined in Development.Benchmark.Rules Methods parseJSON :: Value -> Parser GitCommit # parseJSONList :: Value -> Parser [GitCommit] #
Binary GitCommit Source #
Instance details Defined in Development.Benchmark.Rules Methods put :: GitCommit -> Put # get :: Get GitCommit # putList :: [GitCommit] -> Put #
NFData GitCommit Source #
Instance details Defined in Development.Benchmark.Rules Methods rnf :: GitCommit -> () #
type Rep GitCommit Source #
Instance details Defined in Development.Benchmark.Rules type Rep GitCommit = D1 ('MetaData "GitCommit" "Development.Benchmark.Rules" "shake-bench-0.1.0.3-9iZtOhtMvLmBAUDRjvlAYZ" 'False) (C1 ('MetaCons "GitCommit" 'PrefixI 'True) ((S1 ('MetaSel ('Just "gitName") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 Text) :: S1 ('MetaSel ('Just "name") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 (Maybe Text))) :: (S1 ('MetaSel ('Just "parent") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 (Maybe Text)) :*: S1 ('MetaSel ('Just "include") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 Bool))))