#!/usr/bin/env bash set -e # Invoke with hevm e.g. # hevm compliance --tests ~/ethereum-tests --group VM --skip quadratic --html HEVM=${HEVM:-hevm} if [[ "$#" -lt 1 ]]; then echo >&2 "usage: $(basename "$0") <tests-dir>" exit 1 fi tests=$1 html=$2 match=$3 skip=$4 timeout=${5:-10} _html() { cat <<. <!doctype html> <title>hevm test results</title> <style> * { font-family: "latin modern mono", "fantasque sans mono", inconsolata, menlo, monospace; font-size: 22px; line-height: 26px; } body { margin: 2rem; } header { text-align: center; margin: 4rem 0; } table { border-collapse: collapse; width: 100%; } tr:nth-child(even) { background: rgba(0, 0, 0, 0.05); } td:not(:first-child):not(:last-child) { padding: 0 1rem; } .category { opacity: 0.6; text-align: right } a { color: darkblue; text-decoration: none; } h1, h2 { text-align: center; margin-top: 2rem } .testcase { font-weight: bold } #failed .testcase { color: rgb(200, 0, 0) } #passed .testcase { color: rgb(0, 150, 0) } #skipped .testcase { color: rgb(0, 100, 250) } </style> <header> <h1>hevm consensus test report</h1> <p>$(date +%Y-%m-%d)</p> <p>$(echo "$npass passed, $nfail failed, $nskip skipped")</p> (Test suite: <span class=VMTests</span>VMTests</span> for ConstantinopleFix) </header> <h2>Failed tests</h2> <table id=failed> <tbody> $(echo $failed) </table> <h2>Skipped tests</h2> <table id=skipped> <tbody> $(echo $skipped) </table> <h2>Passed tests</h2> <table id=passed> <tbody> $(echo $passed) </table> . } { cd "$tests" for x in VMTests/*/*; do if [[ $x =~ .*$match.* ]] && [[ -n $skip && $x =~ .*$skip.* ]]; then for job in $(<$x jq '.|keys[]' -r); do echo "$x $job skip" done elif [[ $x =~ .*$match.* ]]; then echo -n "$x " ; "$HEVM" vm-test --file $x --timeout $timeout 2>&1 fi done } | { while read path test outcome; do echo >&2 "$path $test $outcome" category=$(dirname "$path") testcase=$(basename "${path%.json}") row="<tr><td class=testcase>$testcase<td>$outcome<td class=category>$category" row+=$'\n' case $outcome in ok) passed+=$row ;; skip) skipped+=$row ;; timeout) timouts+=row ;; *) failed+=$row ;; esac done sum () { echo -ne "$1" | wc -l | awk '{print $1}'; } nfail=$(sum "$failed") npass=$(sum "$passed") nskip=$(sum "$skipped") ntime=$(sum "$timeouts") echo >&2 "passed: $npass" echo >&2 "failed: $nfail" echo >&2 "timeout: $ntime" echo >&2 "skipped: $nskip" if [[ $html == "True" ]]; then _html fi [[ $nfail -gt 0 ]] && exit 1 || exit 0 }