#!/usr/bin/env bash set -e # Invoke with hevm e.g. # hevm compliance --tests ~/ethereum-tests --skip modexp --timeout 20 --html HEVM=${HEVM:-hevm} if [[ "$#" -lt 1 ]]; then echo >&2 "usage: $(basename "$0") <tests-dir>" exit 1 fi tests=$1 html=$2 match=$3 skip=$4 timeout=${5:-10} _html () { cat <<. <!doctype html> <title>hevm test results</title> <style> * { font-family: "latin modern mono", "fantasque sans mono", inconsolata, menlo, monospace; font-size: 22px; line-height: 26px; } body { margin: 2rem; } header { text-align: center; margin: 4rem 0; } table { border-collapse: collapse; width: 100%; } tr:nth-child(even) { background: rgba(0, 0, 0, 0.05); } td:not(:first-child):not(:last-child) { padding: 0 1rem; } .category { opacity: 0.6; text-align: right } a { color: darkblue; text-decoration: none; } h1, h2 { text-align: center; margin-top: 2rem } .testcase { font-weight: bold } #failed .testcase { color: rgb(200, 0, 0) } #balancefailed .testcase { color: rgb(200, 200, 0) } #passed .testcase { color: rgb(0, 150, 0) } #skipped .testcase { color: rgb(0, 100, 250) } #timeout .testcase { color: rgb(0, 0, 250) } </style> <header> <h1>hevm consensus test report</h1> <p>$(date +%Y-%m-%d)</p> <p>$(echo "$npass passed, $nbal bad-balance, $nnon bad-nonce, $nstr bad-storage, $nfail failed, $nskip skipped, $ntime timeout")</p> (Test suite: <span class=GeneralStateTests</span>GeneralStateTests</span> for Berlin) </header> <h2>Failed tests</h2> <table id=failed> <tbody> $(echo $noncefailed) $(echo $storagefailed) $(echo $failed) </table> <h2>Failed tests (due to balance only)</h2> <table id=balancefailed> <tbody> $(echo $balancefailed) </table> <h2>Timeout tests</h2> <table id=timeout> <tbody> $(echo $timeouts) </table> <h2>Skipped tests</h2> <table id=skipped> <tbody> $(echo $skipped) </table> <h2>Passed tests</h2> <table id=passed> <tbody> $(echo $passed) </table> . } shopt -s nocasematch { cd "$tests" for x in BlockchainTests/GeneralStateTests/*/*; do if [ -d $x ]; then for y in $x/*; do if [[ $y =~ .*$match.* ]] && [[ -n $skip && $y =~ .*$skip.* ]]; then for job in $(<$y jq '.|keys[]' -r); do echo -n "$job " ; echo "skip" done elif [[ $y =~ .*$match.* ]]; then set +e "$HEVM" bc-test --file $y --timeout $timeout 2>&1 set -e fi done else if [[ $x =~ .*$match.* ]] && [[ -n $skip && $x =~ .*$skip.* ]]; then for job in $(<$x jq '.|keys[]' -r); do echo -n "$job " ; echo "skip" done elif [[ $x =~ .*$match.* ]]; then set +e "$HEVM" bc-test --file $x --timeout $timeout 2>&1 set -e fi fi done } | { while read test outcome; do echo >&2 "$test $outcome" row="<tr><td class=testcase>$test<td>$outcome" row+=$'\n' case $outcome in ok) passed+=$row ;; bad-balance) balancefailed+=$row ;; bad-nonce) noncefailed+=$row ;; bad-storage) storagefailed+=$row ;; timeout) timeouts+=$row ;; skip) skipped+=$row ;; *) failed+=$row ;; esac done sum () { echo -ne "$1" | wc -l | awk '{print $1}'; } npass=$(sum "$passed") nbal=$(sum "$balancefailed") nnon=$(sum "$noncefailed") nstr=$(sum "$storagefailed") nfail=$(sum "$failed") ntime=$(sum "$timeouts") nskip=$(sum "$skipped") echo >&2 "passed: $npass" echo >&2 "bad-balance: $nbal" echo >&2 "bad-nonce: $nnon" echo >&2 "bad-storage: $nstr" echo >&2 "failed: $nfail" echo >&2 "timeout: $ntime" echo >&2 "skipped: $nskip" if [[ $html == "True" ]]; then _html fi nbad=$(($nbal + $nnon + $nstr + $nfail)) [[ $nbad -gt 0 ]] && exit 1 || exit 0 }