883 lines
30 KiB
Bash
883 lines
30 KiB
Bash
#!/bin/sh
|
||
|
||
# $Id: runall.sh 580483 2007-09-28 20:55:52Z sebor $
|
||
#
|
||
##############################################################################
|
||
#
|
||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||
# contributor license agreements. See the NOTICE file distributed
|
||
# with this work for additional information regarding copyright
|
||
# ownership. The ASF licenses this file to you under the Apache
|
||
# License, Version 2.0 (the "License"); you may not use this file
|
||
# except in compliance with the License. You may obtain a copy of
|
||
# the License at
|
||
#
|
||
# http://www.apache.org/licenses/LICENSE-2.0
|
||
#
|
||
# Unless required by applicable law or agreed to in writing, software
|
||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||
# implied. See the License for the specific language governing
|
||
# permissions and limitations under the License.
|
||
#
|
||
# Copyright 1999-2007 Rogue Wave Software, Inc.
|
||
#
|
||
##############################################################################
|
||
#
|
||
# NAME
|
||
# runall.sh, run - run programs and analyze output
|
||
#
|
||
# SYNOPSIS
|
||
# runall.sh [option(s)...] [file(s)...]
|
||
#
|
||
# DESCRIPTION
|
||
# Takes a list of files as arguments. The files are assumed to be
|
||
# executable; if a file isn't executable, it is assumed to have a
|
||
# .o suffix, in which case it is considered as an object file and
|
||
# its existence is taken as success. If the file isn't executable
|
||
# and doesn't have a .o suffix, it is considered a failure.
|
||
#
|
||
# Runs all executables named on the command line giving each a
|
||
# fixed amount of time to complete. If the program fails to exit
|
||
# within the alotted amount of time, it is killed via a
|
||
# signal. Collects and reports statistics on each executable with
|
||
# the -p option tries to post a result file into a revison control
|
||
# system (Perforce).
|
||
#
|
||
# OPTIONS
|
||
# -a <age>
|
||
# Do not run programs, just collect output unless output
|
||
# file is older than `age' minutes.
|
||
#
|
||
# -c or --color
|
||
# Produce colorized output (default). Requires xterm.
|
||
#
|
||
# -d <text>
|
||
# Use `text' as a description when submitting a
|
||
# report. If `text' is an existing file, use the contents
|
||
# of the file as the description. Meaningful only with -p.
|
||
#
|
||
# -l <fname> or --log <fname>
|
||
# Append the file named `fname' to the end of the result
|
||
# file before posting. Meaningful only with -p.
|
||
#
|
||
# --label <name>
|
||
# Label files with the label `name', or generate own label
|
||
# name if "--label -". The generated label will have the
|
||
# following format: stdlib-<compiler>-<os>-<timestamp>.
|
||
# Meaningful only with -p.
|
||
#
|
||
# -m or --mono or --nocolor
|
||
# Produce monochrome output (no escape sequences). Default
|
||
# for all terminals other than xterm.
|
||
#
|
||
# -o <fname>
|
||
# Redirect process output to the named file. The option
|
||
# affects only the next executable to be run, it is reset
|
||
# for all others.
|
||
#
|
||
# -p <postdir>
|
||
# Run tests and examples and post results in a revision
|
||
# control system (Perforce by default).
|
||
#
|
||
# --ph Run plumhall LVS tests
|
||
#
|
||
# --rate <percent>
|
||
# Minimum pass rate (in percent) to post. If the pass rate
|
||
# is lower, send email to $P4USER notifying that post
|
||
# failed. Meaningful only with -p.
|
||
#
|
||
# -r <fname>
|
||
# Compare results against `fname' and report differences as
|
||
# regressions.
|
||
#
|
||
# -s <signo>
|
||
# Set the signal to be sent to the hung process. SIGKILL
|
||
# (signo 9) is default.
|
||
#
|
||
# -t <sec>
|
||
# Set process timeout to `sec' seconds. Negative value of
|
||
# `sec' allows an infinite timeout.
|
||
#
|
||
# -T <ts0,ts1,ts2>
|
||
# Specifies the timestamps corresponding to the local times
|
||
# at which the following events started: configuration, lib
|
||
# build, and tests (and examples) build, in that order.
|
||
# Meaningful only with -p.
|
||
#
|
||
# -X <text>
|
||
# Specifies additional command line options to be passed to
|
||
# the executable being run.
|
||
#
|
||
# -x <dir>
|
||
# Run subsequent filenames as examples and diff output
|
||
# against files in the respective subdirectories of `dir'
|
||
# (manual/ or tutorial/), if they exist.
|
||
#
|
||
##############################################################################
|
||
|
||
|
||
##############################################################################
|
||
# GLOBAL VARIABLES
|
||
##############################################################################
|
||
|
||
TIMEOUT=10 # seconds by default
|
||
SIGNO=9 # signal to be sent by default (SIGKILL by default)
|
||
REGRESS="." # regression file (cwd by default, ignored)
|
||
LINES=24 # default number of continuous lines
|
||
MANDIR="-" # directory containing manual (none by default)
|
||
MIN_PASS_RATE=70 # minimum acceptable pass rate
|
||
POSTFILE="." # posted results file name
|
||
POSTDESC="test results" # description used when posting
|
||
LABEL="-" # generate a label when posting
|
||
OUTFILE="." # default output file name
|
||
AGEMAX=0 # if non-zero will not run tests, just looks at
|
||
# test results not older than AGEMAX minutes
|
||
XOPTS="-D ." # additional options to be passed to executable
|
||
BUILDLOG=/dev/null # log to append to results when posting
|
||
TIMESTAMPS="-" # times when lib and tests builds started
|
||
|
||
case "`uname`" in
|
||
CYGWIN*)
|
||
DIFF_STRIP_CR="--strip-trailing-cr";; # strip trailing CR on CygWin
|
||
esac
|
||
|
||
|
||
##############################################################################
|
||
# FUNCTION DEFINITIONS
|
||
##############################################################################
|
||
|
||
setcolors ()
|
||
{
|
||
COLORS=$1
|
||
|
||
if [ $COLORS -ne 0 ] ; then
|
||
ESC0="[30;30;0m" # escape sequence turning off any formatting
|
||
|
||
CLR_BLACK="[30;30;1m"
|
||
CLR_RED="[30;31;1m"
|
||
CLR_GREEN="[30;32;1m"
|
||
CLR_BLUE="[30;34;1m"
|
||
CLR_YELLOW="[30;33;1m"
|
||
CLR_MAGENTA="[30;35;1m"
|
||
CLR_CYAN="[30;36;1m"
|
||
CLR_GRAY="[30;37;1m"
|
||
|
||
CLR_ERROR="[41;33;1m" # serious error (core dump, etc)
|
||
CLR_SUCCESS="[42;37;1m" # important improvement (e.g., regress < 0)
|
||
CLR_WARN="[41;33;7;1m" # potential problem (decrease in assertins)
|
||
else
|
||
unset ESC0
|
||
unset CLR_BLACK
|
||
unset CLR_RED
|
||
unset CLR_GREEN
|
||
unset CLR_BLUE
|
||
unset CLR_YELLOW
|
||
unset CLR_CYAN
|
||
unset CLR_GRAY
|
||
unset CLR_ERROR
|
||
unset CLR_SUCCESS
|
||
unset CLR_WARN
|
||
fi
|
||
}
|
||
|
||
# post results in Perforce
|
||
post_results ()
|
||
{
|
||
p4 opened 2>&1 | grep "File(s) not opened on this client."
|
||
if [ $? -ne 0 ] ; then
|
||
echo "Warning: files opened on $P4CLIENT."
|
||
# exit 1
|
||
fi
|
||
|
||
# create a label containing all files in our client view
|
||
if [ "$LABEL" ] ; then
|
||
[ "$LABEL" = "-" ] && LABEL="stdlib-$POSTFILE-`date +%m.%d.%y-%H:%M`"
|
||
( echo Label: $LABEL ;
|
||
echo View: ;
|
||
p4 client -o | sed -n "s:\(//.*\) //.*:\1:p" ) | p4 label -i
|
||
fi
|
||
|
||
# see if we can check the file out
|
||
P4PATH=`p4 edit $POSTDIR/$POSTFILE 2>&1`
|
||
echo $P4PATH | grep " - file(s) not on client." > /dev/null 2>&1
|
||
|
||
# add the file if it is not in p4
|
||
[ $? -eq 0 ] && P4PATH=`p4 add $POSTDIR/$POSTFILE`
|
||
|
||
[ ! -x "$POSTDIR" ] && echo "$POSTDIR doesn't exist, can't post"
|
||
|
||
# copy postfile (and a log, if any) where it can be checked in from
|
||
cat $POSTFILE $BUILDLOG > $POSTDIR/$POSTFILE
|
||
|
||
# remove junk p4 appends to the end of file names
|
||
P4PATH=`echo $P4PATH | sed "s/#[1-9][0-9]* - .*$//"`
|
||
|
||
echo "executing: p4 submit -i $P4PATH"
|
||
|
||
# submit into Perforce
|
||
cat << EOF | p4 submit -i $P4PATH
|
||
Change: new
|
||
Description:
|
||
`cat /tmp/summary-$$`
|
||
Files: $P4PATH
|
||
EOF
|
||
|
||
# clean up
|
||
rm /tmp/summary-$$
|
||
}
|
||
|
||
# send mail with postfile (containing summary info) to p4 user
|
||
send_mail ()
|
||
{
|
||
# check for mailx to use -s subject
|
||
if type mailx > /dev/null 2>&1 ; then
|
||
mailcmd=mailx -s "could not post $POSTFILE"
|
||
elif [ `uname` = Linux ] ; then
|
||
# mail on Linux understands -s
|
||
mailcmd=mail -s "could not post $POSTFILE"
|
||
else
|
||
mailcmd=mail
|
||
fi
|
||
email=`p4 user -o | sed -n "s/^Email:.\(.*\)/\1/p"`
|
||
cat $POSTFILE | $mailcmd $email
|
||
}
|
||
|
||
|
||
# format a line on output
|
||
# $1: STATUS, $2: ASSRTS, $3 HIGH, $4 DELTA, $5 FAILED, $6 LOW,
|
||
# $7: REGRESS, $8: PERCNT
|
||
format ()
|
||
{
|
||
if [ $4 -ne 0 ] ; then
|
||
ESC2="$CLR_BLACK"
|
||
ESC4="$ESC2"
|
||
fi
|
||
|
||
# colorize regression field (numeric $1 okay, non-numeric considered error)
|
||
if [ $7 -gt 0 ] || [ `expr $1 + 0 >/dev/null 2>&1; echo $?` -eq 2 ] ; then
|
||
ESC7="$CLR_ERROR"
|
||
elif [ $7 -lt 0 ] ; then
|
||
ESC7="$CLR_SUCCESS"
|
||
fi
|
||
|
||
printf "$ESC1" >&2 ; printf " %6.6s" $1 ; printf "$ESC0" >&2
|
||
|
||
# skip negative value (indicates that values don't make sense)
|
||
if [ $2 -ge 0 ] ; then
|
||
printf "$ESC2" >&2 ; printf " %6d" $2 ; printf "$ESC0" >&2
|
||
printf "$ESC3" >&2 ; printf " %6d" $3 ; printf "$ESC0" >&2
|
||
printf "$ESC4" >&2 ; printf " %6d" $4 ; printf "$ESC0" >&2
|
||
printf "$ESC5" >&2 ; printf " %6d" $5 ; printf "$ESC0" >&2
|
||
printf "$ESC6" >&2 ; printf " %6d" $6 ; printf "$ESC0" >&2
|
||
printf "$ESC7" >&2 ; printf " %6d" $7 ; printf "$ESC0" >&2
|
||
printf "$ESC8" >&2 ; printf " %5d%%" $8 ; printf "$ESC0" >&2
|
||
printf "$ESC0" >&2
|
||
fi
|
||
|
||
echo
|
||
}
|
||
|
||
# get signal name from signal number (first argument)
|
||
signame ()
|
||
{
|
||
n=`expr \( \( $1 - 1 \) % 4 + 1 \) \* 2`
|
||
n=`kill -l | grep ".$1)" | head -1 | awk "{ print $"$n" }" | sed "s/SIG//"`
|
||
|
||
[ "$n" = "" ] && n="SIG$1"
|
||
echo $n
|
||
}
|
||
|
||
# compmute the elapsed time between adjacent arguments, each of which is
|
||
# expected to be in date +%T format, result is sent to stdout in +%T format
|
||
elapsed_time ()
|
||
{
|
||
first=$1
|
||
t0=`echo $1 | awk 'BEGIN { FS=":" } { print $1 * 3600 + $2 * 60 + $3 }'`
|
||
shift
|
||
for t in $* ; do
|
||
t1=`echo $t | awk 'BEGIN { FS=":" } { print $1 * 3600 + $2 * 60 + $3 }'`
|
||
[ $t1 -lt $t0 ] && t1=`expr $t1 + 24 \* 60 \* 60`
|
||
hrs=`expr \( $t1 - $t0 \) / 3600`
|
||
if [ $hrs -gt 0 ] ; then
|
||
printf "%d:%02d:%02d " $hrs \
|
||
`expr \( \( $t1 - $t0 \) % 3600 \) / 60` \
|
||
`expr \( $t1 - $t0 \) % 60`
|
||
else
|
||
printf "%d:%02d " `expr \( \( $t1 - $t0 \) % 3600 \) / 60` \
|
||
`expr \( $t1 - $t0 \) % 60`
|
||
fi
|
||
t0=$t1
|
||
done
|
||
|
||
# compute the difference between the first and the last args
|
||
[ $# -gt 2 ] && elapsed_time $first $t
|
||
}
|
||
|
||
|
||
fmat_diff ()
|
||
{
|
||
# search file for the first occurrence of a number followed by
|
||
# the string in $2; keep number and exit immediately to save time
|
||
d=`sed -n "/$2/{s/[^0-9]*\([1-9][0-9]*\).*$2.*/\1/p;q
|
||
}" $REGRESS`
|
||
|
||
if [ "$d" != "" ] ; then
|
||
d=`expr $1 - $d`
|
||
# output only if difference is non-zero
|
||
[ $d -ne 0 ] && printf "(%+d) " $d
|
||
fi
|
||
unset d
|
||
}
|
||
|
||
##############################################################################
|
||
# MAIN LOOP
|
||
##############################################################################
|
||
|
||
# ensure consistent behavior of POSIX utilities such as grep and sed
|
||
LANG=POSIX
|
||
|
||
# show help and exit when invoked with no arguments
|
||
if [ $# -eq 0 ] ; then
|
||
# print out top of this file (between the ###-lines)
|
||
# sed expression split up to satisfy non-GNU sed
|
||
sed -n "/^###/!D; {: print
|
||
n; /^###/q; s/^#//p; t print
|
||
}" $0
|
||
exit 0
|
||
fi
|
||
|
||
# reset the terminal settings on exit
|
||
trap 'printf "$ESC0" >&2 && exit' 0 2 3 15
|
||
|
||
SIGNAME=`signame $SIGNO`
|
||
|
||
HEADER="NAME STATUS ASSRTS HIGH DELTA FAILED LOW RGRESS PERCNT"
|
||
|
||
# write out our command line - uncomment for debugging
|
||
# echo $0 $*
|
||
|
||
tst_count=0 # number of tests tried
|
||
tst_pass=0 # number of successfully run tests (perhaps with assertions)
|
||
tst_ncomp=0 # number of tests that failed to compile
|
||
tst_nlink=0 # number of tests that failed to link
|
||
tst_fail=0 # number of tests run to completion but with failures
|
||
tst_core=0 # number of tests that dumped core
|
||
tst_kill=0 # number of tests that were forcefully killed
|
||
tst_rgrs=0 # number of tests with regressions
|
||
|
||
asrt_count=0 # total number of assertions
|
||
asrt_dlta=0 # increase in assertions from last run
|
||
asrt_pass=0 # total number of passed assertions
|
||
asrt_fail=0 # total number of failed assertions
|
||
asrt_rgrs=0 # increase in failed assertions from last run
|
||
asrt_hi=0 # the total highest number of failed assertions ever
|
||
asrt_lo=0 # the total lowest number of failed assertions ever
|
||
|
||
t0=`date +%T` # timestamp
|
||
|
||
# get $LINES and $COLUMNS set if possible
|
||
type resize > /dev/null 2>&1 && eval `resize` 2>&1 > /dev/null
|
||
[ -z "$LINES" ] && LINES=40
|
||
[ -z "$COLUMNS" ] && COLUMNS=80
|
||
|
||
if [ "$TERM" = "xterm" ] ;then
|
||
setcolors 1 # turn on colorized output
|
||
else
|
||
setcolors 0 # disable colorization
|
||
fi
|
||
|
||
for i do
|
||
# process command line options
|
||
|
||
# avoid the unportable [ ! "$foo" ]
|
||
if [ "" = "$AGEMAX" ] ; then
|
||
AGEMAX=$i
|
||
continue
|
||
elif [ "$i" = "-a" ] ; then
|
||
unset AGEMAX
|
||
continue
|
||
elif [ "" = "$POSTDESC" ] ; then
|
||
POSTDESC="$i"
|
||
continue
|
||
elif [ "$i" = "-d" ] ; then
|
||
unset POSTDESC
|
||
continue
|
||
elif [ "$i" = "-m" ] || [ "$i" = "--mono" ] || [ "$i" = "--nocolor" ]; then
|
||
setcolors 0
|
||
continue
|
||
elif [ "$i" = "-c" ] || [ "$i" = "--color" ] ; then
|
||
setcolors 1
|
||
continue
|
||
elif [ "" = "$BUILDLOG" ] ; then
|
||
BUILDLOG=$i
|
||
continue
|
||
elif [ "$i" = "-l" ] || [ "$i" = "--log" ]; then
|
||
unset BUILDLOG
|
||
continue
|
||
elif [ "" = "$LABEL" ] ; then
|
||
LABEL=$i
|
||
continue
|
||
elif [ "$i" = "--label" ] ; then
|
||
unset LABEL
|
||
continue
|
||
elif [ "" = "$REGRESS" ] ; then
|
||
REGRESS=$i
|
||
continue
|
||
elif [ "$i" = "-r" ] ; then
|
||
unset REGRESS
|
||
continue
|
||
elif [ "" = "$MIN_PASS_RATE" ] ; then
|
||
MIN_PASS_RATE=$i
|
||
continue
|
||
elif [ "$i" = "--rate" ] ; then
|
||
unset MIN_PASS_RATE
|
||
continue
|
||
elif [ "" = "$TIMEOUT" ] ; then
|
||
TIMEOUT=$i
|
||
continue
|
||
elif [ "$i" = "-t" ] ; then
|
||
unset TIMEOUT
|
||
continue;
|
||
elif [ "" = "$TIMESTAMPS" ] ; then
|
||
TIMESTAMPS=$i
|
||
continue
|
||
elif [ "$i" = "-T" ] ; then
|
||
unset TIMESTAMPS
|
||
continue;
|
||
elif [ "" = "$SIGNO" ] ; then
|
||
SIGNO=$i
|
||
SIGNAME=`signame $SIGNO`
|
||
continue
|
||
elif [ "$i" = "-s" ] ; then
|
||
unset SIGNO
|
||
continue
|
||
elif [ "" = "$MANDIR" ] ; then
|
||
MANDIR=$i/manual
|
||
TUTDIR=$i/tutorial
|
||
continue
|
||
elif [ "" = "$XOPTS" ] ; then
|
||
XOPTS="$i"
|
||
continue
|
||
elif [ "$i" = "-X" ] ; then
|
||
unset XOPTS
|
||
continue
|
||
elif [ "$i" = "-x" ] ; then
|
||
unset MANDIR
|
||
continue
|
||
elif [ "$i" = "--ph" ] ; then
|
||
PLUMHALL=1
|
||
MANDIR='-' # indicates we're running tests
|
||
continue
|
||
elif [ "" = "$POSTFILE" ] ; then
|
||
POSTDIR=`dirname $i`
|
||
POSTFILE=`basename $i`
|
||
continue
|
||
elif [ "$i" = "-p" ] ; then
|
||
unset POSTFILE
|
||
continue
|
||
elif [ "" = "$OUTFILE" ] ; then
|
||
OUTFILE=$i
|
||
continue
|
||
elif [ "$i" = "-o" ] ; then
|
||
unset OUTFILE
|
||
continue
|
||
fi;
|
||
|
||
n_asrt=0 # number of assertions in this test
|
||
d_asrt=0 # increase in assertions from last run
|
||
p_asrt=0 # passed assertions in this test
|
||
f_asrt=0 # failed assertions in this test
|
||
r_asrt=0 # number of regressions
|
||
h_asrt=0 # the highest number of assertions ever tested
|
||
l_asrt=0 # the least number of assertions ever tested
|
||
pcnt_asrt=0 # percentage of passed assertions in this test
|
||
no_asserts=0 # non-zero if output doesn't contain assertion tags
|
||
exit_status=0 # successful exit status by default (will be overridden)
|
||
|
||
ESC1="$ESC0" ; ESC2="$ESC0" ; ESC3="$ESC0" ; ESC4="$ESC0"
|
||
ESC5="$ESC0" ; ESC6="$ESC0" ; ESC7="$ESC0" ; ESC8="$ESC0"
|
||
|
||
if [ `expr $tst_count % $LINES` -eq 0 ] ; then
|
||
# redirect test number to stderr in order not to mess up diff
|
||
echo ; printf "$CLR_BLACK" >&2 ; printf "%s" "--- " >&2
|
||
echo "$HEADER" ; printf "$ESC0" >&2
|
||
fi
|
||
|
||
tst_count=`expr $tst_count + 1`
|
||
printf "%3d " $tst_count 1>&2
|
||
|
||
# strip trailing `.sh' from filename if present
|
||
file=`basename $i | sed "s/\.sh//"`
|
||
printf "%-18.18s" $file
|
||
|
||
# if an executable file exists _or_ if AGEMAX is not 0
|
||
if [ -x "$i" ] || [ $AGEMAX -ne 0 ] ; then
|
||
# get the name of input file if any (/dev/null by default)
|
||
if [ -f $MANDIR/in/$file.in ] ; then
|
||
input=$MANDIR/in/$file.in
|
||
elif [ -f $TUTDIR/in/$file.in ] ; then
|
||
input=$TUTDIR/in/$file.in
|
||
else
|
||
input=/dev/null
|
||
fi
|
||
|
||
if [ "$OUTFILE" = "." ] ; then
|
||
output="$i.out"
|
||
else
|
||
output=$OUTFILE
|
||
fi
|
||
|
||
if [ $AGEMAX -eq 0 ] ; then
|
||
|
||
# save cursor position in case the shell (bash 2 on SunOS)
|
||
# reports when a process gets a signal despite redirection
|
||
[ "$COLORS" != "0" ] && printf "7" >&2
|
||
|
||
# set alarm in process (in case we can't kill it if it hangs)
|
||
alrm=`expr $TIMEOUT + 3`
|
||
|
||
# run test in the background, redirect I/O to a file
|
||
# stderr redirected to /dev/null to prevent compilation
|
||
# warnings in "script" mode from causing false errors
|
||
$i $XOPTS -O $output -L $alrm -M "`uname -a`" \
|
||
> $output < $input 2>/dev/null &
|
||
|
||
# get the background process' pid
|
||
testpid=$!
|
||
|
||
# start a subshell to kill hung process
|
||
if [ $TIMEOUT -ge 0 ] ; then
|
||
(t=$TIMEOUT
|
||
while [ $t -ne 0 ]; do
|
||
# exit right away if process doesn't exist
|
||
ps -p $testpid > /dev/null 2>&1 || exit 0
|
||
|
||
# show countdown if colorized
|
||
[ "$COLORS" != "0" ] \
|
||
&& printf "$CLR_WARN %6d$ESC0[7D" $t >&2
|
||
t=`expr $t - 1`
|
||
sleep 1
|
||
done
|
||
kill -s $SIGNO $testpid > /dev/null 2>&1
|
||
)&
|
||
fi
|
||
|
||
# we'll test for this later
|
||
kill_status=`expr 128 + $SIGNO`
|
||
|
||
# wait for test to exit (on its own or forcefully)
|
||
wait $testpid
|
||
exit_status=$?
|
||
|
||
# restore cursor position
|
||
[ "$COLORS" != "0" ] && printf "8" >&2
|
||
else
|
||
# set kill status to be different than exit_status
|
||
kill_status=1
|
||
found=`find ./$output -mmin -$AGEMAX`
|
||
if [ "" = "$found" ] ; then
|
||
exit_status=-1
|
||
else
|
||
# indicate that output file was stale (older than AGEMAX)
|
||
exit_status=0
|
||
fi
|
||
fi
|
||
|
||
if [ $exit_status -lt 126 ] && [ "$PLUMHALL" ]; then
|
||
exit_status=0 #plum hall exit status is number of failed tests
|
||
fi
|
||
|
||
if [ $exit_status -eq 0 ]; then
|
||
if [ "$MANDIR" = "-" ] ; then # running tests
|
||
|
||
# make sure output file exists
|
||
if [ -f "$output" ] ; then
|
||
tst_pass=`expr $tst_pass + 1`
|
||
|
||
# grep for the number of total and failed assertions
|
||
if [ "$PLUMHALL" ]; then
|
||
f_asrt=`grep "errors detected" $output \
|
||
| sed "s/\*\*\*\*\* \([0-9]*\) .*/\1/"`
|
||
n_success=`grep "successful test case" $output \
|
||
| sed "s/\*\*\*\*\* \([0-9]*\) .*/\1/"`
|
||
if [ "$f_asrt" ] && [ "$n_success" ]; then
|
||
n_asrt=`expr $n_success + $f_asrt`
|
||
fi
|
||
else
|
||
n_asrt=`sed "s/[^ -~]/ /g" $output \
|
||
| grep "## Assertions" \
|
||
| tail -1 | sed "s/[^0-9]//g"`
|
||
f_asrt=`sed "s/[^ -~]/ /g" $output \
|
||
| grep "## FailedAssertions" \
|
||
| tail -1 | sed "s/[^0-9]//g"`
|
||
fi
|
||
|
||
if [ "" = "$n_asrt" ] ; then
|
||
n_asrt=0
|
||
no_assert=1
|
||
# no regression line found, something's fishy
|
||
exit_status="FORMAT" ; ESC1="$CLR_WARN"
|
||
fi
|
||
|
||
[ "" = "$f_asrt" ] && f_asrt=0
|
||
|
||
# increment the number of tests not passing at 100%
|
||
[ $f_asrt -gt 0 ] && tst_fail=`expr $tst_fail + 1`
|
||
|
||
# compute percentage of passed assertions
|
||
p_asrt=`expr $n_asrt - $f_asrt`
|
||
asrt_count=`expr $asrt_count + $n_asrt`
|
||
asrt_pass=`expr $asrt_pass + $p_asrt`
|
||
asrt_fail=`expr $asrt_fail + $f_asrt`
|
||
|
||
[ $f_asrt -gt 0 ] && ESC5="$CLR_RED"
|
||
|
||
# prevent division by zero
|
||
[ $n_asrt -ne 0 ] \
|
||
&& pcnt_asrt=`expr 100 \* $p_asrt / $n_asrt`
|
||
else # output file not found
|
||
exit_status="OUTPUT" ; ESC1="$CLR_ERROR"
|
||
fi
|
||
else # running examples (expect output)
|
||
no_asserts=1
|
||
|
||
# get the name of file against which to diff the output
|
||
if [ -f $MANDIR/out/$file.out ] ; then
|
||
diff_file=$MANDIR/out/$file.out
|
||
elif [ -f $TUTDIR/out/$file.out ] ; then
|
||
diff_file=$TUTDIR/out/$file.out
|
||
else
|
||
unset diff_file
|
||
fi
|
||
|
||
if [ "$diff_file" != "" ] ; then
|
||
diff $DIFF_STRIP_CR $diff_file $output >/dev/null 2>&1
|
||
if [ $? -eq 0 ] ; then
|
||
tst_pass=`expr $tst_pass + 1`
|
||
else
|
||
exit_status="DIFF" ; ESC1="$CLR_ERROR"
|
||
fi
|
||
else
|
||
exit_status="OUTPUT" ; ESC1="$CLR_WARN"
|
||
tst_pass=`expr $tst_pass + 1`
|
||
fi
|
||
ESC2="$ESC0" ; ESC3="$ESC2" ; ESC4="$ESC2" ; ESC5="$ESC2"
|
||
ESC6="$ESC2" ; ESC7="$ESC2" ; ESC8="$ESC2"
|
||
fi
|
||
elif [ $exit_status -eq 126 ] ; then
|
||
exit_status="EXIST" # can't find executable
|
||
ESC1="$CLR_WARN"
|
||
elif [ $exit_status -eq 127 ] ; then
|
||
exit_status="EXEC" # can't execute (check LD_LIBRARY_PATH)
|
||
ESC1="$CLR_WARN"
|
||
elif [ $exit_status -eq $kill_status ] ; then
|
||
exit_status=`signame $SIGNO` # forcefully killed
|
||
ESC1="$CLR_ERROR"
|
||
tst_kill=`expr $tst_kill + 1`
|
||
elif [ $exit_status -ge 128 ] ; then
|
||
n=`expr $exit_status - 128`
|
||
exit_status=`signame $n`
|
||
ESC1="$CLR_ERROR"
|
||
tst_core=`expr $tst_core + 1`
|
||
elif [ $exit_status -eq -1 ] ; then
|
||
exit_status="STALE"
|
||
ESC1="$CLR_WARN"
|
||
else
|
||
ESC1="$CLR_WARN"
|
||
fi
|
||
elif [ -f $i.o ] ; then
|
||
exit_status="LINK" # .o exists but no executable, failed to link
|
||
ESC1="$CLR_ERROR"
|
||
tst_nlink=`expr $tst_nlink + 1`
|
||
else
|
||
# file isn't executable, see if it's an object file...
|
||
suffix=`echo $i | sed "s/.*\(\..*\)/\1/"`
|
||
if [ "$suffix" = ".o" ] && [ -f "$i" ] ; then
|
||
no_asserts=1
|
||
tst_pass=`expr $tst_pass + 1`
|
||
else
|
||
exit_status="COMP" # no .o, failed to compile
|
||
ESC1="$CLR_ERROR"
|
||
tst_ncomp=`expr $tst_ncomp + 1`
|
||
fi
|
||
fi
|
||
|
||
if [ -f "$REGRESS" ] ; then
|
||
# find line for same executable in a previous report file
|
||
line=`grep "^$file[ .] " $REGRESS | head -n 1`
|
||
|
||
# compute difference in tried assertions
|
||
d_asrt=`echo $line | awk '{ print $3 }'`
|
||
d_asrt=`expr $n_asrt - 0$d_asrt`
|
||
|
||
# only consider if current exit status is 0 (success)
|
||
if [ "$exit_status" = "0" ] ; then
|
||
|
||
# regressions make sense only if previous exit status was 0
|
||
if [ "`echo $line | awk '{ print $2 }'`" = "0" ] ; then
|
||
r_asrt=`echo $line | awk '{ print $6 }'`
|
||
r_asrt=`expr $f_asrt - 0$r_asrt`
|
||
fi
|
||
|
||
# see if the lowest number of failed assertions improved
|
||
l_asrt=`echo $line | awk '{ print $7 }'`
|
||
[ "" = "$l_asrt" ] && l_asrt=0
|
||
[ $l_asrt -gt $f_asrt ] && l_asrt=$f_asrt && ESC6="$CLR_SUCCESS"
|
||
else
|
||
# carry over previous low assertions on failure
|
||
l_asrt=`echo $line | awk '{ print $7 }'`
|
||
[ "" = "$l_asrt" ] && l_asrt=0
|
||
|
||
# count the test as regressing if the last exit status was 0
|
||
old_status=`echo $line | awk '{ print $2 }'`
|
||
[ "$old_status" = "0" ] && tst_rgrs=`expr $tst_rgrs + 1`
|
||
fi
|
||
|
||
# adjust highest number of assertions achieved
|
||
h_asrt=`echo $line | awk '{ print $4 }'`
|
||
[ "" = "$h_asrt" ] && h_asrt=0
|
||
[ $h_asrt -lt $n_asrt ] && h_asrt=$n_asrt && ESC3="$CLR_BLACK"
|
||
|
||
# increment cumulative totals
|
||
asrt_dlta=`expr $asrt_dlta + $d_asrt`
|
||
asrt_rgrs=`expr $asrt_rgrs + $r_asrt`
|
||
|
||
[ $r_asrt -gt 0 ] && tst_rgrs=`expr $tst_rgrs + 1`
|
||
else
|
||
l_asrt=$f_asrt
|
||
h_asrt=$n_asrt
|
||
fi
|
||
|
||
# get rid of leading zeros (to prevent octal conversion by printf)
|
||
l_asrt=`expr $l_asrt + 0`
|
||
|
||
asrt_hi=`expr $h_asrt + $asrt_hi`
|
||
asrt_lo=`expr $l_asrt + $asrt_lo`
|
||
|
||
# set number of assertions negative if assertion tags not found
|
||
[ $no_asserts -ne 0 ] && n_asrt=-1
|
||
|
||
# careful with leading zeros (interpreted as octal with %d)
|
||
format $exit_status $n_asrt $h_asrt $d_asrt \
|
||
$f_asrt $l_asrt $r_asrt $pcnt_asrt
|
||
|
||
# reset name of output file
|
||
OUTFILE="."
|
||
done
|
||
|
||
# compute pass rate as percentage of tests with perfect score
|
||
# out of the total number of tests tried
|
||
pass_rate=0
|
||
[ $tst_count -gt 0 ] && pass_rate=`expr \( 100 \* \( $tst_pass - $tst_fail \) \) / $tst_count`
|
||
|
||
if [ $tst_count -gt 1 ] ; then
|
||
|
||
[ $asrt_count -ne 0 ] && pcnt_asrt=`expr 100 \* $asrt_pass / $asrt_count`
|
||
|
||
# show cumulative results
|
||
printf "\n--- " 1>&2
|
||
printf "%-18.18s" ""
|
||
|
||
ESC1="" ; ESC2="" ; ESC3="" ; ESC4=""
|
||
ESC5="" ; ESC6="" ; ESC7="" ; ESC8=""
|
||
|
||
# careful with leading zeros (interpreted as octal with %d)
|
||
format $tst_pass $asrt_count $asrt_hi $asrt_dlta \
|
||
$asrt_fail $asrt_lo $asrt_rgrs $pcnt_asrt
|
||
|
||
echo ; printf "${CLR_BLACK} " >&2
|
||
echo " FAIL COMP LINK ASSERT CORE KILL RGRESS";
|
||
printf "$ESC0 " >&2
|
||
printf " %6d %6d %6d %6d %6d %6d %6d\n" \
|
||
`expr $tst_count - $tst_pass` \
|
||
$tst_ncomp $tst_nlink $tst_fail $tst_core $tst_kill $tst_rgrs
|
||
echo
|
||
|
||
if [ -f "$REGRESS" ] ; then
|
||
|
||
# compute and format summary regressions wrt the previous report
|
||
# each var will be empty when there's no change, else "(+/-N) "
|
||
d_count=`fmat_diff $tst_count "programs"`
|
||
d_ncomp=`fmat_diff $tst_ncomp "failed to compile"`
|
||
d_nlink=`fmat_diff $tst_nlink "failed to link"`
|
||
d_fail=`fmat_diff $tst_fail "failed at least one assertion"`
|
||
d_core=`fmat_diff $tst_core "exited with a signal"`
|
||
d_kill=`fmat_diff $tst_kill "killed with signal"`
|
||
d_rate=`fmat_diff $pass_rate "pass rate"`
|
||
|
||
# special treatment (no variable $tst_nzero exists)
|
||
d_nzero=`expr $tst_count - $tst_pass`
|
||
d_nzero=`fmat_diff $d_nzero "exited with non-zero status"`
|
||
|
||
if [ -f "$POSTFILE" ] ; then
|
||
# modify REGRESS to be the p4 path name and rev # of what
|
||
# regressions are being reported against (used in description)
|
||
tmp=`p4 have $REGRESS | sed "s/ - .*//" 2>/dev/null`
|
||
[ $? -eq 0 ] && REGRESS="$tmp"
|
||
fi
|
||
regress_line="$asrt_rgrs regressions (baseline was $REGRESS)"
|
||
else
|
||
regress_line="no regressions (this is baseline)"
|
||
fi
|
||
|
||
# present summary only for more files than fits on screen or when posting
|
||
if [ $tst_count -ge $LINES -o -f "$POSTFILE" ] ; then
|
||
|
||
# compute and format timestamps
|
||
if [ "$TIMESTAMPS" != "-" ] ; then
|
||
# expect: "h0:m0:s0,h1:m1:s1,h2:m2:s2"
|
||
TIMESTAMPS="$TIMESTAMPS $t0 `date +%T`"
|
||
# replace commas with spaces
|
||
TIMESTAMPS=`echo $TIMESTAMPS | sed "s/,/ /g"`
|
||
TIMESTAMPS=`elapsed_time $TIMESTAMPS | awk '{ print \
|
||
"cfg: " $1 " lib: " $2 " tst: " $3 " run: " $4 " all: " $5 }'`
|
||
fi
|
||
|
||
# create a file containing summary info about this run
|
||
cat << EOF | tee /tmp/summary-$$
|
||
|
||
Generated by $LOGNAME@`uname -n` running `uname -sr`
|
||
|
||
`if [ -f "$POSTDESC" ]; then cat "$POSTDESC" ; else echo $POSTDESC; fi`
|
||
|
||
`[ "$TIMESTAMPS" != "-" ] && echo "Times: $TIMESTAMPS"`
|
||
|
||
Summary: out of $tst_count ${d_count}programs
|
||
`expr $tst_count - $tst_pass` ${d_nzero}exited with non-zero status
|
||
$tst_ncomp ${d_ncomp}failed to compile
|
||
$tst_nlink ${d_nlink}failed to link
|
||
$tst_fail ${d_fail}failed at least one assertion
|
||
$tst_core ${d_core}exited with a signal
|
||
$tst_kill ${d_kill}killed with signal $SIGNO
|
||
$regress_line
|
||
$pass_rate% ${d_rate}pass rate
|
||
|
||
EOF
|
||
fi
|
||
|
||
else
|
||
echo
|
||
fi
|
||
|
||
|
||
# post results in revision control system
|
||
if [ -f "$POSTFILE" ] ; then
|
||
# make sure minimum pass rate is achieved
|
||
if [ $pass_rate -lt $MIN_PASS_RATE ] ; then
|
||
# send email to p4 user if not passing at a desired rate
|
||
send_mail
|
||
echo "pass rate < $MIN_PASS_RATE%, can't post"
|
||
elif [ "$P4CLIENT" != "" ] ; then
|
||
# post results in Perforce
|
||
post_results
|
||
else
|
||
send_mail
|
||
echo "P4CLIENT not defined, can't post"
|
||
fi
|
||
|
||
elif [ "$POSTFILE" != "." ] ; then
|
||
echo "$POSTFILE not found, can't post"
|
||
exit 1
|
||
fi
|