2 # The perl script requires the following variables to be bound
3 # to something meaningful before it will operate correctly:
10 # * a program to run (1st arg)
11 # * some "command-line opts" ( -O<opt1> -O<opt2> ... )
12 # [default: anything on the cmd line this script doesn't recognise ]
13 # the first opt not starting w/ "-" is taken to be an input
14 # file and (if it exists) is grepped for "what's going on here"
16 # * a file to feed to stdin ( -i<file> ) [default: /dev/null ]
17 # * a "time" command to use (-t <cmd>).
19 # * alternatively, a "-script <script>" argument says: run the
20 # named Bourne-shell script to do the test. It's passed the
21 # pgm-to-run as the one-and-only arg.
23 # Run the program with those options and that input, and check:
26 # * an expected exit status ( -x <val> ) [ default 0 ]
27 # * expected output on stdout ( -o1 <file> ) [ default /dev/null ]
28 # ( we'll accept one of several...)
29 # * expected output on stderr ( -o2 <file> ) [ default /dev/null ]
30 # ( we'll accept one of several...)
32 # (if the expected-output files' names end in .Z, then
33 # they are uncompressed before doing the comparison)
35 # (This is supposed to be a "prettier" replacement for runstdtest.)
39 # -accept-output replace output files with the ones actually generated by running
42 ($Pgm = $0) =~ s|.*/||;
52 $PgmStdinFile = '/dev/null';
53 if ( $ENV{'TMPDIR'} ) { # where to make tmp file names
54 $TmpPrefix = $ENV{'TMPDIR'};
56 $TmpPrefix ="$DEFAULT_TMPDIR";
57 $ENV{'TMPDIR'} = "$DEFAULT_TMPDIR"; # set the env var as well
59 # If this is Cygwin, ignore eol and CR characters.
60 # Perhaps required for MSYS too, although the cygpath
61 # bit is hopefully unnecessary.
62 if ( `uname | grep CYGWIN` ) {
63 $CONTEXT_DIFF=$CONTEXT_DIFF . " --strip-trailing-cr" ;
64 $TmpPrefix = `cygpath -m $TmpPrefix | tr -d \\\\n`;
66 $ScriptFile = "$TmpPrefix/run_me$$";
67 $DefaultStdoutFile = "$TmpPrefix/no_stdout$$"; # can't use /dev/null (e.g. Alphas)
68 $DefaultStderrFile = "$TmpPrefix/no_stderr$$";
74 $StatsFile = "$TmpPrefix/stats$$";
75 $CachegrindStats = "cachegrind.out.summary";
76 $SysSpecificTiming = '';
77 $SysCPUCounting = 0; # Use CPU counters
81 die "$Pgm: program to run not given as first argument\n" if $#ARGV < 0;
82 $ToRun = $ARGV[0]; shift(@ARGV);
83 # avoid picking up same-named thing from somewhere else on $PATH...
84 $ToRun = "./$ToRun" if -e "./$ToRun";
86 arg: while ($_ = $ARGV[0]) {
89 /^--$/ && do { # let anything past after --
90 push(@PgmArgs, @ARGV);
93 /^-v$/ && do { $Verbose = 1; next arg; };
94 /^-accept-output-stderr$/ && do { $SaveStderr = 1; next arg; };
95 /^-accept-output-stdout$/ && do { $SaveStdout = 1; next arg; };
96 /^-accept-output$/ && do { $SaveStdout = 1; $SaveStderr = 1; next arg; };
98 /^-stdout-binary/ && do { $StdoutBinary=1; next arg; };
99 /^-stderr-binary/ && do { $StderrBinary=1; next arg; };
101 /^-O(.*)/ && do { push(@PgmArgs, &grab_arg_arg('-O',$1)); next arg; };
102 /^-i(.*)/ && do { $PgmStdinFile = &grab_arg_arg('-i',$1);
104 print STDERR "$Pgm: bogus -i input file: $PgmStdinFile\n"
105 if $PgmStdinFile !~ /^\/dev\/.*$/ && ! -f $PgmStdinFile;
107 /^-fail/ && do { $PgmFail=1; next arg; };
108 /^-x(.*)/ && do { $PgmExitStatus = &grab_arg_arg('-x',$1);
110 print STDERR "$Pgm: bogus -x expected exit status: $PgmExitStatus\n"
111 if $PgmExitStatus !~ /^\d+$/;
113 /^-o1(.*)/ && do { $out_file = &grab_arg_arg('-o1',$1);
114 push(@PgmStdoutFile, $out_file);
116 /^-o2(.*)/ && do { $out_file = &grab_arg_arg('-o2',$1);
117 push(@PgmStderrFile, $out_file);
119 /^-prescript(.*)/ && do { $PreScript = &grab_arg_arg('-prescript',$1);
121 /^-postscript(.*)/ && do { $PostScript = &grab_arg_arg('-postscript',$1);
123 /^-script/ && do { print STDERR "$Pgm: -script argument is obsolete;\nUse -prescript and -postscript instead.\n";
126 /^-(ghc|hbc)-timing$/ && do { $SysSpecificTiming = $1;
128 /^-cpu-counting-(.)$/ && do { $SysCPUCounting = "$1";
130 /^-cachegrind$/ && do { $SysSpecificTiming = 'ghc-instrs';
133 /^-t(.*)/ && do { $TimeCmd = &grab_arg_arg('-t', $1); next arg; };
135 # anything else is taken to be a pgm arg
139 foreach $out_file ( @PgmStdoutFile ) {
140 if ( ! -f $out_file && !$SaveStdout ) {
141 print STDERR "$Pgm: warning: expected-stdout file missing: $out_file\n";
146 foreach $out_file ( @PgmStderrFile ) {
147 if ( ! -f $out_file && !$SaveStderr ) {
148 print STDERR "$Pgm: warning: expected-stderr file missing: $out_file\n";
155 # add on defaults if none specified
156 @PgmStdoutFile = ( $DefaultStdoutFile ) if $#PgmStdoutFile < 0;
157 @PgmStderrFile = ( $DefaultStderrFile ) if $#PgmStderrFile < 0;
159 # tidy up the pgm args:
160 # (1) look for the "first input file"
161 # and grep it for "interesting" comments (-- !!! )
162 # (2) quote any args w/ whitespace in them.
164 foreach $a ( @PgmArgs ) {
165 if (! $grep_done && $a !~ /^-/ && -f $a) {
166 print `egrep "^--[ ]?!!!" $a`;
169 if ($a =~ /\s/ || $a =~ /'/) {
170 $a =~ s/'/\\'/g; # backslash the quotes;
171 $a = "\"$a\""; # quote the arg
175 # deal with system-specific timing options
177 if ( $SysSpecificTiming =~ /^ghc/ ) {
178 if ($SysCPUCounting) {
179 # Count specified CPU events
180 $cpu_counting_ghc = "-a$SysCPUCounting";
182 $cpu_counting_ghc = "";
184 $TimingMagic = "+RTS -S$StatsFile $cpu_counting_ghc -RTS"
185 } elsif ( $SysSpecificTiming eq 'hbc' ) {
186 $TimingMagic = "-S$StatsFile";
189 if ($PreScript ne '') {
191 $PreScriptLines = `cat $PreScript`;
192 $PreScriptLines =~ s/\r//g;
194 $PreScriptLines = '';
197 if ($PostScript ne '') {
199 $PostScriptLines = `cat $PostScript`;
200 $PostScriptLines =~ s/\r//g;
201 $PostScriptLines =~ s#\$o1#$TmpPrefix/runtest$$.1#gm;
202 $PostScriptLines =~ s#\$o2#$TmpPrefix/runtest$$.2#gm;
203 # The postfix 'm' deals with recent versions of
204 # Perl that removed the $* feature
206 $PostScriptLines = '';
209 # OK, so we're gonna do the normal thing...
211 if ($Cachegrind eq 'yes') {
212 $CachegrindPrefix = "valgrind --tool=cachegrind --log-fd=9 9>$CachegrindStats";
214 $CachegrindPrefix = '';
217 $Script = <<EOSCRIPT;
221 rm -f $DefaultStdoutFile $DefaultStderrFile
222 cat /dev/null > $DefaultStdoutFile
223 cat /dev/null > $DefaultStderrFile
226 $TimeCmd /bin/sh -c \'$CachegrindPrefix $ToRun $TimingMagic @PgmArgs < $PgmStdinFile 1> $TmpPrefix/runtest$$.1.raw 2> $TmpPrefix/runtest$$.2.raw 3> $TmpPrefix/runtest$$.3.raw\'
228 if [ "$StdoutBinary" = "0" ]; then
229 dos2unix < $TmpPrefix/runtest$$.1.raw > $TmpPrefix/runtest$$.1
231 cp $TmpPrefix/runtest$$.1.raw $TmpPrefix/runtest$$.1
233 if [ "$StderrBinary" = "0" ]; then
234 dos2unix < $TmpPrefix/runtest$$.2.raw > $TmpPrefix/runtest$$.2
236 cp $TmpPrefix/runtest$$.2.raw $TmpPrefix/runtest$$.2
238 dos2unix < $TmpPrefix/runtest$$.3.raw > $TmpPrefix/runtest$$.3
239 if [ \$progexit -eq 0 ] && [ $PgmFail -ne 0 ]; then
240 echo $ToRun @PgmArgs \\< $PgmStdinFile
241 echo "****" expected a failure, but was successful
244 if [ \$progexit -ne $PgmExitStatus ] && [ $PgmFail -eq 0 ]; then
245 echo $ToRun @PgmArgs \\< $PgmStdinFile
246 echo "****" expected exit status $PgmExitStatus not seen \\; got \$progexit
251 for out_file in @PgmStdoutFile ; do
252 if sed "s/
\r\$//" $TmpPrefix/runtest$$.1 | cmp -s \$out_file - ; then
256 if [ \$hit = 'NO' ] ; then
257 echo $ToRun @PgmArgs \\< $PgmStdinFile
258 echo expected stdout not matched by reality
259 orig_file="$PgmStdoutFile[0]";
260 [ ! -f \$orig_file ] && orig_file="/dev/null"
261 ${CONTEXT_DIFF} \$orig_file $TmpPrefix/runtest$$.1
265 if [ $SaveStdout = 1 ] &&
266 [ $PgmStdoutFile[0] != $DefaultStdoutFile ] && [ -s $TmpPrefix/runtest$$.1 ]; then
267 echo Saving away stdout output in $PgmStdoutFile[0] ...
268 if [ -f $PgmStdoutFile[0] ]; then
269 rm -f $PgmStdoutFile[0].bak
270 cp $PgmStdoutFile[0] $PgmStdoutFile[0].bak
272 cp $TmpPrefix/runtest$$.1 $PgmStdoutFile[0]
277 for out_file in @PgmStderrFile ; do
278 if sed "s/
\r\$//" $TmpPrefix/runtest$$.2 | cmp -s \$out_file - ; then
282 if [ \$hit = 'NO' ] ; then
283 echo $ToRun @PgmArgs \\< $PgmStdinFile
284 echo expected stderr not matched by reality
285 orig_file="$PgmStderrFile[0]"
286 [ ! -f \$orig_file ] && orig_file="/dev/null"
287 ${CONTEXT_DIFF} \$orig_file $TmpPrefix/runtest$$.2
291 if [ $SaveStderr = 1 ] &&
292 [ $PgmStderrFile[0] != $DefaultStderrFile ] && [ -s $TmpPrefix/runtest$$.2 ]; then
293 echo Saving away stderr output in $PgmStderrFile[0] ...
294 if [ -f $PgmStderrFile[0] ]; then
295 rm -f $PgmStderrFile[0].bak
296 cp $PgmStderrFile[0] $PgmStderrFile[0].bak
298 cp $TmpPrefix/runtest$$.2 $PgmStderrFile[0]
301 ${RM} core $ToRunOrig.spix $DefaultStdoutFile $DefaultStderrFile $TmpPrefix/runtest$$.1 $TmpPrefix/runtest$$.2 $TmpPrefix/runtest$$.3 $TmpPrefix/runtest$$.1.raw $TmpPrefix/runtest$$.2.raw $TmpPrefix/runtest$$.3.raw
305 # bung script into a file
306 open(SCR, "> $ScriptFile") || die "Failed opening script file $ScriptFile!\n";
308 close(SCR) || die "Failed closing script file!\n";
309 chmod 0755, $ScriptFile;
311 print STDERR $Script if $Verbose;
313 &run_something($ScriptFile);
315 if ( $SysSpecificTiming eq '' ) {
321 &process_stats_file();
322 &process_cachegrind_files() if $Cachegrind eq 'yes';
324 # print out what we found
325 print STDERR "<<$SysSpecificTiming: ";
326 if ( $Cachegrind ne 'yes') {
327 print STDERR "$BytesAlloc bytes, $GCs GCs, $AvgResidency/$MaxResidency avg/max bytes residency ($ResidencySamples samples), $GCWork bytes GC work, ${TotMem}M in use, $InitTime INIT ($InitElapsed elapsed), $MutTime MUT ($MutElapsed elapsed), $GcTime GC ($GcElapsed elapsed), $Gc0Time GC(0) ($Gc0Elapsed elapsed), $Gc1Time GC(1) ($Gc1Elapsed elapsed), $Balance balance$Counters";
329 print STDERR "$BytesAlloc bytes, $GCs GCs, $AvgResidency/$MaxResidency avg/max bytes residency ($ResidencySamples samples), $GCWork bytes GC work, ${TotMem}M in use, $InitTime INIT ($InitElapsed elapsed), $MutTime MUT ($MutElapsed elapsed), $GcTime GC ($GcElapsed elapsed), $Gc0Time GC(0) ($Gc0Elapsed elapsed), $Gc1Time GC(1) ($Gc1Elapsed elapsed), $Balance balance, $TotInstrs instructions, $TotReads memory reads, $TotWrites memory writes, $TotMisses L2 cache misses";
331 print STDERR " :$SysSpecificTiming>>\n";
339 local($option, $rest_of_arg) = @_;
341 if ($rest_of_arg ne "") {
342 return($rest_of_arg);
343 } elsif ($#ARGV >= 0) {
344 local($temp) = $ARGV[0]; shift(@ARGV);
347 print STDERR "$Pgm: no argument following $option option\n";
353 local($str_to_do) = @_;
355 # print STDERR "$str_to_do\n" if $Verbose;
357 local($return_val) = 0;
361 if ($return_val != 0) {
362 #ToDo: this return-value mangling is wrong
363 # local($die_msg) = "$Pgm: execution of the $tidy_name had trouble";
364 # $die_msg .= " (program not found)" if $return_val == 255;
365 # $die_msg .= " ($!)" if $Verbose && $! != 0;
370 exit (($return_val == 0) ? 0 : 1);
374 sub process_stats_file {
376 # OK, process system-specific stats file
377 if ( $SysSpecificTiming =~ /^ghc/ ) {
379 #NB: nearly the same as in GHC driver's -ghc-timing stuff
381 open(STATS, $StatsFile) || die "Failed when opening $StatsFile\n";
383 local($max_live) = 0;
384 local($tot_live) = 0; # for calculating residency stuff
385 local($tot_samples) = 0;
386 local($into_gc_counters) = 0; # once we reach into the GC counters part
387 local($counters) = "";
388 local($counter) = "";
395 if (! /Gen:\s+0/ && /^\s*\d+\s+\d+\s+(\d+)\s+\d+\.\d+/ ) {
396 $max_live = $1 if $max_live < $1;
401 $BytesAlloc = $1 if /^\s*([0-9,]+) bytes allocated in the heap/;
403 if (/^\s*([0-9,]+) bytes copied during GC/) {
409 # if ( /^\s*([0-9,]+) bytes maximum residency .* (\d+) sample/ ) {
410 # $MaxResidency = $1; $ResidencySamples = $2;
413 $GCs += $1 if /^\s*Generation\s*\d+:\s*([0-9,]+) collections/;
415 if ( /^\s+([0-9]+)\s+M[Bb] total memory/ ) {
419 if ( /^\s*INIT\s+time\s*(-*\d+\.\d\d)s\s*\(\s*(-*\d+\.\d\d)s elapsed\)/ ) {
420 $InitTime = $1; $InitElapsed = $2;
421 } elsif ( /^\s*MUT\s+time\s*(-*\d+\.\d\d)s\s*\(\s*(-*\d+\.\d\d)s elapsed\)/ ) {
422 $MutTime = $1; $MutElapsed = $2;
423 } elsif ( /^\s*GC\s+time\s*(-*\d+\.\d\d)s\s*\(\s*(-*\d+\.\d\d)s elapsed\)/ ) {
424 $GcTime = $1; $GcElapsed = $2;
427 if (/Generation (\d+):\s*\d+ collections,\s*\d+ parallel,\s*(-*\d+\.\d\d)s\s*,\s*(-*\d+\.\d\d)s elapsed/) {
429 $Gc0Time = $2; $Gc0Elapsed = $3;
431 $Gc1Time = $2; $Gc1Elapsed = $3;
435 if (/work balance: ([0-9.]+)/) {
440 if ( /CPU GC counters/ ) {
441 # Counters that follow correspond to GC
442 $into_gc_counters = 1;
445 if ( /^\s+\(([A-Z_0-9]+)\)\s+:\s+([0-9,]+)/ ) {
448 $count =~ s/,//g; # Remove commas in numbers
449 # Pretty printing elements of a list with type [(String,[Float])]
450 # It's a bit lame for passing values but it works.
451 if($into_gc_counters) {
452 $counters = "$counters(\"GC:$counter\",[$count]),";
454 $counters = "$counters(\"$counter\",[$count]),";
458 if ( /^\s+\(([A-Z_0-9]+)\)\s+\%\s+of\s+\(([A-Z_0-9]+)\)\s+:\s+([0-9.]+)/ ) {
459 $counter = "$1/$2"; # Relative quantity
461 # Pretty printing elements of a list with type [(String,[Float])]
462 # It's a bit lame for passing values but it works.
463 if($into_gc_counters) {
464 $counters = "$counters(\"GC:$counter\",[$count]),";
466 $counters = "$counters(\"$counter\",[$count]),";
471 close(STATS) || die "Failed when closing $StatsFile\n";
472 if ( $tot_samples > 0 ) {
473 $ResidencySamples = $tot_samples;
474 $MaxResidency = $max_live;
475 $AvgResidency = int ($tot_live / $tot_samples) ;
478 if ( length($counters) == 0 ) {
481 chop($counters); # remove trailing comma from the last entry
482 $Counters = " [$counters]";
485 } elsif ( $SysSpecificTiming eq 'hbc' ) {
487 open(STATS, $StatsFile) || die "Failed when opening $StatsFile\n";
489 $BytesAlloc = $1 if /^\s*([0-9]+) bytes allocated from the heap/;
491 $GCs = $1 if /^\s*([0-9]+) GCs?,$/;
493 if ( /^\s*(\d+\.\d\d) \((\d+\.\d)\) seconds total time,$/ ) {
494 $MutTime = $1; $MutElapsed = $2; # will fix up later
496 $InitTime = 0; $InitElapsed = 0; # hbc doesn't report these
498 } elsif ( /^\s*(\d+\.\d\d) \((\d+\.\d)\) seconds GC time/ ) {
499 $GcTime = $1; $GcElapsed = $2;
501 # fix up mutator time now
502 $MutTime = sprintf("%.2f", ($MutTime - $GcTime));
503 $MutElapsed = sprintf("%.1f", ($MutElapsed - $GcElapsed));
506 close(STATS) || die "Failed when closing $StatsFile\n";
509 # warn about what we didn't find
510 print STDERR "Warning: BytesAlloc not found in stats file\n" unless defined($BytesAlloc);
511 print STDERR "Warning: GCs not found in stats file\n" unless defined($GCs);
512 print STDERR "Warning: InitTime not found in stats file\n" unless defined($InitTime);
513 print STDERR "Warning: InitElapsed not found in stats file\n" unless defined($InitElapsed);
514 print STDERR "Warning: MutTime not found in stats file\n" unless defined($MutTime);
515 print STDERR "Warning: MutElapsed not found in stats file\n" unless defined($MutElapsed);
516 print STDERR "Warning: GcTime inot found in stats file\n" unless defined($GcTime);
517 print STDERR "Warning: GcElapsed not found in stats file\n" unless defined($GcElapsed);
518 print STDERR "Warning: total memory not found in stats file\n" unless defined($TotMem);
519 print STDERR "Warning: GC work not found in stats file\n" unless defined($GCWork);
520 print STDERR "Warning: Gc0Time not found in stats file\n" unless defined($Gc0Time);
521 print STDERR "Warning: Gc0Elapsed not found in stats file\n" unless defined($Gc0Elapsed);
522 print STDERR "Warning: Gc1Time not found in stats file\n" unless defined($Gc1Time);
523 print STDERR "Warning: Gc1Elapsed not found in stats file\n" unless defined($Gc1Elapsed);
524 print STDERR "Warning: Balance not found in stats file\n" unless defined($Gc1Elapsed);
526 # things we didn't necessarily expect to find
527 $MaxResidency = 0 unless defined($MaxResidency);
528 $AvgResidency = 0 unless defined($AvgResidency);
529 $ResidencySamples = 0 unless defined($ResidencySamples);
530 $Gc0Time = 0.0 unless defined($Gc0Time);
531 $Gc0Elapsed = 0.0 unless defined($Gc0Elapsed);
532 $Gc1Time = 0.0 unless defined($Gc1Time);
533 $Gc1Elapsed = 0.0 unless defined($Gc1Elapsed);
536 $BytesAlloc =~ s/,//g;
537 $MaxResidency =~ s/,//g;
540 $InitElapsed =~ s/,//g;
542 $MutElapsed =~ s/,//g;
544 $GcElapsed =~ s/,//g;
547 sub process_cachegrind_files {
549 open(STATS, "< $CachegrindStats") || die("Can't open $CachegrindStats\n");
552 /^==\d+==\s+I\s+refs:\s+([0-9,]*)/ && do {
554 $TotInstrs =~ s/,//g;
557 /^==\d+==\s+D\s+refs:\s+[0-9,]+\s+\(([0-9,]+)\s+rd\s+\+\s+([0-9,]+)\s+wr/ && do {
561 $TotWrites =~ s/,//g;
564 /^==\d+==\s+L2d\s+misses:\s+([0-9,]+)/ && do {
566 $TotMisses =~ s/,//g;