4 print STDERR "Usage: $0 profile compiler glob-pattern\n";
9 # Expected value constants
11 my $EXPECTING_WRONG_ERROR = 1;
12 my $EXPECTING_NO_ERROR = 2;
13 my %expecting_map = ();
16 my $profile = $ARGV [0];
17 my $compile = $ARGV [1];
18 my $files = $ARGV [2];
20 if (open (EXPECT_WRONG, "<$profile-expect-wrong-error")) {
21 $expecting_map{$_} = $EXPECTING_WRONG_ERROR
23 chomp, # remove trailing \n
24 s/\#.*//g, # remove # style comments
25 s/\s//g; # remove whitespace
26 $_ eq "" ? () : $_; # now copy over non empty stuff
32 if (open (EXPECT_NO, "<$profile-expect-no-error")) {
33 $expecting_map{$_} = $EXPECTING_NO_ERROR
35 chomp, # remove trailing \n
36 s/\#.*//g, # remove # style comments
37 s/\s//g; # remove whitespace
38 $_ eq "" ? () : $_; # now copy over non empty stuff
44 if (open (IGNORE, "<$profile-ignore-tests")) {
47 chomp, # remove trailing \n
48 s/\#.*//g, # remove # style comments
49 s/\s//g; # remove whitespace
50 $_ eq "" ? () : $_; # now copy over non empty stuff
56 my $RESULT_UNEXPECTED_CORRECT_ERROR = 1;
57 my $RESULT_CORRECT_ERROR = 2;
58 my $RESULT_UNEXPECTED_INCORRECT_ERROR = 3;
59 my $RESULT_EXPECTED_INCORRECT_ERROR = 4;
60 my $RESULT_UNEXPECTED_NO_ERROR = 5;
61 my $RESULT_EXPECTED_NO_ERROR = 6;
62 my $RESULT_UNEXPECTED_CRASH = 7;
67 "UNEXPECTED INCORRECT ERROR",
69 "UNEXPECTED NO ERROR",
87 foreach (glob ($files)) {
89 my ($error_number) = (/[a-z]*(\d+)(-\d+)?\.cs/);
90 my $options = `sed -n 's,^// Compiler options:,,p' $_`;
94 next if exists $ignore_map {$_};
96 my $testlogfile="$profile-$_.log";
97 system "$compile --expect-error $error_number $options $_ > $testlogfile 2>&1";
101 my $exit_value = $? >> 8;
105 if ($exit_value > 2) {
106 if (exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_WRONG_ERROR) {
107 $status = $RESULT_EXPECTED_INCORRECT_ERROR;
109 $status = $RESULT_UNEXPECTED_CRASH;
113 if ($exit_value == 0) {
114 system "rm -f $testlogfile";
115 $status = $RESULT_UNEXPECTED_CORRECT_ERROR if exists $expecting_map {$_};
116 $status = $RESULT_CORRECT_ERROR unless exists $expecting_map {$_};
119 if ($exit_value == 1) {
120 $status = $RESULT_UNEXPECTED_INCORRECT_ERROR unless exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_WRONG_ERROR;
121 $status = $RESULT_EXPECTED_INCORRECT_ERROR if exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_WRONG_ERROR;
124 if ($exit_value == 2) {
125 $status = $RESULT_UNEXPECTED_NO_ERROR unless exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_NO_ERROR;
126 $status = $RESULT_EXPECTED_NO_ERROR if exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_NO_ERROR;
130 push @{@status_items [($status - 1)]}, $_;
131 print "@statuses[($status - 1)]\n";
132 $results_map{$_} = $status;
135 my $correct = scalar @{@status_items [($RESULT_CORRECT_ERROR - 1)]};
136 print $correct, " correctly detected errors (", sprintf("%.2f",($correct / $total) * 100), " %) \n\n";
138 if (scalar @{@status_items [($RESULT_UNEXPECTED_CRASH - 1)]} > 0) {
139 print scalar @{@status_items [($RESULT_UNEXPECTED_CRASH - 1)]}, " compiler crashes\n";
140 print, print "\n" foreach @{@status_items [($RESULT_UNEXPECTED_CRASH - 1)]};
143 if (scalar @{@status_items [($RESULT_UNEXPECTED_CORRECT_ERROR - 1)]} > 0) {
144 print scalar @{@status_items [($RESULT_UNEXPECTED_CORRECT_ERROR - 1)]}, " fixed error report, remove it from expect-wrong-error or expect-no-error !\n";
145 print, print "\n" foreach @{@status_items [($RESULT_UNEXPECTED_CORRECT_ERROR - 1)]};
148 if (scalar @{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]} > 0) {
149 print scalar @{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]}, " new incorrect error report(s) !\n";
150 print, print "\n" foreach @{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]};
153 if (scalar @{@status_items [($RESULT_UNEXPECTED_NO_ERROR - 1)]} > 0) {
154 print scalar @{@status_items [($RESULT_UNEXPECTED_NO_ERROR - 1)]}, " new missing error report !\n";
155 print, print "\n" foreach @{@status_items [($RESULT_UNEXPECTED_NO_ERROR - 1)]};
159 scalar @{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]} +
160 scalar @{@status_items [($RESULT_UNEXPECTED_NO_ERROR - 1)]} +
161 scalar @{@status_items [($RESULT_UNEXPECTED_CRASH - 1)]}