mcs/errors/do-tests.pl

   1 #!/usr/bin/perl -w
   2
   3 unless ($#ARGV == 2) {
   4     print STDERR "Usage: $0 profile compiler glob-pattern\n";
   5     exit 1;
   6 }
   7
   8 #
   9 # Expected value constants
  10 #
  11 my $EXPECTING_WRONG_ERROR = 1;
  12 my $EXPECTING_NO_ERROR    = 2;
  13 my %expecting_map = ();
  14 my %ignore_map = ();
  15
  16 my $profile = $ARGV [0];
  17 my $compile = $ARGV [1];
  18 my $files   = $ARGV [2];
  19
  20 if (open (EXPECT_WRONG, "<$profile-expect-wrong-error")) {
  21         $expecting_map{$_} = $EXPECTING_WRONG_ERROR
  22         foreach map {
  23                 chomp,                     # remove trailing \n
  24                 s/\#.*//g,                 # remove # style comments
  25                 s/\s//g;                    # remove whitespace
  26                 $_ eq "" ? () : $_;        # now copy over non empty stuff
  27         } <EXPECT_WRONG>;
  28
  29         close EXPECT_WRONG;
  30 }
  31
  32 if (open (EXPECT_NO, "<$profile-expect-no-error")) {
  33         $expecting_map{$_} = $EXPECTING_NO_ERROR
  34         foreach map {
  35                 chomp,                     # remove trailing \n
  36                 s/\#.*//g,                 # remove # style comments
  37                 s/\s//g;                    # remove whitespace
  38                 $_ eq "" ? () : $_;        # now copy over non empty stuff
  39         } <EXPECT_NO>;
  40
  41         close EXPECT_NO;
  42 }
  43
  44 if (open (IGNORE, "<$profile-ignore-tests")) {
  45         $ignore_map{$_} = 1
  46         foreach map {
  47                 chomp,                     # remove trailing \n
  48                 s/\#.*//g,                 # remove # style comments
  49                 s/\s//g;                    # remove whitespace
  50                 $_ eq "" ? () : $_;        # now copy over non empty stuff
  51         } <IGNORE>;
  52
  53         close IGNORE;
  54 }
  55
  56 my $RESULT_UNEXPECTED_CORRECT_ERROR     = 1;
  57 my $RESULT_CORRECT_ERROR                = 2;
  58 my $RESULT_UNEXPECTED_INCORRECT_ERROR   = 3;
  59 my $RESULT_EXPECTED_INCORRECT_ERROR     = 4;
  60 my $RESULT_UNEXPECTED_NO_ERROR          = 5;
  61 my $RESULT_EXPECTED_NO_ERROR            = 6;
  62 my $RESULT_UNEXPECTED_CRASH             = 7;
  63
  64 my @statuses = (
  65         "UNEXPECTED SUCCESS",
  66         "SUCCESS",
  67         "UNEXPECTED INCORRECT ERROR",
  68         "INCORRECT ERROR",
  69         "UNEXPECTED NO ERROR",
  70         "NO ERROR",
  71         "UNEXPECTED CRASH"
  72 );
  73
  74 my @status_items = (
  75         [],
  76         [],
  77         [],
  78         [],
  79         [],
  80         [],
  81         [],
  82 );
  83
  84 my %results_map = ();
  85 my $total = 0;
  86
  87 foreach (glob ($files)) {
  88         print "$_";
  89         my ($error_number) = (/[a-z]*(\d+)(-\d+)?\.cs/);
  90         my $options = `sed -n 's,^// Compiler options:,,p' $_`;
  91         chomp $options;
  92         print "...";
  93
  94         if (exists $ignore_map {$_}) {
  95             print "IGNORED\n";
  96             next;
  97         }
  98
  99         $total++;
 100         my $testlogfile="$profile-$_.log";
 101         system "$compile --expect-error $error_number $options -out:$profile-$_.junk $_ > $testlogfile 2>&1";
 102
 103         exit 1 if $? & 127;
 104
 105         my $exit_value = $? >> 8;
 106
 107         my $status;
 108
 109         if ($exit_value > 2) {
 110                 if (exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_WRONG_ERROR) {
 111                         $status = $RESULT_EXPECTED_INCORRECT_ERROR;
 112                 } else {
 113                         $status = $RESULT_UNEXPECTED_CRASH;
 114                 }
 115         }
 116
 117         if ($exit_value == 0) {
 118                 system "rm -f $testlogfile";
 119                 $status = $RESULT_UNEXPECTED_CORRECT_ERROR     if     exists $expecting_map {$_};
 120                 $status = $RESULT_CORRECT_ERROR                unless exists $expecting_map {$_};
 121         }
 122
 123         if ($exit_value == 1) {
 124                 $status = $RESULT_UNEXPECTED_INCORRECT_ERROR   unless exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_WRONG_ERROR;
 125                 $status = $RESULT_EXPECTED_INCORRECT_ERROR     if     exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_WRONG_ERROR;
 126         }
 127
 128         if ($exit_value == 2) {
 129                 $status = $RESULT_UNEXPECTED_NO_ERROR          unless exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_NO_ERROR;
 130                 $status = $RESULT_EXPECTED_NO_ERROR            if     exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_NO_ERROR;
 131         }
 132
 133
 134         push @{@status_items [($status - 1)]}, $_;
 135         print "@statuses[($status - 1)]\n";
 136         $results_map{$_} = $status;
 137 }
 138 print "\n";
 139 my $correct = scalar @{@status_items [($RESULT_CORRECT_ERROR              - 1)]};
 140 print $correct, " correctly detected errors (", sprintf("%.2f",($correct / $total) * 100), " %) \n\n";
 141
 142 if (scalar @{@status_items [($RESULT_UNEXPECTED_CRASH - 1)]} > 0) {
 143     print scalar @{@status_items [($RESULT_UNEXPECTED_CRASH - 1)]}, " compiler crashes\n";
 144     print, print "\n" foreach @{@status_items [($RESULT_UNEXPECTED_CRASH - 1)]};
 145 }
 146
 147 if (scalar @{@status_items [($RESULT_UNEXPECTED_CORRECT_ERROR - 1)]} > 0) {
 148     print scalar @{@status_items [($RESULT_UNEXPECTED_CORRECT_ERROR   - 1)]}, " fixed error report(s), remove it from expect-wrong-error or expect-no-error !\n";
 149     print, print "\n" foreach @{@status_items [($RESULT_UNEXPECTED_CORRECT_ERROR - 1)]};
 150 }
 151
 152 if (scalar @{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]} > 0) {
 153     print scalar @{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]}, " new incorrect error report(s) !\n";
 154     print, print "\n" foreach @{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]};
 155 }
 156
 157 if (scalar @{@status_items [($RESULT_UNEXPECTED_NO_ERROR - 1)]} > 0) {
 158     print scalar @{@status_items [($RESULT_UNEXPECTED_NO_ERROR        - 1)]}, " new missing error report(s) !\n";
 159     print, print "\n" foreach @{@status_items [($RESULT_UNEXPECTED_NO_ERROR - 1)]};
 160 }
 161
 162 exit (
 163         scalar @{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]} +
 164         scalar @{@status_items [($RESULT_UNEXPECTED_NO_ERROR        - 1)]} +
 165         scalar @{@status_items [($RESULT_UNEXPECTED_CRASH           - 1)]}
 166 ) == 0 ? 0 : 1;