mcs/errors/do-tests.pl

   1 #!/usr/bin/perl -w
   2
   3 unless ($#ARGV == 2) {
   4     print STDERR "Usage: $0 profile compiler glob-pattern\n";
   5     exit 1;
   6 }
   7
   8 #
   9 # Expected value constants
  10 #
  11 my $EXPECTING_WRONG_ERROR = 1;
  12 my $EXPECTING_NO_ERROR    = 2;
  13 my %expecting_map = ();
  14
  15 my $profile = $ARGV [0];
  16 my $compile = $ARGV [1];
  17 my $files   = $ARGV [2];
  18
  19 if (open (EXPECT_WRONG, "<$profile-expect-wrong-error")) {
  20         $expecting_map{$_} = $EXPECTING_WRONG_ERROR
  21         foreach map {
  22                 chomp,                     # remove trailing \n
  23                 s/\#.*//g,                 # remove # style comments
  24                 s/ //g;                    # remove whitespace
  25                 $_ eq "" ? () : $_;        # now copy over non empty stuff
  26         } <EXPECT_WRONG>;
  27
  28         close EXPECT_WRONG;
  29 }
  30
  31 if (open (EXPECT_NO, "<$profile-expect-no-error")) {
  32         $expecting_map{$_} = $EXPECTING_NO_ERROR
  33         foreach map {
  34                 chomp,                     # remove trailing \n
  35                 s/\#.*//g,                 # remove # style comments
  36                 s/ //g;                    # remove whitespace
  37                 $_ eq "" ? () : $_;        # now copy over non empty stuff
  38         } <EXPECT_NO>;
  39
  40         close EXPECT_NO;
  41 }
  42 my $RESULT_UNEXPECTED_CORRECT_ERROR     = 1;
  43 my $RESULT_CORRECT_ERROR                = 2;
  44 my $RESULT_UNEXPECTED_INCORRECT_ERROR   = 3;
  45 my $RESULT_EXPECTED_INCORRECT_ERROR     = 4;
  46 my $RESULT_UNEXPECTED_NO_ERROR          = 5;
  47 my $RESULT_EXPECTED_NO_ERROR            = 6;
  48
  49 my @statuses = (
  50         "UNEXPECTED SUCCESS",
  51         "SUCCESS",
  52         "UNEXPECTED INCORRECT ERROR",
  53         "INCORRECT ERROR",
  54         "UNEXPECTED NO ERROR",
  55         "NO ERROR",
  56 );
  57
  58 my @status_items = (
  59         [],
  60         [],
  61         [],
  62         [],
  63         [],
  64         [],
  65 );
  66
  67 my %results_map = ();
  68
  69 foreach (glob ($files)) {
  70         print "$_";
  71         my ($error_number) = (/[a-z]*(\d+)(-\d+)?\.cs/);
  72         my $options = `sed -n 's,^// Compiler options:,,p' $_`;
  73         chomp $options;
  74         print "...";
  75
  76         my $testlogfile="$_.log";
  77         system "$compile --expect-error $error_number $options $_ > $testlogfile 2>&1";
  78
  79         exit 1 if $? & 127;
  80
  81         my $exit_value = $? >> 8;
  82
  83         die "unexpected return from mcs" if $exit_value > 2;
  84
  85         my $status;
  86
  87         if ($exit_value == 0) {
  88                 system "rm -f $testlogfile";
  89                 $status = $RESULT_UNEXPECTED_CORRECT_ERROR     if     exists $expecting_map {$_};
  90                 $status = $RESULT_CORRECT_ERROR                unless exists $expecting_map {$_};
  91         }
  92
  93         if ($exit_value == 1) {
  94                 $status = $RESULT_UNEXPECTED_INCORRECT_ERROR   unless exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_WRONG_ERROR;
  95                 $status = $RESULT_EXPECTED_INCORRECT_ERROR     if     exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_WRONG_ERROR;
  96         }
  97
  98         if ($exit_value == 2) {
  99                 $status = $RESULT_UNEXPECTED_NO_ERROR          unless exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_NO_ERROR;
 100                 $status = $RESULT_EXPECTED_NO_ERROR            if     exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_NO_ERROR;
 101         }
 102
 103
 104         push @{@status_items [($status - 1)]}, $_;
 105         print "@statuses[($status - 1)]\n";
 106         $results_map{$_} = $status;
 107 }
 108 print scalar @{@status_items [($RESULT_CORRECT_ERROR              - 1)]}, " Correct errors\n";
 109 print scalar @{@status_items [($RESULT_EXPECTED_INCORRECT_ERROR   - 1)]}, " Incorrect errors\n";
 110 print scalar @{@status_items [($RESULT_EXPECTED_NO_ERROR          - 1)]}, " No errors\n";
 111
 112 print scalar @{@status_items [($RESULT_UNEXPECTED_CORRECT_ERROR   - 1)]}, " Unexpected correct errors\n";
 113 print, print "\n" foreach @{@status_items [($RESULT_UNEXPECTED_CORRECT_ERROR - 1)]};
 114
 115 print scalar @{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]}, " Unexpected incorrect errors\n";
 116 print, print "\n" foreach @{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]};
 117
 118 print scalar @{@status_items [($RESULT_UNEXPECTED_NO_ERROR        - 1)]}, " Unexpected no errors\n";
 119 print, print "\n" foreach @{@status_items [($RESULT_UNEXPECTED_NO_ERROR - 1)]};
 120
 121 print "\n";
 122
 123 print "OVERALL:";
 124 print scalar @{@status_items [($RESULT_CORRECT_ERROR - 1)]}, " tests succeeded\n";
 125 print scalar @{@status_items [($RESULT_EXPECTED_INCORRECT_ERROR - 1)]} + scalar @{@status_items [($RESULT_EXPECTED_NO_ERROR - 1)]}, " known errors\n";
 126 print scalar @{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]} + scalar @{@status_items [($RESULT_UNEXPECTED_NO_ERROR - 1)]}, " errors\n";
 127 print scalar @{@status_items [($RESULT_UNEXPECTED_CORRECT_ERROR   - 1)]}, " new tests passing\n";
 128
 129 exit (
 130         scalar @{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]} +
 131         scalar @{@status_items [($RESULT_UNEXPECTED_NO_ERROR        - 1)]} +
 132         scalar @{@status_items [($RESULT_UNEXPECTED_CORRECT_ERROR   - 1)]}
 133 ) == 0 ? 0 : 1;