mcs/errors/do-tests.pl

   1 #!/usr/bin/perl -w
   2
   3 unless ($#ARGV == 2) {
   4     print STDERR "Usage: $0 profile compiler glob-pattern\n";
   5     exit 1;
   6 }
   7
   8 #
   9 # Expected value constants
  10 #
  11 my $EXPECTING_WRONG_ERROR = 1;
  12 my $EXPECTING_NO_ERROR    = 2;
  13 my %expecting_map = ();
  14
  15 my $profile = $ARGV [0];
  16 my $compile = $ARGV [1];
  17 my $files   = $ARGV [2];
  18
  19 if (open (EXPECT_WRONG, "<$profile-expect-wrong-error")) {
  20         $expecting_map{$_} = $EXPECTING_WRONG_ERROR
  21         foreach map {
  22                 chomp,                     # remove trailing \n
  23                 s/\#.*//g,                 # remove # style comments
  24                 s/ //g;                    # remove whitespace
  25                 $_ eq "" ? () : $_;        # now copy over non empty stuff
  26         } <EXPECT_WRONG>;
  27
  28         close EXPECT_WRONG;
  29 }
  30
  31 if (open (EXPECT_NO, "<$profile-expect-no-error")) {
  32         $expecting_map{$_} = $EXPECTING_NO_ERROR
  33         foreach map {
  34                 chomp,                     # remove trailing \n
  35                 s/\#.*//g,                 # remove # style comments
  36                 s/ //g;                    # remove whitespace
  37                 $_ eq "" ? () : $_;        # now copy over non empty stuff
  38         } <EXPECT_NO>;
  39
  40         close EXPECT_NO;
  41 }
  42 my $RESULT_UNEXPECTED_CORRECT_ERROR     = 1;
  43 my $RESULT_CORRECT_ERROR                = 2;
  44 my $RESULT_UNEXPECTED_INCORRECT_ERROR   = 3;
  45 my $RESULT_EXPECTED_INCORRECT_ERROR     = 4;
  46 my $RESULT_UNEXPECTED_NO_ERROR          = 5;
  47 my $RESULT_EXPECTED_NO_ERROR            = 6;
  48
  49 my @statuses = (
  50         "UNEXPECTED SUCCESS",
  51         "SUCCESS",
  52         "UNEXPECTED INCORRECT ERROR",
  53         "INCORRECT ERROR",
  54         "UNEXPECTED NO ERROR",
  55         "NO ERROR",
  56 );
  57
  58 my @status_items = (
  59         [],
  60         [],
  61         [],
  62         [],
  63         [],
  64         [],
  65 );
  66
  67 my %results_map = ();
  68
  69 foreach (glob ($files)) {
  70         print "$_";
  71         my ($error_number) = (/[a-z]*(\d+)(-\d+)?\.cs/);
  72         my $options = `sed -n 's,^// Compiler options:,,p' $_`;
  73         chomp $options;
  74         print "...";
  75
  76         system "$compile $options $_ --expect-error $error_number > /dev/null";
  77
  78         exit 1 if $? & 127;
  79
  80         my $exit_value = $? >> 8;
  81
  82         die "unexpected return from mcs" if $exit_value > 2;
  83
  84         my $status;
  85
  86         if ($exit_value == 0) {
  87                 $status = $RESULT_UNEXPECTED_CORRECT_ERROR     if     exists $expecting_map {$_};
  88                 $status = $RESULT_CORRECT_ERROR                unless exists $expecting_map {$_};
  89         }
  90
  91         if ($exit_value == 1) {
  92                 $status = $RESULT_UNEXPECTED_INCORRECT_ERROR   unless exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_WRONG_ERROR;
  93                 $status = $RESULT_EXPECTED_INCORRECT_ERROR     if     exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_WRONG_ERROR;
  94         }
  95
  96         if ($exit_value == 2) {
  97                 $status = $RESULT_UNEXPECTED_NO_ERROR          unless exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_NO_ERROR;
  98                 $status = $RESULT_EXPECTED_NO_ERROR            if     exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_NO_ERROR;
  99         }
 100
 101
 102         push @{@status_items [($status - 1)]}, $_;
 103         print "@statuses[($status - 1)]\n";
 104         $results_map{$_} = $status;
 105 }
 106 print scalar @{@status_items [($RESULT_CORRECT_ERROR              - 1)]}, " Correct errors\n";
 107 print scalar @{@status_items [($RESULT_EXPECTED_INCORRECT_ERROR   - 1)]}, " Incorrect errors\n";
 108 print scalar @{@status_items [($RESULT_EXPECTED_NO_ERROR          - 1)]}, " No errors\n";
 109
 110 print scalar @{@status_items [($RESULT_UNEXPECTED_CORRECT_ERROR   - 1)]}, " Unexpected correct errors\n";
 111 print, print "\n" foreach @{@status_items [($RESULT_UNEXPECTED_CORRECT_ERROR - 1)]};
 112
 113 print scalar @{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]}, " Unexpected incorrect errors\n";
 114 print, print "\n" foreach @{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]};
 115
 116 print scalar @{@status_items [($RESULT_UNEXPECTED_NO_ERROR        - 1)]}, " Unexpected no errors\n";
 117 print, print "\n" foreach @{@status_items [($RESULT_UNEXPECTED_NO_ERROR - 1)]};
 118
 119 print "\n";
 120
 121 print "OVERALL:";
 122 print scalar @{@status_items [($RESULT_CORRECT_ERROR - 1)]}, " tests succeeded\n";
 123 print scalar @{@status_items [($RESULT_EXPECTED_INCORRECT_ERROR - 1)]} + scalar @{@status_items [($RESULT_EXPECTED_NO_ERROR - 1)]}, " known errors\n";
 124 print scalar @{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]} + scalar @{@status_items [($RESULT_UNEXPECTED_NO_ERROR - 1)]}, " errors\n";
 125 print scalar @{@status_items [($RESULT_UNEXPECTED_CORRECT_ERROR   - 1)]}, " new tests passing\n";
 126
 127 exit (
 128         scalar @{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]} +
 129         scalar @{@status_items [($RESULT_UNEXPECTED_NO_ERROR        - 1)]} +
 130         scalar @{@status_items [($RESULT_UNEXPECTED_CORRECT_ERROR   - 1)]}
 131 ) == 0 ? 0 : 1;