mono/tests/verifier/AssemblyRunner.cs

   1 //
   2 // AssemblyRunner.cs
   3 //
   4 // Author:
   5 //   Rodrigo Kumpera (rkumpera@novell.com)
   6 //
   7 // Copyright (C) 2007 Novell, Inc (http://www.novell.com)
   8 //
   9 // Licensed under the MIT license. See LICENSE file in the project root for full license information.//
  10 //
  11 using System;
  12 using System.IO;
  13 using System.Net;
  14 using System.Diagnostics;
  15 using System.Security;
  16 using System.Security.Policy;
  17 using System.Security.Permissions;
  18 using System.Threading;
  19
  20
  21 namespace AssemblyRunner {
  22
  23         public enum RunResult {
  24                 valid,
  25                 unverifiable,
  26                 invalid,
  27                 strict,
  28                 none
  29         }
  30
  31         public class Runner {
  32             private static string program = "\"%COMSPEC%\"";
  33
  34                 static String ExecuteAndFetchStderr (String command) {
  35                         ProcessStartInfo psi = new ProcessStartInfo (Environment.ExpandEnvironmentVariables (program), "/c " + command);
  36                         psi.CreateNoWindow = true;
  37                         psi.UseShellExecute = false;
  38                         psi.RedirectStandardError = true;
  39
  40                     string stderr=null;
  41                     Process activeProcess=null;
  42
  43                         Thread stderrThread = new Thread (new ThreadStart (delegate {
  44                                 if (activeProcess != null)
  45                                         stderr = activeProcess.StandardError.ReadToEnd ();
  46                         }));
  47                         activeProcess = Process.Start (psi);
  48
  49                         stderrThread.Start ();
  50                 activeProcess.WaitForExit();
  51                         stderrThread.Join();
  52                         activeProcess.Close ();
  53                         return stderr;
  54             }
  55
  56                 static int ExecuteAndFetchReturnCode (String command) {
  57                         ProcessStartInfo psi = new ProcessStartInfo (Environment.ExpandEnvironmentVariables (program), "/c " + command);
  58                         psi.CreateNoWindow = true;
  59                         psi.UseShellExecute = false;
  60                         psi.RedirectStandardError = true;
  61
  62                     Process activeProcess = Process.Start (psi);
  63
  64                 activeProcess.WaitForExit();
  65                         int res = activeProcess.ExitCode;
  66                         activeProcess.Close ();
  67
  68                         return res;
  69             }
  70
  71                 static AppDomain NewDomain () {
  72                         PolicyStatement statement = new PolicyStatement(new PermissionSet(PermissionState.None),PolicyStatementAttribute.Nothing);
  73                         PermissionSet ps = new PermissionSet(PermissionState.None);
  74                         ps.AddPermission(new SecurityPermission(SecurityPermissionFlag.Execution));
  75                         ps.AddPermission(new SecurityPermission(SecurityPermissionFlag.Assertion));
  76                         ps.AddPermission(new SecurityPermission(SecurityPermissionFlag.ControlAppDomain));
  77                         ps.AddPermission(new SecurityPermission(SecurityPermissionFlag.ControlDomainPolicy));
  78                         ps.AddPermission(new SecurityPermission(SecurityPermissionFlag.ControlEvidence));
  79                         ps.AddPermission(new SecurityPermission(SecurityPermissionFlag.ControlPolicy));
  80                         ps.AddPermission(new SecurityPermission(SecurityPermissionFlag.ControlPrincipal));
  81                         ps.AddPermission(new SecurityPermission(SecurityPermissionFlag.ControlThread));
  82                         ps.AddPermission(new SecurityPermission(SecurityPermissionFlag.Infrastructure));
  83                         ps.AddPermission(new SecurityPermission(SecurityPermissionFlag.RemotingConfiguration));
  84                         ps.AddPermission(new SecurityPermission(SecurityPermissionFlag.SerializationFormatter));
  85                         ps.AddPermission(new FileIOPermission(PermissionState.Unrestricted));
  86                         ps.AddPermission(new EnvironmentPermission(PermissionState.Unrestricted));
  87                         ps.AddPermission(new ReflectionPermission(PermissionState.Unrestricted));
  88                         ps.AddPermission(new RegistryPermission(PermissionState.Unrestricted));
  89                         ps.AddPermission(new IsolatedStorageFilePermission(PermissionState.Unrestricted));
  90                         ps.AddPermission(new EventLogPermission(PermissionState.Unrestricted));
  91                         ps.AddPermission(new PerformanceCounterPermission(PermissionState.Unrestricted));
  92                         ps.AddPermission(new DnsPermission(PermissionState.Unrestricted));
  93                         ps.AddPermission(new UIPermission(PermissionState.Unrestricted));
  94                         PolicyStatement statement1 = new PolicyStatement(ps,PolicyStatementAttribute.Exclusive);
  95                         CodeGroup group;
  96                         group = new UnionCodeGroup(new AllMembershipCondition(),statement);
  97                         group.AddChild(new UnionCodeGroup(new ZoneMembershipCondition(SecurityZone.MyComputer),statement1));
  98                         PolicyLevel level = PolicyLevel.CreateAppDomainLevel();
  99                         level.RootCodeGroup = group;
 100
 101                         AppDomain domain = AppDomain.CreateDomain ("test");
 102                         domain.SetAppDomainPolicy(level);
 103                         return domain;
 104                 }
 105
 106                 static RunResult testWithAppDomain (String path, String assemblyName) {
 107                         AppDomain domain = NewDomain ();
 108                         try {
 109                                 domain.ExecuteAssembly (path);
 110                                 return RunResult.valid;
 111                         } catch (InvalidProgramException) {
 112                                 return RunResult.invalid;
 113                         } catch (FileLoadException) {
 114                                 return RunResult.invalid;
 115                         } catch (VerificationException) {
 116                                 return RunResult.unverifiable;
 117                         } catch (TypeInitializationException ve) {
 118                                 if (ve.InnerException is VerificationException)
 119                                         return RunResult.unverifiable;
 120                                 Console.WriteLine ("Warning: test {0} thrown exception: {1} ", assemblyName, ve.InnerException);
 121                                 return RunResult.invalid;
 122                         } catch (MissingMemberException) {
 123                                 return RunResult.invalid;
 124                         } catch (MemberAccessException) {
 125                                 return RunResult.unverifiable;
 126                         } catch (TypeLoadException) {
 127                                 return RunResult.invalid;
 128                         } catch (BadImageFormatException) {
 129                                 return RunResult.invalid;
 130                         } catch (Exception e) {
 131                                 Console.WriteLine ("Warning: test {0} thrown exception {1}", assemblyName, e);
 132                                 return RunResult.valid;
 133                         } finally {
 134                                 AppDomain.Unload (domain);
 135                         }
 136                 }
 137
 138                 /*
 139                  * This test with runtime is usefull to assert if the code is unverifiable but not invalid.
 140                  * This test should be used to diagnose if it's the case of code that was reported as invalid but actually is unverifiable.
 141                  */
 142                 static RunResult testWithRuntime (String path) {
 143                         String stderr = ExecuteAndFetchStderr (path);
 144                         String[] knownErrors = new String[] {
 145                                 "MissingMethodException",
 146                                 "InvalidProgramException",
 147                                 "FileLoadException",
 148                                 "BadImageFormatException",
 149                                 "TypeLoadException",
 150                                 "VerificationException"
 151                         };
 152
 153                         foreach (String str in knownErrors) {
 154                                 if (stderr.IndexOf (str) >= 0)
 155                                         return RunResult.invalid;
 156                         }
 157                         return RunResult.valid;
 158                 }
 159
 160                 /*
 161                  * This test can only assert if the code is unverifiable or not. Use it
 162                  * to check if it's the case for a strict check or the code is verifiable.
 163                  */
 164                 static RunResult testWithPeverify (String path) {
 165                         if (ExecuteAndFetchReturnCode ("peverify "+path) == 0)
 166                                 return RunResult.valid;
 167                         return RunResult.unverifiable;
 168                 }
 169
 170                 static RunResult decide (RunResult ad, RunResult rt, RunResult pv, String testName) {
 171                         if (ad == RunResult.valid) {
 172                                 if (rt != RunResult.valid) {
 173                                         Console.WriteLine ("Warning: test {0} returned valid under AD but {1} under runtime. PV said {2}, using runtime choice", testName, rt, pv);
 174                                         return rt;
 175                                 }
 176                                 if (pv != RunResult.valid)
 177                                         return RunResult.strict;
 178                                 return RunResult.valid;
 179                         }
 180
 181                         if (ad == RunResult.unverifiable) {
 182                                 //the rt test cannot complain about unverifiable
 183
 184                                 if (pv == RunResult.valid)
 185                                         Console.WriteLine ("Warning: test {0} returned unverifiable under AD but {1} under PV, using AD choice", testName, pv);
 186
 187                                 if (rt == RunResult.invalid) {
 188                                         /*This warning doesn't help a lot since there are cases which this happens
 189                                         Console.WriteLine ("Warning: test {0} returned unverifiable under AD but {1} under runtime. PV said {2}, using runtime choice", testName, rt, pv);
 190                                         */
 191
 192                                         return rt;
 193                                 }
 194
 195                                 return RunResult.unverifiable;
 196                         }
 197
 198                         if (ad == RunResult.invalid) {
 199                                 //in some cases the runtime throws exceptions meant for invalid code but the code is only unverifiable
 200                                 //we double check that by checking if rt returns ok
 201
 202                                 if (pv == RunResult.valid)
 203                                         Console.WriteLine ("Warning: test {0} returned invalid under AD but {1} under PV, using AD choice", testName, pv);
 204
 205                                 if (rt == RunResult.valid)
 206                                         return RunResult.unverifiable;
 207
 208                                 return RunResult.invalid;
 209                         }
 210                         Console.WriteLine ("ERROR: test {0} returned an unknown result under ad {1}, runtime said {2} and PV {3} -- FIXME --", testName, ad, rt, pv);
 211                         return RunResult.none;
 212                 }
 213
 214                 static void executeTest (String path, String assemblyName, RunResult expected) {
 215                         RunResult ad = testWithAppDomain (path, assemblyName);
 216                         RunResult rt = testWithRuntime (path);
 217                         RunResult pv = testWithPeverify (path);
 218
 219                         RunResult veredict = decide (ad, rt, pv, assemblyName);
 220                         if (veredict != expected)
 221                                 Console.WriteLine ("ERROR: test {0} expected {1} but got {2} AD {3} RT {4} PV {5}", assemblyName, expected, veredict, ad, rt, pv);
 222                 }
 223
 224                 public static void Main (String[] args) {
 225                         String dirName = ".";
 226                         if (args.Length > 0)
 227                                 dirName = args[0];
 228                         DirectoryInfo dir = new DirectoryInfo (dirName);
 229                         foreach (FileInfo file in dir.GetFiles ()) {
 230                                 try {
 231                                         RunResult rr = RunResult.none;
 232                                         if (file.Name.StartsWith ("strict_"))
 233                                                 rr = RunResult.strict;
 234                                         else if (file.Name.StartsWith ("valid_"))
 235                                                 rr = RunResult.valid;
 236                                         else if (file.Name.StartsWith ("unverifiable_"))
 237                                                 rr = RunResult.unverifiable;
 238                                         else if (file.Name.StartsWith ("invalid_"))
 239                                                 rr = RunResult.invalid;
 240
 241                                         if (file.Name.EndsWith (".exe") && rr != RunResult.none)
 242                                                 executeTest (file.FullName, file.Name, rr);
 243                                 } catch (Exception e) {
 244                                         Console.WriteLine ("Warning: test {0} thrown exception {1}", file.FullName, e);
 245                                 }
 246                         }
 247                 }
 248         }
 249 }