mono/tests/verifier/AssemblyRunner.cs

   1 //
   2 // AssemblyRunner.cs
   3 //
   4 // Author:
   5 //   Rodrigo Kumpera (rkumpera@novell.com)
   6 //
   7 // Copyright (C) 2007 Novell, Inc (http://www.novell.com)
   8 //
   9 // Permission is hereby granted, free of charge, to any person obtaining
  10 // a copy of this software and associated documentation files (the
  11 // "Software"), to deal in the Software without restriction, including
  12 // without limitation the rights to use, copy, modify, merge, publish,
  13 // distribute, sublicense, and/or sell copies of the Software, and to
  14 // permit persons to whom the Software is furnished to do so, subject to
  15 // the following conditions:
  16 //
  17 // The above copyright notice and this permission notice shall be
  18 // included in all copies or substantial portions of the Software.
  19 //
  20 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  21 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  22 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  23 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  24 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  25 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  26 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  27 //
  28 using System;
  29 using System.IO;
  30 using System.Net;
  31 using System.Diagnostics;
  32 using System.Security;
  33 using System.Security.Policy;
  34 using System.Security.Permissions;
  35 using System.Threading;
  36
  37
  38 namespace AssemblyRunner {
  39
  40         public enum RunResult {
  41                 valid,
  42                 unverifiable,
  43                 invalid,
  44                 strict,
  45                 none
  46         }
  47
  48         public class Runner {
  49             private static string program = "\"%COMSPEC%\"";
  50
  51                 static String ExecuteAndFetchStderr (String command) {
  52                         ProcessStartInfo psi = new ProcessStartInfo (Environment.ExpandEnvironmentVariables (program), "/c " + command);
  53                         psi.CreateNoWindow = true;
  54                         psi.UseShellExecute = false;
  55                         psi.RedirectStandardError = true;
  56
  57                     string stderr=null;
  58                     Process activeProcess=null;
  59
  60                         Thread stderrThread = new Thread (new ThreadStart (delegate {
  61                                 if (activeProcess != null)
  62                                         stderr = activeProcess.StandardError.ReadToEnd ();
  63                         }));
  64                         activeProcess = Process.Start (psi);
  65
  66                         stderrThread.Start ();
  67                 activeProcess.WaitForExit();
  68                         stderrThread.Join();
  69                         activeProcess.Close ();
  70                         return stderr;
  71             }
  72
  73                 static int ExecuteAndFetchReturnCode (String command) {
  74                         ProcessStartInfo psi = new ProcessStartInfo (Environment.ExpandEnvironmentVariables (program), "/c " + command);
  75                         psi.CreateNoWindow = true;
  76                         psi.UseShellExecute = false;
  77                         psi.RedirectStandardError = true;
  78
  79                     Process activeProcess = Process.Start (psi);
  80
  81                 activeProcess.WaitForExit();
  82                         int res = activeProcess.ExitCode;
  83                         activeProcess.Close ();
  84
  85                         return res;
  86             }
  87
  88                 static AppDomain NewDomain () {
  89                         PolicyStatement statement = new PolicyStatement(new PermissionSet(PermissionState.None),PolicyStatementAttribute.Nothing);
  90                         PermissionSet ps = new PermissionSet(PermissionState.None);
  91                         ps.AddPermission(new SecurityPermission(SecurityPermissionFlag.Execution));
  92                         ps.AddPermission(new SecurityPermission(SecurityPermissionFlag.Assertion));
  93                         ps.AddPermission(new SecurityPermission(SecurityPermissionFlag.ControlAppDomain));
  94                         ps.AddPermission(new SecurityPermission(SecurityPermissionFlag.ControlDomainPolicy));
  95                         ps.AddPermission(new SecurityPermission(SecurityPermissionFlag.ControlEvidence));
  96                         ps.AddPermission(new SecurityPermission(SecurityPermissionFlag.ControlPolicy));
  97                         ps.AddPermission(new SecurityPermission(SecurityPermissionFlag.ControlPrincipal));
  98                         ps.AddPermission(new SecurityPermission(SecurityPermissionFlag.ControlThread));
  99                         ps.AddPermission(new SecurityPermission(SecurityPermissionFlag.Infrastructure));
 100                         ps.AddPermission(new SecurityPermission(SecurityPermissionFlag.RemotingConfiguration));
 101                         ps.AddPermission(new SecurityPermission(SecurityPermissionFlag.SerializationFormatter));
 102                         ps.AddPermission(new FileIOPermission(PermissionState.Unrestricted));
 103                         ps.AddPermission(new EnvironmentPermission(PermissionState.Unrestricted));
 104                         ps.AddPermission(new ReflectionPermission(PermissionState.Unrestricted));
 105                         ps.AddPermission(new RegistryPermission(PermissionState.Unrestricted));
 106                         ps.AddPermission(new IsolatedStorageFilePermission(PermissionState.Unrestricted));
 107                         ps.AddPermission(new EventLogPermission(PermissionState.Unrestricted));
 108                         ps.AddPermission(new PerformanceCounterPermission(PermissionState.Unrestricted));
 109                         ps.AddPermission(new DnsPermission(PermissionState.Unrestricted));
 110                         ps.AddPermission(new UIPermission(PermissionState.Unrestricted));
 111                         PolicyStatement statement1 = new PolicyStatement(ps,PolicyStatementAttribute.Exclusive);
 112                         CodeGroup group;
 113                         group = new UnionCodeGroup(new AllMembershipCondition(),statement);
 114                         group.AddChild(new UnionCodeGroup(new ZoneMembershipCondition(SecurityZone.MyComputer),statement1));
 115                         PolicyLevel level = PolicyLevel.CreateAppDomainLevel();
 116                         level.RootCodeGroup = group;
 117
 118                         AppDomain domain = AppDomain.CreateDomain ("test");
 119                         domain.SetAppDomainPolicy(level);
 120                         return domain;
 121                 }
 122
 123                 static RunResult testWithAppDomain (String path, String assemblyName) {
 124                         AppDomain domain = NewDomain ();
 125                         try {
 126                                 domain.ExecuteAssembly (path);
 127                                 return RunResult.valid;
 128                         } catch (InvalidProgramException) {
 129                                 return RunResult.invalid;
 130                         } catch (FileLoadException) {
 131                                 return RunResult.invalid;
 132                         } catch (VerificationException) {
 133                                 return RunResult.unverifiable;
 134                         } catch (TypeInitializationException ve) {
 135                                 if (ve.InnerException is VerificationException)
 136                                         return RunResult.unverifiable;
 137                                 Console.WriteLine ("Warning: test {0} thrown exception: {1} ", assemblyName, ve.InnerException);
 138                                 return RunResult.invalid;
 139                         } catch (MissingMemberException) {
 140                                 return RunResult.invalid;
 141                         } catch (MemberAccessException) {
 142                                 return RunResult.unverifiable;
 143                         } catch (TypeLoadException) {
 144                                 return RunResult.invalid;
 145                         } catch (BadImageFormatException) {
 146                                 return RunResult.invalid;
 147                         } catch (Exception e) {
 148                                 Console.WriteLine ("Warning: test {0} thrown exception {1}", assemblyName, e);
 149                                 return RunResult.valid;
 150                         } finally {
 151                                 AppDomain.Unload (domain);
 152                         }
 153                 }
 154
 155                 /*
 156                  * This test with runtime is usefull to assert if the code is unverifiable but not invalid.
 157                  * This test should be used to diagnose if it's the case of code that was reported as invalid but actually is unverifiable.
 158                  */
 159                 static RunResult testWithRuntime (String path) {
 160                         String stderr = ExecuteAndFetchStderr (path);
 161                         String[] knownErrors = new String[] {
 162                                 "MissingMethodException",
 163                                 "InvalidProgramException",
 164                                 "FileLoadException",
 165                                 "BadImageFormatException",
 166                                 "TypeLoadException",
 167                                 "VerificationException"
 168                         };
 169
 170                         foreach (String str in knownErrors) {
 171                                 if (stderr.IndexOf (str) >= 0)
 172                                         return RunResult.invalid;
 173                         }
 174                         return RunResult.valid;
 175                 }
 176
 177                 /*
 178                  * This test can only assert if the code is unverifiable or not. Use it
 179                  * to check if it's the case for a strict check or the code is verifiable.
 180                  */
 181                 static RunResult testWithPeverify (String path) {
 182                         if (ExecuteAndFetchReturnCode ("peverify "+path) == 0)
 183                                 return RunResult.valid;
 184                         return RunResult.unverifiable;
 185                 }
 186
 187                 static RunResult decide (RunResult ad, RunResult rt, RunResult pv, String testName) {
 188                         if (ad == RunResult.valid) {
 189                                 if (rt != RunResult.valid) {
 190                                         Console.WriteLine ("Warning: test {0} returned valid under AD but {1} under runtime. PV said {2}, using runtime choice", testName, rt, pv);
 191                                         return rt;
 192                                 }
 193                                 if (pv != RunResult.valid)
 194                                         return RunResult.strict;
 195                                 return RunResult.valid;
 196                         }
 197
 198                         if (ad == RunResult.unverifiable) {
 199                                 //the rt test cannot complain about unverifiable
 200
 201                                 if (pv == RunResult.valid)
 202                                         Console.WriteLine ("Warning: test {0} returned unverifiable under AD but {1} under PV, using AD choice", testName, pv);
 203
 204                                 if (rt == RunResult.invalid) {
 205                                         /*This warning doesn't help a lot since there are cases which this happens
 206                                         Console.WriteLine ("Warning: test {0} returned unverifiable under AD but {1} under runtime. PV said {2}, using runtime choice", testName, rt, pv);
 207                                         */
 208
 209                                         return rt;
 210                                 }
 211
 212                                 return RunResult.unverifiable;
 213                         }
 214
 215                         if (ad == RunResult.invalid) {
 216                                 //in some cases the runtime throws exceptions meant for invalid code but the code is only unverifiable
 217                                 //we double check that by checking if rt returns ok
 218
 219                                 if (pv == RunResult.valid)
 220                                         Console.WriteLine ("Warning: test {0} returned invalid under AD but {1} under PV, using AD choice", testName, pv);
 221
 222                                 if (rt == RunResult.valid)
 223                                         return RunResult.unverifiable;
 224
 225                                 return RunResult.invalid;
 226                         }
 227                         Console.WriteLine ("ERROR: test {0} returned an unknown result under ad {1}, runtime said {2} and PV {3} -- FIXME --", testName, ad, rt, pv);
 228                         return RunResult.none;
 229                 }
 230
 231                 static void executeTest (String path, String assemblyName, RunResult expected) {
 232                         RunResult ad = testWithAppDomain (path, assemblyName);
 233                         RunResult rt = testWithRuntime (path);
 234                         RunResult pv = testWithPeverify (path);
 235
 236                         RunResult veredict = decide (ad, rt, pv, assemblyName);
 237                         if (veredict != expected)
 238                                 Console.WriteLine ("ERROR: test {0} expected {1} but got {2} AD {3} RT {4} PV {5}", assemblyName, expected, veredict, ad, rt, pv);
 239                 }
 240
 241                 public static void Main (String[] args) {
 242                         String dirName = ".";
 243                         if (args.Length > 0)
 244                                 dirName = args[0];
 245                         DirectoryInfo dir = new DirectoryInfo (dirName);
 246                         foreach (FileInfo file in dir.GetFiles ()) {
 247                                 try {
 248                                         RunResult rr = RunResult.none;
 249                                         if (file.Name.StartsWith ("strict_"))
 250                                                 rr = RunResult.strict;
 251                                         else if (file.Name.StartsWith ("valid_"))
 252                                                 rr = RunResult.valid;
 253                                         else if (file.Name.StartsWith ("unverifiable_"))
 254                                                 rr = RunResult.unverifiable;
 255                                         else if (file.Name.StartsWith ("invalid_"))
 256                                                 rr = RunResult.invalid;
 257
 258                                         if (file.Name.EndsWith (".exe") && rr != RunResult.none)
 259                                                 executeTest (file.FullName, file.Name, rr);
 260                                 } catch (Exception e) {
 261                                         Console.WriteLine ("Warning: test {0} thrown exception {1}", file.FullName, e);
 262                                 }
 263                         }
 264                 }
 265         }
 266 }