Add symbol file testing to compiler tester
[mono.git] / mcs / tools / compiler-tester / compiler-tester.cs
index 10665a0ea3f44b9d02d44d1759739242cd1f63fa..84d878193cd98153ccff1bd97d874efbaf38be5d 100644 (file)
@@ -3,10 +3,9 @@
 //
 // Author:
 //   Marek Safar (marek.safar@gmail.com)
-//
-
 //
 // Copyright (C) 2008, 2009 Novell, Inc (http://www.novell.com)
+// Copyright (C) 2012 Xamarin Inc (http://www.xamarin.com)
 //
 // Permission is hereby granted, free of charge, to any person obtaining
 // a copy of this software and associated documentation files (the
@@ -35,6 +34,9 @@ using System.Reflection;
 using System.Text;
 using System.Collections;
 using System.Xml;
+using System.Collections.Generic;
+using Mono.CompilerServices.SymbolWriter;
+using System.Globalization;
 
 namespace TestRunner {
 
@@ -180,12 +182,9 @@ namespace TestRunner {
 
                        public VerificationData (string test_file)
                        {
-#if NET_2_0
                                this.test_file = test_file;
-#endif                         
                        }
 
-#if NET_2_0
                        string test_file;
 
                        public static VerificationData FromFile (string name, XmlReader r)
@@ -243,7 +242,6 @@ namespace TestRunner {
 
                                w.WriteEndElement ();
                        }
-#endif
 
                        public MethodData FindMethodData (string method_name, string declaring_type)
                        {
@@ -611,7 +609,8 @@ namespace TestRunner {
                        LoadError,
                        XmlError,
                        Success,
-                       ILError
+                       ILError,
+                       DebugError
                }
 
                public PositiveChecker (ITester tester, string verif_file):
@@ -725,13 +724,13 @@ namespace TestRunner {
                                        return true;
 
                                if (md.ILSize > il_size) {
-                                       checker.LogFileLine (test.FileName, "{0} (code size reduction {1} -> {2})", m_name, md.ILSize, il_size);
+                                       checker.LogFileLine (test.FileName, "{0} (code size reduction {1} -> {2})", decl_type + ": " + m_name, md.ILSize, il_size);
                                        md.ILSize = il_size;
                                        return true;
                                }
 
                                checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError,
-                                       string.Format ("{0} (code size {1} -> {2})", m_name, md.ILSize, il_size));
+                                       string.Format ("{0} (code size {1} -> {2})", decl_type + ": " + m_name, md.ILSize, il_size));
 
                                md.ILSize = il_size;
 
@@ -740,11 +739,10 @@ namespace TestRunner {
 
                        static int GetILSize (MethodBase mi)
                        {
-#if NET_2_0
                                MethodBody body = mi.GetMethodBody ();
                                if (body != null)
                                        return body.GetILAsByteArray ().Length;
-#endif
+
                                return 0;
                        }
 
@@ -850,7 +848,7 @@ namespace TestRunner {
                                        string ref_file = filename.Replace (".cs", "-ref.xml");
                                        try {
 #if !NET_2_1
-                                               XmlComparer.Compare (ref_file, doc_output);
+                                               new XmlComparer ("doc").Compare (ref_file, doc_output);
 #endif
                                        } catch (Exception e) {
                                                HandleFailure (filename, TestResult.XmlError, e.Message);
@@ -864,6 +862,21 @@ namespace TestRunner {
                                                if (!tester.CheckILSize (pt, this, file))
                                                        return false;
                                        }
+
+                                       if (filename.StartsWith ("test-debug", StringComparison.OrdinalIgnoreCase)) {
+                                               MonoSymbolFile mdb_file = MonoSymbolFile.ReadSymbolFile (file + ".mdb");
+                                               var mdb_xml_file = mdb_file.FileName + ".xml";
+                                               ConvertSymbolFileToXml (mdb_file, mdb_xml_file);
+
+                                               var ref_file = filename.Replace(".cs", "-ref.xml");
+                                               try {
+                                                       new XmlComparer ("symbols").Compare (ref_file, mdb_xml_file);
+                                               } catch (Exception e) {
+                                                       HandleFailure (filename, TestResult.DebugError, e.Message);
+                                                       return false;
+                                               }
+                                       }
+
                                }
                        } finally {
                                if (domain != null)
@@ -874,6 +887,80 @@ namespace TestRunner {
                        return true;
                }
 
+               static void ConvertSymbolFileToXml (MonoSymbolFile symbolFile, string xmlFile)
+               {
+                       using (XmlTextWriter writer = new XmlTextWriter (xmlFile, Encoding.UTF8)) {
+                               writer.Formatting = Formatting.Indented;
+
+                               writer.WriteStartDocument ();
+
+                               writer.WriteStartElement ("symbols");
+
+                               writer.WriteStartElement ("files");
+                               foreach (var file in symbolFile.Sources) {
+                                       writer.WriteStartElement ("file");
+                                       writer.WriteAttributeString ("id", file.Index.ToString ());
+                                       writer.WriteAttributeString ("name", Path.GetFileName (file.FileName));
+                                       writer.WriteEndElement ();
+                               }
+                               writer.WriteEndElement ();
+
+                               writer.WriteStartElement ("methods");
+                               foreach (var method in symbolFile.Methods) {
+                                       writer.WriteStartElement ("method");
+                                       writer.WriteAttributeString ("token", IntToHex (method.Token));
+
+                                       var il_entries = method.GetLineNumberTable ();
+                                       writer.WriteStartElement ("sequencepoints");
+                                       foreach (var entry in il_entries.LineNumbers) {
+                                               writer.WriteStartElement ("entry");
+                                               writer.WriteAttributeString ("il", IntToHex (entry.Offset));
+                                               writer.WriteAttributeString ("row", entry.Row.ToString ());
+                                               writer.WriteAttributeString ("file_ref", entry.File.ToString ());
+                                               writer.WriteAttributeString ("hidden", BoolToString (entry.IsHidden));
+                                               writer.WriteEndElement ();
+                                       }
+                                       writer.WriteEndElement ();
+
+                                       writer.WriteStartElement ("locals");
+                                       foreach (var local in method.GetLocals ()) {
+                                               writer.WriteStartElement ("entry");
+                                               writer.WriteAttributeString ("name", local.Name);
+                                               writer.WriteAttributeString ("il_index", local.Index.ToString ());
+                                               writer.WriteAttributeString ("scope_ref", local.BlockIndex.ToString ());
+                                               writer.WriteEndElement ();
+                                       }
+                                       writer.WriteEndElement ();
+
+                                       writer.WriteStartElement ("scopes");
+                                       foreach (var scope in method.GetCodeBlocks ()) {
+                                               writer.WriteStartElement ("entry");
+                                               writer.WriteAttributeString ("index", scope.Index.ToString ());
+                                               writer.WriteAttributeString ("start", IntToHex (scope.StartOffset));
+                                               writer.WriteAttributeString ("end", IntToHex (scope.EndOffset));
+                                               writer.WriteEndElement ();
+                                       }
+                                       writer.WriteEndElement ();
+
+                                       writer.WriteEndElement ();
+                               }
+                               writer.WriteEndElement ();
+
+                               writer.WriteEndElement ();
+                               writer.WriteEndDocument ();
+                       }
+               }
+
+               static string IntToHex (int value)
+               {
+                       return "0x" + value.ToString ("x", CultureInfo.InvariantCulture);
+               }
+
+               static string BoolToString (bool value)
+               {
+                       return value ? "true" : "false";
+               }
+
                protected override TestCase CreateTestCase (string filename, string [] options, string [] deps)
                {
                        return new PositiveTestCase (filename, options, deps);
@@ -929,6 +1016,10 @@ namespace TestRunner {
                                        }
                                        extra = null;
                                        break;
+
+                               case TestResult.DebugError:
+                                       LogFileLine (file, "REGRESSION (SUCCESS -> SYMBOL FILE ERROR)");
+                                       break;
                        }
 
                        if (extra != null)
@@ -941,11 +1032,7 @@ namespace TestRunner {
                public override void Initialize ()
                {
                        if (verif_file != null) {
-#if NET_2_0
                                LoadVerificationData (verif_file);
-#else
-                               throw new NotSupportedException ();
-#endif
                        }
 
                        base.Initialize ();
@@ -956,15 +1043,10 @@ namespace TestRunner {
                        base.CleanUp ();
 
                        if (update_verif_file) {
-#if NET_2_0
                                UpdateVerificationData (verif_file);
-#else
-                               throw new NotSupportedException ();
-#endif
                        }
                }
 
-#if NET_2_0
                void LoadVerificationData (string file)
                {
                        LogLine ("Loading verification data from `{0}' ...", file);
@@ -1001,7 +1083,6 @@ namespace TestRunner {
                                w.WriteEndElement ();
                        }
                }
-#endif
        }
 
        class NegativeChecker: Checker
@@ -1054,11 +1135,7 @@ namespace TestRunner {
                                // Some error tests require to have different error text for different runtimes.
                                if (filtered.StartsWith ("//GMCS")) {
                                        row = 1;
-#if !NET_2_0
-                                       return true;
-#else
                                        return AnalyzeTestFile(file, ref row, line, ref compiler_options, ref dependencies);
-#endif
                                }
 
                                check_error_line = !filtered.StartsWith ("//Line:0");
@@ -1152,8 +1229,13 @@ namespace TestRunner {
                                                if (check_msg) {
                                                        int first = line.IndexOf (':');
                                                        int second = line.IndexOf (':', first + 1);
-                                                       if (second == -1 || !check_error_line)
+                                                       if (line.IndexOf ("Warning as Error: ", first, StringComparison.Ordinal) > 0) {
+                                                               if (check_error_line) {
+                                                                       second = line.IndexOf (':', second + 1);
+                                                               }
+                                                       } else if (second == -1 || !check_error_line) {
                                                                second = first;
+                                                       }
 
                                                        string msg = line.Substring (second + 1).TrimEnd ('.').Trim ();
                                                        if (msg != expected_message && msg != expected_message.Replace ('`', '\'')) {
@@ -1362,7 +1444,7 @@ namespace TestRunner {
                                return 1;
                        }
 
-                       ArrayList files = new ArrayList ();
+                       var files = new List<string> ();
                        switch (test_pattern) {
                        case "v1":
                                files.AddRange (Directory.GetFiles (".", positive ? "test*.cs" : "cs*.cs"));
@@ -1384,7 +1466,19 @@ namespace TestRunner {
                        }
 
                        checker.Initialize ();
+/*
+                       files.Sort ((a, b) => {
+                               if (a.EndsWith ("-lib.cs", StringComparison.Ordinal)) {
+                                       if (!b.EndsWith ("-lib.cs", StringComparison.Ordinal))
+                                               return -1;
+                               } else if (b.EndsWith ("-lib.cs", StringComparison.Ordinal)) {
+                                       if (!a.EndsWith ("-lib.cs", StringComparison.Ordinal))
+                                               return 1;
+                               }
 
+                               return a.CompareTo (b);
+                       });
+*/
                        foreach (string s in files) {
                                string filename = Path.GetFileName (s);
                                if (Char.IsUpper (filename, 0)) { // Windows hack