From: Mark Probst Date: Sat, 5 Dec 2015 00:30:24 +0000 (-0500) Subject: Automatic optimization bug bisector. X-Git-Url: http://wien.tomnetworks.com/gitweb/?a=commitdiff_plain;h=121f253944a52fbe65a3f0ef00d2c3dc27578fa5;p=mono.git Automatic optimization bug bisector. --- diff --git a/man/mono.1 b/man/mono.1 index b92c29473fa..c1efea05956 100644 --- a/man/mono.1 +++ b/man/mono.1 @@ -602,6 +602,16 @@ Currently this option is only supported on Linux. The maintainer options are only used by those developing the runtime itself, and not typically of interest to runtime users or developers. .TP +\fB--bisect=optimization:filename\fR +This flag is used by the automatic optimization bug bisector. It +takes an optimization flag and a filename of a file containing a list +of full method names, one per line. When it compiles one of the +methods in the file it will use the optimization given, in addition to +the optimizations that are otherwise enabled. Note that if the +optimization is enabled by default, you should disable it with `-O`, +otherwise it will just apply to every method, whether it's in the file +or not. +.TP \fB--break method\fR Inserts a breakpoint before the method whose name is `method' (namespace.class:methodname). Use `Main' as method name to insert a diff --git a/mono/mini/driver.c b/mono/mini/driver.c index 7f35d2549db..09da6d2875e 100644 --- a/mono/mini/driver.c +++ b/mono/mini/driver.c @@ -150,7 +150,7 @@ extern char *nacl_mono_path; #define EXCLUDED_FROM_ALL (MONO_OPT_SHARED | MONO_OPT_PRECOMP | MONO_OPT_UNSAFE | MONO_OPT_GSHAREDVT | MONO_OPT_FLOAT32) static guint32 -parse_optimizations (guint32 opt, const char* p) +parse_optimizations (guint32 opt, const char* p, gboolean cpu_opts) { guint32 exclude = 0; const char *n; @@ -160,8 +160,10 @@ parse_optimizations (guint32 opt, const char* p) mono_hwcap_init (); /* call out to cpu detection code here that sets the defaults ... */ - opt |= mono_arch_cpu_optimizations (&exclude); - opt &= ~exclude; + if (cpu_opts) { + opt |= mono_arch_cpu_optimizations (&exclude); + opt &= ~exclude; + } if (!p) return opt; @@ -287,12 +289,12 @@ mono_parse_default_optimizations (const char* p) { guint32 opt; - opt = parse_optimizations (DEFAULT_OPTIMIZATIONS, p); + opt = parse_optimizations (DEFAULT_OPTIMIZATIONS, p, TRUE); return opt; } -static char* -opt_descr (guint32 flags) { +char* +mono_opt_descr (guint32 flags) { GString *str = g_string_new (""); int i, need_comma; @@ -361,7 +363,7 @@ mini_regression_step (MonoImage *image, int verbose, int *total_run, int *total, int i; mono_set_defaults (verbose, opt_flags); - n = opt_descr (opt_flags); + n = mono_opt_descr (opt_flags); g_print ("Test run: image=%s, opts=%s\n", mono_image_get_filename (image), n); g_free (n); cfailed = failed = run = code_size = 0; @@ -455,7 +457,7 @@ mini_regression (MonoImage *image, int verbose, int *total_run) fprintf (mini_stats_fd, "$graph->set_legend(qw("); for (opt = 0; opt < G_N_ELEMENTS (opt_sets); opt++) { guint32 opt_flags = opt_sets [opt]; - n = opt_descr (opt_flags); + n = mono_opt_descr (opt_flags); if (!n [0]) n = (char *)"none"; if (opt) @@ -1374,10 +1376,10 @@ mono_jit_parse_options (int argc, char * argv[]) opt->soft_breakpoints = TRUE; opt->explicit_null_checks = TRUE; } else if (strncmp (argv [i], "--optimize=", 11) == 0) { - opt = parse_optimizations (opt, argv [i] + 11); + opt = parse_optimizations (opt, argv [i] + 11, TRUE); mono_set_optimizations (opt); } else if (strncmp (argv [i], "-O=", 3) == 0) { - opt = parse_optimizations (opt, argv [i] + 3); + opt = parse_optimizations (opt, argv [i] + 3, TRUE); mono_set_optimizations (opt); } else if (strcmp (argv [i], "--trace") == 0) { trace_options = (char*)""; @@ -1589,7 +1591,7 @@ mono_main (int argc, char* argv[]) } else if (strncmp (argv [i], "--single-method=", 16) == 0) { char *full_opts = g_strdup_printf ("-all,%s", argv [i] + 16); action = DO_SINGLE_METHOD_REGRESSION; - mono_single_method_regression_opt = parse_optimizations (opt, full_opts); + mono_single_method_regression_opt = parse_optimizations (opt, full_opts, TRUE); g_free (full_opts); } else if (strcmp (argv [i], "--verbose") == 0 || strcmp (argv [i], "-v") == 0) { mini_verbose++; @@ -1639,9 +1641,20 @@ mono_main (int argc, char* argv[]) } mini_stats_fd = fopen (argv [++i], "w+"); } else if (strncmp (argv [i], "--optimize=", 11) == 0) { - opt = parse_optimizations (opt, argv [i] + 11); + opt = parse_optimizations (opt, argv [i] + 11, TRUE); } else if (strncmp (argv [i], "-O=", 3) == 0) { - opt = parse_optimizations (opt, argv [i] + 3); + opt = parse_optimizations (opt, argv [i] + 3, TRUE); + } else if (strncmp (argv [i], "--bisect=", 9) == 0) { + char *param = argv [i] + 9; + char *sep = strchr (param, ':'); + if (!sep) { + fprintf (stderr, "Error: --bisect requires OPT:FILENAME\n"); + return 1; + } + char *opt_string = g_strndup (param, sep - param); + guint32 opt = parse_optimizations (0, opt_string, FALSE); + g_free (opt_string); + mono_set_bisect_methods (opt, sep + 1); } else if (strcmp (argv [i], "--gc=sgen") == 0) { switch_gc (argv, "sgen"); } else if (strcmp (argv [i], "--gc=boehm") == 0) { @@ -2158,7 +2171,7 @@ mono_main (int argc, char* argv[]) fprintf (mini_stats_fd, "["); for (i = 0; i < G_N_ELEMENTS (opt_sets); i++) { opt = opt_sets [i]; - n = opt_descr (opt); + n = mono_opt_descr (opt); if (!n [0]) n = "none"; fprintf (mini_stats_fd, "\"%s\",", n); diff --git a/mono/mini/mini-runtime.c b/mono/mini/mini-runtime.c index b78a31d35c4..f569e5c3539 100644 --- a/mono/mini/mini-runtime.c +++ b/mono/mini/mini-runtime.c @@ -2096,6 +2096,32 @@ mono_jit_find_compiled_method_with_jit_info (MonoDomain *domain, MonoMethod *met return NULL; } +static guint32 bisect_opt = 0; +static GHashTable *bisect_methods_hash = NULL; + +void +mono_set_bisect_methods (guint32 opt, const char *method_list_filename) +{ + FILE *file; + char method_name [2048]; + + bisect_opt = opt; + bisect_methods_hash = g_hash_table_new (g_str_hash, g_str_equal); + g_assert (bisect_methods_hash); + + file = fopen (method_list_filename, "r"); + g_assert (file); + + while (fgets (method_name, sizeof (method_name), file)) { + size_t len = strlen (method_name); + g_assert (len > 0); + g_assert (method_name [len - 1] == '\n'); + method_name [len - 1] = 0; + g_hash_table_insert (bisect_methods_hash, g_strdup (method_name), GINT_TO_POINTER (1)); + } + g_assert (feof (file)); +} + gboolean mono_do_single_method_regression = FALSE; guint32 mono_single_method_regression_opt = 0; MonoMethod *mono_current_single_method; @@ -2107,6 +2133,13 @@ mono_get_optimizations_for_method (MonoMethod *method, guint32 default_opt) { g_assert (method); + if (bisect_methods_hash) { + char *name = mono_method_full_name (method, TRUE); + void *res = g_hash_table_lookup (bisect_methods_hash, name); + g_free (name); + if (res) + return default_opt | bisect_opt; + } if (!mono_do_single_method_regression) return default_opt; if (!mono_current_single_method) { diff --git a/mono/mini/mini.c b/mono/mini/mini.c index 230062fdf33..f766fcdb96f 100644 --- a/mono/mini/mini.c +++ b/mono/mini/mini.c @@ -2566,10 +2566,13 @@ mono_codegen (MonoCompile *cfg) if (cfg->verbose_level > 0) { char* nm = mono_method_full_name (cfg->method, TRUE); - g_print ("Method %s emitted at %p to %p (code length %d) [%s]\n", + char *opt_descr = mono_opt_descr (cfg->opt); + g_print ("Method %s emitted at %p to %p (code length %d) [%s] with opts %s\n", nm, - cfg->native_code, cfg->native_code + cfg->code_len, cfg->code_len, cfg->domain->friendly_name); + cfg->native_code, cfg->native_code + cfg->code_len, cfg->code_len, cfg->domain->friendly_name, + opt_descr); g_free (nm); + g_free (opt_descr); } { diff --git a/mono/mini/mini.h b/mono/mini/mini.h index 8c6913add30..58d7c105ebf 100644 --- a/mono/mini/mini.h +++ b/mono/mini/mini.h @@ -2218,7 +2218,9 @@ void mini_jit_init (void); void mini_jit_cleanup (void); void mono_disable_optimizations (guint32 opts); void mono_set_optimizations (guint32 opts); +void mono_set_bisect_methods (guint32 opt, const char *method_list_filename); guint32 mono_get_optimizations_for_method (MonoMethod *method, guint32 default_opt); +char* mono_opt_descr (guint32 flags); void mono_set_verbose_level (guint32 level); MonoJumpInfoToken* mono_jump_info_token_new (MonoMemPool *mp, MonoImage *image, guint32 token); MonoJumpInfoToken* mono_jump_info_token_new2 (MonoMemPool *mp, MonoImage *image, guint32 token, MonoGenericContext *context); diff --git a/tools/crash-bisector/Program.cs b/tools/crash-bisector/Program.cs new file mode 100644 index 00000000000..86cc8f00bcf --- /dev/null +++ b/tools/crash-bisector/Program.cs @@ -0,0 +1,238 @@ +using System; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using System.IO; +using System.Text.RegularExpressions; +using System.Collections.Generic; + +namespace crashbisector +{ + class BisectInfo { + const int timeout = 60; + + public string MonoPath { get; set; } + public string OptName { get; set; } + public IEnumerable Args { get; set; } + Random rand; + + public BisectInfo () { + rand = new Random (); + } + + string Run (string bisectArg) { + var args = Args; + + if (bisectArg == null) { + args = new string[] { "-v" }.Concat (args); + } else { + args = new string[] { bisectArg }.Concat (args); + } + var startInfo = new ProcessStartInfo { + FileName = MonoPath, + Arguments = string.Join (" ", args), + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + startInfo.EnvironmentVariables.Add ("MONO_DEBUG", "no-gdb-backtrace"); + + using (var process = Process.Start (startInfo)) { + var stdoutTask = Task.Factory.StartNew (() => new StreamReader (process.StandardOutput.BaseStream).ReadToEnd (), TaskCreationOptions.LongRunning); + var stderrTask = Task.Factory.StartNew (() => new StreamReader (process.StandardError.BaseStream).ReadToEnd (), TaskCreationOptions.LongRunning); + + var success = process.WaitForExit (timeout < 0 ? -1 : (Math.Min (Int32.MaxValue / 1000, timeout) * 1000)); + if (!success || process.ExitCode != 0) { + return null; + } + + var stdout = stdoutTask.Result; + + return stdout; + } + } + + bool RunWithMethods (IEnumerable methods) { + var path = Path.GetTempFileName (); + File.WriteAllLines (path, methods); + var stdout = Run (String.Format ("--bisect={0}:{1}", OptName, path)); + File.Delete (path); + return stdout != null; + } + + IEnumerable EliminationOrder (int numChunks) { + var chunks = new int [numChunks]; + for (var i = 0; i < numChunks; ++i) + chunks [i] = i; + for (var i = 0; i < numChunks; ++i) { + var j = rand.Next (i, numChunks); + var tmp = chunks [i]; + chunks [i] = chunks [j]; + chunks [j] = tmp; + } + return chunks; + } + + bool TryEliminate (IEnumerable methods, int chunkSize) { + var count = methods.Count (); + if (chunkSize < 1 || chunkSize > count) + throw new Exception ("I can't do math."); + + var numChunks = (count + chunkSize - 1) / chunkSize; + foreach (var i in EliminationOrder (numChunks)) { + var firstIndex = i * chunkSize; + var lastPlusOneIndex = (i + 1) * chunkSize; + var methodsLeft = methods.Take (firstIndex).Concat (methods.Skip (lastPlusOneIndex)); + + if (chunkSize == 1) + Console.WriteLine ("Running without method at position {0}", firstIndex); + else + Console.WriteLine ("Running without methods at positions {0} to {1}", firstIndex, lastPlusOneIndex - 1); + var success = RunWithMethods (methodsLeft); + Console.WriteLine ("Crashed: {0}", !success); + + if (!success) { + Console.WriteLine ("Eliminating further from {0} methods.", methodsLeft.Count ()); + return EliminationStep (methodsLeft); + } + } + + return false; + } + + bool EliminationStep (IEnumerable methods) { + var count = methods.Count (); + + if (count < 2) { + Console.WriteLine ("Can't eliminate further. Methods required to crash are:\n{0}", + string.Join ("\n", methods)); + return true; + } + + if (count >= 9) { + var chunkSize = (int)Math.Floor (Math.Sqrt (count)); + Console.WriteLine ("Trying eliminating chunks of {0}.", chunkSize); + if (TryEliminate (methods, chunkSize)) + return true; + Console.WriteLine ("Chunks didn't succeed, eliminating individual methods."); + } + + if (TryEliminate (methods, 1)) + return true; + + Console.WriteLine ("Couldn't eliminate any method. Methods required to crash are:\n{0}", + string.Join ("\n", methods)); + return true; + } + + bool BisectStep (IEnumerable methods) { + var count = methods.Count (); + + if (count == 0) { + Console.WriteLine ("Error: No methods left - what happened?"); + return false; + } + if (count == 1) { + Console.WriteLine ("Found the offending method: {0}", methods.First ()); + return true; + } + + var half = count / 2; + var firstHalf = methods.Take (half); + var secondHalf = methods.Skip (half); + Console.WriteLine ("Splitting into two halves: {0} and {1} methods.", firstHalf.Count (), secondHalf.Count ()); + + Console.WriteLine ("Running first half."); + var firstSuccess = RunWithMethods (firstHalf); + Console.WriteLine ("Crashed: {0}", !firstSuccess); + + if (!firstSuccess) { + Console.WriteLine ("Continuing with first half."); + return BisectStep (firstHalf); + } + + Console.WriteLine ("Running second half."); + var secondSuccess = RunWithMethods (secondHalf); + Console.WriteLine ("Crashed: {0}", !secondSuccess); + + if (!secondSuccess) { + Console.WriteLine ("Continuing with second half."); + return BisectStep (secondHalf); + } + + Console.WriteLine ("Error: Both halves succeeded, can't bisect. Trying elimination."); + return EliminationStep (methods); + } + + public bool Bisect () { + Console.WriteLine ("Running to gather methods."); + var stdout = Run (null); + if (stdout == null) { + Console.Error.WriteLine ("Error: Failed to execute without optimization."); + Environment.Exit (1); + } + + var regex = new Regex ("converting[^\n]* method ([^\n]+)\n"); + var matches = regex.Matches (stdout); + var methods = new List (); + foreach (Match match in matches) { + var method = match.Groups [1].Value; + methods.Add (method); + } + + Console.WriteLine ("Bisecting {0} methods.", methods.Count); + + Console.WriteLine ("Running with all methods, just to make sure."); + var success = RunWithMethods (methods); + if (success) { + Console.WriteLine ("Error: Ran successfully with all methods optimized. Nothing to bisect."); + return false; + } + Console.WriteLine ("Crashed. Bisecting."); + return BisectStep (methods); + } + } + + class MainClass + { + static void UsageAndExit (int exitCode) { + Console.Error.WriteLine ("Usage: crash-bisector.exe --mono MONO-EXECUTABLE --opt OPTION-NAME -- MONO-ARG ..."); + Environment.Exit (exitCode); + } + + public static void Main (string[] args) + { + string monoPath = null; + string optName = null; + + var argIndex = 0; + while (argIndex < args.Length) { + if (args [argIndex] == "--mono") { + monoPath = args [argIndex + 1]; + argIndex += 2; + } else if (args [argIndex] == "--opt") { + optName = args [argIndex + 1]; + argIndex += 2; + } else if (args [argIndex] == "--help") { + UsageAndExit (0); + } else if (args [argIndex] == "--") { + argIndex += 1; + break; + } else { + UsageAndExit (1); + } + } + + if (monoPath == null || optName == null || argIndex == args.Length) + UsageAndExit (1); + + var bisectInfo = new BisectInfo { + MonoPath = monoPath, + OptName = optName, + Args = args.Skip (argIndex) + }; + var success = bisectInfo.Bisect (); + Environment.Exit (success ? 0 : 1); + } + } +} diff --git a/tools/crash-bisector/Properties/AssemblyInfo.cs b/tools/crash-bisector/Properties/AssemblyInfo.cs new file mode 100644 index 00000000000..c9d6924ee4a --- /dev/null +++ b/tools/crash-bisector/Properties/AssemblyInfo.cs @@ -0,0 +1,27 @@ +using System.Reflection; +using System.Runtime.CompilerServices; + +// Information about this assembly is defined by the following attributes. +// Change them to the values specific to your project. + +[assembly: AssemblyTitle ("crash-bisector")] +[assembly: AssemblyDescription ("")] +[assembly: AssemblyConfiguration ("")] +[assembly: AssemblyCompany ("")] +[assembly: AssemblyProduct ("")] +[assembly: AssemblyCopyright ("schani")] +[assembly: AssemblyTrademark ("")] +[assembly: AssemblyCulture ("")] + +// The assembly version has the format "{Major}.{Minor}.{Build}.{Revision}". +// The form "{Major}.{Minor}.*" will automatically update the build and revision, +// and "{Major}.{Minor}.{Build}.*" will update just the revision. + +[assembly: AssemblyVersion ("1.0.*")] + +// The following attributes are used to specify the signing key for the assembly, +// if desired. See the Mono documentation for more information about signing. + +//[assembly: AssemblyDelaySign(false)] +//[assembly: AssemblyKeyFile("")] + diff --git a/tools/crash-bisector/README.md b/tools/crash-bisector/README.md new file mode 100644 index 00000000000..96424f58fbf --- /dev/null +++ b/tools/crash-bisector/README.md @@ -0,0 +1,51 @@ +# Crash Bisector + +It is often difficult finding a bug in an optimization pass. The test +case for which the optimization produces incorrect results or crashes +the program might have thousands of methods compiled, and the bug +might only show up in one or a handful of them. It would be much +easier if you knew which methods specifically trigger the bug. + +This tool automates the search for those methods. Given some +reasonable conditions it will find a (locally) minimal set of methods +for which, if a given optimization is applied to them, a test case +will fail or crash. + +You will need a test case for which Mono either crashes with your +optimization, or returns a non-zero exit status. The bisector will +then run the test case without your optimization, gathering a list of +all the methods that are compiled. It will then start bisecting this +list, applying the optimization to only one half of the methods, +checking whether the test still fails. + +At some point bisecting will either terminate with a single method +that still makes the test fail, or it will come to a point where a set +of methods makes the test fail, but either half of that set will not. +In that case it will start trying to remove smaller subsets of +methods, until at some point no single method can be removed anymore, +i.e., all the methods in the set must be optimized for the test to +fail. + +## Usage + +You run it like so: + + mono crash-bisector.exe --mono ../mini/mono-sgen --opt free-regions -- generics-sharing.2.exe + +Here the optimization is `free-regions` and the test case is +`generics-sharing.2.exe`. + +Note that if the optimization you're debugging is turned on by default +you'll have to pass a `-O` option to Mono to turn it off, like so: + + mono crash-bisector.exe --mono ../mini/mono-sgen --opt intrins -- -O=-intrins generics-sharing.2.exe + +## Assumptions + +The bisector assumes that each run of your test case compiles the same +methods, and that applying your optimization to some of them doesn't +change which methods are compiled. + +The test case is assumed to succeed or fail deterministically. + +The optimization bug must also be deterministic. diff --git a/tools/crash-bisector/crash-bisector.csproj b/tools/crash-bisector/crash-bisector.csproj new file mode 100644 index 00000000000..69ac6cdc992 --- /dev/null +++ b/tools/crash-bisector/crash-bisector.csproj @@ -0,0 +1,40 @@ + + + + Debug + x86 + {5726F59B-A5CF-4DE1-9E55-4F9188A00CBA} + Exe + crashbisector + crash-bisector + v4.5 + + + true + full + false + bin\Debug + DEBUG; + prompt + 4 + true + x86 + + + full + true + bin\Release + prompt + 4 + true + x86 + + + + + + + + + + \ No newline at end of file diff --git a/tools/crash-bisector/crash-bisector.sln b/tools/crash-bisector/crash-bisector.sln new file mode 100644 index 00000000000..5f10c6935ab --- /dev/null +++ b/tools/crash-bisector/crash-bisector.sln @@ -0,0 +1,17 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 2012 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "crash-bisector", "crash-bisector.csproj", "{5726F59B-A5CF-4DE1-9E55-4F9188A00CBA}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|x86 = Debug|x86 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {5726F59B-A5CF-4DE1-9E55-4F9188A00CBA}.Debug|x86.ActiveCfg = Debug|x86 + {5726F59B-A5CF-4DE1-9E55-4F9188A00CBA}.Debug|x86.Build.0 = Debug|x86 + {5726F59B-A5CF-4DE1-9E55-4F9188A00CBA}.Release|x86.ActiveCfg = Release|x86 + {5726F59B-A5CF-4DE1-9E55-4F9188A00CBA}.Release|x86.Build.0 = Release|x86 + EndGlobalSection +EndGlobal