From ac1ce7eb4d81bf741a009236425603d99cdf1e7a Mon Sep 17 00:00:00 2001 From: Alexander Kyte Date: Wed, 31 May 2017 14:13:09 -0400 Subject: [PATCH] [runtime] Build stripper and use with mkbundle Preliminary linker support ran into dependency-finding bug with the monolinker.exe binary. Fix postponed until later. --- mcs/build/tests.make | 17 +++-- mcs/tools/Makefile | 2 +- mcs/tools/mkbundle/mkbundle.cs | 124 +++++++++++++++++++++++++++------ 3 files changed, 116 insertions(+), 27 deletions(-) diff --git a/mcs/build/tests.make b/mcs/build/tests.make index 25bd27957ee..b9f869e651d 100644 --- a/mcs/build/tests.make +++ b/mcs/build/tests.make @@ -149,17 +149,22 @@ ifdef PLATFORM_AOT_SUFFIX MKBUNDLE_TEST_BIN = $(TEST_HARNESS).static MKBUNDLE_EXE = $(topdir)/class/lib/$(PROFILE)/mkbundle.exe -BUNDLED_ASSEMBLIES := $(sort $(patsubst .//%,%,$(filter-out %.exe.static %.dll.dll %.exe.dll %bare% %plaincore% %secxml% %Facades% %ilasm%,$(filter %.dll %.exe,$(wildcard $(topdir)/class/lib/$(PROFILE)/*))))) +# Pattern based on the one in AOT_PROFILE_ASSEMBLIES +# It's easier if you read it backwards. +# What we do here is get the files in the profile directory that end in "test.dll" or are prefixed with nunit (filter) +# and then strip out everything that we expect to live outside the top level (filter-out) +TEST_ASSEMBLIES:=$(sort $(patsubst .//%,%,$(filter-out %.exe.static %.dll.dll %.exe.dll %bare% %plaincore% %secxml% %Facades% %ilasm%,$(filter %.dll,$(wildcard $(topdir)/class/lib/$(PROFILE)/tests/*))))) $(MKBUNDLE_EXE): $(topdir)/tools/mkbundle/mkbundle.cs make -C $(topdir)/tools/mkbundle -mkbundle-all-tests: $(MKBUNDLE_EXE) +mkbundle-all-tests: $(Q_AOT) $(MAKE) -C $(topdir)/class do-test - $(Q_AOT) $(MAKE) $(MKBUNDLE_TEST_BIN) # recursive make re-computes variables for BUNDLED_ASSEMBLIES + $(Q_AOT) $(MAKE) -C $(topdir)/tools/mkbundle + $(Q_AOT) $(MAKE) $(MKBUNDLE_TEST_BIN) # recursive make re-computes variables for TEST_ASSEMBLIES -$(MKBUNDLE_TEST_BIN): $(BUNDLED_ASSEMBLIES) $(TEST_HARNESS) - $(Q_AOT) MONO_PATH="$(dir $<)" PKG_CONFIG_PATH="$(topdir)/../data" $(RUNTIME) $(RUNTIME_FLAGS) $(MKBUNDLE_EXE) -L $(topdir)/class/lib/$(PROFILE) -v --deps $(TEST_HARNESS) $(BUNDLED_ASSEMBLIES) -o $(MKBUNDLE_TEST_BIN) --aot-mode $(AOT_MODE) --aot-runtime $(RUNTIME) --aot-args $(AOT_BUILD_ATTRS) --in-tree $(topdir)/.. +$(MKBUNDLE_TEST_BIN): $(TEST_ASSEMBLIES) $(TEST_HARNESS) $(MKBUNDLE_EXE) + $(Q_AOT) MONO_PATH="$(topdir)/class/lib/$(BUILD_TOOLS_PROFILE)" PKG_CONFIG_PATH="$(topdir)/../data" $(RUNTIME) $(RUNTIME_FLAGS) $(MKBUNDLE_EXE) -L $(topdir)/class/lib/$(PROFILE) -v --deps $(TEST_HARNESS) $(TEST_ASSEMBLIES) -o $(MKBUNDLE_TEST_BIN) --aot-mode $(AOT_MODE) --aot-runtime $(RUNTIME) --aot-args $(AOT_BUILD_ATTRS) --in-tree $(topdir)/.. --cil-strip $(topdir)/class/lib/$(BUILD_TOOLS_PROFILE)/mono-cil-strip.exe --managed-linker $(topdir)/class/lib/$(BUILD_TOOLS_PROFILE)/monolinker.exe endif # PLATFORM_AOT_SUFFIX @@ -170,7 +175,7 @@ TEST_HARNESS_EXEC = $(TEST_RUNTIME) $(RUNTIME_FLAGS) $(AOT_RUN_FLAGS) $(TEST_HAR endif ## FIXME: i18n problem in the 'sed' command below -run-test-lib: test-local test-local-aot-compile patch-nunitlite-appconfig $(MKBUNDLE_TEST_BIN) +run-test-lib: test-local test-local-aot-compile patch-nunitlite-appconfig ok=:; \ PATH="$(TEST_RUNTIME_WRAPPERS_PATH):$(PATH)" MONO_REGISTRY_PATH="$(HOME)/.mono/registry" MONO_TESTS_IN_PROGRESS="yes" $(TEST_HARNESS_EXEC) $(test_assemblies) $(NOSHADOW_FLAG) $(TEST_HARNESS_FLAGS) $(LOCAL_TEST_HARNESS_FLAGS) $(TEST_HARNESS_EXCLUDES) $(LABELS_ARG) -format:nunit2 -result:TestResult-$(PROFILE).xml $(FIXTURE_ARG) $(TESTNAME_ARG)|| ok=false; \ if [ ! -f "TestResult-$(PROFILE).xml" ]; then echo "The test runner didn't produce a test result XML, probably due to a crash of the runtime. Check the log for more details." > TestResult-$(PROFILE).xml; fi; \ diff --git a/mcs/tools/Makefile b/mcs/tools/Makefile index ec1f2bd2d51..5fe9c2b40bc 100644 --- a/mcs/tools/Makefile +++ b/mcs/tools/Makefile @@ -49,7 +49,7 @@ net_4_5_dirs := \ mono-symbolicate \ linker-analyzer -build_SUBDIRS = resgen gacutil security culevel cil-stringreplacer commoncryptogenerator resx2sr linker +build_SUBDIRS = resgen gacutil security culevel cil-stringreplacer commoncryptogenerator resx2sr linker cil-strip net_4_5_SUBDIRS = gacutil net_4_x_SUBDIRS = gacutil net_4_5_PARALLEL_SUBDIRS = $(net_4_5_dirs) diff --git a/mcs/tools/mkbundle/mkbundle.cs b/mcs/tools/mkbundle/mkbundle.cs index f9b44245154..27fdcdddb6d 100755 --- a/mcs/tools/mkbundle/mkbundle.cs +++ b/mcs/tools/mkbundle/mkbundle.cs @@ -65,12 +65,26 @@ class MakeBundle { static string fetch_target = null; static bool custom_mode = true; static string embedded_options = null; - static string runtime = null; + + static string runtime_bin = null; + + static string runtime { + get { + if (runtime_bin == null && IsUnix) + runtime_bin = Process.GetCurrentProcess().MainModule.FileName; + return runtime_bin; + } + + set { runtime_bin = value; } + } + static bool aot_compile = false; static string aot_args = "static"; static string aot_mode = ""; static string aot_runtime = null; static int? aot_dedup_assembly = null; + static string cil_strip_path = null; + static string managed_linker_path = null; static string sdk_path = null; static string lib_path = null; static Dictionary environment = new Dictionary(); @@ -366,6 +380,20 @@ class MakeBundle { } in_tree = args [++i]; break; + case "--managed-linker": + if (i+1 == top) { + Console.WriteLine ("Usage: --managed-linker "); + return 1; + } + managed_linker_path = args [++i]; + break; + case "--cil-strip": + if (i+1 == top) { + Console.WriteLine ("Usage: --cil-strip "); + return 1; + } + cil_strip_path = args [++i]; + break; case "--aot-runtime": if (i+1 == top) { Console.WriteLine ("Usage: --aot-runtime "); @@ -475,6 +503,8 @@ class MakeBundle { if (!QueueAssembly (files, file)) return 1; + PreprocessAssemblies (assemblies, files); + if (aot_compile) AotCompile (files); @@ -695,12 +725,8 @@ class MakeBundle { static bool GeneratePackage (List files) { if (runtime == null){ - if (IsUnix) - runtime = Process.GetCurrentProcess().MainModule.FileName; - else { - Error ("You must specify at least one runtime with --runtime or --cross"); - Environment.Exit (1); - } + Error ("You must specify at least one runtime with --runtime or --cross"); + Environment.Exit (1); } if (!File.Exists (runtime)){ Error ($"The specified runtime at {runtime} does not exist"); @@ -1468,13 +1494,9 @@ void mono_register_config_for_assembly (const char* assembly_name, cons if (aot_runtime == null) aot_runtime = runtime; - if (aot_runtime == null || aot_runtime.Length == 0){ - if (IsUnix) - aot_runtime = Process.GetCurrentProcess().MainModule.FileName; - else { - Error ("You must specify at least one aot runtime with --runtime or --cross or --aot_runtime when AOT compiling"); - Environment.Exit (1); - } + if (aot_runtime == null) { + Error ("You must specify at least one aot runtime with --runtime or --cross or --aot_runtime when AOT compiling"); + Environment.Exit (1); } var aot_mode_string = ""; @@ -1488,6 +1510,8 @@ void mono_register_config_for_assembly (const char* assembly_name, cons all_assemblies = new StringBuilder(""); } + Console.WriteLine ("Aoting files:"); + for (int i=0; i < files.Count; i++) { var fileName = files [i]; string path = LocateFile (new Uri (fileName).LocalPath); @@ -1499,11 +1523,11 @@ void mono_register_config_for_assembly (const char* assembly_name, cons if (aot_dedup_assembly != null && i != (int) aot_dedup_assembly) { all_assemblies.Append (path); all_assemblies.Append (" "); - Execute (String.Format ("{0} --aot={1},outfile={2}{3}{4} {5}", - aot_runtime, aot_args, outPath, aot_mode_string, dedup_mode_string, path)); + Execute (String.Format ("MONO_PATH={6} {0} --aot={1},outfile={2}{3}{4} {5}", + aot_runtime, aot_args, outPath, aot_mode_string, dedup_mode_string, path, Path.GetDirectoryName (path))); } else { - Execute (String.Format ("{0} --aot={1},outfile={2}{3} {4}", - aot_runtime, aot_args, outPath, aot_mode_string, path)); + Execute (String.Format ("MONO_PATH={5} {0} --aot={1},outfile={2}{3} {4}", + aot_runtime, aot_args, outPath, aot_mode_string, path, Path.GetDirectoryName (path))); } } if (aot_dedup_assembly != null) { @@ -1512,8 +1536,68 @@ void mono_register_config_for_assembly (const char* assembly_name, cons string path = LocateFile (filePath); dedup_mode_string = String.Format (",dedup-include={0}", Path.GetFileName(filePath)); string outPath = String.Format ("{0}.aot_out", path); - Execute (String.Format ("{0} --aot={1},outfile={2}{3}{4} {5} {6}", - aot_runtime, aot_args, outPath, aot_mode_string, dedup_mode_string, path, all_assemblies.ToString ())); + Execute (String.Format ("MONO_PATH={7} {0} --aot={1},outfile={2}{3}{4} {5} {6}", + aot_runtime, aot_args, outPath, aot_mode_string, dedup_mode_string, path, all_assemblies.ToString (), Path.GetDirectoryName (path))); + } + + if ((aot_mode == "full" || aot_mode == "llvmonly") && cil_strip_path != null) { + for (int i=0; i < files.Count; i++) { + var inName = new Uri (files [i]).LocalPath; + var cmd = String.Format ("{0} {1} {2}", aot_runtime, cil_strip_path, inName); + Execute (cmd); + } + } + } + + static void LinkManaged (List files, string outDir) + { + if (managed_linker_path == null) + return; + + var paths = new StringBuilder (""); + foreach (var file in files) { + paths.Append (" -a "); + paths.Append (new Uri (file).LocalPath); + } + + var cmd = String.Format ("{0} {1} -b true -out {2} {3} -c link -p copy ", runtime, managed_linker_path, outDir, paths.ToString ()); + Execute (cmd); + } + + static void PreprocessAssemblies (List chosenFiles, List files) + { + if (aot_mode == "" || (cil_strip_path == null && managed_linker_path == null)) + return; + + var temp_dir = Path.Combine(Directory.GetCurrentDirectory(), "temp_assemblies"); + var d_info = new DirectoryInfo (temp_dir); + if (d_info.Exists) { + Console.WriteLine ("Removing previous build cache at {0}", temp_dir); + d_info.Delete (true); + } + d_info.Create (); + + //if (managed_linker_path != null) { + //LinkManaged (chosenFiles, temp_dir); + + //// Replace list with new list of files + //files.Clear (); + //Console.WriteLine ("Iterating {0}", temp_dir); + //d_info = new DirectoryInfo (temp_dir); + //foreach (var file in d_info.GetFiles ()) { + //files.Append (String.Format ("file:///{0}", file)); + //Console.WriteLine (String.Format ("file:///{0}", file)); + //} + //return; + //} + + // Fix file references + for (int i=0; i < files.Count; i++) { + var inName = new Uri (files [i]).LocalPath; + var outName = Path.Combine (temp_dir, Path.GetFileName (inName)); + //if (managed_linker_path == null) + File.Copy (inName, outName); + files [i] = files[i].Replace (Path.GetDirectoryName (inName), temp_dir); } } -- 2.25.1