From 66a493cefe3eae0b8a136f6a046f01bb76be33b0 Mon Sep 17 00:00:00 2001 From: Miguel de Icaza Date: Mon, 11 Apr 2016 10:42:51 -0400 Subject: [PATCH] Compiler-less mkbundle support in Mono. This is the support that makes it so that mkbundle can create self-contained executables without requiring a native toolchain to be present. Also is the foundation to allow cross-compilation of self-contained executables. This works by concatenating the runtime with the various dependencies and configuration files, attaching a directory at the end of the file and a signature to flag this as a runtime that has embedded resources in the executable file itself. This works by modifying the runtime startup to check for a special signature at the end of the main executable, if it is not present, execution resumes as usual. Otherwise a directory is read from the end of the file which contains both the kind of file (assembly, configuration file, and so on), as well as pointer to the contents in the file. Then the regular Mono APIs to load resources are used to load these.This is the support that makes it so that mkbundle can create self-contained executables without requiring a native toolchain to be present. Also is the foundation to allow cross-compilation of self-contained executables. This works by concatenating the runtime with the various dependencies and configuration files, attaching a directory at the end of the file and a signature to flag this as a runtime that has embedded resources in the executable file itself. This works by modifying the runtime startup to check for a special signature at the end of the main executable, if it is not present, execution resumes as usual. Otherwise a directory is read from the end of the file which contains both the kind of file (assembly, configuration file, and so on), as well as pointer to the contents in the file. Then the regular Mono APIs to load resources are used to load these. --- mcs/tools/mkbundle/Makefile | 2 +- mcs/tools/mkbundle/mkbundle.cs | 204 +++++++++++++++++++++++++++++--- mono/mini/driver.c | 209 ++++++++++++++++++++++----------- mono/mini/main.c | 134 ++++++++++++++++++++- mono/mini/mini.h | 2 + msvc/mono.def | 1 + msvc/monosgen.def | 1 + 7 files changed, 464 insertions(+), 89 deletions(-) diff --git a/mcs/tools/mkbundle/Makefile b/mcs/tools/mkbundle/Makefile index 867c1d3e2df..05faf5e8c68 100644 --- a/mcs/tools/mkbundle/Makefile +++ b/mcs/tools/mkbundle/Makefile @@ -11,7 +11,7 @@ RESOURCE_FILES = $(OTHER_RES) LOCAL_MCS_FLAGS= $(OTHER_RES:%=-resource:%) LOCAL_MCS_FLAGS += -d:STATIC,NO_SYMBOL_WRITER,NO_AUTHENTICODE -LIB_REFS = System.Xml System +LIB_REFS = System.Xml System System.Core EXTRA_DISTFILES = $(RESOURCE_FILES) diff --git a/mcs/tools/mkbundle/mkbundle.cs b/mcs/tools/mkbundle/mkbundle.cs index 57bae7e4c1d..6395e90ff41 100755 --- a/mcs/tools/mkbundle/mkbundle.cs +++ b/mcs/tools/mkbundle/mkbundle.cs @@ -7,6 +7,11 @@ // Miguel de Icaza // // (C) Novell, Inc 2004 +// (C) 2016 Xamarin Inc +// +// Missing features: +// * Implement --cross, --local-targets, --list-targets, --no-auto-fetch +// * concatenate target with package to form native binary // using System; using System.Diagnostics; @@ -17,8 +22,8 @@ using System.IO.Compression; using System.Runtime.InteropServices; using System.Text; using IKVM.Reflection; - - +using System.Linq; +using System.Diagnostics; using System.Threading.Tasks; class MakeBundle { @@ -41,6 +46,9 @@ class MakeBundle { static bool skip_scan; static string ctor_func; static bool quiet; + static bool custom_mode = true; + static string embedded_options = null; + static string runtime = null; static int Main (string [] args) { @@ -56,6 +64,15 @@ class MakeBundle { Help (); return 1; + case "--simple": + custom_mode = false; + autodeps = true; + break; + + case "--custom": + custom_mode = true; + break; + case "-c": compile_only = true; break; @@ -68,6 +85,20 @@ class MakeBundle { output = args [++i]; break; + case "--options": + if (i+1 == top){ + Help (); + return 1; + } + embedded_options = args [++i]; + break; + case "--runtime": + if (i+1 == top){ + Help (); + return 1; + } + runtime = args [++i]; + break; case "-oo": if (i+1 == top){ Help (); @@ -95,6 +126,7 @@ class MakeBundle { case "--keeptemp": keeptemp = true; break; + case "--static": static_link = true; break; @@ -197,9 +229,11 @@ class MakeBundle { foreach (string file in assemblies) if (!QueueAssembly (files, file)) return 1; - - GenerateBundles (files); - //GenerateJitWrapper (); + + if (custom_mode) + GenerateBundles (files); + else + GeneratePackage (files); return 0; } @@ -264,6 +298,138 @@ class MakeBundle { ts.WriteLine (); } + + class PackageMaker { + Dictionary> locations = new Dictionary> (); + const int align = 4096; + Stream package; + + public PackageMaker (string output) + { + package = File.Create (output, 128*1024); + if (IsUnix){ + File.SetAttributes (output, unchecked ((FileAttributes) 0x80000000)); + } + } + + public int AddFile (string fname) + { + using (Stream fileStream = File.OpenRead (fname)){ + var ret = fileStream.Length; + + Console.WriteLine ("At {0:x} with input {1}", package.Position, fileStream.Length); + fileStream.CopyTo (package); + package.Position = package.Position + (align - (package.Position % align)); + + return (int) ret; + } + } + + public void Add (string entry, string fname) + { + var p = package.Position; + var size = AddFile (fname); + + locations [entry] = Tuple.Create(p, size); + } + + public void AddString (string entry, string text) + { + var bytes = Encoding.UTF8.GetBytes (text); + locations [entry] = Tuple.Create (package.Position, bytes.Length); + package.Write (bytes, 0, bytes.Length); + package.Position = package.Position + (align - (package.Position % align)); + } + + public void Dump () + { + foreach (var floc in locations.Keys){ + Console.WriteLine ($"{floc} at {locations[floc]:x}"); + } + } + + public void WriteIndex () + { + var indexStart = package.Position; + var binary = new BinaryWriter (package); + + binary.Write (locations.Count); + foreach (var entry in from entry in locations orderby entry.Value.Item1 ascending select entry){ + var bytes = Encoding.UTF8.GetBytes (entry.Key); + binary.Write (bytes.Length+1); + binary.Write (bytes); + binary.Write ((byte) 0); + binary.Write (entry.Value.Item1); + binary.Write (entry.Value.Item2); + } + binary.Write (indexStart); + binary.Write (Encoding.UTF8.GetBytes ("xmonkeysloveplay")); + binary.Flush (); + } + + public void Close () + { + WriteIndex (); + package.Close (); + package = null; + } + } + + static bool MaybeAddFile (PackageMaker maker, string code, string file) + { + if (file == null) + return true; + + if (!File.Exists (file)){ + Console.Error.WriteLine ("The file {0} does not exist", file); + return false; + } + maker.Add (code, file); + return true; + } + + static bool GeneratePackage (List files) + { + if (runtime == null){ + if (IsUnix) + runtime = Process.GetCurrentProcess().MainModule.FileName; + else { + Console.Error.WriteLine ("You must specify at least one runtime with --runtime or --cross"); + Environment.Exit (1); + } + } + if (!File.Exists (runtime)){ + Console.Error.WriteLine ($"The specified runtime at {runtime} does not exist"); + Environment.Exit (1); + } + + if (ctor_func != null){ + Console.Error.WriteLine ("--static-ctor not supported with package bundling, you must use native compilation for this"); + return false; + } + + var maker = new PackageMaker (output); + maker.AddFile (runtime); + + foreach (var url in files){ + string fname = LocateFile (new Uri (url).LocalPath); + string aname = Path.GetFileName (fname); + + maker.Add ("assembly:" + aname, fname); + if (File.Exists (fname + ".config")) + maker.Add ("config:" + aname, fname + ".config"); + } + if (!MaybeAddFile (maker, "systemconfig:", config_file) || !MaybeAddFile (maker, "machineconfig:", machine_config_file)) + return false; + + if (config_dir != null) + maker.Add ("config_dir:", config_dir); + if (embedded_options != null) + maker.AddString ("options:", embedded_options); + maker.Dump (); + maker.Close (); + return true; + } static void GenerateBundles (List files) { @@ -756,25 +922,35 @@ void mono_register_config_for_assembly (const char* assembly_name, cons { Console.WriteLine ("Usage is: mkbundle [options] assembly1 [assembly2...]\n\n" + "Options:\n" + - " -c Produce stub only, do not compile\n" + + " --config F Bundle system config file `F'\n" + + " --config-dir D Set MONO_CFG_DIR to `D'\n" + + " --deps Turns on automatic dependency embedding (default on simple)\n" + + " -L path Adds `path' to the search path for assemblies\n" + + " --machine-config F Use the given file as the machine.config for the application.\n" + " -o out Specifies output filename\n" + + " --nodeps Turns off automatic dependency embedding (default on custom)\n" + + " --skip-scan Skip scanning assemblies that could not be loaded (but still embed them).\n" + + "\n" + + "--simple Simple mode does not require a C toolchain and can cross compile\n" + + " --cross TARGET Generates a binary for the given TARGET\n"+ + " --local-targets Lists locally available targets\n" + + " --list-targets [SERVER] Lists available targets on the remote server\n" + + " --no-auto-fetch Prevents the tool from auto-fetching a TARGET\n" + + " --options OPTIONS Embed the specified Mono command line options on target\n" + + " --runtime RUNTIME Manually specifies the Mono runtime to use\n" + + "\n" + + "--custom Builds a custom launcher, options for --custom\n" + + " -c Produce stub only, do not compile\n" + " -oo obj Specifies output filename for helper object file\n" + - " -L path Adds `path' to the search path for assemblies\n" + - " --nodeps Turns off automatic dependency embedding (default)\n" + - " --deps Turns on automatic dependency embedding\n" + " --dos2unix[=true|false]\n" + " When no value provided, or when `true` specified\n" + " `dos2unix` will be invoked to convert paths on Windows.\n" + " When `--dos2unix=false` used, dos2unix is NEVER used.\n" + " --keeptemp Keeps the temporary files\n" + - " --config F Bundle system config file `F'\n" + - " --config-dir D Set MONO_CFG_DIR to `D'\n" + - " --machine-config F Use the given file as the machine.config for the application.\n" + " --static Statically link to mono libs\n" + " --nomain Don't include a main() function, for libraries\n" + - " --custom-main C Link the specified compilation unit (.c or .obj) with entry point/init code\n" + + " --custom-main C Link the specified compilation unit (.c or .obj) with entry point/init code\n" + " -z Compress the assemblies before embedding.\n" + - " --skip-scan Skip scanning assemblies that could not be loaded (but still embed them).\n" + " --static-ctor ctor Add a constructor call to the supplied function.\n" + " You need zlib development headers and libraries.\n"); } diff --git a/mono/mini/driver.c b/mono/mini/driver.c index f1c27d7dc40..e39a9410234 100644 --- a/mono/mini/driver.c +++ b/mono/mini/driver.c @@ -2377,83 +2377,154 @@ mono_set_crash_chaining (gboolean chain_crashes) mono_do_crash_chaining = chain_crashes; } -void -mono_parse_env_options (int *ref_argc, char **ref_argv []) +/** + * mono_parse_options_from: + * @options: string containing strings + * @ref_argc: pointer to the argc variable that might be updated + * @ref_argv: pointer to the argv string vector variable that might be updated + * + * This function parses the contents of the `MONO_ENV_OPTIONS` + * environment variable as if they were parsed by a command shell + * splitting the contents by spaces into different elements of the + * @argv vector. This method supports quoting with both the " and ' + * characters. Inside quoting, spaces and tabs are significant, + * otherwise, they are considered argument separators. + * + * The \ character can be used to escape the next character which will + * be added to the current element verbatim. Typically this is used + * inside quotes. If the quotes are not balanced, this method + * + * If the environment variable is empty, no changes are made + * to the values pointed by @ref_argc and @ref_argv. + * + * Otherwise the @ref_argv is modified to point to a new array that contains + * all the previous elements contained in the vector, plus the values parsed. + * The @argc is updated to match the new number of parameters. + * + * Returns: The value NULL is returned on success, otherwise a g_strdup allocated + * string is returned (this is an alias to malloc under normal circumstances) that + * contains the error message that happened during parsing. + */ +char * +mono_parse_options_from (const char *options, int *ref_argc, char **ref_argv []) { int argc = *ref_argc; char **argv = *ref_argv; - - const char *env_options = g_getenv ("MONO_ENV_OPTIONS"); - if (env_options != NULL){ - GPtrArray *array = g_ptr_array_new (); - GString *buffer = g_string_new (""); - const char *p; - unsigned i; - gboolean in_quotes = FALSE; - char quote_char = '\0'; - - for (p = env_options; *p; p++){ - switch (*p){ - case ' ': case '\t': - if (!in_quotes) { - if (buffer->len != 0){ - g_ptr_array_add (array, g_strdup (buffer->str)); - g_string_truncate (buffer, 0); - } - } else { - g_string_append_c (buffer, *p); - } - break; - case '\\': - if (p [1]){ - g_string_append_c (buffer, p [1]); - p++; - } - break; - case '\'': - case '"': - if (in_quotes) { - if (quote_char == *p) - in_quotes = FALSE; - else - g_string_append_c (buffer, *p); - } else { - in_quotes = TRUE; - quote_char = *p; + GPtrArray *array = g_ptr_array_new (); + GString *buffer = g_string_new (""); + const char *p; + unsigned i; + gboolean in_quotes = FALSE; + char quote_char = '\0'; + + if (options == NULL) + return NULL; + + for (p = options; *p; p++){ + switch (*p){ + case ' ': case '\t': + if (!in_quotes) { + if (buffer->len != 0){ + g_ptr_array_add (array, g_strdup (buffer->str)); + g_string_truncate (buffer, 0); } - break; - default: + } else { g_string_append_c (buffer, *p); - break; } + break; + case '\\': + if (p [1]){ + g_string_append_c (buffer, p [1]); + p++; + } + break; + case '\'': + case '"': + if (in_quotes) { + if (quote_char == *p) + in_quotes = FALSE; + else + g_string_append_c (buffer, *p); + } else { + in_quotes = TRUE; + quote_char = *p; + } + break; + default: + g_string_append_c (buffer, *p); + break; } - if (in_quotes) { - fprintf (stderr, "Unmatched quotes in value of MONO_ENV_OPTIONS: [%s]\n", env_options); - exit (1); - } - - if (buffer->len != 0) - g_ptr_array_add (array, g_strdup (buffer->str)); - g_string_free (buffer, TRUE); + } + if (in_quotes) + return g_strdup_printf ("Unmatched quotes in value: [%s]\n", options); + + if (buffer->len != 0) + g_ptr_array_add (array, g_strdup (buffer->str)); + g_string_free (buffer, TRUE); - if (array->len > 0){ - int new_argc = array->len + argc; - char **new_argv = g_new (char *, new_argc + 1); - int j; + if (array->len > 0){ + int new_argc = array->len + argc; + char **new_argv = g_new (char *, new_argc + 1); + int j; - new_argv [0] = argv [0]; - - /* First the environment variable settings, to allow the command line options to override */ - for (i = 0; i < array->len; i++) - new_argv [i+1] = (char *)g_ptr_array_index (array, i); - i++; - for (j = 1; j < argc; j++) - new_argv [i++] = argv [j]; - new_argv [i] = NULL; - - *ref_argc = new_argc; - *ref_argv = new_argv; - } - g_ptr_array_free (array, TRUE); + new_argv [0] = argv [0]; + + /* First the environment variable settings, to allow the command line options to override */ + for (i = 0; i < array->len; i++) + new_argv [i+1] = (char *)g_ptr_array_index (array, i); + i++; + for (j = 1; j < argc; j++) + new_argv [i++] = argv [j]; + new_argv [i] = NULL; + + *ref_argc = new_argc; + *ref_argv = new_argv; } + g_ptr_array_free (array, TRUE); + return NULL; +} + +/** + * mono_parse_env_options: + * @ref_argc: pointer to the argc variable that might be updated + * @ref_argv: pointer to the argv string vector variable that might be updated + * + * This function parses the contents of the `MONO_ENV_OPTIONS` + * environment variable as if they were parsed by a command shell + * splitting the contents by spaces into different elements of the + * @argv vector. This method supports quoting with both the " and ' + * characters. Inside quoting, spaces and tabs are significant, + * otherwise, they are considered argument separators. + * + * The \ character can be used to escape the next character which will + * be added to the current element verbatim. Typically this is used + * inside quotes. If the quotes are not balanced, this method + * + * If the environment variable is empty, no changes are made + * to the values pointed by @ref_argc and @ref_argv. + * + * Otherwise the @ref_argv is modified to point to a new array that contains + * all the previous elements contained in the vector, plus the values parsed. + * The @argc is updated to match the new number of parameters. + * + * If there is an error parsing, this method will terminate the process by + * calling exit(1). + * + * An alternative to this method that allows an arbitrary string to be parsed + * and does not exit on error is the `api:mono_parse_options_from`. + */ +void +mono_parse_env_options (int *ref_argc, char **ref_argv []) +{ + char *ret; + + const char *env_options = g_getenv ("MONO_ENV_OPTIONS"); + if (env_options == NULL) + return; + ret = mono_parse_options_from (env_options, ref_argc, ref_argv); + if (ret == NULL) + return; + fprintf (stderr, "%s", ret); + exit (1); } + diff --git a/mono/mini/main.c b/mono/mini/main.c index 4ef344ecdd9..de8ab61e3d5 100644 --- a/mono/mini/main.c +++ b/mono/mini/main.c @@ -1,10 +1,18 @@ #include +#include +#include +#include #include "mini.h" -#ifndef HOST_WIN32 -#ifndef BUILDVER_INCLUDED -#include "buildver-boehm.h" +#ifdef HAVE_UNISTD_H +# include #endif +#ifdef HOST_WIN32 +# include +#else +# ifndef BUILDVER_INCLUDED +# include "buildver-boehm.h" +# endif #endif /* @@ -20,6 +28,114 @@ mono_main_with_options (int argc, char *argv []) return mono_main (argc, argv); } +#define STREAM_INT(x) (*(uint32_t*)x) +#define STREAM_LONG(x) (*(uint64_t*)x) + +static gboolean +probe_embedded (const char *program, int *ref_argc, char **ref_argv []) +{ + MonoBundledAssembly last = { NULL, 0, 0 }; + char sigbuffer [16+sizeof (uint64_t)]; + gboolean status = FALSE; + uint64_t directory_location; + off_t sigstart, baseline = 0; + uint64_t directory_size; + char *directory, *p; + int items, i; + unsigned char *mapaddress = NULL; + void *maphandle = NULL; + GArray *assemblies; + char *entry_point = NULL; + char **new_argv; + int j; + + int fd = open (program, O_RDONLY); + if (fd == -1) + return FALSE; + if ((sigstart = lseek (fd, -(16+sizeof(uint64_t)), SEEK_END)) == -1) + goto doclose; + if (read (fd, sigbuffer, sizeof (sigbuffer)) == -1) + goto doclose; + if (memcmp (sigbuffer+sizeof(uint64_t), "xmonkeysloveplay", 16) != 0) + goto doclose; + directory_location = GUINT64_FROM_LE ((*(uint64_t *) &sigbuffer [0])); + if (lseek (fd, directory_location, SEEK_SET) == -1) + goto doclose; + directory_size = sigstart-directory_location; + directory = g_malloc (directory_size); + if (directory == NULL) + goto doclose; + if (read (fd, directory, directory_size) == -1) + goto dofree; + + items = STREAM_INT (directory); + p = directory+4; + + assemblies = g_array_new (0, 0, sizeof (MonoBundledAssembly*)); + for (i = 0; i < items; i++){ + char *kind; + int strsize = STREAM_INT (p); + uint64_t offset, item_size; + kind = p+4; + p += 4 + strsize; + offset = STREAM_LONG(p); + p += 8; + item_size = STREAM_INT (p); + p += 4; + + if (mapaddress == NULL){ + mapaddress = mono_file_map (directory_location-offset, MONO_MMAP_READ | MONO_MMAP_PRIVATE, fd, offset, &maphandle); + if (mapaddress == NULL){ + perror ("Error mapping file"); + exit (1); + } + baseline = offset; + } + if (strncmp (kind, "assembly:", strlen ("assembly:")) == 0){ + char *aname = kind + strlen ("assembly:"); + MonoBundledAssembly mba = { aname, mapaddress + offset - baseline, item_size }, *ptr; + ptr = g_new (MonoBundledAssembly, 1); + memcpy (ptr, &mba, sizeof (MonoBundledAssembly)); + g_array_append_val (assemblies, ptr); + if (entry_point == NULL) + entry_point = aname; + } else if (strncmp (kind, "config:", strlen ("config:")) == 0){ + printf ("c-Found: %s %llx\n", kind, offset); + char *config = kind + strlen ("config:"); + char *aname = g_strdup (config); + aname [strlen(aname)-strlen(".config")] = 0; + mono_register_config_for_assembly (aname, config); + } else if (strncmp (kind, "system_config:", strlen ("system_config:")) == 0){ + printf ("TODO s-Found: %s %llx\n", kind, offset); + } else if (strncmp (kind, "options:", strlen ("options:")) == 0){ + mono_parse_options_from (kind + strlen("options:"), ref_argc, ref_argv); + } else if (strncmp (kind, "config_dir:", strlen ("config_dir:")) == 0){ + printf ("TODO Found: %s %llx\n", kind, offset); + } else { + fprintf (stderr, "Unknown stream on embedded package: %s\n", kind); + exit (1); + } + } + g_array_append_val (assemblies, last); + + mono_register_bundled_assemblies ((const MonoBundledAssembly **) assemblies->data); + new_argv = g_new (char *, (*ref_argc)+1); + for (j = 0; j < *ref_argc; j++) + new_argv [j] = (*ref_argv)[j]; + new_argv [j] = entry_point; + *ref_argv = new_argv; + (*ref_argc)++; + + return TRUE; + +dofree: + g_free (directory); +doclose: + if (!status) + close (fd); + return status; +} + #ifdef HOST_WIN32 #include @@ -27,11 +143,13 @@ mono_main_with_options (int argc, char *argv []) int main (void) { + TCHAR szFileName[MAX_PATH]; int argc; gunichar2** argvw; gchar** argv; int i; - + DWORD count; + argvw = CommandLineToArgvW (GetCommandLine (), &argc); argv = g_new0 (gchar*, argc + 1); for (i = 0; i < argc; i++) @@ -40,6 +158,11 @@ main (void) LocalFree (argvw); + if ((count = GetModuleFileName (NULL, szFileName, MAX_PATH)) != 0){ + char *entry = g_utf16_to_utf8 (szFileName, count, NULL, NULL, NULL); + probe_embedded (entry, &argc, &argv); + } + return mono_main_with_options (argc, argv); } @@ -49,7 +172,8 @@ int main (int argc, char* argv[]) { mono_build_date = build_date; - + + probe_embedded (argv [0], &argc, &argv); return mono_main_with_options (argc, argv); } diff --git a/mono/mini/mini.h b/mono/mini/mini.h index e63fd7aa222..31591493070 100644 --- a/mono/mini/mini.h +++ b/mono/mini/mini.h @@ -2286,6 +2286,8 @@ mono_bb_last_inst (MonoBasicBlock *bb, int filter) MONO_API int mono_main (int argc, char* argv[]); MONO_API void mono_set_defaults (int verbose_level, guint32 opts); MONO_API void mono_parse_env_options (int *ref_argc, char **ref_argv []); +MONO_API char *mono_parse_options_from (const char *options, int *ref_argc, char **ref_argv []); + MonoDomain* mini_init (const char *filename, const char *runtime_version); void mini_cleanup (MonoDomain *domain); MONO_API MonoDebugOptions *mini_get_debug_options (void); diff --git a/msvc/mono.def b/msvc/mono.def index 8392a8c693d..487513f2c36 100644 --- a/msvc/mono.def +++ b/msvc/mono.def @@ -659,6 +659,7 @@ mono_pagesize mono_param_get_objects mono_parse_default_optimizations mono_parse_env_options +mono_parse_options_from mono_path_canonicalize mono_path_resolve_symlinks mono_pe_file_open diff --git a/msvc/monosgen.def b/msvc/monosgen.def index eee61f02426..d686ca0d909 100644 --- a/msvc/monosgen.def +++ b/msvc/monosgen.def @@ -661,6 +661,7 @@ mono_pagesize mono_param_get_objects mono_parse_default_optimizations mono_parse_env_options +mono_parse_options_from mono_path_canonicalize mono_path_resolve_symlinks mono_pe_file_open -- 2.25.1