Merge pull request #5560 from kumpera/wasm-work-p3
[mono.git] / mcs / tools / mkbundle / mkbundle.cs
index 447540860f50024ec14df5685b4cf49d5fc3218d..d4ef1cbb5520e4f8c493fdcce14d61e2b21730f1 100755 (executable)
@@ -3,6 +3,16 @@
 //
 // Based on the `make-bundle' Perl script written by Paolo Molaro (lupus@debian.org)
 //
+// TODO:
+//   [x] Rename the paths for the zip file that is downloaded
+//   [x] Update documentation with new flag
+//   [x] Load internationalized assemblies
+//   [x] Dependencies - if System.dll -> include Mono.Security.* (not needed, automatic)
+//   [x] --list-targets should download from a different url
+//   [x] --fetch-target should unpack zip file
+//   [x] Update --cross to use not a runtime, but an SDK
+//   [x] Update --local-targets to show the downloaded SDKs
+//
 // Author:
 //   Miguel de Icaza
 //
@@ -39,6 +49,7 @@ class MakeBundle {
        static string machine_config_file = null;
        static string config_dir = null;
        static string style = "linux";
+       static bool bundled_header = false;
        static string os_message = "";
        static bool compress;
        static bool nomain;
@@ -52,6 +63,8 @@ class MakeBundle {
        static bool custom_mode = true;
        static string embedded_options = null;
        static string runtime = null;
+       static string sdk_path = null;
+       static string lib_path = null;
        static Dictionary<string,string> environment = new Dictionary<string,string>();
        static string [] i18n = new string [] {
                "West",
@@ -74,7 +87,7 @@ class MakeBundle {
                link_paths.Add (".");
 
                DetectOS ();
-               
+
                for (int i = 0; i < top; i++){
                        switch (args [i]){
                        case "--help": case "-h": case "-?":
@@ -121,6 +134,8 @@ class MakeBundle {
                                        Help (); 
                                        return 1;
                                }
+                               if (sdk_path != null || runtime != null)
+                                       Error ("You can only specify one of --runtime, --sdk or --cross");
                                custom_mode = false;
                                autodeps = true;
                                cross_target = args [++i];
@@ -146,10 +161,8 @@ class MakeBundle {
                                        alias = lspec.Substring (0, p);
                                        path = lspec.Substring (p+1);
                                }
-                               if (!File.Exists (path)){
-                                       Console.Error.WriteLine ($"The specified library file {path} does not exist");
-                                       return 1;
-                               }
+                               if (!File.Exists (path))
+                                       Error ($"The specified library file {path} does not exist");
                                libraries [alias] = path;
                                break;
 
@@ -162,10 +175,10 @@ class MakeBundle {
                                break;
 
                        case "--list-targets":
+                               CommandLocalTargets ();
                                var wc = new WebClient ();
-                               var s = wc.DownloadString (new Uri (target_server + "target-list.txt"));
-                               Console.WriteLine ("Cross-compilation targets available:\n" + s);
-                               
+                               var s = wc.DownloadString (new Uri (target_server + "target-sdks.txt"));
+                               Console.WriteLine ("Targets available for download with --fetch-target:\n" + s);
                                return 0;
                                
                        case "--target-server":
@@ -191,11 +204,24 @@ class MakeBundle {
                                }
                                embedded_options = args [++i];
                                break;
+                       case "--sdk":
+                               if (i + 1 == top) {
+                                       Help ();
+                                       return 1;
+                               }
+                               custom_mode = false;
+                               autodeps = true;
+                               sdk_path = args [++i];
+                               if (cross_target != null || runtime != null)
+                                       Error ("You can only specify one of --runtime, --sdk or --cross");
+                               break;
                        case "--runtime":
                                if (i+1 == top){
                                        Help (); 
                                        return 1;
                                }
+                               if (sdk_path != null || cross_target != null)
+                                       Error ("You can only specify one of --runtime, --sdk or --cross");
                                custom_mode = false;
                                autodeps = true;
                                runtime = args [++i];
@@ -283,7 +309,7 @@ class MakeBundle {
                                case "linux":
                                        break;
                                default:
-                                       Console.Error.WriteLine ("Invalid style '{0}' - only 'windows', 'mac' and 'linux' are supported for --style argument", style);
+                                       Error ("Invalid style '{0}' - only 'windows', 'mac' and 'linux' are supported for --style argument", style);
                                        return 1;
                                }
                                        
@@ -322,27 +348,48 @@ class MakeBundle {
                                else
                                        environment.Add (env.Substring (0, p), env.Substring (p+1));
                                break;
+                       case "--bundled-header":
+                               bundled_header = true;
+                               break;
                        default:
                                sources.Add (args [i]);
                                break;
                        }
 
                }
+               // Modern bundling starts here
+               if (!custom_mode){
+                       if (runtime != null){
+                               // Nothing to do here, the user has chosen to manually specify --runtime nad libraries
+                       } else if (sdk_path != null) {
+                               VerifySdk (sdk_path);
+                       } else if (cross_target == "default" || cross_target == null){
+                               sdk_path = Path.GetFullPath (Path.Combine (Process.GetCurrentProcess().MainModule.FileName, "..", ".."));
+                               VerifySdk (sdk_path);
+                       } else {
+                               sdk_path = Path.Combine (targets_dir, cross_target);
+                               Console.WriteLine ("From: " + sdk_path);
+                               VerifySdk (sdk_path);
+                       }
+               }
 
                if (fetch_target != null){
-                       var truntime = Path.Combine (targets_dir, fetch_target, "mono");
-                       Directory.CreateDirectory (Path.GetDirectoryName (truntime));
+                       var directory = Path.Combine (targets_dir, fetch_target);
+                       var zip_download = Path.Combine (directory, "sdk.zip");
+                       Directory.CreateDirectory (directory);
                        var wc = new WebClient ();
                        var uri = new Uri ($"{target_server}{fetch_target}");
                        try {
                                if (!quiet){
-                                       Console.WriteLine ($"Downloading runtime {uri} to {truntime}");
+                                       Console.WriteLine ($"Downloading runtime {uri} to {zip_download}");
                                }
                                
-                               wc.DownloadFile (uri, truntime);
+                               wc.DownloadFile (uri, zip_download);
+                               ZipFile.ExtractToDirectory(zip_download, directory);
+                               File.Delete (zip_download);
                        } catch {
                                Console.Error.WriteLine ($"Failure to download the specified runtime from {uri}");
-                               File.Delete (truntime);
+                               File.Delete (zip_download);
                                return 1;
                        }
                        return 0;
@@ -359,48 +406,41 @@ class MakeBundle {
                }
 
                List<string> assemblies = LoadAssemblies (sources);
+               LoadLocalizedAssemblies (assemblies);
                List<string> files = new List<string> ();
                foreach (string file in assemblies)
                        if (!QueueAssembly (files, file))
                                return 1;
                if (custom_mode)
                        GenerateBundles (files);
-               else {
-                       if (cross_target == "default")
-                               runtime = null;
-                       else {
-                               if (runtime == null){
-                                       if (cross_target == null){
-                                               Console.Error.WriteLine ("you should specify either a --runtime or a --cross compilation target");
-                                               Environment.Exit (1);
-                                       }
-                                       runtime = Path.Combine (targets_dir, cross_target, "mono");
-                                       if (!File.Exists (runtime)){
-                                               Console.Error.WriteLine ($"The runtime for the {cross_target} does not exist, use --fetch-target {cross_target} to download first");
-                                               return 1;
-                                       }
-                               } else {
-                                       if (!File.Exists (runtime)){
-                                               Console.Error.WriteLine ($"The Mono runtime specified with --runtime does not exist");
-                                               return 1;
-                                       }
-                               }
-                               
-                       }
+               else 
                        GeneratePackage (files);
-               }
+
                Console.WriteLine ("Generated {0}", output);
 
                return 0;
        }
 
+       static void VerifySdk (string path)
+       {
+               if (!Directory.Exists (path))
+                       Error ($"The specified SDK path does not exist: {path}");
+               runtime = Path.Combine (sdk_path, "bin", "mono");
+               if (!File.Exists (runtime))
+                       Error ($"The SDK location does not contain a {path}/bin/mono runtime");
+               lib_path = Path.Combine (path, "lib", "mono", "4.5");
+               if (!Directory.Exists (lib_path))
+                       Error ($"The SDK location does not contain a {path}/lib/mono/4.5 directory");
+               link_paths.Add (lib_path);
+       }
+
        static string targets_dir = Path.Combine (Environment.GetFolderPath (Environment.SpecialFolder.Personal), ".mono", "targets");
        
        static void CommandLocalTargets ()
        {
                string [] targets;
 
-               Console.WriteLine ("Available targets:");
+               Console.WriteLine ("Available targets locally:");
                Console.WriteLine ("\tdefault\t- Current System Mono");
                try {
                        targets = Directory.GetDirectories (targets_dir);
@@ -408,7 +448,7 @@ class MakeBundle {
                        return;
                }
                foreach (var target in targets){
-                       var p = Path.Combine (target, "mono");
+                       var p = Path.Combine (target, "bin", "mono");
                        if (File.Exists (p))
                                Console.WriteLine ("\t{0}", Path.GetFileName (target));
                }
@@ -580,10 +620,12 @@ class MakeBundle {
                        return true;
                
                if (!File.Exists (file)){
-                       Console.Error.WriteLine ("The file {0} does not exist", file);
+                       Error ("The file {0} does not exist", file);
                        return false;
                }
                maker.Add (code, file);
+               // add a space after code (="systemconfig:" or "machineconfig:")
+               Console.WriteLine (code + " " + file);
                return true;
        }
        
@@ -593,17 +635,17 @@ class MakeBundle {
                        if (IsUnix)
                                runtime = Process.GetCurrentProcess().MainModule.FileName;
                        else {
-                               Console.Error.WriteLine ("You must specify at least one runtime with --runtime or --cross");
+                               Error ("You must specify at least one runtime with --runtime or --cross");
                                Environment.Exit (1);
                        }
                }
                if (!File.Exists (runtime)){
-                       Console.Error.WriteLine ($"The specified runtime at {runtime} does not exist");
+                       Error ($"The specified runtime at {runtime} does not exist");
                        Environment.Exit (1);
                }
                
                if (ctor_func != null){
-                       Console.Error.WriteLine ("--static-ctor not supported with package bundling, you must use native compilation for this");
+                       Error ("--static-ctor not supported with package bundling, you must use native compilation for this");
                        return false;
                }
                
@@ -619,15 +661,17 @@ class MakeBundle {
                        Console.WriteLine ("     Assembly: " + fname);
                        if (File.Exists (fname + ".config")){
                                maker.Add ("config:" + aname, fname + ".config");
-                               Console.WriteLine ("       Config: " + runtime);
+                               Console.WriteLine ("       Config: " + fname + ".config");
                        }
                }
                
                if (!MaybeAddFile (maker, "systemconfig:", config_file) || !MaybeAddFile (maker, "machineconfig:", machine_config_file))
                        return false;
 
-               if (config_dir != null)
+               if (config_dir != null){
                        maker.Add ("config_dir:", config_dir);
+                       Console.WriteLine ("   Config_dir: " + config_dir );
+               }
                if (embedded_options != null)
                        maker.AddString ("options:", embedded_options);
                if (environment.Count > 0){
@@ -664,10 +708,10 @@ class MakeBundle {
                        using (StreamWriter tc = new StreamWriter (File.Create (temp_c))) {
                        string prog = null;
 
-#if XAMARIN_ANDROID
-                       tc.WriteLine ("/* This source code was produced by mkbundle, do not edit */");
-                       tc.WriteLine ("\n#ifndef NULL\n#define NULL (void *)0\n#endif");
-                       tc.WriteLine (@"
+                       if (bundled_header) {
+                               tc.WriteLine ("/* This source code was produced by mkbundle, do not edit */");
+                               tc.WriteLine ("\n#ifndef NULL\n#define NULL (void *)0\n#endif");
+                               tc.WriteLine (@"
 typedef struct {
        const char *name;
        const unsigned char *data;
@@ -676,10 +720,10 @@ typedef struct {
 void          mono_register_bundled_assemblies (const MonoBundledAssembly **assemblies);
 void          mono_register_config_for_assembly (const char* assembly_name, const char* config_xml);
 ");
-#else
-                       tc.WriteLine ("#include <mono/metadata/mono-config.h>");
-                       tc.WriteLine ("#include <mono/metadata/assembly.h>\n");
-#endif
+                       } else {
+                               tc.WriteLine ("#include <mono/metadata/mono-config.h>");
+                               tc.WriteLine ("#include <mono/metadata/assembly.h>\n");
+                       }
 
                        if (compress) {
                                tc.WriteLine ("typedef struct _compressed_data {");
@@ -786,7 +830,7 @@ void          mono_register_config_for_assembly (const char* assembly_name, cons
                                try {
                                        conf = File.OpenRead (config_file);
                                } catch {
-                                       Error (String.Format ("Failure to open {0}", config_file));
+                                       Error ("Failure to open {0}", config_file);
                                        return;
                                }
                                if (!quiet)
@@ -805,7 +849,7 @@ void          mono_register_config_for_assembly (const char* assembly_name, cons
                                try {
                                        conf = File.OpenRead (machine_config_file);
                                } catch {
-                                       Error (String.Format ("Failure to open {0}", machine_config_file));
+                                       Error ("Failure to open {0}", machine_config_file);
                                        return;
                                }
                                if (!quiet)
@@ -965,6 +1009,7 @@ void          mono_register_config_for_assembly (const char* assembly_name, cons
                        else
                        {
                                string zlib = (compress ? "-lz" : "");
+                               string objc = (style == "osx" ? "-framework CoreFoundation -lobjc" : "");
                                string debugging = "-g";
                                string cc = GetEnv("CC", "cc");
                                string cmd = null;
@@ -978,16 +1023,16 @@ void          mono_register_config_for_assembly (const char* assembly_name, cons
                                                smonolib = "`pkg-config --variable=libdir mono-2`/libmono-2.0.a ";
                                        else
                                                smonolib = "-Wl,-Bstatic -lmono-2.0 -Wl,-Bdynamic ";
-                                       cmd = String.Format("{4} -o '{2}' -Wall `pkg-config --cflags mono-2` {0} {3} " +
+                                       cmd = String.Format("{4} -o '{2}' -Wall {5} `pkg-config --cflags mono-2` {0} {3} " +
                                                "`pkg-config --libs-only-L mono-2` " + smonolib +
                                                "`pkg-config --libs-only-l mono-2 | sed -e \"s/\\-lmono-2.0 //\"` {1}",
-                                               temp_c, temp_o, output, zlib, cc);
+                                               temp_c, temp_o, output, zlib, cc, objc);
                                }
                                else
                                {
 
-                                       cmd = String.Format("{4} " + debugging + " -o '{2}' -Wall {0} `pkg-config --cflags --libs mono-2` {3} {1}",
-                                               temp_c, temp_o, output, zlib, cc);
+                                       cmd = String.Format("{4} " + debugging + " -o '{2}' -Wall {5} {0} `pkg-config --cflags --libs mono-2` {3} {1}",
+                                               temp_c, temp_o, output, zlib, cc, objc);
                                }
                                Execute (cmd);
                        }
@@ -1014,11 +1059,9 @@ void          mono_register_config_for_assembly (const char* assembly_name, cons
                List<string> assemblies = new List<string> ();
                bool error = false;
 
-               var other = i18n.Select (x=> "I18N." + x + (x.Length > 0 ? "." : "") + "dll");
-               
-               foreach (string name in sources.Concat (other)){
+               foreach (string name in sources){
                        try {
-                               Assembly a = LoadAssembly (name);
+                               Assembly a = LoadAssemblyFile (name);
 
                                if (a == null){
                                        error = true;
@@ -1037,17 +1080,61 @@ void          mono_register_config_for_assembly (const char* assembly_name, cons
                        }
                }
 
-               if (error)
+               if (error) {
+                       Error ("Couldn't load one or more of the assemblies.");
                        Environment.Exit (1);
+               }
 
                return assemblies;
        }
+
+       static void LoadLocalizedAssemblies (List<string> assemblies)
+       {
+               var other = i18n.Select (x => "I18N." + x + (x.Length > 0 ? "." : "") + "dll");
+               string error = null;
+
+               foreach (string name in other) {
+                       try {
+                               Assembly a = LoadAssembly (name);
+
+                               if (a == null) {
+                                       error = "Failed to load " + name;
+                                       continue;
+                               }
+
+                               assemblies.Add (a.CodeBase);
+                       } catch (Exception) {
+                               if (skip_scan) {
+                                       if (!quiet)
+                                               Console.WriteLine ("File will not be scanned: {0}", name);
+                                       assemblies.Add (new Uri (new FileInfo (name).FullName).ToString ());
+                               } else {
+                                       throw;
+                               }
+                       }
+               }
+
+               if (error != null) {
+                       Console.Error.WriteLine ("Failure to load i18n assemblies, the following directories were searched for the assemblies:");
+                       foreach (var path in link_paths){
+                               Console.Error.WriteLine ("   Path: " + path);
+                       }
+                       if (custom_mode){
+                               Console.WriteLine ("In Custom mode, you need to provide the directory to lookup assemblies from using -L");
+                       }
+                       
+                       Error ("Couldn't load one or more of the i18n assemblies: " + error);
+                       Environment.Exit (1);
+               }
+       }
+
        
        static readonly Universe universe = new Universe ();
        static readonly Dictionary<string, string> loaded_assemblies = new Dictionary<string, string> ();
 
        public static string GetAssemblyName (string path)
        {
+               string resourcePathSeparator = style == "windows" ? "\\\\" : "/";
                string name = Path.GetFileName (path);
 
                // A bit of a hack to support satellite assemblies. They all share the same name but
@@ -1057,7 +1144,7 @@ void          mono_register_config_for_assembly (const char* assembly_name, cons
                        string dir = Path.GetDirectoryName (path);
                        int idx = dir.LastIndexOf (Path.DirectorySeparatorChar);
                        if (idx >= 0) {
-                               name = dir.Substring (idx + 1) + Path.DirectorySeparatorChar + name;
+                               name = dir.Substring (idx + 1) + resourcePathSeparator + name;
                                Console.WriteLine ($"Storing satellite assembly '{path}' with name '{name}'");
                        } else if (!quiet)
                                Console.WriteLine ($"Warning: satellite assembly {path} doesn't have locale path prefix, name conflicts possible");
@@ -1076,7 +1163,7 @@ void          mono_register_config_for_assembly (const char* assembly_name, cons
                var name = GetAssemblyName (path);
                string found;
                if (loaded_assemblies.TryGetValue (name, out found)) {
-                       Error (string.Format ("Duplicate assembly name `{0}'. Both `{1}' and `{2}' use same assembly name.", name, path, found));
+                       Error ("Duplicate assembly name `{0}'. Both `{1}' and `{2}' use same assembly name.", name, path, found);
                        return false;
                }
 
@@ -1087,9 +1174,18 @@ void          mono_register_config_for_assembly (const char* assembly_name, cons
                        return true;
                try {
                        Assembly a = universe.LoadFile (path);
+                       if (a == null) {
+                               Error ("Unable to to load assembly `{0}'", path);
+                               return false;
+                       }
 
                        foreach (AssemblyName an in a.GetReferencedAssemblies ()) {
-                               a = universe.Load (an.FullName);
+                               a = LoadAssembly (an.Name);
+                               if (a == null) {
+                                       Error ("Unable to load assembly `{0}' referenced by `{1}'", an.Name, path);
+                                       return false;
+                               }
+
                                if (!QueueAssembly (files, a.CodeBase))
                                        return false;
                        }
@@ -1101,56 +1197,63 @@ void          mono_register_config_for_assembly (const char* assembly_name, cons
                return true;
        }
 
-       static Assembly LoadAssembly (string assembly)
+       //
+       // Loads an assembly from a specific path
+       //
+       static Assembly LoadAssemblyFile (string assembly)
        {
-               Assembly a;
+               Assembly a = null;
                
                try {
-                       char[] path_chars = { '/', '\\' };
+                       if (!quiet)
+                               Console.WriteLine ("Attempting to load assembly: {0}", assembly);
+                       a = universe.LoadFile (assembly);
+                       if (!quiet)
+                               Console.WriteLine ("Assembly {0} loaded successfully.", assembly);
                        
-                       if (assembly.IndexOfAny (path_chars) != -1) {
-                               a = universe.LoadFile (assembly);
-                       } else {
-                               string ass = assembly;
-                               if (ass.EndsWith (".dll"))
-                                       ass = assembly.Substring (0, assembly.Length - 4);
-                               a = universe.Load (ass);
-                       }
-                       return a;
                } catch (FileNotFoundException){
-                       string total_log = "";
-                       
-                       foreach (string dir in link_paths){
-                               string full_path = Path.Combine (dir, assembly);
-                               if (!assembly.EndsWith (".dll") && !assembly.EndsWith (".exe"))
-                                       full_path += ".dll";
-                               
-                               try {
-                                       a = universe.LoadFile (full_path);
-                                       return a;
-                               } catch (FileNotFoundException ff) {
-                                       total_log += ff.FusionLog;
-                                       continue;
-                               }
-                       }
-                       Error ("Cannot find assembly `" + assembly + "'" );
-                       if (!quiet)
-                               Console.WriteLine ("Log: \n" + total_log);
+                       Error ($"Cannot find assembly `{assembly}'");
                } catch (IKVM.Reflection.BadImageFormatException f) {
                        if (skip_scan)
                                throw;
-                       Error ("Cannot load assembly (bad file format) " + f.Message);
+                       Error ($"Cannot load assembly (bad file format) " + f.Message);
                } catch (FileLoadException f){
-                       Error ("Cannot load assembly " + f.Message);
+                       Error ($"Cannot load assembly " + f.Message);
                } catch (ArgumentNullException){
-                       Error("Cannot load assembly (null argument)");
+                       Error( $"Cannot load assembly (null argument)");
                }
-               return null;
+               return a;
        }
 
-       static void Error (string msg)
+       //
+       // Loads an assembly from any of the link directories provided
+       //
+       static Assembly LoadAssembly (string assembly)
+       {
+               string total_log = "";
+               foreach (string dir in link_paths){
+                       string full_path = Path.Combine (dir, assembly);
+                       if (!quiet)
+                               Console.WriteLine ("Attempting to load assembly from: " + full_path);
+                       if (!assembly.EndsWith (".dll") && !assembly.EndsWith (".exe"))
+                               full_path += ".dll";
+                       
+                       try {
+                               var a = universe.LoadFile (full_path);
+                               return a;
+                       } catch (FileNotFoundException ff) {
+                               total_log += ff.FusionLog;
+                               continue;
+                       }
+               }
+               if (!quiet)
+                       Console.WriteLine ("Log: \n" + total_log);
+               return null;
+       }
+       
+       static void Error (string msg, params object [] args)
        {
-               Console.Error.WriteLine ("ERROR: " + msg);
+               Console.Error.WriteLine ("ERROR: {0}", string.Format (msg, args));
                Environment.Exit (1);
        }
 
@@ -1168,16 +1271,19 @@ void          mono_register_config_for_assembly (const char* assembly_name, cons
                                   "    --skip-scan          Skip scanning assemblies that could not be loaded (but still embed them).\n" +
                                   "    --i18n ENCODING      none, all or comma separated list of CJK, MidWest, Other, Rare, West.\n" +
                                   "    -v                   Verbose output\n" + 
+                                  "    --bundled-header     Do not attempt to include 'mono-config.h'. Define the entry points directly in the generated code\n" +
                                   "\n" + 
                                   "--simple   Simple mode does not require a C toolchain and can cross compile\n" + 
                                   "    --cross TARGET       Generates a binary for the given TARGET\n"+
                                   "    --env KEY=VALUE      Hardcodes an environment variable for the target\n" +
+                                  "    --fetch-target NAME  Downloads the target SDK from the remote server\n" + 
                                   "    --library [LIB,]PATH Bundles the specified dynamic library to be used at runtime\n" +
                                   "                         LIB is optional shortname for file located at PATH\n" + 
                                   "    --list-targets       Lists available targets on the remote server\n" +
                                   "    --local-targets      Lists locally available targets\n" +
                                   "    --options OPTIONS    Embed the specified Mono command line options on target\n" +
                                   "    --runtime RUNTIME    Manually specifies the Mono runtime to use\n" +
+                                  "    --sdk PATH           Use a Mono SDK root location instead of a target\n" + 
                                   "    --target-server URL  Specified a server to download targets from, default is " + target_server + "\n" +
                                   "\n" +
                                   "--custom   Builds a custom launcher, options for --custom\n" +
@@ -1308,7 +1414,7 @@ void          mono_register_config_for_assembly (const char* assembly_name, cons
                        p.WaitForExit ();
                        int ret = p.ExitCode;
                        if (ret != 0){
-                               Error (String.Format("[Fail] {0}", ret));
+                               Error ("[Fail] {0}", ret);
                        }
                }
        }