Merge pull request #610 from metanest/tests_gc_descriptors_vpath_build_2013Mar_spike
[mono.git] / mcs / tools / mkbundle / mkbundle.cs
1 //
2 // mkbundle: tool to create bundles.
3 //
4 // Based on the `make-bundle' Perl script written by Paolo Molaro (lupus@debian.org)
5 //
6 // Author:
7 //   Miguel de Icaza
8 //
9 // (C) Novell, Inc 2004
10 //
11 using System;
12 using System.Diagnostics;
13 using System.Xml;
14 using System.Collections.Generic;
15 using System.Reflection;
16 using System.IO;
17 using System.IO.Compression;
18 using System.Runtime.InteropServices;
19 using System.Text;
20
21
22 #if NET_4_5
23 using System.Threading.Tasks;
24 #endif
25
26 class MakeBundle {
27         static string output = "a.out";
28         static string object_out = null;
29         static List<string> link_paths = new List<string> ();
30         static bool autodeps = false;
31         static bool keeptemp = false;
32         static bool compile_only = false;
33         static bool static_link = false;
34         static string config_file = null;
35         static string machine_config_file = null;
36         static string config_dir = null;
37         static string style = "linux";
38         static bool compress;
39         static bool nomain;
40         
41         static int Main (string [] args)
42         {
43                 List<string> sources = new List<string> ();
44                 int top = args.Length;
45                 link_paths.Add (".");
46
47                 DetectOS ();
48                 
49                 for (int i = 0; i < top; i++){
50                         switch (args [i]){
51                         case "--help": case "-h": case "-?":
52                                 Help ();
53                                 return 1;
54
55                         case "-c":
56                                 compile_only = true;
57                                 break;
58                                 
59                         case "-o": 
60                                 if (i+1 == top){
61                                         Help (); 
62                                         return 1;
63                                 }
64                                 output = args [++i];
65                                 break;
66
67                         case "-oo":
68                                 if (i+1 == top){
69                                         Help (); 
70                                         return 1;
71                                 }
72                                 object_out = args [++i];
73                                 break;
74
75                         case "-L":
76                                 if (i+1 == top){
77                                         Help (); 
78                                         return 1;
79                                 }
80                                 link_paths.Add (args [++i]);
81                                 break;
82
83                         case "--nodeps":
84                                 autodeps = false;
85                                 break;
86
87                         case "--deps":
88                                 autodeps = true;
89                                 break;
90
91                         case "--keeptemp":
92                                 keeptemp = true;
93                                 break;
94                         case "--static":
95                                 static_link = true;
96                                 Console.WriteLine ("Note that statically linking the LGPL Mono runtime has more licensing restrictions than dynamically linking.");
97                                 Console.WriteLine ("See http://www.mono-project.com/Licensing for details on licensing.");
98                                 break;
99                         case "--config":
100                                 if (i+1 == top) {
101                                         Help ();
102                                         return 1;
103                                 }
104
105                                 config_file = args [++i];
106                                 break;
107                         case "--machine-config":
108                                 if (i+1 == top) {
109                                         Help ();
110                                         return 1;
111                                 }
112
113                                 machine_config_file = args [++i];
114
115                                 Console.WriteLine ("WARNING:\n  Check that the machine.config file you are bundling\n  doesn't contain sensitive information specific to this machine.");
116                                 break;
117                         case "--config-dir":
118                                 if (i+1 == top) {
119                                         Help ();
120                                         return 1;
121                                 }
122
123                                 config_dir = args [++i];
124                                 break;
125                         case "-z":
126                                 compress = true;
127                                 break;
128                         case "--nomain":
129                                 nomain = true;
130                                 break;
131                         case "--style":
132                                 if (i+1 == top) {
133                                         Help ();
134                                         return 1;
135                                 }
136                                 style = args [++i];
137                                 switch (style) {
138                                 case "windows":
139                                 case "mac":
140                                 case "linux":
141                                         break;
142                                 default:
143                                         Console.Error.WriteLine ("Invalid style '{0}' - only 'windows', 'mac' and 'linux' are supported for --style argument", style);
144                                         return 1;
145                                 }
146                                         
147                                 break;
148                         default:
149                                 sources.Add (args [i]);
150                                 break;
151                         }
152                         
153                         if (static_link && style == "windows") {
154                                 Console.Error.WriteLine ("The option `{0}' is not supported on this platform.", args [i]);
155                                 return 1;
156                         }
157                 }
158
159                 Console.WriteLine ("Sources: {0} Auto-dependencies: {1}", sources.Count, autodeps);
160                 if (sources.Count == 0 || output == null) {
161                         Help ();
162                         Environment.Exit (1);
163                 }
164
165                 List<Assembly> assemblies = LoadAssemblies (sources);
166                 List<string> files = new List<string> ();
167                 foreach (Assembly a in assemblies)
168                         QueueAssembly (files, a.CodeBase);
169                         
170                 // Special casing mscorlib.dll: any specified mscorlib.dll cannot be loaded
171                 // by Assembly.ReflectionFromLoadFrom(). Instead the fx assembly which runs
172                 // mkbundle.exe is loaded, which is not what we want.
173                 // So, replace it with whatever actually specified.
174                 foreach (string srcfile in sources) {
175                         if (Path.GetFileName (srcfile) == "mscorlib.dll") {
176                                 foreach (string file in files) {
177                                         if (Path.GetFileName (new Uri (file).LocalPath) == "mscorlib.dll") {
178                                                 files.Remove (file);
179                                                 files.Add (new Uri (Path.GetFullPath (srcfile)).LocalPath);
180                                                 break;
181                                         }
182                                 }
183                                 break;
184                         }
185                 }
186
187                 GenerateBundles (files);
188                 //GenerateJitWrapper ();
189                 
190                 return 0;
191         }
192
193         static void WriteSymbol (StreamWriter sw, string name, long size)
194         {
195                 switch (style){
196                 case "linux":
197                         sw.WriteLine (
198                                 ".globl {0}\n" +
199                                 "\t.section .rodata\n" +
200                                 "\t.p2align 5\n" +
201                                 "\t.type {0}, \"object\"\n" +
202                                 "\t.size {0}, {1}\n" +
203                                 "{0}:\n",
204                                 name, size);
205                         break;
206                 case "osx":
207                         sw.WriteLine (
208                                 "\t.section __TEXT,__text,regular,pure_instructions\n" + 
209                                 "\t.globl _{0}\n" +
210                                 "\t.data\n" +
211                                 "\t.align 4\n" +
212                                 "_{0}:\n",
213                                 name, size);
214                         break;
215                 case "windows":
216                         sw.WriteLine (
217                                 ".globl _{0}\n" +
218                                 "\t.section .rdata,\"dr\"\n" +
219                                 "\t.align 32\n" +
220                                 "_{0}:\n",
221                                 name, size);
222                         break;
223                 }
224         }
225         
226         static string [] chars = new string [256];
227         
228         static void WriteBuffer (StreamWriter ts, Stream stream, byte[] buffer)
229         {
230                 int n;
231                 
232                 // Preallocate the strings we need.
233                 if (chars [0] == null) {
234                         for (int i = 0; i < chars.Length; i++)
235                                 chars [i] = string.Format ("{0}", i.ToString ());
236                 }
237
238                 while ((n = stream.Read (buffer, 0, buffer.Length)) != 0) {
239                         int count = 0;
240                         for (int i = 0; i < n; i++) {
241                                 if (count % 32 == 0) {
242                                         ts.Write ("\n\t.byte ");
243                                 } else {
244                                         ts.Write (",");
245                                 }
246                                 ts.Write (chars [buffer [i]]);
247                                 count ++;
248                         }
249                 }
250
251                 ts.WriteLine ();
252         }
253         
254         static void GenerateBundles (List<string> files)
255         {
256                 string temp_s = "temp.s"; // Path.GetTempFileName ();
257                 string temp_c = "temp.c";
258                 string temp_o = "temp.o";
259
260                 if (compile_only)
261                         temp_c = output;
262                 if (object_out != null)
263                         temp_o = object_out;
264                 
265                 try {
266                         List<string> c_bundle_names = new List<string> ();
267                         List<string[]> config_names = new List<string[]> ();
268                         byte [] buffer = new byte [8192];
269
270                         using (StreamWriter ts = new StreamWriter (File.Create (temp_s))) {
271                         using (StreamWriter tc = new StreamWriter (File.Create (temp_c))) {
272                         string prog = null;
273
274                         tc.WriteLine ("/* This source code was produced by mkbundle, do not edit */");
275                         tc.WriteLine ("#include <mono/metadata/mono-config.h>");
276                         tc.WriteLine ("#include <mono/metadata/assembly.h>\n");
277
278                         if (compress) {
279                                 tc.WriteLine ("typedef struct _compressed_data {");
280                                 tc.WriteLine ("\tMonoBundledAssembly assembly;");
281                                 tc.WriteLine ("\tint compressed_size;");
282                                 tc.WriteLine ("} CompressedAssembly;\n");
283                         }
284
285                         object monitor = new object ();
286
287                         var streams = new Dictionary<string, Stream> ();
288                         var sizes = new Dictionary<string, long> ();
289
290                         // Do the file reading and compression in parallel
291                         Action<string> body = delegate (string url) {
292                                 string fname = new Uri (url).LocalPath;
293                                 Stream stream = File.OpenRead (fname);
294
295                                 long real_size = stream.Length;
296                                 int n;
297                                 if (compress) {
298                                         MemoryStream ms = new MemoryStream ();
299                                         GZipStream deflate = new GZipStream (ms, CompressionMode.Compress, leaveOpen:true);
300                                         while ((n = stream.Read (buffer, 0, buffer.Length)) != 0){
301                                                 deflate.Write (buffer, 0, n);
302                                         }
303                                         stream.Close ();
304                                         deflate.Close ();
305                                         byte [] bytes = ms.GetBuffer ();
306                                         stream = new MemoryStream (bytes, 0, (int) ms.Length, false, false);
307                                 }
308
309                                 lock (monitor) {
310                                         streams [url] = stream;
311                                         sizes [url] = real_size;
312                                 }
313                         };
314
315 #if NET_4_5
316                         Parallel.ForEach (files, body);
317 #else
318                         foreach (var url in files)
319                                 body (url);
320 #endif
321
322                         // The non-parallel part
323                         foreach (var url in files) {
324                                 string fname = new Uri (url).LocalPath;
325                                 string aname = Path.GetFileName (fname);
326                                 string encoded = aname.Replace ("-", "_").Replace (".", "_");
327
328                                 if (prog == null)
329                                         prog = aname;
330
331                                 var stream = streams [url];
332                                 var real_size = sizes [url];
333
334                                 Console.WriteLine ("   embedding: " + fname);
335
336                                 WriteSymbol (ts, "assembly_data_" + encoded, stream.Length);
337                         
338                                 WriteBuffer (ts, stream, buffer);
339
340                                 if (compress) {
341                                         tc.WriteLine ("extern const unsigned char assembly_data_{0} [];", encoded);
342                                         tc.WriteLine ("static CompressedAssembly assembly_bundle_{0} = {{{{\"{1}\"," +
343                                                                   " assembly_data_{0}, {2}}}, {3}}};",
344                                                                   encoded, aname, real_size, stream.Length);
345                                         double ratio = ((double) stream.Length * 100) / real_size;
346                                         Console.WriteLine ("   compression ratio: {0:.00}%", ratio);
347                                 } else {
348                                         tc.WriteLine ("extern const unsigned char assembly_data_{0} [];", encoded);
349                                         tc.WriteLine ("static const MonoBundledAssembly assembly_bundle_{0} = {{\"{1}\", assembly_data_{0}, {2}}};",
350                                                                   encoded, aname, real_size);
351                                 }
352                                 stream.Close ();
353
354                                 c_bundle_names.Add ("assembly_bundle_" + encoded);
355
356                                 try {
357                                         FileStream cf = File.OpenRead (fname + ".config");
358                                         Console.WriteLine (" config from: " + fname + ".config");
359                                         tc.WriteLine ("extern const unsigned char assembly_config_{0} [];", encoded);
360                                         WriteSymbol (ts, "assembly_config_" + encoded, cf.Length);
361                                         WriteBuffer (ts, cf, buffer);
362                                         ts.WriteLine ();
363                                         config_names.Add (new string[] {aname, encoded});
364                                 } catch (FileNotFoundException) {
365                                         /* we ignore if the config file doesn't exist */
366                                 }
367                         }
368
369                         if (config_file != null){
370                                 FileStream conf;
371                                 try {
372                                         conf = File.OpenRead (config_file);
373                                 } catch {
374                                         Error (String.Format ("Failure to open {0}", config_file));
375                                         return;
376                                 }
377                                 Console.WriteLine ("System config from: " + config_file);
378                                 tc.WriteLine ("extern const char system_config;");
379                                 WriteSymbol (ts, "system_config", config_file.Length);
380
381                                 WriteBuffer (ts, conf, buffer);
382                                 // null terminator
383                                 ts.Write ("\t.byte 0\n");
384                                 ts.WriteLine ();
385                         }
386
387                         if (machine_config_file != null){
388                                 FileStream conf;
389                                 try {
390                                         conf = File.OpenRead (machine_config_file);
391                                 } catch {
392                                         Error (String.Format ("Failure to open {0}", machine_config_file));
393                                         return;
394                                 }
395                                 Console.WriteLine ("Machine config from: " + machine_config_file);
396                                 tc.WriteLine ("extern const char machine_config;");
397                                 WriteSymbol (ts, "machine_config", machine_config_file.Length);
398
399                                 WriteBuffer (ts, conf, buffer);
400                                 ts.Write ("\t.byte 0\n");
401                                 ts.WriteLine ();
402                         }
403                         ts.Close ();
404                         
405                         Console.WriteLine ("Compiling:");
406                         string cmd = String.Format ("{0} -o {1} {2} ", GetEnv ("AS", "as"), temp_o, temp_s);
407                         int ret = Execute (cmd);
408                         if (ret != 0){
409                                 Error ("[Fail]");
410                                 return;
411                         }
412
413                         if (compress)
414                                 tc.WriteLine ("\nstatic const CompressedAssembly *compressed [] = {");
415                         else
416                                 tc.WriteLine ("\nstatic const MonoBundledAssembly *bundled [] = {");
417
418                         foreach (string c in c_bundle_names){
419                                 tc.WriteLine ("\t&{0},", c);
420                         }
421                         tc.WriteLine ("\tNULL\n};\n");
422                         tc.WriteLine ("static char *image_name = \"{0}\";", prog);
423
424                         tc.WriteLine ("\nstatic void install_dll_config_files (void) {\n");
425                         foreach (string[] ass in config_names){
426                                 tc.WriteLine ("\tmono_register_config_for_assembly (\"{0}\", assembly_config_{1});\n", ass [0], ass [1]);
427                         }
428                         if (config_file != null)
429                                 tc.WriteLine ("\tmono_config_parse_memory (&system_config);\n");
430                         if (machine_config_file != null)
431                                 tc.WriteLine ("\tmono_register_machine_config (&machine_config);\n");
432                         tc.WriteLine ("}\n");
433
434                         if (config_dir != null)
435                                 tc.WriteLine ("static const char *config_dir = \"{0}\";", config_dir);
436                         else
437                                 tc.WriteLine ("static const char *config_dir = NULL;");
438
439                         Stream template_stream;
440                         if (compress) {
441                                 template_stream = Assembly.GetAssembly (typeof(MakeBundle)).GetManifestResourceStream ("template_z.c");
442                         } else {
443                                 template_stream = Assembly.GetAssembly (typeof(MakeBundle)).GetManifestResourceStream ("template.c");
444                         }
445
446                         StreamReader s = new StreamReader (template_stream);
447                         string template = s.ReadToEnd ();
448                         tc.Write (template);
449
450                         if (!nomain) {
451                                 Stream template_main_stream = Assembly.GetAssembly (typeof(MakeBundle)).GetManifestResourceStream ("template_main.c");
452                                 StreamReader st = new StreamReader (template_main_stream);
453                                 string maintemplate = st.ReadToEnd ();
454                                 tc.Write (maintemplate);
455                         }
456                         
457                         tc.Close ();
458
459                         if (compile_only)
460                                 return;
461
462                         string zlib = (compress ? "-lz" : "");
463                         string debugging = "-g";
464                         string cc = GetEnv ("CC", IsUnix ? "cc" : "gcc -mno-cygwin");
465
466                         if (style == "linux")
467                                 debugging = "-ggdb";
468                         if (static_link) {
469                                 string smonolib;
470                                 if (style == "osx")
471                                         smonolib = "`pkg-config --variable=libdir mono-2`/libmono-2.0.a ";
472                                 else
473                                         smonolib = "-Wl,-Bstatic -lmono-2.0 -Wl,-Bdynamic ";
474                                 cmd = String.Format ("{4} -o {2} -Wall `pkg-config --cflags mono-2` {0} {3} " +
475                                                      "`pkg-config --libs-only-L mono-2` " + smonolib +
476                                                      "`pkg-config --libs-only-l mono-2 | sed -e \"s/\\-lmono-2.0 //\"` {1}",
477                                                      temp_c, temp_o, output, zlib, cc);
478                         } else {
479                                 
480                                 cmd = String.Format ("{4} " + debugging + " -o {2} -Wall {0} `pkg-config --cflags --libs mono-2` {3} {1}",
481                                                      temp_c, temp_o, output, zlib, cc);
482                         }
483                             
484                         ret = Execute (cmd);
485                         if (ret != 0){
486                                 Error ("[Fail]");
487                                 return;
488                         }
489                         Console.WriteLine ("Done");
490                         }
491                         }
492                 } finally {
493                         if (!keeptemp){
494                                 if (object_out == null){
495                                         File.Delete (temp_o);
496                                 }
497                                 if (!compile_only){
498                                         File.Delete (temp_c);
499                                 }
500                                 File.Delete (temp_s);
501                         }
502                 }
503         }
504         
505         static List<Assembly> LoadAssemblies (List<string> sources)
506         {
507                 List<Assembly> assemblies = new List<Assembly> ();
508                 bool error = false;
509                 
510                 foreach (string name in sources){
511                         Assembly a = LoadAssembly (name);
512
513                         if (a == null){
514                                 error = true;
515                                 continue;
516                         }
517                         
518                         assemblies.Add (a);
519                 }
520
521                 if (error)
522                         Environment.Exit (1);
523
524                 return assemblies;
525         }
526         
527         static void QueueAssembly (List<string> files, string codebase)
528         {
529                 if (files.Contains (codebase))
530                         return;
531
532                 files.Add (codebase);
533                 Assembly a = Assembly.LoadFrom (new Uri(codebase).LocalPath);
534
535                 if (!autodeps)
536                         return;
537                 
538                 foreach (AssemblyName an in a.GetReferencedAssemblies ()) {
539                         a = Assembly.Load (an);
540                         QueueAssembly (files, a.CodeBase);
541                 }
542         }
543
544         static Assembly LoadAssembly (string assembly)
545         {
546                 Assembly a;
547                 
548                 try {
549                         char[] path_chars = { '/', '\\' };
550                         
551                         if (assembly.IndexOfAny (path_chars) != -1) {
552                                 a = Assembly.LoadFrom (assembly);
553                         } else {
554                                 string ass = assembly;
555                                 if (ass.EndsWith (".dll"))
556                                         ass = assembly.Substring (0, assembly.Length - 4);
557                                 a = Assembly.Load (ass);
558                         }
559                         return a;
560                 } catch (FileNotFoundException){
561                         string total_log = "";
562                         
563                         foreach (string dir in link_paths){
564                                 string full_path = Path.Combine (dir, assembly);
565                                 if (!assembly.EndsWith (".dll") && !assembly.EndsWith (".exe"))
566                                         full_path += ".dll";
567                                 
568                                 try {
569                                         a = Assembly.LoadFrom (full_path);
570                                         return a;
571                                 } catch (FileNotFoundException ff) {
572                                         total_log += ff.FusionLog;
573                                         continue;
574                                 }
575                         }
576                         Error ("Cannot find assembly `" + assembly + "'" );
577                         Console.WriteLine ("Log: \n" + total_log);
578                 } catch (BadImageFormatException f) {
579                         Error ("Cannot load assembly (bad file format)" + f.FusionLog);
580                 } catch (FileLoadException f){
581                         Error ("Cannot load assembly " + f.FusionLog);
582                 } catch (ArgumentNullException){
583                         Error("Cannot load assembly (null argument)");
584                 }
585                 return null;
586         }
587
588         static void Error (string msg)
589         {
590                 Console.Error.WriteLine (msg);
591                 Environment.Exit (1);
592         }
593
594         static void Help ()
595         {
596                 Console.WriteLine ("Usage is: mkbundle [options] assembly1 [assembly2...]\n\n" +
597                                    "Options:\n" +
598                                    "    -c                  Produce stub only, do not compile\n" +
599                                    "    -o out              Specifies output filename\n" +
600                                    "    -oo obj             Specifies output filename for helper object file\n" +
601                                    "    -L path             Adds `path' to the search path for assemblies\n" +
602                                    "    --nodeps            Turns off automatic dependency embedding (default)\n" +
603                                    "    --deps              Turns on automatic dependency embedding\n" +
604                                    "    --keeptemp          Keeps the temporary files\n" +
605                                    "    --config F          Bundle system config file `F'\n" +
606                                    "    --config-dir D      Set MONO_CFG_DIR to `D'\n" +
607                                    "    --machine-config F  Use the given file as the machine.config for the application.\n" +
608                                    "    --static            Statically link to mono libs\n" +
609                                    "    --nomain            Don't include a main() function, for libraries\n" +
610                                    "    -z                  Compress the assemblies before embedding.\n" +
611                                    "                        You need zlib development headers and libraries.\n");
612         }
613
614         [DllImport ("libc")]
615         static extern int system (string s);
616         [DllImport ("libc")]
617         static extern int uname (IntPtr buf);
618                 
619         static void DetectOS ()
620         {
621                 if (!IsUnix) {
622                         Console.WriteLine ("OS is: Windows");
623                         style = "windows";
624                         return;
625                 }
626
627                 IntPtr buf = Marshal.AllocHGlobal (8192);
628                 if (uname (buf) != 0){
629                         Console.WriteLine ("Warning: Unable to detect OS");
630                         Marshal.FreeHGlobal (buf);
631                         return;
632                 }
633                 string os = Marshal.PtrToStringAnsi (buf);
634                 Console.WriteLine ("OS is: " + os);
635                 if (os == "Darwin")
636                         style = "osx";
637                 
638                 Marshal.FreeHGlobal (buf);
639         }
640
641         static bool IsUnix {
642                 get {
643                         int p = (int) Environment.OSVersion.Platform;
644                         return ((p == 4) || (p == 128) || (p == 6));
645                 }
646         }
647
648         static int Execute (string cmdLine)
649         {
650                 if (IsUnix) {
651                         Console.WriteLine (cmdLine);
652                         return system (cmdLine);
653                 }
654
655                 // on Windows, we have to pipe the output of a
656                 // `cmd` interpolation to dos2unix, because the shell does not
657                 // strip the CRLFs generated by the native pkg-config distributed
658                 // with Mono.
659                 StringBuilder b = new StringBuilder ();
660                 int count = 0;
661                 for (int i = 0; i < cmdLine.Length; i++) {
662                         if (cmdLine [i] == '`') {
663                                 if (count % 2 != 0) {
664                                         b.Append ("|dos2unix");
665                                 }
666                                 count++;
667                         }
668                         b.Append (cmdLine [i]);
669                 }
670                 cmdLine = b.ToString ();
671                 Console.WriteLine (cmdLine);
672                         
673                 ProcessStartInfo psi = new ProcessStartInfo ();
674                 psi.UseShellExecute = false;
675                 psi.FileName = "sh";
676                 psi.Arguments = String.Format ("-c \"{0}\"", cmdLine);
677
678                 using (Process p = Process.Start (psi)) {
679                         p.WaitForExit ();
680                         return p.ExitCode;
681                 }
682         }
683
684         static string GetEnv (string name, string defaultValue) 
685         {
686                 string s = Environment.GetEnvironmentVariable (name);
687                 return s != null ? s : defaultValue;
688         }
689 }