Merge pull request #3766 from BrzVlad/feature-default-conc
[mono.git] / mcs / tools / mkbundle / mkbundle.cs
1 //
2 // mkbundle: tool to create bundles.
3 //
4 // Based on the `make-bundle' Perl script written by Paolo Molaro (lupus@debian.org)
5 //
6 // Author:
7 //   Miguel de Icaza
8 //
9 // (C) Novell, Inc 2004
10 // (C) 2016 Xamarin Inc
11 //
12 // Missing features:
13 // * Add support for packaging native libraries, extracting at runtime and setting the library path.
14 // * Implement --list-targets lists all the available remote targets
15 //
16 using System;
17 using System.Diagnostics;
18 using System.Xml;
19 using System.Collections.Generic;
20 using System.IO;
21 using System.IO.Compression;
22 using System.Runtime.InteropServices;
23 using System.Text;
24 using IKVM.Reflection;
25 using System.Linq;
26 using System.Net;
27 using System.Threading.Tasks;
28
29 class MakeBundle {
30         static string output = "a.out";
31         static string object_out = null;
32         static List<string> link_paths = new List<string> ();
33         static Dictionary<string,string> libraries = new Dictionary<string,string> ();
34         static bool autodeps = false;
35         static bool keeptemp = false;
36         static bool compile_only = false;
37         static bool static_link = false;
38         static string config_file = null;
39         static string machine_config_file = null;
40         static string config_dir = null;
41         static string style = "linux";
42         static string os_message = "";
43         static bool compress;
44         static bool nomain;
45         static string custom_main = null;
46         static bool? use_dos2unix = null;
47         static bool skip_scan;
48         static string ctor_func;
49         static bool quiet = true;
50         static string cross_target = null;
51         static string fetch_target = null;
52         static bool custom_mode = true;
53         static string embedded_options = null;
54         static string runtime = null;
55         static Dictionary<string,string> environment = new Dictionary<string,string>();
56         static string [] i18n = new string [] {
57                 "West",
58                 ""
59         };
60         static string [] i18n_all = new string [] {
61                 "CJK", 
62                 "MidEast",
63                 "Other",
64                 "Rare",
65                 "West",
66                 ""
67         };
68         static string target_server = "https://download.mono-project.com/runtimes/raw/";
69         
70         static int Main (string [] args)
71         {
72                 List<string> sources = new List<string> ();
73                 int top = args.Length;
74                 link_paths.Add (".");
75
76                 DetectOS ();
77                 
78                 for (int i = 0; i < top; i++){
79                         switch (args [i]){
80                         case "--help": case "-h": case "-?":
81                                 Help ();
82                                 return 1;
83
84                         case "--simple":
85                                 custom_mode = false;
86                                 autodeps = true;
87                                 break;
88
89                         case "-v":
90                                 quiet = false;
91                                 break;
92                                 
93                         case "--i18n":
94                                 if (i+1 == top){
95                                         Help ();
96                                         return 1;
97                                 }
98                                 var iarg = args [++i];
99                                 if (iarg == "all")
100                                         i18n = i18n_all;
101                                 else if (iarg == "none")
102                                         i18n = new string [0];
103                                 else
104                                         i18n = iarg.Split (',');
105                                 break;
106                                 
107                         case "--custom":
108                                 custom_mode = true;
109                                 break;
110                                 
111                         case "-c":
112                                 compile_only = true;
113                                 break;
114
115                         case "--local-targets":
116                                 CommandLocalTargets ();
117                                 return 0;
118
119                         case "--cross":
120                                 if (i+1 == top){
121                                         Help (); 
122                                         return 1;
123                                 }
124                                 custom_mode = false;
125                                 autodeps = true;
126                                 cross_target = args [++i];
127                                 break;
128
129                         case "--library":
130                                 if (i+1 == top){
131                                         Help (); 
132                                         return 1;
133                                 }
134                                 if (custom_mode){
135                                         Console.Error.WriteLine ("--library can only be used with --simple/--runtime/--cross mode");
136                                         Help ();
137                                         return 1;
138                                 }
139                                 var lspec = args [++i];
140                                 var p = lspec.IndexOf (",");
141                                 string alias, path;
142                                 if (p == -1){
143                                         alias = Path.GetFileName (lspec);
144                                         path = lspec;
145                                 } else {
146                                         alias = lspec.Substring (0, p);
147                                         path = lspec.Substring (p+1);
148                                 }
149                                 if (!File.Exists (path)){
150                                         Console.Error.WriteLine ($"The specified library file {path} does not exist");
151                                         return 1;
152                                 }
153                                 libraries [alias] = path;
154                                 break;
155
156                         case "--fetch-target":
157                                 if (i+1 == top){
158                                         Help (); 
159                                         return 1;
160                                 }
161                                 fetch_target = args [++i];
162                                 break;
163
164                         case "--list-targets":
165                                 var wc = new WebClient ();
166                                 var s = wc.DownloadString (new Uri (target_server + "target-list.txt"));
167                                 Console.WriteLine ("Cross-compilation targets available:\n" + s);
168                                 
169                                 return 0;
170                                 
171                         case "--target-server":
172                                 if (i+1 == top){
173                                         Help (); 
174                                         return 1;
175                                 }
176                                 target_server = args [++i];
177                                 break;
178
179                         case "-o": 
180                                 if (i+1 == top){
181                                         Help (); 
182                                         return 1;
183                                 }
184                                 output = args [++i];
185                                 break;
186
187                         case "--options":
188                                 if (i+1 == top){
189                                         Help (); 
190                                         return 1;
191                                 }
192                                 embedded_options = args [++i];
193                                 break;
194                         case "--runtime":
195                                 if (i+1 == top){
196                                         Help (); 
197                                         return 1;
198                                 }
199                                 custom_mode = false;
200                                 autodeps = true;
201                                 runtime = args [++i];
202                                 break;
203                         case "-oo":
204                                 if (i+1 == top){
205                                         Help (); 
206                                         return 1;
207                                 }
208                                 object_out = args [++i];
209                                 break;
210
211                         case "-L":
212                                 if (i+1 == top){
213                                         Help (); 
214                                         return 1;
215                                 }
216                                 link_paths.Add (args [++i]);
217                                 break;
218
219                         case "--nodeps":
220                                 autodeps = false;
221                                 break;
222
223                         case "--deps":
224                                 autodeps = true;
225                                 break;
226
227                         case "--keeptemp":
228                                 keeptemp = true;
229                                 break;
230                                 
231                         case "--static":
232                                 static_link = true;
233                                 break;
234                         case "--config":
235                                 if (i+1 == top) {
236                                         Help ();
237                                         return 1;
238                                 }
239
240                                 config_file = args [++i];
241                                 break;
242                         case "--machine-config":
243                                 if (i+1 == top) {
244                                         Help ();
245                                         return 1;
246                                 }
247
248                                 machine_config_file = args [++i];
249
250                                 if (!quiet)
251                                         Console.WriteLine ("WARNING:\n  Check that the machine.config file you are bundling\n  doesn't contain sensitive information specific to this machine.");
252                                 break;
253                         case "--config-dir":
254                                 if (i+1 == top) {
255                                         Help ();
256                                         return 1;
257                                 }
258
259                                 config_dir = args [++i];
260                                 break;
261                         case "-z":
262                                 compress = true;
263                                 break;
264                         case "--nomain":
265                                 nomain = true;
266                                 break;
267                         case "--custom-main":
268                                 if (i+1 == top) {
269                                         Help ();
270                                         return 1;
271                                 }
272                                 custom_main = args [++i];
273                                 break;
274                         case "--style":
275                                 if (i+1 == top) {
276                                         Help ();
277                                         return 1;
278                                 }
279                                 style = args [++i];
280                                 switch (style) {
281                                 case "windows":
282                                 case "mac":
283                                 case "linux":
284                                         break;
285                                 default:
286                                         Console.Error.WriteLine ("Invalid style '{0}' - only 'windows', 'mac' and 'linux' are supported for --style argument", style);
287                                         return 1;
288                                 }
289                                         
290                                 break;
291                         case "--skip-scan":
292                                 skip_scan = true;
293                                 break;
294                         case "--static-ctor":
295                                 if (i+1 == top) {
296                                         Help ();
297                                         return 1;
298                                 }
299                                 ctor_func = args [++i];
300                                 break;
301                         case "--dos2unix":
302                         case "--dos2unix=true":
303                                 use_dos2unix = true;
304                                 break;
305                         case "--dos2unix=false":
306                                 use_dos2unix = false;
307                                 break;
308                         case "-q":
309                         case "--quiet":
310                                 quiet = true;
311                                 break;
312                         case "-e":
313                         case "--env":
314                                 if (i+1 == top) {
315                                         Help ();
316                                         return 1;
317                                 }
318                                 var env = args [++i];
319                                 p = env.IndexOf ('=');
320                                 if (p == -1)
321                                         environment.Add (env, "");
322                                 else
323                                         environment.Add (env.Substring (0, p), env.Substring (p+1));
324                                 break;
325                         default:
326                                 sources.Add (args [i]);
327                                 break;
328                         }
329
330                 }
331
332                 if (fetch_target != null){
333                         var truntime = Path.Combine (targets_dir, fetch_target, "mono");
334                         Directory.CreateDirectory (Path.GetDirectoryName (truntime));
335                         var wc = new WebClient ();
336                         var uri = new Uri ($"{target_server}{fetch_target}");
337                         try {
338                                 if (!quiet){
339                                         Console.WriteLine ($"Downloading runtime {uri} to {truntime}");
340                                 }
341                                 
342                                 wc.DownloadFile (uri, truntime);
343                         } catch {
344                                 Console.Error.WriteLine ($"Failure to download the specified runtime from {uri}");
345                                 File.Delete (truntime);
346                                 return 1;
347                         }
348                         return 0;
349                 }
350                 
351                 if (!quiet) {
352                         Console.WriteLine (os_message);
353                         Console.WriteLine ("Sources: {0} Auto-dependencies: {1}", sources.Count, autodeps);
354                 }
355
356                 if (sources.Count == 0 || output == null) {
357                         Help ();
358                         Environment.Exit (1);
359                 }
360
361                 List<string> assemblies = LoadAssemblies (sources);
362                 List<string> files = new List<string> ();
363                 foreach (string file in assemblies)
364                         if (!QueueAssembly (files, file))
365                                 return 1;
366                 if (custom_mode)
367                         GenerateBundles (files);
368                 else {
369                         if (cross_target == "default")
370                                 runtime = null;
371                         else {
372                                 if (runtime == null){
373                                         if (cross_target == null){
374                                                 Console.Error.WriteLine ("you should specify either a --runtime or a --cross compilation target");
375                                                 Environment.Exit (1);
376                                         }
377                                         runtime = Path.Combine (targets_dir, cross_target, "mono");
378                                         if (!File.Exists (runtime)){
379                                                 Console.Error.WriteLine ($"The runtime for the {cross_target} does not exist, use --fetch-target {cross_target} to download first");
380                                                 return 1;
381                                         }
382                                 } else {
383                                         if (!File.Exists (runtime)){
384                                                 Console.Error.WriteLine ($"The Mono runtime specified with --runtime does not exist");
385                                                 return 1;
386                                         }
387                                 }
388                                 
389                         }
390                         GeneratePackage (files);
391                 }
392                 Console.WriteLine ("Generated {0}", output);
393
394                 return 0;
395         }
396
397         static string targets_dir = Path.Combine (Environment.GetFolderPath (Environment.SpecialFolder.Personal), ".mono", "targets");
398         
399         static void CommandLocalTargets ()
400         {
401                 string [] targets;
402
403                 Console.WriteLine ("Available targets:");
404                 Console.WriteLine ("\tdefault\t- Current System Mono");
405                 try {
406                         targets = Directory.GetDirectories (targets_dir);
407                 } catch {
408                         return;
409                 }
410                 foreach (var target in targets){
411                         var p = Path.Combine (target, "mono");
412                         if (File.Exists (p))
413                                 Console.WriteLine ("\t{0}", Path.GetFileName (target));
414                 }
415         }
416
417         static void WriteSymbol (StreamWriter sw, string name, long size)
418         {
419                 switch (style){
420                 case "linux":
421                         sw.WriteLine (
422                                 ".globl {0}\n" +
423                                 "\t.section .rodata\n" +
424                                 "\t.p2align 5\n" +
425                                 "\t.type {0}, \"object\"\n" +
426                                 "\t.size {0}, {1}\n" +
427                                 "{0}:\n",
428                                 name, size);
429                         break;
430                 case "osx":
431                         sw.WriteLine (
432                                 "\t.section __TEXT,__text,regular,pure_instructions\n" + 
433                                 "\t.globl _{0}\n" +
434                                 "\t.data\n" +
435                                 "\t.align 4\n" +
436                                 "_{0}:\n",
437                                 name, size);
438                         break;
439                 case "windows":
440                         sw.WriteLine (
441                                 ".globl _{0}\n" +
442                                 "\t.section .rdata,\"dr\"\n" +
443                                 "\t.align 32\n" +
444                                 "_{0}:\n",
445                                 name, size);
446                         break;
447                 }
448         }
449         
450         static string [] chars = new string [256];
451         
452         static void WriteBuffer (StreamWriter ts, Stream stream, byte[] buffer)
453         {
454                 int n;
455                 
456                 // Preallocate the strings we need.
457                 if (chars [0] == null) {
458                         for (int i = 0; i < chars.Length; i++)
459                                 chars [i] = string.Format ("{0}", i.ToString ());
460                 }
461
462                 while ((n = stream.Read (buffer, 0, buffer.Length)) != 0) {
463                         int count = 0;
464                         for (int i = 0; i < n; i++) {
465                                 if (count % 32 == 0) {
466                                         ts.Write ("\n\t.byte ");
467                                 } else {
468                                         ts.Write (",");
469                                 }
470                                 ts.Write (chars [buffer [i]]);
471                                 count ++;
472                         }
473                 }
474
475                 ts.WriteLine ();
476         }
477
478         class PackageMaker {
479                 Dictionary<string, Tuple<long,int>> locations = new Dictionary<string, Tuple<long,int>> ();
480                 const int align = 4096;
481                 Stream package;
482                 
483                 public PackageMaker (string output)
484                 {
485                         package = File.Create (output, 128*1024);
486                         if (IsUnix){
487                                 File.SetAttributes (output, unchecked ((FileAttributes) 0x80000000));
488                         }
489                 }
490
491                 public int AddFile (string fname)
492                 {
493                         using (Stream fileStream = File.OpenRead (fname)){
494                                 var ret = fileStream.Length;
495
496                                 if (!quiet)
497                                         Console.WriteLine ("At {0:x} with input {1}", package.Position, fileStream.Length);
498                                 fileStream.CopyTo (package);
499                                 package.Position = package.Position + (align - (package.Position % align));
500
501                                 return (int) ret;
502                         }
503                 }
504                 
505                 public void Add (string entry, string fname)
506                 {
507                         var p = package.Position;
508                         var size = AddFile (fname);
509                         
510                         locations [entry] = Tuple.Create(p, size);
511                 }
512
513                 public void AddString (string entry, string text)
514                 {
515                         var bytes = Encoding.UTF8.GetBytes (text);
516                         locations [entry] = Tuple.Create (package.Position, bytes.Length);
517                         package.Write (bytes, 0, bytes.Length);
518                         package.Position = package.Position + (align - (package.Position % align));
519                 }
520
521                 public void AddStringPair (string entry, string key, string value)
522                 {
523                         var kbytes = Encoding.UTF8.GetBytes (key);
524                         var vbytes = Encoding.UTF8.GetBytes (value);
525
526                         Console.WriteLine ("ADDING {0} to {1}", key, value);
527                         if (kbytes.Length > 255){
528                                 Console.WriteLine ("The key value can not exceed 255 characters: " + key);
529                                 Environment.Exit (1);
530                         }
531                                 
532                         locations [entry] = Tuple.Create (package.Position, kbytes.Length+vbytes.Length+3);
533                         package.WriteByte ((byte)kbytes.Length);
534                         package.Write (kbytes, 0, kbytes.Length);
535                         package.WriteByte (0);
536                         package.Write (vbytes, 0, vbytes.Length);
537                         package.WriteByte (0);
538                         package.Position = package.Position + (align - (package.Position % align));
539                 }
540
541                 public void Dump ()
542                 {
543                         if (quiet)
544                                 return;
545                         foreach (var floc in locations.Keys){
546                                 Console.WriteLine ($"{floc} at {locations[floc]:x}");
547                         }
548                 }
549
550                 public void WriteIndex ()
551                 {
552                         var indexStart = package.Position;
553                         var binary = new BinaryWriter (package);
554
555                         binary.Write (locations.Count);
556                         foreach (var entry in from entry in locations orderby entry.Value.Item1 ascending select entry){
557                                 var bytes = Encoding.UTF8.GetBytes (entry.Key);
558                                 binary.Write (bytes.Length+1);
559                                 binary.Write (bytes);
560                                 binary.Write ((byte) 0);
561                                 binary.Write (entry.Value.Item1);
562                                 binary.Write (entry.Value.Item2);
563                         }
564                         binary.Write (indexStart);
565                         binary.Write (Encoding.UTF8.GetBytes ("xmonkeysloveplay"));
566                         binary.Flush ();
567                 }
568                 
569                 public void Close ()
570                 {
571                         WriteIndex ();
572                         package.Close ();
573                         package = null;
574                 }
575         }
576
577         static bool MaybeAddFile (PackageMaker maker, string code, string file)
578         {
579                 if (file == null)
580                         return true;
581                 
582                 if (!File.Exists (file)){
583                         Console.Error.WriteLine ("The file {0} does not exist", file);
584                         return false;
585                 }
586                 maker.Add (code, file);
587                 return true;
588         }
589         
590         static bool GeneratePackage (List<string> files)
591         {
592                 if (runtime == null){
593                         if (IsUnix)
594                                 runtime = Process.GetCurrentProcess().MainModule.FileName;
595                         else {
596                                 Console.Error.WriteLine ("You must specify at least one runtime with --runtime or --cross");
597                                 Environment.Exit (1);
598                         }
599                 }
600                 if (!File.Exists (runtime)){
601                         Console.Error.WriteLine ($"The specified runtime at {runtime} does not exist");
602                         Environment.Exit (1);
603                 }
604                 
605                 if (ctor_func != null){
606                         Console.Error.WriteLine ("--static-ctor not supported with package bundling, you must use native compilation for this");
607                         return false;
608                 }
609                 
610                 var maker = new PackageMaker (output);
611                 Console.WriteLine ("Using runtime: " + runtime);
612                 maker.AddFile (runtime);
613                 
614                 foreach (var url in files){
615                         string fname = LocateFile (new Uri (url).LocalPath);
616                         string aname = MakeBundle.GetAssemblyName (fname);
617
618                         maker.Add ("assembly:" + aname, fname);
619                         Console.WriteLine ("     Assembly: " + fname);
620                         if (File.Exists (fname + ".config")){
621                                 maker.Add ("config:" + aname, fname + ".config");
622                                 Console.WriteLine ("       Config: " + runtime);
623                         }
624                 }
625                 
626                 if (!MaybeAddFile (maker, "systemconfig:", config_file) || !MaybeAddFile (maker, "machineconfig:", machine_config_file))
627                         return false;
628
629                 if (config_dir != null)
630                         maker.Add ("config_dir:", config_dir);
631                 if (embedded_options != null)
632                         maker.AddString ("options:", embedded_options);
633                 if (environment.Count > 0){
634                         foreach (var key in environment.Keys)
635                                 maker.AddStringPair ("env:" + key, key, environment [key]);
636                 }
637                 if (libraries.Count > 0){
638                         foreach (var alias_and_path in libraries){
639                                 Console.WriteLine ("     Library:  " + alias_and_path.Value);
640                                 maker.Add ("library:" + alias_and_path.Key, alias_and_path.Value);
641                         }
642                 }
643                 maker.Dump ();
644                 maker.Close ();
645                 return true;
646         }
647         
648         static void GenerateBundles (List<string> files)
649         {
650                 string temp_s = "temp.s"; // Path.GetTempFileName ();
651                 string temp_c = "temp.c";
652                 string temp_o = "temp.o";
653
654                 if (compile_only)
655                         temp_c = output;
656                 if (object_out != null)
657                         temp_o = object_out;
658                 
659                 try {
660                         List<string> c_bundle_names = new List<string> ();
661                         List<string[]> config_names = new List<string[]> ();
662
663                         using (StreamWriter ts = new StreamWriter (File.Create (temp_s))) {
664                         using (StreamWriter tc = new StreamWriter (File.Create (temp_c))) {
665                         string prog = null;
666
667 #if XAMARIN_ANDROID
668                         tc.WriteLine ("/* This source code was produced by mkbundle, do not edit */");
669                         tc.WriteLine ("\n#ifndef NULL\n#define NULL (void *)0\n#endif");
670                         tc.WriteLine (@"
671 typedef struct {
672         const char *name;
673         const unsigned char *data;
674         const unsigned int size;
675 } MonoBundledAssembly;
676 void          mono_register_bundled_assemblies (const MonoBundledAssembly **assemblies);
677 void          mono_register_config_for_assembly (const char* assembly_name, const char* config_xml);
678 ");
679 #else
680                         tc.WriteLine ("#include <mono/metadata/mono-config.h>");
681                         tc.WriteLine ("#include <mono/metadata/assembly.h>\n");
682 #endif
683
684                         if (compress) {
685                                 tc.WriteLine ("typedef struct _compressed_data {");
686                                 tc.WriteLine ("\tMonoBundledAssembly assembly;");
687                                 tc.WriteLine ("\tint compressed_size;");
688                                 tc.WriteLine ("} CompressedAssembly;\n");
689                         }
690
691                         object monitor = new object ();
692
693                         var streams = new Dictionary<string, Stream> ();
694                         var sizes = new Dictionary<string, long> ();
695
696                         // Do the file reading and compression in parallel
697                         Action<string> body = delegate (string url) {
698                                 string fname = LocateFile (new Uri (url).LocalPath);
699                                 Stream stream = File.OpenRead (fname);
700
701                                 long real_size = stream.Length;
702                                 int n;
703                                 if (compress) {
704                                         byte[] cbuffer = new byte [8192];
705                                         MemoryStream ms = new MemoryStream ();
706                                         GZipStream deflate = new GZipStream (ms, CompressionMode.Compress, leaveOpen:true);
707                                         while ((n = stream.Read (cbuffer, 0, cbuffer.Length)) != 0){
708                                                 deflate.Write (cbuffer, 0, n);
709                                         }
710                                         stream.Close ();
711                                         deflate.Close ();
712                                         byte [] bytes = ms.GetBuffer ();
713                                         stream = new MemoryStream (bytes, 0, (int) ms.Length, false, false);
714                                 }
715
716                                 lock (monitor) {
717                                         streams [url] = stream;
718                                         sizes [url] = real_size;
719                                 }
720                         };
721
722                         //#if NET_4_5
723 #if FALSE
724                         Parallel.ForEach (files, body);
725 #else
726                         foreach (var url in files)
727                                 body (url);
728 #endif
729
730                         // The non-parallel part
731                         byte [] buffer = new byte [8192];
732                         // everything other than a-zA-Z0-9_ needs to be escaped in asm symbols.
733                         var symbolEscapeRE = new System.Text.RegularExpressions.Regex ("[^\\w_]");
734                         foreach (var url in files) {
735                                 string fname = LocateFile (new Uri (url).LocalPath);
736                                 string aname = MakeBundle.GetAssemblyName (fname);
737                                 string encoded = symbolEscapeRE.Replace (aname, "_");
738
739                                 if (prog == null)
740                                         prog = aname;
741
742                                 var stream = streams [url];
743                                 var real_size = sizes [url];
744
745                                 if (!quiet)
746                                         Console.WriteLine ("   embedding: " + fname);
747
748                                 WriteSymbol (ts, "assembly_data_" + encoded, stream.Length);
749                         
750                                 WriteBuffer (ts, stream, buffer);
751
752                                 if (compress) {
753                                         tc.WriteLine ("extern const unsigned char assembly_data_{0} [];", encoded);
754                                         tc.WriteLine ("static CompressedAssembly assembly_bundle_{0} = {{{{\"{1}\"," +
755                                                                   " assembly_data_{0}, {2}}}, {3}}};",
756                                                                   encoded, aname, real_size, stream.Length);
757                                         if (!quiet) {
758                                                 double ratio = ((double) stream.Length * 100) / real_size;
759                                                 Console.WriteLine ("   compression ratio: {0:.00}%", ratio);
760                                         }
761                                 } else {
762                                         tc.WriteLine ("extern const unsigned char assembly_data_{0} [];", encoded);
763                                         tc.WriteLine ("static const MonoBundledAssembly assembly_bundle_{0} = {{\"{1}\", assembly_data_{0}, {2}}};",
764                                                                   encoded, aname, real_size);
765                                 }
766                                 stream.Close ();
767
768                                 c_bundle_names.Add ("assembly_bundle_" + encoded);
769
770                                 try {
771                                         FileStream cf = File.OpenRead (fname + ".config");
772                                         if (!quiet)
773                                                 Console.WriteLine (" config from: " + fname + ".config");
774                                         tc.WriteLine ("extern const unsigned char assembly_config_{0} [];", encoded);
775                                         WriteSymbol (ts, "assembly_config_" + encoded, cf.Length);
776                                         WriteBuffer (ts, cf, buffer);
777                                         ts.WriteLine ();
778                                         config_names.Add (new string[] {aname, encoded});
779                                 } catch (FileNotFoundException) {
780                                         /* we ignore if the config file doesn't exist */
781                                 }
782                         }
783
784                         if (config_file != null){
785                                 FileStream conf;
786                                 try {
787                                         conf = File.OpenRead (config_file);
788                                 } catch {
789                                         Error (String.Format ("Failure to open {0}", config_file));
790                                         return;
791                                 }
792                                 if (!quiet)
793                                         Console.WriteLine ("System config from: " + config_file);
794                                 tc.WriteLine ("extern const char system_config;");
795                                 WriteSymbol (ts, "system_config", config_file.Length);
796
797                                 WriteBuffer (ts, conf, buffer);
798                                 // null terminator
799                                 ts.Write ("\t.byte 0\n");
800                                 ts.WriteLine ();
801                         }
802
803                         if (machine_config_file != null){
804                                 FileStream conf;
805                                 try {
806                                         conf = File.OpenRead (machine_config_file);
807                                 } catch {
808                                         Error (String.Format ("Failure to open {0}", machine_config_file));
809                                         return;
810                                 }
811                                 if (!quiet)
812                                         Console.WriteLine ("Machine config from: " + machine_config_file);
813                                 tc.WriteLine ("extern const char machine_config;");
814                                 WriteSymbol (ts, "machine_config", machine_config_file.Length);
815
816                                 WriteBuffer (ts, conf, buffer);
817                                 ts.Write ("\t.byte 0\n");
818                                 ts.WriteLine ();
819                         }
820                         ts.Close ();
821
822                         if (compress)
823                                 tc.WriteLine ("\nstatic const CompressedAssembly *compressed [] = {");
824                         else
825                                 tc.WriteLine ("\nstatic const MonoBundledAssembly *bundled [] = {");
826
827                         foreach (string c in c_bundle_names){
828                                 tc.WriteLine ("\t&{0},", c);
829                         }
830                         tc.WriteLine ("\tNULL\n};\n");
831                         tc.WriteLine ("static char *image_name = \"{0}\";", prog);
832
833                         if (ctor_func != null) {
834                                 tc.WriteLine ("\nextern void {0} (void);", ctor_func);
835                                 tc.WriteLine ("\n__attribute__ ((constructor)) static void mono_mkbundle_ctor (void)");
836                                 tc.WriteLine ("{{\n\t{0} ();\n}}", ctor_func);
837                         }
838
839                         tc.WriteLine ("\nstatic void install_dll_config_files (void) {\n");
840                         foreach (string[] ass in config_names){
841                                 tc.WriteLine ("\tmono_register_config_for_assembly (\"{0}\", assembly_config_{1});\n", ass [0], ass [1]);
842                         }
843                         if (config_file != null)
844                                 tc.WriteLine ("\tmono_config_parse_memory (&system_config);\n");
845                         if (machine_config_file != null)
846                                 tc.WriteLine ("\tmono_register_machine_config (&machine_config);\n");
847                         tc.WriteLine ("}\n");
848
849                         if (config_dir != null)
850                                 tc.WriteLine ("static const char *config_dir = \"{0}\";", config_dir);
851                         else
852                                 tc.WriteLine ("static const char *config_dir = NULL;");
853
854                         Stream template_stream;
855                         if (compress) {
856                                 template_stream = System.Reflection.Assembly.GetAssembly (typeof(MakeBundle)).GetManifestResourceStream ("template_z.c");
857                         } else {
858                                 template_stream = System.Reflection.Assembly.GetAssembly (typeof(MakeBundle)).GetManifestResourceStream ("template.c");
859                         }
860
861                         StreamReader s = new StreamReader (template_stream);
862                         string template = s.ReadToEnd ();
863                         tc.Write (template);
864
865                         if (!nomain && custom_main == null) {
866                                 Stream template_main_stream = System.Reflection.Assembly.GetAssembly (typeof(MakeBundle)).GetManifestResourceStream ("template_main.c");
867                                 StreamReader st = new StreamReader (template_main_stream);
868                                 string maintemplate = st.ReadToEnd ();
869                                 tc.Write (maintemplate);
870                         }
871
872                         tc.Close ();
873
874                         string assembler = GetEnv("AS", "as");
875                         string as_cmd = String.Format("{0} -o {1} {2} ", assembler, temp_o, temp_s);
876                         Execute(as_cmd);
877
878                         if (compile_only)
879                                 return;
880
881                         if (!quiet)
882                                 Console.WriteLine("Compiling:");
883
884                         if (style == "windows")
885                         {
886
887                                 Func<string, string> quote = (pp) => { return "\"" + pp + "\""; };
888
889                                 string compiler = GetEnv("CC", "cl.exe");
890                                 string winsdkPath = GetEnv("WINSDK", @"C:\Program Files (x86)\Windows Kits\8.1");
891                                 string vsPath = GetEnv("VSINCLUDE", @"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC");
892                                 string monoPath = GetEnv("MONOPREFIX", @"C:\Program Files (x86)\Mono");
893
894                                 string[] includes = new string[] {winsdkPath + @"\Include\um", winsdkPath + @"\Include\shared", vsPath + @"\include", monoPath + @"\include\mono-2.0", "." };
895                                 // string[] libs = new string[] { winsdkPath + @"\Lib\winv6.3\um\x86" , vsPath + @"\lib" };
896                                 var linkLibraries = new string[] {  "kernel32.lib",
897                                                                                                 "version.lib",
898                                                                                                 "Ws2_32.lib",
899                                                                                                 "Mswsock.lib",
900                                                                                                 "Psapi.lib",
901                                                                                                 "shell32.lib",
902                                                                                                 "OleAut32.lib",
903                                                                                                 "ole32.lib",
904                                                                                                 "winmm.lib",
905                                                                                                 "user32.lib",
906                                                                                                 "libvcruntime.lib",
907                                                                                                 "advapi32.lib",
908                                                                                                 "OLDNAMES.lib",
909                                                                                                 "libucrt.lib" };
910
911                                 string glue_obj = "mkbundle_glue.obj";
912                                 string monoLib;
913
914                                 if (static_link)
915                                         monoLib = LocateFile (monoPath + @"\lib\monosgen-2.0-static.lib");
916
917                                 else {
918                                         Console.WriteLine ("WARNING: Dynamically linking the Mono runtime on Windows is not a tested option.");
919                                         monoLib = LocateFile (monoPath + @"\lib\monosgen-2.0.lib");
920                                         LocateFile (monoPath + @"\lib\monosgen-2.0.dll"); // in this case, the .lib is just the import library, and the .dll is also needed
921                                 }
922
923                                 var compilerArgs = new List<string>();
924                                 compilerArgs.Add("/MT");
925
926                                 foreach (string include in includes)
927                                         compilerArgs.Add(String.Format ("/I {0}", quote (include)));
928
929                                 if (!nomain || custom_main != null) {
930                                         compilerArgs.Add(quote(temp_c));
931                                         compilerArgs.Add(quote(temp_o));
932                                         if (custom_main != null)
933                                                 compilerArgs.Add(quote(custom_main));
934                                         compilerArgs.Add(quote(monoLib));
935                                         compilerArgs.Add("/link");
936                                         compilerArgs.Add("/NODEFAULTLIB");
937                                         compilerArgs.Add("/SUBSYSTEM:windows");
938                                         compilerArgs.Add("/ENTRY:mainCRTStartup");
939                                         compilerArgs.AddRange(linkLibraries);
940                                         compilerArgs.Add("/out:"+ output);
941
942                                         string cl_cmd = String.Format("{0} {1}", compiler, String.Join(" ", compilerArgs.ToArray()));
943                                         Execute (cl_cmd);
944                                 }
945                                 else
946                                 {
947                                         // we are just creating a .lib
948                                         compilerArgs.Add("/c"); // compile only
949                                         compilerArgs.Add(temp_c);
950                                         compilerArgs.Add(String.Format("/Fo" + glue_obj)); // .obj output name
951
952                                         string cl_cmd = String.Format("{0} {1}", compiler, String.Join(" ", compilerArgs.ToArray()));
953                                         Execute (cl_cmd);
954
955                                         string librarian = GetEnv ("LIB", "lib.exe");
956                                         var librarianArgs = new List<string> ();
957                                         librarianArgs.Add (String.Format ("/out:{0}.lib" + output));
958                                         librarianArgs.Add (temp_o);
959                                         librarianArgs.Add (glue_obj);
960                                         librarianArgs.Add (monoLib);
961                                         string lib_cmd = String.Format("{0} {1}", librarian, String.Join(" ", librarianArgs.ToArray()));
962                                         Execute (lib_cmd);
963                                 }
964                         }
965                         else
966                         {
967                                 string zlib = (compress ? "-lz" : "");
968                                 string debugging = "-g";
969                                 string cc = GetEnv("CC", "cc");
970                                 string cmd = null;
971
972                                 if (style == "linux")
973                                         debugging = "-ggdb";
974                                 if (static_link)
975                                 {
976                                         string smonolib;
977                                         if (style == "osx")
978                                                 smonolib = "`pkg-config --variable=libdir mono-2`/libmono-2.0.a ";
979                                         else
980                                                 smonolib = "-Wl,-Bstatic -lmono-2.0 -Wl,-Bdynamic ";
981                                         cmd = String.Format("{4} -o '{2}' -Wall `pkg-config --cflags mono-2` {0} {3} " +
982                                                 "`pkg-config --libs-only-L mono-2` " + smonolib +
983                                                 "`pkg-config --libs-only-l mono-2 | sed -e \"s/\\-lmono-2.0 //\"` {1}",
984                                                 temp_c, temp_o, output, zlib, cc);
985                                 }
986                                 else
987                                 {
988
989                                         cmd = String.Format("{4} " + debugging + " -o '{2}' -Wall {0} `pkg-config --cflags --libs mono-2` {3} {1}",
990                                                 temp_c, temp_o, output, zlib, cc);
991                                 }
992                                 Execute (cmd);
993                         }
994
995                         if (!quiet)
996                                 Console.WriteLine ("Done");
997                 }
998         }
999                 } finally {
1000                         if (!keeptemp){
1001                                 if (object_out == null){
1002                                         File.Delete (temp_o);
1003                                 }
1004                                 if (!compile_only){
1005                                         File.Delete (temp_c);
1006                                 }
1007                                 File.Delete (temp_s);
1008                         }
1009                 }
1010         }
1011         
1012         static List<string> LoadAssemblies (List<string> sources)
1013         {
1014                 List<string> assemblies = new List<string> ();
1015                 bool error = false;
1016
1017                 var other = i18n.Select (x=> "I18N." + x + (x.Length > 0 ? "." : "") + "dll");
1018                 
1019                 foreach (string name in sources.Concat (other)){
1020                         try {
1021                                 Assembly a = LoadAssembly (name);
1022
1023                                 if (a == null){
1024                                         error = true;
1025                                         continue;
1026                                 }
1027                         
1028                                 assemblies.Add (a.CodeBase);
1029                         } catch (Exception) {
1030                                 if (skip_scan) {
1031                                         if (!quiet)
1032                                                 Console.WriteLine ("File will not be scanned: {0}", name);
1033                                         assemblies.Add (new Uri (new FileInfo (name).FullName).ToString ());
1034                                 } else {
1035                                         throw;
1036                                 }
1037                         }
1038                 }
1039
1040                 if (error)
1041                         Environment.Exit (1);
1042
1043                 return assemblies;
1044         }
1045         
1046         static readonly Universe universe = new Universe ();
1047         static readonly Dictionary<string, string> loaded_assemblies = new Dictionary<string, string> ();
1048
1049         public static string GetAssemblyName (string path)
1050         {
1051                 string name = Path.GetFileName (path);
1052
1053                 // A bit of a hack to support satellite assemblies. They all share the same name but
1054                 // are placed in subdirectories named after the locale they implement. Also, all of
1055                 // them end in .resources.dll, therefore we can use that to detect the circumstances.
1056                 if (name.EndsWith (".resources.dll", StringComparison.OrdinalIgnoreCase)) {
1057                         string dir = Path.GetDirectoryName (path);
1058                         int idx = dir.LastIndexOf (Path.DirectorySeparatorChar);
1059                         if (idx >= 0) {
1060                                 name = dir.Substring (idx + 1) + Path.DirectorySeparatorChar + name;
1061                                 Console.WriteLine ($"Storing satellite assembly '{path}' with name '{name}'");
1062                         } else if (!quiet)
1063                                 Console.WriteLine ($"Warning: satellite assembly {path} doesn't have locale path prefix, name conflicts possible");
1064                 }
1065
1066                 return name;
1067         }
1068
1069         static bool QueueAssembly (List<string> files, string codebase)
1070         {
1071                 //Console.WriteLine ("CODE BASE IS {0}", codebase);
1072                 if (files.Contains (codebase))
1073                         return true;
1074
1075                 var path = new Uri(codebase).LocalPath;
1076                 var name = GetAssemblyName (path);
1077                 string found;
1078                 if (loaded_assemblies.TryGetValue (name, out found)) {
1079                         Error (string.Format ("Duplicate assembly name `{0}'. Both `{1}' and `{2}' use same assembly name.", name, path, found));
1080                         return false;
1081                 }
1082
1083                 loaded_assemblies.Add (name, path);
1084
1085                 files.Add (codebase);
1086                 if (!autodeps)
1087                         return true;
1088                 try {
1089                         Assembly a = universe.LoadFile (path);
1090
1091                         foreach (AssemblyName an in a.GetReferencedAssemblies ()) {
1092                                 a = universe.Load (an.FullName);
1093                                 if (!QueueAssembly (files, a.CodeBase))
1094                                         return false;
1095                         }
1096                 } catch (Exception) {
1097                         if (!skip_scan)
1098                                 throw;
1099                 }
1100
1101                 return true;
1102         }
1103
1104         static Assembly LoadAssembly (string assembly)
1105         {
1106                 Assembly a;
1107                 
1108                 try {
1109                         char[] path_chars = { '/', '\\' };
1110                         
1111                         if (assembly.IndexOfAny (path_chars) != -1) {
1112                                 a = universe.LoadFile (assembly);
1113                         } else {
1114                                 string ass = assembly;
1115                                 if (ass.EndsWith (".dll"))
1116                                         ass = assembly.Substring (0, assembly.Length - 4);
1117                                 a = universe.Load (ass);
1118                         }
1119                         return a;
1120                 } catch (FileNotFoundException){
1121                         string total_log = "";
1122                         
1123                         foreach (string dir in link_paths){
1124                                 string full_path = Path.Combine (dir, assembly);
1125                                 if (!assembly.EndsWith (".dll") && !assembly.EndsWith (".exe"))
1126                                         full_path += ".dll";
1127                                 
1128                                 try {
1129                                         a = universe.LoadFile (full_path);
1130                                         return a;
1131                                 } catch (FileNotFoundException ff) {
1132                                         total_log += ff.FusionLog;
1133                                         continue;
1134                                 }
1135                         }
1136                         Error ("Cannot find assembly `" + assembly + "'" );
1137                         if (!quiet)
1138                                 Console.WriteLine ("Log: \n" + total_log);
1139                 } catch (IKVM.Reflection.BadImageFormatException f) {
1140                         if (skip_scan)
1141                                 throw;
1142                         Error ("Cannot load assembly (bad file format) " + f.Message);
1143                 } catch (FileLoadException f){
1144                         Error ("Cannot load assembly " + f.Message);
1145                 } catch (ArgumentNullException){
1146                         Error("Cannot load assembly (null argument)");
1147                 }
1148                 return null;
1149         }
1150
1151         static void Error (string msg)
1152         {
1153                 Console.Error.WriteLine ("ERROR: " + msg);
1154                 Environment.Exit (1);
1155         }
1156
1157         static void Help ()
1158         {
1159                 Console.WriteLine ("Usage is: mkbundle [options] assembly1 [assembly2...]\n\n" +
1160                                    "Options:\n" +
1161                                    "    --config F           Bundle system config file `F'\n" +
1162                                    "    --config-dir D       Set MONO_CFG_DIR to `D'\n" +
1163                                    "    --deps               Turns on automatic dependency embedding (default on simple)\n" +
1164                                    "    -L path              Adds `path' to the search path for assemblies\n" +
1165                                    "    --machine-config F   Use the given file as the machine.config for the application.\n" +
1166                                    "    -o out               Specifies output filename\n" +
1167                                    "    --nodeps             Turns off automatic dependency embedding (default on custom)\n" +
1168                                    "    --skip-scan          Skip scanning assemblies that could not be loaded (but still embed them).\n" +
1169                                    "    --i18n ENCODING      none, all or comma separated list of CJK, MidWest, Other, Rare, West.\n" +
1170                                    "    -v                   Verbose output\n" + 
1171                                    "\n" + 
1172                                    "--simple   Simple mode does not require a C toolchain and can cross compile\n" + 
1173                                    "    --cross TARGET       Generates a binary for the given TARGET\n"+
1174                                    "    --env KEY=VALUE      Hardcodes an environment variable for the target\n" +
1175                                    "    --library [LIB,]PATH Bundles the specified dynamic library to be used at runtime\n" +
1176                                    "                         LIB is optional shortname for file located at PATH\n" + 
1177                                    "    --list-targets       Lists available targets on the remote server\n" +
1178                                    "    --local-targets      Lists locally available targets\n" +
1179                                    "    --options OPTIONS    Embed the specified Mono command line options on target\n" +
1180                                    "    --runtime RUNTIME    Manually specifies the Mono runtime to use\n" +
1181                                    "    --target-server URL  Specified a server to download targets from, default is " + target_server + "\n" +
1182                                    "\n" +
1183                                    "--custom   Builds a custom launcher, options for --custom\n" +
1184                                    "    -c                   Produce stub only, do not compile\n" +
1185                                    "    -oo obj              Specifies output filename for helper object file\n" +
1186                                    "    --dos2unix[=true|false]\n" +
1187                                    "                         When no value provided, or when `true` specified\n" +
1188                                    "                         `dos2unix` will be invoked to convert paths on Windows.\n" +
1189                                    "                         When `--dos2unix=false` used, dos2unix is NEVER used.\n" +
1190                                    "    --keeptemp           Keeps the temporary files\n" +
1191                                    "    --static             Statically link to mono libs\n" +
1192                                    "    --nomain             Don't include a main() function, for libraries\n" +
1193                                    "    --custom-main C      Link the specified compilation unit (.c or .obj) with entry point/init code\n" +
1194                                    "    -z                   Compress the assemblies before embedding.\n" +
1195                                    "    --static-ctor ctor   Add a constructor call to the supplied function.\n" +
1196                                    "                         You need zlib development headers and libraries.\n");
1197         }
1198
1199         [DllImport ("libc")]
1200         static extern int system (string s);
1201         [DllImport ("libc")]
1202         static extern int uname (IntPtr buf);
1203                 
1204         static void DetectOS ()
1205         {
1206                 if (!IsUnix) {
1207                         os_message = "OS is: Windows";
1208                         style = "windows";
1209                         return;
1210                 }
1211
1212                 IntPtr buf = Marshal.AllocHGlobal (8192);
1213                 if (uname (buf) != 0){
1214                         os_message = "Warning: Unable to detect OS";
1215                         Marshal.FreeHGlobal (buf);
1216                         return;
1217                 }
1218                 string os = Marshal.PtrToStringAnsi (buf);
1219                 os_message = "OS is: " + os;
1220                 if (os == "Darwin")
1221                         style = "osx";
1222                 
1223                 Marshal.FreeHGlobal (buf);
1224         }
1225
1226         static bool IsUnix {
1227                 get {
1228                         int p = (int) Environment.OSVersion.Platform;
1229                         return ((p == 4) || (p == 128) || (p == 6));
1230                 }
1231         }
1232
1233         static void Execute (string cmdLine)
1234         {
1235                 if (IsUnix) {
1236                         if (!quiet)
1237                                 Console.WriteLine ("[execute cmd]: " + cmdLine);
1238                         int ret = system (cmdLine);
1239                         if (ret != 0)
1240                         {
1241                                 Error(String.Format("[Fail] {0}", ret));
1242                         }
1243                         return;
1244                 }
1245
1246                 // on Windows, we have to pipe the output of a
1247                 // `cmd` interpolation to dos2unix, because the shell does not
1248                 // strip the CRLFs generated by the native pkg-config distributed
1249                 // with Mono.
1250                 //
1251                 // But if it's *not* on cygwin, just skip it.
1252
1253                 // check if dos2unix is applicable.
1254                 if (use_dos2unix == true)
1255                         try {
1256                         var info = new ProcessStartInfo ("dos2unix");
1257                         info.CreateNoWindow = true;
1258                         info.RedirectStandardInput = true;
1259                         info.UseShellExecute = false;
1260                         var dos2unix = Process.Start (info);
1261                         dos2unix.StandardInput.WriteLine ("aaa");
1262                         dos2unix.StandardInput.WriteLine ("\u0004");
1263                         dos2unix.StandardInput.Close ();
1264                         dos2unix.WaitForExit ();
1265                         if (dos2unix.ExitCode == 0)
1266                                 use_dos2unix = true;
1267                 } catch {
1268                         Console.WriteLine("Warning: dos2unix not found");
1269                         use_dos2unix = false;
1270                 }
1271
1272                 if (use_dos2unix == null)
1273                         use_dos2unix = false;
1274
1275                 ProcessStartInfo psi = new ProcessStartInfo();
1276                 psi.UseShellExecute = false;
1277
1278                 // if there is no dos2unix, just run cmd /c.
1279                 if (use_dos2unix == false)
1280                 {
1281                         psi.FileName = "cmd";
1282                         psi.Arguments = String.Format("/c \"{0}\"", cmdLine);
1283                 }
1284                 else
1285                 {
1286                         psi.FileName = "sh";
1287                         StringBuilder b = new StringBuilder();
1288                         int count = 0;
1289                         for (int i = 0; i < cmdLine.Length; i++)
1290                         {
1291                                 if (cmdLine[i] == '`')
1292                                 {
1293                                         if (count % 2 != 0)
1294                                         {
1295                                                 b.Append("|dos2unix");
1296                                         }
1297                                         count++;
1298                                 }
1299                                 b.Append(cmdLine[i]);
1300                         }
1301                         cmdLine = b.ToString();
1302                         psi.Arguments = String.Format("-c \"{0}\"", cmdLine);
1303                 }
1304
1305                 if (!quiet)
1306                         Console.WriteLine(cmdLine);
1307                 using (Process p = Process.Start (psi)) {
1308                         p.WaitForExit ();
1309                         int ret = p.ExitCode;
1310                         if (ret != 0){
1311                                 Error (String.Format("[Fail] {0}", ret));
1312                         }
1313                 }
1314         }
1315
1316         static string GetEnv(string name, string defaultValue)
1317         {
1318                 string val = Environment.GetEnvironmentVariable(name);
1319                 if (val != null)
1320                 {
1321                         if (!quiet)
1322                                 Console.WriteLine("{0} = {1}", name, val);
1323                 }
1324                 else
1325                 {
1326                         val = defaultValue;
1327                         if (!quiet)
1328                                 Console.WriteLine("{0} = {1} (default)", name, val);
1329                 }
1330                 return val;
1331         }
1332
1333         static string LocateFile(string default_path)
1334         {
1335                 var override_path = Path.Combine(Directory.GetCurrentDirectory(), Path.GetFileName(default_path));
1336                 if (File.Exists(override_path))
1337                         return override_path;
1338                 else if (File.Exists(default_path))
1339                         return default_path;
1340                 else
1341                         throw new FileNotFoundException(default_path);
1342         }
1343 }