2007-09-05 Marek Safar <marek.safar@gmail.com>
[mono.git] / tools / locale-builder / Driver.cs
index 9d1b2de2efa1ea1afa7c9470cf29c0404ca36830..bcdf6c861a5a364f1af2bdae435091b39609dc3b 100644 (file)
@@ -3,8 +3,9 @@
 //
 // Author(s):
 //  Jackson Harper (jackson@ximian.com)
+//  Atsushi Enomoto (atsushi@ximian.com)
 //
-// (C) 2004 Novell, Inc (http://www.novell.com)
+// (C) 2004-2005 Novell, Inc (http://www.novell.com)
 //
 
 
@@ -46,7 +47,10 @@ namespace Mono.Tools.LocaleBuilder {
                 private ArrayList cultures;
                 private Hashtable langs;
                 private Hashtable currency_types;
-                
+                private Hashtable regions;
+
+               private XPathDocument lcids_doc;
+
                // The lang is the language that display names will be displayed in
                public string Lang {
                        get {
@@ -73,22 +77,81 @@ namespace Mono.Tools.LocaleBuilder {
 
                public void Run ()
                {
+                       lcids_doc = GetXPathDocument ("lcids.xml");
+
                        Regex locales_regex = null;
                        if (Locales != null)
                                locales_regex = new Regex (Locales);
 
                         langs = new Hashtable ();
                         cultures = new ArrayList ();
+                        regions = new Hashtable ();
+
+                       LookupRegions ();
 
                         LookupCurrencyTypes ();
 
                        foreach (string file in Directory.GetFiles ("locales", "*.xml")) {
                                string fn = Path.GetFileNameWithoutExtension (file);
+                               if (fn == "hy_AM")
+                                       continue; // see bug #75499
                                if (locales_regex == null || locales_regex.IsMatch (fn)) {
                                        ParseLocale (fn);
                                 }
                        }
 
+                       /* FIXME: This is hacky.
+                        * Since there is only langs/zh.xml while there are
+                        * two "zh" languages (CHS and CHT), there should be
+                        * different language profiles and we are not likely
+                        * to add lang/* files. So here I just clone zh-CHS
+                        * as zh-CHT
+                        */
+                        foreach (CultureInfoEntry e in cultures) {
+                               if (e.Name == "zh-CHS") {
+                                       CultureInfoEntry t =
+                                               CultureInfoEntry.ShallowCopy (e);
+                                       t.Language = "zh-CHT";
+                                       LookupLcids (t, true);
+                                       cultures.Add (t);
+                                       break;
+                               }
+                        }
+
+                       ArrayList regionList = new ArrayList (regions.Values);
+                       regionList.Sort (RegionComparer.Instance);
+                       int number = 0;
+                       foreach (RegionInfoEntry r in regionList)
+                               r.RegionId = number++;
+
+                       foreach (CultureInfoEntry e in cultures) {
+                               int lcid = int.Parse (e.Lcid.Substring (2),
+                                       NumberStyles.HexNumber);
+                               int idx;
+                               int start = e.Name.IndexOf ('-') + 1;
+                               if (start == 0)
+                                       continue;
+                               for (idx = start; idx < e.Name.Length; idx++)
+                                       if (!Char.IsLetter (e.Name [idx]))
+                                               break;
+                               if (start == idx) {
+                                       Console.Error.WriteLine ("Culture {0} {1} is not mappable to Region.", e.Lcid, e.Name);
+                                       continue;
+                               }
+                               string name = e.Name.Substring (start, idx - start);
+                               RegionInfoEntry rm = null;
+                               foreach (RegionInfoEntry r in regions.Values)
+                                       if (r.ISO2Name == name) {
+                                               rm = r;
+                                               break;
+                                       }
+                               if (rm == null) {
+                                       Console.Error.WriteLine ("No definition for region {0}", name);
+                                       continue;
+                               }
+                               e.RegionId = rm.RegionId;
+                       }
+
                         /**
                          * Dump each table individually. Using StringBuilders
                          * because it is easier to debug, should switch to just
@@ -103,6 +166,7 @@ namespace Mono.Tools.LocaleBuilder {
                                 writer.WriteLine ("\n");
 
                                 writer.WriteLine ("#define NUM_CULTURE_ENTRIES " + cultures.Count);
+                                writer.WriteLine ("#define NUM_REGION_ENTRIES " + regionList.Count);
                                 writer.WriteLine ("\n");
 
                                 // Sort the cultures by lcid
@@ -173,6 +237,32 @@ namespace Mono.Tools.LocaleBuilder {
                                 writer.Write (builder);
                                 writer.WriteLine ("};\n\n");
 
+                               builder = new StringBuilder ();
+                               int rcount = 0;
+                               foreach (RegionInfoEntry r in regionList) {
+                                       r.AppendTableRow (builder);
+                                       if (++rcount != regionList.Count)
+                                               builder.Append (',');
+                                       builder.Append ('\n');
+                               }
+                               writer.WriteLine ("static const RegionInfoEntry region_entries [] = {");
+                               writer.Write (builder);
+                               writer.WriteLine ("};\n\n");
+
+                                builder = new StringBuilder ();
+                               rcount = 0;
+                               foreach (RegionInfoEntry ri in regionList) {
+                                        builder.Append ("\t{" + Entry.EncodeStringIdx (ri.ISO2Name) + ", ");
+                                        builder.Append (ri.RegionId + "}");
+                                        if (++rcount < regionList.Count)
+                                                builder.Append (',');
+                                        builder.Append ('\n');
+                                }
+
+                                writer.WriteLine ("static const RegionInfoNameEntry region_name_entries [] = {");
+                                writer.Write (builder);
+                                writer.WriteLine ("};\n\n");
+
                                 writer.WriteLine ("static const char locale_strings [] = {");
                                 writer.Write (Entry.GetStrings ());
                                 writer.WriteLine ("};\n\n");
@@ -183,9 +273,15 @@ namespace Mono.Tools.LocaleBuilder {
 
                private XPathDocument GetXPathDocument (string path)
                {
-                       XmlTextReader xtr = new XmlTextReader (path);
-                       xtr.XmlResolver = null;
-                       return new XPathDocument (xtr);
+                       XmlTextReader xtr = null;
+                       try {
+                               xtr = new XmlTextReader (path);
+                               xtr.XmlResolver = null;
+                               return new XPathDocument (xtr);
+                       } finally {
+                               if (xtr != null)
+                                       xtr.Close ();
+                       }
                }
 
                private string GetShortName (string lang)
@@ -250,8 +346,8 @@ namespace Mono.Tools.LocaleBuilder {
                         if (ci == null)
                                 return;
 
-                        if (langs [ci.Language] == null) {
-                                if (!ParseLang (ci.Language)) // If we can't parse the lang we cant have the locale
+                        if (langs [GetLanguageFixed (ci)] == null) {
+                                if (!ParseLang (GetLanguageFixed (ci))) // If we can't parse the lang we cant have the locale
                                         return;
                         }
 
@@ -260,7 +356,10 @@ namespace Mono.Tools.LocaleBuilder {
 
                private CultureInfoEntry LookupCulture (string locale)
                {
-                        XPathDocument doc = GetXPathDocument (Path.Combine ("locales", locale + ".xml"));
+                       string path = Path.Combine ("locales", locale + ".xml");
+                       if (!File.Exists (path))
+                               return null;
+                        XPathDocument doc = GetXPathDocument (path);
                         XPathNavigator nav = doc.CreateNavigator ();
                        CultureInfoEntry ci = new CultureInfoEntry ();
                        string supp;
@@ -292,11 +391,11 @@ namespace Mono.Tools.LocaleBuilder {
                         nav = doc.CreateNavigator ();
                         Lookup (nav, ci);
 
-                       doc = GetXPathDocument (Path.Combine ("langs", GetShortName (ci.Language) + ".xml"));
+                       doc = GetXPathDocument (Path.Combine ("langs", GetShortName (GetLanguageFixed (ci)) + ".xml"));
                        nav = doc.CreateNavigator ();
                        Lookup (nav, ci);
 
-                       supp = Path.Combine ("supp", ci.Language + ".xml");
+                       supp = Path.Combine ("supp", GetLanguageFixed (ci) + ".xml");
                        if (File.Exists (supp)) {
                                doc = GetXPathDocument (supp);
                                nav = doc.CreateNavigator ();
@@ -323,6 +422,19 @@ namespace Mono.Tools.LocaleBuilder {
                        LookupNumberInfo (nav, ci);
                }
 
+               private string GetLanguageFixed (CultureInfoEntry ci)
+               {
+                       // This is a hack, but without it nb-NO and nn-NO won't work.
+                       if (ci.Territory == "NO") {
+                               switch (ci.Language) {
+                               case "nb":
+                               case "nn":
+                                       return "no";
+                               }
+                       }
+                       return ci.Language;
+               }
+
                private void LookupNames (CultureInfoEntry ci)
                {
                        XPathDocument doc = GetXPathDocument (Path.Combine ("langs", GetShortName (Lang) + ".xml"));
@@ -341,12 +453,22 @@ namespace Mono.Tools.LocaleBuilder {
                        if (ci.Language == Lang) {
                                ci.NativeName = ci.DisplayName;
                        } else {
-                               doc = GetXPathDocument (Path.Combine ("langs", GetShortName (ci.Language) + ".xml"));
+                               // FIXME: We use ci.Language here.
+                               // This is nothing more than hack for nb-NO and nn-NO
+                               // where Parent of them is nn (not nb or nn).
+                               string lang = ci.Language;
+                               doc = GetXPathDocument (Path.Combine ("langs", GetShortName (lang) + ".xml"));
                                nav = doc.CreateNavigator ();
                                ci.NativeName = LookupFullName (ci, nav);
                        }
                }
 
+               private void AddPattern (ArrayList al, string pattern)
+               {
+                       if (!al.Contains (pattern))
+                               al.Add (pattern);
+               }
+
                private void LookupDateTimeInfo (XPathNavigator nav, CultureInfoEntry ci)
                {
                        /**
@@ -442,10 +564,13 @@ namespace Mono.Tools.LocaleBuilder {
                                                ext = df_nav.Select ("extraPatterns/pattern");
                                                if (ext.MoveNext ()) {
                                                        df.LongDatePatterns.Clear ();
+                                                       AddPattern (df.LongDatePatterns, df.LongDatePattern);
                                                        do {
                                                                df.LongDatePatterns.Add (ext.Current.Value);
                                                        } while (ext.MoveNext ());
                                                }
+                                               else
+                                                       AddPattern (df.LongDatePatterns, df.LongDatePattern);
                                                break;
                                        case "short":
                                                if (value != null)
@@ -453,10 +578,13 @@ namespace Mono.Tools.LocaleBuilder {
                                                ext = df_nav.Select ("extraPatterns/pattern");
                                                if (ext.MoveNext ()) {
                                                        df.ShortDatePatterns.Clear ();
+                                                       AddPattern (df.ShortDatePatterns, df.ShortDatePattern);
                                                        do {
                                                                df.ShortDatePatterns.Add (ext.Current.Value);
                                                        } while (ext.MoveNext ());
                                                }
+                                               else
+                                                       AddPattern (df.ShortDatePatterns, df.ShortDatePattern);
                                                break;
                                        case "year_month":
                                                if (value != null)
@@ -484,10 +612,13 @@ namespace Mono.Tools.LocaleBuilder {
                                                ext = df_nav.Select ("extraPatterns/pattern");
                                                if (ext.MoveNext ()) {
                                                        df.LongTimePatterns.Clear ();
+                                                       AddPattern (df.LongTimePatterns, df.LongTimePattern);
                                                        do {
                                                                df.LongTimePatterns.Add (ext.Current.Value);
                                                        } while (ext.MoveNext ());
                                                }
+                                               else
+                                                       AddPattern (df.LongTimePatterns, df.LongTimePattern);
                                                break;
                                        case "short":
                                                if (value != null)
@@ -495,10 +626,13 @@ namespace Mono.Tools.LocaleBuilder {
                                                ext = df_nav.Select ("extraPatterns/pattern");
                                                if (ext.MoveNext ()) {
                                                        df.ShortTimePatterns.Clear ();
+                                                       AddPattern (df.ShortTimePatterns, df.ShortTimePattern);
                                                        do {
                                                                df.ShortTimePatterns.Add (ext.Current.Value);
                                                        } while (ext.MoveNext ());
                                                }
+                                               else
+                                                       AddPattern (df.ShortTimePatterns, df.ShortTimePattern);
                                                break;
                                        }
                                }
@@ -821,12 +955,11 @@ namespace Mono.Tools.LocaleBuilder {
 
                private bool LookupLcids (CultureInfoEntry ci, bool lang)
                {
-                       XPathDocument doc = GetXPathDocument ("lcids.xml");
-                       XPathNavigator nav = doc.CreateNavigator ();
+                       XPathNavigator nav = lcids_doc.CreateNavigator ();
                        string name = ci.Name;
                        // Language name does not always consist of locale name.
                        // (for zh-* it must be either zh-CHS or zh-CHT)
-                       string langName = ci.Language;
+                       string langName = GetLanguageFixed (ci);
 
 //                        if (ci.Territory != null)
 //                                name += "-" + ci.Territory;
@@ -839,8 +972,8 @@ namespace Mono.Tools.LocaleBuilder {
 
                                 if (ci.Territory != null) {
                                         file = Path.Combine ("locales", ci.Language + "_" + ci.Territory + ".xml");
-                                        File.Delete (file);
                                         Console.WriteLine ("deleting file:  " + file);
+                                        File.Delete (file);
                                 }
 
                                return false;
@@ -873,6 +1006,9 @@ namespace Mono.Tools.LocaleBuilder {
                        string pre = "ldml/localeDisplayNames/";
                        string ret;
 
+                       // FIXME: We use ci.Language here.
+                       // This is nothing more than hack for nb-NO or nn-NO
+                       // where Parent of them is nn (not nb or nn).
                        ret = (string) nav.Evaluate ("string("+
                                        pre + "languages/language[@type='" + GetShortName (ci.Language) + "'])");
 
@@ -884,6 +1020,42 @@ namespace Mono.Tools.LocaleBuilder {
                        return ret;
                }
 
+               private void LookupRegions ()
+               {
+                        XPathDocument doc = GetXPathDocument ("supplementalData.xml");
+                       XPathNavigator nav = doc.CreateNavigator ();
+                       XPathNodeIterator ni = nav.Select ("supplementalData/currencyData/region");
+                       while (ni.MoveNext ()) {
+                               string territory = (string) ni.Current.GetAttribute ("iso3166", String.Empty);
+                                string currency = (string) ni.Current.Evaluate ("string(currency/@iso4217)");
+                               RegionInfoEntry region = new RegionInfoEntry ();
+                               region.ISO2Name = territory.ToUpper ();
+                               region.ISOCurrencySymbol = currency;
+                               regions [territory] = region;
+                       }
+
+                        doc = GetXPathDocument ("langs/en.xml");
+                       nav = doc.CreateNavigator ();
+                       ni = nav.Select ("/ldml/localeDisplayNames/territories/territory");
+                       while (ni.MoveNext ()) {
+                               RegionInfoEntry r = (RegionInfoEntry)
+                                       regions [ni.Current.GetAttribute ("type", "")];
+                               if (r == null)
+                                       continue;
+                               r.EnglishName = ni.Current.Value;
+                       }
+
+                       Hashtable curNames = new Hashtable ();
+                       ni = nav.Select ("/ldml/numbers/currencies/currency");
+                       while (ni.MoveNext ())
+                               curNames [ni.Current.GetAttribute ("type", "")] =
+                                       ni.Current.Evaluate ("string (displayName)");
+
+                       foreach (RegionInfoEntry r in regions.Values)
+                               r.CurrencyEnglishName =
+                                       (string) curNames [r.ISOCurrencySymbol];
+               }
+
                 private void LookupCurrencyTypes ()
                 {
                         XPathDocument doc = GetXPathDocument ("supplementalData.xml");
@@ -899,7 +1071,7 @@ namespace Mono.Tools.LocaleBuilder {
                        }
                 }
 
-                static string control_chars = "ghmsftz";
+                static string control_chars = "eghmsftz";
 
                 // HACK: We are trying to build year_month and month_day patterns from the full pattern.
                 private void ParseFullDateFormat (DateTimeFormatEntry df, string full)
@@ -909,8 +1081,9 @@ namespace Mono.Tools.LocaleBuilder {
                         string year_month = String.Empty;
                         bool in_month_data = false;
                         bool in_year_data = false;
-                        int month_end = 0;
-                        int year_end = 0;
+                       int day_start = 0, day_end = 0;
+                        int month_start = 0, month_end = 0;
+                        int year_start = 0, year_end = 0;
                        bool inquote = false;
                         
                         for (int i = 0; i < full.Length; i++) {
@@ -920,18 +1093,24 @@ namespace Mono.Tools.LocaleBuilder {
                                         year_month += c;
                                         in_year_data = true;
                                         in_month_data = true;
-                                        month_end = month_day.Length;
+                                       if (month_start == 0)
+                                               month_start = i;
+                                        month_end = i;
                                         year_end = year_month.Length;
                                 } else if (!inquote && Char.ToLower (c) == 'd') {
                                         month_day += c;
                                         in_month_data = true;
                                         in_year_data = false;
-                                        month_end = month_day.Length;
+                                       if (day_start == 0)
+                                               day_start = i;
+                                        day_end = i;
                                 } else if (!inquote && Char.ToLower (c) == 'y') {
                                         year_month += c;
                                         in_year_data = true;
                                         in_month_data = false;
-                                        year_end = year_month.Length;
+                                       if (year_start == 0)
+                                               year_start = i;
+                                        year_end = i;
                                 } else if (!inquote && control_chars.IndexOf (Char.ToLower (c)) >= 0) {
                                         in_year_data = false;
                                         in_month_data = false;
@@ -948,15 +1127,35 @@ namespace Mono.Tools.LocaleBuilder {
                         }
 
                         if (month_day != String.Empty) {
-                                month_day = month_day.Substring (0, month_end);
-                                df.MonthDayPattern = month_day;
+                                //month_day = month_day.Substring (0, month_end);
+                                df.MonthDayPattern = TrimPattern (month_day);
                         }
                         if (year_month != String.Empty) {
-                                year_month = year_month.Substring (0, year_end);
-                                df.YearMonthPattern = year_month;
+                                //year_month = year_month.Substring (0, year_end);
+                                df.YearMonthPattern = TrimPattern (year_month);
                         }
                 }
 
+               string TrimPattern (string p)
+               {
+                       int idx = 0;
+                       p = p.Trim ().TrimEnd (',');
+                       idx = p.LastIndexOf ("' de '"); // spanish dates
+                       if (idx > 0)
+                               p = p.Substring (0, idx);
+                       idx = p.LastIndexOf ("' ta '"); // finnish
+                       if (idx > 0)
+                               p = p.Substring (0, idx);
+                       idx = p.LastIndexOf ("'ren'"); // euskara
+                       if (idx > 0)
+                               p = p.Replace ("'ren'", "").Trim ();
+                       idx = p.LastIndexOf ("'a'"); // estonian
+                       if (idx > 0)
+                               p = p.Substring (0, idx);
+
+                       return p.Replace ("'ta '", "'ta'"); // finnish
+               }
+
                 private class LcidComparer : IComparer {
 
                         public int Compare (object a, object b)
@@ -975,9 +1174,34 @@ namespace Mono.Tools.LocaleBuilder {
                                 CultureInfoEntry aa = (CultureInfoEntry) a;
                                 CultureInfoEntry bb = (CultureInfoEntry) b;
 
-                                return aa.Name.ToLower ().CompareTo (bb.Name.ToLower ());
+                                return String.CompareOrdinal(aa.Name.ToLower (), bb.Name.ToLower ());
                         }
                 }
+
+               class RegionComparer : IComparer
+               {
+                       public static RegionComparer Instance = new RegionComparer ();
+                       
+                       public int Compare (object o1, object o2)
+                       {
+                               RegionInfoEntry r1 = (RegionInfoEntry) o1;
+                               RegionInfoEntry r2 = (RegionInfoEntry) o2;
+                               return String.CompareOrdinal (
+                                       r1.ISO2Name, r2.ISO2Name);
+                       }
+               }
+
+               class RegionLCIDMap
+               {
+                       public RegionLCIDMap (int lcid, int regionId)
+                       {
+                               LCID = lcid;
+                               RegionId = regionId;
+                       }
+
+                       public int LCID;
+                       public int RegionId;
+               }
         }
 }