mcs/class/corlib/Mono.Globalization.Unicode/MSCompatUnicodeTable.cs

   1 #define USE_MANAGED_RESOURCE
   2 //#define USE_C_HEADER
   3
   4 //
   5 // MSCompatUnicodeTable.cs : Handles Windows-like sortket tables.
   6 //
   7 // Author:
   8 //      Atsushi Enomoto  <atsushi@ximian.com>
   9 //
  10 // Copyright (C) 2005 Novell, Inc (http://www.novell.com)
  11 //
  12 // Permission is hereby granted, free of charge, to any person obtaining
  13 // a copy of this software and associated documentation files (the
  14 // "Software"), to deal in the Software without restriction, including
  15 // without limitation the rights to use, copy, modify, merge, publish,
  16 // distribute, sublicense, and/or sell copies of the Software, and to
  17 // permit persons to whom the Software is furnished to do so, subject to
  18 // the following conditions:
  19 //
  20 // The above copyright notice and this permission notice shall be
  21 // included in all copies or substantial portions of the Software.
  22 //
  23 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  27 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  28 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  29 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  30 //
  31
  32 using System;
  33 using System.Collections;
  34 using System.Globalization;
  35 using System.Reflection;
  36 using System.Runtime.CompilerServices;
  37 using System.Runtime.InteropServices;
  38
  39 using UUtil = Mono.Globalization.Unicode.MSCompatUnicodeTableUtil;
  40
  41 namespace Mono.Globalization.Unicode
  42 {
  43         internal class TailoringInfo
  44         {
  45                 public readonly int LCID;
  46                 public readonly int TailoringIndex;
  47                 public readonly int TailoringCount;
  48                 public readonly bool FrenchSort;
  49
  50                 public TailoringInfo (int lcid, int tailoringIndex, int tailoringCount, bool frenchSort)
  51                 {
  52                         LCID = lcid;
  53                         TailoringIndex = tailoringIndex;
  54                         TailoringCount = tailoringCount;
  55                         FrenchSort = frenchSort;
  56                 }
  57         }
  58
  59         #region Tailoring support classes
  60         // Possible mapping types are:
  61         //
  62         //      - string to string (ReplacementMap)
  63         //      - string to SortKey (SortKeyMap)
  64         //      - diacritical byte to byte (DiacriticalMap)
  65         //
  66         // There could be mapping from string to sortkeys, but
  67         // for now there is none as such.
  68         //
  69         internal class Contraction
  70         {
  71                 public int Index;
  72                 public readonly char [] Source;
  73                 // only either of them is used.
  74                 public readonly string Replacement;
  75                 public readonly byte [] SortKey;
  76
  77                 public Contraction (int index, char [] source,
  78                         string replacement, byte [] sortkey)
  79                 {
  80                         Index = index;
  81                         Source = source;
  82                         Replacement = replacement;
  83                         SortKey = sortkey;
  84                 }
  85         }
  86
  87         internal class ContractionComparer : IComparer
  88         {
  89                 public static readonly ContractionComparer Instance =
  90                         new ContractionComparer ();
  91
  92                 public int Compare (object o1, object o2)
  93                 {
  94                         Contraction c1 = (Contraction) o1;
  95                         Contraction c2 = (Contraction) o2;
  96                         char [] a1 = c1.Source;
  97                         char [] a2 = c2.Source;
  98                         int min = a1.Length > a2.Length ?
  99                                 a2.Length : a1.Length;
 100                         for (int i = 0; i < min; i++)
 101                                 if (a1 [i] != a2 [i])
 102                                         return a1 [i] - a2 [i];
 103                         if (a1.Length != a2.Length)
 104                                 return a1.Length - a2.Length;
 105                         // This makes the sorting stable, since we are using Array.Sort () which is
 106                         // not stable
 107                         return c1.Index - c2.Index;
 108                 }
 109         }
 110
 111         internal class Level2Map
 112         {
 113                 public byte Source;
 114                 public byte Replace;
 115
 116                 public Level2Map (byte source, byte replace)
 117                 {
 118                         Source = source;
 119                         Replace = replace;
 120                 }
 121         }
 122
 123         internal class Level2MapComparer : IComparer
 124         {
 125                 public static readonly Level2MapComparer Instance =
 126                         new Level2MapComparer ();
 127
 128                 public int Compare (object o1, object o2)
 129                 {
 130                         Level2Map m1 = (Level2Map) o1;
 131                         Level2Map m2 = (Level2Map) o2;
 132                         return (m1.Source - m2.Source);
 133                 }
 134         }
 135
 136         #endregion
 137
 138         unsafe internal class MSCompatUnicodeTable
 139         {
 140                 public static int MaxExpansionLength = 3;
 141
 142                 static readonly byte* ignorableFlags;
 143                 static readonly byte* categories;
 144                 static readonly byte* level1;
 145                 static readonly byte* level2;
 146                 static readonly byte* level3;
 147 //              static readonly ushort* widthCompat;
 148 #if USE_C_HEADER
 149                 static readonly char* tailoring;
 150 #endif
 151                 static byte* cjkCHScategory;
 152                 static byte* cjkCHTcategory;
 153                 static byte* cjkJAcategory;
 154                 static byte* cjkKOcategory;
 155                 static byte* cjkCHSlv1;
 156                 static byte* cjkCHTlv1;
 157                 static byte* cjkJAlv1;
 158                 static byte* cjkKOlv1;
 159                 static byte* cjkKOlv2;
 160
 161                 const int ResourceVersionSize = 1;
 162
 163                 public static TailoringInfo GetTailoringInfo (int lcid)
 164                 {
 165                         for (int i = 0; i < tailoringInfos.Length; i++)
 166                                 if (tailoringInfos [i].LCID == lcid)
 167                                         return tailoringInfos [i];
 168                         return null;
 169                 }
 170
 171                 unsafe public static void BuildTailoringTables (CultureInfo culture,
 172                         TailoringInfo t,
 173                         ref Contraction [] contractions,
 174                         ref Level2Map [] diacriticals)
 175                 {
 176                         // collect tailoring entries.
 177                         ArrayList cmaps = new ArrayList ();
 178                         ArrayList dmaps = new ArrayList ();
 179                         int iindex = 0;
 180                         fixed (char* tarr = tailoringArr){
 181                                 int idx = t.TailoringIndex;
 182                                 int end = idx + t.TailoringCount;
 183                                 while (idx < end) {
 184                                         int ss = idx + 1;
 185                                         char [] src = null;
 186                                         switch (tarr [idx]) {
 187                                         case '\x1': // SortKeyMap
 188                                                 idx++;
 189                                                 while (tarr [ss] != 0)
 190                                                         ss++;
 191                                                 src = new char [ss - idx];
 192                                                 //                                      Array.Copy (tarr, idx, src, 0, ss - idx);
 193                                                 Marshal.Copy ((IntPtr) (tarr + idx), src, 0, ss - idx);
 194                                                 byte [] sortkey = new byte [4];
 195                                                 for (int i = 0; i < 4; i++)
 196                                                         sortkey [i] = (byte) tarr [ss + 1 + i];
 197                                                 cmaps.Add (new Contraction (iindex,
 198                                                                             src, null, sortkey));
 199                                                 // it ends with 0
 200                                                 idx = ss + 6;
 201                                                 iindex ++;
 202                                                 break;
 203                                         case '\x2': // DiacriticalMap
 204                                                 dmaps.Add (new Level2Map (
 205                                                                           (byte) tarr [idx + 1],
 206                                                                           (byte) tarr [idx + 2]));
 207                                                 idx += 3;
 208                                                 break;
 209                                         case '\x3': // ReplacementMap
 210                                                 idx++;
 211                                                 while (tarr [ss] != 0)
 212                                                         ss++;
 213                                                 src = new char [ss - idx];
 214                                                 //                                      Array.Copy (tarr, idx, src, 0, ss - idx);
 215                                                 Marshal.Copy ((IntPtr) (tarr + idx), src, 0, ss - idx);
 216                                                 ss++;
 217                                                 int l = ss;
 218                                                 while (tarr [l] != 0)
 219                                                         l++;
 220                                                 string r = new string (tarr, ss, l - ss);
 221                                                 cmaps.Add (new Contraction (iindex,
 222                                                                             src, r, null));
 223                                                 idx = l + 1;
 224                                                 iindex ++;
 225                                                 break;
 226                                         default:
 227                                                 throw new NotImplementedException (String.Format ("Mono INTERNAL ERROR (Should not happen): Collation tailoring table is broken for culture {0} ({1}) at 0x{2:X}", culture.LCID, culture.Name, idx));
 228                                         }
 229                                 }
 230                         }
 231                         cmaps.Sort (ContractionComparer.Instance);
 232                         dmaps.Sort (Level2MapComparer.Instance);
 233                         contractions = cmaps.ToArray (typeof (Contraction))
 234                                 as Contraction [];
 235                         diacriticals = dmaps.ToArray (typeof (Level2Map))
 236                                 as Level2Map [];
 237                 }
 238
 239                 static void SetCJKReferences (string name,
 240                         ref CodePointIndexer cjkIndexer,
 241                         ref byte* catTable, ref byte* lv1Table,
 242                         ref CodePointIndexer lv2Indexer, ref byte* lv2Table)
 243                 {
 244                         // as a part of mscorlib.dll, this invocation is
 245                         // somewhat extraneous (pointers were already assigned).
 246
 247                         switch (name) {
 248                         case "zh-CHS":
 249                                 catTable = cjkCHScategory;
 250                                 lv1Table = cjkCHSlv1;
 251                                 cjkIndexer = UUtil.CjkCHS;
 252                                 break;
 253                         case "zh-CHT":
 254                                 catTable = cjkCHTcategory;
 255                                 lv1Table = cjkCHTlv1;
 256                                 cjkIndexer = UUtil.Cjk;
 257                                 break;
 258                         case "ja":
 259                                 catTable = cjkJAcategory;
 260                                 lv1Table = cjkJAlv1;
 261                                 cjkIndexer = UUtil.Cjk;
 262                                 break;
 263                         case "ko":
 264                                 catTable = cjkKOcategory;
 265                                 lv1Table = cjkKOlv1;
 266                                 lv2Table = cjkKOlv2;
 267                                 cjkIndexer = UUtil.Cjk;
 268                                 lv2Indexer = UUtil.Cjk;
 269                                 break;
 270                         }
 271                 }
 272
 273                 public static byte Category (int cp)
 274                 {
 275                         return categories [UUtil.Category.ToIndex (cp)];
 276                 }
 277
 278                 public static byte Level1 (int cp)
 279                 {
 280                         return level1 [UUtil.Level1.ToIndex (cp)];
 281                 }
 282
 283                 public static byte Level2 (int cp)
 284                 {
 285                         return level2 [UUtil.Level2.ToIndex (cp)];
 286                 }
 287
 288                 public static byte Level3 (int cp)
 289                 {
 290                         return level3 [UUtil.Level3.ToIndex (cp)];
 291                 }
 292
 293                 public static bool IsSortable (string s)
 294                 {
 295                         foreach (char c in s)
 296                                 if (!IsSortable (c))
 297                                         return false;
 298                         return true;
 299                 }
 300
 301                 public static bool IsSortable (int cp)
 302                 {
 303                         // LAMESPEC: they should strictly match with
 304                         // IsIgnorable() result, but sometimes it does not.
 305                         if (!IsIgnorable (cp))
 306                                 return true;
 307                         switch (cp) {
 308                         case 0:
 309                         case 0x0640:
 310                         case 0xFEFF:
 311                                 return true;
 312                         }
 313                         return 0x180B <= cp && cp <= 0x180E ||
 314                                 0x200C <= cp && cp <= 0x200F ||
 315                                 0x202A <= cp && cp <= 0x202E ||
 316                                 0x206A <= cp && cp <= 0x206F ||
 317                                 0x200C <= cp && cp <= 0x200F ||
 318                                 0xFFF9 <= cp && cp <= 0xFFFD;
 319                 }
 320
 321                 public static bool IsIgnorable (int cp)
 322                 {
 323                         return IsIgnorable (cp, 1);
 324                 }
 325
 326                 public static bool IsIgnorable (int cp, byte flag)
 327                 {
 328                         if (cp == 0)
 329                                 return true;
 330                         if ((flag & 1) != 0) {
 331                                 UnicodeCategory uc = Char.GetUnicodeCategory ((char) cp);
 332                                 // This check eliminates some extraneous code areas
 333                                 if (uc == UnicodeCategory.OtherNotAssigned)
 334                                         return true;
 335                                 // Some characters in Surrogate area are ignored.
 336                                 if (0xD880 <= cp && cp < 0xDB80)
 337                                         return true;
 338                         }
 339                         int i = UUtil.Ignorable.ToIndex (cp);
 340                         return i >= 0 && (ignorableFlags [i] & flag) != 0;
 341                 }
 342                 // Verifier:
 343                 // for (int i = 0; i <= char.MaxValue; i++)
 344                 //      if (Char.GetUnicodeCategory ((char) i)
 345                 //              == UnicodeCategory.OtherNotAssigned
 346                 //              && ignorableFlags [i] != 7)
 347                 //              Console.WriteLine ("{0:X04}", i);
 348
 349                 public static bool IsIgnorableSymbol (int cp)
 350                 {
 351                         return IsIgnorable (cp, 2);
 352 //                      int i = UUtil.Ignorable.ToIndex (cp);
 353 //                      return i >= 0 && (ignorableFlags [i] & 0x2) != 0;
 354                 }
 355
 356                 public static bool IsIgnorableNonSpacing (int cp)
 357                 {
 358                         return IsIgnorable (cp, 4);
 359 //                      int i = UUtil.Ignorable.ToIndex (cp);
 360 //                      return i >= 0 && (ignorableFlags [i] & 0x4) != 0;
 361
 362                         // It could be implemented this way, but the above
 363                         // is faster.
 364 //                      return categories [UUtil.Category.ToIndex (cp)] == 1;
 365                 }
 366
 367                 public static int ToKanaTypeInsensitive (int i)
 368                 {
 369                         // Note that IgnoreKanaType does not treat half-width
 370                         // katakana as equivalent to full-width ones.
 371
 372                         // Thus, it is so simple ;-)
 373                         return (0x3041 <= i && i <= 0x3094) ? i + 0x60 : i;
 374                 }
 375
 376                 // Note that currently indexer optimizes this table a lot,
 377                 // which might have resulted in bugs.
 378                 public static int ToWidthCompat (int i)
 379                 {
 380                         if (i < 0x2190)
 381                                 return i;
 382                         if (i > 0xFF00) {
 383                                 if (i <= 0xFF5E)
 384                                         return i - 0xFF00 + 0x20;
 385                                 switch (i) {
 386                                 case 0xFFE0:
 387                                         return 0xA2;
 388                                 case 0xFFE1:
 389                                         return 0xA3;
 390                                 case 0xFFE2:
 391                                         return 0xAC;
 392                                 case 0xFFE3:
 393                                         return 0xAF;
 394                                 case 0xFFE4:
 395                                         return 0xA6;
 396                                 case 0xFFE5:
 397                                         return 0xA5;
 398                                 case 0xFFE6:
 399                                         return 0x20A9;
 400                                 }
 401                         }
 402                         if (i > 0x32FE)
 403                                 return i;
 404
 405                         if (i <= 0x2193)
 406                                 return 0xFFE9 - 0x2190 + i;
 407                         if (i < 0x2502)
 408                                 return i;
 409                         if (i <= 0x25CB) {
 410                                 switch (i) {
 411                                 case 0x2502:
 412                                         return 0xFFE8;
 413                                 case 0x25A0:
 414                                         return 0xFFED;
 415                                 case 0x25CB:
 416                                         return 0xFFEE;
 417                                 default:
 418                                         return i;
 419                                 }
 420                         }
 421                         if (i < 0x3000)
 422                                 return i;
 423                         if (i < 0x3131) {
 424                                 switch (i) {
 425                                 case 0x3000:
 426                                         return 0x20;
 427                                 case 0x3001:
 428                                         return 0xFF64;
 429                                 case 0x3002:
 430                                         return 0xFF61;
 431                                 case 0x300C:
 432                                         return 0xFF62;
 433                                 case 0x300D:
 434                                         return 0xFF63;
 435                                 case 0x30FB:
 436                                         return 0xFF65;
 437                                 // Other Kana compat characters' width
 438                                 // compatibility is considered in special weight.
 439                                 default:
 440                                         return i;
 441                                 }
 442                         }
 443                         if (i < 0x3164) { // Hangul compat
 444                                 return i - 0x3130 + 0xFFA0;
 445                         }
 446                         if (i == 0x3164)
 447                                 return 0xFFA0;
 448                         // 0x32D0-0x32FE are Kana compat characters, whose
 449                         // width compatibility is considered in special weight.
 450                         return i;
 451                 }
 452
 453                 #region Level 4 properties (Kana)
 454
 455                 public static bool HasSpecialWeight (char c)
 456                 {
 457                         if (c < '\u3041')
 458                                 return false;
 459                         else if ('\uFF66' <= c && c < '\uFF9E')
 460                                 return true;
 461                         else if ('\u3300' <= c)
 462                                 return false;
 463                         else if (c < '\u309D')
 464                                 return (c < '\u3099');
 465                         else if (c < '\u3100')
 466                                 return c != '\u30FB';
 467                         else if (c < '\u32D0')
 468                                 return false;
 469                         else if (c < '\u32FF')
 470                                 return true;
 471                         return false;
 472                 }
 473
 474                 // FIXME: it should be removed at some stage
 475                 // (will become unused).
 476                 public static byte GetJapaneseDashType (char c)
 477                 {
 478                         switch (c) {
 479                         case '\u309D':
 480                         case '\u309E':
 481                         case '\u30FD':
 482                         case '\u30FE':
 483                         case '\uFF70':
 484                                 return 4;
 485                         case '\u30FC':
 486                                 return 5;
 487                         }
 488                         return 3;
 489                 }
 490
 491                 public static bool IsHalfWidthKana (char c)
 492                 {
 493                         return '\uFF66' <= c && c <= '\uFF9D';
 494                 }
 495
 496                 public static bool IsHiragana (char c)
 497                 {
 498                         return '\u3041' <= c && c <= '\u3094';
 499                 }
 500
 501                 public static bool IsJapaneseSmallLetter (char c)
 502                 {
 503                         if ('\uFF67' <= c && c <= '\uFF6F')
 504                                 return true;
 505                         if ('\u3040' < c && c < '\u30FA') {
 506                                 switch (c) {
 507                                 case '\u3041':
 508                                 case '\u3043':
 509                                 case '\u3045':
 510                                 case '\u3047':
 511                                 case '\u3049':
 512                                 case '\u3063':
 513                                 case '\u3083':
 514                                 case '\u3085':
 515                                 case '\u3087':
 516                                 case '\u308E':
 517                                 case '\u30A1':
 518                                 case '\u30A3':
 519                                 case '\u30A5':
 520                                 case '\u30A7':
 521                                 case '\u30A9':
 522                                 case '\u30C3':
 523                                 case '\u30E3':
 524                                 case '\u30E5':
 525                                 case '\u30E7':
 526                                 case '\u30EE':
 527                                 case '\u30F5':
 528                                 case '\u30F6':
 529                                         return true;
 530                                 }
 531                         }
 532                         return false;
 533                 }
 534
 535                 #endregion
 536
 537 #if GENERATE_TABLE
 538
 539                 public static readonly bool IsReady = true; // always
 540
 541                 static MSCompatUnicodeTable ()
 542                 {
 543                         throw new Exception ("This code should not be used");
 544
 545                         fixed (byte* tmp = ignorableFlagsArr) {
 546                                 ignorableFlags = tmp;
 547                         }
 548                         fixed (byte* tmp = categoriesArr) {
 549                                 categories = tmp;
 550                         }
 551                         fixed (byte* tmp = level1Arr) {
 552                                 level1 = tmp;
 553                         }
 554                         fixed (byte* tmp = level2Arr) {
 555                                 level2 = tmp;
 556                         }
 557                         fixed (byte* tmp = level3Arr) {
 558                                 level3 = tmp;
 559                         }
 560 //                      fixed (ushort* tmp = widthCompatArr) {
 561 //                              widthCompat = tmp;
 562 //                      }
 563                         fixed (char* tmp = tailoringArr) {
 564                                 tailoring = tmp;
 565                         }
 566                         fixed (byte* tmp = cjkCHSArr) {
 567                                 cjkCHScategory = tmp;
 568                                 cjkCHSlv1 = tmp + cjkCHSArrLength;
 569                         }
 570                         fixed (byte* tmp = cjkCHTArr) {
 571                                 cjkCHTcategory = tmp;
 572                                 cjkCHTlv1 = tmp + cjkCHTArrLength;
 573                         }
 574                         fixed (byte* tmp = cjkJAArr) {
 575                                 cjkJAcategory = tmp;
 576                                 cjkJAlv1 = tmp + cjkJAArrLength;
 577                         }
 578                         fixed (byte* tmp = cjkKOArr) {
 579                                 cjkKOcategory = tmp;
 580                                 cjkKOlv1 = tmp + cjkKOArrLength;
 581                         }
 582                         fixed (byte* tmp = cjkKOlv2Arr) {
 583                                 cjkKOlv2 = tmp;
 584                         }
 585                 }
 586
 587                 public static void FillCJK (string name,
 588                         ref CodePointIndexer cjkIndexer,
 589                         ref byte* catTable, ref byte* lv1Table,
 590                         ref CodePointIndexer cjkLv2Indexer,
 591                         ref byte* lv2Table)
 592                 {
 593                         SetCJKReferences (name, ref cjkIndexer,
 594                                 ref catTable, ref lv1Table,
 595                                 ref cjkLv2Indexer, ref lv2Table);
 596                 }
 597 #else
 598
 599 #if !USE_C_HEADER
 600                 static readonly char [] tailoringArr;
 601 #endif
 602                 static readonly TailoringInfo [] tailoringInfos;
 603                 static object forLock = new object ();
 604                 public static readonly bool isReady;
 605
 606                 public static bool IsReady {
 607                         get { return isReady; }
 608                 }
 609
 610 #if USE_MANAGED_RESOURCE
 611                 static IntPtr GetResource (string name)
 612                 {
 613                         int size;
 614                         Module module;
 615                         return Assembly.GetExecutingAssembly ().GetManifestResourceInternal (name, out size, out module);
 616                 }
 617 #elif USE_C_HEADER
 618                 const int CollationTableIdxIgnorables = 0;
 619                 const int CollationTableIdxCategory = 1;
 620                 const int CollationTableIdxLevel1 = 2;
 621                 const int CollationTableIdxLevel2 = 3;
 622                 const int CollationTableIdxLevel3 = 4;
 623                 const int CollationTableIdxTailoringInfos = 5;
 624                 const int CollationTableIdxTailoringChars = 6;
 625                 const int CollationTableIdxCjkCHS = 7;
 626                 const int CollationTableIdxCjkCHT = 8;
 627                 const int CollationTableIdxCjkJA = 9;
 628                 const int CollationTableIdxCjkKO = 10;
 629                 const int CollationTableIdxCjkKOLv2 = 11;
 630
 631                 [MethodImplAttribute (MethodImplOptions.InternalCall)]
 632                 static extern void load_collation_resource (int resource_index, byte** data);
 633 #else
 634                 static readonly string corlibPath = Assembly.GetExecutingAssembly ().Location;
 635
 636                 const int CollationResourceCore = 0;
 637                 const int CollationResourceCJKCHS = 1;
 638                 const int CollationResourceCJKCHT = 2;
 639                 const int CollationResourceCJKJA = 3;
 640                 const int CollationResourceCJKKO = 4;
 641                 const int CollationResourceCJKKOlv2 = 5;
 642                 const int CollationResourceTailoring = 6;
 643
 644                 [MethodImplAttribute (MethodImplOptions.InternalCall)]
 645                 static extern void load_collation_resource (string path, int resource_index, byte** data, int* size);
 646 #endif
 647
 648                 static uint UInt32FromBytePtr (byte* raw, uint idx)
 649                 {
 650                         return (uint) (raw [idx] + (raw [idx + 1] << 8)
 651                                 + (raw [idx + 2] << 16) + (raw [idx + 3] << 24));
 652                 }
 653
 654                 static MSCompatUnicodeTable ()
 655                 {
 656 #if USE_C_HEADER
 657                         byte* raw;
 658                         uint* tailor;
 659                         uint size;
 660                         uint idx = 0;
 661
 662                         lock (forLock) {
 663                                 load_collation_resource (CollationTableIdxIgnorables, &raw);
 664                                 ignorableFlags = raw;
 665                                 load_collation_resource (CollationTableIdxCategory, &raw);
 666                                 categories = raw;
 667                                 load_collation_resource (CollationTableIdxLevel1, &raw);
 668                                 level1 = raw;
 669                                 load_collation_resource (CollationTableIdxLevel2, &raw);
 670                                 level2 = raw;
 671                                 load_collation_resource (CollationTableIdxLevel3, &raw);
 672                                 level3 = raw;
 673                                 load_collation_resource (CollationTableIdxTailoringInfos, &raw);
 674                                 tailor = (uint*) raw;
 675                                 load_collation_resource (CollationTableIdxTailoringChars, &raw);
 676                                 tailoring = (char*) raw;
 677                         }
 678
 679                         idx = 0;
 680                         uint count = tailor [idx++];
 681                         tailoringInfos = new TailoringInfo [count];
 682                         for (int i = 0; i < count; i++) {
 683                                 int i1 = (int) tailor [idx++];
 684                                 int i2 = (int) tailor [idx++];
 685                                 int i3 = (int) tailor [idx++];
 686                                 TailoringInfo ti = new TailoringInfo (
 687                                         i1, i2, i3, tailor [idx++] != 0);
 688                                 tailoringInfos [i] = ti;
 689                         }
 690
 691                         isReady = true;
 692 #else
 693
 694                         byte* raw;
 695                         byte* tailor;
 696                         uint size;
 697                         uint idx = 0;
 698
 699 #if USE_MANAGED_RESOURCE
 700                         IntPtr ptr = GetResource ("collation.core.bin");
 701                         if (ptr == IntPtr.Zero)
 702                                 return;
 703                         raw = (byte*) ((void*) ptr);
 704                         ptr = GetResource ("collation.tailoring.bin");
 705                         if (ptr == IntPtr.Zero)
 706                                 return;
 707                         tailor = (byte*) ((void*) ptr);
 708 #else
 709                         int rawsize;
 710                         int trawsize;
 711
 712                         lock (forLock) {
 713                                 load_collation_resource (corlibPath, CollationResourceCore, &raw, &rawsize);
 714                                 load_collation_resource (corlibPath, CollationResourceTailoring, &tailor, &trawsize);
 715                                 load_collation_resource (corlibPath, CollationResourceTailoringChars, &tailorChars, &trawsize);
 716                         }
 717 #endif
 718
 719                         if (raw == null || tailor == null)
 720                                 return;
 721                         // check resource version
 722                         if (raw [0] != UUtil.ResourceVersion ||
 723                                 tailor [0] != UUtil.ResourceVersion)
 724                                 return;
 725
 726                         idx = 1;
 727                         size = UInt32FromBytePtr (raw, idx);
 728                         idx += 4;
 729                         ignorableFlags = raw + idx;
 730                         idx += size;
 731
 732                         size = UInt32FromBytePtr (raw, idx);
 733                         idx += 4;
 734                         categories = raw + idx;
 735                         idx += size;
 736
 737                         size = UInt32FromBytePtr (raw, idx);
 738                         idx += 4;
 739                         level1 = raw + idx;
 740                         idx += size;
 741
 742                         size = UInt32FromBytePtr (raw, idx);
 743                         idx += 4;
 744                         level2 = raw + idx;
 745                         idx += size;
 746
 747                         size = UInt32FromBytePtr (raw, idx);
 748                         idx += 4;
 749                         level3 = raw + idx;
 750                         idx += size;
 751
 752 //                      size = UInt32FromBytePtr (raw, idx);
 753 //                      idx += 4;
 754 //                      widthCompat = (ushort*) (raw + idx);
 755 //                      idx += size * 2;
 756
 757                         // tailoring
 758
 759                         idx = 1;
 760                         uint count = UInt32FromBytePtr (tailor, idx);
 761                         idx += 4;
 762                         tailoringInfos = new TailoringInfo [count];
 763                         for (int i = 0; i < count; i++) {
 764                                 int i1 = (int) UInt32FromBytePtr (tailor, idx);
 765                                 idx += 4;
 766                                 int i2 = (int) UInt32FromBytePtr (tailor, idx);
 767                                 idx += 4;
 768                                 int i3 = (int) UInt32FromBytePtr (tailor, idx);
 769                                 idx += 4;
 770                                 TailoringInfo ti = new TailoringInfo (
 771                                         i1, i2, i3, tailor [idx++] != 0);
 772                                 tailoringInfos [i] = ti;
 773                         }
 774                         idx += 2; // dummy
 775                         // tailorings
 776                         count = UInt32FromBytePtr (tailor, idx);
 777                         idx += 4;
 778
 779                         tailoringArr = new char [count];
 780                         for (int i = 0; i < count; i++, idx += 2)
 781                                 tailoringArr [i] = (char) (tailor [idx] + (tailor [idx + 1] << 8));
 782                         isReady = true;
 783 #endif
 784                 }
 785
 786                 public static void FillCJK (string culture,
 787                         ref CodePointIndexer cjkIndexer,
 788                         ref byte* catTable,
 789                         ref byte* lv1Table,
 790                         ref CodePointIndexer lv2Indexer,
 791                         ref byte* lv2Table)
 792                 {
 793                         lock (forLock) {
 794                                 FillCJKCore (culture, ref cjkIndexer,
 795                                         ref catTable, ref lv1Table,
 796                                         ref lv2Indexer, ref lv2Table);
 797                                 SetCJKReferences (culture, ref cjkIndexer,
 798                                         ref catTable, ref lv1Table,
 799                                         ref lv2Indexer, ref lv2Table);
 800                         }
 801                 }
 802
 803                 static void FillCJKCore (string culture,
 804                         ref CodePointIndexer cjkIndexer,
 805                         ref byte* catTable, ref byte* lv1Table,
 806                         ref CodePointIndexer cjkLv2Indexer, ref byte* lv2Table)
 807                 {
 808                         if (!IsReady)
 809                                 return;
 810
 811                         string name = null;
 812                         switch (culture) {
 813                         case "zh-CHS":
 814                                 name = "cjkCHS";
 815                                 catTable = cjkCHScategory;
 816                                 lv1Table = cjkCHSlv1;
 817                                 break;
 818                         case "zh-CHT":
 819                                 name = "cjkCHT";
 820                                 catTable = cjkCHTcategory;
 821                                 lv1Table = cjkCHTlv1;
 822                                 break;
 823                         case "ja":
 824                                 name = "cjkJA";
 825                                 catTable = cjkJAcategory;
 826                                 lv1Table = cjkJAlv1;
 827                                 break;
 828                         case "ko":
 829                                 name = "cjkKO";
 830                                 catTable = cjkKOcategory;
 831                                 lv1Table = cjkKOlv1;
 832                                 break;
 833                         }
 834
 835                         if (name == null || lv1Table != null)
 836                                 return;
 837
 838                         byte* raw;
 839                         uint idx = 0;
 840 #if USE_MANAGED_RESOURCE
 841                         string filename =
 842                                 String.Format ("collation.{0}.bin", name);
 843                         IntPtr ptr = GetResource (filename);
 844                         if (ptr == IntPtr.Zero)
 845                                 return;
 846                         raw = (byte*) ((void*) ptr);
 847                         idx += ResourceVersionSize;
 848 #elif USE_C_HEADER
 849                         int residx = -1;
 850                         switch (culture) {
 851                         case "zh-CHS": residx = CollationTableIdxCjkCHS; break;
 852                         case "zh-CHT": residx = CollationTableIdxCjkCHT; break;
 853                         case "ja": residx = CollationTableIdxCjkJA; break;
 854                         case "ko": residx = CollationTableIdxCjkKO; break;
 855                         }
 856                         if (residx < 0)
 857                                 return;
 858                         load_collation_resource (residx, &raw);
 859 #else
 860                         int size;
 861                         int residx = -1;
 862                         switch (culture) {
 863                         case "zh-CHS": residx = CollationResourceCJKCHS; break;
 864                         case "zh-CHT": residx = CollationResourceCJKCHT; break;
 865                         case "ja": residx = CollationResourceCJKJA; break;
 866                         case "ko": residx = CollationResourceCJKKO; break;
 867                         }
 868                         if (residx < 0)
 869                                 return;
 870                         load_collation_resource (corlibPath, residx, &raw, &size);
 871                         idx += ResourceVersionSize;
 872 #endif
 873                         uint count = UInt32FromBytePtr (raw, idx);
 874                         idx += 4;
 875                         catTable = (byte*) raw + idx;
 876                         lv1Table = (byte*) raw + idx + count;
 877
 878                         switch (culture) {
 879                         case "zh-CHS":
 880                                 cjkCHScategory = catTable;
 881                                 cjkCHSlv1 = lv1Table;
 882                                 break;
 883                         case "zh-CHT":
 884                                 cjkCHTcategory = catTable;
 885                                 cjkCHTlv1 = lv1Table;
 886                                 break;
 887                         case "ja":
 888                                 cjkJAcategory = catTable;
 889                                 cjkJAlv1 = lv1Table;
 890                                 break;
 891                         case "ko":
 892                                 cjkKOcategory = catTable;
 893                                 cjkKOlv1 = lv1Table;
 894                                 break;
 895                         }
 896
 897                         if (name != "cjkKO")
 898                                 return;
 899 #if USE_MANAGED_RESOURCE
 900                         ptr = GetResource ("collation.cjkKOlv2.bin");
 901                         if (ptr == IntPtr.Zero)
 902                                 return;
 903                         raw = (byte*) ((void*) ptr);
 904                         idx = ResourceVersionSize + 4;
 905 #elif USE_C_HEADER
 906                         load_collation_resource (CollationTableIdxCjkKOLv2, &raw);
 907 #else
 908                         load_collation_resource (corlibPath, CollationResourceCJKKOlv2, &raw, &size);
 909                         idx = ResourceVersionSize + 4;
 910 #endif
 911                         cjkKOlv2 = raw + idx;
 912                         lv2Table = cjkKOlv2;
 913                 }
 914         }
 915 }
 916 #endif
 917
 918
 919                 // For "categories", 0 means no primary weight. 6 means
 920                 // variable weight
 921                 // For expanded character the value is never filled (i.e. 0).
 922                 // Those arrays will be split into blocks (<3400 and >F800)
 923                 // level 4 is computed.
 924
 925                 // public static bool HasSpecialWeight (char c)
 926                 // { return level1 [(int) c] == 6; }
 927
 928                 //
 929                 // autogenerated code or icall to fill array runs here
 930                 //
 931