mcs/class/corlib/Mono.Globalization.Unicode/MSCompatUnicodeTable.cs

   1 #define USE_MANAGED_RESOURCE
   2 //#define USE_C_HEADER
   3
   4 //
   5 // MSCompatUnicodeTable.cs : Handles Windows-like sortket tables.
   6 //
   7 // Author:
   8 //      Atsushi Enomoto  <atsushi@ximian.com>
   9 //
  10 // Copyright (C) 2005 Novell, Inc (http://www.novell.com)
  11 //
  12 // Permission is hereby granted, free of charge, to any person obtaining
  13 // a copy of this software and associated documentation files (the
  14 // "Software"), to deal in the Software without restriction, including
  15 // without limitation the rights to use, copy, modify, merge, publish,
  16 // distribute, sublicense, and/or sell copies of the Software, and to
  17 // permit persons to whom the Software is furnished to do so, subject to
  18 // the following conditions:
  19 //
  20 // The above copyright notice and this permission notice shall be
  21 // included in all copies or substantial portions of the Software.
  22 //
  23 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  27 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  28 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  29 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  30 //
  31
  32 using System;
  33 using System.Collections;
  34 using System.Globalization;
  35 using System.Reflection;
  36 using System.Runtime.CompilerServices;
  37 using System.Runtime.InteropServices;
  38
  39 using UUtil = Mono.Globalization.Unicode.MSCompatUnicodeTableUtil;
  40
  41 namespace Mono.Globalization.Unicode
  42 {
  43         internal class TailoringInfo
  44         {
  45                 public readonly int LCID;
  46                 public readonly int TailoringIndex;
  47                 public readonly int TailoringCount;
  48                 public readonly bool FrenchSort;
  49
  50                 public TailoringInfo (int lcid, int tailoringIndex, int tailoringCount, bool frenchSort)
  51                 {
  52                         LCID = lcid;
  53                         TailoringIndex = tailoringIndex;
  54                         TailoringCount = tailoringCount;
  55                         FrenchSort = frenchSort;
  56                 }
  57         }
  58
  59         #region Tailoring support classes
  60         // Possible mapping types are:
  61         //
  62         //      - string to string (ReplacementMap)
  63         //      - string to SortKey (SortKeyMap)
  64         //      - diacritical byte to byte (DiacriticalMap)
  65         //
  66         // There could be mapping from string to sortkeys, but
  67         // for now there is none as such.
  68         //
  69         internal class Contraction
  70         {
  71                 public readonly char [] Source;
  72                 // only either of them is used.
  73                 public readonly string Replacement;
  74                 public readonly byte [] SortKey;
  75
  76                 public Contraction (char [] source,
  77                         string replacement, byte [] sortkey)
  78                 {
  79                         Source = source;
  80                         Replacement = replacement;
  81                         SortKey = sortkey;
  82                 }
  83         }
  84
  85         internal class ContractionComparer : IComparer
  86         {
  87                 public static readonly ContractionComparer Instance =
  88                         new ContractionComparer ();
  89
  90                 public int Compare (object o1, object o2)
  91                 {
  92                         Contraction c1 = (Contraction) o1;
  93                         Contraction c2 = (Contraction) o2;
  94                         char [] a1 = c1.Source;
  95                         char [] a2 = c2.Source;
  96                         int min = a1.Length > a2.Length ?
  97                                 a2.Length : a1.Length;
  98                         for (int i = 0; i < min; i++)
  99                                 if (a1 [i] != a2 [i])
 100                                         return a1 [i] - a2 [i];
 101                         return a1.Length - a2.Length;
 102                 }
 103         }
 104
 105         internal class Level2Map
 106         {
 107                 public byte Source;
 108                 public byte Replace;
 109
 110                 public Level2Map (byte source, byte replace)
 111                 {
 112                         Source = source;
 113                         Replace = replace;
 114                 }
 115         }
 116
 117         internal class Level2MapComparer : IComparer
 118         {
 119                 public static readonly Level2MapComparer Instance =
 120                         new Level2MapComparer ();
 121
 122                 public int Compare (object o1, object o2)
 123                 {
 124                         Level2Map m1 = (Level2Map) o1;
 125                         Level2Map m2 = (Level2Map) o2;
 126                         return (m1.Source - m2.Source);
 127                 }
 128         }
 129
 130         #endregion
 131
 132         unsafe internal class MSCompatUnicodeTable
 133         {
 134                 public static int MaxExpansionLength = 3;
 135
 136                 static readonly byte* ignorableFlags;
 137                 static readonly byte* categories;
 138                 static readonly byte* level1;
 139                 static readonly byte* level2;
 140                 static readonly byte* level3;
 141 //              static readonly ushort* widthCompat;
 142 #if USE_C_HEADER
 143                 static readonly char* tailoring;
 144 #endif
 145                 static byte* cjkCHScategory;
 146                 static byte* cjkCHTcategory;
 147                 static byte* cjkJAcategory;
 148                 static byte* cjkKOcategory;
 149                 static byte* cjkCHSlv1;
 150                 static byte* cjkCHTlv1;
 151                 static byte* cjkJAlv1;
 152                 static byte* cjkKOlv1;
 153                 static byte* cjkKOlv2;
 154
 155                 const int ResourceVersionSize = 1;
 156
 157                 public static TailoringInfo GetTailoringInfo (int lcid)
 158                 {
 159                         for (int i = 0; i < tailoringInfos.Length; i++)
 160                                 if (tailoringInfos [i].LCID == lcid)
 161                                         return tailoringInfos [i];
 162                         return null;
 163                 }
 164
 165                 unsafe public static void BuildTailoringTables (CultureInfo culture,
 166                         TailoringInfo t,
 167                         ref Contraction [] contractions,
 168                         ref Level2Map [] diacriticals)
 169                 {
 170                         // collect tailoring entries.
 171                         ArrayList cmaps = new ArrayList ();
 172                         ArrayList dmaps = new ArrayList ();
 173                         fixed (char* tarr = tailoringArr){
 174                                 int idx = t.TailoringIndex;
 175                                 int end = idx + t.TailoringCount;
 176                                 while (idx < end) {
 177                                         int ss = idx + 1;
 178                                         char [] src = null;
 179                                         switch (tarr [idx]) {
 180                                         case '\x1': // SortKeyMap
 181                                                 idx++;
 182                                                 while (tarr [ss] != 0)
 183                                                         ss++;
 184                                                 src = new char [ss - idx];
 185                                                 //                                      Array.Copy (tarr, idx, src, 0, ss - idx);
 186                                                 Marshal.Copy ((IntPtr) (tarr + idx), src, 0, ss - idx);
 187                                                 byte [] sortkey = new byte [4];
 188                                                 for (int i = 0; i < 4; i++)
 189                                                         sortkey [i] = (byte) tarr [ss + 1 + i];
 190                                                 cmaps.Add (new Contraction (
 191                                                                             src, null, sortkey));
 192                                                 // it ends with 0
 193                                                 idx = ss + 6;
 194                                                 break;
 195                                         case '\x2': // DiacriticalMap
 196                                                 dmaps.Add (new Level2Map (
 197                                                                           (byte) tarr [idx + 1],
 198                                                                           (byte) tarr [idx + 2]));
 199                                                 idx += 3;
 200                                                 break;
 201                                         case '\x3': // ReplacementMap
 202                                                 idx++;
 203                                                 while (tarr [ss] != 0)
 204                                                         ss++;
 205                                                 src = new char [ss - idx];
 206                                                 //                                      Array.Copy (tarr, idx, src, 0, ss - idx);
 207                                                 Marshal.Copy ((IntPtr) (tarr + idx), src, 0, ss - idx);
 208                                                 ss++;
 209                                                 int l = ss;
 210                                                 while (tarr [l] != 0)
 211                                                         l++;
 212                                                 string r = new string (tarr, ss, l - ss);
 213                                                 cmaps.Add (new Contraction (
 214                                                                             src, r, null));
 215                                                 idx = l + 1;
 216                                                 break;
 217                                         default:
 218                                                 throw new NotImplementedException (String.Format ("Mono INTERNAL ERROR (Should not happen): Collation tailoring table is broken for culture {0} ({1}) at 0x{2:X}", culture.LCID, culture.Name, idx));
 219                                         }
 220                                 }
 221                         }
 222                         cmaps.Sort (ContractionComparer.Instance);
 223                         dmaps.Sort (Level2MapComparer.Instance);
 224                         contractions = cmaps.ToArray (typeof (Contraction))
 225                                 as Contraction [];
 226                         diacriticals = dmaps.ToArray (typeof (Level2Map))
 227                                 as Level2Map [];
 228                 }
 229
 230                 static void SetCJKReferences (string name,
 231                         ref CodePointIndexer cjkIndexer,
 232                         ref byte* catTable, ref byte* lv1Table,
 233                         ref CodePointIndexer lv2Indexer, ref byte* lv2Table)
 234                 {
 235                         // as a part of mscorlib.dll, this invocation is
 236                         // somewhat extraneous (pointers were already assigned).
 237
 238                         switch (name) {
 239                         case "zh-CHS":
 240                                 catTable = cjkCHScategory;
 241                                 lv1Table = cjkCHSlv1;
 242                                 cjkIndexer = UUtil.CjkCHS;
 243                                 break;
 244                         case "zh-CHT":
 245                                 catTable = cjkCHTcategory;
 246                                 lv1Table = cjkCHTlv1;
 247                                 cjkIndexer = UUtil.Cjk;
 248                                 break;
 249                         case "ja":
 250                                 catTable = cjkJAcategory;
 251                                 lv1Table = cjkJAlv1;
 252                                 cjkIndexer = UUtil.Cjk;
 253                                 break;
 254                         case "ko":
 255                                 catTable = cjkKOcategory;
 256                                 lv1Table = cjkKOlv1;
 257                                 lv2Table = cjkKOlv2;
 258                                 cjkIndexer = UUtil.Cjk;
 259                                 lv2Indexer = UUtil.Cjk;
 260                                 break;
 261                         }
 262                 }
 263
 264                 public static byte Category (int cp)
 265                 {
 266                         return categories [UUtil.Category.ToIndex (cp)];
 267                 }
 268
 269                 public static byte Level1 (int cp)
 270                 {
 271                         return level1 [UUtil.Level1.ToIndex (cp)];
 272                 }
 273
 274                 public static byte Level2 (int cp)
 275                 {
 276                         return level2 [UUtil.Level2.ToIndex (cp)];
 277                 }
 278
 279                 public static byte Level3 (int cp)
 280                 {
 281                         return level3 [UUtil.Level3.ToIndex (cp)];
 282                 }
 283
 284                 public static bool IsSortable (string s)
 285                 {
 286                         foreach (char c in s)
 287                                 if (!IsSortable (c))
 288                                         return false;
 289                         return true;
 290                 }
 291
 292                 public static bool IsSortable (int cp)
 293                 {
 294                         // LAMESPEC: they should strictly match with
 295                         // IsIgnorable() result, but sometimes it does not.
 296                         if (!IsIgnorable (cp))
 297                                 return true;
 298                         switch (cp) {
 299                         case 0:
 300                         case 0x0640:
 301                         case 0xFEFF:
 302                                 return true;
 303                         }
 304                         return 0x180B <= cp && cp <= 0x180E ||
 305                                 0x200C <= cp && cp <= 0x200F ||
 306                                 0x202A <= cp && cp <= 0x202E ||
 307                                 0x206A <= cp && cp <= 0x206F ||
 308                                 0x200C <= cp && cp <= 0x200F ||
 309                                 0xFFF9 <= cp && cp <= 0xFFFD;
 310                 }
 311
 312                 public static bool IsIgnorable (int cp)
 313                 {
 314                         return IsIgnorable (cp, 1);
 315                 }
 316
 317                 public static bool IsIgnorable (int cp, byte flag)
 318                 {
 319                         if (cp == 0)
 320                                 return false;
 321                         if ((flag & 1) != 0) {
 322                                 UnicodeCategory uc = Char.GetUnicodeCategory ((char) cp);
 323                                 // This check eliminates some extraneous code areas
 324                                 if (uc == UnicodeCategory.OtherNotAssigned)
 325                                         return true;
 326                                 // Some characters in Surrogate area are ignored.
 327                                 if (0xD880 <= cp && cp < 0xDB80)
 328                                         return true;
 329                         }
 330                         int i = UUtil.Ignorable.ToIndex (cp);
 331                         return i >= 0 && (ignorableFlags [i] & flag) != 0;
 332                 }
 333                 // Verifier:
 334                 // for (int i = 0; i <= char.MaxValue; i++)
 335                 //      if (Char.GetUnicodeCategory ((char) i)
 336                 //              == UnicodeCategory.OtherNotAssigned
 337                 //              && ignorableFlags [i] != 7)
 338                 //              Console.WriteLine ("{0:X04}", i);
 339
 340                 public static bool IsIgnorableSymbol (int cp)
 341                 {
 342                         return IsIgnorable (cp, 2);
 343 //                      int i = UUtil.Ignorable.ToIndex (cp);
 344 //                      return i >= 0 && (ignorableFlags [i] & 0x2) != 0;
 345                 }
 346
 347                 public static bool IsIgnorableNonSpacing (int cp)
 348                 {
 349                         return IsIgnorable (cp, 4);
 350 //                      int i = UUtil.Ignorable.ToIndex (cp);
 351 //                      return i >= 0 && (ignorableFlags [i] & 0x4) != 0;
 352
 353                         // It could be implemented this way, but the above
 354                         // is faster.
 355 //                      return categories [UUtil.Category.ToIndex (cp)] == 1;
 356                 }
 357
 358                 public static int ToKanaTypeInsensitive (int i)
 359                 {
 360                         // Note that IgnoreKanaType does not treat half-width
 361                         // katakana as equivalent to full-width ones.
 362
 363                         // Thus, it is so simple ;-)
 364                         return (0x3041 <= i && i <= 0x3094) ? i + 0x60 : i;
 365                 }
 366
 367                 // Note that currently indexer optimizes this table a lot,
 368                 // which might have resulted in bugs.
 369                 public static int ToWidthCompat (int i)
 370                 {
 371                         if (i < 0x2190)
 372                                 return i;
 373                         if (i > 0xFF00) {
 374                                 if (i <= 0xFF5E)
 375                                         return i - 0xFF00 + 0x20;
 376                                 switch (i) {
 377                                 case 0xFFE0:
 378                                         return 0xA2;
 379                                 case 0xFFE1:
 380                                         return 0xA3;
 381                                 case 0xFFE2:
 382                                         return 0xAC;
 383                                 case 0xFFE3:
 384                                         return 0xAF;
 385                                 case 0xFFE4:
 386                                         return 0xA6;
 387                                 case 0xFFE5:
 388                                         return 0xA5;
 389                                 case 0xFFE6:
 390                                         return 0x20A9;
 391                                 }
 392                         }
 393                         if (i > 0x32FE)
 394                                 return i;
 395
 396                         if (i <= 0x2193)
 397                                 return 0xFFE9 - 0x2190 + i;
 398                         if (i < 0x2502)
 399                                 return i;
 400                         if (i <= 0x25CB) {
 401                                 switch (i) {
 402                                 case 0x2502:
 403                                         return 0xFFE8;
 404                                 case 0x25A0:
 405                                         return 0xFFED;
 406                                 case 0x25CB:
 407                                         return 0xFFEE;
 408                                 default:
 409                                         return i;
 410                                 }
 411                         }
 412                         if (i < 0x3000)
 413                                 return i;
 414                         if (i < 0x3131) {
 415                                 switch (i) {
 416                                 case 0x3000:
 417                                         return 0x20;
 418                                 case 0x3001:
 419                                         return 0xFF64;
 420                                 case 0x3002:
 421                                         return 0xFF61;
 422                                 case 0x300C:
 423                                         return 0xFF62;
 424                                 case 0x300D:
 425                                         return 0xFF63;
 426                                 case 0x30FB:
 427                                         return 0xFF65;
 428                                 // Other Kana compat characters' width
 429                                 // compatibility is considered in special weight.
 430                                 default:
 431                                         return i;
 432                                 }
 433                         }
 434                         if (i < 0x3164) { // Hangul compat
 435                                 return i - 0x3130 + 0xFFA0;
 436                         }
 437                         if (i == 0x3164)
 438                                 return 0xFFA0;
 439                         // 0x32D0-0x32FE are Kana compat characters, whose
 440                         // width compatibility is considered in special weight.
 441                         return i;
 442                 }
 443
 444                 #region Level 4 properties (Kana)
 445
 446                 public static bool HasSpecialWeight (char c)
 447                 {
 448                         if (c < '\u3041')
 449                                 return false;
 450                         else if ('\uFF66' <= c && c < '\uFF9E')
 451                                 return true;
 452                         else if ('\u3300' <= c)
 453                                 return false;
 454                         else if (c < '\u309D')
 455                                 return (c < '\u3099');
 456                         else if (c < '\u3100')
 457                                 return c != '\u30FB';
 458                         else if (c < '\u32D0')
 459                                 return false;
 460                         else if (c < '\u32FF')
 461                                 return true;
 462                         return false;
 463                 }
 464
 465                 // FIXME: it should be removed at some stage
 466                 // (will become unused).
 467                 public static byte GetJapaneseDashType (char c)
 468                 {
 469                         switch (c) {
 470                         case '\u309D':
 471                         case '\u309E':
 472                         case '\u30FD':
 473                         case '\u30FE':
 474                         case '\uFF70':
 475                                 return 4;
 476                         case '\u30FC':
 477                                 return 5;
 478                         }
 479                         return 3;
 480                 }
 481
 482                 public static bool IsHalfWidthKana (char c)
 483                 {
 484                         return '\uFF66' <= c && c <= '\uFF9D';
 485                 }
 486
 487                 public static bool IsHiragana (char c)
 488                 {
 489                         return '\u3041' <= c && c <= '\u3094';
 490                 }
 491
 492                 public static bool IsJapaneseSmallLetter (char c)
 493                 {
 494                         if ('\uFF67' <= c && c <= '\uFF6F')
 495                                 return true;
 496                         if ('\u3040' < c && c < '\u30FA') {
 497                                 switch (c) {
 498                                 case '\u3041':
 499                                 case '\u3043':
 500                                 case '\u3045':
 501                                 case '\u3047':
 502                                 case '\u3049':
 503                                 case '\u3063':
 504                                 case '\u3083':
 505                                 case '\u3085':
 506                                 case '\u3087':
 507                                 case '\u308E':
 508                                 case '\u30A1':
 509                                 case '\u30A3':
 510                                 case '\u30A5':
 511                                 case '\u30A7':
 512                                 case '\u30A9':
 513                                 case '\u30C3':
 514                                 case '\u30E3':
 515                                 case '\u30E5':
 516                                 case '\u30E7':
 517                                 case '\u30EE':
 518                                 case '\u30F5':
 519                                 case '\u30F6':
 520                                         return true;
 521                                 }
 522                         }
 523                         return false;
 524                 }
 525
 526                 #endregion
 527
 528 #if GENERATE_TABLE
 529
 530                 public static readonly bool IsReady = true; // always
 531
 532                 static MSCompatUnicodeTable ()
 533                 {
 534                         throw new Exception ("This code should not be used");
 535
 536                         fixed (byte* tmp = ignorableFlagsArr) {
 537                                 ignorableFlags = tmp;
 538                         }
 539                         fixed (byte* tmp = categoriesArr) {
 540                                 categories = tmp;
 541                         }
 542                         fixed (byte* tmp = level1Arr) {
 543                                 level1 = tmp;
 544                         }
 545                         fixed (byte* tmp = level2Arr) {
 546                                 level2 = tmp;
 547                         }
 548                         fixed (byte* tmp = level3Arr) {
 549                                 level3 = tmp;
 550                         }
 551 //                      fixed (ushort* tmp = widthCompatArr) {
 552 //                              widthCompat = tmp;
 553 //                      }
 554                         fixed (char* tmp = tailoringArr) {
 555                                 tailoring = tmp;
 556                         }
 557                         fixed (byte* tmp = cjkCHSArr) {
 558                                 cjkCHScategory = tmp;
 559                                 cjkCHSlv1 = tmp + cjkCHSArrLength;
 560                         }
 561                         fixed (byte* tmp = cjkCHTArr) {
 562                                 cjkCHTcategory = tmp;
 563                                 cjkCHTlv1 = tmp + cjkCHTArrLength;
 564                         }
 565                         fixed (byte* tmp = cjkJAArr) {
 566                                 cjkJAcategory = tmp;
 567                                 cjkJAlv1 = tmp + cjkJAArrLength;
 568                         }
 569                         fixed (byte* tmp = cjkKOArr) {
 570                                 cjkKOcategory = tmp;
 571                                 cjkKOlv1 = tmp + cjkKOArrLength;
 572                         }
 573                         fixed (byte* tmp = cjkKOlv2Arr) {
 574                                 cjkKOlv2 = tmp;
 575                         }
 576                 }
 577
 578                 public static void FillCJK (string name,
 579                         ref CodePointIndexer cjkIndexer,
 580                         ref byte* catTable, ref byte* lv1Table,
 581                         ref CodePointIndexer cjkLv2Indexer,
 582                         ref byte* lv2Table)
 583                 {
 584                         SetCJKReferences (name, ref cjkIndexer,
 585                                 ref catTable, ref lv1Table,
 586                                 ref cjkLv2Indexer, ref lv2Table);
 587                 }
 588 #else
 589
 590 #if !USE_C_HEADER
 591                 static readonly char [] tailoringArr;
 592 #endif
 593                 static readonly TailoringInfo [] tailoringInfos;
 594                 static object forLock = new object ();
 595                 public static readonly bool isReady;
 596
 597                 public static bool IsReady {
 598                         get { return isReady; }
 599                 }
 600
 601 #if USE_MANAGED_RESOURCE
 602                 static IntPtr GetResource (string name)
 603                 {
 604                         int size;
 605                         Module module;
 606                         return Assembly.GetExecutingAssembly ().GetManifestResourceInternal (name, out size, out module);
 607                 }
 608 #elif USE_C_HEADER
 609                 const int CollationTableIdxIgnorables = 0;
 610                 const int CollationTableIdxCategory = 1;
 611                 const int CollationTableIdxLevel1 = 2;
 612                 const int CollationTableIdxLevel2 = 3;
 613                 const int CollationTableIdxLevel3 = 4;
 614                 const int CollationTableIdxTailoringInfos = 5;
 615                 const int CollationTableIdxTailoringChars = 6;
 616                 const int CollationTableIdxCjkCHS = 7;
 617                 const int CollationTableIdxCjkCHT = 8;
 618                 const int CollationTableIdxCjkJA = 9;
 619                 const int CollationTableIdxCjkKO = 10;
 620                 const int CollationTableIdxCjkKOLv2 = 11;
 621
 622                 [MethodImplAttribute (MethodImplOptions.InternalCall)]
 623                 static extern void load_collation_resource (int resource_index, byte** data);
 624 #else
 625                 static readonly string corlibPath = Assembly.GetExecutingAssembly ().Location;
 626
 627                 const int CollationResourceCore = 0;
 628                 const int CollationResourceCJKCHS = 1;
 629                 const int CollationResourceCJKCHT = 2;
 630                 const int CollationResourceCJKJA = 3;
 631                 const int CollationResourceCJKKO = 4;
 632                 const int CollationResourceCJKKOlv2 = 5;
 633                 const int CollationResourceTailoring = 6;
 634
 635                 [MethodImplAttribute (MethodImplOptions.InternalCall)]
 636                 static extern void load_collation_resource (string path, int resource_index, byte** data, int* size);
 637 #endif
 638
 639                 static uint UInt32FromBytePtr (byte* raw, uint idx)
 640                 {
 641                         return (uint) (raw [idx] + (raw [idx + 1] << 8)
 642                                 + (raw [idx + 2] << 16) + (raw [idx + 3] << 24));
 643                 }
 644
 645                 static MSCompatUnicodeTable ()
 646                 {
 647 #if USE_C_HEADER
 648                         byte* raw;
 649                         uint* tailor;
 650                         uint size;
 651                         uint idx = 0;
 652
 653                         lock (forLock) {
 654                                 load_collation_resource (CollationTableIdxIgnorables, &raw);
 655                                 ignorableFlags = raw;
 656                                 load_collation_resource (CollationTableIdxCategory, &raw);
 657                                 categories = raw;
 658                                 load_collation_resource (CollationTableIdxLevel1, &raw);
 659                                 level1 = raw;
 660                                 load_collation_resource (CollationTableIdxLevel2, &raw);
 661                                 level2 = raw;
 662                                 load_collation_resource (CollationTableIdxLevel3, &raw);
 663                                 level3 = raw;
 664                                 load_collation_resource (CollationTableIdxTailoringInfos, &raw);
 665                                 tailor = (uint*) raw;
 666                                 load_collation_resource (CollationTableIdxTailoringChars, &raw);
 667                                 tailoring = (char*) raw;
 668                         }
 669
 670                         idx = 0;
 671                         uint count = tailor [idx++];
 672                         tailoringInfos = new TailoringInfo [count];
 673                         for (int i = 0; i < count; i++) {
 674                                 int i1 = (int) tailor [idx++];
 675                                 int i2 = (int) tailor [idx++];
 676                                 int i3 = (int) tailor [idx++];
 677                                 TailoringInfo ti = new TailoringInfo (
 678                                         i1, i2, i3, tailor [idx++] != 0);
 679                                 tailoringInfos [i] = ti;
 680                         }
 681
 682                         isReady = true;
 683 #else
 684
 685                         byte* raw;
 686                         byte* tailor;
 687                         uint size;
 688                         uint idx = 0;
 689
 690 #if USE_MANAGED_RESOURCE
 691                         IntPtr ptr = GetResource ("collation.core.bin");
 692                         if (ptr == IntPtr.Zero)
 693                                 return;
 694                         raw = (byte*) ((void*) ptr);
 695                         ptr = GetResource ("collation.tailoring.bin");
 696                         if (ptr == IntPtr.Zero)
 697                                 return;
 698                         tailor = (byte*) ((void*) ptr);
 699 #else
 700                         int rawsize;
 701                         int trawsize;
 702
 703                         lock (forLock) {
 704                                 load_collation_resource (corlibPath, CollationResourceCore, &raw, &rawsize);
 705                                 load_collation_resource (corlibPath, CollationResourceTailoring, &tailor, &trawsize);
 706                                 load_collation_resource (corlibPath, CollationResourceTailoringChars, &tailorChars, &trawsize);
 707                         }
 708 #endif
 709
 710                         if (raw == null || tailor == null)
 711                                 return;
 712                         // check resource version
 713                         if (raw [0] != UUtil.ResourceVersion ||
 714                                 tailor [0] != UUtil.ResourceVersion)
 715                                 return;
 716
 717                         idx = 1;
 718                         size = UInt32FromBytePtr (raw, idx);
 719                         idx += 4;
 720                         ignorableFlags = raw + idx;
 721                         idx += size;
 722
 723                         size = UInt32FromBytePtr (raw, idx);
 724                         idx += 4;
 725                         categories = raw + idx;
 726                         idx += size;
 727
 728                         size = UInt32FromBytePtr (raw, idx);
 729                         idx += 4;
 730                         level1 = raw + idx;
 731                         idx += size;
 732
 733                         size = UInt32FromBytePtr (raw, idx);
 734                         idx += 4;
 735                         level2 = raw + idx;
 736                         idx += size;
 737
 738                         size = UInt32FromBytePtr (raw, idx);
 739                         idx += 4;
 740                         level3 = raw + idx;
 741                         idx += size;
 742
 743 //                      size = UInt32FromBytePtr (raw, idx);
 744 //                      idx += 4;
 745 //                      widthCompat = (ushort*) (raw + idx);
 746 //                      idx += size * 2;
 747
 748                         // tailoring
 749
 750                         idx = 1;
 751                         uint count = UInt32FromBytePtr (tailor, idx);
 752                         idx += 4;
 753                         tailoringInfos = new TailoringInfo [count];
 754                         for (int i = 0; i < count; i++) {
 755                                 int i1 = (int) UInt32FromBytePtr (tailor, idx);
 756                                 idx += 4;
 757                                 int i2 = (int) UInt32FromBytePtr (tailor, idx);
 758                                 idx += 4;
 759                                 int i3 = (int) UInt32FromBytePtr (tailor, idx);
 760                                 idx += 4;
 761                                 TailoringInfo ti = new TailoringInfo (
 762                                         i1, i2, i3, tailor [idx++] != 0);
 763                                 tailoringInfos [i] = ti;
 764                         }
 765                         idx += 2; // dummy
 766                         // tailorings
 767                         count = UInt32FromBytePtr (tailor, idx);
 768                         idx += 4;
 769
 770                         tailoringArr = new char [count];
 771                         for (int i = 0; i < count; i++, idx += 2)
 772                                 tailoringArr [i] = (char) (tailor [idx] + (tailor [idx + 1] << 8));
 773                         isReady = true;
 774 #endif
 775                 }
 776
 777                 public static void FillCJK (string culture,
 778                         ref CodePointIndexer cjkIndexer,
 779                         ref byte* catTable,
 780                         ref byte* lv1Table,
 781                         ref CodePointIndexer lv2Indexer,
 782                         ref byte* lv2Table)
 783                 {
 784                         lock (forLock) {
 785                                 FillCJKCore (culture, ref cjkIndexer,
 786                                         ref catTable, ref lv1Table,
 787                                         ref lv2Indexer, ref lv2Table);
 788                                 SetCJKReferences (culture, ref cjkIndexer,
 789                                         ref catTable, ref lv1Table,
 790                                         ref lv2Indexer, ref lv2Table);
 791                         }
 792                 }
 793
 794                 static void FillCJKCore (string culture,
 795                         ref CodePointIndexer cjkIndexer,
 796                         ref byte* catTable, ref byte* lv1Table,
 797                         ref CodePointIndexer cjkLv2Indexer, ref byte* lv2Table)
 798                 {
 799                         if (!IsReady)
 800                                 return;
 801
 802                         string name = null;
 803                         switch (culture) {
 804                         case "zh-CHS":
 805                                 name = "cjkCHS";
 806                                 catTable = cjkCHScategory;
 807                                 lv1Table = cjkCHSlv1;
 808                                 break;
 809                         case "zh-CHT":
 810                                 name = "cjkCHT";
 811                                 catTable = cjkCHTcategory;
 812                                 lv1Table = cjkCHTlv1;
 813                                 break;
 814                         case "ja":
 815                                 name = "cjkJA";
 816                                 catTable = cjkJAcategory;
 817                                 lv1Table = cjkJAlv1;
 818                                 break;
 819                         case "ko":
 820                                 name = "cjkKO";
 821                                 catTable = cjkKOcategory;
 822                                 lv1Table = cjkKOlv1;
 823                                 break;
 824                         }
 825
 826                         if (name == null || lv1Table != null)
 827                                 return;
 828
 829                         byte* raw;
 830                         uint idx = 0;
 831 #if USE_MANAGED_RESOURCE
 832                         string filename =
 833                                 String.Format ("collation.{0}.bin", name);
 834                         IntPtr ptr = GetResource (filename);
 835                         if (ptr == IntPtr.Zero)
 836                                 return;
 837                         raw = (byte*) ((void*) ptr);
 838                         idx += ResourceVersionSize;
 839 #elif USE_C_HEADER
 840                         int residx = -1;
 841                         switch (culture) {
 842                         case "zh-CHS": residx = CollationTableIdxCjkCHS; break;
 843                         case "zh-CHT": residx = CollationTableIdxCjkCHT; break;
 844                         case "ja": residx = CollationTableIdxCjkJA; break;
 845                         case "ko": residx = CollationTableIdxCjkKO; break;
 846                         }
 847                         if (residx < 0)
 848                                 return;
 849                         load_collation_resource (residx, &raw);
 850 #else
 851                         int size;
 852                         int residx = -1;
 853                         switch (culture) {
 854                         case "zh-CHS": residx = CollationResourceCJKCHS; break;
 855                         case "zh-CHT": residx = CollationResourceCJKCHT; break;
 856                         case "ja": residx = CollationResourceCJKJA; break;
 857                         case "ko": residx = CollationResourceCJKKO; break;
 858                         }
 859                         if (residx < 0)
 860                                 return;
 861                         load_collation_resource (corlibPath, residx, &raw, &size);
 862                         idx += ResourceVersionSize;
 863 #endif
 864                         uint count = UInt32FromBytePtr (raw, idx);
 865                         idx += 4;
 866                         catTable = (byte*) raw + idx;
 867                         lv1Table = (byte*) raw + idx + count;
 868
 869                         switch (culture) {
 870                         case "zh-CHS":
 871                                 cjkCHScategory = catTable;
 872                                 cjkCHSlv1 = lv1Table;
 873                                 break;
 874                         case "zh-CHT":
 875                                 cjkCHTcategory = catTable;
 876                                 cjkCHTlv1 = lv1Table;
 877                                 break;
 878                         case "ja":
 879                                 cjkJAcategory = catTable;
 880                                 cjkJAlv1 = lv1Table;
 881                                 break;
 882                         case "ko":
 883                                 cjkKOcategory = catTable;
 884                                 cjkKOlv1 = lv1Table;
 885                                 break;
 886                         }
 887
 888                         if (name != "cjkKO")
 889                                 return;
 890 #if USE_MANAGED_RESOURCE
 891                         ptr = GetResource ("collation.cjkKOlv2.bin");
 892                         if (ptr == IntPtr.Zero)
 893                                 return;
 894                         raw = (byte*) ((void*) ptr);
 895                         idx = ResourceVersionSize + 4;
 896 #elif USE_C_HEADER
 897                         load_collation_resource (CollationTableIdxCjkKOLv2, &raw);
 898 #else
 899                         load_collation_resource (corlibPath, CollationResourceCJKKOlv2, &raw, &size);
 900                         idx = ResourceVersionSize + 4;
 901 #endif
 902                         cjkKOlv2 = raw + idx;
 903                         lv2Table = cjkKOlv2;
 904                 }
 905         }
 906 }
 907 #endif
 908
 909
 910                 // For "categories", 0 means no primary weight. 6 means
 911                 // variable weight
 912                 // For expanded character the value is never filled (i.e. 0).
 913                 // Those arrays will be split into blocks (<3400 and >F800)
 914                 // level 4 is computed.
 915
 916                 // public static bool HasSpecialWeight (char c)
 917                 // { return level1 [(int) c] == 6; }
 918
 919                 //
 920                 // autogenerated code or icall to fill array runs here
 921                 //
 922