mcs/class/corlib/Mono.Globalization.Unicode/MSCompatUnicodeTable.cs

   1 #define USE_MANAGED_RESOURCE
   2 //#define USE_C_HEADER
   3
   4 //
   5 // MSCompatUnicodeTable.cs : Handles Windows-like sortket tables.
   6 //
   7 // Author:
   8 //      Atsushi Enomoto  <atsushi@ximian.com>
   9 //
  10 // Copyright (C) 2005 Novell, Inc (http://www.novell.com)
  11 //
  12 // Permission is hereby granted, free of charge, to any person obtaining
  13 // a copy of this software and associated documentation files (the
  14 // "Software"), to deal in the Software without restriction, including
  15 // without limitation the rights to use, copy, modify, merge, publish,
  16 // distribute, sublicense, and/or sell copies of the Software, and to
  17 // permit persons to whom the Software is furnished to do so, subject to
  18 // the following conditions:
  19 //
  20 // The above copyright notice and this permission notice shall be
  21 // included in all copies or substantial portions of the Software.
  22 //
  23 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  27 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  28 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  29 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  30 //
  31
  32 using System;
  33 using System.Collections.Generic;
  34 using System.Globalization;
  35 using System.Reflection;
  36 using System.Runtime.CompilerServices;
  37 using System.Runtime.InteropServices;
  38
  39 using UUtil = Mono.Globalization.Unicode.MSCompatUnicodeTableUtil;
  40
  41 namespace Mono.Globalization.Unicode
  42 {
  43         internal class TailoringInfo
  44         {
  45                 public readonly int LCID;
  46                 public readonly int TailoringIndex;
  47                 public readonly int TailoringCount;
  48                 public readonly bool FrenchSort;
  49
  50                 public TailoringInfo (int lcid, int tailoringIndex, int tailoringCount, bool frenchSort)
  51                 {
  52                         LCID = lcid;
  53                         TailoringIndex = tailoringIndex;
  54                         TailoringCount = tailoringCount;
  55                         FrenchSort = frenchSort;
  56                 }
  57         }
  58
  59         #region Tailoring support classes
  60         // Possible mapping types are:
  61         //
  62         //      - string to string (ReplacementMap)
  63         //      - string to SortKey (SortKeyMap)
  64         //      - diacritical byte to byte (DiacriticalMap)
  65         //
  66         // There could be mapping from string to sortkeys, but
  67         // for now there is none as such.
  68         //
  69         internal class Contraction
  70         {
  71                 public int Index;
  72                 public readonly char [] Source;
  73                 // only either of them is used.
  74                 public readonly string Replacement;
  75                 public readonly byte [] SortKey;
  76
  77                 public Contraction (int index, char [] source,
  78                         string replacement, byte [] sortkey)
  79                 {
  80                         Index = index;
  81                         Source = source;
  82                         Replacement = replacement;
  83                         SortKey = sortkey;
  84                 }
  85         }
  86
  87         internal class ContractionComparer : IComparer<Contraction>
  88         {
  89                 public static readonly ContractionComparer Instance =
  90                         new ContractionComparer ();
  91
  92                 public int Compare (Contraction c1, Contraction c2)
  93                 {
  94                         char [] a1 = c1.Source;
  95                         char [] a2 = c2.Source;
  96                         int min = a1.Length > a2.Length ?
  97                                 a2.Length : a1.Length;
  98                         for (int i = 0; i < min; i++)
  99                                 if (a1 [i] != a2 [i])
 100                                         return a1 [i] - a2 [i];
 101                         if (a1.Length != a2.Length)
 102                                 return a1.Length - a2.Length;
 103                         // This makes the sorting stable, since we are using Array.Sort () which is
 104                         // not stable
 105                         return c1.Index - c2.Index;
 106                 }
 107         }
 108
 109         internal class Level2Map
 110         {
 111                 public byte Source;
 112                 public byte Replace;
 113
 114                 public Level2Map (byte source, byte replace)
 115                 {
 116                         Source = source;
 117                         Replace = replace;
 118                 }
 119         }
 120
 121         #endregion
 122
 123         unsafe internal class MSCompatUnicodeTable
 124         {
 125                 public static int MaxExpansionLength = 3;
 126
 127                 static readonly byte* ignorableFlags;
 128                 static readonly byte* categories;
 129                 static readonly byte* level1;
 130                 static readonly byte* level2;
 131                 static readonly byte* level3;
 132 //              static readonly ushort* widthCompat;
 133 #if USE_C_HEADER
 134                 static readonly char* tailoring;
 135 #endif
 136                 static byte* cjkCHScategory;
 137                 static byte* cjkCHTcategory;
 138                 static byte* cjkJAcategory;
 139                 static byte* cjkKOcategory;
 140                 static byte* cjkCHSlv1;
 141                 static byte* cjkCHTlv1;
 142                 static byte* cjkJAlv1;
 143                 static byte* cjkKOlv1;
 144                 static byte* cjkKOlv2;
 145
 146                 const int ResourceVersionSize = 1;
 147
 148                 public static TailoringInfo GetTailoringInfo (int lcid)
 149                 {
 150                         for (int i = 0; i < tailoringInfos.Length; i++)
 151                                 if (tailoringInfos [i].LCID == lcid)
 152                                         return tailoringInfos [i];
 153                         return null;
 154                 }
 155
 156                 unsafe public static void BuildTailoringTables (CultureInfo culture,
 157                         TailoringInfo t,
 158                         ref Contraction [] contractions,
 159                         ref Level2Map [] diacriticals)
 160                 {
 161                         // collect tailoring entries.
 162                         var cmaps = new List<Contraction> ();
 163                         var dmaps = new List<Level2Map> ();
 164                         int iindex = 0;
 165                         fixed (char* tarr = tailoringArr){
 166                                 int idx = t.TailoringIndex;
 167                                 int end = idx + t.TailoringCount;
 168                                 while (idx < end) {
 169                                         int ss = idx + 1;
 170                                         char [] src = null;
 171                                         switch (tarr [idx]) {
 172                                         case '\x1': // SortKeyMap
 173                                                 idx++;
 174                                                 while (tarr [ss] != 0)
 175                                                         ss++;
 176                                                 src = new char [ss - idx];
 177                                                 //                                      Array.Copy (tarr, idx, src, 0, ss - idx);
 178                                                 Marshal.Copy ((IntPtr) (tarr + idx), src, 0, ss - idx);
 179                                                 byte [] sortkey = new byte [4];
 180                                                 for (int i = 0; i < 4; i++)
 181                                                         sortkey [i] = (byte) tarr [ss + 1 + i];
 182                                                 cmaps.Add (new Contraction (iindex,
 183                                                                             src, null, sortkey));
 184                                                 // it ends with 0
 185                                                 idx = ss + 6;
 186                                                 iindex ++;
 187                                                 break;
 188                                         case '\x2': // DiacriticalMap
 189                                                 dmaps.Add (new Level2Map (
 190                                                                           (byte) tarr [idx + 1],
 191                                                                           (byte) tarr [idx + 2]));
 192                                                 idx += 3;
 193                                                 break;
 194                                         case '\x3': // ReplacementMap
 195                                                 idx++;
 196                                                 while (tarr [ss] != 0)
 197                                                         ss++;
 198                                                 src = new char [ss - idx];
 199                                                 //                                      Array.Copy (tarr, idx, src, 0, ss - idx);
 200                                                 Marshal.Copy ((IntPtr) (tarr + idx), src, 0, ss - idx);
 201                                                 ss++;
 202                                                 int l = ss;
 203                                                 while (tarr [l] != 0)
 204                                                         l++;
 205                                                 string r = new string (tarr, ss, l - ss);
 206                                                 cmaps.Add (new Contraction (iindex,
 207                                                                             src, r, null));
 208                                                 idx = l + 1;
 209                                                 iindex ++;
 210                                                 break;
 211                                         default:
 212                                                 throw new NotImplementedException (String.Format ("Mono INTERNAL ERROR (Should not happen): Collation tailoring table is broken for culture {0} ({1}) at 0x{2:X}", culture.LCID, culture.Name, idx));
 213                                         }
 214                                 }
 215                         }
 216                         cmaps.Sort (ContractionComparer.Instance);
 217                         dmaps.Sort ((a, b) => a.Source - b.Source);
 218                         contractions = cmaps.ToArray ();
 219                         diacriticals = dmaps.ToArray ();
 220                 }
 221
 222                 static void SetCJKReferences (string name,
 223                         ref CodePointIndexer cjkIndexer,
 224                         ref byte* catTable, ref byte* lv1Table,
 225                         ref CodePointIndexer lv2Indexer, ref byte* lv2Table)
 226                 {
 227                         // as a part of mscorlib.dll, this invocation is
 228                         // somewhat extraneous (pointers were already assigned).
 229
 230                         switch (name) {
 231                         case "zh-CHS":
 232                                 catTable = cjkCHScategory;
 233                                 lv1Table = cjkCHSlv1;
 234                                 cjkIndexer = UUtil.CjkCHS;
 235                                 break;
 236                         case "zh-CHT":
 237                                 catTable = cjkCHTcategory;
 238                                 lv1Table = cjkCHTlv1;
 239                                 cjkIndexer = UUtil.Cjk;
 240                                 break;
 241                         case "ja":
 242                                 catTable = cjkJAcategory;
 243                                 lv1Table = cjkJAlv1;
 244                                 cjkIndexer = UUtil.Cjk;
 245                                 break;
 246                         case "ko":
 247                                 catTable = cjkKOcategory;
 248                                 lv1Table = cjkKOlv1;
 249                                 lv2Table = cjkKOlv2;
 250                                 cjkIndexer = UUtil.Cjk;
 251                                 lv2Indexer = UUtil.Cjk;
 252                                 break;
 253                         }
 254                 }
 255
 256                 public static byte Category (int cp)
 257                 {
 258                         return categories [UUtil.Category.ToIndex (cp)];
 259                 }
 260
 261                 public static byte Level1 (int cp)
 262                 {
 263                         return level1 [UUtil.Level1.ToIndex (cp)];
 264                 }
 265
 266                 public static byte Level2 (int cp)
 267                 {
 268                         return level2 [UUtil.Level2.ToIndex (cp)];
 269                 }
 270
 271                 public static byte Level3 (int cp)
 272                 {
 273                         return level3 [UUtil.Level3.ToIndex (cp)];
 274                 }
 275
 276                 public static bool IsSortable (string s)
 277                 {
 278                         foreach (char c in s)
 279                                 if (!IsSortable (c))
 280                                         return false;
 281                         return true;
 282                 }
 283
 284                 public static bool IsSortable (int cp)
 285                 {
 286                         // LAMESPEC: they should strictly match with
 287                         // IsIgnorable() result, but sometimes it does not.
 288                         if (!IsIgnorable (cp))
 289                                 return true;
 290                         switch (cp) {
 291                         case 0:
 292                         case 0x0640:
 293                         case 0xFEFF:
 294                                 return true;
 295                         }
 296                         return 0x180B <= cp && cp <= 0x180E ||
 297                                 0x200C <= cp && cp <= 0x200F ||
 298                                 0x202A <= cp && cp <= 0x202E ||
 299                                 0x206A <= cp && cp <= 0x206F ||
 300                                 0x200C <= cp && cp <= 0x200F ||
 301                                 0xFFF9 <= cp && cp <= 0xFFFD;
 302                 }
 303
 304                 public static bool IsIgnorable (int cp)
 305                 {
 306                         return IsIgnorable (cp, 1);
 307                 }
 308
 309                 public static bool IsIgnorable (int cp, byte flag)
 310                 {
 311                         if (cp == 0)
 312                                 return true;
 313                         if ((flag & 1) != 0) {
 314                                 UnicodeCategory uc = Char.GetUnicodeCategory ((char) cp);
 315                                 // This check eliminates some extraneous code areas
 316                                 if (uc == UnicodeCategory.OtherNotAssigned)
 317                                         return true;
 318                                 // Some characters in Surrogate area are ignored.
 319                                 if (0xD880 <= cp && cp < 0xDB80)
 320                                         return true;
 321                         }
 322                         int i = UUtil.Ignorable.ToIndex (cp);
 323                         return i >= 0 && (ignorableFlags [i] & flag) != 0;
 324                 }
 325                 // Verifier:
 326                 // for (int i = 0; i <= char.MaxValue; i++)
 327                 //      if (Char.GetUnicodeCategory ((char) i)
 328                 //              == UnicodeCategory.OtherNotAssigned
 329                 //              && ignorableFlags [i] != 7)
 330                 //              Console.WriteLine ("{0:X04}", i);
 331
 332                 public static bool IsIgnorableSymbol (int cp)
 333                 {
 334                         return IsIgnorable (cp, 2);
 335 //                      int i = UUtil.Ignorable.ToIndex (cp);
 336 //                      return i >= 0 && (ignorableFlags [i] & 0x2) != 0;
 337                 }
 338
 339                 public static bool IsIgnorableNonSpacing (int cp)
 340                 {
 341                         return IsIgnorable (cp, 4);
 342 //                      int i = UUtil.Ignorable.ToIndex (cp);
 343 //                      return i >= 0 && (ignorableFlags [i] & 0x4) != 0;
 344
 345                         // It could be implemented this way, but the above
 346                         // is faster.
 347 //                      return categories [UUtil.Category.ToIndex (cp)] == 1;
 348                 }
 349
 350                 public static int ToKanaTypeInsensitive (int i)
 351                 {
 352                         // Note that IgnoreKanaType does not treat half-width
 353                         // katakana as equivalent to full-width ones.
 354
 355                         // Thus, it is so simple ;-)
 356                         return (0x3041 <= i && i <= 0x3094) ? i + 0x60 : i;
 357                 }
 358
 359                 // Note that currently indexer optimizes this table a lot,
 360                 // which might have resulted in bugs.
 361                 public static int ToWidthCompat (int i)
 362                 {
 363                         if (i < 0x2190)
 364                                 return i;
 365                         if (i > 0xFF00) {
 366                                 if (i <= 0xFF5E)
 367                                         return i - 0xFF00 + 0x20;
 368                                 switch (i) {
 369                                 case 0xFFE0:
 370                                         return 0xA2;
 371                                 case 0xFFE1:
 372                                         return 0xA3;
 373                                 case 0xFFE2:
 374                                         return 0xAC;
 375                                 case 0xFFE3:
 376                                         return 0xAF;
 377                                 case 0xFFE4:
 378                                         return 0xA6;
 379                                 case 0xFFE5:
 380                                         return 0xA5;
 381                                 case 0xFFE6:
 382                                         return 0x20A9;
 383                                 }
 384                         }
 385                         if (i > 0x32FE)
 386                                 return i;
 387
 388                         if (i <= 0x2193)
 389                                 return 0xFFE9 - 0x2190 + i;
 390                         if (i < 0x2502)
 391                                 return i;
 392                         if (i <= 0x25CB) {
 393                                 switch (i) {
 394                                 case 0x2502:
 395                                         return 0xFFE8;
 396                                 case 0x25A0:
 397                                         return 0xFFED;
 398                                 case 0x25CB:
 399                                         return 0xFFEE;
 400                                 default:
 401                                         return i;
 402                                 }
 403                         }
 404                         if (i < 0x3000)
 405                                 return i;
 406                         if (i < 0x3131) {
 407                                 switch (i) {
 408                                 case 0x3000:
 409                                         return 0x20;
 410                                 case 0x3001:
 411                                         return 0xFF64;
 412                                 case 0x3002:
 413                                         return 0xFF61;
 414                                 case 0x300C:
 415                                         return 0xFF62;
 416                                 case 0x300D:
 417                                         return 0xFF63;
 418                                 case 0x30FB:
 419                                         return 0xFF65;
 420                                 // Other Kana compat characters' width
 421                                 // compatibility is considered in special weight.
 422                                 default:
 423                                         return i;
 424                                 }
 425                         }
 426                         if (i < 0x3164) { // Hangul compat
 427                                 return i - 0x3130 + 0xFFA0;
 428                         }
 429                         if (i == 0x3164)
 430                                 return 0xFFA0;
 431                         // 0x32D0-0x32FE are Kana compat characters, whose
 432                         // width compatibility is considered in special weight.
 433                         return i;
 434                 }
 435
 436                 #region Level 4 properties (Kana)
 437
 438                 public static bool HasSpecialWeight (char c)
 439                 {
 440                         if (c < '\u3041')
 441                                 return false;
 442                         else if ('\uFF66' <= c && c < '\uFF9E')
 443                                 return true;
 444                         else if ('\u3300' <= c)
 445                                 return false;
 446                         else if (c < '\u309D')
 447                                 return (c < '\u3099');
 448                         else if (c < '\u3100')
 449                                 return c != '\u30FB';
 450                         else if (c < '\u32D0')
 451                                 return false;
 452                         else if (c < '\u32FF')
 453                                 return true;
 454                         return false;
 455                 }
 456
 457                 // FIXME: it should be removed at some stage
 458                 // (will become unused).
 459                 public static byte GetJapaneseDashType (char c)
 460                 {
 461                         switch (c) {
 462                         case '\u309D':
 463                         case '\u309E':
 464                         case '\u30FD':
 465                         case '\u30FE':
 466                         case '\uFF70':
 467                                 return 4;
 468                         case '\u30FC':
 469                                 return 5;
 470                         }
 471                         return 3;
 472                 }
 473
 474                 public static bool IsHalfWidthKana (char c)
 475                 {
 476                         return '\uFF66' <= c && c <= '\uFF9D';
 477                 }
 478
 479                 public static bool IsHiragana (char c)
 480                 {
 481                         return '\u3041' <= c && c <= '\u3094';
 482                 }
 483
 484                 public static bool IsJapaneseSmallLetter (char c)
 485                 {
 486                         if ('\uFF67' <= c && c <= '\uFF6F')
 487                                 return true;
 488                         if ('\u3040' < c && c < '\u30FA') {
 489                                 switch (c) {
 490                                 case '\u3041':
 491                                 case '\u3043':
 492                                 case '\u3045':
 493                                 case '\u3047':
 494                                 case '\u3049':
 495                                 case '\u3063':
 496                                 case '\u3083':
 497                                 case '\u3085':
 498                                 case '\u3087':
 499                                 case '\u308E':
 500                                 case '\u30A1':
 501                                 case '\u30A3':
 502                                 case '\u30A5':
 503                                 case '\u30A7':
 504                                 case '\u30A9':
 505                                 case '\u30C3':
 506                                 case '\u30E3':
 507                                 case '\u30E5':
 508                                 case '\u30E7':
 509                                 case '\u30EE':
 510                                 case '\u30F5':
 511                                 case '\u30F6':
 512                                         return true;
 513                                 }
 514                         }
 515                         return false;
 516                 }
 517
 518                 #endregion
 519
 520 #if GENERATE_TABLE
 521
 522                 public static readonly bool IsReady = true; // always
 523
 524                 static MSCompatUnicodeTable ()
 525                 {
 526                         throw new Exception ("This code should not be used");
 527
 528                         fixed (byte* tmp = ignorableFlagsArr) {
 529                                 ignorableFlags = tmp;
 530                         }
 531                         fixed (byte* tmp = categoriesArr) {
 532                                 categories = tmp;
 533                         }
 534                         fixed (byte* tmp = level1Arr) {
 535                                 level1 = tmp;
 536                         }
 537                         fixed (byte* tmp = level2Arr) {
 538                                 level2 = tmp;
 539                         }
 540                         fixed (byte* tmp = level3Arr) {
 541                                 level3 = tmp;
 542                         }
 543 //                      fixed (ushort* tmp = widthCompatArr) {
 544 //                              widthCompat = tmp;
 545 //                      }
 546                         fixed (char* tmp = tailoringArr) {
 547                                 tailoring = tmp;
 548                         }
 549                         fixed (byte* tmp = cjkCHSArr) {
 550                                 cjkCHScategory = tmp;
 551                                 cjkCHSlv1 = tmp + cjkCHSArrLength;
 552                         }
 553                         fixed (byte* tmp = cjkCHTArr) {
 554                                 cjkCHTcategory = tmp;
 555                                 cjkCHTlv1 = tmp + cjkCHTArrLength;
 556                         }
 557                         fixed (byte* tmp = cjkJAArr) {
 558                                 cjkJAcategory = tmp;
 559                                 cjkJAlv1 = tmp + cjkJAArrLength;
 560                         }
 561                         fixed (byte* tmp = cjkKOArr) {
 562                                 cjkKOcategory = tmp;
 563                                 cjkKOlv1 = tmp + cjkKOArrLength;
 564                         }
 565                         fixed (byte* tmp = cjkKOlv2Arr) {
 566                                 cjkKOlv2 = tmp;
 567                         }
 568                 }
 569
 570                 public static void FillCJK (string name,
 571                         ref CodePointIndexer cjkIndexer,
 572                         ref byte* catTable, ref byte* lv1Table,
 573                         ref CodePointIndexer cjkLv2Indexer,
 574                         ref byte* lv2Table)
 575                 {
 576                         SetCJKReferences (name, ref cjkIndexer,
 577                                 ref catTable, ref lv1Table,
 578                                 ref cjkLv2Indexer, ref lv2Table);
 579                 }
 580 #else
 581
 582 #if !USE_C_HEADER
 583                 static readonly char [] tailoringArr;
 584 #endif
 585                 static readonly TailoringInfo [] tailoringInfos;
 586                 static object forLock = new object ();
 587                 public static readonly bool isReady;
 588
 589                 public static bool IsReady {
 590                         get { return isReady; }
 591                 }
 592
 593 #if USE_MANAGED_RESOURCE
 594                 static IntPtr GetResource (string name)
 595                 {
 596                         int size;
 597                         Module module;
 598                         return Assembly.GetExecutingAssembly ().GetManifestResourceInternal (name, out size, out module);
 599                 }
 600 #elif USE_C_HEADER
 601                 const int CollationTableIdxIgnorables = 0;
 602                 const int CollationTableIdxCategory = 1;
 603                 const int CollationTableIdxLevel1 = 2;
 604                 const int CollationTableIdxLevel2 = 3;
 605                 const int CollationTableIdxLevel3 = 4;
 606                 const int CollationTableIdxTailoringInfos = 5;
 607                 const int CollationTableIdxTailoringChars = 6;
 608                 const int CollationTableIdxCjkCHS = 7;
 609                 const int CollationTableIdxCjkCHT = 8;
 610                 const int CollationTableIdxCjkJA = 9;
 611                 const int CollationTableIdxCjkKO = 10;
 612                 const int CollationTableIdxCjkKOLv2 = 11;
 613
 614                 [MethodImplAttribute (MethodImplOptions.InternalCall)]
 615                 static extern void load_collation_resource (int resource_index, byte** data);
 616 #else
 617                 static readonly string corlibPath = Assembly.GetExecutingAssembly ().Location;
 618
 619                 const int CollationResourceCore = 0;
 620                 const int CollationResourceCJKCHS = 1;
 621                 const int CollationResourceCJKCHT = 2;
 622                 const int CollationResourceCJKJA = 3;
 623                 const int CollationResourceCJKKO = 4;
 624                 const int CollationResourceCJKKOlv2 = 5;
 625                 const int CollationResourceTailoring = 6;
 626
 627                 [MethodImplAttribute (MethodImplOptions.InternalCall)]
 628                 static extern void load_collation_resource (string path, int resource_index, byte** data, int* size);
 629 #endif
 630
 631                 static uint UInt32FromBytePtr (byte* raw, uint idx)
 632                 {
 633                         return (uint) (raw [idx] + (raw [idx + 1] << 8)
 634                                 + (raw [idx + 2] << 16) + (raw [idx + 3] << 24));
 635                 }
 636
 637                 static MSCompatUnicodeTable ()
 638                 {
 639 #if USE_C_HEADER
 640                         byte* raw;
 641                         uint* tailor;
 642                         uint size;
 643                         uint idx = 0;
 644
 645                         lock (forLock) {
 646                                 load_collation_resource (CollationTableIdxIgnorables, &raw);
 647                                 ignorableFlags = raw;
 648                                 load_collation_resource (CollationTableIdxCategory, &raw);
 649                                 categories = raw;
 650                                 load_collation_resource (CollationTableIdxLevel1, &raw);
 651                                 level1 = raw;
 652                                 load_collation_resource (CollationTableIdxLevel2, &raw);
 653                                 level2 = raw;
 654                                 load_collation_resource (CollationTableIdxLevel3, &raw);
 655                                 level3 = raw;
 656                                 load_collation_resource (CollationTableIdxTailoringInfos, &raw);
 657                                 tailor = (uint*) raw;
 658                                 load_collation_resource (CollationTableIdxTailoringChars, &raw);
 659                                 tailoring = (char*) raw;
 660                         }
 661
 662                         idx = 0;
 663                         uint count = tailor [idx++];
 664                         tailoringInfos = new TailoringInfo [count];
 665                         for (int i = 0; i < count; i++) {
 666                                 int i1 = (int) tailor [idx++];
 667                                 int i2 = (int) tailor [idx++];
 668                                 int i3 = (int) tailor [idx++];
 669                                 TailoringInfo ti = new TailoringInfo (
 670                                         i1, i2, i3, tailor [idx++] != 0);
 671                                 tailoringInfos [i] = ti;
 672                         }
 673
 674                         isReady = true;
 675 #else
 676
 677                         byte* raw;
 678                         byte* tailor;
 679                         uint size;
 680                         uint idx = 0;
 681
 682 #if USE_MANAGED_RESOURCE
 683                         IntPtr ptr = GetResource ("collation.core.bin");
 684                         if (ptr == IntPtr.Zero)
 685                                 return;
 686                         raw = (byte*) ((void*) ptr);
 687                         ptr = GetResource ("collation.tailoring.bin");
 688                         if (ptr == IntPtr.Zero)
 689                                 return;
 690                         tailor = (byte*) ((void*) ptr);
 691 #else
 692                         int rawsize;
 693                         int trawsize;
 694
 695                         lock (forLock) {
 696                                 load_collation_resource (corlibPath, CollationResourceCore, &raw, &rawsize);
 697                                 load_collation_resource (corlibPath, CollationResourceTailoring, &tailor, &trawsize);
 698                                 load_collation_resource (corlibPath, CollationResourceTailoringChars, &tailorChars, &trawsize);
 699                         }
 700 #endif
 701
 702                         if (raw == null || tailor == null)
 703                                 return;
 704                         // check resource version
 705                         if (raw [0] != UUtil.ResourceVersion ||
 706                                 tailor [0] != UUtil.ResourceVersion)
 707                                 return;
 708
 709                         idx = 1;
 710                         size = UInt32FromBytePtr (raw, idx);
 711                         idx += 4;
 712                         ignorableFlags = raw + idx;
 713                         idx += size;
 714
 715                         size = UInt32FromBytePtr (raw, idx);
 716                         idx += 4;
 717                         categories = raw + idx;
 718                         idx += size;
 719
 720                         size = UInt32FromBytePtr (raw, idx);
 721                         idx += 4;
 722                         level1 = raw + idx;
 723                         idx += size;
 724
 725                         size = UInt32FromBytePtr (raw, idx);
 726                         idx += 4;
 727                         level2 = raw + idx;
 728                         idx += size;
 729
 730                         size = UInt32FromBytePtr (raw, idx);
 731                         idx += 4;
 732                         level3 = raw + idx;
 733                         idx += size;
 734
 735 //                      size = UInt32FromBytePtr (raw, idx);
 736 //                      idx += 4;
 737 //                      widthCompat = (ushort*) (raw + idx);
 738 //                      idx += size * 2;
 739
 740                         // tailoring
 741
 742                         idx = 1;
 743                         uint count = UInt32FromBytePtr (tailor, idx);
 744                         idx += 4;
 745                         tailoringInfos = new TailoringInfo [count];
 746                         for (int i = 0; i < count; i++) {
 747                                 int i1 = (int) UInt32FromBytePtr (tailor, idx);
 748                                 idx += 4;
 749                                 int i2 = (int) UInt32FromBytePtr (tailor, idx);
 750                                 idx += 4;
 751                                 int i3 = (int) UInt32FromBytePtr (tailor, idx);
 752                                 idx += 4;
 753                                 TailoringInfo ti = new TailoringInfo (
 754                                         i1, i2, i3, tailor [idx++] != 0);
 755                                 tailoringInfos [i] = ti;
 756                         }
 757                         idx += 2; // dummy
 758                         // tailorings
 759                         count = UInt32FromBytePtr (tailor, idx);
 760                         idx += 4;
 761
 762                         tailoringArr = new char [count];
 763                         for (int i = 0; i < count; i++, idx += 2)
 764                                 tailoringArr [i] = (char) (tailor [idx] + (tailor [idx + 1] << 8));
 765                         isReady = true;
 766 #endif
 767                 }
 768
 769                 public static void FillCJK (string culture,
 770                         ref CodePointIndexer cjkIndexer,
 771                         ref byte* catTable,
 772                         ref byte* lv1Table,
 773                         ref CodePointIndexer lv2Indexer,
 774                         ref byte* lv2Table)
 775                 {
 776                         lock (forLock) {
 777                                 FillCJKCore (culture, ref cjkIndexer,
 778                                         ref catTable, ref lv1Table,
 779                                         ref lv2Indexer, ref lv2Table);
 780                                 SetCJKReferences (culture, ref cjkIndexer,
 781                                         ref catTable, ref lv1Table,
 782                                         ref lv2Indexer, ref lv2Table);
 783                         }
 784                 }
 785
 786                 static void FillCJKCore (string culture,
 787                         ref CodePointIndexer cjkIndexer,
 788                         ref byte* catTable, ref byte* lv1Table,
 789                         ref CodePointIndexer cjkLv2Indexer, ref byte* lv2Table)
 790                 {
 791                         if (!IsReady)
 792                                 return;
 793
 794                         string name = null;
 795                         switch (culture) {
 796                         case "zh-CHS":
 797                                 name = "cjkCHS";
 798                                 catTable = cjkCHScategory;
 799                                 lv1Table = cjkCHSlv1;
 800                                 break;
 801                         case "zh-CHT":
 802                                 name = "cjkCHT";
 803                                 catTable = cjkCHTcategory;
 804                                 lv1Table = cjkCHTlv1;
 805                                 break;
 806                         case "ja":
 807                                 name = "cjkJA";
 808                                 catTable = cjkJAcategory;
 809                                 lv1Table = cjkJAlv1;
 810                                 break;
 811                         case "ko":
 812                                 name = "cjkKO";
 813                                 catTable = cjkKOcategory;
 814                                 lv1Table = cjkKOlv1;
 815                                 break;
 816                         }
 817
 818                         if (name == null || lv1Table != null)
 819                                 return;
 820
 821                         byte* raw;
 822                         uint idx = 0;
 823 #if USE_MANAGED_RESOURCE
 824                         string filename =
 825                                 String.Format ("collation.{0}.bin", name);
 826                         IntPtr ptr = GetResource (filename);
 827                         if (ptr == IntPtr.Zero)
 828                                 return;
 829                         raw = (byte*) ((void*) ptr);
 830                         idx += ResourceVersionSize;
 831 #elif USE_C_HEADER
 832                         int residx = -1;
 833                         switch (culture) {
 834                         case "zh-CHS": residx = CollationTableIdxCjkCHS; break;
 835                         case "zh-CHT": residx = CollationTableIdxCjkCHT; break;
 836                         case "ja": residx = CollationTableIdxCjkJA; break;
 837                         case "ko": residx = CollationTableIdxCjkKO; break;
 838                         }
 839                         if (residx < 0)
 840                                 return;
 841                         load_collation_resource (residx, &raw);
 842 #else
 843                         int size;
 844                         int residx = -1;
 845                         switch (culture) {
 846                         case "zh-CHS": residx = CollationResourceCJKCHS; break;
 847                         case "zh-CHT": residx = CollationResourceCJKCHT; break;
 848                         case "ja": residx = CollationResourceCJKJA; break;
 849                         case "ko": residx = CollationResourceCJKKO; break;
 850                         }
 851                         if (residx < 0)
 852                                 return;
 853                         load_collation_resource (corlibPath, residx, &raw, &size);
 854                         idx += ResourceVersionSize;
 855 #endif
 856                         uint count = UInt32FromBytePtr (raw, idx);
 857                         idx += 4;
 858                         catTable = (byte*) raw + idx;
 859                         lv1Table = (byte*) raw + idx + count;
 860
 861                         switch (culture) {
 862                         case "zh-CHS":
 863                                 cjkCHScategory = catTable;
 864                                 cjkCHSlv1 = lv1Table;
 865                                 break;
 866                         case "zh-CHT":
 867                                 cjkCHTcategory = catTable;
 868                                 cjkCHTlv1 = lv1Table;
 869                                 break;
 870                         case "ja":
 871                                 cjkJAcategory = catTable;
 872                                 cjkJAlv1 = lv1Table;
 873                                 break;
 874                         case "ko":
 875                                 cjkKOcategory = catTable;
 876                                 cjkKOlv1 = lv1Table;
 877                                 break;
 878                         }
 879
 880                         if (name != "cjkKO")
 881                                 return;
 882 #if USE_MANAGED_RESOURCE
 883                         ptr = GetResource ("collation.cjkKOlv2.bin");
 884                         if (ptr == IntPtr.Zero)
 885                                 return;
 886                         raw = (byte*) ((void*) ptr);
 887                         idx = ResourceVersionSize + 4;
 888 #elif USE_C_HEADER
 889                         load_collation_resource (CollationTableIdxCjkKOLv2, &raw);
 890 #else
 891                         load_collation_resource (corlibPath, CollationResourceCJKKOlv2, &raw, &size);
 892                         idx = ResourceVersionSize + 4;
 893 #endif
 894                         cjkKOlv2 = raw + idx;
 895                         lv2Table = cjkKOlv2;
 896                 }
 897         }
 898 }
 899 #endif
 900
 901
 902                 // For "categories", 0 means no primary weight. 6 means
 903                 // variable weight
 904                 // For expanded character the value is never filled (i.e. 0).
 905                 // Those arrays will be split into blocks (<3400 and >F800)
 906                 // level 4 is computed.
 907
 908                 // public static bool HasSpecialWeight (char c)
 909                 // { return level1 [(int) c] == 6; }
 910
 911                 //
 912                 // autogenerated code or icall to fill array runs here
 913                 //
 914