mcs/class/corlib/Mono.Globalization.Unicode/create-mscompat-collation-table.cs

   1 //
   2 // create-mscompat-collation-table.cs : generates Windows-like sortkey tables.
   3 //
   4 // Author:
   5 //      Atsushi Enomoto  <atsushi@ximian.com>
   6 //
   7 // Copyright (C) 2005 Novell, Inc (http://www.novell.com)
   8 //
   9 // Permission is hereby granted, free of charge, to any person obtaining
  10 // a copy of this software and associated documentation files (the
  11 // "Software"), to deal in the Software without restriction, including
  12 // without limitation the rights to use, copy, modify, merge, publish,
  13 // distribute, sublicense, and/or sell copies of the Software, and to
  14 // permit persons to whom the Software is furnished to do so, subject to
  15 // the following conditions:
  16 //
  17 // The above copyright notice and this permission notice shall be
  18 // included in all copies or substantial portions of the Software.
  19 //
  20 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  21 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  22 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  23 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  24 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  25 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  26 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  27 //
  28
  29 //
  30 // There are two kind of sort keys : which are computed and which are laid out
  31 // as an indexed array. Computed sort keys are:
  32 //
  33 //      - Surrogate
  34 //      - PrivateUse
  35 //
  36 // Though it is possible to "compute" level 3 weights, they are still dumped
  37 // to an array to avoid execution cost.
  38 //
  39 #define Binary
  40
  41 using System;
  42 using System.IO;
  43 using System.Collections;
  44 using System.Globalization;
  45 using System.Text;
  46 using System.Xml;
  47
  48 using UUtil = Mono.Globalization.Unicode.MSCompatUnicodeTableUtil;
  49
  50 namespace Mono.Globalization.Unicode
  51 {
  52         internal class MSCompatSortKeyTableGenerator
  53         {
  54                 public static void Main (string [] args)
  55                 {
  56                         new MSCompatSortKeyTableGenerator ().Run (args);
  57                 }
  58
  59                 const int DecompositionWide = 1; // fixed
  60                 const int DecompositionSub = 2; // fixed
  61                 const int DecompositionSmall = 3;
  62                 const int DecompositionIsolated = 4;
  63                 const int DecompositionInitial = 5;
  64                 const int DecompositionFinal = 6;
  65                 const int DecompositionMedial = 7;
  66                 const int DecompositionNoBreak = 8;
  67                 const int DecompositionVertical = 9;
  68                 const int DecompositionFraction = 0xA;
  69                 const int DecompositionFont = 0xB;
  70                 const int DecompositionSuper = 0xC; // fixed
  71                 const int DecompositionFull = 0xE;
  72                 const int DecompositionNarrow = 0xD;
  73                 const int DecompositionCircle = 0xF;
  74                 const int DecompositionSquare = 0x10;
  75                 const int DecompositionCompat = 0x11;
  76                 const int DecompositionCanonical = 0x12;
  77
  78                 TextWriter CSResult = Console.Out;
  79                 TextWriter CResult = TextWriter.Null;
  80
  81                 byte [] fillIndex = new byte [256]; // by category
  82                 CharMapEntry [] map = new CharMapEntry [char.MaxValue + 1];
  83
  84                 char [] specialIgnore = new char [] {
  85                         '\u3099', '\u309A', '\u309B', '\u309C', '\u0BCD',
  86                         '\u0E47', '\u0E4C', '\uFF9E', '\uFF9F'
  87                         };
  88
  89                 // FIXME: need more love (as always)
  90                 char [] alphabets = new char [] {'A', 'B', 'C', 'D', 'E', 'F',
  91                         'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q',
  92                         'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
  93                         '\u0292', '\u01BE', '\u0298'};
  94                 byte [] alphaWeights = new byte [] {
  95                         2, 9, 0xA, 0x1A, 0x21,
  96                         0x23, 0x25, 0x2C, 0x32, 0x35,
  97                         0x36, 0x48, 0x51, 0x70, 0x7C,
  98                         0x7E, 0x89, 0x8A, 0x91, 0x99,
  99                         0x9F, 0xA2, 0xA4, 0xA6, 0xA7,
 100                         0xA9, 0xAA, 0xB3, 0xB4};
 101
 102                 bool [] isSmallCapital = new bool [char.MaxValue + 1];
 103                 bool [] isUppercase = new bool [char.MaxValue + 1];
 104
 105                 byte [] decompType = new byte [char.MaxValue + 1];
 106                 int [] decompIndex = new int [char.MaxValue + 1];
 107                 int [] decompLength = new int [char.MaxValue + 1];
 108                 int [] decompValues;
 109                 decimal [] decimalValue = new decimal [char.MaxValue + 1];
 110
 111                 byte [] diacritical = new byte [char.MaxValue + 1];
 112
 113                 string [] diacritics = new string [] {
 114                         // LATIN, CYRILLIC etc.
 115                         "VERTICAL LINE ABOVE", "UPTURN", "DOUBLE-STRUCK",
 116                         "ABKHASIAN",
 117                         "MIDDLE HOOK", "WITH VERTICAL LINE ABOVE;", "WITH TONOS",
 118                         "WITH ACUTE ACCENT;", "WITH GRAVE ACCENT;",
 119                         "WITH ACUTE;", "WITH GRAVE;",
 120                         //
 121                         "WITH DOT ABOVE;", " MIDDLE DOT;",
 122                         "WITH CIRCUMFLEX ACCENT;", "WITH CIRCUMFLEX;",
 123                         "WITH DIALYTIKA;",
 124                         "WITH DIAERESIS;", "WITH CARON;", "WITH BREVE;",
 125                         "DIALYTIKA TONOS", "DIALYTIKA AND TONOS",
 126                         "ABKHASIAN CHE WITH DESCENDER",
 127                         "WITH MACRON;", "WITH TILDE;", "WITH RING ABOVE;",
 128                         "WITH OGONEK;", "WITH CEDILLA;",
 129                         //
 130                         " DOUBLE ACUTE;", " ACUTE AND DOT ABOVE;",
 131                         "WITH STROKE;", " CIRCUMFLEX AND ACUTE;",
 132                         "STROKE OVERLAY",
 133                         " DIAERESIS AND ACUTE;", "WITH CIRCUMFLEX AND GRAVE;", " L SLASH;",
 134                         " DIAERESIS AND GRAVE;",
 135                         " BREVE AND ACUTE;",
 136                         " CARON AND DOT ABOVE;", " BREVE AND GRAVE;",
 137                         " MACRON AND ACUTE;",
 138                         " MACRON AND GRAVE;",
 139                         //
 140                         " DIAERESIS AND CARON", " DOT ABOVE AND MACRON", " TILDE AND ACUTE",
 141                         " RING ABOVE AND ACUTE",
 142                         " DIAERESIS AND MACRON", " CEDILLA AND ACUTE", " MACRON AND DIAERESIS",
 143                         " CIRCUMFLEX AND TILDE",
 144                         " TILDE AND DIAERESIS",
 145                         " STROKE AND ACUTE",
 146                         " BREVE AND TILDE",
 147                         " CEDILLA AND BREVE",
 148                         " OGONEK AND MACRON",
 149                         // 0x40
 150                         "WITH OVERLINE", "DOUBLE VERTICAL LINE ABOVE",
 151                         "WITH HOOK;", "LEFT HOOK;", " WITH HOOK ABOVE;",
 152                         " DOUBLE GRAVE",
 153                         " INVERTED BREVE",
 154                         "ROMAN NUMERAL",
 155                         " PRECEDED BY APOSTROPHE",
 156                         "WITH HORN;",
 157                         " LINE BELOW;", " CIRCUMFLEX AND HOOK ABOVE",
 158                         " PALATAL HOOK",
 159                         " DOT BELOW;",
 160                         " RETROFLEX;", "DIAERESIS BELOW", "RETROFLEX HOOK",
 161                         " RING BELOW", "LOW VERTICAL LINE",
 162                         //
 163                         " CIRCUMFLEX BELOW", "HORN AND ACUTE",
 164                         " BREVE BELOW;", " HORN AND GRAVE",
 165                         " LOW MACRON",
 166                         " TILDE BELOW",
 167                         " TOPBAR",
 168                         " DOT BELOW AND DOT ABOVE",
 169                         " RIGHT HALF RING", " HORN AND TILDE",
 170                         " CIRCUMFLEX AND DOT BELOW",
 171                         " BREVE AND DOT BELOW",
 172                         " DOT BELOW AND MACRON",
 173                         " TONE TWO",
 174                         " HORN AND HOOK ABOVE",
 175                         " HORN AND DOT",
 176                         // CIRCLED, PARENTHESIZED and so on
 177                         "CIRCLED DIGIT", "CIRCLED NUMBER", "CIRCLED LATIN",
 178                         "CIRCLED KATAKANA", "CIRCLED SANS-SERIF",
 179                         "PARENTHESIZED DIGIT", "PARENTHESIZED NUMBER", "PARENTHESIZED LATIN",
 180                         };
 181                 byte [] diacriticWeights = new byte [] {
 182                         // LATIN.
 183                         3, 3, 3, 5, 5, 5, 5,
 184                         0xE, 0xF,
 185                         0xE, 0xF,
 186                         //
 187                         0x10, 0x11, 0x12, 0x12, 0x13, 0x13, 0x14, 0x15, 0x16,
 188                         0x16, 0x17, 0x17, 0x19, 0x1A, 0x1B, 0x1C,
 189                         //
 190                         0x1D, 0x1D, 0x1E, 0x1E, 0x1E, 0x1F, 0x1F, 0x1F,
 191                         0x20, 0x21, 0x22, 0x22, 0x23, 0x24,
 192                         //
 193                         0x25, 0x25, 0x25, 0x26, 0x28, 0x28, 0x28,
 194                         0x29, 0x2A, 0x2B, 0x2C, 0x2F, 0x30,
 195                         //
 196                         0x40, 0x41, 0x43, 0x43, 0x43, 0x44, 0x46, 0x47, 0x48,
 197                         0x52, 0x55, 0x55, 0x57, 0x58, 0x59, 0x59, 0x59,
 198                         0x5A, 0x5A,
 199                         //
 200                         0x60, 0x60, 0x61, 0x61, 0x62, 0x63, 0x68, 0x68,
 201                         0x69, 0x69, 0x6A, 0x6D, 0x6E,
 202                         0x87, 0x95, 0xAA,
 203                         // CIRCLED, PARENTHESIZED and so on.
 204                         0xEE, 0xEE, 0xEE, 0xEE, 0xEE,
 205                         0xF3, 0xF3, 0xF3
 206                         };
 207
 208                 int [] numberSecondaryWeightBounds = new int [] {
 209                         0x660, 0x680, 0x6F0, 0x700, 0x960, 0x970,
 210                         0x9E0, 0x9F0, 0x9F4, 0xA00, 0xA60, 0xA70,
 211                         0xAE0, 0xAF0, 0xB60, 0xB70, 0xBE0, 0xC00,
 212                         0xC60, 0xC70, 0xCE0, 0xCF0, 0xD60, 0xD70,
 213                         0xE50, 0xE60, 0xED0, 0xEE0
 214                         };
 215
 216                 char [] orderedGurmukhi;
 217                 char [] orderedGujarati;
 218                 char [] orderedGeorgian;
 219                 char [] orderedThaana;
 220
 221                 static readonly char [] orderedTamilConsonants = new char [] {
 222                         // based on traditional Tamil consonants, except for
 223                         // Grantha (where Microsoft breaks traditionalism).
 224                         // http://www.angelfire.com/empire/thamizh/padanGaL
 225                         '\u0B95', '\u0B99', '\u0B9A', '\u0B9E', '\u0B9F',
 226                         '\u0BA3', '\u0BA4', '\u0BA8', '\u0BAA', '\u0BAE',
 227                         '\u0BAF', '\u0BB0', '\u0BB2', '\u0BB5', '\u0BB4',
 228                         '\u0BB3', '\u0BB1', '\u0BA9', '\u0B9C', '\u0BB8',
 229                         '\u0BB7', '\u0BB9'};
 230
 231                 // cp -> character name (only for some characters)
 232                 ArrayList sortableCharNames = new ArrayList ();
 233
 234                 // cp -> arrow value (int)
 235                 ArrayList arrowValues = new ArrayList ();
 236
 237                 // cp -> box value (int)
 238                 ArrayList boxValues = new ArrayList ();
 239
 240                 // cp -> level1 value
 241                 Hashtable arabicLetterPrimaryValues = new Hashtable ();
 242
 243                 // letterName -> cp
 244                 Hashtable arabicNameMap = new Hashtable ();
 245
 246                 // cp -> Hashtable [decompType] -> cp
 247                 Hashtable nfkdMap = new Hashtable ();
 248
 249                 // Latin letter -> ArrayList [int]
 250                 Hashtable latinMap = new Hashtable ();
 251
 252                 ArrayList jisJapanese = new ArrayList ();
 253                 ArrayList nonJisJapanese = new ArrayList ();
 254
 255                 ushort [] cjkJA = new ushort [char.MaxValue +1];// - 0x4E00];
 256                 ushort [] cjkCHS = new ushort [char.MaxValue +1];// - 0x3100];
 257                 ushort [] cjkCHT = new ushort [char.MaxValue +1];// - 0x4E00];
 258                 ushort [] cjkKO = new ushort [char.MaxValue +1];// - 0x4E00];
 259                 byte [] cjkKOlv2 = new byte [char.MaxValue +1];// - 0x4E00];
 260
 261                 byte [] ignorableFlags = new byte [char.MaxValue + 1];
 262
 263                 static double [] unicodeAge = new double [char.MaxValue + 1];
 264
 265                 ArrayList tailorings = new ArrayList ();
 266
 267                 void Run (string [] args)
 268                 {
 269                         string dirname = args.Length == 0 ? "downloaded" : args [0];
 270                         ParseSources (dirname);
 271                         Console.Error.WriteLine ("parse done.");
 272
 273                         ModifyParsedValues ();
 274                         GenerateCore ();
 275                         Console.Error.WriteLine ("generation done.");
 276                         CResult = new StreamWriter ("collation-tables.h", false);
 277                         Serialize ();
 278                         CResult.Close ();
 279                         Console.Error.WriteLine ("serialization done.");
 280 /*
 281 StreamWriter sw = new StreamWriter ("agelog.txt");
 282 for (int i = 0; i < char.MaxValue; i++) {
 283 bool shouldBe = false;
 284 switch (Char.GetUnicodeCategory ((char) i)) {
 285 case UnicodeCategory.Format: case UnicodeCategory.OtherNotAssigned:
 286         shouldBe = true; break;
 287 }
 288 if (unicodeAge [i] >= 3.1)
 289         shouldBe = true;
 290 //if (IsIgnorable (i) != shouldBe)
 291 sw.WriteLine ("{1} {2} {3} {0:X04} {4} {5}", i, unicodeAge [i], IsIgnorable (i), IsIgnorableSymbol (i), char.GetUnicodeCategory ((char) i), IsIgnorable (i) != shouldBe ? '!' : ' ');
 292 }
 293 sw.Close ();
 294 */
 295                 }
 296
 297                 byte [] CompressArray (byte [] source, CodePointIndexer i)
 298                 {
 299                         return (byte []) CodePointIndexer.CompressArray  (
 300                                 source, typeof (byte), i);
 301                 }
 302
 303                 ushort [] CompressArray (ushort [] source, CodePointIndexer i)
 304                 {
 305                         return (ushort []) CodePointIndexer.CompressArray  (
 306                                 source, typeof (ushort), i);
 307                 }
 308
 309                 void WriteByte (byte value)
 310                 {
 311
 312                 }
 313
 314                 void Serialize ()
 315                 {
 316                         // Tailorings
 317                         SerializeTailorings ();
 318
 319                         byte [] categories = new byte [map.Length];
 320                         byte [] level1 = new byte [map.Length];
 321                         byte [] level2 = new byte [map.Length];
 322                         byte [] level3 = new byte [map.Length];
 323 // widthCompat is now removed from the mapping table.
 324 // If it turned out that it is still required, grep this source and uncomment
 325 // widthCompat related lines. FIXME: remove those lines in the future.
 326 //                      ushort [] widthCompat = new ushort [map.Length];
 327                         for (int i = 0; i < map.Length; i++) {
 328                                 categories [i] = map [i].Category;
 329                                 level1 [i] = map [i].Level1;
 330                                 level2 [i] = map [i].Level2;
 331                                 level3 [i] = ComputeLevel3Weight ((char) i);
 332 /*
 333                                 // For Japanese Half-width characters, don't
 334                                 // map widthCompat. It is IgnoreKanaType that
 335                                 // handles those width differences.
 336                                 if (0xFF6D <= i && i <= 0xFF9D)
 337                                         continue;
 338                                 switch (decompType [i]) {
 339                                 case DecompositionNarrow:
 340                                 case DecompositionWide:
 341                                 case DecompositionSuper:
 342                                 case DecompositionSub:
 343                                         // they are always 1 char
 344                                         widthCompat [i] = (ushort) decompValues [decompIndex [i]];
 345                                         break;
 346                                 }
 347 */
 348                         }
 349
 350                         // compress
 351                         ignorableFlags = CompressArray (ignorableFlags,
 352                                 UUtil.Ignorable);
 353                         categories = CompressArray (categories, UUtil.Category);
 354                         level1 = CompressArray (level1, UUtil.Level1);
 355                         level2 = CompressArray (level2, UUtil.Level2);
 356                         level3 = CompressArray (level3, UUtil.Level3);
 357 //                      widthCompat = (ushort []) CodePointIndexer.CompressArray (
 358 //                              widthCompat, typeof (ushort), UUtil.WidthCompat);
 359                         cjkCHS = CompressArray (cjkCHS, UUtil.CjkCHS);
 360                         cjkCHT = CompressArray (cjkCHT,UUtil.Cjk);
 361                         cjkJA = CompressArray (cjkJA, UUtil.Cjk);
 362                         cjkKO = CompressArray (cjkKO, UUtil.Cjk);
 363                         cjkKOlv2 = CompressArray (cjkKOlv2, UUtil.Cjk);
 364
 365                         // Ignorables
 366                         CResult.WriteLine ("static const guint8  collation_table_ignorableFlags [] = {");
 367                         CSResult.WriteLine ("static readonly byte [] ignorableFlagsArr = new byte [] {");
 368 #if Binary
 369                         MemoryStream ms = new MemoryStream ();
 370                         BinaryWriter binary = new BinaryWriter (ms);
 371                         binary.Write (UUtil.ResourceVersion);
 372                         binary.Write (ignorableFlags.Length);
 373 #endif
 374                         for (int i = 0; i < ignorableFlags.Length; i++) {
 375                                 byte value = ignorableFlags [i];
 376                                 if (value < 10)
 377                                         CSResult.Write ("{0},", value);
 378                                 else
 379                                         CSResult.Write ("0x{0:X02},", value);
 380                                 CResult.Write ("{0},", value);
 381 #if Binary
 382                                 binary.Write (value);
 383 #endif
 384                                 if ((i & 0xF) == 0xF) {
 385                                         CSResult.WriteLine ("// {0:X04}",
 386                                                 UUtil.Ignorable.ToCodePoint (i - 0xF));
 387                                         CResult.WriteLine ();
 388                                 }
 389                         }
 390                         CResult.WriteLine ("0};");
 391                         CSResult.WriteLine ("};");
 392                         CSResult.WriteLine ();
 393
 394                         // Primary category
 395                         CResult.WriteLine ("static const guint8 collation_table_category [] = {");
 396                         CSResult.WriteLine ("static readonly byte [] categoriesArr = new byte [] {");
 397 #if Binary
 398                         binary.Write (categories.Length);
 399 #endif
 400                         for (int i = 0; i < categories.Length; i++) {
 401                                 byte value = categories [i];
 402                                 if (value < 10)
 403                                         CSResult.Write ("{0},", value);
 404                                 else
 405                                         CSResult.Write ("0x{0:X02},", value);
 406                                 CResult.Write ("{0},", value);
 407 #if Binary
 408                                 binary.Write (value);
 409 #endif
 410                                 if ((i & 0xF) == 0xF) {
 411                                         CSResult.WriteLine ("// {0:X04}",
 412                                                 UUtil.Category.ToCodePoint (i - 0xF));
 413                                         CResult.WriteLine ();
 414                                 }
 415                         }
 416                         CResult.WriteLine ("};");
 417                         CSResult.WriteLine ("};");
 418                         CSResult.WriteLine ();
 419
 420                         // Primary weight value
 421                         CResult.WriteLine ("static const guint8 collation_table_level1 [] = {");
 422                         CSResult.WriteLine ("static readonly byte [] level1Arr = new byte [] {");
 423 #if Binary
 424                         binary.Write (level1.Length);
 425 #endif
 426                         for (int i = 0; i < level1.Length; i++) {
 427                                 byte value = level1 [i];
 428                                 if (value < 10)
 429                                         CSResult.Write ("{0},", value);
 430                                 else
 431                                         CSResult.Write ("0x{0:X02},", value);
 432                                 CResult.Write ("{0},", value);
 433 #if Binary
 434                                 binary.Write (value);
 435 #endif
 436                                 if ((i & 0xF) == 0xF) {
 437                                         CSResult.WriteLine ("// {0:X04}",
 438                                                 UUtil.Level1.ToCodePoint (i - 0xF));
 439                                         CResult.WriteLine ();
 440                                 }
 441                         }
 442                         CResult.WriteLine ("0};");
 443                         CSResult.WriteLine ("};");
 444                         CSResult.WriteLine ();
 445
 446                         // Secondary weight
 447                         CResult.WriteLine ("static const guint8 collation_table_level2 [] = {");
 448                         CSResult.WriteLine ("static readonly byte [] level2Arr = new byte [] {");
 449 #if Binary
 450                         binary.Write (level2.Length);
 451 #endif
 452                         for (int i = 0; i < level2.Length; i++) {
 453                                 byte value = level2 [i];
 454                                 if (value < 10)
 455                                         CSResult.Write ("{0},", value);
 456                                 else
 457                                         CSResult.Write ("0x{0:X02},", value);
 458                                 CResult.Write ("{0},", value);
 459 #if Binary
 460                                 binary.Write (value);
 461 #endif
 462                                 if ((i & 0xF) == 0xF) {
 463                                         CSResult.WriteLine ("// {0:X04}",
 464                                                 UUtil.Level2.ToCodePoint (i - 0xF));
 465                                         CResult.WriteLine ();
 466                                 }
 467                         }
 468                         CResult.WriteLine ("0};");
 469                         CSResult.WriteLine ("};");
 470                         CSResult.WriteLine ();
 471
 472                         // Thirtiary weight
 473                         CResult.WriteLine ("static const guint8 collation_table_level3 [] = {");
 474                         CSResult.WriteLine ("static readonly byte [] level3Arr = new byte [] {");
 475 #if Binary
 476                         binary.Write (level3.Length);
 477 #endif
 478                         for (int i = 0; i < level3.Length; i++) {
 479                                 byte value = level3 [i];
 480                                 if (value < 10)
 481                                         CSResult.Write ("{0},", value);
 482                                 else
 483                                         CSResult.Write ("0x{0:X02},", value);
 484                                 CResult.Write ("{0},", value);
 485 #if Binary
 486                                 binary.Write (value);
 487 #endif
 488                                 if ((i & 0xF) == 0xF) {
 489                                         CSResult.WriteLine ("// {0:X04}",
 490                                                 UUtil.Level3.ToCodePoint (i - 0xF));
 491                                         CResult.WriteLine ();
 492                                 }
 493                         }
 494                         CResult.WriteLine ("0};");
 495                         CSResult.WriteLine ("};");
 496                         CSResult.WriteLine ();
 497
 498 /*
 499                         // Width insensitivity mappings
 500                         // (for now it is more lightweight than dumping the
 501                         // entire NFKD table).
 502                         CResult.WriteLine ("static const guint16* widthCompat [] = {");
 503                         CSResult.WriteLine ("static readonly ushort [] widthCompatArr = new ushort [] {");
 504 #if Binary
 505                         binary.Write (widthCompat.Length);
 506 #endif
 507                         for (int i = 0; i < widthCompat.Length; i++) {
 508                                 ushort value = widthCompat [i];
 509                                 if (value < 10)
 510                                         CSResult.Write ("{0},", value);
 511                                 else
 512                                         CSResult.Write ("0x{0:X02},", value);
 513                                 CResult.Write ("{0},", value);
 514 #if Binary
 515                                 binary.Write (value);
 516 #endif
 517                                 if ((i & 0xF) == 0xF) {
 518                                         CSResult.WriteLine ("// {0:X04}",
 519                                                 UUtil.WidthCompat.ToCodePoint (i - 0xF));
 520                                         CResult.WriteLine ();
 521                                 }
 522                         }
 523                         CResult.WriteLine ("0};");
 524                         CSResult.WriteLine ("};");
 525                         CSResult.WriteLine ();
 526 */
 527
 528 #if Binary
 529                         using (FileStream fs = File.Create ("../resources/collation.core.bin")) {
 530                                 byte [] array = ms.ToArray ();
 531                                 fs.Write (array, 0, array.Length);
 532                         }
 533 #endif
 534
 535                         // CJK
 536                         SerializeCJK ("cjkCHS", cjkCHS, char.MaxValue);
 537                         SerializeCJK ("cjkCHT", cjkCHT, 0x9FB0);
 538                         SerializeCJK ("cjkJA", cjkJA, 0x9FB0);
 539                         SerializeCJK ("cjkKO", cjkKO, 0x9FB0);
 540                         SerializeCJK ("cjkKOlv2", cjkKOlv2, 0x9FB0);
 541                 }
 542
 543                 void SerializeCJK (string name, ushort [] cjk, int max_unused)
 544                 {
 545 //                      CResult.WriteLine ("static const int collation_table_collation_cjk_{0}_size [] = {1};", name, cjk.Length);
 546                         CSResult.WriteLine ("const int {0}ArrLength = {1};", name, cjk.Length);
 547
 548                         int len = cjk.Length;
 549                         CResult.WriteLine ("static const guint8 collation_table_collation_cjk_{0} [] = {{", name);
 550                         CSResult.WriteLine ("static byte [] {0}Arr = new byte [] {{", name);
 551                         // the actual length is *2
 552                         for (int i = 0; i < 4; i++, len /= 256) {
 553                                 CResult.Write ("{0},", len & 0xFF);
 554                                 CSResult.Write ("0x{0:X04},", len & 0xFF);
 555                         }
 556                         CResult.WriteLine ();
 557                         CSResult.WriteLine ();
 558 #if Binary
 559                         MemoryStream ms = new MemoryStream ();
 560                         BinaryWriter binary = new BinaryWriter (ms);
 561                         binary.Write (UUtil.ResourceVersion);
 562                         binary.Write (cjk.Length); // the actual size is *2.
 563 #endif
 564                         // category
 565                         for (int i = 0; i < cjk.Length; i++) {
 566 //                              if (i == max)
 567 //                                      break;
 568                                 byte value = (byte) (cjk [i] >> 8);
 569                                 if (value < 10)
 570                                         CSResult.Write ("{0},", value);
 571                                 else
 572                                         CSResult.Write ("0x{0:X02},", value);
 573                                 CResult.Write ("{0},", value);
 574 #if Binary
 575                                 binary.Write (value);
 576 #endif
 577                                 if ((i & 0xF) == 0xF) {
 578                                         CSResult.WriteLine ("// {0:X04}", i - 0xF);
 579                                         CResult.WriteLine ();
 580                                 }
 581                         }
 582
 583                         // level 1
 584                         for (int i = 0; i < cjk.Length; i++) {
 585 //                              if (i == max)
 586 //                                      break;
 587                                 byte value = (byte) (cjk [i] & 0xFF);
 588                                 if (value < 10)
 589                                         CSResult.Write ("{0},", value);
 590                                 else
 591                                         CSResult.Write ("0x{0:X02},", value);
 592                                 CResult.Write ("{0},", value);
 593 #if Binary
 594                                 binary.Write (value);
 595 #endif
 596                                 if ((i & 0xF) == 0xF) {
 597                                         CSResult.WriteLine ("// {0:X04}", i - 0xF);
 598                                         CResult.WriteLine ();
 599                                 }
 600                         }
 601
 602                         CResult.WriteLine ("0};");
 603                         CSResult.WriteLine ("};");
 604                         CSResult.WriteLine ();
 605 #if Binary
 606                         using (FileStream fs = File.Create (String.Format ("../resources/collation.{0}.bin", name))) {
 607                                 byte [] array = ms.ToArray ();
 608                                 fs.Write (array, 0, array.Length);
 609                         }
 610 #endif
 611                 }
 612
 613                 void SerializeCJK (string name, byte [] cjk, int max)
 614                 {
 615                         CResult.WriteLine ("static const guint8 collation_table_collation_cjk_{0} [] = {{", name);
 616                         CSResult.WriteLine ("static byte [] {0}Arr = new byte [] {{", name);
 617 #if Binary
 618                         MemoryStream ms = new MemoryStream ();
 619                         BinaryWriter binary = new BinaryWriter (ms);
 620                         binary.Write (UUtil.ResourceVersion);
 621 #endif
 622                         for (int i = 0; i < cjk.Length; i++) {
 623                                 if (i == max)
 624                                         break;
 625                                 byte value = cjk [i];
 626                                 if (value < 10)
 627                                         CSResult.Write ("{0},", value);
 628                                 else
 629                                         CSResult.Write ("0x{0:X02},", value);
 630                                 CResult.Write ("{0},", value);
 631 #if Binary
 632                                 binary.Write (value);
 633 #endif
 634                                 if ((i & 0xF) == 0xF) {
 635                                         CSResult.WriteLine ("// {0:X04}", i - 0xF);
 636                                         CResult.WriteLine ();
 637                                 }
 638                         }
 639                         CResult.WriteLine ("0};");
 640                         CSResult.WriteLine ("};");
 641                         CSResult.WriteLine ();
 642 #if Binary
 643                         using (FileStream fs = File.Create (String.Format ("../resources/collation.{0}.bin", name))) {
 644                                 byte [] array = ms.ToArray ();
 645                                 fs.Write (array, 0, array.Length);
 646                         }
 647 #endif
 648                 }
 649
 650                 void SerializeTailorings ()
 651                 {
 652                         Hashtable indexes = new Hashtable ();
 653                         Hashtable counts = new Hashtable ();
 654                         CResult.WriteLine ("static const guint16 collation_table_tailoring [] = {");
 655                         CSResult.WriteLine ("static char [] tailoringArr = new char [] {");
 656                         int count = 0;
 657 #if Binary
 658                         MemoryStream ms = new MemoryStream ();
 659                         BinaryWriter binary = new BinaryWriter (ms);
 660                         // Here we don't need to output resource version.
 661                         // This is cached.
 662 #endif
 663                         foreach (Tailoring t in tailorings) {
 664                                 if (t.Alias != 0)
 665                                         continue;
 666                                 CResult.Write ("/*{0}*/", t.LCID);
 667                                 CSResult.Write ("/*{0}*/", t.LCID);
 668                                 indexes.Add (t.LCID, count);
 669                                 char [] values = t.ItemToCharArray ();
 670                                 counts.Add (t.LCID, values.Length);
 671                                 foreach (char c in values) {
 672                                         CSResult.Write ("'\\x{0:X}', ", (int) c);
 673                                         CResult.Write ("{0},", (int) c);
 674                                         if (++count % 16 == 0) {
 675                                                 CSResult.WriteLine (" // {0:X04}", count - 16);
 676                                                 CResult.WriteLine ();
 677                                         }
 678 #if Binary
 679                                         binary.Write ((ushort) c);
 680 #endif
 681                                 }
 682                         }
 683                         CResult.WriteLine ("0};");
 684                         CSResult.WriteLine ("};");
 685
 686                         CResult.WriteLine ("static const guint32 collation_table_tailoring_infos [] = {");
 687                         CResult.WriteLine ("{0}, /*count*/", tailorings.Count);
 688                         CSResult.WriteLine ("static TailoringInfo [] tailoringInfos = new TailoringInfo [] {");
 689 #if Binary
 690                         byte [] rawdata = ms.ToArray ();
 691                         ms = new MemoryStream ();
 692                         binary = new BinaryWriter (ms);
 693                         binary.Write (UUtil.ResourceVersion);
 694                         binary.Write (tailorings.Count);
 695 #endif
 696                         foreach (Tailoring t in tailorings) {
 697                                 int target = t.Alias != 0 ? t.Alias : t.LCID;
 698                                 if (!indexes.ContainsKey (target)) {
 699                                         throw new Exception (String.Format ("WARNING: no corresponding definition for tailoring alias. From {0} to {1}", t.LCID, t.Alias));
 700                                         continue;
 701                                 }
 702                                 int idx = (int) indexes [target];
 703                                 int cnt = (int) counts [target];
 704                                 bool french = t.FrenchSort;
 705                                 if (t.Alias != 0)
 706                                         foreach (Tailoring t2 in tailorings)
 707                                                 if (t2.LCID == t.LCID)
 708                                                         french = t2.FrenchSort;
 709                                 CSResult.WriteLine ("new TailoringInfo ({0}, 0x{1:X}, {2}, {3}), ", t.LCID, idx, cnt, french ? "true" : "false");
 710                                 CResult.WriteLine ("{0},{1},{2},{3},", t.LCID, idx, cnt, french ? 1 : 0);
 711 #if Binary
 712                                 binary.Write (t.LCID);
 713                                 binary.Write (idx);
 714                                 binary.Write (cnt);
 715                                 binary.Write (french);
 716 #endif
 717                         }
 718                         CResult.WriteLine ("0};");
 719                         CSResult.WriteLine ("};");
 720 #if Binary
 721                         binary.Write ((byte) 0xFF);
 722                         binary.Write ((byte) 0xFF);
 723                         binary.Write (rawdata.Length / 2);
 724                         binary.Write (rawdata, 0, rawdata.Length);
 725
 726
 727                         using (FileStream fs = File.Create ("../resources/collation.tailoring.bin")) {
 728                                 byte [] array = ms.ToArray ();
 729                                 fs.Write (array, 0, array.Length);
 730                         }
 731 #endif
 732                 }
 733
 734                 #region Parse
 735
 736                 void ParseSources (string dirname)
 737                 {
 738                         string unidata =
 739                                 dirname + "/UnicodeData.txt";
 740                         string derivedCoreProps =
 741                                 dirname + "/DerivedCoreProperties.txt";
 742                         string scripts =
 743                                 dirname + "/Scripts.txt";
 744                         string cp932 =
 745                                 dirname + "/CP932.TXT";
 746                         string derivedAge =
 747                                 dirname + "/DerivedAge.txt";
 748                         string chXML = dirname + "/common/collation/zh.xml";
 749                         string jaXML = dirname + "/common/collation/ja.xml";
 750                         string koXML = dirname + "/common/collation/ko.xml";
 751
 752                         ParseDerivedAge (derivedAge);
 753
 754                         FillIgnorables ();
 755
 756                         ParseJISOrder (cp932); // in prior to ParseUnidata()
 757                         ParseUnidata (unidata);
 758                         ModifyUnidata ();
 759                         ParseDerivedCoreProperties (derivedCoreProps);
 760                         ParseScripts (scripts);
 761                         ParseCJK (chXML, jaXML, koXML);
 762
 763                         ParseTailorings ("mono-tailoring-source.txt");
 764                 }
 765
 766                 void ParseTailorings (string filename)
 767                 {
 768                         Tailoring t = null;
 769                         int line = 0;
 770                         using (StreamReader sr = new StreamReader (filename)) {
 771                                 try {
 772                                         while (sr.Peek () >= 0) {
 773                                                 line++;
 774                                                 ProcessTailoringLine (ref t,
 775                                                         sr.ReadLine ().Trim ());
 776                                         }
 777                                 } catch (Exception) {
 778                                         Console.Error.WriteLine ("ERROR at line {0}", line);
 779                                         throw;
 780                                 }
 781                         }
 782                 }
 783
 784                 // For now this is enough.
 785                 string ParseTailoringSourceValue (string s)
 786                 {
 787                         StringBuilder sb = new StringBuilder ();
 788                         for (int i = 0; i < s.Length; i++) {
 789                                 if (i + 5 < s.Length &&
 790                                         s [i] == '\\' && s [i + 1] == 'u') {
 791                                         sb.Append (
 792                                                 (char) int.Parse (
 793                                                         s.Substring (i + 2, 4),
 794                                                         NumberStyles.HexNumber),
 795                                                 1);
 796                                         i += 5;
 797                                 }
 798                                 else
 799                                         sb.Append (s [i]);
 800                         }
 801                         return sb.ToString ();
 802                 }
 803
 804                 void ProcessTailoringLine (ref Tailoring t, string s)
 805                 {
 806                         int idx = s.IndexOf ('#');
 807                         if (idx > 0)
 808                                 s = s.Substring (0, idx).Trim ();
 809                         if (s.Length == 0 || s [0] == '#')
 810                                 return;
 811                         if (s [0] == '@') {
 812                                 idx = s.IndexOf ('=');
 813                                 if (idx > 0)
 814                                         t = new Tailoring (
 815                                                 int.Parse (s.Substring (1, idx - 1)),
 816                                                 int.Parse (s.Substring (idx + 1)));
 817                                 else
 818                                         t = new Tailoring (int.Parse (s.Substring (1)));
 819                                 tailorings.Add (t);
 820                                 return;
 821                         }
 822                         if (s.StartsWith ("*FrenchSort")) {
 823                                 t.FrenchSort = true;
 824                                 return;
 825                         }
 826                         string d = "*Diacritical";
 827                         if (s.StartsWith (d)) {
 828                                 idx = s.IndexOf ("->");
 829                                 t.AddDiacriticalMap (
 830                                         byte.Parse (s.Substring (d.Length, idx - d.Length).Trim (),
 831                                                 NumberStyles.HexNumber),
 832                                         byte.Parse (s.Substring (idx + 2).Trim (),
 833                                                 NumberStyles.HexNumber));
 834                                 return;
 835                         }
 836                         idx = s.IndexOf (':');
 837                         if (idx > 0) {
 838                                 string source = s.Substring (0, idx).Trim ();
 839                                 string [] l = s.Substring (idx + 1).Trim ().Split (' ');
 840                                 byte [] b = new byte [4];
 841                                 for (int i = 0; i < 4; i++) {
 842                                         if (l [i] == "*")
 843                                                 b [i] = 0;
 844                                         else
 845                                                 b [i] = byte.Parse (l [i],
 846                                                         NumberStyles.HexNumber);
 847                                 }
 848                                 t.AddSortKeyMap (ParseTailoringSourceValue (source),
 849                                         b);
 850                         }
 851                         idx = s.IndexOf ('=');
 852                         if (idx > 0)
 853                                 t.AddReplacementMap (
 854                                         ParseTailoringSourceValue (
 855                                                 s.Substring (0, idx).Trim ()),
 856                                         ParseTailoringSourceValue (
 857                                                 s.Substring (idx + 1).Trim ()));
 858                 }
 859
 860                 void ParseDerivedAge (string filename)
 861                 {
 862                         using (StreamReader file =
 863                                 new StreamReader (filename)) {
 864                                 while (file.Peek () >= 0) {
 865                                         string s = file.ReadLine ();
 866                                         int idx = s.IndexOf ('#');
 867                                         if (idx >= 0)
 868                                                 s = s.Substring (0, idx);
 869                                         idx = s.IndexOf (';');
 870                                         if (idx < 0)
 871                                                 continue;
 872
 873                                         string cpspec = s.Substring (0, idx);
 874                                         idx = cpspec.IndexOf ("..");
 875                                         NumberStyles nf = NumberStyles.HexNumber |
 876                                                 NumberStyles.AllowTrailingWhite;
 877                                         int cp = int.Parse (idx < 0 ? cpspec : cpspec.Substring (0, idx), nf);
 878                                         int cpEnd = idx < 0 ? cp : int.Parse (cpspec.Substring (idx + 2), nf);
 879                                         string value = s.Substring (cpspec.Length + 1).Trim ();
 880
 881                                         // FIXME: use index
 882                                         if (cp > char.MaxValue)
 883                                                 continue;
 884
 885                                         double v = double.Parse (value);
 886                                         for (int i = cp; i <= cpEnd; i++)
 887                                                 unicodeAge [i] = v;
 888                                 }
 889                         }
 890                         unicodeAge [0] = double.MaxValue; // never be supported
 891                 }
 892
 893                 void ParseUnidata (string filename)
 894                 {
 895                         ArrayList decompValues = new ArrayList ();
 896                         using (StreamReader unidata =
 897                                 new StreamReader (filename)) {
 898                                 for (int line = 1; unidata.Peek () >= 0; line++) {
 899                                         try {
 900                                                 ProcessUnidataLine (unidata.ReadLine (), decompValues);
 901                                         } catch (Exception) {
 902                                                 Console.Error.WriteLine ("**** At line " + line);
 903                                                 throw;
 904                                         }
 905                                 }
 906                         }
 907                         this.decompValues = (int [])
 908                                 decompValues.ToArray (typeof (int));
 909                 }
 910
 911                 char previousLatinTarget = char.MinValue;
 912                 byte [] diacriticalOffset = new byte ['Z' - 'A' + 1];
 913
 914                 void ProcessUnidataLine (string s, ArrayList decompValues)
 915                 {
 916                         int idx = s.IndexOf ('#');
 917                         if (idx >= 0)
 918                                 s = s.Substring (0, idx);
 919                         idx = s.IndexOf (';');
 920                         if (idx < 0)
 921                                 return;
 922                         int cp = int.Parse (s.Substring (0, idx), NumberStyles.HexNumber);
 923                         string [] values = s.Substring (idx + 1).Split (';');
 924
 925                         // FIXME: use index
 926                         if (cp > char.MaxValue)
 927                                 return;
 928                         if (IsIgnorable (cp))
 929                                 return;
 930
 931                         string name = values [0];
 932
 933                         // SPECIAL CASE: rename some characters for diacritical
 934                         // remapping. FIXME: why are they different?
 935                         // FIXME: it's still not working.
 936                         if (cp == 0x018B || cp == 0x018C)
 937                                 name = name.Replace ("TOPBAR", "STROKE");
 938
 939                         // isSmallCapital
 940                         if (s.IndexOf ("SMALL CAPITAL") > 0)
 941                                 isSmallCapital [cp] = true;
 942
 943                         // latin mapping by character name
 944                         if (s.IndexOf ("LATIN") >= 0) {
 945                                 int lidx = s.IndexOf ("LETTER DOTLESS ");
 946                                 int offset = lidx + 15;
 947                                 if (lidx < 0) {
 948                                         lidx = s.IndexOf ("LETTER TURNED ");
 949                                         offset = lidx + 14;
 950                                 }
 951                                 if (lidx < 0) {
 952                                         lidx = s.IndexOf ("LETTER CAPITAL ");
 953                                         offset = lidx + 15;
 954                                 }
 955                                 if (lidx < 0) {
 956                                         lidx = s.IndexOf ("LETTER SCRIPT ");
 957                                         offset = lidx + 14;
 958                                 }
 959                                 if (lidx < 0) {
 960                                         lidx = s.IndexOf ("LETTER ");
 961                                         offset = lidx + 7;
 962                                 }
 963                                 char c = lidx > 0 ? s [offset] : char.MinValue;
 964                                 char n = s [offset + 1];
 965                                 char target = char.MinValue;
 966                                 if ('A' <= c && c <= 'Z' &&
 967                                         (n == ' ') || n == ';') {
 968                                         target = c;
 969                                         // FIXME: After 'Z', I cannot reset this state.
 970                                         previousLatinTarget = c == 'Z' ? char.MinValue : c;
 971                                 }
 972
 973                                 if (s.Substring (offset).StartsWith ("ALPHA"))
 974                                         target = 'A';
 975                                 else if (s.Substring (offset).StartsWith ("TONE SIX"))
 976                                         target = 'B';
 977                                 else if (s.Substring (offset).StartsWith ("OPEN O"))
 978                                         target = 'C';
 979                                 else if (s.Substring (offset).StartsWith ("ETH"))
 980                                         target = 'D';
 981                                 else if (s.Substring (offset).StartsWith ("SCHWA"))
 982                                         target = 'E';
 983                                 else if (s.Substring (offset).StartsWith ("OI;")) // 01A2,01A3
 984                                         target = 'O';
 985                                 else if (s.Substring (offset).StartsWith ("YR;")) // 01A2,01A3
 986                                         target = 'R';
 987                                 else if (s.Substring (offset).StartsWith ("TONE TWO"))
 988                                         target = 'S';
 989                                 else if (s.Substring (offset).StartsWith ("ESH"))
 990                                         target = 'S';
 991                                 else if (s.Substring (offset).StartsWith ("OUNCE"))
 992                                         target = 'Z';
 993
 994                                 // For remaining IPA chars, direct mapping is
 995                                 // much faster.
 996                                 switch (cp) {
 997                                 case 0x0166: case 0x0167:
 998                                         // Though they are 'T', they have different weight
 999                                         target = char.MinValue; break;
1000                                 case 0x0299: target = 'B'; break;
1001                                 case 0x029A: target = 'E'; break;
1002                                 case 0x029B: target = 'G'; break;
1003                                 case 0x029C: target = 'H'; break;
1004                                 case 0x029D: target = 'J'; break;
1005                                 case 0x029E: target = 'K'; break;
1006                                 case 0x029F: target = 'L'; break;
1007                                 case 0x02A0: target = 'Q'; break;
1008                                 case 0x02A7: target = 'T'; break;
1009                                 case 0x02A8: target = 'T'; break;
1010                                 }
1011
1012                                 if (target == char.MinValue)
1013                                         target = previousLatinTarget;
1014
1015                                 if (target != char.MinValue) {
1016                                         ArrayList entry = (ArrayList) latinMap [target];
1017                                         if (entry == null) {
1018                                                 entry = new ArrayList ();
1019                                                 latinMap [target] = entry;
1020                                         }
1021                                         entry.Add (cp);
1022                                         // FIXME: This secondary weight is hack.
1023                                         // They are here because they must not
1024                                         // be identical to the corresponding
1025                                         // ASCII latins.
1026                                         if (c != target && diacritical [cp] == 0) {
1027                                                 diacriticalOffset [c - 'A']++;
1028                                                 diacritical [cp] = (byte) (diacriticalOffset [c - 'A'] + 0x7C);
1029                                         }
1030                                 }
1031                         }
1032
1033                         // Arrow names
1034                         if (0x2000 <= cp && cp < 0x3000) {
1035                                 int value = 0;
1036                                 // SPECIAL CASES. FIXME: why?
1037                                 switch (cp) {
1038                                 case 0x21C5: value = -1; break; // E2
1039                                 case 0x261D: value = 1; break;
1040                                 case 0x27A6: value = 3; break;
1041                                 case 0x21B0: value = 7; break;
1042                                 case 0x21B1: value = 3; break;
1043                                 case 0x21B2: value = 7; break;
1044                                 case 0x21B4: value = 5; break;
1045                                 case 0x21B5: value = 7; break;
1046                                 case 0x21B9: value = -1; break; // E1
1047                                 case 0x21CF: value = 7; break;
1048                                 case 0x21D0: value = 3; break;
1049                                 }
1050                                 string [] arrowTargets = new string [] {
1051                                         "",
1052                                         "UPWARDS",
1053                                         "NORTH EAST",
1054                                         "RIGHTWARDS",
1055                                         "SOUTH EAST",
1056                                         "DOWNWARDS",
1057                                         "SOUTH WEST",
1058                                         "LEFTWARDS",
1059                                         "NORTH WEST",
1060                                         "LEFT RIGHT",
1061                                         "UP DOWN",
1062                                         };
1063                                 if (s.IndexOf ("RIGHTWARDS") >= 0 &&
1064                                         s.IndexOf ("LEFTWARDS") >= 0)
1065                                         value = 0xE1 - 0xD8;
1066                                 else if (s.IndexOf ("UPWARDS") >= 0 &&
1067                                         s.IndexOf ("DOWNWARDS") >= 0)
1068                                         value = 0xE2 - 0xD8;
1069                                 else if (s.IndexOf ("ARROW") >= 0 &&
1070                                         s.IndexOf ("COMBINING") < 0 &&
1071                                         s.IndexOf ("CLOCKWISE") >= 0)
1072                                         value = s.IndexOf ("ANTICLOCKWISE") >= 0 ? 0xE4 - 0xD8 : 0xE3 - 0xD8;
1073                                 if (value == 0)
1074                                         for (int i = 1; value == 0 && i < arrowTargets.Length; i++)
1075                                                 if (s.IndexOf (arrowTargets [i]) > 0 &&
1076                                                         s.IndexOf ("BARB " + arrowTargets [i]) < 0 &&
1077                                                         s.IndexOf (" OVER") < 0
1078                                                 )
1079                                                         value = i;
1080                                 if (value > 0)
1081                                         arrowValues.Add (new DictionaryEntry (
1082                                                 cp, value));
1083                         }
1084
1085                         // Box names
1086                         if (0x2500 <= cp && cp < 0x2600) {
1087                                 int value = int.MinValue;
1088                                 // flags:
1089                                 // up:1 down:2 right:4 left:8 vert:16 horiz:32
1090                                 // [h,rl] [r] [l]
1091                                 // [v,ud] [u] [d]
1092                                 // [dr] [dl] [ur] [ul]
1093                                 // [vr,udr] [vl,vdl]
1094                                 // [hd,rld] [hu,rlu]
1095                                 // [hv,udrl,rlv,udh]
1096                                 ArrayList flags = new ArrayList (new int [] {
1097                                         32, 8 + 4, 8, 4,
1098                                         16, 1 + 2, 1, 2,
1099                                         4 + 2, 8 + 2, 4 + 1, 8 + 1,
1100                                         16 + 4, 1 + 2 + 4, 16 + 8, 1 + 2 + 8,
1101                                         32 + 2, 4 + 8 + 2, 32 + 1, 4 + 8 + 1,
1102                                         16 + 32, 1 + 2 + 4 + 8, 4 + 8 + 16, 1 + 2 + 32
1103                                         });
1104                                 byte [] offsets = new byte [] {
1105                                         0, 0, 1, 2,
1106                                         3, 3, 4, 5,
1107                                         6, 7, 8, 9,
1108                                         10, 10, 11, 11,
1109                                         12, 12, 13, 13,
1110                                         14, 14, 14, 14};
1111                                 if (s.IndexOf ("BOX DRAWINGS ") >= 0) {
1112                                         int flag = 0;
1113                                         if (s.IndexOf (" UP") >= 0)
1114                                                 flag |= 1;
1115                                         if (s.IndexOf (" DOWN") >= 0)
1116                                                 flag |= 2;
1117                                         if (s.IndexOf (" RIGHT") >= 0)
1118                                                 flag |= 4;
1119                                         if (s.IndexOf (" LEFT") >= 0)
1120                                                 flag |= 8;
1121                                         if (s.IndexOf (" VERTICAL") >= 0)
1122                                                 flag |= 16;
1123                                         if (s.IndexOf (" HORIZONTAL") >= 0)
1124                                                 flag |= 32;
1125
1126                                         int fidx = flags.IndexOf (flag);
1127                                         if (fidx >= 0)
1128                                                 value = offsets [fidx];
1129                                 } else if (s.IndexOf ("BLOCK") >= 0) {
1130                                         if (s.IndexOf ("ONE EIGHTH") >= 0)
1131                                                 value = 0x12;
1132                                         else if (s.IndexOf ("ONE QUARTER") >= 0)
1133                                                 value = 0x13;
1134                                         else if (s.IndexOf ("THREE EIGHTHS") >= 0)
1135                                                 value = 0x14;
1136                                         else if (s.IndexOf ("HALF") >= 0)
1137                                                 value = 0x15;
1138                                         else if (s.IndexOf ("FIVE EIGHTHS") >= 0)
1139                                                 value = 0x16;
1140                                         else if (s.IndexOf ("THREE QUARTERS") >= 0)
1141                                                 value = 0x17;
1142                                         else if (s.IndexOf ("SEVEN EIGHTHS") >= 0)
1143                                                 value = 0x18;
1144                                         else
1145                                                 value = 0x19;
1146                                 }
1147                                 else if (s.IndexOf ("SHADE") >= 0)
1148                                         value = 0x19;
1149                                 else if (s.IndexOf ("SQUARE") >= 0)
1150                                         value = 0xBC - 0xE5;
1151                                 else if (s.IndexOf ("VERTICAL RECTANGLE") >= 0)
1152                                         value = 0xBE - 0xE5;
1153                                 else if (s.IndexOf ("RECTANGLE") >= 0)
1154                                         value = 0xBD - 0xE5;
1155                                 else if (s.IndexOf ("PARALLELOGRAM") >= 0)
1156                                         value = 0xBF - 0xE5;
1157                                 else if (s.IndexOf ("TRIANGLE") >= 0) {
1158                                         if (s.IndexOf ("UP-POINTING") >= 0)
1159                                                 value = 0xC0 - 0xE5;
1160                                         else if (s.IndexOf ("RIGHT-POINTING") >= 0)
1161                                                 value = 0xC1 - 0xE5;
1162                                         else if (s.IndexOf ("DOWN-POINTING") >= 0)
1163                                                 value = 0xC2 - 0xE5;
1164                                         else if (s.IndexOf ("LEFT-POINTING") >= 0)
1165                                                 value = 0xC3 - 0xE5;
1166                                 }
1167                                 else if (s.IndexOf ("POINTER") >= 0) {
1168                                         if (s.IndexOf ("RIGHT-POINTING") >= 0)
1169                                                 value = 0xC4 - 0xE5;
1170                                         else if (s.IndexOf ("LEFT-POINTING") >= 0)
1171                                                 value = 0xC5 - 0xE5;
1172                                 }
1173                                 else if (s.IndexOf ("DIAMOND") >= 0)
1174                                         value = 0xC6 - 0xE5;
1175                                 else if (s.IndexOf ("FISHEYE") >= 0)
1176                                         value = 0xC7 - 0xE5;
1177                                 else if (s.IndexOf ("LOZENGE") >= 0)
1178                                         value = 0xC8 - 0xE5;
1179                                 else if (s.IndexOf ("BULLSEYE") >= 0)
1180                                         value = 0xC9 - 0xE5;
1181                                 else if (s.IndexOf ("CIRCLE") >= 0) {
1182                                         if (cp == 0x25D6) // it could be IndexOf ("LEFT HALF BLACK CIRCLE")
1183                                                 value = 0xCA - 0xE5;
1184                                         else if (cp == 0x25D7) // it could be IndexOf ("RIGHT HALF BLACK CIRCLE")
1185                                                 value = 0xCB - 0xE5;
1186                                         else
1187                                                 value = 0xC9 - 0xE5;
1188                                 }
1189                                 else if (s.IndexOf ("BULLET") >= 0)
1190                                         value = 0xCC - 0xE5;
1191                                 if (0x25DA <= cp && cp <= 0x25E5)
1192                                         value = 0xCD + cp - 0x25DA - 0xE5;
1193
1194                                 // SPECIAL CASE: BOX DRAWING DIAGONAL patterns
1195                                 switch (cp) {
1196                                 case 0x2571: value = 0xF; break;
1197                                 case 0x2572: value = 0x10; break;
1198                                 case 0x2573: value = 0x11; break;
1199                                 }
1200                                 if (value != int.MinValue)
1201                                         boxValues.Add (new DictionaryEntry (
1202                                                 cp, value));
1203                         }
1204
1205                         // For some characters store the name and sort later
1206                         // to determine sorting.
1207                         if (0x2100 <= cp && cp <= 0x213F &&
1208                                 Char.IsSymbol ((char) cp))
1209                                 sortableCharNames.Add (
1210                                         new DictionaryEntry (cp, name));
1211                         else if (0x3380 <= cp && cp <= 0x33DD)
1212                                 sortableCharNames.Add (new DictionaryEntry (
1213                                         cp, name.Substring (7)));
1214
1215                         if (Char.GetUnicodeCategory ((char) cp) ==
1216                                 UnicodeCategory.MathSymbol) {
1217                                 if (name.StartsWith ("CIRCLED "))
1218                                         diacritical [cp] = 0xEE;
1219                                 if (name.StartsWith ("SQUARED "))
1220                                         diacritical [cp] = 0xEF;
1221                         }
1222
1223                         // diacritical weights by character name
1224 if (diacritics.Length != diacriticWeights.Length)
1225 throw new Exception (String.Format ("Should not happen. weights are {0} while labels are {1}", diacriticWeights.Length, diacritics.Length));
1226                         for (int d = diacritics.Length - 1; d >= 0; d--) {
1227                                 if (s.IndexOf (diacritics [d]) > 0) {
1228                                         diacritical [cp] += diacriticWeights [d];
1229                                         if (s.IndexOf ("COMBINING") >= 0)
1230                                                 diacritical [cp] -= (byte) 2;
1231                                         break;
1232                                 }
1233                                 // also process "COMBINING blah" here
1234                                 // For now it is limited to cp < 0x0370
1235 //                              if (cp < 0x0300 || cp >= 0x0370)
1236 //                                      continue;
1237                                 string tmp = diacritics [d].TrimEnd (';');
1238                                 if (tmp.IndexOf ("WITH ") == 0)
1239                                         tmp = tmp.Substring (4);
1240                                 tmp = String.Concat ("COMBINING", (tmp [0] != ' ' ? " " : ""), tmp);
1241                                 if (name == tmp) {
1242                                         diacritical [cp] = (byte) (diacriticWeights [d] - 2);
1243                                         break;
1244                                 }
1245 //if (name == tmp)
1246 //Console.Error.WriteLine ("======= {2:X04} : '{0}' / '{1}'", name, tmp, cp);
1247                         }
1248                         // Two-step grep required for it.
1249                         if (s.IndexOf ("FULL STOP") > 0 &&
1250                                 (s.IndexOf ("DIGIT") > 0 || s.IndexOf ("NUMBER") > 0))
1251                                 diacritical [cp] |= 0xF4;
1252                         if (s.StartsWith ("SCRIPT") || s.IndexOf (" SCRIPT ") > 0)
1253                                 diacritical [cp] = (byte) (s.IndexOf ("SMALL") > 0 ? 3 :
1254                                         s.IndexOf ("CAPITAL") > 0 ? 5 : 4);
1255
1256                         // Arabic letter name
1257                         if (0x0621 <= cp && cp <= 0x064A &&
1258                                 Char.GetUnicodeCategory ((char) cp)
1259                                 == UnicodeCategory.OtherLetter) {
1260                                 byte value = (byte) (arabicNameMap.Count * 4 + 0x0B);
1261                                 switch (cp) {
1262                                 case 0x0621:
1263                                 case 0x0624:
1264                                 case 0x0626:
1265                                         // hamza, waw, yeh ... special cases.
1266                                         value = 0x07;
1267                                         break;
1268                                 case 0x0649:
1269                                 case 0x064A:
1270                                         value = 0x77; // special cases.
1271                                         break;
1272                                 default:
1273                                         // Get primary letter name i.e.
1274                                         // XXX part of ARABIC LETTER XXX yyy
1275                                         // e.g. that of "TEH MARBUTA" is "TEH".
1276                                         string letterName =
1277                                                 (cp == 0x0640) ?
1278                                                 // 0x0640 is special: it does
1279                                                 // not start with ARABIC LETTER
1280                                                 name :
1281                                                 name.Substring (14);
1282                                         int tmpIdx = letterName.IndexOf (' ');
1283                                         letterName = tmpIdx < 0 ? letterName : letterName.Substring (0, tmpIdx);
1284 //Console.Error.WriteLine ("Arabic name for {0:X04} is {1}", cp, letterName);
1285                                         if (arabicNameMap.ContainsKey (letterName))
1286                                                 value = (byte) arabicLetterPrimaryValues [arabicNameMap [letterName]];
1287                                         else
1288                                                 arabicNameMap [letterName] = cp;
1289                                         break;
1290                                 }
1291                                 arabicLetterPrimaryValues [cp] = value;
1292                         }
1293
1294                         // Japanese square letter
1295                         if (0x3300 <= cp && cp <= 0x3357)
1296                                 if (!ExistsJIS (cp))
1297                                         nonJisJapanese.Add (new NonJISCharacter (cp, name));
1298
1299                         // normalizationType
1300                         string decomp = values [4];
1301                         idx = decomp.IndexOf ('<');
1302                         if (idx >= 0) {
1303                                 switch (decomp.Substring (idx + 1, decomp.IndexOf ('>') - 1)) {
1304                                 case "full":
1305                                         decompType [cp] = DecompositionFull;
1306                                         break;
1307                                 case "sub":
1308                                         decompType [cp] = DecompositionSub;
1309                                         break;
1310                                 case "super":
1311                                         decompType [cp] = DecompositionSuper;
1312                                         break;
1313                                 case "small":
1314                                         decompType [cp] = DecompositionSmall;
1315                                         break;
1316                                 case "isolated":
1317                                         decompType [cp] = DecompositionIsolated;
1318                                         break;
1319                                 case "initial":
1320                                         decompType [cp] = DecompositionInitial;
1321                                         break;
1322                                 case "final":
1323                                         decompType [cp] = DecompositionFinal;
1324                                         break;
1325                                 case "medial":
1326                                         decompType [cp] = DecompositionMedial;
1327                                         break;
1328                                 case "noBreak":
1329                                         decompType [cp] = DecompositionNoBreak;
1330                                         break;
1331                                 case "compat":
1332                                         decompType [cp] = DecompositionCompat;
1333                                         break;
1334                                 case "fraction":
1335                                         decompType [cp] = DecompositionFraction;
1336                                         break;
1337                                 case "font":
1338                                         decompType [cp] = DecompositionFont;
1339                                         break;
1340                                 case "circle":
1341                                         decompType [cp] = DecompositionCircle;
1342                                         break;
1343                                 case "square":
1344                                         decompType [cp] = DecompositionSquare;
1345                                         break;
1346                                 case "wide":
1347                                         decompType [cp] = DecompositionWide;
1348                                         break;
1349                                 case "narrow":
1350                                         decompType [cp] = DecompositionNarrow;
1351                                         break;
1352                                 case "vertical":
1353                                         decompType [cp] = DecompositionVertical;
1354                                         break;
1355                                 default:
1356                                         throw new Exception ("Support NFKD type : " + decomp);
1357                                 }
1358                         }
1359                         else
1360                                 decompType [cp] = DecompositionCanonical;
1361                         decomp = idx < 0 ? decomp : decomp.Substring (decomp.IndexOf ('>') + 2);
1362                         if (decomp.Length > 0) {
1363
1364                                 string [] velems = decomp.Split (' ');
1365                                 int didx = decompValues.Count;
1366                                 decompIndex [cp] = didx;
1367                                 foreach (string v in velems)
1368                                         decompValues.Add (int.Parse (v, NumberStyles.HexNumber));
1369                                 decompLength [cp] = velems.Length;
1370
1371                                 // [decmpType] -> this_cp
1372                                 int targetCP = (int) decompValues [didx];
1373                                 // for "(x)" it specially maps to 'x' .
1374                                 // FIXME: check if it is sane
1375                                 if (velems.Length == 3 &&
1376                                         (int) decompValues [didx] == '(' &&
1377                                         (int) decompValues [didx + 2] == ')')
1378                                         targetCP = (int) decompValues [didx + 1];
1379                                 // special: 0x215F "1/"
1380                                 else if (cp == 0x215F)
1381                                         targetCP = '1';
1382                                 else if (velems.Length > 1 &&
1383                                         (targetCP < 0x4C00 || 0x9FBB < targetCP))
1384                                         // skip them, except for CJK ideograph compat
1385                                         targetCP = 0;
1386
1387                                 if (targetCP != 0) {
1388                                         Hashtable entry = (Hashtable) nfkdMap [targetCP];
1389                                         if (entry == null) {
1390                                                 entry = new Hashtable ();
1391                                                 nfkdMap [targetCP] = entry;
1392                                         }
1393                                         entry [(byte) decompType [cp]] = cp;
1394                                 }
1395                         }
1396                         // numeric values
1397                         if (values [5].Length > 0)
1398                                 decimalValue [cp] = decimal.Parse (values [5]);
1399                         else if (values [6].Length > 0)
1400                                 decimalValue [cp] = decimal.Parse (values [6]);
1401                         else if (values [7].Length > 0) {
1402                                 string decstr = values [7];
1403                                 idx = decstr.IndexOf ('/');
1404                                 if (cp == 0x215F) // special. "1/"
1405                                         decimalValue [cp] = 0x1;
1406                                 else if (idx > 0)
1407                                         // m/n
1408                                         decimalValue [cp] =
1409                                                 decimal.Parse (decstr.Substring (0, idx))
1410                                                 / decimal.Parse (decstr.Substring (idx + 1));
1411                                 else if (decstr [0] == '(' &&
1412                                         decstr [decstr.Length - 1] == ')')
1413                                         // (n)
1414                                         decimalValue [cp] =
1415                                                 decimal.Parse (decstr.Substring (1, decstr.Length - 2));
1416                                 else if (decstr [decstr.Length - 1] == '.')
1417                                         // n.
1418                                         decimalValue [cp] =
1419                                                 decimal.Parse (decstr.Substring (0, decstr.Length - 1));
1420                                 else
1421                                         decimalValue [cp] = decimal.Parse (decstr);
1422                         }
1423                 }
1424
1425                 void ParseDerivedCoreProperties (string filename)
1426                 {
1427                         // IsUppercase
1428                         using (StreamReader file =
1429                                 new StreamReader (filename)) {
1430                                 for (int line = 1; file.Peek () >= 0; line++) {
1431                                         try {
1432                                                 ProcessDerivedCorePropLine (file.ReadLine ());
1433                                         } catch (Exception) {
1434                                                 Console.Error.WriteLine ("**** At line " + line);
1435                                                 throw;
1436                                         }
1437                                 }
1438                         }
1439                 }
1440
1441                 void ProcessDerivedCorePropLine (string s)
1442                 {
1443                         int idx = s.IndexOf ('#');
1444                         if (idx >= 0)
1445                                 s = s.Substring (0, idx);
1446                         idx = s.IndexOf (';');
1447                         if (idx < 0)
1448                                 return;
1449                         string cpspec = s.Substring (0, idx);
1450                         idx = cpspec.IndexOf ("..");
1451                         NumberStyles nf = NumberStyles.HexNumber |
1452                                 NumberStyles.AllowTrailingWhite;
1453                         int cp = int.Parse (idx < 0 ? cpspec : cpspec.Substring (0, idx), nf);
1454                         int cpEnd = idx < 0 ? cp : int.Parse (cpspec.Substring (idx + 2), nf);
1455                         string value = s.Substring (cpspec.Length + 1).Trim ();
1456
1457                         // FIXME: use index
1458                         if (cp > char.MaxValue)
1459                                 return;
1460
1461                         switch (value) {
1462                         case "Uppercase":
1463                                 for (int x = cp; x <= cpEnd; x++)
1464                                         isUppercase [x] = true;
1465                                 break;
1466                         }
1467                 }
1468
1469                 void ParseScripts (string filename)
1470                 {
1471                         ArrayList gurmukhi = new ArrayList ();
1472                         ArrayList gujarati = new ArrayList ();
1473                         ArrayList georgian = new ArrayList ();
1474                         ArrayList thaana = new ArrayList ();
1475
1476                         using (StreamReader file =
1477                                 new StreamReader (filename)) {
1478                                 while (file.Peek () >= 0) {
1479                                         string s = file.ReadLine ();
1480                                         int idx = s.IndexOf ('#');
1481                                         if (idx >= 0)
1482                                                 s = s.Substring (0, idx);
1483                                         idx = s.IndexOf (';');
1484                                         if (idx < 0)
1485                                                 continue;
1486
1487                                         string cpspec = s.Substring (0, idx);
1488                                         idx = cpspec.IndexOf ("..");
1489                                         NumberStyles nf = NumberStyles.HexNumber |
1490                                                 NumberStyles.AllowTrailingWhite;
1491                                         int cp = int.Parse (idx < 0 ? cpspec : cpspec.Substring (0, idx), nf);
1492                                         int cpEnd = idx < 0 ? cp : int.Parse (cpspec.Substring (idx + 2), nf);
1493                                         string value = s.Substring (cpspec.Length + 1).Trim ();
1494
1495                                         // FIXME: use index
1496                                         if (cp > char.MaxValue)
1497                                                 continue;
1498
1499                                         switch (value) {
1500                                         case "Gurmukhi":
1501                                                 for (int x = cp; x <= cpEnd; x++)
1502                                                         if (!IsIgnorable (x))
1503                                                                 gurmukhi.Add ((char) x);
1504                                                 break;
1505                                         case "Gujarati":
1506                                                 for (int x = cp; x <= cpEnd; x++)
1507                                                         if (!IsIgnorable (x))
1508                                                                 gujarati.Add ((char) x);
1509                                                 break;
1510                                         case "Georgian":
1511                                                 for (int x = cp; x <= cpEnd; x++)
1512                                                         if (!IsIgnorable (x))
1513                                                                 georgian.Add ((char) x);
1514                                                 break;
1515                                         case "Thaana":
1516                                                 for (int x = cp; x <= cpEnd; x++)
1517                                                         if (!IsIgnorable (x))
1518                                                                 thaana.Add ((char) x);
1519                                                 break;
1520                                         }
1521                                 }
1522                         }
1523                         gurmukhi.Sort (UCAComparer.Instance);
1524                         gujarati.Sort (UCAComparer.Instance);
1525                         georgian.Sort (UCAComparer.Instance);
1526                         thaana.Sort (UCAComparer.Instance);
1527                         orderedGurmukhi = (char []) gurmukhi.ToArray (typeof (char));
1528                         orderedGujarati = (char []) gujarati.ToArray (typeof (char));
1529                         orderedGeorgian = (char []) georgian.ToArray (typeof (char));
1530                         orderedThaana = (char []) thaana.ToArray (typeof (char));
1531                 }
1532
1533                 void ParseJISOrder (string filename)
1534                 {
1535                         int line = 1;
1536                         try {
1537                                 using (StreamReader file =
1538                                         new StreamReader (filename)) {
1539                                         for (;file.Peek () >= 0; line++)
1540                                                 ProcessJISOrderLine (file.ReadLine ());
1541                                 }
1542                         } catch (Exception) {
1543                                 Console.Error.WriteLine ("---- line {0}", line);
1544                                 throw;
1545                         }
1546                 }
1547
1548                 char [] ws = new char [] {'\t', ' '};
1549
1550                 void ProcessJISOrderLine (string s)
1551                 {
1552                         int idx = s.IndexOf ('#');
1553                         if (idx >= 0)
1554                                 s = s.Substring (0, idx).Trim ();
1555                         if (s.Length == 0)
1556                                 return;
1557                         idx = s.IndexOfAny (ws);
1558                         if (idx < 0)
1559                                 return;
1560                         // They start with "0x" so cut them out.
1561                         int jis = int.Parse (s.Substring (2, idx - 2), NumberStyles.HexNumber);
1562                         int cp = int.Parse (s.Substring (idx).Trim ().Substring (2), NumberStyles.HexNumber);
1563                         jisJapanese.Add (new JISCharacter (cp, jis));
1564                 }
1565
1566                 void ParseCJK (string zhXML, string jaXML, string koXML)
1567                 {
1568                         XmlDocument doc = new XmlDocument ();
1569                         doc.XmlResolver = null;
1570                         int v;
1571                         string s;
1572                         string category;
1573                         int offset;
1574                         ushort [] arr;
1575
1576                         // Chinese Simplified
1577                         category = "chs";
1578                         arr = cjkCHS;
1579                         offset = 0;//char.MaxValue - arr.Length;
1580                         doc.Load (zhXML);
1581                         s = doc.SelectSingleNode ("/ldml/collations/collation[@type='pinyin']/rules/pc").InnerText;
1582                         v = 0x8008;
1583                         foreach (char c in s) {
1584                                 if (c < '\u3100')
1585                                         Console.Error.WriteLine ("---- warning: for {0} {1:X04} is omitted which should be {2:X04}", category, (int) c, v);
1586                                 else {
1587                                         arr [(int) c - offset] = (ushort) v++;
1588                                         if (v % 256 == 0)
1589                                                 v += 2;
1590                                 }
1591                         }
1592
1593                         // Chinese Traditional
1594                         category = "cht";
1595                         arr = cjkCHT;
1596                         offset = 0;//char.MaxValue - arr.Length;
1597                         s = doc.SelectSingleNode ("/ldml/collations/collation[@type='stroke']/rules/pc").InnerText;
1598                         v = 0x8002;
1599                         foreach (char c in s) {
1600                                 if (c < '\u4E00')
1601                                         Console.Error.WriteLine ("---- warning: for {0} {1:X04} is omitted which should be {2:X04}", category, (int) c, v);
1602                                 else {
1603                                         arr [(int) c - offset] = (ushort) v++;
1604                                         if (v % 256 == 0)
1605                                                 v += 2;
1606                                 }
1607                         }
1608
1609                         // Japanese
1610                         category = "ja";
1611                         arr = cjkJA;
1612                         offset = 0;//char.MaxValue - arr.Length;
1613
1614                         // SPECIAL CASES
1615                         arr [0x4EDD] = 0x8002; // Chinese repetition mark?
1616                         arr [0x337B] = 0x8004; // Those 4 characters are Gengou
1617                         arr [0x337E] = 0x8005;
1618                         arr [0x337D] = 0x8006;
1619                         arr [0x337C] = 0x8007;
1620
1621                         v = 0x8008;
1622                         foreach (JISCharacter jc in jisJapanese) {
1623                                 if (jc.JIS < 0x8800)
1624                                         continue;
1625                                 char c = (char) jc.CP;
1626
1627                                 if (c < '\u4E00')
1628                                         // Console.Error.WriteLine ("---- warning: for {0} {1:X04} is omitted which should be {2:X04}", category, (int) c, v);
1629                                         continue;
1630                                 else {
1631                                         arr [(int) c - offset] = (ushort) v++;
1632                                         if (v % 256 == 0)
1633                                                 v += 2;
1634
1635                                         // SPECIAL CASES:
1636                                         if (c == '\u662D') // U+337C
1637                                                 continue;
1638                                         if (c == '\u5927') // U+337D
1639                                                 continue;
1640                                         if (c == '\u5E73') // U+337B
1641                                                 continue;
1642                                         if (c == '\u660E') // U+337E
1643                                                 continue;
1644                                         if (c == '\u9686') // U+F9DC
1645                                                 continue;
1646
1647                                         // FIXME: there are still remaining
1648                                         // characters after U+FA0C.
1649 //                                      for (int k = 0; k < char.MaxValue; k++) {
1650                                         for (int k = 0; k < '\uFA0D'; k++) {
1651                                                 if (decompIndex [k] == 0 || IsIgnorable (k))
1652                                                         continue;
1653                                                 if (decompValues [decompIndex [k]] == c /*&&
1654                                                         decompLength [k] == 1*/ ||
1655                                                         decompLength [k] == 3 &&
1656                                                         decompValues [decompIndex [k] + 1] == c) {
1657                                                         arr [k - offset] = (ushort) v++;
1658                                                         if (v % 256 == 0)
1659                                                                 v += 2;
1660                                                 }
1661                                         }
1662                                 }
1663                         }
1664
1665                         // Korean
1666                         // Korean weight is somewhat complex. It first shifts
1667                         // Hangul category from 52-x to 80-x (they are anyways
1668                         // computed). CJK ideographs are placed at secondary
1669                         // weight, like XX YY 01 zz 01, where XX and YY are
1670                         // corresponding "reset" value and zz is 41,43,45...
1671                         //
1672                         // Unlike chs,cht and ja, Korean value is a combined
1673                         // ushort which is computed as category
1674                         //
1675                         category = "ko";
1676                         arr = cjkKO;
1677                         offset = 0;//char.MaxValue - arr.Length;
1678                         doc.Load (koXML);
1679                         foreach (XmlElement reset in doc.SelectNodes ("/ldml/collations/collation/rules/reset")) {
1680                                 XmlElement sc = (XmlElement) reset.NextSibling;
1681                                 // compute "category" and "level 1" for the
1682                                 // target "reset" Hangle syllable
1683                                 char rc = reset.InnerText [0];
1684                                 int ri = ((int) rc - 0xAC00) + 1;
1685                                 ushort p = (ushort)
1686                                         ((ri / 254) * 256 + (ri % 254) + 2);
1687                                 // Place the characters after the target.
1688                                 s = sc.InnerText;
1689                                 v = 0x41;
1690                                 foreach (char c in s) {
1691                                         arr [(int) c - offset] = p;
1692                                         cjkKOlv2 [(int) c - offset] = (byte) v;
1693                                         v += 2;
1694                                 }
1695                         }
1696                 }
1697
1698                 #endregion
1699
1700                 #region Generation
1701
1702                 void FillIgnorables ()
1703                 {
1704                         for (int i = 0; i <= char.MaxValue; i++) {
1705                                 if (Char.GetUnicodeCategory ((char) i) ==
1706                                         UnicodeCategory.OtherNotAssigned)
1707                                         continue;
1708                                 if (IsIgnorable (i))
1709                                         ignorableFlags [i] |= 1;
1710                                 if (IsIgnorableSymbol (i))
1711                                         ignorableFlags [i] |= 2;
1712                                 if (IsIgnorableNonSpacing (i))
1713                                         ignorableFlags [i] |= 4;
1714                         }
1715                 }
1716
1717                 void ModifyUnidata ()
1718                 {
1719                         ArrayList decompValues = new ArrayList (this.decompValues);
1720
1721                         // Hebrew uppercase letters.
1722                         foreach (int i in new int []
1723                                 {0x05DB, 0x05DE, 0x05E0, 0x05E4, 0x05E6})
1724                                 isUppercase [i] = true;
1725
1726
1727                         // Modify some decomposition equivalence
1728                         for (int i = 0xFE31; i <= 0xFE34; i++) {
1729                                 decompType [i] = 0;
1730                                 decompIndex [i] = 0;
1731                                 decompLength [i] = 0;
1732                         }
1733                         decompType [0x037E] = 0;
1734                         decompIndex [0x037E] = 0;
1735                         decompLength [0x037E] = 0;
1736
1737                         // Hangzhou numbers
1738                         for (int i = 0x3021; i <= 0x3029; i++)
1739                                 diacritical [i] = 0x4E;
1740                         // Korean parens numbers
1741                         for (int i = 0x3200; i <= 0x321C; i++)
1742                                 diacritical [i] = 0xA;
1743                         for (int i = 0x3260; i <= 0x327B; i++)
1744                                 diacritical [i] = 0xC;
1745
1746                         // LAMESPEC: these remapping should not be done.
1747                         // Windows have incorrect CJK compat mappings.
1748                         decompValues [decompIndex [0x32A9]] = 0x91AB;
1749                         decompLength [0x323B] = 1;
1750                         decompValues [decompIndex [0x323B]] = 0x5B78;
1751                         decompValues [decompIndex [0x32AB]] = 0x5B78;
1752                         decompValues [decompIndex [0x32A2]] = 0x5BEB;
1753                         decompLength [0x3238] = 1;
1754                         decompValues [decompIndex [0x3238]] = 0x52DE;
1755                         decompValues [decompIndex [0x3298]] = 0x52DE;
1756
1757                         // LAMESPEC: custom remapping (which is not bugs but not fine, non-standard compliant things)
1758                         decompIndex [0xFA0C] = decompValues.Count;
1759                         decompValues.Add ((int) 0x5140);
1760                         decompLength [0xFA0C] = 1;
1761                         decompIndex [0xF929] = decompLength [0xF929] = 0;
1762
1763                         decompValues [decompIndex [0xF92C]] = 0x90DE;
1764
1765                         decompIndex [0x2125] = decompValues.Count;
1766                         decompValues.Add ((int) 0x005A);
1767                         decompLength [0x2125] = 1;
1768                         decompType [0x2125] = DecompositionFont;
1769
1770                         this.decompValues = decompValues.ToArray (typeof (int)) as int [];
1771                 }
1772
1773                 void ModifyParsedValues ()
1774                 {
1775                         // Sometimes STROKE don't work fine
1776                         diacritical [0xD8] = diacritical [0xF8] = 0x21;
1777                         diacritical [0x141] = diacritical [0x142] = 0x1F;
1778                         // FIXME: why?
1779                         diacritical [0xAA] = diacritical [0xBA] = 3;
1780                         diacritical [0xD0] = diacritical [0xF0] = 0x68;
1781                         diacritical [0x131] = 3;
1782                         diacritical [0x138] = 3;
1783                         // TOPBAR does not work as an identifier for the weight
1784                         diacritical [0x182] = diacritical [0x183] = 0x68; // B
1785                         diacritical [0x18B] = diacritical [0x18C] = 0x1E; // D
1786                         // TONE TWO
1787                         diacritical [0x1A7] = diacritical [0x1A8] = 0x87;
1788                         // TONE SIX
1789                         diacritical [0x184] = diacritical [0x185] = 0x87;
1790                         // OPEN E
1791                         diacritical [0x190] = diacritical [0x25B] = 0x7B;
1792                         // There are many letters w/ diacritical weight 0x7B
1793                         diacritical [0x0192] = diacritical [0x0194] =
1794                         diacritical [0x0195] = diacritical [0x0196] =
1795                         diacritical [0x019C] = diacritical [0x019E] =
1796                         diacritical [0x01A6] = diacritical [0x01B1] =
1797                         diacritical [0x01B2] = diacritical [0x01BF] = 0x7B;
1798                         // ... as well as 0x7C
1799                         diacritical [0x01A2] = diacritical [0x01A3] = 0x7C;
1800
1801                         // <font> NFKD characters seem to have diacritical
1802                         // weight as 3,4,5... but the order does not look
1803                         // by codepoint and I have no idea how they are sorted.
1804                         diacritical [0x210E] = 3;
1805                         diacritical [0x210F] = 0x68;
1806                         diacritical [0x2110] = 4;
1807                         diacritical [0x2111] = 5;
1808                         diacritical [0x2112] = 4;
1809                         diacritical [0x2113] = 4;
1810                         diacritical [0x211B] = 4;
1811                         diacritical [0x211C] = 5;
1812
1813                         // some cyrillic diacritical weight. They seem to be
1814                         // based on old character names, so it's quicker to
1815                         // set them directly here.
1816                         // FIXME: they are by mostly unknown reason
1817                         diacritical [0x0496] = diacritical [0x0497] = 7;
1818                         diacritical [0x0498] = diacritical [0x0499] = 0x1A;
1819                         diacritical [0x049A] = diacritical [0x049B] = 0x17;
1820                         diacritical [0x049C] = diacritical [0x049D] = 9;
1821                         diacritical [0x049E] = diacritical [0x049F] = 4;
1822                         diacritical [0x04A0] = diacritical [0x04A1] = 0xA;
1823                         diacritical [0x04A2] = diacritical [0x04A3] = 7;
1824                         diacritical [0x04A4] = diacritical [0x04A5] = 8;
1825                         diacritical [0x04AA] = diacritical [0x04AB] = 0x1A; // ES CEDILLA?
1826                         diacritical [0x04AC] = diacritical [0x04AD] = 7; // RIGHT DESCENDER? but U+4B2
1827                         diacritical [0x04AE] = diacritical [0x04AF] = 0xB; // STRAIGHT U?
1828                         diacritical [0x04B2] = diacritical [0x04B3] = 0x17; // RIGHT DESCENDER? but U+4AC
1829                         diacritical [0x04B4] = diacritical [0x04B5] = 3;
1830                         diacritical [0x04B6] = 8;
1831                         diacritical [0x04B7] = 7;
1832                         diacritical [0x04B8] = diacritical [0x04B9] = 9;
1833                         diacritical [0x04BA] = diacritical [0x04BB] = 9;
1834
1835                         // number, secondary weights
1836                         byte weight = 0x38;
1837                         int [] numarr = numberSecondaryWeightBounds;
1838                         for (int i = 0; i < numarr.Length; i += 2, weight++)
1839                                 for (int cp = numarr [i]; cp < numarr [i + 1]; cp++)
1840                                         if (Char.IsNumber ((char) cp))
1841                                                 diacritical [cp] = weight;
1842
1843                         // Gurmukhi special letters' diacritical weight
1844                         for (int i = 0x0A50; i < 0x0A60; i++)
1845                                 diacritical [i] = 4;
1846                         // Oriya special letters' diacritical weight
1847                         for (int i = 0x0B5C; i < 0x0B60; i++)
1848                                 diacritical [i] = 6;
1849
1850                         // Update name part of named characters
1851                         for (int i = 0; i < sortableCharNames.Count; i++) {
1852                                 DictionaryEntry de =
1853                                         (DictionaryEntry) sortableCharNames [i];
1854                                 int cp = (int) de.Key;
1855                                 string renamed = null;
1856                                 switch (cp) {
1857                                 case 0x2101: renamed = "A_1"; break;
1858                                 case 0x33C3: renamed = "A_2"; break;
1859                                 case 0x2105: renamed = "C_1"; break;
1860                                 case 0x2106: renamed = "C_2"; break;
1861                                 case 0x211E: renamed = "R1"; break;
1862                                 case 0x211F: renamed = "R2"; break;
1863                                 // Remove some of them!
1864                                 case 0x2103:
1865                                 case 0x2109:
1866                                 case 0x2116:
1867                                 case 0x2117:
1868                                 case 0x2118:
1869                                 case 0x2125:
1870                                 case 0x2127:
1871                                 case 0x2129:
1872                                 case 0x212E:
1873                                 case 0x2132:
1874                                         sortableCharNames.RemoveAt (i);
1875                                         i--;
1876                                         continue;
1877                                 }
1878                                 if (renamed != null)
1879                                         sortableCharNames [i] =
1880                                                 new DictionaryEntry (cp, renamed);
1881                         }
1882                 }
1883
1884                 void GenerateCore ()
1885                 {
1886                         UnicodeCategory uc;
1887
1888                         #region Specially ignored // 01
1889                         // This will raise "Defined" flag up.
1890                         // FIXME: Check If it is really fine. Actually for
1891                         // Japanese voice marks this code does remapping.
1892                         foreach (char c in specialIgnore)
1893                                 map [(int) c] = new CharMapEntry (0, 0, 0);
1894                         #endregion
1895
1896                         #region Extenders (FF FF)
1897                         fillIndex [0xFF] = 0xFF;
1898                         char [] specialBiggest = new char [] {
1899                                 '\u3005', '\u3031', '\u3032', '\u309D',
1900                                 '\u309E', '\u30FC', '\u30FD', '\u30FE',
1901                                 '\uFE7C', '\uFE7D', '\uFF70'};
1902                         foreach (char c in specialBiggest)
1903                                 AddCharMap (c, 0xFF, 0);
1904                         #endregion
1905
1906                         #region Variable weights
1907                         // Controls : 06 03 - 06 3D
1908                         fillIndex [0x6] = 3;
1909                         for (int i = 0; i < 65536; i++) {
1910                                 if (IsIgnorable (i))
1911                                         continue;
1912                                 char c = (char) i;
1913                                 uc = Char.GetUnicodeCategory (c);
1914                                 // NEL is whitespace but not ignored here.
1915                                 if (uc == UnicodeCategory.Control &&
1916                                         !Char.IsWhiteSpace (c) || c == '\u0085')
1917                                         AddCharMap (c, 6, 1);
1918                         }
1919
1920                         // Apostrophe 06 80
1921                         fillIndex [0x6] = 0x80;
1922                         AddCharMap ('\'', 6, 0);
1923                         AddCharMap ('\uFF07', 6, 1);
1924                         AddCharMap ('\uFE63', 6, 1);
1925
1926                         // SPECIAL CASE: fill FE32 here in prior to be added
1927                         // at 2013. Windows does not always respect NFKD.
1928                         map [0xFE32] = new CharMapEntry (6, 0x90, 0);
1929
1930                         // Hyphen/Dash : 06 81 - 06 90
1931                         for (int i = 0; i < char.MaxValue; i++) {
1932                                 if (!IsIgnorable (i) &&
1933                                         Char.GetUnicodeCategory ((char) i) ==
1934                                         UnicodeCategory.DashPunctuation) {
1935                                         AddCharMapGroup2 ((char) i, 6, 1, 0);
1936                                         if (i == 0x2011) {
1937                                                 // SPECIAL: add 2027 and 2043
1938                                                 // Maybe they are regarded the
1939                                                 // same hyphens in "central"
1940                                                 // position.
1941                                                 AddCharMap ('\u2027', 6, 1);
1942                                                 AddCharMap ('\u2043', 6, 1);
1943                                         }
1944                                 }
1945                         }
1946                         // They are regarded as primarily equivalent to '-'
1947                         map [0x208B] = new CharMapEntry (6, 0x82, 0);
1948                         map [0x207B] = new CharMapEntry (6, 0x82, 0);
1949                         map [0xFF0D] = new CharMapEntry (6, 0x82, 0);
1950
1951                         // Arabic variable weight chars 06 A0 -
1952                         fillIndex [6] = 0xA0;
1953                         // vowels
1954                         for (int i = 0x64B; i <= 0x650; i++)
1955                                 AddArabicCharMap ((char) i, 6, 1, 0);
1956                         // sukun
1957                         AddCharMapGroup ('\u0652', 6, 1, 0);
1958                         // shadda
1959                         AddCharMapGroup ('\u0651', 6, 1, 0);
1960                         #endregion
1961
1962
1963                         #region Nonspacing marks // 01
1964                         // FIXME: 01 03 - 01 B6 ... annoyance :(
1965
1966                         // Combining diacritical marks: 01 DC -
1967
1968                         fillIndex [0x1] = 0x41;
1969                         for (int i = 0x030E; i <= 0x0326; i++)
1970                                 if (!IsIgnorable (i))
1971                                         AddCharMap ((char) i, 0x1, 1);
1972                         for (int i = 0x0329; i <= 0x0334; i++)
1973                                 if (!IsIgnorable (i))
1974                                         AddCharMap ((char) i, 0x1, 1);
1975                         fillIndex [0x1]++;
1976                         for (int i = 0x0339; i <= 0x0341; i++)
1977                                 if (!IsIgnorable (i))
1978                                         AddCharMap ((char) i, 0x1, 1);
1979                         fillIndex [0x1] = 0x74;
1980                         for (int i = 0x0346; i <= 0x0348; i++)
1981                                 if (!IsIgnorable (i))
1982                                         AddCharMap ((char) i, 0x1, 1);
1983                         for (int i = 0x02BE; i <= 0x02BF; i++)
1984                                 if (!IsIgnorable (i))
1985                                         AddCharMap ((char) i, 0x1, 1);
1986                         for (int i = 0x02C1; i <= 0x02C5; i++)
1987                                 if (!IsIgnorable (i))
1988                                         AddCharMap ((char) i, 0x1, 1);
1989                         for (int i = 0x02CE; i <= 0x02CF; i++)
1990                                 if (!IsIgnorable (i))
1991                                         AddCharMap ((char) i, 0x1, 1);
1992                         fillIndex [0x1]++;
1993                         for (int i = 0x02D1; i <= 0x02D3; i++)
1994                                 if (!IsIgnorable (i))
1995                                         AddCharMap ((char) i, 0x1, 1);
1996                         AddCharMap ('\u02DE', 0x1, 1);
1997                         for (int i = 0x02E4; i <= 0x02E9; i++)
1998                                 if (!IsIgnorable (i))
1999                                         AddCharMap ((char) i, 0x1, 1);
2000
2001
2002                         // FIXME: needs more love here (it should eliminate
2003                         // all the hacky code above).
2004                         for (int i = 0x0300; i < 0x0370; i++)
2005                                 if (!IsIgnorable (i) && diacritical [i] != 0
2006                                         && !map [i].Defined)
2007                                         map [i] = new CharMapEntry (
2008                                                 0x1, 0x1, diacritical [i]);
2009
2010                         // Cyrillic and Armenian nonspacing mark
2011                         fillIndex [0x1] = 0x94;
2012                         for (int i = 0x400; i < 0x580; i++)
2013                                 if (!IsIgnorable (i) &&
2014                                         Char.GetUnicodeCategory ((char) i) ==
2015                                         UnicodeCategory.NonSpacingMark)
2016                                         AddCharMap ((char) i, 1, 1);
2017
2018                         fillIndex [0x1] = 0x8D;
2019                         // syriac dotted nonspacing marks (1)
2020                         AddCharMap ('\u0740', 0x1, 1);
2021                         AddCharMap ('\u0741', 0x1, 1);
2022                         AddCharMap ('\u0742', 0x1, 1);
2023                         // syriac oblique nonspacing marks
2024                         AddCharMap ('\u0747', 0x1, 1);
2025                         AddCharMap ('\u0748', 0x1, 1);
2026                         // syriac dotted nonspacing marks (2)
2027                         fillIndex [0x1] = 0x94; // this reset is mandatory
2028                         AddCharMap ('\u0732', 0x1, 1);
2029                         AddCharMap ('\u0735', 0x1, 1);
2030                         AddCharMap ('\u0738', 0x1, 1);
2031                         AddCharMap ('\u0739', 0x1, 1);
2032                         AddCharMap ('\u073C', 0x1, 1);
2033                         // SPECIAL CASES: superscripts
2034                         AddCharMap ('\u073F', 0x1, 1);
2035                         AddCharMap ('\u0711', 0x1, 1);
2036                         // syriac "DOTS"
2037                         for (int i = 0x0743; i <= 0x0746; i++)
2038                                 AddCharMap ((char) i, 0x1, 1);
2039                         for (int i = 0x0730; i <= 0x0780; i++)
2040                                 if (!map [i].Defined &&
2041                                         Char.GetUnicodeCategory ((char) i) ==
2042                                         UnicodeCategory.NonSpacingMark)
2043                                         AddCharMap ((char) i, 0x1, 1);
2044
2045                         // LAMESPEC: It should not stop at '\u20E1'. There are
2046                         // a few more characters (that however results in
2047                         // overflow of level 2 unless we start before 0xDD).
2048                         fillIndex [0x1] = 0xDD;
2049                         for (int i = 0x20D0; i <= 0x20DC; i++)
2050                                 AddCharMap ((char) i, 0x1, 1);
2051                         fillIndex [0x1] = 0xEC;
2052                         for (int i = 0x20DD; i <= 0x20E1; i++)
2053                                 AddCharMap ((char) i, 0x1, 1);
2054                         fillIndex [0x1] = 0x4;
2055                         AddCharMap ('\u0CD5', 0x1, 1);
2056                         AddCharMap ('\u0CD6', 0x1, 1);
2057                         AddCharMap ('\u093C', 0x1, 1);
2058                         for (int i = 0x302A; i <= 0x302D; i++)
2059                                 AddCharMap ((char) i, 0x1, 1);
2060                         AddCharMap ('\u0C55', 0x1, 1);
2061                         AddCharMap ('\u0C56', 0x1, 1);
2062
2063                         fillIndex [0x1] = 0x50; // I wonder how they are sorted
2064                         for (int i = 0x02D4; i <= 0x02D7; i++)
2065                                 AddCharMap ((char) i, 0x1, 1);
2066
2067                         // They are not part of Nonspacing marks, but have
2068                         // only diacritical weight.
2069                         for (int i = 0x3099; i <= 0x309C; i++)
2070                                 map [i] = new CharMapEntry (1, 1, 1);
2071                         map [0xFF9E] = new CharMapEntry (1, 1, 1);
2072                         map [0xFF9F] = new CharMapEntry (1, 1, 2);
2073                         map [0x309D] = new CharMapEntry (0xFF, 0xFF, 1);
2074                         map [0x309E] = new CharMapEntry (0xFF, 0xFF, 1);
2075                         for (int i = 0x30FC; i <= 0x30FE; i++)
2076                                 map [i] = new CharMapEntry (0xFF, 0xFF, 1);
2077
2078                         fillIndex [0x1] = 0xA;
2079                         for (int i = 0x0951; i <= 0x0954; i++)
2080                                 AddCharMap ((char) i, 0x1, 2);
2081
2082                         #endregion
2083
2084
2085                         #region Whitespaces // 07 03 -
2086                         fillIndex [0x7] = 0x2;
2087                         AddCharMap (' ', 0x7, 2);
2088                         AddCharMap ('\u00A0', 0x7, 1);
2089                         for (int i = 9; i <= 0xD; i++)
2090                                 AddCharMap ((char) i, 0x7, 1);
2091                         for (int i = 0x2000; i <= 0x200B; i++)
2092                                 AddCharMap ((char) i, 0x7, 1);
2093
2094                         fillIndex [0x7] = 0x17;
2095                         AddCharMapGroup ('\u2028', 0x7, 1, 0);
2096                         AddCharMapGroup ('\u2029', 0x7, 1, 0);
2097
2098                         // Characters which used to represent layout control.
2099                         // LAMESPEC: Windows developers seem to have thought
2100                         // that those characters are kind of whitespaces,
2101                         // while they aren't.
2102                         AddCharMap ('\u2422', 0x7, 1, 0); // blank symbol
2103                         AddCharMap ('\u2423', 0x7, 1, 0); // open box
2104
2105                         #endregion
2106
2107                         // category 09 - continued symbols from 08
2108                         fillIndex [0x9] = 2;
2109                         // misc tech mark
2110                         for (int cp = 0x2300; cp <= 0x237A; cp++)
2111                                 AddCharMap ((char) cp, 0x9, 1, 0);
2112
2113                         // arrows
2114                         byte [] arrowLv2 = new byte [] {0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3};
2115                         foreach (DictionaryEntry de in arrowValues) {
2116                                 int idx = (int) de.Value;
2117                                 int cp = (int) de.Key;
2118                                 if (map [cp].Defined)
2119                                         continue;
2120                                 fillIndex [0x9] = (byte) (0xD8 + idx);
2121                                 AddCharMapGroup ((char) cp, 0x9, 0, arrowLv2 [idx]);
2122                                 arrowLv2 [idx]++;
2123                         }
2124                         // boxes
2125                         byte [] boxLv2 = new byte [128];
2126                         // 0-63 will be used for those offsets are positive,
2127                         // and 64-127 are for negative ones.
2128                         for (int i = 0; i < boxLv2.Length; i++)
2129                                 boxLv2 [i] = 3;
2130                         foreach (DictionaryEntry de in boxValues) {
2131                                 int cp = (int) de.Key;
2132                                 int off = (int) de.Value;
2133                                 if (map [cp].Defined)
2134                                         continue;
2135                                 if (off < 0) {
2136                                         fillIndex [0x9] = (byte) (0xE5 + off);
2137                                         AddCharMapGroup ((char) cp, 0x9, 0, boxLv2 [128 + off]++);
2138                                 }
2139                                 else {
2140                                         fillIndex [0x9] = (byte) (0xE5 + off);
2141                                         AddCharMapGroup ((char) cp, 0x9, 0, boxLv2 [off]++);
2142                                 }
2143                         }
2144                         // Some special characters (slanted)
2145                         fillIndex [0x9] = 0xF4;
2146                         AddCharMap ('\u2571', 0x9, 3);
2147                         AddCharMap ('\u2572', 0x9, 3);
2148                         AddCharMap ('\u2573', 0x9, 3);
2149
2150                         // FIXME: implement 0A
2151                         #region Symbols
2152                         fillIndex [0xA] = 2;
2153                         // byte currency symbols
2154                         for (int cp = 0; cp < 0x100; cp++) {
2155                                 uc = Char.GetUnicodeCategory ((char) cp);
2156                                 if (!IsIgnorable (cp) &&
2157                                         uc == UnicodeCategory.CurrencySymbol &&
2158                                         cp != '$')
2159                                         AddCharMapGroup ((char) cp, 0xA, 1, 0);
2160                         }
2161                         // byte other symbols
2162                         for (int cp = 0; cp < 0x100; cp++) {
2163                                 if (cp == 0xA6)
2164                                         continue; // SPECIAL: skip FIXME: why?
2165                                 uc = Char.GetUnicodeCategory ((char) cp);
2166                                 if (!IsIgnorable (cp) &&
2167                                         uc == UnicodeCategory.OtherSymbol ||
2168                                         cp == '\u00AC' || cp == '\u00B5' || cp == '\u00B7')
2169                                         AddCharMapGroup ((char) cp, 0xA, 1, 0);
2170                         }
2171                         // U+30FB here
2172                         AddCharMapGroup ('\u30FB', 0xA, 1, 0);
2173
2174                         for (int cp = 0x2020; cp <= 0x2031; cp++)
2175                                 if (Char.IsPunctuation ((char) cp))
2176                                         AddCharMap ((char) cp, 0xA, 1, 0);
2177                         // SPECIAL CASES: why?
2178                         AddCharMap ('\u203B', 0xA, 1, 0);
2179                         AddCharMap ('\u2040', 0xA, 1, 0);
2180                         AddCharMap ('\u2041', 0xA, 1, 0);
2181                         AddCharMap ('\u2042', 0xA, 1, 0);
2182
2183                         for (int cp = 0x20A0; cp <= 0x20AB; cp++)
2184                                 AddCharMap ((char) cp, 0xA, 1, 0);
2185
2186                         // 3004 is skipped at first...
2187                         for (int cp = 0x3010; cp <= 0x3040; cp++)
2188                                 if (Char.IsSymbol ((char) cp))
2189                                         AddCharMap ((char) cp, 0xA, 1, 0);
2190                         // SPECIAL CASES: added here
2191                         AddCharMap ('\u3004', 0xA, 1, 0);
2192                         AddCharMap ('\u327F', 0xA, 1, 0);
2193
2194                         for (int cp = 0x2600; cp <= 0x2613; cp++)
2195                                 AddCharMap ((char) cp, 0xA, 1, 0);
2196                         // Dingbats
2197                         for (int cp = 0x2620; cp <= 0x2770; cp++)
2198                                 if (Char.IsSymbol ((char) cp))
2199                                         AddCharMap ((char) cp, 0xA, 1, 0);
2200                         // OCR
2201                         for (int i = 0x2440; i < 0x2460; i++)
2202                                 AddCharMap ((char) i, 0xA, 1, 0);
2203
2204                         // SPECIAL CASES: why?
2205                         AddCharMap ('\u0E3F', 0xA, 1, 0);
2206                         AddCharMap ('\u2117', 0xA, 1, 0);
2207                         AddCharMap ('\u20AC', 0xA, 1, 0);
2208                         #endregion
2209
2210                         #region Numbers // 0C 02 - 0C E1
2211                         fillIndex [0xC] = 2;
2212
2213                         // 9F8 : Bengali "one less than the denominator"
2214                         AddCharMap ('\u09F8', 0xC, 1, 0x3C);
2215
2216                         ArrayList numbers = new ArrayList ();
2217                         for (int i = 0; i < 65536; i++)
2218                                 if (!IsIgnorable (i) &&
2219                                         Char.IsNumber ((char) i) &&
2220                                         (i < 0x3190 || 0x32C0 < i)) // they are CJK characters
2221                                         numbers.Add (i);
2222
2223                         ArrayList numberValues = new ArrayList ();
2224                         foreach (int i in numbers)
2225                                 numberValues.Add (new DictionaryEntry (i, decimalValue [(char) i]));
2226                         // SPECIAL CASE: Cyrillic Thousand sign
2227                         numberValues.Add (new DictionaryEntry (0x0482, 1000m));
2228                         numberValues.Sort (DecimalDictionaryValueComparer.Instance);
2229
2230 //foreach (DictionaryEntry de in numberValues)
2231 //Console.Error.WriteLine ("****** number {0:X04} : {1} {2}", de.Key, de.Value, decompType [(int) de.Key]);
2232
2233                         // FIXME: fillIndex adjustment lines are too
2234                         // complicated. It must be simpler.
2235                         decimal prevValue = -1;
2236                         foreach (DictionaryEntry de in numberValues) {
2237                                 int cp = (int) de.Key;
2238                                 decimal currValue = (decimal) de.Value;
2239                                 bool addnew = false;
2240                                 if (prevValue < currValue &&
2241                                         prevValue - (int) prevValue == 0 &&
2242                                         prevValue >= 1) {
2243
2244                                         addnew = true;
2245                                         // Process Hangzhou and Roman numbers
2246
2247                                         // There are some SPECIAL cases.
2248                                         if (currValue != 4) // no increment for 4
2249                                                 fillIndex [0xC]++;
2250
2251                                         int xcp;
2252                                         if (currValue <= 13) {
2253                                                 if (currValue == 4)
2254                                                         fillIndex [0xC]++;
2255                                                 // SPECIAL CASE
2256                                                 if (currValue == 11)
2257                                                         AddCharMap ('\u0BF0', 0xC, 1);
2258                                                 xcp = (int) prevValue + 0x2160 - 1;
2259                                                 AddCharMap ((char) xcp, 0xC, 0, diacritical [xcp]);
2260                                                 xcp = (int) prevValue + 0x2170 - 1;
2261                                                 AddCharMap ((char) xcp, 0xC, 0, diacritical [xcp]);
2262                                                 fillIndex [0xC]++;
2263                                         }
2264                                         if (currValue < 12)
2265                                                 fillIndex [0xC]++;
2266                                         if (currValue <= 10) {
2267                                                 xcp = (int) prevValue + 0x3021 - 1;
2268                                                 AddCharMap ((char) xcp, 0xC, 0, diacritical [xcp]);
2269                                                 fillIndex [0xC]++;
2270                                         }
2271                                 }
2272                                 if (prevValue < currValue)
2273                                         prevValue = currValue;
2274                                 if (map [cp].Defined)
2275                                         continue;
2276                                 // HangZhou and Roman are add later
2277                                 // (code is above)
2278                                 if (0x3021 <= cp && cp < 0x302A
2279                                         || 0x2160 <= cp && cp < 0x216C
2280                                         || 0x2170 <= cp && cp < 0x217C)
2281                                         continue;
2282
2283                                 if (cp == 0x215B) // FIXME: why?
2284                                         fillIndex [0xC] += 2;
2285                                 else if (cp == 0x3021) // FIXME: why?
2286                                         fillIndex [0xC]++;
2287                                 if (addnew || cp <= '9') {
2288                                         int mod = (int) currValue - 1;
2289                                         int xcp;
2290                                         if (1 <= currValue && currValue <= 11) {
2291                                                 xcp = mod + 0x2776;
2292                                                 AddCharMap ((char) xcp, 0xC, 0, diacritical [xcp]);
2293                                                 xcp = mod + 0x2780;
2294                                                 AddCharMap ((char) xcp, 0xC, 0, diacritical [xcp]);
2295                                                 xcp = mod + 0x278A;
2296                                                 AddCharMap ((char) xcp, 0xC, 0, diacritical [xcp]);
2297                                         }
2298                                         if (1 <= currValue && currValue <= 20) {
2299                                                 xcp = mod + 0x2460;
2300                                                 AddCharMap ((char) xcp, 0xC, 0, diacritical [xcp]);
2301                                                 xcp = mod + 0x2474;
2302                                                 AddCharMap ((char) xcp, 0xC, 0, diacritical [xcp]);
2303                                                 xcp = mod + 0x2488;
2304                                                 AddCharMap ((char) xcp, 0xC, 0, diacritical [xcp]);
2305                                         }
2306                                 }
2307                                 if (addnew && currValue >= 10 && currValue < 13 || cp == 0x09F9)
2308                                         fillIndex [0xC]++;
2309                                 AddCharMapGroup ((char) cp, 0xC, 0, diacritical [cp], true);
2310
2311                                 switch (cp) {
2312                                 // Maybe Bengali digit numbers do not increase
2313                                 // indexes, but 0x09E6 does.
2314                                 case 0x09E7: case 0x09E8: case 0x09E9:
2315                                 case 0x09EA:
2316                                 // SPECIAL CASES
2317                                 case 0x0BF0: case 0x2180: case 0x2181:
2318                                         break;
2319                                 // SPECIAL CASE
2320                                 case 0x0BF1:
2321                                         fillIndex [0xC]++;
2322                                         break;
2323                                 default:
2324                                         if (currValue < 11 || currValue == 1000)
2325                                                 fillIndex [0xC]++;
2326                                         break;
2327                                 }
2328
2329                                 // Add special cases that are not regarded as
2330                                 // numbers in UnicodeCategory speak.
2331                                 if (cp == '5') {
2332                                         // TONE FIVE
2333                                         AddCharMapGroup ('\u01BD', 0xC, 0, 0);
2334                                         AddCharMapGroup ('\u01BC', 0xC, 1, 0);
2335                                 }
2336                                 else if (cp == '2' || cp == '6') // FIXME: why?
2337                                         fillIndex [0xC]++;
2338                         }
2339
2340                         // 221E: infinity
2341                         fillIndex [0xC] = 0xFF;
2342                         AddCharMap ('\u221E', 0xC, 1);
2343                         #endregion
2344
2345                         #region Letters and NonSpacing Marks (general)
2346
2347                         // ASCII Latin alphabets
2348                         for (int i = 0; i < alphabets.Length; i++)
2349                                 AddAlphaMap (alphabets [i], 0xE, alphaWeights [i]);
2350
2351                         // non-ASCII Latin alphabets
2352                         // FIXME: there is no such characters that are placed
2353                         // *after* "alphabets" array items. This is nothing
2354                         // more than a hack that creates dummy weight for
2355                         // primary characters.
2356                         for (int i = 0x0080; i < 0x0300; i++) {
2357                                 if (!Char.IsLetter ((char) i))
2358                                         continue;
2359                                 // For those Latin Letters which has NFKD are
2360                                 // not added as independent primary character.
2361                                 if (decompIndex [i] != 0)
2362                                         continue;
2363                                 // SPECIAL CASES:
2364                                 // 1.some alphabets have primarily
2365                                 //   equivalent ASCII alphabets.
2366                                 // 2.some have independent primary weights,
2367                                 //   but inside a-to-z range.
2368                                 // 3.there are some expanded characters that
2369                                 //   are not part of Unicode Standard NFKD.
2370                                 // 4. some characters are letter in IsLetter
2371                                 //   but not in sortkeys (maybe unicode version
2372                                 //   difference caused it).
2373                                 switch (i) {
2374                                 // 1. skipping them does not make sense
2375 //                              case 0xD0: case 0xF0: case 0x131: case 0x138:
2376 //                              case 0x184: case 0x185: case 0x186: case 0x189:
2377 //                              case 0x18D: case 0x18E: case 0x18F: case 0x190:
2378 //                              case 0x194: case 0x195: case 0x196: case 0x19A:
2379 //                              case 0x19B: case 0x19C:
2380                                 // 2. skipping them does not make sense
2381 //                              case 0x14A: // Ng
2382 //                              case 0x14B: // ng
2383                                 // 3.
2384                                 case 0xC6: // AE
2385                                 case 0xE6: // ae
2386                                 case 0xDE: // Icelandic Thorn
2387                                 case 0xFE: // Icelandic Thorn
2388                                 case 0xDF: // German ss
2389                                 case 0xFF: // German ss
2390                                 // 4.
2391                                 case 0x1C0: case 0x1C1: case 0x1C2: case 0x1C3:
2392                                 // not classified yet
2393 //                              case 0x1A6: case 0x1A7: case 0x1A8: case 0x1A9:
2394 //                              case 0x1AA: case 0x1B1: case 0x1B7: case 0x1B8:
2395 //                              case 0x1B9: case 0x1BA: case 0x1BB: case 0x1BF:
2396 //                              case 0x1DD:
2397                                         continue;
2398                                 }
2399                                 AddCharMapGroup ((char) i, 0xE, 1, 0);
2400                         }
2401
2402                         // IPA extensions
2403                         // FIXME: this results in not equivalent values to
2404                         // Windows, but is safer for comparison.
2405                         char [] ipaArray = new char [0x300 - 0x250 + 0x20];
2406                         for (int i = 0x40; i < 0x60; i++)
2407                                 if (Char.IsLetter ((char) i))
2408                                         ipaArray [i - 0x40] = (char) (i);
2409                         for (int i = 0x250; i < 0x300; i++)
2410                                 if (Char.IsLetter ((char) i))
2411                                         ipaArray [i - 0x250 + 0x20] = (char) i;
2412                         Array.Sort (ipaArray, UCAComparer.Instance);
2413                         int targetASCII = 0;
2414                         byte latinDiacritical = 0x7B;
2415                         foreach (char c in ipaArray) {
2416                                 if (c <= 'Z') {
2417                                         targetASCII = c;
2418                                         latinDiacritical = 0x7B;
2419                                 }
2420                                 else
2421                                         map [(int) c] = new CharMapEntry (
2422                                                 0xE,
2423                                                 map [targetASCII].Level1,
2424                                                 latinDiacritical++);
2425                         }
2426
2427                         // Greek and Coptic
2428
2429                         // FIXME: this is (mysterious and) incomplete.
2430                         for (int i = 0x0380; i < 0x0400; i++)
2431                                 if (diacritical [i] == 0 &&
2432                                         decompLength [i] == 1 &&
2433                                         decompType [i] == DecompositionCompat)
2434                                         diacritical [i] = 3;
2435
2436                         fillIndex [0xF] = 2;
2437                         for (int i = 0x0391; i < 0x03AA; i++)
2438                                 if (i != 0x03A2)
2439                                         AddCharMap ((char) i, 0xF, 1,
2440                                                 diacritical [i]);
2441                         fillIndex [0xF] = 2;
2442                         for (int i = 0x03B1; i < 0x03CA; i++)
2443                                 if (i != 0x03C2)
2444                                         AddCharMap ((char) i, 0xF, 1,
2445                                                 diacritical [i]);
2446                         // Final Sigma
2447                         map [0x03C2] = new CharMapEntry (0xF,
2448                                 map [0x03C3].Level1, map [0x03C3].Level2);
2449
2450                         fillIndex [0xF] = 0x40;
2451                         for (int i = 0x03DA; i < 0x03F0; i++)
2452                                 AddCharMap ((char) i, 0xF,
2453                                         (byte) (i % 2 == 0 ? 0 : 2),
2454                                         diacritical [i]);
2455
2456                         // NFKD
2457                         for (int i = 0x0386; i <= 0x0400; i++)
2458                                 FillLetterNFKD (i, true, true);
2459
2460                         // Cyrillic.
2461                         // Cyrillic letters are sorted like Latin letters i.e.
2462                         // containing culture-specific letters between the
2463                         // standard Cyrillic sequence.
2464                         //
2465                         // We can't use UCA here; it has different sorting.
2466                         char [] orderedCyrillic = new char [] {
2467                                 '\u0430', '\u0431', '\u0432', '\u0433', '\u0434',
2468                                 '\u0452', // DJE for Serbocroatian
2469                                 '\u0435',
2470                                 '\u0454', // IE for Ukrainian
2471                                 '\u0436', '\u0437',
2472                                 '\u0455', // DZE
2473                                 '\u0438',
2474                                 '\u0456', // Byelorussian-Ukrainian I
2475                                 '\u0457', // YI
2476                                 '\u0439',
2477                                 '\u0458', // JE
2478                                 '\u043A', '\u043B',
2479                                 '\u0459', // LJE
2480                                 '\u043C', '\u043D',
2481                                 '\u045A', // NJE
2482                                 '\u043E',
2483                                 // 4E9 goes here.
2484                                 '\u043F', '\u0440', '\u0441', '\u0442',
2485                                 '\u045B', // TSHE for Serbocroatian
2486                                 '\u0443',
2487                                 '\u045E', // Short U for Byelorussian
2488                                 '\u04B1', // Straight U w/ stroke (diacritical!)
2489                                 '\u0444', '\u0445', '\u0446', '\u0447',
2490                                 '\u045F', // DZHE
2491                                 '\u0448', '\u0449', '\u044A', '\u044B', '\u044C',
2492                                 '\u044D', '\u044E', '\u044F'};
2493
2494                         // For some characters here is a map to basic cyrillic
2495                         // letters. See UnicodeData.txt character names for
2496                         // the sources. Here I simply declare an equiv. array.
2497                         // The content characters are map from U+490(,491),
2498                         // skipping small letters.
2499                         char [] cymap_src = new char [] {
2500                                 '\u0433', '\u0433', '\u0433', '\u0436',
2501                                 '\u0437', '\u043A', '\u043A', '\u043A',
2502                                 '\u043A', '\u043D', '\u043D', '\u043F',
2503                                 '\u0445', '\u0441', '\u0442', '\u0443',
2504                                 '\u0443', '\u0445', '\u0446', '\u0447',
2505                                 '\u0447', '\u0432', '\u0435', '\u0435',
2506                                 '\u0406', '\u0436', '\u043A', '\u043D',
2507                                 '\u0447', '\u0435'};
2508
2509                         fillIndex [0x10] = 0x8D;
2510                         for (int i = 0x0460; i < 0x0481; i++) {
2511                                 if (Char.IsLetter ((char) i)) {
2512                                         if (i == 0x0476)
2513                                                 // U+476/477 have the same
2514                                                 // primary weight as U+474/475.
2515                                                 fillIndex [0x10] -= 3;
2516                                         AddLetterMap ((char) i, 0x10, 3);
2517                                 }
2518                         }
2519
2520                         fillIndex [0x10] = 0x6;
2521                         for (int i = 0; i < orderedCyrillic.Length; i++) {
2522                                 char c = Char.ToUpper (orderedCyrillic [i], CultureInfo.InvariantCulture);
2523                                 if (!IsIgnorable ((int) c) &&
2524                                         Char.IsLetter (c) &&
2525                                         !map [c].Defined) {
2526                                         AddLetterMap (c, 0x10, 0);
2527                                         fillIndex [0x10] += 3;
2528                                 }
2529                         }
2530
2531                         // NFKD
2532                         for (int i = 0x0401; i <= 0x045F; i++)
2533                                 FillLetterNFKD (i, false, false);
2534
2535                         for (int i = 0; i < cymap_src.Length; i++) {
2536                                 char c = cymap_src [i];
2537                                 fillIndex [0x10] = map [c].Level1;
2538                                 int c2 = 0x0490 + i * 2;
2539                                 AddLetterMapCore ((char) c2, 0x10, 0, diacritical [c2], false);
2540                         }
2541
2542                         // Armenian
2543                         fillIndex [0x11] = 0x3;
2544                         fillIndex [0x1] = 0x98;
2545                         for (int i = 0x0531; i < 0x0586; i++) {
2546                                 if (i == 0x0559 || i == 0x55A)
2547                                         AddCharMap ((char) i, 1, 1);
2548                                 if (Char.IsLetter ((char) i))
2549                                         AddLetterMap ((char) i, 0x11, 1);
2550                         }
2551
2552                         // Hebrew
2553                         // -Letters
2554                         fillIndex [0x12] = 0x2;
2555                         for (int i = 0x05D0; i < 0x05FF; i++)
2556                                 if (Char.IsLetter ((char) i)) {
2557                                         if (isUppercase [i]) {
2558                                                 fillIndex [0x12]--;
2559                                                 AddLetterMap ((char) i, 0x12, 2);
2560                                         }
2561                                         else
2562                                                 AddLetterMap ((char) i, 0x12, 1);
2563                                 }
2564                         // -Accents
2565                         fillIndex [0x1] = 0x3;
2566                         for (int i = 0x0591; i <= 0x05C2; i++) {
2567                                 if (i == 0x05A3 || i == 0x05BB)
2568                                         fillIndex [0x1]++;
2569                                 if (i != 0x05BE)
2570                                         AddCharMap ((char) i, 0x1, 1);
2571                         }
2572
2573                         // Arabic
2574                         fillIndex [0x1] = 0x8E;
2575                         fillIndex [0x13] = 0x3;
2576                         for (int i = 0x0621; i <= 0x064A; i++) {
2577                                 // Abjad
2578                                 if (Char.GetUnicodeCategory ((char) i)
2579                                         != UnicodeCategory.OtherLetter) {
2580                                         // FIXME: arabic nonspacing marks are
2581                                         // in different order.
2582                                         AddCharMap ((char) i, 0x1, 1);
2583                                         continue;
2584                                 }
2585 //                              map [i] = new CharMapEntry (0x13,
2586 //                                      (byte) arabicLetterPrimaryValues [i], 1);
2587                                 fillIndex [0x13] =
2588                                         (byte) arabicLetterPrimaryValues [i];
2589                                 byte formDiacritical = 8; // default
2590                                 // SPECIAL CASES:
2591                                 switch (i) {
2592                                 case 0x0622: formDiacritical = 9; break;
2593                                 case 0x0623: formDiacritical = 0xA; break;
2594                                 case 0x0624: formDiacritical = 5; break;
2595                                 case 0x0625: formDiacritical = 0xB; break;
2596                                 case 0x0626: formDiacritical = 7; break;
2597                                 case 0x0649: formDiacritical = 5; break;
2598                                 case 0x064A: formDiacritical = 7; break;
2599                                 }
2600 //                              AddLetterMapCore ((char) i, 0x13, 1, formDiacritical, false);
2601                                 AddArabicCharMap ((char) i, 0x13, 1, formDiacritical);
2602                         }
2603                         for (int i = 0x0670; i < 0x0673; i++)
2604                                 map [i] = new CharMapEntry (0x13, 0xB, (byte) (0xC + i - 0x670));
2605                         fillIndex [0x13] = 0x84;
2606                         for (int i = 0x0674; i < 0x06D6; i++)
2607                                 if (Char.IsLetter ((char) i))
2608                                         AddLetterMapCore ((char) i, 0x13, 1, 0, false);
2609
2610                         // Devanagari
2611
2612                         // FIXME: this could be fixed in more decent way
2613                         for (int i = 0x0958; i <= 0x095F; i++)
2614                                 diacritical [i] = 8;
2615
2616                         // FIXME: it does seem straight codepoint mapping.
2617                         fillIndex [0x14] = 04;
2618                         for (int i = 0x0901; i < 0x0905; i++)
2619                                 if (!IsIgnorable (i))
2620                                         AddLetterMap ((char) i, 0x14, 2);
2621                         fillIndex [0x14] = 0xB;
2622                         for (int i = 0x0905; i < 0x093A; i++) {
2623                                 if (i == 0x0928)
2624                                         AddCharMap ('\u0929', 0x14, 0, 8);
2625                                 if (i == 0x0930)
2626                                         AddCharMap ('\u0931', 0x14, 0, 8);
2627                                 if (i == 0x0933)
2628                                         AddCharMap ('\u0934', 0x14, 0, 8);
2629                                 if (Char.IsLetter ((char) i))
2630                                         AddLetterMap ((char) i, 0x14, 4);
2631                                 if (i == 0x090B)
2632                                         AddCharMap ('\u0960', 0x14, 4);
2633                                 if (i == 0x090C)
2634                                         AddCharMap ('\u0961', 0x14, 4);
2635                         }
2636                         fillIndex [0x14] = 0xDA;
2637                         for (int i = 0x093E; i < 0x0945; i++)
2638                                 if (!IsIgnorable (i))
2639                                         AddLetterMap ((char) i, 0x14, 2);
2640                         fillIndex [0x14] = 0xEC;
2641                         for (int i = 0x0945; i < 0x094F; i++)
2642                                 if (!IsIgnorable (i))
2643                                         AddLetterMap ((char) i, 0x14, 2);
2644
2645                         // Bengali
2646                         // -Letters
2647                         fillIndex [0x15] = 02;
2648                         for (int i = 0x0980; i < 0x9FF; i++) {
2649                                 if (IsIgnorable (i))
2650                                         continue;
2651                                 if (i == 0x09E0)
2652                                         fillIndex [0x15] = 0x3B;
2653                                 switch (Char.GetUnicodeCategory ((char) i)) {
2654                                 case UnicodeCategory.NonSpacingMark:
2655                                 case UnicodeCategory.DecimalDigitNumber:
2656                                 case UnicodeCategory.OtherNumber:
2657                                         continue;
2658                                 }
2659                                 AddLetterMap ((char) i, 0x15, 1);
2660                         }
2661                         // -Signs
2662                         fillIndex [0x1] = 0x3;
2663                         for (int i = 0x0981; i < 0x0A00; i++)
2664                                 if (Char.GetUnicodeCategory ((char) i) ==
2665                                         UnicodeCategory.NonSpacingMark)
2666                                         AddCharMap ((char) i, 0x1, 1);
2667
2668                         // Gurmukhi. orderedGurmukhi is from UCA
2669                         // FIXME: it does not look equivalent to UCA.
2670                         fillIndex [0x16] = 04;
2671                         fillIndex [0x1] = 3;
2672                         for (int i = 0; i < orderedGurmukhi.Length; i++) {
2673                                 char c = orderedGurmukhi [i];
2674                                 if (IsIgnorable ((int) c))
2675                                         continue;
2676                                 if (IsIgnorableNonSpacing (c)) {
2677                                         AddLetterMap (c, 0x1, 1);
2678                                         continue;
2679                                 }
2680                                 if (c == '\u0A3C' || c == '\u0A4D' ||
2681                                         '\u0A66' <= c && c <= '\u0A71')
2682                                         continue;
2683                                 // SPECIAL CASES
2684                                 byte shift = 4;
2685                                 switch (c) {
2686                                 case '\u0A33': case '\u0A36': case '\u0A16':
2687                                 case '\u0A17': case '\u0A5B': case '\u0A5E':
2688                                         shift = 0;
2689                                         break;
2690                                 }
2691                                 if (c == '\u0A3E') // Skip
2692                                         fillIndex [0x16] = 0xC0;
2693                                 AddLetterMap (c, 0x16, shift);
2694                         }
2695
2696                         // Gujarati. orderedGujarati is from UCA
2697                         fillIndex [0x17] = 0x4;
2698                         // nonspacing marks
2699                         map [0x0A4D] = new CharMapEntry (1, 0, 0x3);
2700                         map [0x0ABD] = new CharMapEntry (1, 0, 0x3);
2701                         map [0x0A3C] = new CharMapEntry (1, 0, 0x4);
2702                         map [0x0A71] = new CharMapEntry (1, 0, 0x6);
2703                         map [0x0ABC] = new CharMapEntry (1, 0, 0xB);
2704                         map [0x0A70] = new CharMapEntry (1, 0, 0xE);
2705                         // letters go first.
2706                         for (int i = 0; i < orderedGujarati.Length; i++) {
2707                                 // SPECIAL CASE
2708                                 char c = orderedGujarati [i];
2709                                 if (Char.IsLetter (c)) {
2710                                         // SPECIAL CASES
2711                                         if (c == '\u0AB3' || c == '\u0A32')
2712                                                 continue;
2713                                         if (c == '\u0A33') {
2714                                                 AddCharMap ('\u0A32', 0x17, 0);
2715                                                 AddCharMap ('\u0A33', 0x17, 4, 4);
2716                                                 continue;
2717                                         }
2718                                         if (c == '\u0A8B')
2719                                                 AddCharMap ('\u0AE0', 0x17, 0, 5);
2720                                         AddCharMap (c, 0x17, 4);
2721
2722                                         if (c == '\u0AB9')
2723                                                 AddCharMap ('\u0AB3', 0x17, 6);
2724                                 }
2725                         }
2726                         // non-letters
2727                         byte gujaratiShift = 4;
2728                         fillIndex [0x17] = 0xC0;
2729                         for (int i = 0; i < orderedGujarati.Length; i++) {
2730                                 char c = orderedGujarati [i];
2731                                 if (fillIndex [0x17] == 0xCC)
2732                                         gujaratiShift = 3;
2733                                 if (!Char.IsLetter (c)) {
2734                                         // SPECIAL CASES
2735                                         if (c == '\u0A82')
2736                                                 AddCharMap ('\u0A81', 0x17, 2);
2737                                         if (c == '\u0AC2')
2738                                                 fillIndex [0x17]++;
2739                                         AddLetterMap (c, 0x17, gujaratiShift);
2740                                 }
2741                         }
2742
2743                         // Oriya
2744                         fillIndex [0x1] = 03;
2745                         fillIndex [0x18] = 02;
2746                         for (int i = 0x0B00; i < 0x0B7F; i++) {
2747                                 switch (Char.GetUnicodeCategory ((char) i)) {
2748                                 case UnicodeCategory.NonSpacingMark:
2749                                 case UnicodeCategory.DecimalDigitNumber:
2750                                         AddLetterMap ((char) i, 0x1, 1);
2751                                         continue;
2752                                 }
2753                                 AddLetterMapCore ((char) i, 0x18, 1, 0, true);
2754                         }
2755
2756                         // Tamil
2757                         fillIndex [0x19] = 2;
2758                         AddCharMap ('\u0BD7', 0x19, 0);
2759                         fillIndex [0x19] = 0xA;
2760                         // vowels
2761                         for (int i = 0x0B82; i <= 0x0B94; i++)
2762                                 if (!IsIgnorable ((char) i))
2763                                         AddCharMap ((char) i, 0x19, 2);
2764                         // special vowel
2765                         fillIndex [0x19] = 0x28;
2766                         // The array for Tamil consonants is a constant.
2767                         // Windows have almost similar sequence to TAM from
2768                         // tamilnet but a bit different in Grantha.
2769                         for (int i = 0; i < orderedTamilConsonants.Length; i++)
2770                                 AddLetterMap (orderedTamilConsonants [i], 0x19, 4);
2771                         // combining marks
2772                         fillIndex [0x19] = 0x82;
2773                         for (int i = 0x0BBE; i < 0x0BCD; i++)
2774                                 if (Char.GetUnicodeCategory ((char) i) ==
2775                                         UnicodeCategory.SpacingCombiningMark
2776                                         || i == 0x0BC0)
2777                                         AddLetterMap ((char) i, 0x19, 2);
2778
2779                         // Telugu
2780                         fillIndex [0x1A] = 0x4;
2781                         for (int i = 0x0C00; i < 0x0C62; i++) {
2782                                 if (i == 0x0C55 || i == 0x0C56)
2783                                         continue; // skip
2784                                 AddCharMap ((char) i, 0x1A, 3);
2785                                 char supp = (i == 0x0C0B) ? '\u0C60':
2786                                         i == 0x0C0C ? '\u0C61' : char.MinValue;
2787                                 if (supp == char.MinValue)
2788                                         continue;
2789                                 AddCharMap (supp, 0x1A, 3);
2790                         }
2791
2792                         // Kannada
2793                         fillIndex [0x1B] = 4;
2794                         for (int i = 0x0C80; i < 0x0CE5; i++) {
2795                                 if (i == 0x0CD5 || i == 0x0CD6)
2796                                         continue; // ignore
2797                                 if (i == 0x0CB1 || i == 0x0CB3 || i == 0x0CDE)
2798                                         continue; // shift after 0xCB9
2799                                 AddCharMap ((char) i, 0x1B, 3);
2800                                 if (i == 0x0CB9) {
2801                                         // SPECIAL CASES: but why?
2802                                         AddCharMap ('\u0CB1', 0x1B, 3); // RRA
2803                                         AddCharMap ('\u0CB3', 0x1B, 3); // LLA
2804                                         AddCharMap ('\u0CDE', 0x1B, 3); // FA
2805                                 }
2806                                 if (i == 0x0CB2)
2807                                         AddCharMap ('\u0CE1', 0x1B, 3); // vocalic LL
2808                         }
2809
2810                         // Malayalam
2811                         fillIndex [0x1C] = 2;
2812                         fillIndex [0x1] = 3;
2813                         for (int i = 0x0D02; i < 0x0D61; i++) {
2814                                 // FIXME: I avoided MSCompatUnicodeTable usage
2815                                 // here (it results in recursion). So check if
2816                                 // using NonSpacingMark makes sense or not.
2817                                 if (Char.GetUnicodeCategory ((char) i) != UnicodeCategory.NonSpacingMark)
2818 //                              if (!MSCompatUnicodeTable.IsIgnorable ((char) i))
2819                                         AddCharMap ((char) i, 0x1C, 1);
2820                                 else if (!IsIgnorable ((char) i))
2821                                         AddCharMap ((char) i, 1, 1);
2822                         }
2823
2824                         // Thai ... note that it breaks 0x1E wall after E2B!
2825                         // Also, all Thai characters have level 2 value 3.
2826                         fillIndex [0x1E] = 2;
2827                         fillIndex [0x1] = 3;
2828                         for (int i = 0xE40; i <= 0xE44; i++)
2829                                 AddCharMap ((char) i, 0x1E, 1, 3);
2830                         for (int i = 0xE01; i < 0xE2B; i++)
2831                                 AddCharMap ((char) i, 0x1E, 6, 3);
2832                         fillIndex [0x1F] = 5;
2833                         for (int i = 0xE2B; i < 0xE30; i++)
2834                                 AddCharMap ((char) i, 0x1F, 6, 3);
2835                         fillIndex [0x1F] = 0x1E;
2836                         for (int i = 0xE30; i < 0xE3B; i++)
2837                                 AddCharMap ((char) i, 0x1F, 1, 3);
2838                         // some Thai characters remains.
2839                         char [] specialThai = new char [] {'\u0E45', '\u0E46',
2840                                 '\u0E4E', '\u0E4F', '\u0E5A', '\u0E5B'};
2841                         foreach (char c in specialThai)
2842                                 AddCharMap (c, 0x1F, 1, 3);
2843
2844                         for (int i = 0xE00; i < 0xE80; i++)
2845                                 if (Char.GetUnicodeCategory ((char) i) ==
2846                                         UnicodeCategory.NonSpacingMark)
2847                                         AddCharMap ((char) i, 1, 1);
2848
2849                         // Lao
2850                         fillIndex [0x1F] = 2;
2851                         fillIndex [0x1] = 3;
2852                         for (int i = 0xE80; i < 0xEDF; i++) {
2853                                 if (IsIgnorable ((char) i))
2854                                         continue;
2855                                 else if (Char.IsLetter ((char) i))
2856                                         AddCharMap ((char) i, 0x1F, 1);
2857                                 else if (Char.GetUnicodeCategory ((char) i) ==
2858                                         UnicodeCategory.NonSpacingMark)
2859                                         AddCharMap ((char) i, 1, 1);
2860                         }
2861
2862                         // Georgian. orderedGeorgian is from UCA DUCET.
2863                         fillIndex [0x21] = 5;
2864                         for (int i = 0; i < orderedGeorgian.Length; i++) {
2865                                 char c = orderedGeorgian [i];
2866                                 if (map [(int) c].Defined)
2867                                         continue;
2868                                 AddCharMap (c, 0x21, 0);
2869                                 if (c < '\u10F6')
2870                                         AddCharMap ((char) (c - 0x30), 0x21, 0);
2871                                 fillIndex [0x21] += 5;
2872                         }
2873
2874                         // Japanese Kana.
2875                         fillIndex [0x22] = 2;
2876                         int kanaOffset = 0x3041;
2877                         byte [] kanaLines = new byte [] {2, 2, 2, 2, 1, 3, 1, 2, 1};
2878
2879                         for (int gyo = 0; gyo < 9; gyo++) {
2880                                 for (int dan = 0; dan < 5; dan++) {
2881                                         if (gyo == 7 && dan % 2 == 1) {
2882                                                 // 'ya'-gyo
2883                                                 fillIndex [0x22]++;
2884                                                 kanaOffset -= 2; // There is no space for yi and ye.
2885                                                 continue;
2886                                         }
2887                                         int cp = kanaOffset + dan * kanaLines [gyo];
2888                                         // small lines (a-gyo, ya-gyo)
2889                                         if (gyo == 0 || gyo == 7) {
2890                                                 AddKanaMap (cp, 1); // small
2891                                                 AddKanaMap (cp + 1, 1);
2892                                         }
2893                                         else
2894                                                 AddKanaMap (cp, kanaLines [gyo]);
2895                                         fillIndex [0x22]++;
2896
2897                                         if (cp == 0x30AB) {
2898                                                 // add small 'ka' (before normal one)
2899                                                 AddKanaMap (0x30F5, 1);
2900                                                 kanaOffset++;
2901                                         }
2902                                         if (cp == 0x30B1) {
2903                                                 // add small 'ke' (before normal one)
2904                                                 AddKanaMap (0x30F6, 1);
2905                                                 kanaOffset++;
2906                                         }
2907                                         if (cp == 0x3061) {
2908                                                 // add small 'Tsu' (before normal one)
2909                                                 AddKanaMap (0x3063, 1);
2910                                                 kanaOffset++;
2911                                         }
2912                                 }
2913                                 fillIndex [0x22] += 3;
2914                                 kanaOffset += 5 * kanaLines [gyo];
2915                         }
2916
2917                         // Wa-gyo is almost special, so I just manually add.
2918                         AddLetterMap ((char) 0x308E, 0x22, 0);
2919                         AddLetterMap ((char) (0x308E + 0x60), 0x22, 0);
2920                         AddLetterMap ((char) 0x308F, 0x22, 0);
2921                         AddLetterMap ((char) (0x308F + 0x60), 0x22, 0);
2922                         fillIndex [0x22]++;
2923                         AddLetterMap ((char) 0x3090, 0x22, 0);
2924                         AddLetterMap ((char) (0x3090 + 0x60), 0x22, 0);
2925                         fillIndex [0x22] += 2;
2926                         // no "Wu" in Japanese.
2927                         AddLetterMap ((char) 0x3091, 0x22, 0);
2928                         AddLetterMap ((char) (0x3091 + 0x60), 0x22, 0);
2929                         fillIndex [0x22]++;
2930                         AddLetterMap ((char) 0x3092, 0x22, 0);
2931                         AddLetterMap ((char) (0x3092 + 0x60), 0x22, 0);
2932                         // Nn
2933                         fillIndex [0x22] = 0x80;
2934                         AddLetterMap ((char) 0x3093, 0x22, 0);
2935                         AddLetterMap ((char) (0x3093 + 0x60), 0x22, 0);
2936
2937                         map [0x3094] = new CharMapEntry (map [0x30A6].Category,
2938                                 map [0x30A6].Level1, 3);// voiced hiragana U
2939                         map [0x30F4] = new CharMapEntry (map [0x30A6].Category,
2940                                 map [0x30A6].Level1, 3);// voiced katakana U
2941
2942                         map [0x30F5] = new CharMapEntry (map [0x30AB].Category,
2943                                 map [0x30AB].Level1, 0);// small katakana Ka
2944                         map [0x30F6] = new CharMapEntry (map [0x30B1].Category,
2945                                 map [0x30B1].Level1, 0);// small katakana Ke
2946                         // voiced Wa lines
2947                         for (int i = 0x30F7; i < 0x30FB; i++)
2948                                 map [i] = new CharMapEntry (map [i - 8].Category,
2949                                         map [i - 8].Level1,
2950                                         3);
2951
2952                         // JIS Japanese square chars.
2953                         fillIndex [0x22] = 0x97;
2954                         jisJapanese.Sort (JISComparer.Instance);
2955                         foreach (JISCharacter j in jisJapanese)
2956                                 if (0x3300 <= j.CP && j.CP <= 0x3357)
2957                                         AddCharMap ((char) j.CP, 0x22, 1);
2958                         // non-JIS Japanese square chars.
2959                         nonJisJapanese.Sort (NonJISComparer.Instance);
2960                         foreach (NonJISCharacter j in nonJisJapanese)
2961                                 AddCharMap ((char) j.CP, 0x22, 1);
2962
2963                         // Bopomofo
2964                         fillIndex [0x23] = 0x02;
2965                         for (int i = 0x3105; i <= 0x312C; i++)
2966                                 AddCharMap ((char) i, 0x23, 1);
2967
2968                         // Estrangela: ancient Syriac
2969                         fillIndex [0x24] = 0x0B;
2970                         // FIXME: is 0x71E really alternative form?
2971                         ArrayList syriacAlternatives = new ArrayList (
2972                                 new int [] {0x714, 0x716, 0x71C, 0x71E, 0x724, 0x727});
2973                         for (int i = 0x0710; i <= 0x072C; i++) {
2974                                 if (i == 0x0711) // NonSpacingMark
2975                                         continue;
2976                                 if (syriacAlternatives.Contains (i))
2977                                         continue;
2978                                 AddCharMap ((char) i, 0x24, 4);
2979                                 // FIXME: why?
2980                                 if (i == 0x721)
2981                                         fillIndex [0x24]++;
2982                         }
2983                         foreach (int cp in syriacAlternatives)
2984                                 map [cp] = new CharMapEntry (0x24,
2985                                         (byte) (map [cp - 1].Level1 + 2),
2986                                         0);
2987                         // FIXME: Syriac NonSpacingMark should go here.
2988
2989                         // Thaana
2990                         // FIXME: it turned out that it does not look like UCA
2991                         fillIndex [0x24] = 0x6E;
2992                         fillIndex [0x1] = 0xAC;
2993                         for (int i = 0; i < orderedThaana.Length; i++) {
2994                                 char c = orderedThaana [i];
2995                                 if (IsIgnorableNonSpacing ((int) c))
2996                                         AddCharMap (c, 1, 1);
2997                                 AddCharMap (c, 0x24, 2);
2998                                 if (c == '\u0782') // SPECIAL CASE: why?
2999                                         fillIndex [0x24] += 2;
3000                         }
3001                         #endregion
3002
3003                         // FIXME: Add more culture-specific letters (that are
3004                         // not supported in Windows collation) here.
3005
3006                         // Surrogate ... they are computed.
3007
3008                         #region Hangul
3009                         // Hangul.
3010                         //
3011                         // Unlike UCA Windows Hangul sequence mixes Jongseong
3012                         // with Choseong sequence as well as Jungseong,
3013                         // adjusted to have the same primary weight for the
3014                         // same base character. So it is impossible to compute
3015                         // those sort keys.
3016                         //
3017                         // Here I introduce an ordered sequence of mixed
3018                         // 'commands' and 'characters' that is similar to
3019                         // LDML text:
3020                         //      - ',' increases primary weight.
3021                         //      - [A B] means a range, increasing index
3022                         //      - {A B} means a range, without increasing index
3023                         //      - '=' is no operation (it means the characters
3024                         //        of both sides have the same weight).
3025                         //      - '>' inserts a Hangul Syllable block that
3026                         //        contains 0x251 characters.
3027                         //      - '<' decreases the index
3028                         //      - '0'-'9' means skip count
3029                         //      - whitespaces are ignored
3030                         //
3031
3032                         string hangulSequence =
3033                         + "\u1100=\u11A8 > \u1101=\u11A9 >"
3034                         + "\u11C3, \u11AA, \u11C4, \u1102=\u11AB >"
3035                         + "<{\u1113 \u1116}, \u3165,"
3036                                 + "\u11C5, \u11C6=\u3166,, \u11C7, \u11C8,"
3037                                 + "\u11AC, \u11C9, \u11AD, \u1103=\u11AE  >"
3038                         + "<\u1117, \u11CA, \u1104, \u11CB > \u1105=\u11AF >"
3039                         + "<{\u1118 \u111B}, \u11B0, [\u11CC \u11D0], \u11B1,"
3040                                 + "[\u11D1 \u11D2], \u11B2,"
3041                                 + "[\u11D3 \u11D5], \u11B3,"
3042                                 + "[\u11D6 \u11D7], \u11B4, \u11B5,"
3043                                 + "\u11B6=\u11D8, \u3140,, \u11D9, \u1106=\u11B7 >"
3044                         + "<{\u111C \u111D}, [\u11DA \u11E2], \u1107=\u11B8 >"
3045                         + "<{\u111E \u1120}, \u3172,, \u3173, \u11E3, \u1108 >"
3046                         + "<{\u1121 \u112C}, \u3144 \u11B9, \u3174, \u3175,,,, "
3047                                 + "\u3176,, \u3177, [\u11E4 \u11E6] \u3178,"
3048                                 + "\u3179, \u1109=\u11BA,,, \u3214=\u3274 <>"
3049                         + "<{\u112D \u1133}, \u11E7 \u317A, \u317B, \u317C "
3050                                 + "[\u11E8 \u11E9],, \u11EA \u317D,, \u110A=\u11BB,,, >"
3051                         + "<{\u1134 \u1140}, \u317E,,,,,,, \u11EB,"
3052                                 + "\u110B=\u11BC, [\u1161 \u11A2], \u1160 >"
3053                         + "<{\u1141 \u114C}, \u3180=\u11EE, \u11EC, \u11ED,,,,, "
3054                                 + "\u11F1,, \u11F2,,,"
3055                                 + "\u11EF,,, \u3181=\u11F0, \u110C=\u11BD,, >"
3056                         + "<\u114D, \u110D,,  >"
3057                         + "<{\u114E \u1151},, \u110E=\u11BE,,  >"
3058                         + "<{\u1152 \u1155},,, \u110F=\u11BF >"
3059                         + "\u1110=\u11C0 > \u1111=\u11C1 >"
3060                         + "<\u1156=\u1157, \u11F3, \u11F4, \u1112=\u11C2 >"
3061                         + "<\u1158=\u1159=\u115F, \u3185, \u11F9,"
3062                                 + "[\u11F5 \u11F8]"
3063                         ;
3064
3065                         byte hangulCat = 0x52;
3066                         fillIndex [hangulCat] = 0x2;
3067
3068                         int syllableBlock = 0;
3069                         for (int n = 0; n < hangulSequence.Length; n++) {
3070                                 char c = hangulSequence [n];
3071                                 int start, end;
3072                                 if (Char.IsWhiteSpace (c))
3073                                         continue;
3074                                 switch (c) {
3075                                 case '=':
3076                                         break; // NOP
3077                                 case ',':
3078                                         IncrementSequentialIndex (ref hangulCat);
3079                                         break;
3080                                 case '<':
3081                                         if (fillIndex [hangulCat] == 2)
3082                                                 throw new Exception ("FIXME: handle it correctly (yes it is hacky, it is really unfortunate).");
3083                                         fillIndex [hangulCat]--;
3084                                         break;
3085                                 case '>':
3086                                         IncrementSequentialIndex (ref hangulCat);
3087                                         for (int l = 0; l < 0x15; l++)
3088                                                 for (int v = 0; v < 0x1C; v++) {
3089                                                         AddCharMap (
3090                                                                 (char) (0xAC00 + syllableBlock * 0x1C * 0x15 + l * 0x1C + v), hangulCat, 0);
3091                                                         IncrementSequentialIndex (ref hangulCat);
3092                                                 }
3093                                         syllableBlock++;
3094                                         break;
3095                                 case '[':
3096                                         start = hangulSequence [n + 1];
3097                                         end = hangulSequence [n + 3];
3098                                         for (int i = start; i <= end; i++) {
3099                                                 AddCharMap ((char) i, hangulCat, 0);
3100                                                 if (end > i)
3101                                                         IncrementSequentialIndex (ref hangulCat);
3102                                         }
3103                                         n += 4; // consumes 5 characters for this operation
3104                                         break;
3105                                 case '{':
3106                                         start = hangulSequence [n + 1];
3107                                         end = hangulSequence [n + 3];
3108                                         for (int i = start; i <= end; i++)
3109                                                 AddCharMap ((char) i, hangulCat, 0);
3110                                         n += 4; // consumes 5 characters for this operation
3111                                         break;
3112                                 default:
3113                                         AddCharMap (c, hangulCat, 0);
3114                                         break;
3115                                 }
3116                         }
3117
3118                         // Some Jamo NFKD.
3119                         for (int i = 0x3200; i < 0x3300; i++) {
3120                                 if (IsIgnorable (i) || map [i].Defined)
3121                                         continue;
3122                                 int ch = 0;
3123                                 // w/ bracket
3124                                 if (decompLength [i] == 4 &&
3125                                         decompValues [decompIndex [i]] == '(')
3126                                         ch = decompIndex [i] + 1;
3127                                 // circled
3128                                 else if (decompLength [i] == 2 &&
3129                                         decompValues [decompIndex [i] + 1] == '\u1161')
3130                                         ch = decompIndex [i];
3131                                 else if (decompLength [i] == 1)
3132                                         ch = decompIndex [i];
3133                                 else
3134                                         continue;
3135                                 ch = decompValues [ch];
3136                                 if (ch < 0x1100 || 0x1200 < ch &&
3137                                         ch < 0xAC00 || 0xD800 < ch)
3138                                         continue;
3139
3140                                 // SPECIAL CASE ?
3141                                 int offset = i < 0x3260 ? 1 : 0;
3142                                 if (0x326E <= i && i <= 0x3273)
3143                                         offset = 1;
3144
3145                                 map [i] = new CharMapEntry (map [ch].Category,
3146                                         (byte) (map [ch].Level1 + offset),
3147                                         map [ch].Level2);
3148 //                                      Console.Error.WriteLine ("Jamo {0:X04} -> {1:X04}", i, decompValues [decompIndex [i] + 1]);
3149                         }
3150
3151
3152                         #endregion
3153
3154                         // Letterlike characters and CJK compatibility square
3155                         sortableCharNames.Sort (StringDictionaryValueComparer.Instance);
3156                         int [] counts = new int ['Z' - 'A' + 1];
3157                         char [] namedChars = new char [sortableCharNames.Count];
3158                         int nCharNames = 0;
3159                         foreach (DictionaryEntry de in sortableCharNames) {
3160                                 counts [((string) de.Value) [0] - 'A']++;
3161                                 namedChars [nCharNames++] = (char) ((int) de.Key);
3162                         }
3163                         nCharNames = 0; // reset
3164                         for (int a = 0; a < counts.Length; a++) {
3165                                 fillIndex [0xE] = (byte) (alphaWeights [a + 1] - counts [a]);
3166                                 for (int i = 0; i < counts [a]; i++)
3167 //Console.Error.WriteLine ("---- {0:X04} : {1:x02} / {2} {3}", (int) namedChars [nCharNames], fillIndex [0xE], ((DictionaryEntry) sortableCharNames [nCharNames]).Value, Char.GetUnicodeCategory (namedChars [nCharNames]));
3168                                         AddCharMap (namedChars [nCharNames++], 0xE, 1);
3169                         }
3170
3171                         // CJK unified ideograph.
3172                         byte cjkCat = 0x9E;
3173                         fillIndex [cjkCat] = 0x2;
3174                         for (int cp = 0x4E00; cp <= 0x9FBB; cp++)
3175                                 if (!IsIgnorable (cp))
3176                                         AddCharMapGroupCJK ((char) cp, ref cjkCat);
3177                         // CJK Extensions goes here.
3178                         // LAMESPEC: With this Windows style CJK layout, it is
3179                         // impossible to add more CJK ideograph i.e. 0x9FA6-
3180                         // 0x9FBB can never be added w/o breaking compat.
3181                         for (int cp = 0xF900; cp <= 0xFA2D; cp++)
3182                                 if (!IsIgnorable (cp))
3183                                         AddCharMapGroupCJK ((char) cp, ref cjkCat);
3184
3185                         // PrivateUse ... computed.
3186                         // remaining Surrogate ... computed.
3187
3188                         #region 07 - ASCII non-alphanumeric + 3001, 3002 // 07
3189                         // non-alphanumeric ASCII except for: + - < = > '
3190                         for (int i = 0x21; i < 0x7F; i++) {
3191                                 // SPECIAL CASE: 02C6 looks regarded as
3192                                 // equivalent to '^', which does not conform
3193                                 // to Unicode standard character database.
3194                                 if (i == 0x005B)
3195                                         AddCharMap ('\u2045', 0x7, 0, 0x1C);
3196                                 if (i == 0x005D)
3197                                         AddCharMap ('\u2046', 0x7, 0, 0x1C);
3198                                 if (i == 0x005E)
3199                                         AddCharMap ('\u02C6', 0x7, 0, 3);
3200                                 if (i == 0x0060)
3201                                         AddCharMap ('\u02CB', 0x7, 0, 3);
3202
3203                                 if (Char.IsLetterOrDigit ((char) i)
3204                                         || "+-<=>'".IndexOf ((char) i) >= 0)
3205                                         continue; // they are not added here.
3206
3207                                 AddCharMapGroup2 ((char) i, 0x7, 1, 0);
3208                                 // Insert 3001 after ',' and 3002 after '.'
3209                                 if (i == 0x2C)
3210                                         AddCharMapGroup2 ('\u3001', 0x7, 1, 0);
3211                                 else if (i == 0x2E)
3212                                         AddCharMapGroup2 ('\u3002', 0x7, 1, 0);
3213                                 else if (i == 0x3A)
3214                                         AddCharMap ('\uFE30', 0x7, 1, 0);
3215                         }
3216                         #endregion
3217
3218                         #region 07 - Punctuations and something else
3219                         for (int i = 0xA0; i < char.MaxValue; i++) {
3220                                 if (IsIgnorable (i))
3221                                         continue;
3222
3223                                 // FIXME: actually those reset should not be
3224                                 // done but here I put for easy goal.
3225                                 if (i == 0x05C3)
3226                                         fillIndex [0x7]++;
3227                                 if (i == 0x0700)
3228                                         fillIndex [0x7] = 0xE2;
3229                                 if (i == 0x2016)
3230                                         fillIndex [0x7] = 0x77;
3231                                 if (i == 0x3008)
3232                                         fillIndex [0x7] = 0x93;
3233
3234                                 if (0x02C8 <= i && i <= 0x02CD)
3235                                         continue; // nonspacing marks
3236
3237                                 // SPECIAL CASE: maybe they could be allocated
3238                                 // dummy NFKD mapping and no special processing
3239                                 // would be required here.
3240                                 if (i == 0x00AF)
3241                                         AddCharMap ('\u02C9', 0x7, 0, 3);
3242                                 if (i == 0x00B4)
3243                                         AddCharMap ('\u02CA', 0x7, 0, 3);
3244                                 if (i == 0x02C7)
3245                                         AddCharMap ('\u02D8', 0x7, 0, 3);
3246
3247                                 // SPECIAL CASES:
3248                                 switch (i) {
3249                                 case 0xAB: // 08
3250                                 case 0xB7: // 0A
3251                                 case 0xBB: // 08
3252                                 case 0x02B9: // 01
3253                                 case 0x02BA: // 01
3254                                 case 0x2329: // 09
3255                                 case 0x232A: // 09
3256                                         continue;
3257                                 }
3258
3259                                 switch (Char.GetUnicodeCategory ((char) i)) {
3260                                 case UnicodeCategory.OtherPunctuation:
3261                                 case UnicodeCategory.ClosePunctuation:
3262                                 case UnicodeCategory.OpenPunctuation:
3263                                 case UnicodeCategory.ConnectorPunctuation:
3264                                 case UnicodeCategory.InitialQuotePunctuation:
3265                                 case UnicodeCategory.FinalQuotePunctuation:
3266                                 case UnicodeCategory.ModifierSymbol:
3267                                         // SPECIAL CASES: // 0xA
3268                                         if (0x2020 <= i && i <= 0x2031)
3269                                                 continue;
3270                                         if (i == 0x3003) // added later
3271                                                 continue;
3272                                         AddCharMapGroup2 ((char) i, 0x7, 1, 0);
3273                                         break;
3274                                 default:
3275                                         if (i == 0xA6 || i == 0x1C3 || i == 0x037A) // SPECIAL CASE. FIXME: why?
3276                                                 goto case UnicodeCategory.OtherPunctuation;
3277                                         break;
3278                                 }
3279                         }
3280
3281                         // Control pictures
3282                         // FIXME: it should not need to reset level 1, but
3283                         // it's for easy goal.
3284                         fillIndex [0x7] = 0xB6;
3285                         for (int i = 0x2400; i <= 0x2424; i++)
3286                                 AddCharMap ((char) i, 0x7, 1, 0);
3287
3288                         // FIXME: what are they?
3289                         AddCharMap ('\u3003', 0x7, 1);
3290                         AddCharMap ('\u3006', 0x7, 1);
3291                         AddCharMap ('\u02D0', 0x7, 1);
3292                         AddCharMap ('\u10FB', 0x7, 1);
3293                         AddCharMap ('\u0950', 0x7, 1);
3294                         AddCharMap ('\u093D', 0x7, 1);
3295                         AddCharMap ('\u0964', 0x7, 1);
3296                         AddCharMap ('\u0965', 0x7, 1);
3297                         AddCharMap ('\u0970', 0x7, 1);
3298
3299                         #endregion
3300
3301                         #region category 08 - symbols
3302                         fillIndex [0x8] = 2;
3303                         // Here Windows mapping is not straightforward. It is
3304                         // not based on computation but seems manual sorting.
3305                         AddCharMapGroup ('+', 0x8, 1, 0); // plus
3306                         AddCharMapGroup ('\u2212', 0x8, 1); // minus
3307                         AddCharMapGroup ('\u229D', 0x8, 1); // minus
3308                         AddCharMapGroup ('\u2297', 0x8, 1); // mul
3309                         AddCharMapGroup ('\u2044', 0x8, 1); // div
3310                         AddCharMapGroup ('\u2215', 0x8, 0); // div
3311                         AddCharMapGroup ('\u2298', 0x8, 1); // div slash
3312                         AddCharMapGroup ('\u2217', 0x8, 0); // mul
3313                         AddCharMapGroup ('\u229B', 0x8, 1); // asterisk oper
3314                         AddCharMapGroup ('\u2218', 0x8, 0); // ring
3315                         AddCharMapGroup ('\u229A', 0x8, 1); // ring
3316                         AddCharMapGroup ('\u2219', 0x8, 0); // bullet
3317                         AddCharMapGroup ('\u2299', 0x8, 1); // dot oper
3318                         AddCharMapGroup ('\u2213', 0x8, 1); // minus-or-plus
3319                         AddCharMapGroup ('\u003C', 0x8, 1); // <
3320                         AddCharMapGroup ('\u227A', 0x8, 1); // precedes relation
3321                         AddCharMapGroup ('\u22B0', 0x8, 1); // precedes under relation
3322
3323                         for (int cp = 0; cp < 0x2300; cp++) {
3324                                 if (cp == 0xAC) // SPECIAL CASE: skip
3325                                         continue;
3326                                 if (cp == 0x200) {
3327                                         cp = 0x2200; // skip to 2200
3328                                         fillIndex [0x8] = 0x21;
3329                                 }
3330                                 if (cp == 0x2295)
3331                                         fillIndex [0x8] = 0x3;
3332                                 if (cp == 0x22A2)
3333                                         fillIndex [0x8] = 0xAB;
3334                                 if (cp == 0x22B2)
3335                                         fillIndex [0x8] = 0xB9;
3336                                 if (!map [cp].Defined &&
3337 //                                      Char.GetUnicodeCategory ((char) cp) ==
3338 //                                      UnicodeCategory.MathSymbol)
3339                                         Char.IsSymbol ((char) cp))
3340                                         AddCharMapGroup ((char) cp, 0x8, 1);
3341                                 // SPECIAL CASES: no idea why Windows sorts as such
3342                                 switch (cp) {
3343                                 case 0x3E:
3344                                         AddCharMap ('\u227B', 0x8, 1, 0);
3345                                         AddCharMap ('\u22B1', 0x8, 1, 0);
3346                                         break;
3347                                 case 0xB1:
3348                                         AddCharMapGroup ('\u00AB', 0x8, 1);
3349                                         AddCharMapGroup ('\u226A', 0x8, 1);
3350                                         AddCharMapGroup ('\u00BB', 0x8, 1);
3351                                         AddCharMapGroup ('\u226B', 0x8, 1);
3352                                         break;
3353                                 case 0xF7:
3354                                         AddCharMap ('\u01C0', 0x8, 1, 0);
3355                                         AddCharMap ('\u01C1', 0x8, 1, 0);
3356                                         AddCharMap ('\u01C2', 0x8, 1, 0);
3357                                         break;
3358                                 }
3359                         }
3360                         #endregion
3361
3362                         #region Hack!
3363
3364                         // Characters w/ diacritical marks (NFKD)
3365                         for (int i = 0; i <= char.MaxValue; i++) {
3366                                 if (map [i].Defined || IsIgnorable (i))
3367                                         continue;
3368                                 if (decompIndex [i] == 0)
3369                                         continue;
3370
3371                                 int start = decompIndex [i];
3372                                 int primaryChar = decompValues [start];
3373                                 int secondary = diacritical [i];
3374                                 bool skip = false;
3375                                 int length = decompLength [i];
3376                                 // special processing for parenthesized ones.
3377                                 if (length == 3 &&
3378                                         decompValues [start] == '(' &&
3379                                         decompValues [start + 2] == ')') {
3380                                         primaryChar = decompValues [start + 1];
3381                                         length = 1;
3382                                 }
3383
3384                                 if (map [primaryChar].Level1 == 0)
3385                                         continue;
3386
3387                                 for (int l = 1; l < length; l++) {
3388                                         int c = decompValues [start + l];
3389                                         if (map [c].Level1 != 0)
3390                                                 skip = true;
3391                                         secondary += diacritical [c];
3392                                 }
3393                                 if (skip)
3394                                         continue;
3395                                 map [i] = new CharMapEntry (
3396                                         map [primaryChar].Category,
3397                                         map [primaryChar].Level1,
3398                                         (byte) secondary);
3399
3400                         }
3401
3402                         // Diacritical weight adjustment
3403
3404                         // Arabic Hamzah
3405                         diacritical [0x624] = 0x5;
3406                         diacritical [0x626] = 0x7;
3407                         diacritical [0x622] = 0x9;
3408                         diacritical [0x623] = 0xA;
3409                         diacritical [0x625] = 0xB;
3410                         diacritical [0x649] = 0x5; // 'alif maqs.uurah
3411                         diacritical [0x64A] = 0x7; // Yaa'
3412
3413                         for (int i = 0; i < char.MaxValue; i++) {
3414                                 byte mod = 0;
3415                                 byte cat = map [i].Category;
3416                                 switch (cat) {
3417                                 case 0xE: // Latin diacritics
3418                                 case 0x22: // Japanese: circled characters
3419                                         mod = diacritical [i];
3420                                         break;
3421                                 case 0x13: // Arabic
3422                                         if (i == 0x0621)
3423                                                 break; // 0
3424                                         if (diacritical [i] == 0 && decompLength [i] != 0)
3425                                                 diacritical [i] = map [decompValues [decompIndex [i]]].Level2;
3426                                         if (diacritical [i] == 0 && i >= 0xFE8D)
3427                                                 mod = 0x8; // default for arabic
3428                                         break;
3429                                 }
3430                                 if (0x52 <= cat && cat <= 0x7F) // Hangul
3431                                         mod = diacritical [i];
3432                                 if (mod > 0)
3433                                         map [i] = new CharMapEntry (
3434                                                 cat, map [i].Level1, mod);
3435                         }
3436
3437                         // FIXME: this is halfly hack but those NonSpacingMark
3438                         // characters and still undefined are likely to
3439                         // be nonspacing.
3440                         for (int i = 0; i < char.MaxValue; i++) {
3441                                 if (map [i].Defined ||
3442                                         IsIgnorable (i))
3443                                         continue;
3444                                 switch (i) {
3445                                 // SPECIAL CASES.
3446                                 case 0x02B9:
3447                                 case 0x02BA:
3448                                         break;
3449                                 default:
3450                                         if (Char.GetUnicodeCategory ((char) i) !=
3451                                         UnicodeCategory.NonSpacingMark)
3452                                                 continue;
3453                                         break;
3454                                 }
3455                                 if (diacritical [i] != 0)
3456                                         map [i] = new CharMapEntry (1, 1, diacritical [i]);
3457                                 else
3458                                         AddCharMap ((char) i, 1, 1);
3459                         }
3460
3461                         #endregion
3462                 }
3463
3464                 TextInfo ti = CultureInfo.InvariantCulture.TextInfo;
3465
3466                 private void FillLetterNFKD (int i, bool checkUpper, bool greekRemap)
3467                 {
3468                         if (map [i].Defined)
3469                                 return;
3470                         int up = (int) ti.ToUpper ((char) i);
3471                         if (checkUpper && map [up].Category == 0xF) {
3472                                 if (i == up)
3473                                         return;
3474                                 FillLetterNFKD (up, checkUpper, greekRemap);
3475                                 map [i] = new CharMapEntry (0xF,
3476                                         map [up].Level1,
3477                                         map [up].Level2);
3478                         } else {
3479                                 int idx = decompIndex [i];
3480                                 if (idx == 0)
3481                                         return;
3482                                 int primary = decompValues [decompIndex [i]];
3483                                 FillLetterNFKD (primary, checkUpper, greekRemap);
3484
3485                                 int lv2 = map [primary].Level2;
3486                                 byte off = 0;
3487                                 for (int l = 1; l < decompLength [i]; l++) {
3488                                         int tmp = decompValues [idx + l];
3489                                         if (map [tmp].Category != 1)
3490                                                 return;
3491                                         if (greekRemap && map [tmp].Level2 == 0xC)
3492                                                 off += 3;
3493                                         else
3494                                                 off += map [tmp].Level2;
3495                                 }
3496                                 if (off > 0) {
3497                                         if (lv2 == 0)
3498                                                 lv2 += 2;
3499                                         lv2 += off;
3500                                 }
3501                                 // ... but override if the value already exists.
3502                                 if (diacritical [i] != 0)
3503                                         lv2 = diacritical [i];
3504                                 map [i] = new CharMapEntry (
3505                                         map [primary].Category,
3506                                         map [primary].Level1,
3507                                         (byte) lv2);
3508                         }
3509                 }
3510
3511                 private void IncrementSequentialIndex (ref byte hangulCat)
3512                 {
3513                         fillIndex [hangulCat]++;
3514                         if (fillIndex [hangulCat] == 0) { // overflown
3515                                 hangulCat++;
3516                                 fillIndex [hangulCat] = 0x2;
3517                         }
3518                 }
3519
3520                 // Reset fillIndex to fixed value and call AddLetterMap().
3521                 private void AddAlphaMap (char c, byte category, byte alphaWeight)
3522                 {
3523                         fillIndex [category] = alphaWeight;
3524                         AddLetterMap (c, category, 0);
3525
3526                         ArrayList al = latinMap [c] as ArrayList;
3527                         if (al == null)
3528                                 return;
3529
3530                         foreach (int cp in al)
3531                                 AddLetterMap ((char) cp, category, 0);
3532                 }
3533
3534                 private void AddKanaMap (int i, byte voices)
3535                 {
3536                         for (byte b = 0; b < voices; b++) {
3537                                 char c = (char) (i + b);
3538                                 byte arg = (byte) (b > 0 ? b + 2 : 0);
3539                                 // Hiragana
3540                                 AddLetterMapCore (c, 0x22, 0, arg, false);
3541                                 // Katakana
3542                                 AddLetterMapCore ((char) (c + 0x60), 0x22, 0, arg, false);
3543                         }
3544                 }
3545
3546                 private void AddLetterMap (char c, byte category, byte updateCount)
3547                 {
3548                         AddLetterMapCore (c, category, updateCount, 0, true);
3549                 }
3550
3551                 private void AddLetterMapCore (char c, byte category, byte updateCount, byte level2, bool deferLevel2)
3552                 {
3553                         char c2;
3554                         // <small> updates index
3555                         c2 = ToSmallForm (c);
3556                         if (c2 != c)
3557                                 AddCharMapGroup (c2, category, updateCount, level2, deferLevel2);
3558                         c2 = Char.ToLower (c, CultureInfo.InvariantCulture);
3559                         if (c2 != c && !map [(int) c2].Defined)
3560                                 AddLetterMapCore (c2, category, 0, level2, deferLevel2);
3561                         bool doUpdate = true;
3562                         if (IsIgnorable ((int) c) || map [(int) c].Defined)
3563                                 doUpdate = false;
3564                         else
3565                                 AddCharMapGroup (c, category, 0, level2, deferLevel2);
3566                         if (doUpdate)
3567                                 fillIndex [category] += updateCount;
3568                 }
3569
3570                 private bool AddCharMap (char c, byte category, byte increment)
3571                 {
3572                         return AddCharMap (c, category, increment, 0);
3573                 }
3574
3575                 private bool AddCharMap (char c, byte category, byte increment, byte alt)
3576                 {
3577                         if (IsIgnorable ((int) c) || map [(int) c].Defined)
3578                                 return false; // do nothing
3579                         map [(int) c] = new CharMapEntry (category,
3580                                 category == 1 ? alt : fillIndex [category],
3581                                 category == 1 ? fillIndex [category] : alt);
3582                         fillIndex [category] += increment;
3583                         return true;
3584                 }
3585
3586                 //
3587                 // Adds characters to table in the order below
3588                 // (+ increases weight):
3589                 //      (<small> +)
3590                 //      itself
3591                 //      <fraction>
3592                 //      <full> | <super> | <sub>
3593                 //      <circle> | <wide> (| <narrow>)
3594                 //      +
3595                 //      (vertical +)
3596                 //
3597                 // level2 is fixed (does not increase).
3598                 int [] sameWeightItems = new int [] {
3599                         DecompositionFraction,
3600                         DecompositionFull,
3601                         DecompositionSuper,
3602                         DecompositionSub,
3603                         DecompositionCircle,
3604                         DecompositionWide,
3605                         DecompositionNarrow,
3606                         };
3607                 private void AddCharMapGroup (char c, byte category, byte updateCount)
3608                 {
3609                         AddCharMapGroup (c, category, updateCount, 0, true);
3610                 }
3611
3612                 private void AddCharMapGroup (char c, byte category, byte updateCount, byte level2)
3613                 {
3614                         AddCharMapGroup (c, category, updateCount, level2, false);
3615                 }
3616
3617                 private void AddCharMapGroup (char c, byte category, byte updateCount, byte level2, bool deferLevel2)
3618                 {
3619                         if (map [(int) c].Defined)
3620                                 return;
3621
3622                         if (deferLevel2)
3623                                 level2 = diacritical [(int) c];
3624
3625                         char small = char.MinValue;
3626                         char vertical = char.MinValue;
3627                         Hashtable nfkd = (Hashtable) nfkdMap [(int) c];
3628                         if (nfkd != null) {
3629                                 object smv = nfkd [(byte) DecompositionSmall];
3630                                 if (smv != null)
3631                                         small = (char) ((int) smv);
3632                                 object vv = nfkd [(byte) DecompositionVertical];
3633                                 if (vv != null)
3634                                         vertical = (char) ((int) vv);
3635                         }
3636
3637                         // <small> updates index
3638                         if (small != char.MinValue) {
3639                                 if (level2 == 0 && deferLevel2)
3640                                         level2 = diacritical [small];
3641                                 AddCharMap (small, category, updateCount, level2);
3642                         }
3643
3644                         // itself
3645                         AddCharMap (c, category, 0, level2);
3646
3647                         if (nfkd != null) {
3648                                 foreach (int weight in sameWeightItems) {
3649                                         object wv = nfkd [(byte) weight];
3650                                         if (wv != null) {
3651                                                 if (deferLevel2)
3652                                                         level2 = diacritical [(int) wv];
3653                                                 AddCharMap ((char) ((int) wv), category, 0, level2);
3654                                         }
3655                                 }
3656                         }
3657
3658                         // update index here.
3659                         fillIndex [category] += updateCount;
3660
3661                         if (vertical != char.MinValue) {
3662                                 if (level2 == 0 && deferLevel2)
3663                                         level2 = diacritical [vertical];
3664                                 AddCharMap (vertical, category, updateCount, level2);
3665                         }
3666                 }
3667
3668                 private void AddCharMapCJK (char c, ref byte category)
3669                 {
3670                         AddCharMap (c, category, 0, 0);
3671                         IncrementSequentialIndex (ref category);
3672
3673                         // Special. I wonder why but Windows skips 9E F9.
3674                         if (category == 0x9E && fillIndex [category] == 0xF9)
3675                                 IncrementSequentialIndex (ref category);
3676                 }
3677
3678                 private void AddCharMapGroupCJK (char c, ref byte category)
3679                 {
3680                         AddCharMapCJK (c, ref category);
3681
3682                         // LAMESPEC: see below.
3683                         if (c == '\u5B78') {
3684                                 AddCharMapCJK ('\u32AB', ref category);
3685                                 AddCharMapCJK ('\u323B', ref category);
3686                         }
3687                         if (c == '\u52DE') {
3688                                 AddCharMapCJK ('\u3298', ref category);
3689                                 AddCharMapCJK ('\u3238', ref category);
3690                         }
3691                         if (c == '\u5BEB')
3692                                 AddCharMapCJK ('\u32A2', ref category);
3693                         if (c == '\u91AB')
3694                                 // Especially this mapping order totally does
3695                                 // not make sense to me.
3696                                 AddCharMapCJK ('\u32A9', ref category);
3697
3698                         Hashtable nfkd = (Hashtable) nfkdMap [(int) c];
3699                         if (nfkd == null)
3700                                 return;
3701                         for (byte weight = 0; weight <= 0x12; weight++) {
3702                                 object wv = nfkd [weight];
3703                                 if (wv == null)
3704                                         continue;
3705                                 int w = (int) wv;
3706
3707                                 // Special: they are ignored in this area.
3708                                 // FIXME: check if it is sane
3709                                 if (0xF900 <= w && w <= 0xFAD9)
3710                                         continue;
3711                                 // LAMESPEC: on Windows some of CJK characters
3712                                 // in 3200-32B0 are incorrectly mapped. They
3713                                 // mix Chinise and Japanese Kanji when
3714                                 // ordering those characters.
3715                                 switch (w) {
3716                                 case 0x32A2: case 0x3298: case 0x3238:
3717                                 case 0x32A9: case 0x323B: case 0x32AB:
3718                                         continue;
3719                                 }
3720
3721                                 AddCharMapCJK ((char) w, ref category);
3722                         }
3723                 }
3724
3725                 // For now it is only for 0x7 category.
3726                 private void AddCharMapGroup2 (char c, byte category, byte updateCount, byte level2)
3727                 {
3728                         if (map [(int) c].Defined)
3729                                 return;
3730
3731                         bool updateWeight = false;
3732                         // Process in advance (lower primary weight)
3733                         for (int c2 = 0; c2 < char.MaxValue; c2++) {
3734                                 if (!map [c2].Defined &&
3735                                         decompLength [c2] == 1 &&
3736                                         (int) (decompValues [decompIndex [c2]]) == (int) c) {
3737                                         switch (decompType [c2]) {
3738                                         case DecompositionSmall:
3739                                                 updateWeight = true;
3740                                                 AddCharMap ((char) c2, category,
3741                                                         0, level2);
3742                                                 break;
3743                                         }
3744                                 }
3745                         }
3746                         if (updateWeight)
3747                                 fillIndex [category] = (byte)
3748                                         (fillIndex [category] + updateCount);
3749
3750                         // Identical weight
3751                         for (int c2 = 0; c2 < char.MaxValue; c2++) {
3752                                 if (!map [c2].Defined &&
3753                                         decompLength [c2] == 1 &&
3754                                         (int) (decompValues [decompIndex [c2]]) == (int) c) {
3755                                         switch (decompType [c2]) {
3756                                         case DecompositionSub:
3757                                         case DecompositionSuper:
3758                                         case DecompositionWide:
3759                                         case DecompositionNarrow:
3760                                                 AddCharMap ((char) c2, category,
3761                                                         0, level2);
3762                                                 break;
3763                                         }
3764                                 }
3765                         }
3766
3767                         // itself
3768                         AddCharMap (c, category, updateCount, level2);
3769
3770                         // Since nfkdMap is problematic to have two or more
3771                         // NFKD to an identical character, here I iterate all.
3772                         for (int c2 = 0; c2 < char.MaxValue; c2++) {
3773                                 if (!map [c2].Defined &&
3774                                         decompLength [c2] == 1 &&
3775                                         (int) (decompValues [decompIndex [c2]]) == (int) c) {
3776                                         switch (decompType [c2]) {
3777                                         case DecompositionWide:
3778                                         case DecompositionNarrow:
3779                                         case DecompositionSmall:
3780                                         case DecompositionSub:
3781                                         case DecompositionSuper:
3782                                                 continue;
3783                                         default:
3784                                                 AddCharMap ((char) c2, category, updateCount, level2);
3785                                                 break;
3786                                         }
3787                                 }
3788                         }
3789                 }
3790
3791                 private void AddArabicCharMap (char c, byte category, byte updateCount, byte level2)
3792                 {
3793                         // itself
3794                         AddCharMap (c, category, 0, level2);
3795
3796                         // Since nfkdMap is problematic to have two or more
3797                         // NFKD to an identical character, here I iterate all.
3798                         for (int c2 = 0; c2 < char.MaxValue; c2++) {
3799                                 if (decompLength [c2] == 0)
3800                                         continue;
3801                                 int idx = decompIndex [c2] + decompLength [c2] - 1;
3802                                 if ((int) (decompValues [idx]) == (int) c)
3803                                         AddCharMap ((char) c2, category,
3804                                                 0, level2);
3805                         }
3806                         fillIndex [category] += updateCount;
3807                 }
3808
3809                 char ToSmallForm (char c)
3810                 {
3811                         return ToDecomposed (c, DecompositionSmall, false);
3812                 }
3813
3814                 char ToDecomposed (char c, byte d, bool tail)
3815                 {
3816                         if (decompType [(int) c] != d)
3817                                 return c;
3818                         int idx = decompIndex [(int) c];
3819                         if (tail)
3820                                 idx += decompLength [(int) c] - 1;
3821                         return (char) decompValues [idx];
3822                 }
3823
3824                 bool ExistsJIS (int cp)
3825                 {
3826                         foreach (JISCharacter j in jisJapanese)
3827                                 if (j.CP == cp)
3828                                         return true;
3829                         return false;
3830                 }
3831
3832                 #endregion
3833
3834                 #region Level 3 properties (Case/Width)
3835
3836                 private byte ComputeLevel3Weight (char c)
3837                 {
3838                         byte b = ComputeLevel3WeightRaw (c);
3839                         return b > 0 ? (byte) (b + 2) : b;
3840                 }
3841
3842                 private byte ComputeLevel3WeightRaw (char c) // add 2 for sortkey value
3843                 {
3844                         // CJK compat
3845                         if ('\u3192' <= c && c <= '\u319F')
3846                                 return 0;
3847
3848                         // They have <narrow> NFKD mapping, and on Windows
3849                         // those narrow characters are regarded as "normal",
3850                         // thus those characters themselves are regarded as
3851                         // "wide". grep "<narrow>" and you can pick them up
3852                         // (ignoring Kana, Hangul etc.)
3853                         switch (c) {
3854                         case '\u3002':
3855                         case '\u300C':
3856                         case '\u300D':
3857                         case '\u3001':
3858                         case '\u30FB':
3859                         case '\u2502':
3860                         case '\u2190':
3861                         case '\u2191':
3862                         case '\u2192':
3863                         case '\u2193':
3864                         case '\u25A0':
3865                         case '\u25CB':
3866                                 return 1;
3867                         }
3868                         // Korean
3869                         if ('\u11A8' <= c && c <= '\u11F9')
3870                                 return 2;
3871                         if ('\uFFA0' <= c && c <= '\uFFDC')
3872                                 return 4;
3873                         if ('\u3130' <= c && c <= '\u3164')
3874                                 return 5;
3875                         if ('\u3165' <= c && c <= '\u318E')
3876                                 return 4;
3877                         // Georgian Capital letters
3878                         if ('\u10A0' <= c && c <= '\u10C5')
3879                                 return 0x10;
3880                         // numbers
3881                         if ('\u2776' <= c && c <= '\u277F')
3882                                 return 4;
3883                         if ('\u2780' <= c && c <= '\u2789')
3884                                 return 8;
3885                         if ('\u2776' <= c && c <= '\u2793')
3886                                 return 0xC;
3887                         if ('\u2160' <= c && c <= '\u216F')
3888                                 return 0x10;
3889                         if ('\u2181' <= c && c <= '\u2182')
3890                                 return 0x10;
3891                         // Arabic
3892                         if ('\u2135' <= c && c <= '\u2138')
3893                                 return 4;
3894                         // I believe that Windows has a bug on setting level 3
3895                         // weight here. NFKD results in different values.
3896                         if ('\uFE80' < c && c < '\uFF00') {
3897                                 // 2(Isolated)/8(Final)/0x18(Medial)
3898                                 switch (decompType [(int) c]) {
3899                                 case DecompositionIsolated:
3900                                         return 0; // 2;
3901                                 case DecompositionFinal:
3902                                         return 8;
3903                                 case DecompositionMedial:
3904                                         return 0x18;
3905                                 case DecompositionInitial:
3906                                         return 0x10;
3907                                 }
3908                         }
3909
3910                         // I have no idea why those symbols have level 3 weight
3911                         if (c == '\u2104' || c == '\u212B')
3912                                 return 0x18;
3913                         if ('\u211E' <= c && c <= '\u212B')
3914                                 return 0x10;
3915
3916                         // actually I dunno the reason why they have weights.
3917                         switch (c) {
3918                         case '\u01BC':
3919                                 return 0x10;
3920                         case '\u06A9':
3921                                 return 0x20;
3922                         case '\u06AA':
3923                                 return 0x28;
3924                         // Gurmukhi
3925                         case '\u0A39':
3926                         case '\u0A59':
3927                         case '\u0A5A':
3928                         case '\u0A5B':
3929                         case '\u0A5E':
3930                                 return 0x10;
3931                         }
3932
3933                         byte ret = 0;
3934                         switch (c) {
3935                         case '\u03C2':
3936                         case '\u212B':
3937                                 ret = 8;
3938                                 break;
3939                         case '\uFE42':
3940                                 ret = 0xA;
3941                                 break;
3942                         }
3943
3944                         // misc
3945                         switch (decompType [(int) c]) {
3946                         case DecompositionWide: // <wide>
3947                         case DecompositionSub: // <sub>
3948                         case DecompositionSuper: // <super>
3949                                 ret |= decompType [(int) c];
3950                                 break;
3951                         }
3952                         if (isSmallCapital [(int) c]) // grep "SMALL CAPITAL"
3953                                 ret |= 8;
3954                         if (isUppercase [(int) c]) // DerivedCoreProperties
3955                                 ret |= 0x10;
3956
3957                         return ret;
3958                 }
3959
3960                 #endregion
3961
3962                 #region IsIgnorable
3963 /*
3964                 static bool IsIgnorable (int i)
3965                 {
3966                         if (unicodeAge [i] >= 3.1)
3967                                 return true;
3968                         switch (char.GetUnicodeCategory ((char) i)) {
3969                         case UnicodeCategory.OtherNotAssigned:
3970                         case UnicodeCategory.Format:
3971                                 return true;
3972                         }
3973                         return false;
3974                 }
3975 */
3976
3977                 // FIXME: In the future use DerivedAge.txt to examine character
3978                 // versions and set those ones that have higher version than
3979                 // 1.0 as ignorable.
3980                 static bool IsIgnorable (int i)
3981                 {
3982                         switch (i) {
3983                         case 0:
3984                         // I guess, those characters are added between
3985                         // Unicode 1.0 (LCMapString) and Unicode 3.1
3986                         // (UnicodeCategory), so they used to be
3987                         // something like OtherNotAssigned as of Unicode 1.1.
3988                         case 0x2df: case 0x387:
3989                         case 0x3d7: case 0x3d8: case 0x3d9:
3990                         case 0x3f3: case 0x3f4: case 0x3f5: case 0x3f6:
3991                         case 0x400: case 0x40d: case 0x450: case 0x45d:
3992                         case 0x587: case 0x58a: case 0x5c4: case 0x640:
3993                         case 0x653: case 0x654: case 0x655: case 0x66d:
3994                         case 0xb56:
3995                         case 0x1e9b: case 0x202f: case 0x20ad:
3996                         case 0x20ae: case 0x20af:
3997                         case 0x20e2: case 0x20e3:
3998                         case 0x2139: case 0x213a: case 0x2183:
3999                         case 0x2425: case 0x2426: case 0x2619:
4000                         case 0x2670: case 0x2671: case 0x3007:
4001                         case 0x3190: case 0x3191:
4002                         case 0xfffc: case 0xfffd:
4003                                 return true;
4004                         // exceptional characters filtered by the
4005                         // following conditions. Originally those exceptional
4006                         // ranges are incorrect (they should not be ignored)
4007                         // and most of those characters are unfortunately in
4008                         // those ranges.
4009                         case 0x4d8: case 0x4d9:
4010                         case 0x4e8: case 0x4e9:
4011                         case 0x70F:
4012                         case 0x3036: case 0x303f:
4013                         case 0x337b: case 0xfb1e:
4014                                 return false;
4015                         }
4016
4017                         if (
4018                                 // The whole Sinhala characters.
4019                                 0x0D82 <= i && i <= 0x0DF4
4020                                 // The whole Tibetan characters.
4021                                 || 0x0F00 <= i && i <= 0x0FD1
4022                                 // The whole Myanmar characters.
4023                                 || 0x1000 <= i && i <= 0x1059
4024                                 // The whole Etiopic, Cherokee,
4025                                 // Canadian Syllablic, Ogham, Runic,
4026                                 // Tagalog, Hanunoo, Philippine,
4027                                 // Buhid, Tagbanwa, Khmer and Mongorian
4028                                 // characters.
4029                                 || 0x1200 <= i && i <= 0x1DFF
4030                                 // Greek extension characters.
4031                                 || 0x1F00 <= i && i <= 0x1FFF
4032                                 // The whole Braille characters.
4033                                 || 0x2800 <= i && i <= 0x28FF
4034                                 // CJK radical characters.
4035                                 || 0x2E80 <= i && i <= 0x2EF3
4036                                 // Kangxi radical characters.
4037                                 || 0x2F00 <= i && i <= 0x2FD5
4038                                 // Ideographic description characters.
4039                                 || 0x2FF0 <= i && i <= 0x2FFB
4040                                 // Bopomofo letter and final
4041                                 || 0x31A0 <= i && i <= 0x31B7
4042                                 // White square with quadrant characters.
4043                                 || 0x25F0 <= i && i <= 0x25F7
4044                                 // Ideographic telegraph symbols.
4045                                 || 0x32C0 <= i && i <= 0x32CB
4046                                 || 0x3358 <= i && i <= 0x3370
4047                                 || 0x33E0 <= i && i <= 0x33FF
4048                                 // The whole YI characters.
4049                                 || 0xA000 <= i && i <= 0xA48C
4050                                 || 0xA490 <= i && i <= 0xA4C6
4051                                 // American small ligatures
4052                                 || 0xFB13 <= i && i <= 0xFB17
4053                                 // hebrew, arabic, variation selector.
4054                                 || 0xFB1D <= i && i <= 0xFE2F
4055                                 // Arabic ligatures.
4056                                 || 0xFEF5 <= i && i <= 0xFEFC
4057                                 // FIXME: why are they excluded?
4058                                 || 0x01F6 <= i && i <= 0x01F9
4059                                 || 0x0218 <= i && i <= 0x0233
4060                                 || 0x02A9 <= i && i <= 0x02AD
4061                                 || 0x02EA <= i && i <= 0x02EE
4062                                 || 0x0349 <= i && i <= 0x036F
4063                                 || 0x0488 <= i && i <= 0x048F
4064                                 || 0x04D0 <= i && i <= 0x04FF
4065                                 || 0x0500 <= i && i <= 0x050F // actually it matters only for 2.0
4066                                 || 0x06D6 <= i && i <= 0x06ED
4067                                 || 0x06FA <= i && i <= 0x06FE
4068                                 || 0x2048 <= i && i <= 0x204D
4069                                 || 0x20e4 <= i && i <= 0x20ea
4070                                 || 0x213C <= i && i <= 0x214B
4071                                 || 0x21EB <= i && i <= 0x21FF
4072                                 || 0x22F2 <= i && i <= 0x22FF
4073                                 || 0x237B <= i && i <= 0x239A
4074                                 || 0x239B <= i && i <= 0x23CF
4075                                 || 0x24EB <= i && i <= 0x24FF
4076                                 || 0x2596 <= i && i <= 0x259F
4077                                 || 0x25F8 <= i && i <= 0x25FF
4078                                 || 0x2672 <= i && i <= 0x2689
4079                                 || 0x2768 <= i && i <= 0x2775
4080                                 || 0x27d0 <= i && i <= 0x27ff
4081                                 || 0x2900 <= i && i <= 0x2aff
4082                                 || 0x3033 <= i && i <= 0x303F
4083                                 || 0x31F0 <= i && i <= 0x31FF
4084                                 || 0x3250 <= i && i <= 0x325F
4085                                 || 0x32B1 <= i && i <= 0x32BF
4086                                 || 0x3371 <= i && i <= 0x337B
4087                                 || 0xFA30 <= i && i <= 0xFA6A
4088                         )
4089                                 return true;
4090
4091                         UnicodeCategory uc = Char.GetUnicodeCategory ((char) i);
4092                         switch (uc) {
4093                         case UnicodeCategory.PrivateUse:
4094                         case UnicodeCategory.Surrogate:
4095                                 return false;
4096                         // ignored by nature
4097                         case UnicodeCategory.Format:
4098                         case UnicodeCategory.OtherNotAssigned:
4099                                 return true;
4100                         default:
4101                                 return false;
4102                         }
4103                 }
4104
4105                 // To check IsIgnorable sanity, try the driver below under MS.NET.
4106
4107                 /*
4108                 public static void Main ()
4109                 {
4110                         for (int i = 0; i <= char.MaxValue; i++)
4111                                 Dump (i, IsIgnorable (i));
4112                 }
4113
4114                 static void Dump (int i, bool ignore)
4115                 {
4116                         switch (Char.GetUnicodeCategory ((char) i)) {
4117                         case UnicodeCategory.PrivateUse:
4118                         case UnicodeCategory.Surrogate:
4119                                 return; // check nothing
4120                         }
4121
4122                         string s1 = "";
4123                         string s2 = new string ((char) i, 10);
4124                         int ret = CultureInfo.InvariantCulture.CompareInfo.Compare (s1, s2, CompareOptions.IgnoreCase);
4125                         if ((ret == 0) == ignore)
4126                                 return;
4127                         Console.WriteLine ("{0} : {1:x} {2}", ignore ? "o" : "x", i, Char.GetUnicodeCategory ((char) i));
4128                 }
4129                 */
4130                 #endregion // IsIgnorable
4131
4132                 #region IsIgnorableSymbol
4133                 static bool IsIgnorableSymbol (int i)
4134                 {
4135                         if (IsIgnorable (i))
4136                                 return true;
4137
4138                         switch (i) {
4139                         // *Letter
4140                         case 0x00b5: case 0x01C0: case 0x01C1:
4141                         case 0x01C2: case 0x01C3: case 0x01F6:
4142                         case 0x01F7: case 0x01F8: case 0x01F9:
4143                         case 0x02D0: case 0x02EE: case 0x037A:
4144                         case 0x03D7: case 0x03F3:
4145                         case 0x0400: case 0x040d:
4146                         case 0x0450: case 0x045d:
4147                         case 0x048C: case 0x048D:
4148                         case 0x048E: case 0x048F:
4149                         case 0x0587: case 0x0640: case 0x06E5:
4150                         case 0x06E6: case 0x06FA: case 0x06FB:
4151                         case 0x06FC: case 0x093D: case 0x0950:
4152                         case 0x1E9B: case 0x2139: case 0x3006:
4153                         case 0x3033: case 0x3034: case 0x3035:
4154                         case 0xFE7E: case 0xFE7F:
4155                         // OtherNumber
4156                         case 0x16EE: case 0x16EF: case 0x16F0:
4157                         // LetterNumber
4158                         case 0x2183: // ROMAN NUMERAL REVERSED ONE HUNDRED
4159                         case 0x3007: // IDEOGRAPHIC NUMBER ZERO
4160                         case 0x3038: // HANGZHOU NUMERAL TEN
4161                         case 0x3039: // HANGZHOU NUMERAL TWENTY
4162                         case 0x303a: // HANGZHOU NUMERAL THIRTY
4163                         // OtherSymbol
4164                         case 0x2117:
4165                         case 0x327F:
4166                                 return true;
4167                         // ModifierSymbol
4168                         case 0x02B9: case 0x02BA: case 0x02C2:
4169                         case 0x02C3: case 0x02C4: case 0x02C5:
4170                         case 0x02C8: case 0x02CC: case 0x02CD:
4171                         case 0x02CE: case 0x02CF: case 0x02D2:
4172                         case 0x02D3: case 0x02D4: case 0x02D5:
4173                         case 0x02D6: case 0x02D7: case 0x02DE:
4174                         case 0x02E5: case 0x02E6: case 0x02E7:
4175                         case 0x02E8: case 0x02E9:
4176                         case 0x309B: case 0x309C:
4177                         // OtherPunctuation
4178                         case 0x055A: // American Apos
4179                         case 0x05C0: // Hebrew Punct
4180                         case 0x0E4F: // Thai FONGMAN
4181                         case 0x0E5A: // Thai ANGKHANKHU
4182                         case 0x0E5B: // Thai KHOMUT
4183                         // CurencySymbol
4184                         case 0x09F2: // Bengali Rupee Mark
4185                         case 0x09F3: // Bengali Rupee Sign
4186                         // MathSymbol
4187                         case 0x221e: // INF.
4188                         // OtherSymbol
4189                         case 0x0482:
4190                         case 0x09FA:
4191                         case 0x0B70:
4192                                 return false;
4193                         }
4194
4195                         // *Letter
4196                         if (0xFE70 <= i && i < 0xFE7C // ARABIC LIGATURES B
4197 #if NET_2_0
4198                                 || 0x0501 <= i && i <= 0x0510 // CYRILLIC KOMI
4199                                 || 0xFA30 <= i && i < 0xFA70 // CJK COMPAT
4200 #endif
4201                         )
4202                                 return true;
4203
4204                         UnicodeCategory uc = Char.GetUnicodeCategory ((char) i);
4205                         switch (uc) {
4206                         case UnicodeCategory.Surrogate:
4207                                 return false; // inconsistent
4208
4209                         case UnicodeCategory.SpacingCombiningMark:
4210                         case UnicodeCategory.EnclosingMark:
4211                         case UnicodeCategory.NonSpacingMark:
4212                         case UnicodeCategory.PrivateUse:
4213                                 // NonSpacingMark
4214                                 if (0x064B <= i && i <= 0x0652) // Arabic
4215                                         return true;
4216                                 return false;
4217
4218                         case UnicodeCategory.Format:
4219                         case UnicodeCategory.OtherNotAssigned:
4220                                 return true;
4221
4222                         default:
4223                                 bool use = false;
4224                                 // OtherSymbols
4225                                 if (
4226                                         // latin in a circle
4227                                         0x249A <= i && i <= 0x24E9
4228                                         || 0x2100 <= i && i <= 0x2132
4229                                         // Japanese
4230                                         || 0x3196 <= i && i <= 0x31A0
4231                                         // Korean
4232                                         || 0x3200 <= i && i <= 0x321C
4233                                         // Chinese/Japanese
4234                                         || 0x322A <= i && i <= 0x3243
4235                                         // CJK
4236                                         || 0x3260 <= i && i <= 0x32B0
4237                                         || 0x32D0 <= i && i <= 0x3357
4238                                         || 0x337B <= i && i <= 0x33DD
4239                                 )
4240                                         use = !Char.IsLetterOrDigit ((char) i);
4241                                 if (use)
4242                                         return false;
4243
4244                                 // This "Digit" rule is mystery.
4245                                 // It filters some symbols out.
4246                                 if (Char.IsLetterOrDigit ((char) i))
4247                                         return false;
4248                                 if (Char.IsNumber ((char) i))
4249                                         return false;
4250                                 if (Char.IsControl ((char) i)
4251                                         || Char.IsSeparator ((char) i)
4252                                         || Char.IsPunctuation ((char) i))
4253                                         return true;
4254                                 if (Char.IsSymbol ((char) i))
4255                                         return true;
4256
4257                                 // FIXME: should check more
4258                                 return false;
4259                         }
4260                 }
4261
4262                 // To check IsIgnorableSymbol sanity, try the driver below under MS.NET.
4263 /*
4264                 public static void Main ()
4265                 {
4266                         CompareInfo ci = CultureInfo.InvariantCulture.CompareInfo;
4267                         for (int i = 0; i <= char.MaxValue; i++) {
4268                                 UnicodeCategory uc = Char.GetUnicodeCategory ((char) i);
4269                                 if (uc == UnicodeCategory.Surrogate)
4270                                         continue;
4271
4272                                 bool ret = IsIgnorableSymbol (i);
4273
4274                                 string s1 = "TEST ";
4275                                 string s2 = "TEST " + (char) i;
4276
4277                                 int result = ci.Compare (s1, s2, CompareOptions.IgnoreSymbols);
4278
4279                                 if (ret != (result == 0))
4280                                         Console.WriteLine ("{0} : {1:x}[{2}]({3})",
4281                                                 ret ? "should not ignore" :
4282                                                         "should ignore",
4283                                                 i,(char) i, uc);
4284                         }
4285                 }
4286 */
4287                 #endregion
4288
4289                 #region NonSpacing
4290                 static bool IsIgnorableNonSpacing (int i)
4291                 {
4292                         if (IsIgnorable (i))
4293                                 return true;
4294
4295                         switch (i) {
4296                         case 0x02C8: case 0x02DE: case 0x0559: case 0x055A:
4297                         case 0x05C0: case 0x0ABD: case 0x0CD5: case 0x0CD6:
4298                         case 0x309B: case 0x309C: case 0xFF9E: case 0xFF9F:
4299                                 return true;
4300                         case 0x02D0: case 0x0670: case 0x0901: case 0x0902:
4301                         case 0x094D: case 0x0962: case 0x0963: case 0x0A41:
4302                         case 0x0A42: case 0x0A47: case 0x0A48: case 0x0A4B:
4303                         case 0x0A4C: case 0x0A81: case 0x0A82: case 0x0B82:
4304                         case 0x0BC0: case 0x0CBF: case 0x0CC6: case 0x0CCC:
4305                         case 0x0CCD: case 0x0E4E:
4306                                 return false;
4307                         }
4308
4309                         if (0x02b9 <= i && i <= 0x02c5
4310                                 || 0x02cc <= i && i <= 0x02d7
4311                                 || 0x02e4 <= i && i <= 0x02ef
4312                                 || 0x20DD <= i && i <= 0x20E0
4313                         )
4314                                 return true;
4315
4316                         if (0x064B <= i && i <= 0x00652
4317                                 || 0x0941 <= i && i <= 0x0948
4318                                 || 0x0AC1 <= i && i <= 0x0ACD
4319                                 || 0x0C3E <= i && i <= 0x0C4F
4320                                 || 0x0E31 <= i && i <= 0x0E3F
4321                         )
4322                                 return false;
4323
4324                         return Char.GetUnicodeCategory ((char) i) ==
4325                                 UnicodeCategory.NonSpacingMark;
4326                 }
4327
4328                 // We can reuse IsIgnorableSymbol testcode
4329                 // for IsIgnorableNonSpacing.
4330                 #endregion
4331         }
4332
4333         struct CharMapEntry
4334         {
4335                 public byte Category;
4336                 public byte Level1;
4337                 public byte Level2; // It is always single byte.
4338                 public bool Defined;
4339
4340                 public CharMapEntry (byte category, byte level1, byte level2)
4341                 {
4342                         Category = category;
4343                         Level1 = level1;
4344                         Level2 = level2;
4345                         Defined = true;
4346                 }
4347         }
4348
4349         class JISCharacter
4350         {
4351                 public readonly int CP;
4352                 public readonly int JIS;
4353
4354                 public JISCharacter (int cp, int cpJIS)
4355                 {
4356                         CP = cp;
4357                         JIS = cpJIS;
4358                 }
4359         }
4360
4361         class JISComparer : IComparer
4362         {
4363                 public static readonly JISComparer Instance =
4364                         new JISComparer ();
4365
4366                 public int Compare (object o1, object o2)
4367                 {
4368                         JISCharacter j1 = (JISCharacter) o1;
4369                         JISCharacter j2 = (JISCharacter) o2;
4370                         return j1.JIS - j2.JIS;
4371                 }
4372         }
4373
4374         class NonJISCharacter
4375         {
4376                 public readonly int CP;
4377                 public readonly string Name;
4378
4379                 public NonJISCharacter (int cp, string name)
4380                 {
4381                         CP = cp;
4382                         Name = name;
4383                 }
4384         }
4385
4386         class NonJISComparer : IComparer
4387         {
4388                 public static readonly NonJISComparer Instance =
4389                         new NonJISComparer ();
4390
4391                 public int Compare (object o1, object o2)
4392                 {
4393                         NonJISCharacter j1 = (NonJISCharacter) o1;
4394                         NonJISCharacter j2 = (NonJISCharacter) o2;
4395                         return string.CompareOrdinal (j1.Name, j2.Name);
4396                 }
4397         }
4398
4399         class DecimalDictionaryValueComparer : IComparer
4400         {
4401                 public static readonly DecimalDictionaryValueComparer Instance
4402                         = new DecimalDictionaryValueComparer ();
4403
4404                 private DecimalDictionaryValueComparer ()
4405                 {
4406                 }
4407
4408                 public int Compare (object o1, object o2)
4409                 {
4410                         DictionaryEntry e1 = (DictionaryEntry) o1;
4411                         DictionaryEntry e2 = (DictionaryEntry) o2;
4412                         // FIXME: in case of 0, compare decomposition categories
4413                         int ret = Decimal.Compare ((decimal) e1.Value, (decimal) e2.Value);
4414                         if (ret != 0)
4415                                 return ret;
4416                         int i1 = (int) e1.Key;
4417                         int i2 = (int) e2.Key;
4418                         return i1 - i2;
4419                 }
4420         }
4421
4422         class StringDictionaryValueComparer : IComparer
4423         {
4424                 public static readonly StringDictionaryValueComparer Instance
4425                         = new StringDictionaryValueComparer ();
4426
4427                 private StringDictionaryValueComparer ()
4428                 {
4429                 }
4430
4431                 public int Compare (object o1, object o2)
4432                 {
4433                         DictionaryEntry e1 = (DictionaryEntry) o1;
4434                         DictionaryEntry e2 = (DictionaryEntry) o2;
4435                         int ret = String.Compare ((string) e1.Value, (string) e2.Value);
4436                         if (ret != 0)
4437                                 return ret;
4438                         int i1 = (int) e1.Key;
4439                         int i2 = (int) e2.Key;
4440                         return i1 - i2;
4441                 }
4442         }
4443
4444         class UCAComparer : IComparer
4445         {
4446                 public static readonly UCAComparer Instance
4447                         = new UCAComparer ();
4448
4449                 private UCAComparer ()
4450                 {
4451                 }
4452
4453                 public int Compare (object o1, object o2)
4454                 {
4455                         char i1 = (char) o1;
4456                         char i2 = (char) o2;
4457
4458                         int l1 = CollationElementTable.GetSortKeyCount (i1);
4459                         int l2 = CollationElementTable.GetSortKeyCount (i2);
4460                         int l = l1 > l2 ? l2 : l1;
4461
4462                         for (int i = 0; i < l; i++) {
4463                                 SortKeyValue k1 = CollationElementTable.GetSortKey (i1, i);
4464                                 SortKeyValue k2 = CollationElementTable.GetSortKey (i2, i);
4465                                 int v = k1.Primary - k2.Primary;
4466                                 if (v != 0)
4467                                         return v;
4468                                 v = k1.Secondary - k2.Secondary;
4469                                 if (v != 0)
4470                                         return v;
4471                                 v = k1.Thirtiary - k2.Thirtiary;
4472                                 if (v != 0)
4473                                         return v;
4474                                 v = k1.Quarternary - k2.Quarternary;
4475                                 if (v != 0)
4476                                         return v;
4477                         }
4478                         return l1 - l2;
4479                 }
4480         }
4481
4482         class Tailoring
4483         {
4484                 int lcid;
4485                 int alias;
4486                 bool frenchSort;
4487                 ArrayList items = new ArrayList ();
4488
4489                 public Tailoring (int lcid)
4490                         : this (lcid, 0)
4491                 {
4492                 }
4493
4494                 public Tailoring (int lcid, int alias)
4495                 {
4496                         this.lcid = lcid;
4497                         this.alias = alias;
4498                 }
4499
4500                 public int LCID {
4501                         get { return lcid; }
4502                 }
4503
4504                 public int Alias {
4505                         get { return alias; }
4506                 }
4507
4508                 public bool FrenchSort {
4509                         get { return frenchSort; }
4510                         set { frenchSort = value; }
4511                 }
4512
4513                 public void AddDiacriticalMap (byte target, byte replace)
4514                 {
4515                         items.Add (new DiacriticalMap (target, replace));
4516                 }
4517
4518                 public void AddSortKeyMap (string source, byte [] sortkey)
4519                 {
4520                         items.Add (new SortKeyMap (source, sortkey));
4521                 }
4522
4523                 public void AddReplacementMap (string source, string replace)
4524                 {
4525                         items.Add (new ReplacementMap (source, replace));
4526                 }
4527
4528                 public char [] ItemToCharArray ()
4529                 {
4530                         ArrayList al = new ArrayList ();
4531                         foreach (ITailoringMap m in items)
4532                                 al.AddRange (m.ToCharArray ());
4533                         return al.ToArray (typeof (char)) as char [];
4534                 }
4535
4536                 interface ITailoringMap
4537                 {
4538                         char [] ToCharArray ();
4539                 }
4540
4541                 class DiacriticalMap : ITailoringMap
4542                 {
4543                         public readonly byte Target;
4544                         public readonly byte Replace;
4545
4546                         public DiacriticalMap (byte target, byte replace)
4547                         {
4548                                 Target = target;
4549                                 Replace = replace;
4550                         }
4551
4552                         public char [] ToCharArray ()
4553                         {
4554                                 char [] ret = new char [3];
4555                                 ret [0] = (char) 02; // kind:DiacriticalMap
4556                                 ret [1] = (char) Target;
4557                                 ret [2] = (char) Replace;
4558                                 return ret;
4559                         }
4560                 }
4561
4562                 class SortKeyMap : ITailoringMap
4563                 {
4564                         public readonly string Source;
4565                         public readonly byte [] SortKey;
4566
4567                         public SortKeyMap (string source, byte [] sortkey)
4568                         {
4569                                 Source = source;
4570                                 SortKey = sortkey;
4571                         }
4572
4573                         public char [] ToCharArray ()
4574                         {
4575                                 char [] ret = new char [Source.Length + 7];
4576                                 ret [0] = (char) 01; // kind:SortKeyMap
4577                                 for (int i = 0; i < Source.Length; i++)
4578                                         ret [i + 1] = Source [i];
4579                                 // null terminate
4580                                 for (int i = 0; i < 4; i++)
4581                                         ret [i + Source.Length + 2] = (char) SortKey [i];
4582                                 return ret;
4583                         }
4584                 }
4585
4586                 class ReplacementMap : ITailoringMap
4587                 {
4588                         public readonly string Source;
4589                         public readonly string Replace;
4590
4591                         public ReplacementMap (string source, string replace)
4592                         {
4593                                 Source = source;
4594                                 Replace = replace;
4595                         }
4596
4597                         public char [] ToCharArray ()
4598                         {
4599                                 char [] ret = new char [Source.Length + Replace.Length + 3];
4600                                 ret [0] = (char) 03; // kind:ReplaceMap
4601                                 int pos = 1;
4602                                 for (int i = 0; i < Source.Length; i++)
4603                                         ret [pos++] = Source [i];
4604                                 // null terminate
4605                                 pos++;
4606                                 for (int i = 0; i < Replace.Length; i++)
4607                                         ret [pos++] = Replace [i];
4608                                 // null terminate
4609                                 return ret;
4610                         }
4611                 }
4612         }
4613 }