mcs/class/I18N/CJK/Test/I18N.CJK.Test.cs

   1 //
   2 // I18N.CJK.Test.cs
   3 //
   4 // Author:
   5 //      Atsushi Enomoto  <atsushi@ximian.com>
   6 //
   7 // Copyright (C) 2005 Novell, Inc.  http://www.novell.com
   8 //
   9
  10 using System;
  11 using System.IO;
  12 using System.Text;
  13 using NUnit.Framework;
  14
  15 namespace MonoTests.I18N.CJK
  16 {
  17         [TestFixture]
  18         public class TestCJK
  19         {
  20                 private global::I18N.Common.Manager Manager = global::I18N.Common.Manager.PrimaryManager;
  21
  22                 void AssertEncode (string utf8file, string decfile, int codepage)
  23                 {
  24                         string decoded = null;
  25                         byte [] encoded = null;
  26                         using (StreamReader sr = new StreamReader (utf8file,
  27                                 Encoding.UTF8)) {
  28                                 decoded = sr.ReadToEnd ();
  29                         }
  30                         using (FileStream fs = File.OpenRead (decfile)) {
  31                                 encoded = new byte [fs.Length];
  32                                 fs.Read (encoded, 0, (int) fs.Length);
  33                         }
  34                         Encoding enc = Manager.GetEncoding (codepage);
  35                         byte [] actual;
  36
  37                         // simple string case
  38                         //Assert.AreEqual (encoded.Length,
  39                         //      enc.GetByteCount (decoded),
  40                         //      "GetByteCount(string)");
  41                         actual = enc.GetBytes (decoded);
  42                         Assert.AreEqual (encoded, actual,
  43                                 "GetBytes(string)");
  44
  45                         // simple char[] case
  46                         Assert.AreEqual (encoded.Length,
  47                                 enc.GetByteCount (decoded.ToCharArray (), 0, decoded.Length),
  48                                 "GetByteCount(char[], 0, len)");
  49                         actual = enc.GetBytes (decoded.ToCharArray (), 0, decoded.Length);
  50                         Assert.AreEqual (encoded, actual,
  51                                 "GetBytes(char[], 0, len)");
  52                 }
  53
  54                 void AssertDecode (string utf8file, string decfile, int codepage)
  55                 {
  56                         string decoded = null;
  57                         byte [] encoded = null;
  58                         using (StreamReader sr = new StreamReader (utf8file,
  59                                 Encoding.UTF8)) {
  60                                 decoded = sr.ReadToEnd ();
  61                         }
  62                         using (FileStream fs = File.OpenRead (decfile)) {
  63                                 encoded = new byte [fs.Length];
  64                                 fs.Read (encoded, 0, (int) fs.Length);
  65                         }
  66                         Encoding enc = Manager.GetEncoding (codepage);
  67                         char [] actual;
  68
  69                         Assert.AreEqual (decoded.Length,
  70                                 enc.GetCharCount (encoded, 0, encoded.Length),
  71                                 "GetCharCount(byte[], 0, len)");
  72                         actual = enc.GetChars (encoded, 0, encoded.Length);
  73                         Assert.AreEqual (decoded.ToCharArray (), actual,
  74                                 "GetChars(byte[], 0, len)");
  75                 }
  76
  77                 #region Chinese
  78
  79                 // GB2312
  80
  81                 [Test]
  82                 public void CP936_Encode ()
  83                 {
  84                         AssertEncode ("Test/texts/chinese-utf8.txt", "Test/texts/chinese-936.txt", 936);
  85                 }
  86
  87                 [Test]
  88                 public void CP936_Encode3 ()
  89                 {
  90                         AssertEncode("Test/texts/chinese3-utf8.txt", "Test/texts/chinese3-936.txt", 936);
  91                 }
  92
  93                 [Test]
  94                 public void CP936_Decode ()
  95                 {
  96                         AssertDecode ("Test/texts/chinese-utf8.txt", "Test/texts/chinese-936.txt", 936);
  97                 }
  98
  99                 [Test]
 100                 public void Bug_1531()
 101                 {
 102                         string str = @"wqk=";
 103                         byte[] utf8 = Convert.FromBase64String(str);
 104                         char[] data = Encoding.UTF8.GetChars(utf8);
 105
 106                         var encoding = Manager.GetEncoding("GB2312");
 107                         var result = encoding.GetBytes(data);
 108
 109                         Assert.AreEqual(new byte[] { 63 }, result);
 110                 }
 111
 112                 // BIG5
 113
 114                 [Test]
 115                 public void CP950_Encode ()
 116                 {
 117                         AssertEncode ("Test/texts/chinese2-utf8.txt", "Test/texts/chinese2-950.txt", 950);
 118                 }
 119
 120                 [Test]
 121                 public void CP950_Encode4 ()
 122                 {
 123                         AssertEncode("Test/texts/chinese4-utf8.txt", "Test/texts/chinese4-950.txt", 950);
 124                 }
 125
 126                 [Test]
 127                 public void CP950_Decode ()
 128                 {
 129                         AssertDecode ("Test/texts/chinese2-utf8.txt", "Test/texts/chinese2-950.txt", 950);
 130                 }
 131
 132                 // GB18030
 133
 134                 [Test]
 135                 public void CP54936_Encode ()
 136                 {
 137                         AssertEncode ("Test/texts/chinese-utf8.txt", "Test/texts/chinese-54936.txt", 54936);
 138                 }
 139
 140                 [Test]
 141                 public void CP54936_Decode ()
 142                 {
 143                         AssertDecode ("Test/texts/chinese-utf8.txt", "Test/texts/chinese-54936.txt", 54936);
 144                 }
 145
 146                 #endregion
 147
 148                 #region Japanese
 149
 150                 // Shift_JIS
 151
 152                 [Test]
 153                 public void CP932_Encode ()
 154                 {
 155                         AssertEncode ("Test/texts/japanese-utf8.txt", "Test/texts/japanese-932.txt", 932);
 156                 }
 157
 158                 [Test]
 159                 public void CP932_Decode ()
 160                 {
 161                         AssertDecode ("Test/texts/japanese-utf8.txt", "Test/texts/japanese-932.txt", 932);
 162                 }
 163
 164                 // EUC-JP
 165
 166                 [Test]
 167                 public void CP51932_Encode ()
 168                 {
 169                         AssertEncode ("Test/texts/japanese-utf8.txt", "Test/texts/japanese-51932.txt", 51932);
 170                 }
 171
 172                 [Test]
 173                 public void CP51932_Decode ()
 174                 {
 175                         AssertDecode ("Test/texts/japanese-utf8.txt", "Test/texts/japanese-51932.txt", 51932);
 176                 }
 177
 178                 // ISO-2022-JP
 179
 180                 [Test]
 181                 public void CP50220_Encode ()
 182                 {
 183                         AssertEncode ("Test/texts/japanese2-utf8.txt", "Test/texts/japanese2-50220.txt", 50220);
 184                 }
 185
 186                 [Test]
 187                 public void CP50220_Encode_3 ()
 188                 {
 189                         AssertEncode("Test/texts/japanese3-utf8.txt", "Test/texts/japanese3-50220.txt", 50220);
 190                 }
 191
 192                 [Test]
 193                 public void CP50220_Decode ()
 194                 {
 195                         AssertDecode ("Test/texts/japanese2-utf8.txt", "Test/texts/japanese2-50220.txt", 50220);
 196                 }
 197
 198                 [Test]
 199                 public void CP50221_Encode ()
 200                 {
 201                         AssertEncode ("Test/texts/japanese-utf8.txt", "Test/texts/japanese-50221.txt", 50221);
 202                 }
 203
 204                 [Test]
 205                 public void CP50221_Encode_3()
 206                 {
 207                         AssertEncode("Test/texts/japanese3-utf8.txt", "Test/texts/japanese3-50221.txt", 50221);
 208                 }
 209
 210                 [Test]
 211                 public void CP50221_Decode ()
 212                 {
 213                         AssertDecode ("Test/texts/japanese-utf8.txt", "Test/texts/japanese-50221.txt", 50221);
 214                 }
 215
 216                 [Test]
 217                 public void CP50222_Encode ()
 218                 {
 219                         AssertEncode ("Test/texts/japanese-utf8.txt", "Test/texts/japanese-50222.txt", 50222);
 220                 }
 221
 222                 [Test]
 223                 public void CP50222_Decode ()
 224                 {
 225                         AssertDecode ("Test/texts/japanese-utf8.txt", "Test/texts/japanese-50222.txt", 50222);
 226                 }
 227
 228                 [Test]
 229                 public void CP50220BrokenESC ()
 230                 {
 231                         Assert.AreEqual ("\u001B$0", Manager.GetEncoding (50220).GetString (new byte [] {0x1B, 0x24, 0x30}), "#1");
 232                 }
 233
 234                 [Test]
 235                 public void CP50220BrokenESC2 ()
 236                 {
 237                         // it does not really invoke fallback ...
 238                         Assert.AreEqual ("\u001B$0", Encoding.GetEncoding (50220, new EncoderReplacementFallback (), new DecoderReplacementFallback ("")).GetString (new byte [] {0x1B, 0x24, 0x30}), "#1");
 239                 }
 240
 241                 [Test]
 242                 public void CP50220BrokenESC3 ()
 243                 {
 244                         // neither ...
 245                         Assert.AreEqual ("\u001B$0", Encoding.GetEncoding (50220, new EncoderExceptionFallback (), new DecoderExceptionFallback ()).GetString (new byte [] {0x1B, 0x24, 0x30}), "#2");
 246                 }
 247
 248                 [Test]
 249                 public void Bug77723 ()
 250                 {
 251                         GetBytesAllSingleChars (51932);
 252                 }
 253
 254                 [Test]
 255                 public void Bug77724 ()
 256                 {
 257                         GetBytesAllSingleChars (932);
 258                 }
 259
 260                 [Test]
 261                 public void Bug77307 ()
 262                 {
 263                         GetBytesAllSingleChars (54936);
 264                 }
 265
 266                 void GetBytesAllSingleChars (int enc)
 267                 {
 268                         Encoding e = Manager.GetEncoding (enc);
 269                         for (int i = 0; i < 0x10000; i++)
 270                                 e.GetBytes (new char [] { (char)i });
 271                 }
 272
 273                 void GetCharsAllBytePairs (int enc)
 274                 {
 275                         Encoding e = Manager.GetEncoding (enc);
 276                         byte [] bytes = new byte [2];
 277                         for (int i0 = 0; i0 < 0x100; i0++) {
 278                                 bytes [0] = (byte) i0;
 279                                 for (int i1 = 0; i1 < 0x100; i1++) {
 280                                         bytes [1] = (byte) i1;
 281                                         e.GetChars (bytes);
 282                                 }
 283                         }
 284                 }
 285
 286                 [Test]
 287                 public void Bug77222 ()
 288                 {
 289                         GetCharsAllBytePairs (51932);
 290                 }
 291
 292                 [Test]
 293                 public void Bug77238 ()
 294                 {
 295                         GetCharsAllBytePairs (936);
 296                 }
 297
 298                 [Test]
 299                 public void Bug77306 ()
 300                 {
 301                         GetCharsAllBytePairs (54936);
 302                 }
 303
 304                 [Test]
 305                 public void Bug77298 ()
 306                 {
 307                         GetCharsAllBytePairs (949);
 308                 }
 309
 310                 [Test]
 311                 public void Bug77274 ()
 312                 {
 313                         GetCharsAllBytePairs (950);
 314                 }
 315
 316                 [Test]
 317                 public void Encoder54936Refresh ()
 318                 {
 319                         Encoding e = Manager.GetEncoding ("gb18030");
 320                         Encoder d = e.GetEncoder ();
 321                         byte [] bytes;
 322
 323                         bytes = new byte [4];
 324                         Assert.AreEqual (0, d.GetBytes (new char [] {'\uD800'}, 0, 1, bytes, 0, false), "#1");
 325                         Assert.AreEqual (new byte [] {00, 00, 00, 00},
 326                                 bytes, "#2");
 327
 328                         bytes = new byte [4];
 329                         Assert.AreEqual (4, d.GetBytes (new char [] {'\uDC00'}, 0, 1, bytes, 0, true), "#3");
 330                         Assert.AreEqual (new byte [] {0x90, 0x30, 0x81, 0x30},
 331                                 bytes, "#4");
 332
 333                         bytes = new byte [4];
 334                         Assert.AreEqual (1, d.GetBytes (new char [] {'\uD800'}, 0, 1, bytes, 0, true), "#5");
 335                         Assert.AreEqual (new byte [] {0x3F, 00, 00, 00},
 336                                 bytes, "#6");
 337                 }
 338
 339                 [Test]
 340                 public void Bug491799 ()
 341                 {
 342                         Assert.AreEqual (new byte [] {0xEE, 0xFC},
 343                                            Manager.GetEncoding (932).GetBytes ("\uFF02"));
 344                 }
 345
 346                 [Test]
 347                 public void Decoder932Refresh ()
 348                 {
 349                         Encoding e = Manager.GetEncoding (932);
 350                         Decoder d = e.GetDecoder ();
 351                         char [] chars;
 352
 353                         chars = new char [1];
 354                         Assert.AreEqual (0, d.GetChars (new byte [] {0x81}, 0, 1, chars, 0, false), "#1");
 355                         Assert.AreEqual (new char [] {'\0'}, chars, "#2");
 356
 357                         chars = new char [1];
 358                         Assert.AreEqual (1, d.GetChars (new byte [] {0x81}, 0, 1, chars, 0, true), "#3");
 359                         Assert.AreEqual (new char [] {'\uFF1D'}, chars, "#4");
 360
 361                         chars = new char [1];
 362                         Assert.AreEqual (1, d.GetChars (new byte [] {0x81}, 0, 1, chars, 0, true), "#5");
 363                         Assert.AreEqual (new char [] {'\u30FB'}, chars, "#6");
 364                 }
 365
 366                 [Test]
 367                 public void Decoder51932Refresh ()
 368                 {
 369                         Encoding e = Manager.GetEncoding (51932);
 370                         Decoder d = e.GetDecoder ();
 371                         char [] chars;
 372
 373                         // invalid one
 374                         chars = new char [1];
 375                         Assert.AreEqual (1, d.GetChars (new byte [] {0x81}, 0, 1, chars, 0, false), "#0.1");
 376                         Assert.AreEqual (new char [] {'\u30FB'}, chars, "#0.2");
 377
 378                         // incomplete
 379                         chars = new char [1];
 380                         Assert.AreEqual (0, d.GetChars (new byte [] {0xA1}, 0, 1, chars, 0, false), "#1");
 381                         Assert.AreEqual (new char [] {'\0'}, chars, "#2");
 382
 383                         // became complete
 384                         chars = new char [1];
 385                         Assert.AreEqual (1, d.GetChars (new byte [] {0xA1}, 0, 1, chars, 0, true), "#3");
 386                         Assert.AreEqual (new char [] {'\u3000'}, chars, "#4");
 387
 388                         // incomplete but refreshed
 389                         chars = new char [1];
 390                         Assert.AreEqual (1, d.GetChars (new byte [] {0xA1}, 0, 1, chars, 0, true), "#5");
 391                         Assert.AreEqual (new char [] {'\u30FB'}, chars, "#6");
 392                 }
 393
 394                 [Test]
 395                 public void Decoder936Refresh ()
 396                 {
 397                         Encoding e = Manager.GetEncoding (936);
 398                         Decoder d = e.GetDecoder ();
 399                         char [] chars;
 400
 401                         // incomplete
 402                         chars = new char [1];
 403                         Assert.AreEqual (0, d.GetChars (new byte [] {0xB0}, 0, 1, chars, 0, false), "#1");
 404                         Assert.AreEqual (new char [] {'\0'}, chars, "#2");
 405
 406                         // became complete
 407                         chars = new char [1];
 408                         Assert.AreEqual (1, d.GetChars (new byte [] {0xA1}, 0, 1, chars, 0, false), "#3");
 409                         Assert.AreEqual (new char [] {'\u554A'}, chars, "#4");
 410
 411                         // incomplete but refreshed
 412                         chars = new char [1];
 413                         Assert.AreEqual (1, d.GetChars (new byte [] {0xB0}, 0, 1, chars, 0, true), "#5");
 414                         Assert.AreEqual (new char [] {'?'}, chars, "#6");
 415                 }
 416
 417                 [Test]
 418                 public void Decoder949Refresh ()
 419                 {
 420                         Encoding e = Manager.GetEncoding (949);
 421                         Decoder d = e.GetDecoder ();
 422                         char [] chars;
 423
 424                         // incomplete
 425                         chars = new char [1];
 426                         Assert.AreEqual (0, d.GetChars (new byte [] {0x81}, 0, 1, chars, 0, false), "#1");
 427                         Assert.AreEqual (new char [] {'\0'}, chars, "#2");
 428
 429                         // became complete
 430                         chars = new char [1];
 431                         Assert.AreEqual (1, d.GetChars (new byte [] {0x41}, 0, 1, chars, 0, false), "#3");
 432                         Assert.AreEqual (new char [] {'\uAC02'}, chars, "#4");
 433
 434                         // incomplete but refreshed
 435                         chars = new char [1];
 436                         Assert.AreEqual (1, d.GetChars (new byte [] {0x81}, 0, 1, chars, 0, true), "#5");
 437                         Assert.AreEqual (new char [] {'?'}, chars, "#6");
 438                 }
 439
 440                 [Test]
 441                 public void Decoder950Refresh ()
 442                 {
 443                         Encoding e = Manager.GetEncoding (950);
 444                         Decoder d = e.GetDecoder ();
 445                         char [] chars;
 446
 447                         // incomplete
 448                         chars = new char [1];
 449                         Assert.AreEqual (0, d.GetChars (new byte [] {0xF9}, 0, 1, chars, 0, false), "#1");
 450                         Assert.AreEqual (new char [] {'\0'}, chars, "#2");
 451
 452                         // became complete
 453                         chars = new char [1];
 454                         Assert.AreEqual (1, d.GetChars (new byte [] {0x40}, 0, 1, chars, 0, false), "#3");
 455                         Assert.AreEqual (new char [] {'\u7E98'}, chars, "#4");
 456
 457                         // incomplete but refreshed
 458                         chars = new char [1];
 459                         Assert.AreEqual (1, d.GetChars (new byte [] {0xF9}, 0, 1, chars, 0, true), "#5");
 460                         Assert.AreEqual (new char [] {'?'}, chars, "#6");
 461                 }
 462
 463
 464                 [Test]
 465                 public void Decoder51932NoRefresh ()
 466                 {
 467                         Encoding e = Manager.GetEncoding (51932);
 468                         Decoder d = e.GetDecoder ();
 469                         char [] chars;
 470
 471                         // incomplete
 472                         chars = new char [1];
 473                         Assert.AreEqual (0, d.GetChars (new byte [] {0xA1}, 0, 1, chars, 0), "#1");
 474                         Assert.AreEqual (new char [] {'\0'}, chars, "#2");
 475
 476                         // became complete
 477                         chars = new char [1];
 478                         Assert.AreEqual (1, d.GetChars (new byte [] {0xA1}, 0, 1, chars, 0), "#3");
 479                         Assert.AreEqual (new char [] {'\u3000'}, chars, "#4");
 480
 481                         // incomplete but refreshed
 482                         chars = new char [1];
 483                         Assert.AreEqual (0, d.GetChars (new byte [] {0xA1}, 0, 1, chars, 0), "#5");
 484                         Assert.AreEqual (new char [] {'\0'}, chars, "#6");
 485                 }
 486
 487                 [Test]
 488                 public void Decoder936NoRefresh ()
 489                 {
 490                         Encoding e = Manager.GetEncoding (936);
 491                         Decoder d = e.GetDecoder ();
 492                         char [] chars;
 493
 494                         // incomplete
 495                         chars = new char [1];
 496                         Assert.AreEqual (0, d.GetChars (new byte [] {0xB0}, 0, 1, chars, 0), "#1");
 497                         Assert.AreEqual (new char [] {'\0'}, chars, "#2");
 498
 499                         // became complete
 500                         chars = new char [1];
 501                         Assert.AreEqual (1, d.GetChars (new byte [] {0xA1}, 0, 1, chars, 0), "#3");
 502                         Assert.AreEqual (new char [] {'\u554A'}, chars, "#4");
 503
 504                         // incomplete but refreshed
 505                         chars = new char [1];
 506                         Assert.AreEqual (0, d.GetChars (new byte [] {0xB0}, 0, 1, chars, 0), "#5");
 507                         Assert.AreEqual (new char [] {'\0'}, chars, "#6");
 508                 }
 509
 510                 [Test]
 511                 public void Decoder949NoRefresh ()
 512                 {
 513                         Encoding e = Manager.GetEncoding (949);
 514                         Decoder d = e.GetDecoder ();
 515                         char [] chars;
 516
 517                         // incomplete
 518                         chars = new char [1];
 519                         Assert.AreEqual (0, d.GetChars (new byte [] {0x81}, 0, 1, chars, 0), "#1");
 520                         Assert.AreEqual (new char [] {'\0'}, chars, "#2");
 521
 522                         // became complete
 523                         chars = new char [1];
 524                         Assert.AreEqual (1, d.GetChars (new byte [] {0x41}, 0, 1, chars, 0), "#3");
 525                         Assert.AreEqual (new char [] {'\uAC02'}, chars, "#4");
 526
 527                         // incomplete but refreshed
 528                         chars = new char [1];
 529                         Assert.AreEqual (0, d.GetChars (new byte [] {0x81}, 0, 1, chars, 0), "#5");
 530                         Assert.AreEqual (new char [] {'\0'}, chars, "#6");
 531                 }
 532
 533                 [Test]
 534                 public void Decoder950NoRefresh ()
 535                 {
 536                         Encoding e = Manager.GetEncoding (950);
 537                         Decoder d = e.GetDecoder ();
 538                         char [] chars;
 539
 540                         // incomplete
 541                         chars = new char [1];
 542                         Assert.AreEqual (0, d.GetChars (new byte [] {0xF9}, 0, 1, chars, 0), "#1");
 543                         Assert.AreEqual (new char [] {'\0'}, chars, "#2");
 544
 545                         // became complete
 546                         chars = new char [1];
 547                         Assert.AreEqual (1, d.GetChars (new byte [] {0x40}, 0, 1, chars, 0), "#3");
 548                         Assert.AreEqual (new char [] {'\u7E98'}, chars, "#4");
 549
 550                         // incomplete but refreshed
 551                         chars = new char [1];
 552                         Assert.AreEqual (0, d.GetChars (new byte [] {0xF9}, 0, 1, chars, 0), "#5");
 553                         Assert.AreEqual (new char [] {'\0'}, chars, "#6");
 554                 }
 555
 556                 [Test]
 557                 public void HandleObsoletedESCJ () // bug #398273
 558                 {
 559                         byte [] b = new byte [] {0x64, 0x6f, 0x6e, 0x1b, 0x24, 0x42, 0x21, 0x47, 0x1b, 0x28, 0x4a, 0x74};
 560                         string s = Manager.GetEncoding ("ISO-2022-JP").GetString (b);
 561                         Assert.AreEqual ("don\u2019t", s);
 562
 563                 }
 564
 565                 [Test]
 566                 public void Bug14591 ()
 567                 {
 568                         var expected = "\u4f50\u85e4\u8c4a";
 569                         var text = Encoding.GetEncoding ("iso-2022-jp").GetString (Convert.FromBase64String ("GyRAOjRGI0stGyhK"));
 570                         Assert.AreEqual (expected, text, "#1");
 571                 }
 572                 #endregion
 573
 574                 #region Korean
 575
 576                 [Test]
 577                 public void CP949_Encode ()
 578                 {
 579                         AssertEncode ("Test/texts/korean-utf8.txt", "Test/texts/korean-949.txt", 949);
 580                 }
 581
 582                 [Test]
 583                 public void CP949_Decode ()
 584                 {
 585                         AssertDecode ("Test/texts/korean-utf8.txt", "Test/texts/korean-949.txt", 949);
 586                 }
 587
 588                 #endregion
 589         }
 590 }