5 // Atsushi Enomoto <atsushi@ximian.com>
7 // Copyright (C) 2005 Novell, Inc. http://www.novell.com
13 using NUnit.Framework;
15 namespace MonoTests.I18N.CJK
20 private global::I18N.Common.Manager Manager = global::I18N.Common.Manager.PrimaryManager;
22 void AssertEncode (string utf8file, string decfile, int codepage)
24 string decoded = null;
25 byte [] encoded = null;
26 using (StreamReader sr = new StreamReader (utf8file,
28 decoded = sr.ReadToEnd ();
30 using (FileStream fs = File.OpenRead (decfile)) {
31 encoded = new byte [fs.Length];
32 fs.Read (encoded, 0, (int) fs.Length);
34 Encoding enc = Manager.GetEncoding (codepage);
38 //Assert.AreEqual (encoded.Length,
39 // enc.GetByteCount (decoded),
40 // "GetByteCount(string)");
41 actual = enc.GetBytes (decoded);
42 Assert.AreEqual (encoded, actual,
46 Assert.AreEqual (encoded.Length,
47 enc.GetByteCount (decoded.ToCharArray (), 0, decoded.Length),
48 "GetByteCount(char[], 0, len)");
49 actual = enc.GetBytes (decoded.ToCharArray (), 0, decoded.Length);
50 Assert.AreEqual (encoded, actual,
51 "GetBytes(char[], 0, len)");
54 void AssertDecode (string utf8file, string decfile, int codepage)
56 string decoded = null;
57 byte [] encoded = null;
58 using (StreamReader sr = new StreamReader (utf8file,
60 decoded = sr.ReadToEnd ();
62 using (FileStream fs = File.OpenRead (decfile)) {
63 encoded = new byte [fs.Length];
64 fs.Read (encoded, 0, (int) fs.Length);
66 Encoding enc = Manager.GetEncoding (codepage);
69 Assert.AreEqual (decoded.Length,
70 enc.GetCharCount (encoded, 0, encoded.Length),
71 "GetCharCount(byte[], 0, len)");
72 actual = enc.GetChars (encoded, 0, encoded.Length);
73 Assert.AreEqual (decoded.ToCharArray (), actual,
74 "GetChars(byte[], 0, len)");
82 public void CP936_Encode ()
84 AssertEncode ("Test/texts/chinese-utf8.txt", "Test/texts/chinese-936.txt", 936);
88 public void CP936_Encode3 ()
90 AssertEncode("Test/texts/chinese3-utf8.txt", "Test/texts/chinese3-936.txt", 936);
94 public void CP936_Decode ()
96 AssertDecode ("Test/texts/chinese-utf8.txt", "Test/texts/chinese-936.txt", 936);
100 public void Bug_1531()
102 string str = @"wqk=";
103 byte[] utf8 = Convert.FromBase64String(str);
104 char[] data = Encoding.UTF8.GetChars(utf8);
106 var encoding = Manager.GetEncoding("GB2312");
107 var result = encoding.GetBytes(data);
109 Assert.AreEqual(new byte[] { 63 }, result);
115 public void CP950_Encode ()
117 AssertEncode ("Test/texts/chinese2-utf8.txt", "Test/texts/chinese2-950.txt", 950);
121 public void CP950_Encode4 ()
123 AssertEncode("Test/texts/chinese4-utf8.txt", "Test/texts/chinese4-950.txt", 950);
127 public void CP950_Decode ()
129 AssertDecode ("Test/texts/chinese2-utf8.txt", "Test/texts/chinese2-950.txt", 950);
135 public void CP54936_Encode ()
137 AssertEncode ("Test/texts/chinese-utf8.txt", "Test/texts/chinese-54936.txt", 54936);
141 public void CP54936_Decode ()
143 AssertDecode ("Test/texts/chinese-utf8.txt", "Test/texts/chinese-54936.txt", 54936);
153 public void CP932_Encode ()
155 AssertEncode ("Test/texts/japanese-utf8.txt", "Test/texts/japanese-932.txt", 932);
159 public void CP932_Decode ()
161 AssertDecode ("Test/texts/japanese-utf8.txt", "Test/texts/japanese-932.txt", 932);
167 public void CP51932_Encode ()
169 AssertEncode ("Test/texts/japanese-utf8.txt", "Test/texts/japanese-51932.txt", 51932);
173 public void CP51932_Decode ()
175 AssertDecode ("Test/texts/japanese-utf8.txt", "Test/texts/japanese-51932.txt", 51932);
181 public void CP50220_Encode ()
183 AssertEncode ("Test/texts/japanese2-utf8.txt", "Test/texts/japanese2-50220.txt", 50220);
187 public void CP50220_Encode_3 ()
189 AssertEncode("Test/texts/japanese3-utf8.txt", "Test/texts/japanese3-50220.txt", 50220);
193 public void CP50220_Decode ()
195 AssertDecode ("Test/texts/japanese2-utf8.txt", "Test/texts/japanese2-50220.txt", 50220);
199 public void CP50221_Encode ()
201 AssertEncode ("Test/texts/japanese-utf8.txt", "Test/texts/japanese-50221.txt", 50221);
205 public void CP50221_Encode_3()
207 AssertEncode("Test/texts/japanese3-utf8.txt", "Test/texts/japanese3-50221.txt", 50221);
211 public void CP50221_Decode ()
213 AssertDecode ("Test/texts/japanese-utf8.txt", "Test/texts/japanese-50221.txt", 50221);
217 public void CP50222_Encode ()
219 AssertEncode ("Test/texts/japanese-utf8.txt", "Test/texts/japanese-50222.txt", 50222);
223 public void CP50222_Decode ()
225 AssertDecode ("Test/texts/japanese-utf8.txt", "Test/texts/japanese-50222.txt", 50222);
229 public void CP50220BrokenESC ()
231 Assert.AreEqual ("\u001B$0", Manager.GetEncoding (50220).GetString (new byte [] {0x1B, 0x24, 0x30}), "#1");
235 public void CP50220BrokenESC2 ()
237 // it does not really invoke fallback ...
238 Assert.AreEqual ("\u001B$0", Encoding.GetEncoding (50220, new EncoderReplacementFallback (), new DecoderReplacementFallback ("")).GetString (new byte [] {0x1B, 0x24, 0x30}), "#1");
242 public void CP50220BrokenESC3 ()
245 Assert.AreEqual ("\u001B$0", Encoding.GetEncoding (50220, new EncoderExceptionFallback (), new DecoderExceptionFallback ()).GetString (new byte [] {0x1B, 0x24, 0x30}), "#2");
249 public void Bug77723 ()
251 GetBytesAllSingleChars (51932);
255 public void Bug77724 ()
257 GetBytesAllSingleChars (932);
261 public void Bug77307 ()
263 GetBytesAllSingleChars (54936);
266 void GetBytesAllSingleChars (int enc)
268 Encoding e = Manager.GetEncoding (enc);
269 for (int i = 0; i < 0x10000; i++)
270 e.GetBytes (new char [] { (char)i });
273 void GetCharsAllBytePairs (int enc)
275 Encoding e = Manager.GetEncoding (enc);
276 byte [] bytes = new byte [2];
277 for (int i0 = 0; i0 < 0x100; i0++) {
278 bytes [0] = (byte) i0;
279 for (int i1 = 0; i1 < 0x100; i1++) {
280 bytes [1] = (byte) i1;
287 public void Bug77222 ()
289 GetCharsAllBytePairs (51932);
293 public void Bug77238 ()
295 GetCharsAllBytePairs (936);
299 public void Bug77306 ()
301 GetCharsAllBytePairs (54936);
305 public void Bug77298 ()
307 GetCharsAllBytePairs (949);
311 public void Bug77274 ()
313 GetCharsAllBytePairs (950);
317 public void Encoder54936Refresh ()
319 Encoding e = Manager.GetEncoding ("gb18030");
320 Encoder d = e.GetEncoder ();
323 bytes = new byte [4];
324 Assert.AreEqual (0, d.GetBytes (new char [] {'\uD800'}, 0, 1, bytes, 0, false), "#1");
325 Assert.AreEqual (new byte [] {00, 00, 00, 00},
328 bytes = new byte [4];
329 Assert.AreEqual (4, d.GetBytes (new char [] {'\uDC00'}, 0, 1, bytes, 0, true), "#3");
330 Assert.AreEqual (new byte [] {0x90, 0x30, 0x81, 0x30},
333 bytes = new byte [4];
334 Assert.AreEqual (1, d.GetBytes (new char [] {'\uD800'}, 0, 1, bytes, 0, true), "#5");
335 Assert.AreEqual (new byte [] {0x3F, 00, 00, 00},
340 public void Bug491799 ()
342 Assert.AreEqual (new byte [] {0xEE, 0xFC},
343 Manager.GetEncoding (932).GetBytes ("\uFF02"));
347 public void Decoder932Refresh ()
349 Encoding e = Manager.GetEncoding (932);
350 Decoder d = e.GetDecoder ();
353 chars = new char [1];
354 Assert.AreEqual (0, d.GetChars (new byte [] {0x81}, 0, 1, chars, 0, false), "#1");
355 Assert.AreEqual (new char [] {'\0'}, chars, "#2");
357 chars = new char [1];
358 Assert.AreEqual (1, d.GetChars (new byte [] {0x81}, 0, 1, chars, 0, true), "#3");
359 Assert.AreEqual (new char [] {'\uFF1D'}, chars, "#4");
361 chars = new char [1];
362 Assert.AreEqual (1, d.GetChars (new byte [] {0x81}, 0, 1, chars, 0, true), "#5");
363 Assert.AreEqual (new char [] {'\u30FB'}, chars, "#6");
367 public void Decoder51932Refresh ()
369 Encoding e = Manager.GetEncoding (51932);
370 Decoder d = e.GetDecoder ();
374 chars = new char [1];
375 Assert.AreEqual (1, d.GetChars (new byte [] {0x81}, 0, 1, chars, 0, false), "#0.1");
376 Assert.AreEqual (new char [] {'\u30FB'}, chars, "#0.2");
379 chars = new char [1];
380 Assert.AreEqual (0, d.GetChars (new byte [] {0xA1}, 0, 1, chars, 0, false), "#1");
381 Assert.AreEqual (new char [] {'\0'}, chars, "#2");
384 chars = new char [1];
385 Assert.AreEqual (1, d.GetChars (new byte [] {0xA1}, 0, 1, chars, 0, true), "#3");
386 Assert.AreEqual (new char [] {'\u3000'}, chars, "#4");
388 // incomplete but refreshed
389 chars = new char [1];
390 Assert.AreEqual (1, d.GetChars (new byte [] {0xA1}, 0, 1, chars, 0, true), "#5");
391 Assert.AreEqual (new char [] {'\u30FB'}, chars, "#6");
395 public void Decoder936Refresh ()
397 Encoding e = Manager.GetEncoding (936);
398 Decoder d = e.GetDecoder ();
402 chars = new char [1];
403 Assert.AreEqual (0, d.GetChars (new byte [] {0xB0}, 0, 1, chars, 0, false), "#1");
404 Assert.AreEqual (new char [] {'\0'}, chars, "#2");
407 chars = new char [1];
408 Assert.AreEqual (1, d.GetChars (new byte [] {0xA1}, 0, 1, chars, 0, false), "#3");
409 Assert.AreEqual (new char [] {'\u554A'}, chars, "#4");
411 // incomplete but refreshed
412 chars = new char [1];
413 Assert.AreEqual (1, d.GetChars (new byte [] {0xB0}, 0, 1, chars, 0, true), "#5");
414 Assert.AreEqual (new char [] {'?'}, chars, "#6");
418 public void Decoder949Refresh ()
420 Encoding e = Manager.GetEncoding (949);
421 Decoder d = e.GetDecoder ();
425 chars = new char [1];
426 Assert.AreEqual (0, d.GetChars (new byte [] {0x81}, 0, 1, chars, 0, false), "#1");
427 Assert.AreEqual (new char [] {'\0'}, chars, "#2");
430 chars = new char [1];
431 Assert.AreEqual (1, d.GetChars (new byte [] {0x41}, 0, 1, chars, 0, false), "#3");
432 Assert.AreEqual (new char [] {'\uAC02'}, chars, "#4");
434 // incomplete but refreshed
435 chars = new char [1];
436 Assert.AreEqual (1, d.GetChars (new byte [] {0x81}, 0, 1, chars, 0, true), "#5");
437 Assert.AreEqual (new char [] {'?'}, chars, "#6");
441 public void Decoder950Refresh ()
443 Encoding e = Manager.GetEncoding (950);
444 Decoder d = e.GetDecoder ();
448 chars = new char [1];
449 Assert.AreEqual (0, d.GetChars (new byte [] {0xF9}, 0, 1, chars, 0, false), "#1");
450 Assert.AreEqual (new char [] {'\0'}, chars, "#2");
453 chars = new char [1];
454 Assert.AreEqual (1, d.GetChars (new byte [] {0x40}, 0, 1, chars, 0, false), "#3");
455 Assert.AreEqual (new char [] {'\u7E98'}, chars, "#4");
457 // incomplete but refreshed
458 chars = new char [1];
459 Assert.AreEqual (1, d.GetChars (new byte [] {0xF9}, 0, 1, chars, 0, true), "#5");
460 Assert.AreEqual (new char [] {'?'}, chars, "#6");
465 public void Decoder51932NoRefresh ()
467 Encoding e = Manager.GetEncoding (51932);
468 Decoder d = e.GetDecoder ();
472 chars = new char [1];
473 Assert.AreEqual (0, d.GetChars (new byte [] {0xA1}, 0, 1, chars, 0), "#1");
474 Assert.AreEqual (new char [] {'\0'}, chars, "#2");
477 chars = new char [1];
478 Assert.AreEqual (1, d.GetChars (new byte [] {0xA1}, 0, 1, chars, 0), "#3");
479 Assert.AreEqual (new char [] {'\u3000'}, chars, "#4");
481 // incomplete but refreshed
482 chars = new char [1];
483 Assert.AreEqual (0, d.GetChars (new byte [] {0xA1}, 0, 1, chars, 0), "#5");
484 Assert.AreEqual (new char [] {'\0'}, chars, "#6");
488 public void Decoder936NoRefresh ()
490 Encoding e = Manager.GetEncoding (936);
491 Decoder d = e.GetDecoder ();
495 chars = new char [1];
496 Assert.AreEqual (0, d.GetChars (new byte [] {0xB0}, 0, 1, chars, 0), "#1");
497 Assert.AreEqual (new char [] {'\0'}, chars, "#2");
500 chars = new char [1];
501 Assert.AreEqual (1, d.GetChars (new byte [] {0xA1}, 0, 1, chars, 0), "#3");
502 Assert.AreEqual (new char [] {'\u554A'}, chars, "#4");
504 // incomplete but refreshed
505 chars = new char [1];
506 Assert.AreEqual (0, d.GetChars (new byte [] {0xB0}, 0, 1, chars, 0), "#5");
507 Assert.AreEqual (new char [] {'\0'}, chars, "#6");
511 public void Decoder949NoRefresh ()
513 Encoding e = Manager.GetEncoding (949);
514 Decoder d = e.GetDecoder ();
518 chars = new char [1];
519 Assert.AreEqual (0, d.GetChars (new byte [] {0x81}, 0, 1, chars, 0), "#1");
520 Assert.AreEqual (new char [] {'\0'}, chars, "#2");
523 chars = new char [1];
524 Assert.AreEqual (1, d.GetChars (new byte [] {0x41}, 0, 1, chars, 0), "#3");
525 Assert.AreEqual (new char [] {'\uAC02'}, chars, "#4");
527 // incomplete but refreshed
528 chars = new char [1];
529 Assert.AreEqual (0, d.GetChars (new byte [] {0x81}, 0, 1, chars, 0), "#5");
530 Assert.AreEqual (new char [] {'\0'}, chars, "#6");
534 public void Decoder950NoRefresh ()
536 Encoding e = Manager.GetEncoding (950);
537 Decoder d = e.GetDecoder ();
541 chars = new char [1];
542 Assert.AreEqual (0, d.GetChars (new byte [] {0xF9}, 0, 1, chars, 0), "#1");
543 Assert.AreEqual (new char [] {'\0'}, chars, "#2");
546 chars = new char [1];
547 Assert.AreEqual (1, d.GetChars (new byte [] {0x40}, 0, 1, chars, 0), "#3");
548 Assert.AreEqual (new char [] {'\u7E98'}, chars, "#4");
550 // incomplete but refreshed
551 chars = new char [1];
552 Assert.AreEqual (0, d.GetChars (new byte [] {0xF9}, 0, 1, chars, 0), "#5");
553 Assert.AreEqual (new char [] {'\0'}, chars, "#6");
557 public void HandleObsoletedESCJ () // bug #398273
559 byte [] b = new byte [] {0x64, 0x6f, 0x6e, 0x1b, 0x24, 0x42, 0x21, 0x47, 0x1b, 0x28, 0x4a, 0x74};
560 string s = Manager.GetEncoding ("ISO-2022-JP").GetString (b);
561 Assert.AreEqual ("don\u2019t", s);
566 public void Bug14591 ()
568 var expected = "\u4f50\u85e4\u8c4a";
569 var text = Encoding.GetEncoding ("iso-2022-jp").GetString (Convert.FromBase64String ("GyRAOjRGI0stGyhK"));
570 Assert.AreEqual (expected, text, "#1");
577 public void CP949_Encode ()
579 AssertEncode ("Test/texts/korean-utf8.txt", "Test/texts/korean-949.txt", 949);
583 public void CP949_Decode ()
585 AssertDecode ("Test/texts/korean-utf8.txt", "Test/texts/korean-949.txt", 949);