{
using System;
+using System.Runtime.InteropServices;
[Serializable]
-[MonoTODO ("Fix serialization compatibility with MS.NET")]
+[MonoTODO ("Serialization format not compatible with .NET")]
#if NET_2_0
[MonoTODO ("EncoderFallback is not handled")]
+[ComVisible (true)]
#endif
public class UTF8Encoding : Encoding
{
emitIdentifier = encoderShouldEmitUTF8Identifier;
#if NET_2_0
if (throwOnInvalidBytes)
- DecoderFallback = new DecoderExceptionFallback ();
+ SetFallbackInternal (null, new DecoderExceptionFallback ());
else
- DecoderFallback = new DecoderReplacementFallback (String.Empty);
+ SetFallbackInternal (null, new DecoderReplacementFallback ("\uFFFD"));
#else
throwOnInvalid = throwOnInvalidBytes;
#endif
is_browser_save = true;
is_browser_display = true;
is_mail_news_display = true;
+ is_mail_news_save = true;
windows_code_page = UnicodeEncoding.UNICODE_CODE_PAGE;
}
+ #region GetByteCount()
+
// Internal version of "GetByteCount" which can handle a rolling
// state between multiple calls to this method.
- private static int InternalGetByteCount (char[] chars, int index, int count, uint leftOver, bool flush)
+ private static int InternalGetByteCount (char[] chars, int index, int count, ref char leftOver, bool flush)
{
// Validate the parameters.
if (chars == null) {
throw new ArgumentOutOfRangeException ("count", _("ArgRange_Array"));
}
- // Determine the lengths of all characters.
- char ch;
+ if (index == chars.Length) {
+ if (flush && leftOver != '\0') {
+ // Flush the left-over surrogate pair start.
+ leftOver = '\0';
+ return 3;
+ }
+ return 0;
+ }
+
+ unsafe {
+ fixed (char* cptr = chars) {
+ return InternalGetByteCount (cptr + index, count, ref leftOver, flush);
+ }
+ }
+ }
+
+ private unsafe static int InternalGetByteCount (char* chars, int count, ref char leftOver, bool flush)
+ {
int length = 0;
- uint pair = leftOver;
- while (count > 0) {
- ch = chars[index];
- if (pair == 0) {
- if (ch < '\u0080') {
- ++length;
- } else if (ch < '\u0800') {
- length += 2;
- } else if (ch >= '\uD800' && ch <= '\uDBFF') {
- // This is the start of a surrogate pair.
- pair = (uint)ch;
+ char* end = chars + count;
+ while (chars < end) {
+ if (leftOver == 0) {
+ for (; chars < end; chars++) {
+ if (*chars < '\x80') {
+ ++length;
+ } else if (*chars < '\x800') {
+ length += 2;
+ } else if (*chars < '\uD800' || *chars > '\uDFFF') {
+ length += 3;
+ } else if (*chars <= '\uDBFF') {
+ // This is a surrogate start char, exit the inner loop only
+ // if we don't find the complete surrogate pair.
+ if (chars + 1 < end && chars [1] >= '\uDC00' && chars [1] <= '\uDFFF') {
+ length += 4;
+ chars++;
+ continue;
+ }
+ leftOver = *chars;
+ chars++;
+ break;
+ } else {
+ // We have a surrogate tail without
+ // leading surrogate. In NET_2_0 it
+ // uses fallback. In NET_1_1 we output
+ // wrong surrogate.
+ length += 3;
+ leftOver = '\0';
+ }
+ }
+ } else {
+ if (*chars >= '\uDC00' && *chars <= '\uDFFF') {
+ // We have a correct surrogate pair.
+ length += 4;
+ chars++;
} else {
+ // We have a surrogate start followed by a
+ // regular character. Technically, this is
+ // invalid, but we have to do something.
+ // We write out the surrogate start and then
+ // re-visit the current character again.
length += 3;
}
- } else if (ch >= '\uDC00' && ch <= '\uDFFF') {
- // We have a surrogate pair.
- length += 4;
- pair = 0;
- } else {
- // We have a surrogate start followed by a
- // regular character. Technically, this is
- // invalid, but we have to do something.
- // We write out the surrogate start and then
- // re-visit the current character again.
- length += 3;
- pair = 0;
- continue;
+ leftOver = '\0';
}
- ++index;
- --count;
}
- if (flush && pair != 0) {
+ if (flush) {
// Flush the left-over surrogate pair start.
- length += 3;
+ if (leftOver != '\0') {
+ length += 3;
+ leftOver = '\0';
+ }
}
-
- // Return the final length to the caller.
return length;
}
// Get the number of bytes needed to encode a character buffer.
public override int GetByteCount (char[] chars, int index, int count)
{
- return InternalGetByteCount (chars, index, count, 0, true);
+ char dummy = '\0';
+ return InternalGetByteCount (chars, index, count, ref dummy, true);
}
+#if !NET_2_0
// Convenience wrappers for "GetByteCount".
- public override int GetByteCount (String s)
+ public override int GetByteCount (String chars)
{
// Validate the parameters.
- if (s == null) {
- throw new ArgumentNullException ("s");
+ if (chars == null) {
+ throw new ArgumentNullException ("chars");
}
- // Determine the lengths of all characters.
- char ch;
- int index = 0;
- int count = s.Length;
- int length = 0;
- uint pair;
- while (count > 0) {
- ch = s[index++];
- if (ch < '\u0080') {
- ++length;
- } else if (ch < '\u0800') {
- length += 2;
- } else if (ch >= '\uD800' && ch <= '\uDBFF' && count > 1) {
- // This may be the start of a surrogate pair.
- pair = (uint)(s[index]);
- if (pair >= (uint)0xDC00 && pair <= (uint)0xDFFF) {
- length += 4;
- ++index;
- --count;
- } else {
- length += 3;
- }
- } else {
- length += 3;
+ unsafe {
+ fixed (char* cptr = chars) {
+ char dummy = '\0';
+ return InternalGetByteCount (cptr, chars.Length, ref dummy, true);
}
- --count;
}
+ }
+#endif
- // Return the final length to the caller.
- return length;
+#if NET_2_0
+ [CLSCompliant (false)]
+ [ComVisible (false)]
+ public unsafe override int GetByteCount (char* chars, int count)
+ {
+ if (chars == null)
+ throw new ArgumentNullException ("chars");
+ if (count == 0)
+ return 0;
+ char dummy = '\0';
+ return InternalGetByteCount (chars, count, ref dummy, true);
}
+#endif
+
+ #endregion
+
+ #region GetBytes()
// Internal version of "GetBytes" which can handle a rolling
// state between multiple calls to this method.
private static int InternalGetBytes (char[] chars, int charIndex,
int charCount, byte[] bytes,
- int byteIndex, ref uint leftOver,
+ int byteIndex, ref char leftOver,
bool flush)
{
// Validate the parameters.
throw new ArgumentOutOfRangeException ("byteIndex", _("ArgRange_Array"));
}
- // Convert the characters into bytes.
- char ch;
- int length = bytes.Length;
- uint pair;
- uint left = leftOver;
- int posn = byteIndex;
- while (charCount > 0) {
- // Fetch the next UTF-16 character pair value.
- ch = chars[charIndex++];
- --charCount;
- if (left == 0) {
- if (ch >= '\uD800' && ch <= '\uDBFF') {
- // This is the start of a surrogate pair.
- left = (uint)ch;
- continue;
- } else {
- // This is a regular character.
- pair = (uint)ch;
- }
- } else if (ch >= '\uDC00' && ch <= '\uDFFF') {
- // We have a surrogate pair.
- pair = ((left - (uint)0xD800) << 10) +
- (((uint)ch) - (uint)0xDC00) +
- (uint)0x10000;
- left = 0;
- } else {
- // We have a surrogate start followed by a
- // regular character. Technically, this is
- // invalid, but we have to do something.
- // We write out the surrogate start and then
- // re-visit the current character again.
- pair = (uint)left;
- left = 0;
- --charIndex;
- ++charCount;
+ if (charIndex == chars.Length) {
+ if (flush && leftOver != '\0') {
+#if NET_2_0
+ // FIXME: use EncoderFallback.
+ //
+ // By default it is empty, so I do nothing for now.
+ leftOver = '\0';
+#else
+ // Flush the left-over surrogate pair start.
+ if (byteIndex >= bytes.Length - 3)
+ throw new ArgumentException (_("Arg_InsufficientSpace"), "bytes");
+ bytes [byteIndex++] = 0xEF;
+ bytes [byteIndex++] = 0xBB;
+ bytes [byteIndex++] = 0xBF;
+ leftOver = '\0';
+ return 3;
+#endif
}
+ return 0;
+ }
- // Encode the character pair value.
- if (pair < (uint)0x0080) {
- if (posn >= length) {
- throw new ArgumentException (_("Arg_InsufficientSpace"), "bytes");
- }
- bytes[posn++] = (byte)pair;
- } else if (pair < (uint)0x0800) {
- if ((posn + 2) > length) {
- throw new ArgumentException (_("Arg_InsufficientSpace"), "bytes");
+ unsafe {
+ fixed (char* cptr = chars) {
+ if (bytes.Length == byteIndex)
+ return InternalGetBytes (
+ cptr + charIndex, charCount,
+ null, 0, ref leftOver, flush);
+ fixed (byte *bptr = bytes) {
+ return InternalGetBytes (
+ cptr + charIndex, charCount,
+ bptr + byteIndex, bytes.Length - byteIndex,
+ ref leftOver, flush);
}
- bytes[posn++] = (byte)(0xC0 | (pair >> 6));
- bytes[posn++] = (byte)(0x80 | (pair & 0x3F));
- } else if (pair < (uint)0x10000) {
- if ((posn + 3) > length) {
- throw new ArgumentException (_("Arg_InsufficientSpace"), "bytes");
+ }
+ }
+ }
+
+ private unsafe static int InternalGetBytes (char* chars, int count, byte* bytes, int bcount, ref char leftOver, bool flush)
+ {
+ char* end = chars + count;
+ byte* end_bytes = bytes + bcount;
+ while (chars < end) {
+ if (leftOver == 0) {
+ for (; chars < end; chars++) {
+ int ch = *chars;
+ if (ch < '\x80') {
+ if (bytes >= end_bytes)
+ goto fail_no_space;
+ *bytes++ = (byte)ch;
+ } else if (ch < '\x800') {
+ if (bytes + 1 >= end_bytes)
+ goto fail_no_space;
+ bytes [0] = (byte) (0xC0 | (ch >> 6));
+ bytes [1] = (byte) (0x80 | (ch & 0x3F));
+ bytes += 2;
+ } else if (ch < '\uD800' || ch > '\uDFFF') {
+ if (bytes + 2 >= end_bytes)
+ goto fail_no_space;
+ bytes [0] = (byte) (0xE0 | (ch >> 12));
+ bytes [1] = (byte) (0x80 | ((ch >> 6) & 0x3F));
+ bytes [2] = (byte) (0x80 | (ch & 0x3F));
+ bytes += 3;
+ } else if (ch <= '\uDBFF') {
+ // This is a surrogate char, exit the inner loop.
+ leftOver = *chars;
+ chars++;
+ break;
+ } else {
+ // We have a surrogate tail without
+ // leading surrogate. In NET_2_0 it
+ // uses fallback. In NET_1_1 we output
+ // wrong surrogate.
+ if (bytes + 2 >= end_bytes)
+ goto fail_no_space;
+ bytes [0] = (byte) (0xE0 | (ch >> 12));
+ bytes [1] = (byte) (0x80 | ((ch >> 6) & 0x3F));
+ bytes [2] = (byte) (0x80 | (ch & 0x3F));
+ bytes += 3;
+ leftOver = '\0';
+ }
}
- bytes[posn++] = (byte)(0xE0 | (pair >> 12));
- bytes[posn++] = (byte)(0x80 | ((pair >> 6) & 0x3F));
- bytes[posn++] = (byte)(0x80 | (pair & 0x3F));
} else {
- if ((posn + 4) > length) {
- throw new ArgumentException (_("Arg_InsufficientSpace"), "bytes");
+ if (*chars >= '\uDC00' && *chars <= '\uDFFF') {
+ // We have a correct surrogate pair.
+ int ch = 0x10000 + (int) *chars - 0xDC00 + (((int) leftOver - 0xD800) << 10);
+ if (bytes + 3 >= end_bytes)
+ goto fail_no_space;
+ bytes [0] = (byte) (0xF0 | (ch >> 18));
+ bytes [1] = (byte) (0x80 | ((ch >> 12) & 0x3F));
+ bytes [2] = (byte) (0x80 | ((ch >> 6) & 0x3F));
+ bytes [3] = (byte) (0x80 | (ch & 0x3F));
+ bytes += 4;
+ chars++;
+ } else {
+ // We have a surrogate start followed by a
+ // regular character. Technically, this is
+ // invalid, but we have to do something.
+ // We write out the surrogate start and then
+ // re-visit the current character again.
+ int ch = leftOver;
+ if (bytes + 2 >= end_bytes)
+ goto fail_no_space;
+ bytes [0] = (byte) (0xE0 | (ch >> 12));
+ bytes [1] = (byte) (0x80 | ((ch >> 6) & 0x3F));
+ bytes [2] = (byte) (0x80 | (ch & 0x3F));
+ bytes += 3;
}
- bytes[posn++] = (byte)(0xF0 | (pair >> 18));
- bytes[posn++] = (byte)(0x80 | ((pair >> 12) & 0x3F));
- bytes[posn++] = (byte)(0x80 | ((pair >> 6) & 0x3F));
- bytes[posn++] = (byte)(0x80 | (pair & 0x3F));
+ leftOver = '\0';
}
}
- if (flush && left != 0) {
+ if (flush) {
// Flush the left-over surrogate pair start.
- if ((posn + 3) > length) {
- throw new ArgumentException (_("Arg_InsufficientSpace"), "bytes");
+ if (leftOver != '\0') {
+ int ch = leftOver;
+ if (bytes + 2 < end_bytes) {
+ bytes [0] = (byte) (0xE0 | (ch >> 12));
+ bytes [1] = (byte) (0x80 | ((ch >> 6) & 0x3F));
+ bytes [2] = (byte) (0x80 | (ch & 0x3F));
+ bytes += 3;
+ } else {
+ goto fail_no_space;
+ }
+ leftOver = '\0';
}
- bytes[posn++] = (byte)(0xE0 | (left >> 12));
- bytes[posn++] = (byte)(0x80 | ((left >> 6) & 0x3F));
- bytes[posn++] = (byte)(0x80 | (left & 0x3F));
- left = 0;
}
- leftOver = left;
-
- // Return the final count to the caller.
- return posn - byteIndex;
+ return (int)(bytes - (end_bytes - bcount));
+fail_no_space:
+ throw new ArgumentException ("Insufficient Space", "bytes");
}
// Get the bytes that result from encoding a character buffer.
public override int GetBytes (char[] chars, int charIndex, int charCount,
byte[] bytes, int byteIndex)
{
- uint leftOver = 0;
+ char leftOver = '\0';
return InternalGetBytes (chars, charIndex, charCount, bytes, byteIndex, ref leftOver, true);
}
throw new ArgumentOutOfRangeException ("byteIndex", _("ArgRange_Array"));
}
- // Convert the characters into bytes.
- char ch;
- int length = bytes.Length;
- uint pair;
- int posn = byteIndex;
- while (charCount > 0) {
- // Fetch the next UTF-16 character pair value.
- ch = s[charIndex++];
- if (ch >= '\uD800' && ch <= '\uDBFF' && charCount > 1) {
- // This may be the start of a surrogate pair.
- pair = (uint)(s[charIndex]);
- if (pair >= (uint)0xDC00 && pair <= (uint)0xDFFF) {
- pair = (pair - (uint)0xDC00) +
- ((((uint)ch) - (uint)0xD800) << 10) +
- (uint)0x10000;
- ++charIndex;
- --charCount;
- } else {
- pair = (uint)ch;
- }
- } else {
- pair = (uint)ch;
- }
- --charCount;
+ if (charIndex == s.Length)
+ return 0;
- // Encode the character pair value.
- if (pair < (uint)0x0080) {
- if (posn >= length) {
- throw new ArgumentException (_("Arg_InsufficientSpace"), "bytes");
- }
- bytes[posn++] = (byte)pair;
- } else if (pair < (uint)0x0800) {
- if ((posn + 2) > length) {
- throw new ArgumentException (_("Arg_InsufficientSpace"), "bytes");
+ unsafe {
+ fixed (char* cptr = s) {
+ char dummy = '\0';
+ if (bytes.Length == byteIndex)
+ return InternalGetBytes (
+ cptr + charIndex, charCount,
+ null, 0, ref dummy, true);
+ fixed (byte *bptr = bytes) {
+ return InternalGetBytes (
+ cptr + charIndex, charCount,
+ bptr + byteIndex, bytes.Length - byteIndex,
+ ref dummy, true);
}
- bytes[posn++] = (byte)(0xC0 | (pair >> 6));
- bytes[posn++] = (byte)(0x80 | (pair & 0x3F));
- } else if (pair < (uint)0x10000) {
- if ((posn + 3) > length) {
- throw new ArgumentException (_("Arg_InsufficientSpace"), "bytes");
- }
- bytes[posn++] = (byte)(0xE0 | (pair >> 12));
- bytes[posn++] = (byte)(0x80 | ((pair >> 6) & 0x3F));
- bytes[posn++] = (byte)(0x80 | (pair & 0x3F));
- } else {
- if ((posn + 4) > length) {
- throw new ArgumentException (_("Arg_InsufficientSpace"), "bytes");
- }
- bytes[posn++] = (byte)(0xF0 | (pair >> 18));
- bytes[posn++] = (byte)(0x80 | ((pair >> 12) & 0x3F));
- bytes[posn++] = (byte)(0x80 | ((pair >> 6) & 0x3F));
- bytes[posn++] = (byte)(0x80 | (pair & 0x3F));
}
}
+ }
+
+#if NET_2_0
+ [CLSCompliant (false)]
+ [ComVisible (false)]
+ public unsafe override int GetBytes (char* chars, int charCount, byte* bytes, int byteCount)
+ {
+ if (chars == null)
+ throw new ArgumentNullException ("chars");
+ if (charCount < 0)
+ throw new IndexOutOfRangeException ("charCount");
+ if (bytes == null)
+ throw new ArgumentNullException ("bytes");
+ if (byteCount < 0)
+ throw new IndexOutOfRangeException ("charCount");
- // Return the final count to the caller.
- return posn - byteIndex;
+ if (charCount == 0)
+ return 0;
+
+ char dummy = '\0';
+ if (byteCount == 0)
+ return InternalGetBytes (chars, charCount, null, 0, ref dummy, true);
+ else
+ return InternalGetBytes (chars, charCount, bytes, byteCount, ref dummy, true);
}
+#endif
+
+ #endregion
// Internal version of "GetCharCount" which can handle a rolling
// state between multiple calls to this method.
#if NET_2_0
- // Internal version of "GetCharCount" which can handle a rolling
- // state between multiple calls to this method.
- private static int InternalGetCharCount (
+ private unsafe static int InternalGetCharCount (
byte[] bytes, int index, int count, uint leftOverBits,
- uint leftOverCount, DecoderFallbackBuffer fallbackBuffer, bool flush)
+ uint leftOverCount, object provider,
+ ref DecoderFallbackBuffer fallbackBuffer, ref byte [] bufferArg, bool flush)
#else
- private static int InternalGetCharCount (
+ private unsafe static int InternalGetCharCount (
byte[] bytes, int index, int count, uint leftOverBits,
uint leftOverCount, bool throwOnInvalid, bool flush)
#endif
throw new ArgumentOutOfRangeException ("count", _("ArgRange_Array"));
}
+ if (count == 0)
+ return 0;
+ fixed (byte *bptr = bytes)
+#if NET_2_0
+ return InternalGetCharCount (bptr + index, count,
+ leftOverBits, leftOverCount, provider, ref fallbackBuffer, ref bufferArg, flush);
+#else
+ return InternalGetCharCount (bptr + index, count,
+ leftOverBits, leftOverCount, throwOnInvalid, flush);
+#endif
+ }
+
+#if NET_2_0
+ private unsafe static int InternalGetCharCount (
+ byte* bytes, int count, uint leftOverBits,
+ uint leftOverCount, object provider,
+ ref DecoderFallbackBuffer fallbackBuffer, ref byte [] bufferArg, bool flush)
+#else
+ private unsafe static int InternalGetCharCount (
+ byte* bytes, int count, uint leftOverBits,
+ uint leftOverCount, bool throwOnInvalid, bool flush)
+#endif
+ {
+ int index = 0;
+
+ int length = 0;
+
+ if (leftOverCount == 0) {
+ int end = index + count;
+ for (; index < end; index++, count--) {
+ if (bytes [index] < 0x80)
+ length++;
+ else
+ break;
+ }
+ }
+
// Determine the number of characters that we have.
uint ch;
- int length = 0;
uint leftBits = leftOverBits;
uint leftSoFar = (leftOverCount & (uint)0x0F);
uint leftSize = ((leftOverCount >> 4) & (uint)0x0F);
} else {
// Invalid UTF-8 start character.
#if NET_2_0
- length += Fallback (fallbackBuffer, bytes, index - 1);
+ length += Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, index - 1, 1);
#else
if (throwOnInvalid)
throw new ArgumentException (_("Arg_InvalidUTF8"), "bytes");
}
if (overlong) {
#if NET_2_0
- length += Fallback (fallbackBuffer, bytes, index - 1);
+ length += Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, index - leftSoFar, leftSoFar);
#else
if (throwOnInvalid)
throw new ArgumentException (_("Overlong"), leftBits.ToString ());
length += 2;
} else {
#if NET_2_0
- length += Fallback (fallbackBuffer, bytes, index - 1);
+ length += Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, index - leftSoFar, leftSoFar);
#else
if (throwOnInvalid)
throw new ArgumentException (_("Arg_InvalidUTF8"), "bytes");
} else {
// Invalid UTF-8 sequence: clear and restart.
#if NET_2_0
- length += Fallback (fallbackBuffer, bytes, index - 1);
+ length += Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, index - leftSoFar, leftSoFar);
#else
if (throwOnInvalid)
throw new ArgumentException (_("Arg_InvalidUTF8"), "bytes");
// We had left-over bytes that didn't make up
// a complete UTF-8 character sequence.
#if NET_2_0
- length += Fallback (fallbackBuffer, bytes, index - 1);
+ length += Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, index - leftSoFar, leftSoFar);
#else
if (throwOnInvalid)
throw new ArgumentException (_("Arg_InvalidUTF8"), "bytes");
#if NET_2_0
// for GetCharCount()
- static int Fallback (DecoderFallbackBuffer buffer, byte [] bytes, int index)
+ static unsafe int Fallback (object provider, ref DecoderFallbackBuffer buffer, ref byte [] bufferArg, byte* bytes, long index, uint size)
{
- buffer.Fallback (bytes, index - 1);
- return buffer.Remaining;
+ if (buffer == null) {
+ DecoderFallback fb = provider as DecoderFallback;
+ if (fb != null)
+ buffer = fb.CreateFallbackBuffer ();
+ else
+ buffer = ((Decoder) provider).FallbackBuffer;
+ }
+ if (bufferArg == null)
+ bufferArg = new byte [1];
+ int ret = 0;
+ for (int i = 0; i < size; i++) {
+ bufferArg [0] = bytes [(int) index + i];
+ buffer.Fallback (bufferArg, 0);
+ ret += buffer.Remaining;
+ buffer.Reset ();
+ }
+ return ret;
}
// for GetChars()
- static void Fallback (DecoderFallbackBuffer buffer, byte [] bytes, int byteIndex,
- char [] chars, ref int charIndex)
+ static unsafe void Fallback (object provider, ref DecoderFallbackBuffer buffer, ref byte [] bufferArg, byte* bytes, long byteIndex, uint size,
+ char* chars, ref int charIndex)
{
- buffer.Fallback (bytes, byteIndex - 1);
- while (buffer.Remaining > 0)
- chars [charIndex++] = buffer.GetNextChar ();
+ if (buffer == null) {
+ DecoderFallback fb = provider as DecoderFallback;
+ if (fb != null)
+ buffer = fb.CreateFallbackBuffer ();
+ else
+ buffer = ((Decoder) provider).FallbackBuffer;
+ }
+ if (bufferArg == null)
+ bufferArg = new byte [1];
+ for (int i = 0; i < size; i++) {
+ bufferArg [0] = bytes [byteIndex + i];
+ buffer.Fallback (bufferArg, 0);
+ while (buffer.Remaining > 0)
+ chars [charIndex++] = buffer.GetNextChar ();
+ buffer.Reset ();
+ }
}
#endif
public override int GetCharCount (byte[] bytes, int index, int count)
{
#if NET_2_0
- return InternalGetCharCount (bytes, index, count, 0, 0, DecoderFallback.CreateFallbackBuffer (), true);
+ DecoderFallbackBuffer buf = null;
+ byte [] bufferArg = null;
+ return InternalGetCharCount (bytes, index, count, 0, 0, DecoderFallback, ref buf, ref bufferArg, true);
#else
return InternalGetCharCount (bytes, index, count, 0, 0, throwOnInvalid, true);
#endif
}
+#if NET_2_0
+ [CLSCompliant (false)]
+ [ComVisible (false)]
+ public unsafe override int GetCharCount (byte* bytes, int count)
+ {
+ DecoderFallbackBuffer buf = null;
+ byte [] bufferArg = null;
+ return InternalGetCharCount (bytes, count, 0, 0, DecoderFallback, ref buf, ref bufferArg, true);
+ }
+#endif
+
// Get the characters that result from decoding a byte buffer.
#if NET_2_0
- private static int InternalGetChars (
+ private unsafe static int InternalGetChars (
byte[] bytes, int byteIndex, int byteCount, char[] chars,
int charIndex, ref uint leftOverBits, ref uint leftOverCount,
- DecoderFallbackBuffer fallbackBuffer, bool flush)
+ object provider,
+ ref DecoderFallbackBuffer fallbackBuffer, ref byte [] bufferArg, bool flush)
#else
- private static int InternalGetChars (
+ private unsafe static int InternalGetChars (
byte[] bytes, int byteIndex, int byteCount, char[] chars,
int charIndex, ref uint leftOverBits, ref uint leftOverCount,
bool throwOnInvalid, bool flush)
if (charIndex == chars.Length)
return 0;
+ fixed (char* cptr = chars) {
+#if NET_2_0
+ if (byteCount == 0 || byteIndex == bytes.Length)
+ return InternalGetChars (null, 0, cptr + charIndex, chars.Length - charIndex, ref leftOverBits, ref leftOverCount, provider, ref fallbackBuffer, ref bufferArg, flush);
+ // otherwise...
+ fixed (byte* bptr = bytes)
+ return InternalGetChars (bptr + byteIndex, byteCount, cptr + charIndex, chars.Length - charIndex, ref leftOverBits, ref leftOverCount, provider, ref fallbackBuffer, ref bufferArg, flush);
+#else
+ if (byteCount == 0 || byteIndex == bytes.Length)
+ return InternalGetChars (null, 0, cptr + charIndex, chars.Length - charIndex, ref leftOverBits, ref leftOverCount, throwOnInvalid, flush);
+ // otherwise...
+ fixed (byte* bptr = bytes)
+ return InternalGetChars (bptr + byteIndex, byteCount, cptr + charIndex, chars.Length - charIndex, ref leftOverBits, ref leftOverCount, throwOnInvalid, flush);
+#endif
+ }
+ }
+
+#if NET_2_0
+ private unsafe static int InternalGetChars (
+ byte* bytes, int byteCount, char* chars, int charCount,
+ ref uint leftOverBits, ref uint leftOverCount,
+ object provider,
+ ref DecoderFallbackBuffer fallbackBuffer, ref byte [] bufferArg, bool flush)
+#else
+ private unsafe static int InternalGetChars (
+ byte* bytes, int byteCount, char* chars, int charCount,
+ ref uint leftOverBits, ref uint leftOverCount,
+ bool throwOnInvalid, bool flush)
+#endif
+ {
+ int charIndex = 0, byteIndex = 0;
+ int length = charCount;
+ int posn = charIndex;
+
+ if (leftOverCount == 0) {
+ int end = byteIndex + byteCount;
+ for (; byteIndex < end; posn++, byteIndex++, byteCount--) {
+ if (bytes [byteIndex] < 0x80)
+ chars [posn] = (char) bytes [byteIndex];
+ else
+ break;
+ }
+ }
+
// Convert the bytes into the output buffer.
uint ch;
- int length = chars.Length;
- int posn = charIndex;
uint leftBits = leftOverBits;
uint leftSoFar = (leftOverCount & (uint)0x0F);
uint leftSize = ((leftOverCount >> 4) & (uint)0x0F);
- while (byteCount > 0) {
+
+ int byteEnd = byteIndex + byteCount;
+ for(; byteIndex < byteEnd; byteIndex++) {
// Fetch the next character from the byte buffer.
- ch = (uint)(bytes[byteIndex++]);
- --byteCount;
+ ch = (uint)(bytes[byteIndex]);
if (leftSize == 0) {
// Process a UTF-8 start character.
if (ch < (uint)0x0080) {
} else {
// Invalid UTF-8 start character.
#if NET_2_0
- Fallback (fallbackBuffer, bytes, byteIndex, chars, ref posn);
+ Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, byteIndex, 1, chars, ref posn);
#else
if (throwOnInvalid)
throw new ArgumentException (_("Arg_InvalidUTF8"), "bytes");
}
if (overlong) {
#if NET_2_0
- Fallback (fallbackBuffer, bytes, byteIndex, chars, ref posn);
+ Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, byteIndex - leftSoFar, leftSoFar, chars, ref posn);
#else
if (throwOnInvalid)
throw new ArgumentException (_("Overlong"), leftBits.ToString ());
+#endif
+ }
+ else if ((leftBits & 0xF800) == 0xD800) {
+ // UTF-8 doesn't use surrogate characters
+#if NET_2_0
+ Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, byteIndex - leftSoFar, leftSoFar, chars, ref posn);
+#else
+ if (throwOnInvalid)
+ throw new ArgumentException (_("Arg_InvalidUTF8"), "bytes");
#endif
}
else {
(char)((leftBits & (uint)0x3FF) + (uint)0xDC00);
} else {
#if NET_2_0
- Fallback (fallbackBuffer, bytes, byteIndex, chars, ref posn);
+ Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, byteIndex - leftSoFar, leftSoFar, chars, ref posn);
#else
if (throwOnInvalid)
throw new ArgumentException (_("Arg_InvalidUTF8"), "bytes");
} else {
// Invalid UTF-8 sequence: clear and restart.
#if NET_2_0
- Fallback (fallbackBuffer, bytes, byteIndex, chars, ref posn);
+ Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, byteIndex - leftSoFar, leftSoFar, chars, ref posn);
#else
if (throwOnInvalid)
throw new ArgumentException (_("Arg_InvalidUTF8"), "bytes");
#endif
leftSize = 0;
--byteIndex;
- ++byteCount;
}
}
}
// We had left-over bytes that didn't make up
// a complete UTF-8 character sequence.
#if NET_2_0
- Fallback (fallbackBuffer, bytes, byteIndex, chars, ref posn);
+ Fallback (provider, ref fallbackBuffer, ref bufferArg, bytes, byteIndex - leftSoFar, leftSoFar, chars, ref posn);
#else
if (throwOnInvalid)
throw new ArgumentException (_("Arg_InvalidUTF8"), "bytes");
uint leftOverBits = 0;
uint leftOverCount = 0;
#if NET_2_0
+ DecoderFallbackBuffer buf = null;
+ byte [] bufferArg = null;
return InternalGetChars (bytes, byteIndex, byteCount, chars,
- charIndex, ref leftOverBits, ref leftOverCount, DecoderFallback.CreateFallbackBuffer (), true);
+ charIndex, ref leftOverBits, ref leftOverCount, DecoderFallback, ref buf, ref bufferArg, true);
#else
return InternalGetChars (bytes, byteIndex, byteCount, chars,
charIndex, ref leftOverBits, ref leftOverCount, throwOnInvalid, true);
#endif
}
+#if NET_2_0
+ [CLSCompliant (false)]
+ [ComVisible (false)]
+ public unsafe override int GetChars (byte* bytes, int byteCount, char* chars, int charCount)
+ {
+ DecoderFallbackBuffer buf = null;
+ byte [] bufferArg = null;
+ uint leftOverBits = 0;
+ uint leftOverCount = 0;
+ return InternalGetChars (bytes, byteCount, chars,
+ charCount, ref leftOverBits, ref leftOverCount, DecoderFallback, ref buf, ref bufferArg, true);
+ }
+#endif
+
// Get the maximum number of bytes needed to encode a
// specified number of characters.
public override int GetMaxByteCount (int charCount)
public override Decoder GetDecoder ()
{
#if NET_2_0
- UTF8Decoder ret = new UTF8Decoder ();
- ret.Fallback = DecoderFallback;
- return ret;
+ return new UTF8Decoder (DecoderFallback);
#else
return new UTF8Decoder (throwOnInvalid);
#endif
{
return base.GetHashCode ();
}
-
+
+#if NET_2_0
+ public override int GetByteCount (string chars)
+ {
+ // hmm, does this override make any sense?
+ return base.GetByteCount (chars);
+ }
+
+ [ComVisible (false)]
+ public override string GetString (byte [] bytes, int index, int count)
+ {
+ // hmm, does this override make any sense?
+ return base.GetString (bytes, index, count);
+ }
+#endif
+
+#if !NET_2_0
public override byte [] GetBytes (String s)
{
if (s == null)
GetBytes (s, 0, s.Length, bytes, 0);
return bytes;
}
+#endif
// UTF-8 decoder implementation.
[Serializable]
// Constructor.
#if NET_2_0
- public UTF8Decoder ()
+ public UTF8Decoder (DecoderFallback fallback)
#else
public UTF8Decoder (bool throwOnInvalid)
#endif
{
-#if !NET_2_0
+#if NET_2_0
+ Fallback = fallback;
+#else
this.throwOnInvalid = throwOnInvalid;
#endif
leftOverBits = 0;
public override int GetCharCount (byte[] bytes, int index, int count)
{
#if NET_2_0
+ DecoderFallbackBuffer buf = null;
+ byte [] bufferArg = null;
return InternalGetCharCount (bytes, index, count,
- leftOverBits, leftOverCount, FallbackBuffer, false);
+ leftOverBits, leftOverCount, this, ref buf, ref bufferArg, false);
#else
return InternalGetCharCount (bytes, index, count,
leftOverBits, leftOverCount, throwOnInvalid, false);
int byteCount, char[] chars, int charIndex)
{
#if NET_2_0
+ DecoderFallbackBuffer buf = null;
+ byte [] bufferArg = null;
return InternalGetChars (bytes, byteIndex, byteCount,
- chars, charIndex, ref leftOverBits, ref leftOverCount, FallbackBuffer, false);
+ chars, charIndex, ref leftOverBits, ref leftOverCount, this, ref buf, ref bufferArg, false);
#else
return InternalGetChars (bytes, byteIndex, byteCount,
chars, charIndex, ref leftOverBits, ref leftOverCount, throwOnInvalid, false);
[Serializable]
private class UTF8Encoder : Encoder
{
- private bool emitIdentifier;
- private uint leftOver;
+// private bool emitIdentifier;
+ private char leftOverForCount;
+ private char leftOverForConv;
// Constructor.
public UTF8Encoder (bool emitIdentifier)
{
- this.emitIdentifier = emitIdentifier;
- leftOver = 0;
+// this.emitIdentifier = emitIdentifier;
+ leftOverForCount = '\0';
+ leftOverForConv = '\0';
}
// Override inherited methods.
public override int GetByteCount (char[] chars, int index,
int count, bool flush)
{
- return InternalGetByteCount (chars, index, count, leftOver, flush);
+ return InternalGetByteCount (chars, index, count, ref leftOverForCount, flush);
}
public override int GetBytes (char[] chars, int charIndex,
- int charCount, byte[] bytes, int byteCount, bool flush)
+ int charCount, byte[] bytes, int byteIndex, bool flush)
{
int result;
- result = InternalGetBytes (chars, charIndex, charCount, bytes, byteCount, ref leftOver, flush);
- emitIdentifier = false;
+ result = InternalGetBytes (chars, charIndex, charCount, bytes, byteIndex, ref leftOverForConv, flush);
+// emitIdentifier = false;
return result;
}
+#if NET_2_0
+ public unsafe override int GetByteCount (char* chars, int count, bool flush)
+ {
+ return InternalGetByteCount (chars, count, ref leftOverForCount, flush);
+ }
+
+ public unsafe override int GetBytes (char* chars, int charCount,
+ byte* bytes, int byteCount, bool flush)
+ {
+ int result;
+ result = InternalGetBytes (chars, charCount, bytes, byteCount, ref leftOverForConv, flush);
+// emitIdentifier = false;
+ return result;
+ }
+#endif
+
} // class UTF8Encoder
}; // class UTF8Encoding
projects / mono.git / blobdiff