mcs/class/System/System.Net/WebConnection.cs

   1 //
   2 // System.Net.WebConnection
   3 //
   4 // Authors:
   5 //      Gonzalo Paniagua Javier (gonzalo@ximian.com)
   6 //
   7 // (C) 2003 Ximian, Inc (http://www.ximian.com)
   8 //
   9
  10 //
  11 // Permission is hereby granted, free of charge, to any person obtaining
  12 // a copy of this software and associated documentation files (the
  13 // "Software"), to deal in the Software without restriction, including
  14 // without limitation the rights to use, copy, modify, merge, publish,
  15 // distribute, sublicense, and/or sell copies of the Software, and to
  16 // permit persons to whom the Software is furnished to do so, subject to
  17 // the following conditions:
  18 //
  19 // The above copyright notice and this permission notice shall be
  20 // included in all copies or substantial portions of the Software.
  21 //
  22 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  23 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  24 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  25 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  26 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  27 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  28 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  29 //
  30
  31 using System.IO;
  32 using System.Collections;
  33 using System.Net.Sockets;
  34 using System.Security.Cryptography.X509Certificates;
  35 using System.Text;
  36 using System.Threading;
  37 using System.Diagnostics;
  38 using Mono.Net.Security;
  39
  40 namespace System.Net
  41 {
  42         enum ReadState
  43         {
  44                 None,
  45                 Status,
  46                 Headers,
  47                 Content,
  48                 Aborted
  49         }
  50
  51         class WebConnection
  52         {
  53                 ServicePoint sPoint;
  54                 Stream nstream;
  55                 internal Socket socket;
  56                 object socketLock = new object ();
  57                 IWebConnectionState state;
  58                 WebExceptionStatus status;
  59                 WaitCallback initConn;
  60                 bool keepAlive;
  61                 byte [] buffer;
  62                 static AsyncCallback readDoneDelegate = new AsyncCallback (ReadDone);
  63                 EventHandler abortHandler;
  64                 AbortHelper abortHelper;
  65                 internal WebConnectionData Data;
  66                 bool chunkedRead;
  67                 ChunkStream chunkStream;
  68                 Queue queue;
  69                 bool reused;
  70                 int position;
  71                 HttpWebRequest priority_request;
  72                 NetworkCredential ntlm_credentials;
  73                 bool ntlm_authenticated;
  74                 bool unsafe_sharing;
  75
  76                 enum NtlmAuthState
  77                 {
  78                         None,
  79                         Challenge,
  80                         Response
  81                 }
  82                 NtlmAuthState connect_ntlm_auth_state;
  83                 HttpWebRequest connect_request;
  84
  85                 Exception connect_exception;
  86                 static object classLock = new object ();
  87                 MonoTlsStream tlsStream;
  88
  89 #if MONOTOUCH && !MONOTOUCH_TV && !MONOTOUCH_WATCH
  90                 [System.Runtime.InteropServices.DllImport ("__Internal")]
  91                 static extern void xamarin_start_wwan (string uri);
  92 #endif
  93
  94                 internal ChunkStream ChunkStream {
  95                         get { return chunkStream; }
  96                 }
  97
  98                 public WebConnection (IWebConnectionState wcs, ServicePoint sPoint)
  99                 {
 100                         this.state = wcs;
 101                         this.sPoint = sPoint;
 102                         buffer = new byte [4096];
 103                         Data = new WebConnectionData ();
 104                         initConn = new WaitCallback (state => {
 105                                 try {
 106                                         InitConnection (state);
 107                                 } catch {}
 108                                 });
 109                         queue = wcs.Group.Queue;
 110                         abortHelper = new AbortHelper ();
 111                         abortHelper.Connection = this;
 112                         abortHandler = new EventHandler (abortHelper.Abort);
 113                 }
 114
 115                 class AbortHelper {
 116                         public WebConnection Connection;
 117
 118                         public void Abort (object sender, EventArgs args)
 119                         {
 120                                 WebConnection other = ((HttpWebRequest) sender).WebConnection;
 121                                 if (other == null)
 122                                         other = Connection;
 123                                 other.Abort (sender, args);
 124                         }
 125                 }
 126
 127                 bool CanReuse ()
 128                 {
 129                         // The real condition is !(socket.Poll (0, SelectMode.SelectRead) || socket.Available != 0)
 130                         // but if there's data pending to read (!) we won't reuse the socket.
 131                         return (socket.Poll (0, SelectMode.SelectRead) == false);
 132                 }
 133
 134                 void Connect (HttpWebRequest request)
 135                 {
 136                         lock (socketLock) {
 137                                 if (socket != null && socket.Connected && status == WebExceptionStatus.Success) {
 138                                         // Take the chunked stream to the expected state (State.None)
 139                                         if (CanReuse () && CompleteChunkedRead ()) {
 140                                                 reused = true;
 141                                                 return;
 142                                         }
 143                                 }
 144
 145                                 reused = false;
 146                                 if (socket != null) {
 147                                         socket.Close();
 148                                         socket = null;
 149                                 }
 150
 151                                 chunkStream = null;
 152                                 IPHostEntry hostEntry = sPoint.HostEntry;
 153
 154                                 if (hostEntry == null) {
 155 #if MONOTOUCH && !MONOTOUCH_TV && !MONOTOUCH_WATCH
 156                                         xamarin_start_wwan (sPoint.Address.ToString ());
 157                                         hostEntry = sPoint.HostEntry;
 158                                         if (hostEntry == null) {
 159 #endif
 160                                                 status = sPoint.UsesProxy ? WebExceptionStatus.ProxyNameResolutionFailure :
 161                                                                             WebExceptionStatus.NameResolutionFailure;
 162                                                 return;
 163 #if MONOTOUCH && !MONOTOUCH_TV && !MONOTOUCH_WATCH
 164                                         }
 165 #endif
 166                                 }
 167
 168                                 //WebConnectionData data = Data;
 169                                 foreach (IPAddress address in hostEntry.AddressList) {
 170                                         try {
 171                                                 socket = new Socket (address.AddressFamily, SocketType.Stream, ProtocolType.Tcp);
 172                                         } catch (Exception se) {
 173                                                 // The Socket ctor can throw if we run out of FD's
 174                                                 if (!request.Aborted)
 175                                                                 status = WebExceptionStatus.ConnectFailure;
 176                                                 connect_exception = se;
 177                                                 return;
 178                                         }
 179                                         IPEndPoint remote = new IPEndPoint (address, sPoint.Address.Port);
 180                                         socket.NoDelay = !sPoint.UseNagleAlgorithm;
 181                                         try {
 182                                                 sPoint.KeepAliveSetup (socket);
 183                                         } catch {
 184                                                 // Ignore. Not supported in all platforms.
 185                                         }
 186
 187                                         if (!sPoint.CallEndPointDelegate (socket, remote)) {
 188                                                 socket.Close ();
 189                                                 socket = null;
 190                                                 status = WebExceptionStatus.ConnectFailure;
 191                                         } else {
 192                                                 try {
 193                                                         if (request.Aborted)
 194                                                                 return;
 195                                                         socket.Connect (remote);
 196                                                         status = WebExceptionStatus.Success;
 197                                                         break;
 198                                                 } catch (ThreadAbortException) {
 199                                                         // program exiting...
 200                                                         Socket s = socket;
 201                                                         socket = null;
 202                                                         if (s != null)
 203                                                                 s.Close ();
 204                                                         return;
 205                                                 } catch (ObjectDisposedException) {
 206                                                         // socket closed from another thread
 207                                                         return;
 208                                                 } catch (Exception exc) {
 209                                                         Socket s = socket;
 210                                                         socket = null;
 211                                                         if (s != null)
 212                                                                 s.Close ();
 213                                                         if (!request.Aborted)
 214                                                                 status = WebExceptionStatus.ConnectFailure;
 215                                                         connect_exception = exc;
 216                                                 }
 217                                         }
 218                                 }
 219                         }
 220                 }
 221
 222                 bool CreateTunnel (HttpWebRequest request, Uri connectUri,
 223                                    Stream stream, out byte[] buffer)
 224                 {
 225                         StringBuilder sb = new StringBuilder ();
 226                         sb.Append ("CONNECT ");
 227                         sb.Append (request.Address.Host);
 228                         sb.Append (':');
 229                         sb.Append (request.Address.Port);
 230                         sb.Append (" HTTP/");
 231                         if (request.ServicePoint.ProtocolVersion == HttpVersion.Version11)
 232                                 sb.Append ("1.1");
 233                         else
 234                                 sb.Append ("1.0");
 235
 236                         sb.Append ("\r\nHost: ");
 237                         sb.Append (request.Address.Authority);
 238
 239                         bool ntlm = false;
 240                         var challenge = Data.Challenge;
 241                         Data.Challenge = null;
 242                         var auth_header = request.Headers ["Proxy-Authorization"];
 243                         bool have_auth = auth_header != null;
 244                         if (have_auth) {
 245                                 sb.Append ("\r\nProxy-Authorization: ");
 246                                 sb.Append (auth_header);
 247                                 ntlm = auth_header.ToUpper ().Contains ("NTLM");
 248                         } else if (challenge != null && Data.StatusCode == 407) {
 249                                 ICredentials creds = request.Proxy.Credentials;
 250                                 have_auth = true;
 251
 252                                 if (connect_request == null) {
 253                                         // create a CONNECT request to use with Authenticate
 254                                         connect_request = (HttpWebRequest)WebRequest.Create (
 255                                                 connectUri.Scheme + "://" + connectUri.Host + ":" + connectUri.Port + "/");
 256                                         connect_request.Method = "CONNECT";
 257                                         connect_request.Credentials = creds;
 258                                 }
 259
 260                                 if (creds != null) {
 261                                         for (int i = 0; i < challenge.Length; i++) {
 262                                                 var auth = AuthenticationManager.Authenticate (challenge [i], connect_request, creds);
 263                                                 if (auth == null)
 264                                                         continue;
 265                                                 ntlm = (auth.ModuleAuthenticationType == "NTLM");
 266                                                 sb.Append ("\r\nProxy-Authorization: ");
 267                                                 sb.Append (auth.Message);
 268                                                 break;
 269                                         }
 270                                 }
 271                         }
 272
 273                         if (ntlm) {
 274                                 sb.Append ("\r\nProxy-Connection: keep-alive");
 275                                 connect_ntlm_auth_state++;
 276                         }
 277
 278                         sb.Append ("\r\n\r\n");
 279
 280                         Data.StatusCode = 0;
 281                         byte [] connectBytes = Encoding.Default.GetBytes (sb.ToString ());
 282                         stream.Write (connectBytes, 0, connectBytes.Length);
 283
 284                         int status;
 285                         WebHeaderCollection result = ReadHeaders (stream, out buffer, out status);
 286                         if ((!have_auth || connect_ntlm_auth_state == NtlmAuthState.Challenge) &&
 287                             result != null && status == 407) { // Needs proxy auth
 288                                 var connectionHeader = result ["Connection"];
 289                                 if (socket != null && !string.IsNullOrEmpty (connectionHeader) &&
 290                                     connectionHeader.ToLower() == "close") {
 291                                         // The server is requesting that this connection be closed
 292                                         socket.Close();
 293                                         socket = null;
 294                                 }
 295
 296                                 Data.StatusCode = status;
 297                                 Data.Challenge = result.GetValues ("Proxy-Authenticate");
 298                                 Data.Headers = result;
 299                                 return false;
 300                         }
 301
 302                         if (status != 200) {
 303                                 Data.StatusCode = status;
 304                                 Data.Headers = result;
 305                                 return false;
 306                         }
 307
 308                         return (result != null);
 309                 }
 310
 311                 WebHeaderCollection ReadHeaders (Stream stream, out byte [] retBuffer, out int status)
 312                 {
 313                         retBuffer = null;
 314                         status = 200;
 315
 316                         byte [] buffer = new byte [1024];
 317                         MemoryStream ms = new MemoryStream ();
 318
 319                         while (true) {
 320                                 int n = stream.Read (buffer, 0, 1024);
 321                                 if (n == 0) {
 322                                         HandleError (WebExceptionStatus.ServerProtocolViolation, null, "ReadHeaders");
 323                                         return null;
 324                                 }
 325
 326                                 ms.Write (buffer, 0, n);
 327                                 int start = 0;
 328                                 string str = null;
 329                                 bool gotStatus = false;
 330                                 WebHeaderCollection headers = new WebHeaderCollection ();
 331                                 while (ReadLine (ms.GetBuffer (), ref start, (int) ms.Length, ref str)) {
 332                                         if (str == null) {
 333                                                 int contentLen = 0;
 334                                                 try     {
 335                                                         contentLen = int.Parse(headers["Content-Length"]);
 336                                                 }
 337                                                 catch {
 338                                                         contentLen = 0;
 339                                                 }
 340
 341                                                 if (ms.Length - start - contentLen > 0) {
 342                                                         // we've read more data than the response header and conents,
 343                                                         // give back extra data to the caller
 344                                                         retBuffer = new byte[ms.Length - start - contentLen];
 345                                                         Buffer.BlockCopy(ms.GetBuffer(), start + contentLen, retBuffer, 0, retBuffer.Length);
 346                                                 }
 347                                                 else {
 348                                                         // haven't read in some or all of the contents for the response, do so now
 349                                                         FlushContents(stream, contentLen - (int)(ms.Length - start));
 350                                                 }
 351
 352                                                 return headers;
 353                                         }
 354
 355                                         if (gotStatus) {
 356                                                 headers.Add (str);
 357                                                 continue;
 358                                         }
 359
 360                                         string[] parts = str.Split (' ');
 361                                         if (parts.Length < 2) {
 362                                                 HandleError (WebExceptionStatus.ServerProtocolViolation, null, "ReadHeaders2");
 363                                                 return null;
 364                                         }
 365
 366                                         if (String.Compare (parts [0], "HTTP/1.1", true) == 0)
 367                                                 Data.ProxyVersion = HttpVersion.Version11;
 368                                         else if (String.Compare (parts [0], "HTTP/1.0", true) == 0)
 369                                                 Data.ProxyVersion = HttpVersion.Version10;
 370                                         else {
 371                                                 HandleError (WebExceptionStatus.ServerProtocolViolation, null, "ReadHeaders2");
 372                                                 return null;
 373                                         }
 374
 375                                         status = (int)UInt32.Parse (parts [1]);
 376                                         if (parts.Length >= 3)
 377                                                 Data.StatusDescription = String.Join (" ", parts, 2, parts.Length - 2);
 378
 379                                         gotStatus = true;
 380                                 }
 381                         }
 382                 }
 383
 384                 void FlushContents(Stream stream, int contentLength)
 385                 {
 386                         while (contentLength > 0) {
 387                                 byte[] contentBuffer = new byte[contentLength];
 388                                 int bytesRead = stream.Read(contentBuffer, 0, contentLength);
 389                                 if (bytesRead > 0) {
 390                                         contentLength -= bytesRead;
 391                                 }
 392                                 else {
 393                                         break;
 394                                 }
 395                         }
 396                 }
 397
 398                 bool CreateStream (HttpWebRequest request)
 399                 {
 400                         try {
 401                                 NetworkStream serverStream = new NetworkStream (socket, false);
 402
 403                                 if (request.Address.Scheme == Uri.UriSchemeHttps) {
 404 #if SECURITY_DEP
 405                                         if (!reused || nstream == null || tlsStream == null) {
 406                                                 byte [] buffer = null;
 407                                                 if (sPoint.UseConnect) {
 408                                                         bool ok = CreateTunnel (request, sPoint.Address, serverStream, out buffer);
 409                                                         if (!ok)
 410                                                                 return false;
 411                                                 }
 412                                                 tlsStream = new MonoTlsStream (request, serverStream);
 413                                                 nstream = tlsStream.CreateStream (buffer);
 414                                         }
 415                                         // we also need to set ServicePoint.Certificate
 416                                         // and ServicePoint.ClientCertificate but this can
 417                                         // only be done later (after handshake - which is
 418                                         // done only after a read operation).
 419 #else
 420                                         throw new NotSupportedException ();
 421 #endif
 422                                 } else {
 423                                         nstream = serverStream;
 424                                 }
 425                         } catch (Exception ex) {
 426                                 if (tlsStream != null)
 427                                         status = tlsStream.ExceptionStatus;
 428                                 else if (!request.Aborted)
 429                                         status = WebExceptionStatus.ConnectFailure;
 430                                 connect_exception = ex;
 431                                 return false;
 432                         }
 433
 434                         return true;
 435                 }
 436
 437                 void HandleError (WebExceptionStatus st, Exception e, string where)
 438                 {
 439                         status = st;
 440                         lock (this) {
 441                                 if (st == WebExceptionStatus.RequestCanceled)
 442                                         Data = new WebConnectionData ();
 443                         }
 444
 445                         if (e == null) { // At least we now where it comes from
 446                                 try {
 447                                         throw new Exception (new System.Diagnostics.StackTrace ().ToString ());
 448                                 } catch (Exception e2) {
 449                                         e = e2;
 450                                 }
 451                         }
 452
 453                         HttpWebRequest req = null;
 454                         if (Data != null && Data.request != null)
 455                                 req = Data.request;
 456
 457                         Close (true);
 458                         if (req != null) {
 459                                 req.FinishedReading = true;
 460                                 req.SetResponseError (st, e, where);
 461                         }
 462                 }
 463
 464                 static void ReadDone (IAsyncResult result)
 465                 {
 466                         WebConnection cnc = (WebConnection)result.AsyncState;
 467                         WebConnectionData data = cnc.Data;
 468                         Stream ns = cnc.nstream;
 469                         if (ns == null) {
 470                                 cnc.Close (true);
 471                                 return;
 472                         }
 473
 474                         int nread = -1;
 475                         try {
 476                                 nread = ns.EndRead (result);
 477                         } catch (ObjectDisposedException) {
 478                                 return;
 479                         } catch (Exception e) {
 480                                 if (e.InnerException is ObjectDisposedException)
 481                                         return;
 482
 483                                 cnc.HandleError (WebExceptionStatus.ReceiveFailure, e, "ReadDone1");
 484                                 return;
 485                         }
 486
 487                         if (nread == 0) {
 488                                 cnc.HandleError (WebExceptionStatus.ReceiveFailure, null, "ReadDone2");
 489                                 return;
 490                         }
 491
 492                         if (nread < 0) {
 493                                 cnc.HandleError (WebExceptionStatus.ServerProtocolViolation, null, "ReadDone3");
 494                                 return;
 495                         }
 496
 497                         int pos = -1;
 498                         nread += cnc.position;
 499                         if (data.ReadState == ReadState.None) {
 500                                 Exception exc = null;
 501                                 try {
 502                                         pos = GetResponse (data, cnc.sPoint, cnc.buffer, nread);
 503                                 } catch (Exception e) {
 504                                         exc = e;
 505                                 }
 506
 507                                 if (exc != null || pos == -1) {
 508                                         cnc.HandleError (WebExceptionStatus.ServerProtocolViolation, exc, "ReadDone4");
 509                                         return;
 510                                 }
 511                         }
 512
 513                         if (data.ReadState == ReadState.Aborted) {
 514                                 cnc.HandleError (WebExceptionStatus.RequestCanceled, null, "ReadDone");
 515                                 return;
 516                         }
 517
 518                         if (data.ReadState != ReadState.Content) {
 519                                 int est = nread * 2;
 520                                 int max = (est < cnc.buffer.Length) ? cnc.buffer.Length : est;
 521                                 byte [] newBuffer = new byte [max];
 522                                 Buffer.BlockCopy (cnc.buffer, 0, newBuffer, 0, nread);
 523                                 cnc.buffer = newBuffer;
 524                                 cnc.position = nread;
 525                                 data.ReadState = ReadState.None;
 526                                 InitRead (cnc);
 527                                 return;
 528                         }
 529
 530                         cnc.position = 0;
 531
 532                         WebConnectionStream stream = new WebConnectionStream (cnc, data);
 533                         bool expect_content = ExpectContent (data.StatusCode, data.request.Method);
 534                         string tencoding = null;
 535                         if (expect_content)
 536                                 tencoding = data.Headers ["Transfer-Encoding"];
 537
 538                         cnc.chunkedRead = (tencoding != null && tencoding.IndexOf ("chunked", StringComparison.OrdinalIgnoreCase) != -1);
 539                         if (!cnc.chunkedRead) {
 540                                 stream.ReadBuffer = cnc.buffer;
 541                                 stream.ReadBufferOffset = pos;
 542                                 stream.ReadBufferSize = nread;
 543                                 try {
 544                                         stream.CheckResponseInBuffer ();
 545                                 } catch (Exception e) {
 546                                         cnc.HandleError (WebExceptionStatus.ReceiveFailure, e, "ReadDone7");
 547                                 }
 548                         } else if (cnc.chunkStream == null) {
 549                                 try {
 550                                         cnc.chunkStream = new ChunkStream (cnc.buffer, pos, nread, data.Headers);
 551                                 } catch (Exception e) {
 552                                         cnc.HandleError (WebExceptionStatus.ServerProtocolViolation, e, "ReadDone5");
 553                                         return;
 554                                 }
 555                         } else {
 556                                 cnc.chunkStream.ResetBuffer ();
 557                                 try {
 558                                         cnc.chunkStream.Write (cnc.buffer, pos, nread);
 559                                 } catch (Exception e) {
 560                                         cnc.HandleError (WebExceptionStatus.ServerProtocolViolation, e, "ReadDone6");
 561                                         return;
 562                                 }
 563                         }
 564
 565                         data.stream = stream;
 566
 567                         if (!expect_content)
 568                                 stream.ForceCompletion ();
 569
 570                         data.request.SetResponseData (data);
 571                 }
 572
 573                 static bool ExpectContent (int statusCode, string method)
 574                 {
 575                         if (method == "HEAD")
 576                                 return false;
 577                         return (statusCode >= 200 && statusCode != 204 && statusCode != 304);
 578                 }
 579
 580                 internal static void InitRead (object state)
 581                 {
 582                         WebConnection cnc = (WebConnection) state;
 583                         Stream ns = cnc.nstream;
 584
 585                         try {
 586                                 int size = cnc.buffer.Length - cnc.position;
 587                                 ns.BeginRead (cnc.buffer, cnc.position, size, readDoneDelegate, cnc);
 588                         } catch (Exception e) {
 589                                 cnc.HandleError (WebExceptionStatus.ReceiveFailure, e, "InitRead");
 590                         }
 591                 }
 592
 593                 static int GetResponse (WebConnectionData data, ServicePoint sPoint,
 594                                         byte [] buffer, int max)
 595                 {
 596                         int pos = 0;
 597                         string line = null;
 598                         bool lineok = false;
 599                         bool isContinue = false;
 600                         bool emptyFirstLine = false;
 601                         do {
 602                                 if (data.ReadState == ReadState.Aborted)
 603                                         return -1;
 604
 605                                 if (data.ReadState == ReadState.None) {
 606                                         lineok = ReadLine (buffer, ref pos, max, ref line);
 607                                         if (!lineok)
 608                                                 return 0;
 609
 610                                         if (line == null) {
 611                                                 emptyFirstLine = true;
 612                                                 continue;
 613                                         }
 614                                         emptyFirstLine = false;
 615                                         data.ReadState = ReadState.Status;
 616
 617                                         string [] parts = line.Split (' ');
 618                                         if (parts.Length < 2)
 619                                                 return -1;
 620
 621                                         if (String.Compare (parts [0], "HTTP/1.1", true) == 0) {
 622                                                 data.Version = HttpVersion.Version11;
 623                                                 sPoint.SetVersion (HttpVersion.Version11);
 624                                         } else {
 625                                                 data.Version = HttpVersion.Version10;
 626                                                 sPoint.SetVersion (HttpVersion.Version10);
 627                                         }
 628
 629                                         data.StatusCode = (int) UInt32.Parse (parts [1]);
 630                                         if (parts.Length >= 3)
 631                                                 data.StatusDescription = String.Join (" ", parts, 2, parts.Length - 2);
 632                                         else
 633                                                 data.StatusDescription = "";
 634
 635                                         if (pos >= max)
 636                                                 return pos;
 637                                 }
 638
 639                                 emptyFirstLine = false;
 640                                 if (data.ReadState == ReadState.Status) {
 641                                         data.ReadState = ReadState.Headers;
 642                                         data.Headers = new WebHeaderCollection ();
 643                                         ArrayList headers = new ArrayList ();
 644                                         bool finished = false;
 645                                         while (!finished) {
 646                                                 if (ReadLine (buffer, ref pos, max, ref line) == false)
 647                                                         break;
 648
 649                                                 if (line == null) {
 650                                                         // Empty line: end of headers
 651                                                         finished = true;
 652                                                         continue;
 653                                                 }
 654
 655                                                 if (line.Length > 0 && (line [0] == ' ' || line [0] == '\t')) {
 656                                                         int count = headers.Count - 1;
 657                                                         if (count < 0)
 658                                                                 break;
 659
 660                                                         string prev = (string) headers [count] + line;
 661                                                         headers [count] = prev;
 662                                                 } else {
 663                                                         headers.Add (line);
 664                                                 }
 665                                         }
 666
 667                                         if (!finished)
 668                                                 return 0;
 669
 670                                         // .NET uses ParseHeaders or ParseHeadersStrict which is much better
 671                                         foreach (string s in headers) {
 672
 673                                                 int pos_s = s.IndexOf (':');
 674                                                 if (pos_s == -1)
 675                                                         throw new ArgumentException ("no colon found", "header");
 676
 677                                                 var header = s.Substring (0, pos_s);
 678                                                 var value = s.Substring (pos_s + 1).Trim ();
 679
 680                                                 var h = data.Headers;
 681                                                 if (WebHeaderCollection.AllowMultiValues (header)) {
 682                                                         h.AddInternal (header, value);
 683                                                 } else  {
 684                                                         h.SetInternal (header, value);
 685                                                 }
 686                                         }
 687
 688                                         if (data.StatusCode == (int) HttpStatusCode.Continue) {
 689                                                 sPoint.SendContinue = true;
 690                                                 if (pos >= max)
 691                                                         return pos;
 692
 693                                                 if (data.request.ExpectContinue) {
 694                                                         data.request.DoContinueDelegate (data.StatusCode, data.Headers);
 695                                                         // Prevent double calls when getting the
 696                                                         // headers in several packets.
 697                                                         data.request.ExpectContinue = false;
 698                                                 }
 699
 700                                                 data.ReadState = ReadState.None;
 701                                                 isContinue = true;
 702                                         }
 703                                         else {
 704                                                 data.ReadState = ReadState.Content;
 705                                                 return pos;
 706                                         }
 707                                 }
 708                         } while (emptyFirstLine || isContinue);
 709
 710                         return -1;
 711                 }
 712
 713                 void InitConnection (object state)
 714                 {
 715                         HttpWebRequest request = (HttpWebRequest) state;
 716                         request.WebConnection = this;
 717                         if (request.ReuseConnection)
 718                                 request.StoredConnection = this;
 719
 720                         if (request.Aborted)
 721                                 return;
 722
 723                         keepAlive = request.KeepAlive;
 724                         Data = new WebConnectionData (request);
 725                 retry:
 726                         Connect (request);
 727                         if (request.Aborted)
 728                                 return;
 729
 730                         if (status != WebExceptionStatus.Success) {
 731                                 if (!request.Aborted) {
 732                                         request.SetWriteStreamError (status, connect_exception);
 733                                         Close (true);
 734                                 }
 735                                 return;
 736                         }
 737
 738                         if (!CreateStream (request)) {
 739                                 if (request.Aborted)
 740                                         return;
 741
 742                                 WebExceptionStatus st = status;
 743                                 if (Data.Challenge != null)
 744                                         goto retry;
 745
 746                                 Exception cnc_exc = connect_exception;
 747                                 if (cnc_exc == null && (Data.StatusCode == 401 || Data.StatusCode == 407)) {
 748                                         st = WebExceptionStatus.ProtocolError;
 749                                         if (Data.Headers == null)
 750                                                 Data.Headers = new WebHeaderCollection ();
 751
 752                                         var webResponse = new HttpWebResponse (sPoint.Address, "CONNECT", Data, null);
 753                                         cnc_exc = new WebException (Data.StatusCode == 407 ? "(407) Proxy Authentication Required" : "(401) Unauthorized", null, st, webResponse);
 754                                 }
 755
 756                                 connect_exception = null;
 757                                 request.SetWriteStreamError (st, cnc_exc);
 758                                 Close (true);
 759                                 return;
 760                         }
 761
 762                         request.SetWriteStream (new WebConnectionStream (this, request));
 763                 }
 764
 765 #if MONOTOUCH
 766                 static bool warned_about_queue = false;
 767 #endif
 768
 769                 internal EventHandler SendRequest (HttpWebRequest request)
 770                 {
 771                         if (request.Aborted)
 772                                 return null;
 773
 774                         lock (this) {
 775                                 if (state.TrySetBusy ()) {
 776                                         status = WebExceptionStatus.Success;
 777                                         ThreadPool.QueueUserWorkItem (initConn, request);
 778                                 } else {
 779                                         lock (queue) {
 780 #if MONOTOUCH
 781                                                 if (!warned_about_queue) {
 782                                                         warned_about_queue = true;
 783                                                         Console.WriteLine ("WARNING: An HttpWebRequest was added to the ConnectionGroup queue because the connection limit was reached.");
 784                                                 }
 785 #endif
 786                                                 queue.Enqueue (request);
 787                                         }
 788                                 }
 789                         }
 790
 791                         return abortHandler;
 792                 }
 793
 794                 void SendNext ()
 795                 {
 796                         lock (queue) {
 797                                 if (queue.Count > 0) {
 798                                         SendRequest ((HttpWebRequest) queue.Dequeue ());
 799                                 }
 800                         }
 801                 }
 802
 803                 internal void NextRead ()
 804                 {
 805                         lock (this) {
 806                                 if (Data.request != null)
 807                                         Data.request.FinishedReading = true;
 808                                 string header = (sPoint.UsesProxy) ? "Proxy-Connection" : "Connection";
 809                                 string cncHeader = (Data.Headers != null) ? Data.Headers [header] : null;
 810                                 bool keepAlive = (Data.Version == HttpVersion.Version11 && this.keepAlive);
 811                                 if (Data.ProxyVersion != null && Data.ProxyVersion != HttpVersion.Version11)
 812                                         keepAlive = false;
 813                                 if (cncHeader != null) {
 814                                         cncHeader = cncHeader.ToLower ();
 815                                         keepAlive = (this.keepAlive && cncHeader.IndexOf ("keep-alive", StringComparison.Ordinal) != -1);
 816                                 }
 817
 818                                 if ((socket != null && !socket.Connected) ||
 819                                    (!keepAlive || (cncHeader != null && cncHeader.IndexOf ("close", StringComparison.Ordinal) != -1))) {
 820                                         Close (false);
 821                                 }
 822
 823                                 state.SetIdle ();
 824                                 if (priority_request != null) {
 825                                         SendRequest (priority_request);
 826                                         priority_request = null;
 827                                 } else {
 828                                         SendNext ();
 829                                 }
 830                         }
 831                 }
 832
 833                 static bool ReadLine (byte [] buffer, ref int start, int max, ref string output)
 834                 {
 835                         bool foundCR = false;
 836                         StringBuilder text = new StringBuilder ();
 837
 838                         int c = 0;
 839                         while (start < max) {
 840                                 c = (int) buffer [start++];
 841
 842                                 if (c == '\n') {                        // newline
 843                                         if ((text.Length > 0) && (text [text.Length - 1] == '\r'))
 844                                                 text.Length--;
 845
 846                                         foundCR = false;
 847                                         break;
 848                                 } else if (foundCR) {
 849                                         text.Length--;
 850                                         break;
 851                                 }
 852
 853                                 if (c == '\r')
 854                                         foundCR = true;
 855
 856
 857                                 text.Append ((char) c);
 858                         }
 859
 860                         if (c != '\n' && c != '\r')
 861                                 return false;
 862
 863                         if (text.Length == 0) {
 864                                 output = null;
 865                                 return (c == '\n' || c == '\r');
 866                         }
 867
 868                         if (foundCR)
 869                                 text.Length--;
 870
 871                         output = text.ToString ();
 872                         return true;
 873                 }
 874
 875
 876                 internal IAsyncResult BeginRead (HttpWebRequest request, byte [] buffer, int offset, int size, AsyncCallback cb, object state)
 877                 {
 878                         Stream s = null;
 879                         lock (this) {
 880                                 if (Data.request != request)
 881                                         throw new ObjectDisposedException (typeof (NetworkStream).FullName);
 882                                 if (nstream == null)
 883                                         return null;
 884                                 s = nstream;
 885                         }
 886
 887                         IAsyncResult result = null;
 888                         if (!chunkedRead || (!chunkStream.DataAvailable && chunkStream.WantMore)) {
 889                                 try {
 890                                         result = s.BeginRead (buffer, offset, size, cb, state);
 891                                         cb = null;
 892                                 } catch (Exception) {
 893                                         HandleError (WebExceptionStatus.ReceiveFailure, null, "chunked BeginRead");
 894                                         throw;
 895                                 }
 896                         }
 897
 898                         if (chunkedRead) {
 899                                 WebAsyncResult wr = new WebAsyncResult (cb, state, buffer, offset, size);
 900                                 wr.InnerAsyncResult = result;
 901                                 if (result == null) {
 902                                         // Will be completed from the data in ChunkStream
 903                                         wr.SetCompleted (true, (Exception) null);
 904                                         wr.DoCallback ();
 905                                 }
 906                                 return wr;
 907                         }
 908
 909                         return result;
 910                 }
 911
 912                 internal int EndRead (HttpWebRequest request, IAsyncResult result)
 913                 {
 914                         Stream s = null;
 915                         lock (this) {
 916                                 if (request.Aborted)
 917                                         throw new WebException ("Request aborted", WebExceptionStatus.RequestCanceled);
 918                                 if (Data.request != request)
 919                                         throw new ObjectDisposedException (typeof (NetworkStream).FullName);
 920                                 if (nstream == null)
 921                                         throw new ObjectDisposedException (typeof (NetworkStream).FullName);
 922                                 s = nstream;
 923                         }
 924
 925                         int nbytes = 0;
 926                         bool done = false;
 927                         WebAsyncResult wr = null;
 928                         IAsyncResult nsAsync = ((WebAsyncResult) result).InnerAsyncResult;
 929                         if (chunkedRead && (nsAsync is WebAsyncResult)) {
 930                                 wr = (WebAsyncResult) nsAsync;
 931                                 IAsyncResult inner = wr.InnerAsyncResult;
 932                                 if (inner != null && !(inner is WebAsyncResult)) {
 933                                         nbytes = s.EndRead (inner);
 934                                         done = nbytes == 0;
 935                                 }
 936                         } else if (!(nsAsync is WebAsyncResult)) {
 937                                 nbytes = s.EndRead (nsAsync);
 938                                 wr = (WebAsyncResult) result;
 939                                 done = nbytes == 0;
 940                         }
 941
 942                         if (chunkedRead) {
 943                                 try {
 944                                         chunkStream.WriteAndReadBack (wr.Buffer, wr.Offset, wr.Size, ref nbytes);
 945                                         if (!done && nbytes == 0 && chunkStream.WantMore)
 946                                                 nbytes = EnsureRead (wr.Buffer, wr.Offset, wr.Size);
 947                                 } catch (Exception e) {
 948                                         if (e is WebException)
 949                                                 throw e;
 950
 951                                         throw new WebException ("Invalid chunked data.", e,
 952                                                                 WebExceptionStatus.ServerProtocolViolation, null);
 953                                 }
 954
 955                                 if ((done || nbytes == 0) && chunkStream.ChunkLeft != 0) {
 956                                         HandleError (WebExceptionStatus.ReceiveFailure, null, "chunked EndRead");
 957                                         throw new WebException ("Read error", null, WebExceptionStatus.ReceiveFailure, null);
 958                                 }
 959                         }
 960
 961                         return (nbytes != 0) ? nbytes : -1;
 962                 }
 963
 964                 // To be called on chunkedRead when we can read no data from the ChunkStream yet
 965                 int EnsureRead (byte [] buffer, int offset, int size)
 966                 {
 967                         byte [] morebytes = null;
 968                         int nbytes = 0;
 969                         while (nbytes == 0 && chunkStream.WantMore) {
 970                                 int localsize = chunkStream.ChunkLeft;
 971                                 if (localsize <= 0) // not read chunk size yet
 972                                         localsize = 1024;
 973                                 else if (localsize > 16384)
 974                                         localsize = 16384;
 975
 976                                 if (morebytes == null || morebytes.Length < localsize)
 977                                         morebytes = new byte [localsize];
 978
 979                                 int nread = nstream.Read (morebytes, 0, localsize);
 980                                 if (nread <= 0)
 981                                         return 0; // Error
 982
 983                                 chunkStream.Write (morebytes, 0, nread);
 984                                 nbytes += chunkStream.Read (buffer, offset + nbytes, size - nbytes);
 985                         }
 986
 987                         return nbytes;
 988                 }
 989
 990                 bool CompleteChunkedRead()
 991                 {
 992                         if (!chunkedRead || chunkStream == null)
 993                                 return true;
 994
 995                         while (chunkStream.WantMore) {
 996                                 int nbytes = nstream.Read (buffer, 0, buffer.Length);
 997                                 if (nbytes <= 0)
 998                                         return false; // Socket was disconnected
 999
1000                                 chunkStream.Write (buffer, 0, nbytes);
1001                         }
1002
1003                         return true;
1004                 }
1005
1006                 internal IAsyncResult BeginWrite (HttpWebRequest request, byte [] buffer, int offset, int size, AsyncCallback cb, object state)
1007                 {
1008                         Stream s = null;
1009                         lock (this) {
1010                                 if (Data.request != request)
1011                                         throw new ObjectDisposedException (typeof (NetworkStream).FullName);
1012                                 if (nstream == null)
1013                                         return null;
1014                                 s = nstream;
1015                         }
1016
1017                         IAsyncResult result = null;
1018                         try {
1019                                 result = s.BeginWrite (buffer, offset, size, cb, state);
1020                         } catch (Exception) {
1021                                 status = WebExceptionStatus.SendFailure;
1022                                 throw;
1023                         }
1024
1025                         return result;
1026                 }
1027
1028                 internal bool EndWrite (HttpWebRequest request, bool throwOnError, IAsyncResult result)
1029                 {
1030                         Stream s = null;
1031                         lock (this) {
1032                                 if (status == WebExceptionStatus.RequestCanceled)
1033                                         return true;
1034                                 if (Data.request != request)
1035                                         throw new ObjectDisposedException (typeof (NetworkStream).FullName);
1036                                 if (nstream == null)
1037                                         throw new ObjectDisposedException (typeof (NetworkStream).FullName);
1038                                 s = nstream;
1039                         }
1040
1041                         try {
1042                                 s.EndWrite (result);
1043                                 return true;
1044                         } catch (Exception exc) {
1045                                 status = WebExceptionStatus.SendFailure;
1046                                 if (throwOnError && exc.InnerException != null)
1047                                         throw exc.InnerException;
1048                                 return false;
1049                         }
1050                 }
1051
1052                 internal int Read (HttpWebRequest request, byte [] buffer, int offset, int size)
1053                 {
1054                         Stream s = null;
1055                         lock (this) {
1056                                 if (Data.request != request)
1057                                         throw new ObjectDisposedException (typeof (NetworkStream).FullName);
1058                                 if (nstream == null)
1059                                         return 0;
1060                                 s = nstream;
1061                         }
1062
1063                         int result = 0;
1064                         try {
1065                                 bool done = false;
1066                                 if (!chunkedRead) {
1067                                         result = s.Read (buffer, offset, size);
1068                                         done = (result == 0);
1069                                 }
1070
1071                                 if (chunkedRead) {
1072                                         try {
1073                                                 chunkStream.WriteAndReadBack (buffer, offset, size, ref result);
1074                                                 if (!done && result == 0 && chunkStream.WantMore)
1075                                                         result = EnsureRead (buffer, offset, size);
1076                                         } catch (Exception e) {
1077                                                 HandleError (WebExceptionStatus.ReceiveFailure, e, "chunked Read1");
1078                                                 throw;
1079                                         }
1080
1081                                         if ((done || result == 0) && chunkStream.WantMore) {
1082                                                 HandleError (WebExceptionStatus.ReceiveFailure, null, "chunked Read2");
1083                                                 throw new WebException ("Read error", null, WebExceptionStatus.ReceiveFailure, null);
1084                                         }
1085                                 }
1086                         } catch (Exception e) {
1087                                 HandleError (WebExceptionStatus.ReceiveFailure, e, "Read");
1088                         }
1089
1090                         return result;
1091                 }
1092
1093                 internal bool Write (HttpWebRequest request, byte [] buffer, int offset, int size, ref string err_msg)
1094                 {
1095                         err_msg = null;
1096                         Stream s = null;
1097                         lock (this) {
1098                                 if (Data.request != request)
1099                                         throw new ObjectDisposedException (typeof (NetworkStream).FullName);
1100                                 s = nstream;
1101                                 if (s == null)
1102                                         return false;
1103                         }
1104
1105                         try {
1106                                 s.Write (buffer, offset, size);
1107                         } catch (Exception e) {
1108                                 err_msg = e.Message;
1109                                 WebExceptionStatus wes = WebExceptionStatus.SendFailure;
1110                                 string msg = "Write: " + err_msg;
1111                                 if (e is WebException) {
1112                                         HandleError (wes, e, msg);
1113                                         return false;
1114                                 }
1115
1116                                 HandleError (wes, e, msg);
1117                                 return false;
1118                         }
1119                         return true;
1120                 }
1121
1122                 internal void Close (bool sendNext)
1123                 {
1124                         lock (this) {
1125                                 if (Data != null && Data.request != null && Data.request.ReuseConnection) {
1126                                         Data.request.ReuseConnection = false;
1127                                         return;
1128                                 }
1129
1130                                 if (nstream != null) {
1131                                         try {
1132                                                 nstream.Close ();
1133                                         } catch {}
1134                                         nstream = null;
1135                                 }
1136
1137                                 if (socket != null) {
1138                                         try {
1139                                                 socket.Close ();
1140                                         } catch {}
1141                                         socket = null;
1142                                 }
1143
1144                                 if (ntlm_authenticated)
1145                                         ResetNtlm ();
1146                                 if (Data != null) {
1147                                         lock (Data) {
1148                                                 Data.ReadState = ReadState.Aborted;
1149                                         }
1150                                 }
1151                                 state.SetIdle ();
1152                                 Data = new WebConnectionData ();
1153                                 if (sendNext)
1154                                         SendNext ();
1155
1156                                 connect_request = null;
1157                                 connect_ntlm_auth_state = NtlmAuthState.None;
1158                         }
1159                 }
1160
1161                 void Abort (object sender, EventArgs args)
1162                 {
1163                         lock (this) {
1164                                 lock (queue) {
1165                                         HttpWebRequest req = (HttpWebRequest) sender;
1166                                         if (Data.request == req || Data.request == null) {
1167                                                 if (!req.FinishedReading) {
1168                                                         status = WebExceptionStatus.RequestCanceled;
1169                                                         Close (false);
1170                                                         if (queue.Count > 0) {
1171                                                                 Data.request = (HttpWebRequest) queue.Dequeue ();
1172                                                                 SendRequest (Data.request);
1173                                                         }
1174                                                 }
1175                                                 return;
1176                                         }
1177
1178                                         req.FinishedReading = true;
1179                                         req.SetResponseError (WebExceptionStatus.RequestCanceled, null, "User aborted");
1180                                         if (queue.Count > 0 && queue.Peek () == sender) {
1181                                                 queue.Dequeue ();
1182                                         } else if (queue.Count > 0) {
1183                                                 object [] old = queue.ToArray ();
1184                                                 queue.Clear ();
1185                                                 for (int i = old.Length - 1; i >= 0; i--) {
1186                                                         if (old [i] != sender)
1187                                                                 queue.Enqueue (old [i]);
1188                                                 }
1189                                         }
1190                                 }
1191                         }
1192                 }
1193
1194                 internal void ResetNtlm ()
1195                 {
1196                         ntlm_authenticated = false;
1197                         ntlm_credentials = null;
1198                         unsafe_sharing = false;
1199                 }
1200
1201                 internal bool Connected {
1202                         get {
1203                                 lock (this) {
1204                                         return (socket != null && socket.Connected);
1205                                 }
1206                         }
1207                 }
1208
1209                 // -Used for NTLM authentication
1210                 internal HttpWebRequest PriorityRequest {
1211                         set { priority_request = value; }
1212                 }
1213
1214                 internal bool NtlmAuthenticated {
1215                         get { return ntlm_authenticated; }
1216                         set { ntlm_authenticated = value; }
1217                 }
1218
1219                 internal NetworkCredential NtlmCredential {
1220                         get { return ntlm_credentials; }
1221                         set { ntlm_credentials = value; }
1222                 }
1223
1224                 internal bool UnsafeAuthenticatedConnectionSharing {
1225                         get { return unsafe_sharing; }
1226                         set { unsafe_sharing = value; }
1227                 }
1228                 // -
1229         }
1230 }
1231