mcs/class/referencesource/System/net/System/Net/_Connection.cs

   1 //------------------------------------------------------------------------------
   2 // <copyright file="_Connection.cs" company="Microsoft">
   3 //     Copyright (c) Microsoft Corporation.  All rights reserved.
   4 // </copyright>
   5 //------------------------------------------------------------------------------
   6
   7 namespace System.Net {
   8
   9     using System;
  10     using System.Collections;
  11     using System.Collections.Generic;
  12     using System.Diagnostics;
  13     using System.Net.Sockets;
  14     using System.Threading;
  15     using System.Security;
  16     using System.Globalization;
  17     using System.Net.Configuration;
  18     using System.Diagnostics.CodeAnalysis;
  19
  20     internal enum ReadState {
  21         Start,
  22         StatusLine, // about to parse status line
  23         Headers,    // reading headers
  24         Data        // now read data
  25     }
  26
  27     internal enum DataParseStatus {
  28         NeedMoreData = 0,   // need more data
  29         ContinueParsing,    // continue parsing
  30         Done,               // done
  31         Invalid,            // bad data format
  32         DataTooBig,         // data exceeds the allowed size
  33     }
  34
  35     internal enum WriteBufferState {
  36         Disabled,
  37         Headers,
  38         Buffer,
  39         Playback,
  40     }
  41
  42     // The enum lietrals will be displayed to the user in the exception message
  43     internal enum WebParseErrorSection {
  44         Generic,
  45         ResponseHeader,
  46         ResponseStatusLine,
  47         ResponseBody
  48     }
  49
  50     // The enum literal will be used to look up an error string in the resource file
  51     internal enum WebParseErrorCode {
  52         Generic,
  53         InvalidHeaderName,
  54         InvalidContentLength,
  55         IncompleteHeaderLine,
  56         CrLfError,
  57         InvalidChunkFormat,
  58         UnexpectedServerResponse
  59     }
  60
  61     // Only defined for DataParseStatus.Invalid
  62     struct WebParseError {
  63         public WebParseErrorSection  Section;
  64         public WebParseErrorCode     Code;
  65     }
  66
  67
  68     struct TunnelStateObject {
  69         internal TunnelStateObject(HttpWebRequest r, Connection c){
  70             Connection = c;
  71             OriginalRequest = r;
  72         }
  73
  74         internal Connection Connection;
  75         internal HttpWebRequest OriginalRequest;
  76     }
  77
  78     //
  79     // ConnectionReturnResult - used to spool requests that have been completed,
  80     //  and need to be notified.
  81     //
  82
  83     internal class ConnectionReturnResult {
  84
  85         private static readonly WaitCallback s_InvokeConnectionCallback = new WaitCallback(InvokeConnectionCallback);
  86
  87         private struct RequestContext {
  88             internal HttpWebRequest      Request;
  89             internal object              CoreResponse;
  90
  91             internal RequestContext(HttpWebRequest request, object coreResponse)
  92             {
  93                 Request = request;
  94                 CoreResponse = coreResponse;
  95             }
  96         }
  97
  98         private List<RequestContext>  m_Context;
  99
 100         internal ConnectionReturnResult()
 101         {
 102             m_Context = new List<RequestContext>(5);
 103         }
 104
 105         internal ConnectionReturnResult(int capacity)
 106         {
 107             m_Context = new List<RequestContext>(capacity);
 108         }
 109
 110         internal bool IsNotEmpty {
 111             get {
 112                 return m_Context.Count != 0;
 113             }
 114         }
 115
 116         internal static void Add(ref ConnectionReturnResult returnResult, HttpWebRequest request, CoreResponseData coreResponseData)
 117         {
 118             if (coreResponseData == null)
 119                 throw new InternalException(); //This may cause duplicate requests if we let it through in retail
 120
 121             if (returnResult == null) {
 122                 returnResult = new ConnectionReturnResult();
 123             }
 124
 125 #if DEBUG
 126             //This may cause duplicate requests if we let it through in retail but it's may be expensive to catch here
 127             for (int j = 0; j <  returnResult.m_Context.Count; ++j)
 128                 if ((object)returnResult.m_Context[j].Request == (object) request)
 129                     throw new InternalException();
 130 #endif
 131
 132             returnResult.m_Context.Add(new RequestContext(request, coreResponseData));
 133         }
 134
 135         internal static void AddExceptionRange(ref ConnectionReturnResult returnResult, HttpWebRequest [] requests, Exception exception)
 136         {
 137             AddExceptionRange(ref returnResult, requests, exception, exception);
 138         }
 139         internal static void AddExceptionRange(ref ConnectionReturnResult returnResult, HttpWebRequest [] requests, Exception exception, Exception firstRequestException)
 140         {
 141
 142             //This may cause duplicate requests if we let it through in retail
 143             if (exception == null)
 144                 throw new InternalException();
 145
 146             if (returnResult == null) {
 147                 returnResult = new ConnectionReturnResult(requests.Length);
 148             }
 149             // "abortedRequestExeption" is assigned to the "abortedRequest" or to the very first request if the latest is null
 150             // Everyone else will get "exception"
 151             for (int i = 0; i < requests.Length; ++i)
 152             {
 153 #if DEBUG
 154                 //This may cause duplicate requests if we let it through in retail but it's may be expensive to catch here
 155                 for (int j = 0; j <  returnResult.m_Context.Count; ++j)
 156                     if ((object)returnResult.m_Context[j].Request == (object) requests[i])
 157                         throw new InternalException();
 158 #endif
 159
 160                 if (i == 0)
 161                     returnResult.m_Context.Add(new RequestContext(requests[i], firstRequestException));
 162                 else
 163                     returnResult.m_Context.Add(new RequestContext(requests[i], exception));
 164             }
 165         }
 166
 167         internal static void SetResponses(ConnectionReturnResult returnResult) {
 168             if (returnResult==null){
 169                 return;
 170             }
 171
 172             GlobalLog.Print("ConnectionReturnResult#" + ValidationHelper.HashString(returnResult) + "::SetResponses() count=" + returnResult.m_Context.Count.ToString());
 173             for (int i = 0; i < returnResult.m_Context.Count; i++)
 174             {
 175                 try {
 176                     HttpWebRequest request = returnResult.m_Context[i].Request;
 177 #if DEBUG
 178                     CoreResponseData coreResponseData = returnResult.m_Context[i].CoreResponse as CoreResponseData;
 179                     if (coreResponseData == null)
 180                         GlobalLog.DebugRemoveRequest(request);
 181 #endif
 182                     request.SetAndOrProcessResponse(returnResult.m_Context[i].CoreResponse);
 183                 }
 184                 catch(Exception e) {
 185                     //ASYNCISSUE
 186                     // on error, with more than one callback need to queue others off to another thread
 187
 188                     GlobalLog.Print("ConnectionReturnResult#" + ValidationHelper.HashString(returnResult) + "::Exception"+e);
 189                     returnResult.m_Context.RemoveRange(0,(i+1));
 190                     if (returnResult.m_Context.Count > 0)
 191                     {
 192                         ThreadPool.UnsafeQueueUserWorkItem(s_InvokeConnectionCallback, returnResult);
 193                     }
 194                     throw;
 195                 }
 196             }
 197
 198             returnResult.m_Context.Clear();
 199         }
 200
 201         private static void InvokeConnectionCallback(object objectReturnResult)
 202         {
 203             ConnectionReturnResult returnResult = (ConnectionReturnResult)objectReturnResult;
 204             SetResponses(returnResult);
 205         }
 206     }
 207
 208     //
 209     // Connection - this is the Connection used to parse
 210     //   server responses, queue requests, and pipeline requests
 211     //
 212     internal class Connection : PooledStream {
 213
 214         //
 215         // thread statics - these values must be per thread, because
 216         //  other requests and operations can take place concurrently on this Connection.
 217         //  Our concern is to make sure that a nested call does not get confused with an
 218         //  operation on another thread.   Parameter passing cannot be used, because
 219         //  the call stack may exit and then reenter the same Connection object.
 220         //
 221         [ThreadStatic]
 222         private static int t_SyncReadNesting;
 223
 224         private const int CRLFSize = 2;
 225         private const long c_InvalidContentLength = -2L;
 226
 227         //
 228         // Buffer manager that allocates and reuses 4k buffers.
 229         //
 230         private const int CachedBufferSize = 4096;
 231         private static PinnableBufferCache s_PinnableBufferCache = new PinnableBufferCache("System.Net.Connection", CachedBufferSize);
 232
 233         //
 234         // Little status line holder.
 235         //
 236         private class StatusLineValues
 237         {
 238             internal int MajorVersion;
 239             internal int MinorVersion;
 240             internal int StatusCode;
 241             internal string StatusDescription;
 242         }
 243
 244         private class WaitListItem
 245         {
 246             private HttpWebRequest request;
 247             private long queueStartTime;
 248
 249             public HttpWebRequest Request
 250             {
 251                 get { return request; }
 252             }
 253
 254             public long QueueStartTime
 255             {
 256                 get { return queueStartTime; }
 257             }
 258
 259             public WaitListItem(HttpWebRequest request, long queueStartTime)
 260             {
 261                 this.request = request;
 262                 this.queueStartTime = queueStartTime;
 263             }
 264         }
 265
 266         //
 267         // class members
 268         //
 269         private WebExceptionStatus  m_Error;
 270         internal Exception           m_InnerException;
 271
 272
 273         internal int                m_IISVersion = -1; //-1 means unread
 274         private byte[]              m_ReadBuffer;
 275         private bool                m_ReadBufferFromPinnableCache; // If we get our m_readBuffer from the Pinnable cache we have to explicitly free it
 276         private int                 m_BytesRead;
 277         private int                 m_BytesScanned;
 278         private int                 m_TotalResponseHeadersLength;
 279         private int                 m_MaximumResponseHeadersLength;
 280         private long                m_MaximumUnauthorizedUploadLength;
 281         private CoreResponseData    m_ResponseData;
 282         private ReadState           m_ReadState;
 283         private StatusLineValues    m_StatusLineValues;
 284         private int                 m_StatusState;
 285         private List<WaitListItem>  m_WaitList;
 286         private ArrayList           m_WriteList;
 287         private IAsyncResult        m_LastAsyncResult;
 288         private TimerThread.Timer   m_RecycleTimer;
 289         private WebParseError       m_ParseError;
 290         private bool                m_AtLeastOneResponseReceived;
 291
 292         private static readonly WaitCallback m_PostReceiveDelegate = new WaitCallback(PostReceiveWrapper);
 293         private static readonly AsyncCallback m_ReadCallback = new AsyncCallback(ReadCallbackWrapper);
 294         private static readonly AsyncCallback m_TunnelCallback = new AsyncCallback(TunnelThroughProxyWrapper);
 295         private static byte[] s_NullBuffer = new byte[0];
 296
 297         //
 298         // Abort handling variables. When trying to abort the
 299         // connection, we set Aborted = true, and close m_AbortSocket
 300         // if its non-null. m_AbortDelegate, is returned to every
 301         // request from our SubmitRequest method.  Calling m_AbortDelegate
 302         // drives us into Abort mode.
 303         //
 304         private HttpAbortDelegate m_AbortDelegate;
 305         private ConnectionGroup   m_ConnectionGroup;
 306
 307         private UnlockConnectionDelegate m_ConnectionUnlock;
 308
 309         //
 310         // ReadDone and m_Write - no two vars are so complicated,
 311         //  as these two. Used for m_WriteList managment, most be under crit
 312         //  section when accessing.
 313         //
 314         // ReadDone tracks the item at the end or
 315         //  just recenlty removed from the m_WriteList. While a
 316         //  pending BeginRead is in place, we need this to be false, in
 317         //  order to indicate to tell the WriteDone callback, that we can
 318         //  handle errors/resets.  The only exception is when the m_WriteList
 319         //  is empty, and there are no outstanding requests, then all it can
 320         //  be true.
 321         //
 322         // WriteDone tracks the item just added at the begining of the m_WriteList.
 323         //  this needs to be false while we about to write something, but have not
 324         //  yet begin or finished the write.  Upon completion, its set to true,
 325         //  so that DoneReading/ReadStartNextRequest can close the socket, without fear
 326         //  of a errand writer still banging away on another thread.
 327         //
 328
 329         private DateTime        m_IdleSinceUtc;
 330         private HttpWebRequest  m_LockedRequest;
 331         private HttpWebRequest  m_CurrentRequest; // This is the request whose response is being parsed, same as WriteList[0] but could be different if request was aborted.
 332         private bool m_CanPipeline;
 333         private bool m_Free = true;
 334         private bool m_Idle = true;
 335         private bool m_KeepAlive = true;
 336         private bool m_Pipelining;
 337         private int m_ReservedCount;
 338         private bool m_ReadDone;
 339         private bool m_WriteDone;
 340         private bool m_RemovedFromConnectionList;
 341         private bool m_NonKeepAliveRequestPipelined;
 342
 343         // Pipeline Throttling: m_IsPipelinePaused==true when we stopped and false when it's ok to add to the pipeline.
 344         private bool                m_IsPipelinePaused;
 345         private static int          s_MaxPipelinedCount = 10;
 346         private static int          s_MinPipelinedCount = 5;
 347
 348 #if TRAVE
 349         private bool q_Tunnelling;
 350 #endif
 351
 352         internal override ServicePoint ServicePoint {
 353             get {
 354                 return ConnectionGroup.ServicePoint;
 355             }
 356         }
 357
 358         private ConnectionGroup ConnectionGroup {
 359             get {
 360                 return m_ConnectionGroup;
 361             }
 362         }
 363
 364         //
 365         // LockedRequest is the request that needs exclusive access to this connection
 366         // the ConnectionGroup should proctect the Connection object from any new
 367         // Requests being queued, until this m_LockedRequest is finished.
 368         //
 369         internal HttpWebRequest LockedRequest {
 370             get {
 371                 return m_LockedRequest;
 372             }
 373             set {
 374                 HttpWebRequest myLock = m_LockedRequest;
 375
 376                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::LockedRequest_set() old#"+ ((myLock!=null)?myLock.GetHashCode().ToString():"null") +  " new#" + ((value!=null)?value.GetHashCode().ToString():"null"));
 377
 378                 if ((object)value == (object)myLock)
 379                 {
 380                     if (value != null && (object)value.UnlockConnectionDelegate != (object) m_ConnectionUnlock)
 381                     {
 382                         throw new InternalException();
 383                     }
 384                     return;
 385                 }
 386
 387                 object myDelegate = myLock == null? null: myLock.UnlockConnectionDelegate;
 388                 if (myDelegate != null && (value != null || (object)m_ConnectionUnlock != (object)myDelegate))
 389                     throw new InternalException();
 390
 391                 if (value == null)
 392                 {
 393                     m_LockedRequest = null;
 394                     myLock.UnlockConnectionDelegate = null;
 395                     return;
 396                 }
 397
 398                 UnlockConnectionDelegate chkDelegate = value.UnlockConnectionDelegate;
 399                 //
 400                 // If "value" request was already locking a connection that is not "this", unlock that other connection
 401                 //
 402                 if ((object)chkDelegate != null)
 403                 {
 404                     if ((object)chkDelegate == (object)m_ConnectionUnlock)
 405                         throw new InternalException();
 406
 407                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::LockedRequest_set() Unlocking old request Connection");
 408                     chkDelegate();
 409                 }
 410
 411                 value.UnlockConnectionDelegate = m_ConnectionUnlock;
 412                 m_LockedRequest = value;
 413             }
 414         }
 415
 416
 417         /// <devdoc>
 418         ///    <para>
 419         ///       Delegate called when the request is finished using this Connection
 420         ///         exclusively.  Called in Abort cases and after NTLM authenticaiton completes.
 421         ///    </para>
 422         /// </devdoc>
 423         private void UnlockRequest() {
 424             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::UnlockRequest() LockedRequest#" + ValidationHelper.HashString(LockedRequest));
 425
 426             LockedRequest = null;
 427
 428             if (ConnectionGroup != null) {
 429                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::UnlockRequest() - forcing call to ConnectionGoneIdle()");
 430                 ConnectionGroup.ConnectionGoneIdle();
 431             }
 432         }
 433
 434
 435 #if TRAVE
 436         /*
 437         private string MyLocalEndPoint {
 438             get {
 439                 try {
 440                     return NetworkStream.InternalSocket.LocalEndPoint.ToString();
 441                 }
 442                 catch {
 443                     return "no connection";
 444                 }
 445             }
 446         }
 447         */
 448
 449         private string MyLocalPort {
 450             get {
 451                 try {
 452                     if (NetworkStream == null || !NetworkStream.Connected) {
 453                         return "no connection";
 454                     }
 455                     return ((IPEndPoint)NetworkStream.InternalSocket.LocalEndPoint).Port.ToString();
 456                 }
 457                 catch {
 458                     return "no connection";
 459                 }
 460             }
 461         }
 462 #endif
 463
 464         internal Connection(ConnectionGroup connectionGroup) : base(null) {
 465             //
 466             // add this Connection to the pool in the connection group,
 467             //  keep a weak reference to it
 468             //
 469             m_MaximumUnauthorizedUploadLength = SettingsSectionInternal.Section.MaximumUnauthorizedUploadLength;
 470             if(m_MaximumUnauthorizedUploadLength > 0){
 471                 m_MaximumUnauthorizedUploadLength*=1024;
 472             }
 473             m_ResponseData = new CoreResponseData();
 474             m_ConnectionGroup = connectionGroup;
 475             m_ReadBuffer = s_PinnableBufferCache.AllocateBuffer();
 476             m_ReadBufferFromPinnableCache = true;
 477             m_ReadState = ReadState.Start;
 478             m_WaitList = new List<WaitListItem>();
 479             m_WriteList = new ArrayList();
 480             m_AbortDelegate = new HttpAbortDelegate(AbortOrDisassociate);
 481             m_ConnectionUnlock = new UnlockConnectionDelegate(UnlockRequest);
 482
 483             // for status line parsing
 484             m_StatusLineValues = new StatusLineValues();
 485             m_RecycleTimer = ConnectionGroup.ServicePoint.ConnectionLeaseTimerQueue.CreateTimer();
 486             // the following line must be the last line of the constructor
 487             ConnectionGroup.Associate(this);
 488             m_ReadDone = true;
 489             m_WriteDone = true;
 490             m_Error = WebExceptionStatus.Success;
 491             if (PinnableBufferCacheEventSource.Log.IsEnabled())
 492             {
 493                 PinnableBufferCacheEventSource.Log.DebugMessage1("CTOR: In System.Net.Connection.Connnection", this.GetHashCode());
 494             }
 495         }
 496
 497         ~Connection() {
 498             if (m_ReadBufferFromPinnableCache)
 499             {
 500                 if (PinnableBufferCacheEventSource.Log.IsEnabled())
 501                 {
 502                     PinnableBufferCacheEventSource.Log.DebugMessage1("DTOR: ERROR Needing to Free m_ReadBuffer in Connection Destructor", m_ReadBuffer.GetHashCode());
 503                 }
 504             }
 505             FreeReadBuffer();
 506         }
 507
 508         // If the buffer came from the the pinnable cache, return it to the cache.
 509         // NOTE: This method is called from this object's finalizer and should not access any member objects.
 510         void FreeReadBuffer() {
 511             if (m_ReadBufferFromPinnableCache) {
 512                 s_PinnableBufferCache.FreeBuffer(m_ReadBuffer);
 513                 m_ReadBufferFromPinnableCache = false;
 514             }
 515             m_ReadBuffer = null;
 516         }
 517
 518         protected override void Dispose(bool disposing) {
 519             if (PinnableBufferCacheEventSource.Log.IsEnabled()) {
 520                 PinnableBufferCacheEventSource.Log.DebugMessage1("In System.Net.Connection.Dispose()", this.GetHashCode());
 521             }
 522             FreeReadBuffer();
 523             base.Dispose(disposing);
 524         }
 525
 526         internal int BusyCount {
 527             get {
 528                 return (m_ReadDone?0:1) + 2 * (m_WaitList.Count + m_WriteList.Count) + m_ReservedCount;
 529             }
 530         }
 531
 532         internal int IISVersion{
 533             get{
 534                 return m_IISVersion;
 535             }
 536         }
 537
 538         internal bool AtLeastOneResponseReceived {
 539             get {
 540                 return m_AtLeastOneResponseReceived;
 541             }
 542         }
 543
 544         /*++
 545
 546             SubmitRequest       - Submit a request for sending.
 547
 548             The core submit handler. This is called when a request needs to be
 549             submitted to the network. This routine is asynchronous; the caller
 550             passes in an HttpSubmitDelegate that we invoke when the caller
 551             can use the underlying network. The delegate is invoked with the
 552             stream that it can right to.
 553
 554             On the Sync path, we work by attempting to gain control of the Connection
 555             for writing and reading.  If some other thread is using the Connection,
 556             We wait inside of a LazyAsyncResult until it is availble.
 557
 558
 559             Input:
 560                     request                 - request that's being submitted.
 561                     SubmitDelegate          - Delegate to be invoked.
 562                     forcedsubmit            - Queue the request even if connection is going to close.
 563
 564             Returns:
 565                     true when the request was correctly submitted
 566
 567         --*/
 568         // userReqeustThread says whether we can post IO from this thread or not.
 569         [SuppressMessage("Microsoft.Reliability","CA2002:DoNotLockOnObjectsWithWeakIdentity", Justification="Re-Baseline System violations from 3.5 SP1 due to added parameter")]
 570         internal bool SubmitRequest(HttpWebRequest request, bool forcedsubmit)
 571         {
 572             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest", "request#" + ValidationHelper.HashString(request));
 573             GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest");
 574             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest() Free:" + m_Free.ToString() + " m_WaitList.Count:" + m_WaitList.Count.ToString());
 575
 576             TriState startRequestResult = TriState.Unspecified;
 577             ConnectionReturnResult returnResult = null;
 578             bool expiredIdleConnection = false;
 579
 580             // See if the connection is free, and if the underlying socket or
 581             // stream is set up. If it is, we can assign this connection to the
 582             // request right now. Otherwise we'll have to put this request on
 583             // on the wait list until it its turn.
 584
 585             lock(this)
 586             {
 587                 request.AbortDelegate = m_AbortDelegate;
 588
 589                 if (request.Aborted)
 590                 {
 591                     // Note that request is not on the connection list yet and Abort() will push the response on the request
 592                     GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest - (Request was aborted before being submitted)", true);
 593                     UnlockIfNeeded(request);
 594                     return true;
 595                 }
 596                 //
 597                 // There is a race condition between FindConnection and PrepareCloseConnectionSocket
 598                 // Some request may already try to submit themselves while the connection is dying.
 599                 //
 600                 // Retry if that's the case
 601                 //
 602                 if (!CanBePooled)
 603                 {
 604                     GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest", "false - can't be pooled");
 605                     UnlockIfNeeded(request);
 606                     return false;
 607                 }
 608
 609                 //
 610                 // There is a race condition between SubmitRequest and FindConnection. A non keep-alive request may
 611                 // get submitted on this connection just after we check for it. So make sure that if we are queueing
 612                 // behind non keep-alive request then its a forced submit.
 613                 // Retry if that's not the case.
 614                 //
 615                 if (!forcedsubmit && NonKeepAliveRequestPipelined)
 616                 {
 617                     GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest", "false - behind non keep-alive request");
 618                     UnlockIfNeeded(request);
 619                     return false;
 620                 }
 621
 622                 // See if our timer still matches the SerivcePoint.  If not, get rid of it.
 623                 if (m_RecycleTimer.Duration != ServicePoint.ConnectionLeaseTimerQueue.Duration) {
 624                     m_RecycleTimer.Cancel();
 625                     m_RecycleTimer = ServicePoint.ConnectionLeaseTimerQueue.CreateTimer();
 626                 }
 627
 628                 if (m_RecycleTimer.HasExpired) {
 629                     request.KeepAlive = false;
 630                 }
 631
 632                 //
 633                 // If the connection has already been locked by another request, then
 634                 // we fail the submission on this Connection.
 635                 //
 636
 637                 if (LockedRequest != null && LockedRequest != request) {
 638                     GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest", "false");
 639                     return false;
 640                 }
 641
 642
 643                 //free means no one in the wait list.  We should only add a request
 644                 //if the request can pipeline, or pipelining isn't available
 645
 646                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest WriteDone:" + m_WriteDone.ToString() + ", ReadDone:" + m_ReadDone.ToString() + ", m_WriteList.Count:" + m_WriteList.Count.ToString());
 647
 648                 //
 649                 // If this request is marked as non keep-alive, we should stop pipelining more requests on this
 650                 // connection. The keep-alive context is transfered to the connection from request only after we start
 651                 // receiving response for the request.
 652                 //
 653                 if (!forcedsubmit && !m_NonKeepAliveRequestPipelined) {
 654                     m_NonKeepAliveRequestPipelined = (!request.KeepAlive && !request.NtlmKeepAlive);
 655                 }
 656
 657                 if (m_Free && m_WriteDone && !forcedsubmit && (m_WriteList.Count == 0 || (request.Pipelined && !request.HasEntityBody && m_CanPipeline && m_Pipelining && !m_IsPipelinePaused))) {
 658
 659                     // Connection is free. Mark it as busy and see if underlying
 660                     // socket is up.
 661                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest - Free ");
 662                     m_Free = false;
 663
 664                     // This codepath handles the case where the server has closed the Connection by
 665                     // returning false below: the request will be resubmitted on a different Connection.
 666                     startRequestResult = StartRequest(request, true);
 667                     if (startRequestResult == TriState.Unspecified)
 668                     {
 669                         expiredIdleConnection = true;
 670                         PrepareCloseConnectionSocket(ref returnResult);
 671                         // Hard Close the socket.
 672                         Close(0);
 673                         FreeReadBuffer();       // Do it after close completes to insure buffer not in use
 674                     }
 675                 }
 676                 else {
 677                     m_WaitList.Add(new WaitListItem(request, NetworkingPerfCounters.GetTimestamp()));
 678                     NetworkingPerfCounters.Instance.Increment(NetworkingPerfCounterName.HttpWebRequestQueued);
 679                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest - Request added to WaitList#"+ValidationHelper.HashString(request));
 680 #if TRAVE
 681                     if (q_Tunnelling) {
 682                         GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest() MyLocalPort:" + MyLocalPort + " ERROR: adding HttpWebRequest#" + ValidationHelper.HashString(request) +" to tunnelling WaitList");
 683                     }
 684                     else {
 685                         GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest() MyLocalPort:" + MyLocalPort + " adding HttpWebRequest#" + ValidationHelper.HashString(request) +" to non-tunnelling WaitList m_WaitList.Count:" + m_WaitList.Count);
 686                     }
 687 #endif
 688                     CheckNonIdle();
 689                 }
 690             }
 691
 692             if (expiredIdleConnection)
 693             {
 694                 GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest(), expired idle connection", false);
 695                 ConnectionReturnResult.SetResponses(returnResult);
 696                 return false;
 697             }
 698
 699             GlobalLog.DebugAddRequest(request, this, 0);
 700             if(Logging.On)Logging.Associate(Logging.Web, this, request);
 701
 702             if (startRequestResult != TriState.Unspecified) {
 703                 CompleteStartRequest(true, request, startRequestResult);
 704             }
 705             // On [....], we wait for the Connection to be come availble here,
 706             if (!request.Async)
 707             {
 708                 object responseObject = request.ConnectionAsyncResult.InternalWaitForCompletion();
 709                 ConnectStream writeStream = responseObject as ConnectStream;
 710                 AsyncTriState triStateAsync = null;
 711                 if (writeStream == null)
 712                     triStateAsync = responseObject as AsyncTriState;
 713
 714                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest() Pipelining:"+m_Pipelining);
 715
 716                 if (startRequestResult == TriState.Unspecified && triStateAsync != null) {
 717                     // May need to recreate Connection here (i.e. call Socket.Connect)
 718                     CompleteStartRequest(true, request, triStateAsync.Value);
 719                 }
 720                 else if (writeStream != null)
 721                 {
 722                     // return the Stream to the Request
 723                     request.SetRequestSubmitDone(writeStream);
 724                 }
 725 #if DEBUG
 726                 else if (responseObject is Exception)
 727                 {
 728                     Exception exception = responseObject as Exception;
 729                     WebException webException = responseObject as WebException;
 730                     GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest (SYNC) - Error waiting for a connection: " + exception.Message,
 731                                     "Status:" + (webException == null? exception.GetType().FullName: (webException.Status.ToString() +  " Internal Status: " + webException.InternalStatus.ToString())));
 732                     return true;
 733                 }
 734 #endif
 735             }
 736
 737             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest", true);
 738             return true;
 739         }
 740
 741         private void UnlockIfNeeded(HttpWebRequest request) {
 742             if (LockedRequest == request) {
 743                 UnlockRequest();
 744             }
 745         }
 746
 747         // Wrapper for TriState for marhshalling across Thread boundaries
 748         private class AsyncTriState {
 749             public TriState Value;
 750             public AsyncTriState(TriState newValue) {
 751                 Value = newValue;
 752             }
 753         }
 754
 755         /*++
 756
 757             StartRequest       - Start a request going.
 758
 759             Routine to start a request. Called when we know the connection is
 760             free and we want to get a request going. This routine initializes
 761             some state, adds the request to the write queue, and checks to
 762             see whether or not the underlying connection is alive. If it's
 763             not, it queues a request to get it going. If the connection
 764             was alive we call the callback delegate of the request.
 765
 766             This routine MUST BE called with the critcal section held.
 767
 768             Input:
 769                     request                 - request that's being started.
 770                     canPollRead             - whether the calling code handles
 771                                               Unspecified due to the Connection
 772                                               being closed by the server.
 773
 774             Returns:
 775                     True if request was started, false otherwise.
 776
 777         --*/
 778
 779         private TriState StartRequest(HttpWebRequest request, bool canPollRead)
 780         {
 781             GlobalLog.Enter(
 782                 "Connection#" + ValidationHelper.HashString(this) +
 783                 "::StartRequest",
 784                 "HttpWebRequest#" + ValidationHelper.HashString(request) +
 785                 " WriteDone:"+ m_WriteDone +
 786                 " ReadDone:" + m_ReadDone +
 787                 " WaitList:" + m_WaitList.Count +
 788                 " WriteList:" + m_WriteList.Count);
 789             GlobalLog.ThreadContract(ThreadKinds.Unknown,
 790                 "Connection#" + ValidationHelper.HashString(this) +
 791                 "::StartRequest");
 792
 793             if (m_WriteList.Count == 0)
 794             {
 795                 // check if we consider connection timed out
 796                 if (ServicePoint.MaxIdleTime != -1 &&
 797                     m_IdleSinceUtc != DateTime.MinValue &&
 798                     m_IdleSinceUtc +
 799                         TimeSpan.FromMilliseconds(ServicePoint.MaxIdleTime) <
 800                         DateTime.UtcNow)
 801                 {
 802                     // This idle keep-alive connection timed out.
 803                     GlobalLog.Leave(
 804                         "Connection#" + ValidationHelper.HashString(this) +
 805                         "::StartRequest()" +
 806                         " Expired connection was idle for "
 807                         + (int)
 808                             ((DateTime.UtcNow - m_IdleSinceUtc).TotalSeconds) +
 809                         " secs, request will be retried: #" +
 810                         ValidationHelper.HashString(request));
 811                     return TriState.Unspecified; // don't use it
 812                 } else if (canPollRead) {
 813                     // Not timed out from our perspective but...
 814                     // Check if remote has:
 815                     //   1) closed an idle connection (TCP FIN)
 816                     // or
 817                     //   2) sent some errant data on an idle connection.
 818                     bool pollRead = PollRead();
 819                     if (pollRead) {
 820                         GlobalLog.Leave(
 821                             "Connection#" + ValidationHelper.HashString(this) +
 822                             "::StartRequest() " +
 823                             "Idle connection remotely closed, " +
 824                             "request will be retried: #" +
 825                             ValidationHelper.HashString(request));
 826                         return TriState.Unspecified; // don't use it
 827                     }
 828                 }
 829             }
 830
 831             TriState needReConnect = TriState.False;
 832              // Starting a request means the connection is not idle anymore
 833             m_IdleSinceUtc = DateTime.MinValue;
 834
 835              // Initialze state, and add the request to the write queue.
 836
 837              //
 838              // Note that m_Pipelining shold be only set here but the sanity check is made by the caller
 839              // means if the caller has found that it is safe to pipeline the below result must be true as well
 840              //
 841             if (!m_IsPipelinePaused)
 842                 m_IsPipelinePaused = m_WriteList.Count >= s_MaxPipelinedCount;
 843
 844             m_Pipelining = m_CanPipeline && request.Pipelined && (!request.HasEntityBody);
 845
 846             // start of write process, disable done-ness flag
 847             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::StartRequest() setting WriteDone:" + m_WriteDone.ToString() + " to false");
 848             m_WriteDone = false;
 849             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::StartRequest() m_WriteList adding HttpWebRequest#" + ValidationHelper.HashString(request));
 850             m_WriteList.Add(request);
 851
 852             GlobalLog.Print(m_WriteList.Count+" requests queued");
 853             CheckNonIdle();
 854
 855             // with no network stream around, we will have to create one, therefore, we can't have
 856             //  the possiblity to even have a DoneReading().
 857
 858             if (IsInitalizing)
 859                 needReConnect = TriState.True;
 860
 861 #if TRAVE
 862             if (request.IsTunnelRequest) {
 863                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::StartRequest() MyLocalPort:" + MyLocalPort + " setting Tunnelling to true HttpWebRequest#" + ValidationHelper.HashString(request));
 864                 q_Tunnelling = true;
 865             }
 866             else {
 867                 GlobalLog.Assert(!q_Tunnelling, "Connection#{0}::StartRequest()|MyLocalPort:{1} ERROR: Already tunnelling during non-tunnel request HttpWebRequest#{2}.", ValidationHelper.HashString(this), MyLocalPort, ValidationHelper.HashString(request));
 868             }
 869 #endif
 870
 871             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::StartRequest", needReConnect.ToString());
 872             return needReConnect;
 873         }
 874
 875         private void CompleteStartRequest(bool onSubmitThread, HttpWebRequest request, TriState needReConnect) {
 876             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::CompleteStartRequest", ValidationHelper.HashString(request));
 877             GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection#" + ValidationHelper.HashString(this) + "::CompleteStartRequest");
 878
 879             if (needReConnect == TriState.True) {
 880                 // Socket is not alive.
 881
 882                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CompleteStartRequest() Queue StartConnection Delegate ");
 883                 try {
 884                     if (request.Async) {
 885                         CompleteStartConnection(true, request);
 886                     }
 887                     else if (onSubmitThread) {
 888                         CompleteStartConnection(false, request);
 889                     }
 890                     // else - fall through and wake up other thread
 891                 }
 892                 catch (Exception exception) {
 893                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CompleteStartRequest(): exception: " + exception.ToString());
 894                     if (NclUtilities.IsFatal(exception)) throw;
 895                     //
 896                     // Should not be here because CompleteStartConnection and below tries to catch everything
 897                     //
 898                     GlobalLog.Assert(exception.ToString());
 899                 }
 900
 901                 // If neeeded wake up other thread where SubmitRequest was called
 902                 if (!request.Async) {
 903                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CompleteStartRequest() Invoking Async Result");
 904                     request.ConnectionAsyncResult.InvokeCallback(new AsyncTriState(needReConnect));
 905                 }
 906
 907
 908                 GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::CompleteStartRequest", "needReConnect");
 909                 return;
 910             }
 911
 912
 913             //
 914             // From now on the request.SetRequestSubmitDone must be called or it may hang
 915             // For a [....] request the write side reponse windowwas opened in HttpWebRequest.SubmitRequest
 916             if (request.Async)
 917                 request.OpenWriteSideResponseWindow();
 918
 919
 920             ConnectStream writeStream = new ConnectStream(this, request);
 921
 922             // Call the request to let them know that we have a write-stream, this might invoke Send() call
 923             if (request.Async || onSubmitThread) {
 924                 request.SetRequestSubmitDone(writeStream);
 925             }
 926             else {
 927                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CompleteStartRequest() Invoking Async Result");
 928                 request.ConnectionAsyncResult.InvokeCallback(writeStream);
 929             }
 930             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::CompleteStartRequest");
 931         }
 932
 933         /*++
 934
 935             CheckNextRequest
 936
 937             Gets the next request from the wait queue, if there is one.
 938
 939             Must be called with the crit sec held.
 940
 941
 942         --*/
 943         private HttpWebRequest CheckNextRequest()
 944         {
 945         GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection#" + ValidationHelper.HashString(this) + "::CheckNextRequest");
 946
 947             if (m_WaitList.Count == 0) {
 948                 // We're free now, if we're not going to close the connection soon.
 949                 m_Free = m_KeepAlive;
 950                 return null;
 951             }
 952             if (!CanBePooled) {
 953                 return null;
 954             }
 955
 956             WaitListItem item = m_WaitList[0];
 957             HttpWebRequest nextRequest = item.Request;
 958
 959             if (m_IsPipelinePaused)
 960                 m_IsPipelinePaused = m_WriteList.Count > s_MinPipelinedCount;
 961
 962             if (!nextRequest.Pipelined || nextRequest.HasEntityBody || !m_CanPipeline || !m_Pipelining || m_IsPipelinePaused) {
 963                 if (m_WriteList.Count != 0) {
 964                     nextRequest = null;
 965                 }
 966             }
 967             if (nextRequest != null) {
 968                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CheckNextRequest() Removing request#" + ValidationHelper.HashString(nextRequest) + " from m_WaitList. New Count:" + (m_WaitList.Count - 1).ToString());
 969
 970                 NetworkingPerfCounters.Instance.IncrementAverage(NetworkingPerfCounterName.HttpWebRequestAvgQueueTime,
 971                     item.QueueStartTime);
 972                 m_WaitList.RemoveAt(0);
 973                 CheckIdle();
 974             }
 975             return nextRequest;
 976         }
 977
 978         private void CompleteStartConnection(bool async, HttpWebRequest httpWebRequest)
 979         {
 980             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::CompleteStartConnection",  "async:" + async.ToString() + " httpWebRequest:" + ValidationHelper.HashString(httpWebRequest));
 981             GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection#" + ValidationHelper.HashString(this) + "::CompleteStartConnection");
 982
 983             WebExceptionStatus ws = WebExceptionStatus.ConnectFailure;
 984             m_InnerException = null;
 985             bool success = true;
 986
 987             try {
 988 #if DEBUG
 989                 lock (this)
 990                 {
 991                     // m_WriteList can be empty if request got aborted.  In that case no new requests can come in so it should remain zero.
 992                     if (m_WriteList.Count != 0)
 993                     {
 994                         GlobalLog.Assert(m_WriteList.Count == 1, "Connection#{0}::CompleteStartConnection()|WriteList is not sized 1.", ValidationHelper.HashString(this));
 995                         GlobalLog.Assert((m_WriteList[0] as HttpWebRequest) == httpWebRequest, "Connection#{0}::CompleteStartConnection()|Last request on write list does not match.", ValidationHelper.HashString(this));
 996                     }
 997                 }
 998 #endif
 999
1000                 //
1001                 // we will create a tunnel through a proxy then create
1002                 // and connect the socket we will use for the connection
1003                 // otherwise we will just create a socket and use it
1004                 //
1005                 if ((httpWebRequest.IsWebSocketRequest || httpWebRequest.Address.Scheme == Uri.UriSchemeHttps) &&
1006                     ServicePoint.InternalProxyServicePoint)
1007                 {
1008                     if(!TunnelThroughProxy(ServicePoint.InternalAddress, httpWebRequest,async)) {
1009                         ws = WebExceptionStatus.ConnectFailure;
1010                         success = false;
1011                     }
1012                     if (async && success) {
1013                         return;
1014                     }
1015                 } else {
1016                     if (!Activate(httpWebRequest, async, new GeneralAsyncDelegate(CompleteConnectionWrapper)))
1017                     {
1018                         return;
1019                     }
1020                 }
1021             }
1022             catch (Exception exception) {
1023                 if (m_InnerException == null)
1024                     m_InnerException = exception;
1025
1026                 if (exception is WebException) {
1027                     ws = ((WebException)exception).Status;
1028                 }
1029                 success = false;
1030             }
1031             if(!success)
1032             {
1033                 ConnectionReturnResult returnResult = null;
1034                 HandleError(false, false, ws, ref returnResult);
1035                 ConnectionReturnResult.SetResponses(returnResult);
1036                 GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::CompleteStartConnection Failed to connect.");
1037                 return;
1038             }
1039
1040             // Getting here means we connected synchronously.  Continue with the next step.
1041
1042             CompleteConnection(async, httpWebRequest);
1043             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::CompleteStartConnection");
1044         }
1045
1046         private void CompleteConnectionWrapper(object request, object state)
1047         {
1048 #if DEBUG
1049             using (GlobalLog.SetThreadKind(ThreadKinds.System | ThreadKinds.Async)) {
1050 #endif
1051             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(state) + "::CompleteConnectionWrapper", "request:" + ValidationHelper.HashString(request));
1052
1053             Exception stateException = state as Exception;
1054             if (stateException != null)
1055             {
1056                 GlobalLog.Print("CompleteConnectionWrapper() Request#" + ValidationHelper.HashString(request) + " Connection is in error: " + stateException.ToString());
1057                 ConnectionReturnResult returnResult = null;
1058
1059                 if (m_InnerException == null)
1060                     m_InnerException = stateException;
1061
1062                 HandleError(false, false, WebExceptionStatus.ConnectFailure, ref returnResult);
1063                 ConnectionReturnResult.SetResponses(returnResult);
1064             }
1065             CompleteConnection(true, (HttpWebRequest) request);
1066
1067             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(state) + "::CompleteConnectionWrapper" + (stateException == null? string.Empty: " failed"));
1068 #if DEBUG
1069             }
1070 #endif
1071         }
1072
1073         private void CompleteConnection(bool async, HttpWebRequest request)
1074         {
1075             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::CompleteConnection", "async:" + async.ToString() + " request:" + ValidationHelper.HashString(request));
1076             GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection#" + ValidationHelper.HashString(this) + "::CompleteConnection");
1077
1078             WebExceptionStatus ws = WebExceptionStatus.ConnectFailure;
1079             //
1080             // From now on the request.SetRequestSubmitDone must be called or it may hang
1081             // For a [....] request the write side reponse windowwas opened in HttpWebRequest.SubmitRequest
1082             if (request.Async)
1083                 request.OpenWriteSideResponseWindow();
1084
1085             try
1086             {
1087                 try {
1088 #if !FEATURE_PAL
1089                     if (request.Address.Scheme == Uri.UriSchemeHttps) {
1090                         TlsStream tlsStream = new TlsStream(request.GetRemoteResourceUri().IdnHost,
1091                             NetworkStream, request.ClientCertificates, ServicePoint, request,
1092                             request.Async ? request.GetConnectingContext().ContextCopy : null);
1093                         NetworkStream = tlsStream;
1094                     }
1095 #endif
1096                     ws = WebExceptionStatus.Success;
1097                 }
1098                 catch {
1099                     // The TLS stream could not be created.  Close the current non-TLS stream immediately
1100                     // to prevent any future use of it.  Due to race conditions, the error handling will sometimes
1101                     // try to write (flush) out some of the HTTP headers to the stream as it is closing down the failed
1102                     // HttpWebRequest. This would cause plain text to go on the wire even though the stream should
1103                     // have been TLS encrypted.
1104                     NetworkStream.Close();
1105                     throw;
1106                 }
1107                 finally {
1108                     //
1109                     // There is a ---- with Abort so TlsStream ctor may throw.
1110                     // SetRequestSubmitDone will deal with this kind of errors.
1111                     //
1112
1113                     m_ReadState = ReadState.Start;
1114                     ClearReaderState();
1115
1116                     request.SetRequestSubmitDone(new ConnectStream(this, request));
1117                 }
1118             }
1119             catch (Exception exception)
1120             {
1121                 if (m_InnerException == null)
1122                     m_InnerException = exception;
1123                 WebException webException = exception as WebException;
1124                 if (webException != null)
1125                 {
1126                     ws = webException.Status;
1127                 }
1128             }
1129
1130             if (ws != WebExceptionStatus.Success)
1131             {
1132                 ConnectionReturnResult returnResult = null;
1133                 HandleError(false, false, ws, ref returnResult);
1134                 ConnectionReturnResult.SetResponses(returnResult);
1135
1136                 if (Logging.On) Logging.PrintError(Logging.Web, this, "CompleteConnection", "on error");
1137                 GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::CompleteConnection", "on error");
1138             }
1139             else
1140             {
1141                 GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::CompleteConnection");
1142             }
1143         }
1144
1145         private void InternalWriteStartNextRequest(HttpWebRequest request, ref bool calledCloseConnection, ref TriState startRequestResult, ref HttpWebRequest nextRequest, ref ConnectionReturnResult returnResult) {
1146             GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection#" + ValidationHelper.HashString(this) + "::InternalWriteStartNextRequest");
1147
1148             lock(this) {
1149
1150                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::WriteStartNextRequest() setting WriteDone:" + m_WriteDone.ToString() + " to true");
1151                 m_WriteDone = true;
1152
1153                 //
1154                 // If we're not doing keep alive, and the read on this connection
1155                 // has already completed, now is the time to close the
1156                 // connection.
1157                 //
1158                 //need to wait for read to set the error
1159                 if (!m_KeepAlive || m_Error != WebExceptionStatus.Success || !CanBePooled) {
1160                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::WriteStartNextRequest() m_WriteList.Count:" + m_WriteList.Count);
1161                     if (m_ReadDone) {
1162                         // We could be closing because of an unexpected keep-alive
1163                         // failure, ie we pipelined a few requests and in the middle
1164                         // the remote server stopped doing keep alive. In this
1165                         // case m_Error could be success, which would be misleading.
1166                         // So in that case we'll set it to connection closed.
1167
1168                         if (m_Error == WebExceptionStatus.Success) {
1169                             // Only reason we could have gotten here is because
1170                             // we're not keeping the connection alive.
1171                             m_Error = WebExceptionStatus.KeepAliveFailure;
1172                         }
1173
1174                         // PrepareCloseConnectionSocket is called with the critical section
1175                         // held. Note that we know since it's not a keep-alive
1176                         // connection the read half wouldn't have posted a receive
1177                         // for this connection, so it's OK to call PrepareCloseConnectionSocket now.
1178                         PrepareCloseConnectionSocket(ref returnResult);
1179                         calledCloseConnection = true;
1180                         Close();
1181                     }
1182                     else {
1183                         if (m_Error!=WebExceptionStatus.Success) {
1184                             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::WriteStartNextRequest() a Failure, m_Error = " + m_Error.ToString());
1185                         }
1186                     }
1187                 }
1188                 else {
1189                     // If we're pipelining, we get get the next request going
1190                     // as soon as the write is done. Otherwise we have to wait
1191                     // until both read and write are done.
1192
1193
1194                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::WriteStartNextRequest() Non-Error m_WriteList.Count:" + m_WriteList.Count + " m_WaitList.Count:" + m_WaitList.Count);
1195
1196                     if (m_Pipelining || m_ReadDone)
1197                     {
1198                         nextRequest = CheckNextRequest();
1199                     }
1200                     if (nextRequest != null)
1201                     {
1202                         // This codepath doesn't handle the case where the server has closed the Connection because we
1203                         // just finished using it and didn't get a Connection: close header.
1204                         startRequestResult = StartRequest(nextRequest, false);
1205                         GlobalLog.Assert(startRequestResult != TriState.Unspecified, "WriteStartNextRequest got TriState.Unspecified from StartRequest, things are about to hang!");
1206                     }
1207                 }
1208             } // lock
1209         }
1210
1211         internal void WriteStartNextRequest(HttpWebRequest request, ref ConnectionReturnResult returnResult) {
1212             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::WriteStartNextRequest" + " WriteDone:" + m_WriteDone + " ReadDone:" + m_ReadDone + " WaitList:" + m_WaitList.Count + " WriteList:" + m_WriteList.Count);
1213             GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection#" + ValidationHelper.HashString(this) + "::WriteStartNextRequest");
1214
1215             TriState startRequestResult = TriState.Unspecified;
1216             HttpWebRequest nextRequest = null;
1217             bool calledCloseConnection = false;
1218
1219             InternalWriteStartNextRequest(request, ref calledCloseConnection, ref startRequestResult, ref nextRequest, ref returnResult);
1220
1221             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::WriteStartNextRequest: Pipelining:" + m_Pipelining + " nextRequest#"+ValidationHelper.HashString(nextRequest));
1222
1223             if (!calledCloseConnection && startRequestResult != TriState.Unspecified)
1224             {
1225                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::WriteStartNextRequest calling CompleteStartRequest");
1226                 CompleteStartRequest(false, nextRequest, startRequestResult);
1227             }
1228
1229             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::WriteStartNextRequest");
1230         }
1231
1232
1233         internal void SetLeftoverBytes(byte[] buffer, int bufferOffset, int bufferCount)
1234         {
1235             // The ConnectStream read past the response of its HTTP response (can happen in chunked scenarios).
1236             // Get the buffer containing bytes belonging to the next request and use them for the next request.
1237
1238             if (bufferOffset > 0)
1239             {
1240                 // We need to move leftover bytes to the beginning of the buffer.
1241                 Buffer.BlockCopy(buffer, bufferOffset, buffer, 0, bufferCount);
1242             }
1243
1244             // If we had to reallocate the buffer, we are going to clobber the one that was allocated from the pin friendly cache.
1245             // give it back
1246             if (m_ReadBuffer != buffer)
1247             {
1248                 // if m_ReadBuffer is from the pinnable cache, give it back
1249                 FreeReadBuffer();
1250                 m_ReadBuffer = buffer;
1251             }
1252
1253             m_BytesScanned = 0;
1254             m_BytesRead = bufferCount;
1255         }
1256
1257         /*++
1258
1259             ReadStartNextRequest
1260
1261             This method is called by a stream interface when it's done reading.
1262             We might possible free up the connection for another request here.
1263
1264             Called when we think we might need to start another request because
1265             a read completed.
1266
1267         --*/
1268         internal void ReadStartNextRequest(WebRequest currentRequest, ref ConnectionReturnResult returnResult)
1269         {
1270             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::ReadStartNextRequest" + " WriteDone:" + m_WriteDone + " ReadDone:" + m_ReadDone + " WaitList:" + m_WaitList.Count + " WriteList:" + m_WriteList.Count);
1271             GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection#" + ValidationHelper.HashString(this) + "::ReadStartNextRequest");
1272
1273             HttpWebRequest nextRequest = null;
1274             TriState startRequestResult = TriState.Unspecified;
1275             bool calledCloseConnection = false;
1276             bool mustExit = false;
1277
1278             // ReadStartNextRequest is called by ConnectStream.CallDone: This guarantees that the request
1279             // is done and the response (response stream) was closed. Remove the reservation for the request.
1280             int currentCount = Interlocked.Decrement(ref m_ReservedCount);
1281             GlobalLog.Assert(currentCount >= 0, "m_ReservedCount must not be < 0 when decremented.");
1282
1283             try {
1284                 lock(this) {
1285                     if (m_WriteList.Count > 0 && (object)currentRequest == m_WriteList[0])
1286                     {
1287                         // advance back to state 0
1288                         m_ReadState = ReadState.Start;
1289                         m_WriteList.RemoveAt(0);
1290
1291                         // Must reset ConnectStream here to prevent a leak through the stream of the last request on each connection.
1292                         m_ResponseData.m_ConnectStream = null;
1293
1294                         GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadStartNextRequest() Removed request#" + ValidationHelper.HashString(currentRequest) + " from m_WriteList. New m_WriteList.Count:" + m_WriteList.Count.ToString());
1295                     }
1296                     else
1297                     {
1298                         GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadStartNextRequest() The request#" + ValidationHelper.HashString(currentRequest) + " was disassociated so do nothing.  m_WriteList.Count:" + m_WriteList.Count.ToString());
1299                         mustExit = true;;
1300                     }
1301
1302                     //
1303                     // Since this is called after we're done reading the current
1304                     // request, if we're not doing keepalive and we're done
1305                     // writing we can close the connection now.
1306                     //
1307                     if(!mustExit)
1308                     {
1309                         //
1310                         // m_ReadDone==true is implied because we just finished a request but really the value must still be false here
1311                         //
1312                         if (m_ReadDone)
1313                             throw new InternalException();  // other requests may already started reading on this connection, need a QFE
1314
1315                         if (!m_KeepAlive || m_Error != WebExceptionStatus.Success || !CanBePooled)
1316                         {
1317                             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadStartNextRequest() KeepAlive:" + m_KeepAlive + " WriteDone:" + m_WriteDone);
1318                             // Finished one request and connection is closing.
1319                             // We will not read from this connection so set readDone = true
1320                             m_ReadDone = true;
1321
1322                             if (m_WriteDone)
1323                             {
1324
1325                                 // We could be closing because of an unexpected keep-alive
1326                                 // failure, ie we pipelined a few requests and in the middle
1327                                 // the remote server stopped doing keep alive. In this
1328                                 // case m_Error could be success, which would be misleading.
1329                                 // So in that case we'll set it to KeepAliveFailure.
1330
1331                                 if (m_Error == WebExceptionStatus.Success) {
1332                                     // Only reason we could have gotten here is because
1333                                     // we're not keeping the connection alive.
1334                                     m_Error = WebExceptionStatus.KeepAliveFailure;
1335                                 }
1336
1337                                 // PrepareCloseConnectionSocket has to be called with the critical section held.
1338                                 PrepareCloseConnectionSocket(ref returnResult);
1339                                 calledCloseConnection = true;
1340                                 Close();
1341                             }
1342                         }
1343                         else
1344                         {
1345                             // We try to sort out KeepAliveFailure thing (search by context)
1346                             m_AtLeastOneResponseReceived = true;
1347
1348                             if (m_WriteList.Count != 0)
1349                             {
1350                                 // If a *pipelined* request that is being submitted has finished with the headers, post a receive
1351                                 nextRequest = m_WriteList[0] as HttpWebRequest;
1352                                 // If the active request has not finished its headers we can set m_ReadDone = true
1353                                 // and that will be changed when said request will call CheckStartReceive
1354                                 if (!nextRequest.HeadersCompleted)
1355                                 {
1356                                     nextRequest = null;
1357                                     m_ReadDone = true;
1358                                 }
1359                             }
1360                             // If there are no requests left to write (means pipeline),
1361                             // we can get the next request from wait list going now.
1362                             else
1363                             {
1364                                 m_ReadDone = true;
1365
1366                                 // Sometime we get a response before completing the body in which case
1367                                 // we defer next request to WriteStartNextRequest
1368                                 if (m_WriteDone)
1369                                 {
1370                                     nextRequest = CheckNextRequest();
1371
1372                                     if (nextRequest != null )
1373                                     {
1374                                         // We cannot have HeadersCompleted on the request that was not placed yet on the write list
1375                                         if(nextRequest.HeadersCompleted) // TODO: change to be Assert but only when stress got stable-stable
1376                                             throw new InternalException();
1377
1378                                         // This codepath doesn't handle the case where the server has closed the
1379                                         // Connection because we just finished using it and didn't get a
1380                                         // Connection: close header.
1381                                         startRequestResult = StartRequest(nextRequest, false);
1382                                         GlobalLog.Assert(startRequestResult != TriState.Unspecified, "ReadStartNextRequest got TriState.Unspecified from StartRequest, things are about to hang!");
1383                                     }
1384                                     else
1385                                     {
1386                                         //There are no other requests to process, so make connection avaliable for all
1387                                         m_Free = true;
1388                                     }
1389                                 }
1390                             }
1391                         }
1392                     }
1393                 }
1394             }
1395             finally
1396             {
1397                 CheckIdle();
1398                 //set result here to prevent nesting of readstartnextrequest.
1399                 if(returnResult != null){
1400                     ConnectionReturnResult.SetResponses(returnResult);
1401                 }
1402             }
1403
1404             if(!mustExit && !calledCloseConnection)
1405             {
1406                 if (startRequestResult != TriState.Unspecified)
1407                 {
1408                     CompleteStartRequest(false, nextRequest, startRequestResult);
1409                 }
1410                 else if (nextRequest != null)
1411                 {
1412                     // Handling receive, note that is for pipelinning case only !
1413                     if (!nextRequest.Async)
1414                     {
1415                         nextRequest.ConnectionReaderAsyncResult.InvokeCallback();
1416                     }
1417                     else
1418                     {
1419                         if (m_BytesScanned < m_BytesRead)
1420                         {
1421                             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadStartNextRequest() Calling ReadComplete, bytes unparsed = " + (m_BytesRead - m_BytesScanned));
1422                             ReadComplete(0, WebExceptionStatus.Success);
1423                         }
1424                         else if (Thread.CurrentThread.IsThreadPoolThread)
1425                         {
1426                             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadStartNextRequest() Calling PostReceive().");
1427                             PostReceive();
1428                         }
1429                         else
1430                         {
1431                             // Offload to the threadpool to protect against the case where one request's thread posts IO that another request
1432                             // depends on, but the first thread dies in the mean time.
1433                             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadStartNextRequest() ThreadPool.UnsafeQueueUserWorkItem(m_PostReceiveDelegate, this)");
1434                             ThreadPool.UnsafeQueueUserWorkItem(m_PostReceiveDelegate, this);
1435                         }
1436                     }
1437                 }
1438             }
1439             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::ReadStartNextRequest");
1440         }
1441
1442         internal void MarkAsReserved()
1443         {
1444             // We use an interlock here rather than a lock() to avoid deadlocks in the following situation:
1445             // - ConnectionGroup is holding lock(obj1), calls into Connection which is waiting for lock(obj2)
1446             // - on another thread Connection is holding lock(obj2) and calls into ConnectionGroup which will wait
1447             //   for lock(obj1).
1448             int currentCount = Interlocked.Increment(ref m_ReservedCount);
1449             GlobalLog.Assert(currentCount > 0, "m_ReservedCount must not be less or equal zero after incrementing.");
1450         }
1451
1452         //
1453         //
1454         //
1455         internal void CheckStartReceive(HttpWebRequest request)
1456         {
1457             lock (this)
1458             {
1459                 request.HeadersCompleted = true;
1460                 if (m_WriteList.Count == 0)
1461                 {
1462                     // aborted request, was already dispatched.
1463                     // Note it could have been aborted softly if not the first one in the pipeline
1464                     return;
1465                 }
1466
1467                 // Note we do NOT allow receive if pipelining and the passed request is not the first one on the write queue
1468                 if (!m_ReadDone || m_WriteList[0] != (object)request)
1469                 {
1470                     // ReadStartNextRequest should take care of these cases
1471                     return;
1472                 }
1473                 // Start a receive
1474                 m_ReadDone = false;
1475                 m_CurrentRequest = (HttpWebRequest)m_WriteList[0];
1476             }
1477
1478             if (!request.Async)
1479             {
1480                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CheckStartReceive() SYNC request, calling ConnectionReaderAsyncResult.InvokeCallback()");
1481                 request.ConnectionReaderAsyncResult.InvokeCallback();
1482             }
1483             else if (m_BytesScanned < m_BytesRead)
1484             {
1485                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CheckStartReceive() Calling ReadComplete, bytes unparsed = " + (m_BytesRead - m_BytesScanned));
1486                 ReadComplete(0, WebExceptionStatus.Success);
1487             }
1488             else if (Thread.CurrentThread.IsThreadPoolThread)
1489             {
1490                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CheckStartReceive() Calling PostReceive().");
1491                 PostReceive();
1492             }
1493             else
1494             {
1495                 // Offload to the threadpool to protect against the case where one request's thread posts IO that another request
1496                 // depends on, but the first thread dies in the mean time.
1497                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CheckStartReceive() ThreadPool.UnsafeQueueUserWorkItem(m_PostReceiveDelegate, this)");
1498                 ThreadPool.UnsafeQueueUserWorkItem(m_PostReceiveDelegate, this);
1499             }
1500         }
1501
1502         /*++
1503
1504         Routine Description:
1505
1506            Clears out common member vars used for Status Line parsing
1507
1508         Arguments:
1509
1510            None.
1511
1512         Return Value:
1513
1514            None.
1515
1516         --*/
1517
1518         private void InitializeParseStatusLine() {
1519             m_StatusState = BeforeVersionNumbers;
1520             m_StatusLineValues.MajorVersion = 0;
1521             m_StatusLineValues.MinorVersion = 0;
1522             m_StatusLineValues.StatusCode = 0;
1523             m_StatusLineValues.StatusDescription = null;
1524         }
1525
1526         /*++
1527
1528         Routine Description:
1529
1530            Performs status line parsing on incomming server responses
1531
1532         Arguments:
1533
1534            statusLine - status line that we wish to parse
1535            statusLineLength - length of the array
1536            statusLineInts - array of ints contanes result
1537            statusDescription - string with discription
1538            statusStatus     - state stored between parse attempts
1539
1540         Return Value:
1541
1542            bool - Success true/false
1543
1544         --*/
1545
1546         private const int BeforeVersionNumbers = 0;
1547         private const int MajorVersionNumber   = 1;
1548         private const int MinorVersionNumber   = 2;
1549         private const int StatusCodeNumber     = 3;
1550         private const int AfterStatusCode      = 4;
1551         private const int AfterCarriageReturn  = 5;
1552
1553         private const string BeforeVersionNumberBytes = "HTTP/";
1554
1555         private DataParseStatus ParseStatusLine(
1556                 byte [] statusLine,
1557                 int statusLineLength,
1558                 ref int bytesParsed,
1559                 ref int [] statusLineInts,
1560                 ref string statusDescription,
1561                 ref int statusState,
1562                 ref WebParseError parseError) {
1563             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::ParseStatusLine", statusLineLength.ToString(NumberFormatInfo.InvariantInfo) + ", " + bytesParsed.ToString(NumberFormatInfo.InvariantInfo) +", " +statusState.ToString(NumberFormatInfo.InvariantInfo));
1564             GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection#" + ValidationHelper.HashString(this) + "::ParseStatusLine");
1565             GlobalLog.Assert((statusLineLength - bytesParsed) >= 0, "Connection#{0}::ParseStatusLine()|(statusLineLength - bytesParsed) < 0", ValidationHelper.HashString(this));
1566             //GlobalLog.Dump(statusLine, bytesParsed, statusLineLength);
1567
1568             DataParseStatus parseStatus = DataParseStatus.Done;
1569             int statusLineSize = 0;
1570             int startIndexStatusDescription = -1;
1571             int lastUnSpaceIndex = 0;
1572
1573             //
1574             // While walking the Status Line looking for terminating \r\n,
1575             //   we extract the Major.Minor Versions and Status Code in that order.
1576             //   text and spaces will lie between/before/after the three numbers
1577             //   but the idea is to remember which number we're calculating based on a numeric state
1578             //   If all goes well the loop will churn out an array with the 3 numbers plugged in as DWORDs
1579             //
1580
1581             while ((bytesParsed < statusLineLength) && (statusLine[bytesParsed] != '\r') && (statusLine[bytesParsed] != '\n')) {
1582
1583                 // below should be wrapped in while (response[i] != ' ') to be more robust???
1584                 switch (statusState) {
1585                     case BeforeVersionNumbers:
1586                         if (statusLine[bytesParsed] == '/') {
1587                             //INET_ASSERT(statusState == BeforeVersionNumbers);
1588                             statusState++; // = MajorVersionNumber
1589                         }
1590                         else if (statusLine[bytesParsed] == ' ') {
1591                             statusState = StatusCodeNumber;
1592                         }
1593
1594                         break;
1595
1596                     case MajorVersionNumber:
1597
1598                         if (statusLine[bytesParsed] == '.') {
1599                             //INET_ASSERT(statusState == MajorVersionNumber);
1600                             statusState++; // = MinorVersionNumber
1601                             break;
1602                         }
1603                         // fall through
1604                         goto case MinorVersionNumber;
1605
1606                     case MinorVersionNumber:
1607
1608                         if (statusLine[bytesParsed] == ' ') {
1609                             //INET_ASSERT(statusState == MinorVersionNumber);
1610                             statusState++; // = StatusCodeNumber
1611                             break;
1612                         }
1613                         // fall through
1614                         goto case StatusCodeNumber;
1615
1616                     case StatusCodeNumber:
1617
1618                         if (Char.IsDigit((char)statusLine[bytesParsed])) {
1619                             int val = statusLine[bytesParsed] - '0';
1620                             statusLineInts[statusState] = statusLineInts[statusState] * 10 + val;
1621                         }
1622                         else if (statusLineInts[StatusCodeNumber] > 0) {
1623                             //
1624                             // we eat spaces before status code is found,
1625                             //  once we have the status code we can go on to the next
1626                             //  state on the next non-digit. This is done
1627                             //  to cover cases with several spaces between version
1628                             //  and the status code number.
1629                             //
1630
1631                             statusState++; // = AfterStatusCode
1632                             break;
1633                         }
1634                         else if (!Char.IsWhiteSpace((char) statusLine[bytesParsed])) {
1635                             statusLineInts[statusState] = (int)-1;
1636                         }
1637
1638                         break;
1639
1640                     case AfterStatusCode:
1641                         if (statusLine[bytesParsed] != ' ') {
1642                             lastUnSpaceIndex = bytesParsed;
1643                             if (startIndexStatusDescription == -1) {
1644                                 startIndexStatusDescription = bytesParsed;
1645                             }
1646                         }
1647                         break;
1648
1649                 }
1650                 ++bytesParsed;
1651                 if (m_MaximumResponseHeadersLength>=0 && ++m_TotalResponseHeadersLength>=m_MaximumResponseHeadersLength) {
1652                     parseStatus = DataParseStatus.DataTooBig;
1653                     goto quit;
1654                 }
1655             }
1656
1657             statusLineSize = bytesParsed;
1658
1659             // add to Description if already partialy parsed
1660             if (startIndexStatusDescription != -1) {
1661                 statusDescription +=
1662                     WebHeaderCollection.HeaderEncoding.GetString(
1663                         statusLine,
1664                         startIndexStatusDescription,
1665                         lastUnSpaceIndex - startIndexStatusDescription + 1 );
1666             }
1667
1668             if (bytesParsed == statusLineLength) {
1669                 //
1670                 // response now points one past the end of the buffer. We may be looking
1671                 // over the edge...
1672                 //
1673                 // if we're at the end of the connection then the server sent us an
1674                 // incorrectly formatted response. Probably an error.
1675                 //
1676                 // Otherwise its a partial response. We need more
1677                 //
1678                 parseStatus = DataParseStatus.NeedMoreData;
1679                 //
1680                 // if we really hit the end of the response then update the amount of
1681                 // headers scanned
1682                 //
1683                 GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::ParseStatusLine", parseStatus.ToString());
1684                 return parseStatus;
1685             }
1686
1687             while ((bytesParsed < statusLineLength)
1688                    && ((statusLine[bytesParsed] == '\r') || (statusLine[bytesParsed] == ' '))) {
1689                 ++bytesParsed;
1690                 if (m_MaximumResponseHeadersLength>=0 && ++m_TotalResponseHeadersLength>=m_MaximumResponseHeadersLength) {
1691                     parseStatus = DataParseStatus.DataTooBig;
1692                     goto quit;
1693                 }
1694             }
1695
1696             if (bytesParsed == statusLineLength) {
1697
1698                 //
1699                 // hit end of buffer without finding LF
1700                 //
1701
1702                 parseStatus = DataParseStatus.NeedMoreData;
1703                 goto quit;
1704
1705             }
1706             else if (statusLine[bytesParsed] == '\n') {
1707                 ++bytesParsed;
1708                 if (m_MaximumResponseHeadersLength>=0 && ++m_TotalResponseHeadersLength>=m_MaximumResponseHeadersLength) {
1709                     parseStatus = DataParseStatus.DataTooBig;
1710                     goto quit;
1711                 }
1712                 //
1713                 // if we found the empty line then we are done
1714                 //
1715                 parseStatus = DataParseStatus.Done;
1716             }
1717
1718
1719             //
1720             // Now we have our parsed header to add to the array
1721             //
1722
1723 quit:
1724             if (parseStatus == DataParseStatus.Done && statusState != AfterStatusCode) {
1725                 // need to handle the case where we parse the StatusCode,
1726                 //  but didn't get a status Line, and there was no space afer it.
1727                 if (statusState != StatusCodeNumber || statusLineInts[StatusCodeNumber] <= 0) {
1728                     //
1729                     // we're done with the status line, if we didn't parse all the
1730                     // numbers needed this is invalid protocol on the server
1731                     //
1732                     parseStatus = DataParseStatus.Invalid;
1733                 }
1734             }
1735
1736             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseStatusLine() StatusCode:" + statusLineInts[StatusCodeNumber] + " MajorVersionNumber:" + statusLineInts[MajorVersionNumber] + " MinorVersionNumber:" + statusLineInts[MinorVersionNumber] + " StatusDescription:" + ValidationHelper.ToString(statusDescription));
1737             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::ParseStatusLine", parseStatus.ToString());
1738
1739             if (parseStatus == DataParseStatus.Invalid) {
1740                 parseError.Section = WebParseErrorSection.ResponseStatusLine;
1741                 parseError.Code = WebParseErrorCode.Generic;
1742             }
1743
1744             return parseStatus;
1745         }
1746
1747         // Must all start with a different first character.
1748         private static readonly string[] s_ShortcutStatusDescriptions = new string[] { "OK", "Continue", "Unauthorized" };
1749
1750         //
1751         // Updated version of ParseStatusLine() - secure and fast
1752         //
1753         private static unsafe DataParseStatus ParseStatusLineStrict(
1754                 byte[] statusLine,
1755                 int statusLineLength,
1756                 ref int bytesParsed,
1757                 ref int statusState,
1758                 StatusLineValues statusLineValues,
1759                 int maximumHeaderLength,
1760                 ref int totalBytesParsed,
1761                 ref WebParseError parseError)
1762         {
1763             GlobalLog.Enter("Connection::ParseStatusLineStrict", statusLineLength.ToString(NumberFormatInfo.InvariantInfo) + ", " + bytesParsed.ToString(NumberFormatInfo.InvariantInfo) + ", " + statusState.ToString(NumberFormatInfo.InvariantInfo));
1764             GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection::ParseStatusLineStrict");
1765             GlobalLog.Assert((statusLineLength - bytesParsed) >= 0, "Connection::ParseStatusLineStrict()|(statusLineLength - bytesParsed) < 0");
1766             GlobalLog.Assert(maximumHeaderLength <= 0 || totalBytesParsed <= maximumHeaderLength, "Connection::ParseStatusLineStrict()|Headers already read exceeds limit.");
1767
1768             // Remember where we started.
1769             int initialBytesParsed = bytesParsed;
1770
1771             // Set up parsing status with what will happen if we exceed the buffer.
1772             DataParseStatus parseStatus = DataParseStatus.DataTooBig;
1773             int effectiveMax = maximumHeaderLength <= 0 ? int.MaxValue : (maximumHeaderLength - totalBytesParsed + bytesParsed);
1774             if (statusLineLength < effectiveMax)
1775             {
1776                 parseStatus = DataParseStatus.NeedMoreData;
1777                 effectiveMax = statusLineLength;
1778             }
1779
1780             // sanity check
1781             if (bytesParsed >= effectiveMax)
1782                 goto quit;
1783
1784             fixed (byte* byteBuffer = statusLine)
1785             {
1786                 // Use this switch to jump midway into the action.  They all fall through until the end of the buffer is reached or
1787                 // the status line is fully parsed.
1788                 switch (statusState)
1789                 {
1790                     case BeforeVersionNumbers:
1791                         // This takes advantage of the fact that this token must be the very first thing in the response.
1792                         while (totalBytesParsed - initialBytesParsed + bytesParsed < BeforeVersionNumberBytes.Length)
1793                         {
1794                             if ((byte)BeforeVersionNumberBytes[totalBytesParsed - initialBytesParsed + bytesParsed] != byteBuffer[bytesParsed])
1795                             {
1796                                 parseStatus = DataParseStatus.Invalid;
1797                                 goto quit;
1798                             }
1799
1800                             if(++bytesParsed == effectiveMax)
1801                                 goto quit;
1802                         }
1803
1804                         // When entering the MajorVersionNumber phase, make sure at least one digit is present.
1805                         if (byteBuffer[bytesParsed] == '.')
1806                         {
1807                             parseStatus = DataParseStatus.Invalid;
1808                             goto quit;
1809                         }
1810
1811                         statusState = MajorVersionNumber;
1812                         goto case MajorVersionNumber;
1813
1814                     case MajorVersionNumber:
1815                         while (byteBuffer[bytesParsed] != '.')
1816                         {
1817                             if (byteBuffer[bytesParsed] < '0' || byteBuffer[bytesParsed] > '9')
1818                             {
1819                                 parseStatus = DataParseStatus.Invalid;
1820                                 goto quit;
1821                             }
1822
1823                             statusLineValues.MajorVersion = statusLineValues.MajorVersion * 10 + byteBuffer[bytesParsed] - '0';
1824
1825                             if (++bytesParsed == effectiveMax)
1826                                 goto quit;
1827                         }
1828
1829                         // Need visibility past the dot.
1830                         if (bytesParsed + 1 == effectiveMax)
1831                             goto quit;
1832                         bytesParsed++;
1833
1834                         // When entering the MinorVersionNumber phase, make sure at least one digit is present.
1835                         if (byteBuffer[bytesParsed] == ' ')
1836                         {
1837                             parseStatus = DataParseStatus.Invalid;
1838                             goto quit;
1839                         }
1840
1841                         statusState = MinorVersionNumber;
1842                         goto case MinorVersionNumber;
1843
1844                     case MinorVersionNumber:
1845                         // Only a single SP character is allowed to delimit fields in the status line.
1846                         while (byteBuffer[bytesParsed] != ' ')
1847                         {
1848                             if (byteBuffer[bytesParsed] < '0' || byteBuffer[bytesParsed] > '9')
1849                             {
1850                                 parseStatus = DataParseStatus.Invalid;
1851                                 goto quit;
1852                             }
1853
1854                             statusLineValues.MinorVersion = statusLineValues.MinorVersion * 10 + byteBuffer[bytesParsed] - '0';
1855
1856                             if (++bytesParsed == effectiveMax)
1857                                 goto quit;
1858                         }
1859
1860                         statusState = StatusCodeNumber;
1861
1862                         // Start the status code out as "1".  This will effectively add 1000 to the code.  It's used to count
1863                         // the number of digits to make sure it's three.  At the end, subtract 1000.
1864                         statusLineValues.StatusCode = 1;
1865
1866                         // Move past the space.
1867                         if (++bytesParsed == effectiveMax)
1868                             goto quit;
1869
1870                         goto case StatusCodeNumber;
1871
1872                     case StatusCodeNumber:
1873                         // RFC2616 says codes with an unrecognized first digit
1874                         // should be rejected.  We're allowing the application to define their own "understanding" of
1875                         // 0, 6, 7, 8, and 9xx codes.
1876                         while (byteBuffer[bytesParsed] >= '0' && byteBuffer[bytesParsed] <= '9')
1877                         {
1878                             // Make sure it isn't too big.  The leading '1' will be removed after three digits are read.
1879                             if (statusLineValues.StatusCode >= 1000)
1880                             {
1881                                 parseStatus = DataParseStatus.Invalid;
1882                                 goto quit;
1883                             }
1884
1885                             statusLineValues.StatusCode = statusLineValues.StatusCode * 10 + byteBuffer[bytesParsed] - '0';
1886
1887                             if (++bytesParsed == effectiveMax)
1888                                 goto quit;
1889                         }
1890
1891                         // Make sure there was enough, and exactly one space.
1892                         if (byteBuffer[bytesParsed] != ' ' || statusLineValues.StatusCode < 1000)
1893                         {
1894                             if(byteBuffer[bytesParsed] == '\r' && statusLineValues.StatusCode >= 1000){
1895                                 statusLineValues.StatusCode -= 1000;
1896                                 statusState = AfterCarriageReturn;
1897                                 if (++bytesParsed == effectiveMax)
1898                                     goto quit;
1899                                 goto case AfterCarriageReturn;
1900                             }
1901                             parseStatus = DataParseStatus.Invalid;
1902                             goto quit;
1903                         }
1904
1905                         // Remove the extra leading 1.
1906                         statusLineValues.StatusCode -= 1000;
1907
1908                         statusState = AfterStatusCode;
1909
1910                         // Move past the space.
1911                         if (++bytesParsed == effectiveMax)
1912                             goto quit;
1913
1914                         goto case AfterStatusCode;
1915
1916                     case AfterStatusCode:
1917                     {
1918                         // Check for shortcuts.
1919                         if (statusLineValues.StatusDescription == null)
1920                         {
1921                             foreach (string s in s_ShortcutStatusDescriptions)
1922                             {
1923                                 if (bytesParsed < effectiveMax - s.Length && byteBuffer[bytesParsed] == (byte) s[0])
1924                                 {
1925                                     int i;
1926                                     byte *pBuffer = byteBuffer + bytesParsed + 1;
1927                                     for(i = 1; i < s.Length; i++)
1928                                         if (*(pBuffer++) != (byte) s[i])
1929                                             break;
1930                                     if (i == s.Length)
1931                                     {
1932                                         statusLineValues.StatusDescription = s;
1933                                         bytesParsed += s.Length;
1934                                     }
1935                                     break;
1936                                 }
1937                             }
1938                         }
1939
1940                         int beginning = bytesParsed;
1941
1942                         while (byteBuffer[bytesParsed] != '\r')
1943                         {
1944                             if (byteBuffer[bytesParsed] < ' ' || byteBuffer[bytesParsed] == 127)
1945                             {
1946                                 parseStatus = DataParseStatus.Invalid;
1947                                 goto quit;
1948                             }
1949
1950                             if (++bytesParsed == effectiveMax)
1951                             {
1952                                 string s = WebHeaderCollection.HeaderEncoding.GetString(byteBuffer + beginning, bytesParsed - beginning);
1953                                 if (statusLineValues.StatusDescription == null)
1954                                     statusLineValues.StatusDescription = s;
1955                                 else
1956                                     statusLineValues.StatusDescription += s;
1957
1958                                 goto quit;
1959                             }
1960                         }
1961
1962                         if (bytesParsed > beginning)
1963                         {
1964                             string s = WebHeaderCollection.HeaderEncoding.GetString(byteBuffer + beginning, bytesParsed - beginning);
1965                             if (statusLineValues.StatusDescription == null)
1966                                 statusLineValues.StatusDescription = s;
1967                             else
1968                                 statusLineValues.StatusDescription += s;
1969                         }
1970                         else if (statusLineValues.StatusDescription == null)
1971                         {
1972                             statusLineValues.StatusDescription = "";
1973                         }
1974
1975                         statusState = AfterCarriageReturn;
1976
1977                         // Move past the CR.
1978                         if (++bytesParsed == effectiveMax)
1979                             goto quit;
1980
1981                         goto case AfterCarriageReturn;
1982                     }
1983
1984                     case AfterCarriageReturn:
1985                         if (byteBuffer[bytesParsed] != '\n')
1986                         {
1987                             parseStatus = DataParseStatus.Invalid;
1988                             goto quit;
1989                         }
1990
1991                         parseStatus = DataParseStatus.Done;
1992                         bytesParsed++;
1993                         break;
1994                 }
1995             }
1996
1997 quit:
1998             totalBytesParsed += bytesParsed - initialBytesParsed;
1999
2000             GlobalLog.Print("Connection::ParseStatusLineStrict() StatusCode:" + statusLineValues.StatusCode + " MajorVersionNumber:" + statusLineValues.MajorVersion + " MinorVersionNumber:" + statusLineValues.MinorVersion + " StatusDescription:" + ValidationHelper.ToString(statusLineValues.StatusDescription));
2001             GlobalLog.Leave("Connection::ParseStatusLineStrict", parseStatus.ToString());
2002
2003             if (parseStatus == DataParseStatus.Invalid) {
2004                 parseError.Section = WebParseErrorSection.ResponseStatusLine;
2005                 parseError.Code = WebParseErrorCode.Generic;
2006             }
2007
2008             return parseStatus;
2009         }
2010
2011
2012         /*++
2013
2014         Routine Description:
2015
2016            SetStatusLineParsed - processes the result of status line,
2017              after it has been parsed, reads vars and formats result of parsing
2018
2019         Arguments:
2020
2021            None - uses member vars
2022
2023         Return Value:
2024
2025            None
2026
2027         --*/
2028
2029         private void SetStatusLineParsed() {
2030             // transfer this to response data
2031             m_ResponseData.m_StatusCode = (HttpStatusCode) m_StatusLineValues.StatusCode;
2032             m_ResponseData.m_StatusDescription = m_StatusLineValues.StatusDescription;
2033             m_ResponseData.m_IsVersionHttp11 = m_StatusLineValues.MajorVersion >= 1 && m_StatusLineValues.MinorVersion >= 1;
2034             if (ServicePoint.HttpBehaviour==HttpBehaviour.Unknown || ServicePoint.HttpBehaviour==HttpBehaviour.HTTP11 && !m_ResponseData.m_IsVersionHttp11) {
2035                 // it's only safe to start doing HTTP/1.1 behaviour if the server's version was unknown
2036                 // or if we need to downgrade
2037                 ServicePoint.HttpBehaviour = m_ResponseData.m_IsVersionHttp11 ? HttpBehaviour.HTTP11 : HttpBehaviour.HTTP10;
2038             }
2039
2040             m_CanPipeline = ServicePoint.SupportsPipelining;
2041         }
2042
2043         /*++
2044
2045             ProcessHeaderData - Pulls out Content-length, and other critical
2046                 data from the newly parsed headers
2047
2048             Input:
2049
2050                 Nothing.
2051
2052             Returns:
2053
2054                 long - size of contentLength that we are to use
2055
2056         --*/
2057         private long ProcessHeaderData(ref bool fHaveChunked, HttpWebRequest request, out bool dummyResponseStream)
2058         {
2059             long contentLength = -1;
2060             fHaveChunked = false;
2061             //
2062             // Check for the "Transfer-Encoding" header to contain the "chunked" string
2063             //
2064             string transferEncodingString = m_ResponseData.m_ResponseHeaders[HttpKnownHeaderNames.TransferEncoding];
2065             if (transferEncodingString!=null) {
2066                 transferEncodingString = transferEncodingString.ToLower(CultureInfo.InvariantCulture);
2067                 fHaveChunked = transferEncodingString.IndexOf(HttpWebRequest.ChunkedHeader) != -1;
2068             }
2069
2070             if (!fHaveChunked) {
2071                 //
2072                 // If the response is not chunked, parse the "Content-Length" into a long for data size.
2073                 //
2074                 string contentLengthString = m_ResponseData.m_ResponseHeaders.ContentLength;
2075                 if (contentLengthString!=null) {
2076                     int index = contentLengthString.IndexOf(':');
2077                     if (index!=-1) {
2078                         contentLengthString = contentLengthString.Substring(index + 1);
2079                     }
2080                     bool success = long.TryParse(contentLengthString, NumberStyles.None, CultureInfo.InvariantCulture.NumberFormat, out contentLength);
2081                     if (!success) {
2082                         contentLength = -1;
2083                         //   in some very rare cases, a proxy server may
2084                         //   send us a pair of numbers in comma delimated
2085                         //   fashion, so we need to handle this case
2086                         index = contentLengthString.LastIndexOf(',');
2087                         if (index!=-1) {
2088                             contentLengthString = contentLengthString.Substring(index + 1);
2089                             success = long.TryParse(contentLengthString, NumberStyles.None, CultureInfo.InvariantCulture.NumberFormat, out contentLength);
2090                             if (!success) {
2091                                 contentLength = -1;
2092                             }
2093                         }
2094                     }
2095                     if (contentLength < 0)
2096                     {
2097                         GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::ProcessHeaderData - ContentLength value in header: " + contentLengthString + ", HttpWebRequest#"+ValidationHelper.HashString(m_CurrentRequest));
2098                         contentLength = c_InvalidContentLength; // This will indicate a CL error to the caller
2099                     }
2100                 }
2101             }
2102
2103             // ** else ** signal no content-length present??? or error out?
2104             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ProcessHeaderData() Content-Length parsed:" + contentLength.ToString(NumberFormatInfo.InvariantInfo));
2105
2106             dummyResponseStream = !request.CanGetResponseStream || m_ResponseData.m_StatusCode < HttpStatusCode.OK ||
2107                                   m_ResponseData.m_StatusCode == HttpStatusCode.NoContent || (m_ResponseData.m_StatusCode == HttpStatusCode.NotModified && contentLength < 0) ;
2108
2109             if (m_KeepAlive)
2110             {
2111                 //
2112                 // Deciding on  KEEP ALIVE
2113                 //
2114                 bool resetKeepAlive = false;
2115
2116                 //(1) if no content-length and no chunked, then turn off keep-alive
2117                 //    In some cases, though, Content-Length should be assumed to be 0 based on HTTP RFC 2616
2118                 if (!dummyResponseStream && contentLength < 0 && !fHaveChunked)
2119                 {
2120                     resetKeepAlive = true;
2121                 }
2122                 //(2) A workaround for a failed client ssl session on IIS6
2123                 //    The problem is that we cannot change the connection group name after it gets created.
2124                 //    IIS6 does not close the connection on 403 so all subsequent requests will fail to be authorized on THAT connection.
2125                 //-----------------------------------------------------------------------------------------------
2126                 //5/15/2006
2127                 //[....]
2128                 //The DTS Issue 595216 claims that we are unnecessarily closing the
2129                 //connection on 403 - even if it is a non SSL request. It seems
2130                 //that the original intention is to close the request for SSL requests
2131                 //The following code change would enforce closing onl fo SSL requests.
2132                 //-----------------------------------------------------------------------------------------------
2133                 else if (m_ResponseData.m_StatusCode == HttpStatusCode.Forbidden && NetworkStream is TlsStream)
2134                 {
2135                     resetKeepAlive = true;
2136                 }
2137                 // (3) Possibly cease posting a big body on the connection, was invented mainly for the very first 401 response
2138                 //
2139                 //     This optimization is for the discovery legs only.  For ntlm this is fine, because the 1st actual authleg
2140                 //     is always sent w/ content-length = 0.
2141                 //     For Kerberos preauth, it there could be 1 or 2 auth legs, but we don't know how many there are in advance,
2142                 //     so we don't have a way of eliminating the 1st auth leg.
2143                 else if (m_ResponseData.m_StatusCode > HttpWebRequest.MaxOkStatus &&
2144                          ((request.CurrentMethod == KnownHttpVerb.Post || request.CurrentMethod == KnownHttpVerb.Put) &&
2145                             m_MaximumUnauthorizedUploadLength >= 0 && request.ContentLength > m_MaximumUnauthorizedUploadLength
2146                             && (request.CurrentAuthenticationState == null || request.CurrentAuthenticationState.Module == null)))
2147                 {
2148                     resetKeepAlive = true;
2149                 }
2150                 //(4) for Http/1.0 servers, we can't be sure what their behavior
2151                 //    in this case, so the best thing is to disable KeepAlive unless explicitly set
2152                 //
2153                 else
2154                 {
2155                     //QFE: 4599.
2156                     //Author: [....]
2157                     //in v2.0, in case of SSL Requests through proxy that require NTLM authentication,
2158                     //we are not honoring the Proxy-Connection: Keep-Alive header and
2159                     //closing the connection.
2160                     //
2161                     //In v1.1 we did not have this issue because in v1.1, we would have set an
2162                     //EmptyProxy on the CONNECT request which kind of made it look like the
2163                     //service point is a proxy service point
2164                     //
2165                     //In v2.0, we don't use the GlobalProxySelection.GetEmptyWebProxy we use null
2166                     //to indicate we are not using a proxy.
2167                     //The CONNECT request is a proxy request and the service point is to the
2168                     //proxy.
2169                     //Design Notes
2170                     //------------
2171                     //This is a surgical fix. The "UsesProxySemantics is defined as
2172                     //ServicePoint is a Proxy Service point && (scheme is != https || the request is a tunnel request)
2173                     //Ideally we use one definition of whether we are going trough a proxy or not.
2174                     //The fact is that if you are connecting to a proxy, it is a proxy request and
2175                     //you should honor the Proxy-Connection header.
2176                     //
2177                     //For the purpose of this QFE, when we receive a header we test
2178                     //if this is a Proxy Service Point OR if this is a TUNNEL request
2179
2180
2181
2182                     bool haveClose = false;
2183                     bool haveKeepAlive = false;
2184                     string connection = m_ResponseData.m_ResponseHeaders[HttpKnownHeaderNames.Connection];
2185                     if (connection == null && (
2186                         (ServicePoint.InternalProxyServicePoint) ||
2187                         (request.IsTunnelRequest)))
2188                     {
2189                         connection = m_ResponseData.m_ResponseHeaders[HttpKnownHeaderNames.ProxyConnection];
2190                     }
2191
2192                     if (connection != null) {
2193                         connection = connection.ToLower(CultureInfo.InvariantCulture);
2194                         if (connection.IndexOf("keep-alive") != -1) {
2195                             haveKeepAlive = true;
2196                         }
2197                         else if (connection.IndexOf("close") != -1) {
2198                             haveClose = true;
2199                         }
2200                     }
2201
2202                     if ((haveClose && ServicePoint.HttpBehaviour==HttpBehaviour.HTTP11) ||
2203                         (!haveKeepAlive && ServicePoint.HttpBehaviour<=HttpBehaviour.HTTP10))
2204                     {
2205                         resetKeepAlive = true;
2206                     }
2207                 }
2208
2209
2210                 if (resetKeepAlive)
2211                 {
2212                     lock (this) {
2213                         m_KeepAlive = false;
2214                         m_Free = false;
2215                     }
2216                 }
2217             }
2218
2219             return contentLength;
2220         }
2221
2222         internal bool KeepAlive
2223         {
2224             get
2225             {
2226                 return m_KeepAlive;
2227             }
2228         }
2229
2230         internal bool NonKeepAliveRequestPipelined
2231         {
2232             get
2233             {
2234                 return m_NonKeepAliveRequestPipelined;
2235             }
2236         }
2237
2238         /*++
2239
2240             ParseStreamData
2241
2242             Handles parsing of the blocks of data received after buffer,
2243              distributes the data to stream constructors as needed
2244
2245             returnResult - contains a object containing Requests
2246                 that must be notified upon return from callback
2247
2248         --*/
2249         private DataParseStatus ParseStreamData(ref ConnectionReturnResult returnResult)
2250         {
2251             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::ParseStreamData");
2252
2253             if (m_CurrentRequest == null)
2254             {
2255                 GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::ParseStreamData - Aborted Request, return DataParseStatus.Invalid");
2256                 m_ParseError.Section = WebParseErrorSection.Generic;
2257                 m_ParseError.Code    = WebParseErrorCode.UnexpectedServerResponse;
2258                 return DataParseStatus.Invalid;
2259             }
2260
2261             bool fHaveChunked = false;
2262             bool dummyResponseStream;
2263             // content-length if there is one
2264             long contentLength = ProcessHeaderData(ref fHaveChunked, m_CurrentRequest, out dummyResponseStream);
2265
2266             GlobalLog.Assert(!fHaveChunked || contentLength == -1, "Connection#{0}::ParseStreamData()|fHaveChunked but contentLength != -1", ValidationHelper.HashString(this));
2267
2268             if (contentLength == c_InvalidContentLength)
2269             {
2270                 m_ParseError.Section = WebParseErrorSection.ResponseHeader;
2271                 m_ParseError.Code    = WebParseErrorCode.InvalidContentLength;
2272                 return DataParseStatus.Invalid;
2273             }
2274
2275             // bytes left over that have not been parsed
2276             int bufferLeft = (m_BytesRead - m_BytesScanned);
2277
2278             if (m_ResponseData.m_StatusCode > HttpWebRequest.MaxOkStatus)
2279             {
2280                 // This will tell the request to be prepared for possible connection drop
2281                 // Also that will stop writing on the wire if the connection is not kept alive
2282                 m_CurrentRequest.ErrorStatusCodeNotify(this, m_KeepAlive, false);
2283             }
2284
2285             int bytesToCopy;
2286             //
2287             //  If pipelining, then look for extra data that could
2288             //  be part of of another stream, if its there,
2289             //  then we need to copy it, add it to a stream,
2290             //  and then continue with the next headers
2291             //
2292
2293             if (dummyResponseStream)
2294             {
2295                 bytesToCopy = 0;
2296                 fHaveChunked = false;
2297             }
2298             else
2299             {
2300                 bytesToCopy = -1;
2301
2302                 if (!fHaveChunked && (contentLength <= (long)Int32.MaxValue))
2303                 {
2304                     bytesToCopy = (int)contentLength;
2305                 }
2306             }
2307
2308             DataParseStatus result;
2309
2310             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseStreamData() bytesToCopy:" + bytesToCopy + " bufferLeft:" + bufferLeft);
2311
2312             if (m_CurrentRequest.IsWebSocketRequest && m_ResponseData.m_StatusCode == HttpStatusCode.SwitchingProtocols)
2313             {
2314                 m_ResponseData.m_ConnectStream = new ConnectStream(this, m_ReadBuffer, m_BytesScanned, bufferLeft, bufferLeft, fHaveChunked, m_CurrentRequest);
2315
2316                 // The parsing will be resumed from m_BytesScanned when response stream is closed.
2317                 result = DataParseStatus.Done;
2318                 ClearReaderState();
2319             }
2320             else if (bytesToCopy != -1 && bytesToCopy <= bufferLeft)
2321             {
2322                 m_ResponseData.m_ConnectStream = new ConnectStream(this, m_ReadBuffer, m_BytesScanned, bytesToCopy, dummyResponseStream? 0: contentLength, fHaveChunked, m_CurrentRequest);
2323
2324                 // The parsing will be resumed from m_BytesScanned when response stream is closed.
2325                 result = DataParseStatus.ContinueParsing;
2326                 m_BytesScanned += bytesToCopy;
2327             }
2328             else
2329             {
2330                 m_ResponseData.m_ConnectStream = new ConnectStream(this, m_ReadBuffer, m_BytesScanned, bufferLeft, dummyResponseStream? 0: contentLength, fHaveChunked, m_CurrentRequest);
2331
2332                 // This is the default case where we have a buffer with no more streams except the last one to create so we create it.
2333                 // Note the buffer is fully consumed so we can reset the buffer offests.
2334                 result = DataParseStatus.Done;
2335                 ClearReaderState();
2336             }
2337
2338             m_ResponseData.m_ContentLength = contentLength;
2339             ConnectionReturnResult.Add(ref returnResult, m_CurrentRequest, m_ResponseData.Clone());
2340
2341 #if DEBUG
2342             GlobalLog.DebugUpdateRequest(m_CurrentRequest, this, GlobalLog.WaitingForReadDoneFlag);
2343 #endif
2344
2345             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::ParseStreamData");
2346             return result; // response stream is taking over the reading
2347         }
2348
2349         // Called before restarting Read operations
2350         private void ClearReaderState() {
2351             m_BytesRead    = 0;
2352             m_BytesScanned = 0;
2353         }
2354
2355         /*++
2356
2357             ParseResponseData - Parses the incomming headers, and handles
2358               creation of new streams that are found while parsing, and passes
2359               extra data the new streams
2360
2361             Input:
2362
2363                 returnResult - returns an object containing items that need to be called
2364                     at the end of the read callback
2365
2366             Returns:
2367
2368                 bool - true if one should continue reading more data
2369
2370         --*/
2371         private DataParseStatus ParseResponseData(ref ConnectionReturnResult returnResult, out bool requestDone, out CoreResponseData continueResponseData)
2372         {
2373             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData()");
2374
2375             DataParseStatus parseStatus = DataParseStatus.NeedMoreData;
2376             DataParseStatus parseSubStatus;
2377
2378             // Indicates whether or not at least one whole request was processed in this loop.
2379             // (i.e. Whether ParseStreamData() was called.
2380             requestDone = false;
2381             continueResponseData = null;
2382
2383             // loop in case of multiple sets of headers or streams,
2384             //  that may be generated due to a pipelined response
2385
2386                 // Invariants: at the start of this loop, m_BytesRead
2387                 // is the number of bytes in the buffer, and m_BytesScanned
2388                 // is how many bytes of the buffer we've consumed so far.
2389                 // and the m_ReadState var will be updated at end of
2390                 // each code path, call to this function to reflect,
2391                 // the state, or error condition of the parsing of data
2392                 //
2393                 // We use the following variables in the code below:
2394                 //
2395                 //  m_ReadState - tracks the current state of our Parsing in a
2396                 //      response. z.B.
2397                 //      Start - initial start state and begining of response
2398                 //      StatusLine - the first line sent in response, include status code
2399                 //      Headers - \r\n delimiated Header parsing until we find entity body
2400                 //      Data - Entity Body parsing, if we have all data, we create stream directly
2401                 //
2402                 //  m_ResponseData - An object used to gather Stream, Headers, and other
2403                 //      tidbits so that a request/Response can receive this data when
2404                 //      this code is finished processing
2405                 //
2406                 //  m_ReadBuffer - Of course the buffer of data we are parsing from
2407                 //
2408                 //  m_BytesScanned - The bytes scanned in this buffer so far,
2409                 //      since its always assumed that parse to completion, this
2410                 //      var becomes ended of known data at the end of this function,
2411                 //      based off of 0
2412                 //
2413                 //  m_BytesRead - The total bytes read in buffer, should be const,
2414                 //      till its updated at end of function.
2415                 //
2416
2417                 //
2418                 // Now attempt to parse the data,
2419                 //   we first parse status line,
2420                 //   then read headers,
2421                 //   and finally transfer results to a new stream, and tell request
2422                 //
2423
2424                 switch (m_ReadState) {
2425
2426                     case ReadState.Start:
2427
2428
2429                         if (m_CurrentRequest == null)
2430                         {
2431                             lock (this)
2432                             {
2433                                 if (m_WriteList.Count == 0 || ((m_CurrentRequest = m_WriteList[0] as HttpWebRequest) == null))
2434                                 {
2435                                     m_ParseError.Section = WebParseErrorSection.Generic;
2436                                     m_ParseError.Code    = WebParseErrorCode.Generic;
2437                                     parseStatus = DataParseStatus.Invalid;
2438                                     break;
2439                                 }
2440                             }
2441                         }
2442
2443                         //
2444                         // Start of new response. Transfer the keep-alive context from the corresponding request to
2445                         // the connection
2446                         //
2447                         m_KeepAlive &= (m_CurrentRequest.KeepAlive || m_CurrentRequest.NtlmKeepAlive);
2448
2449                         m_MaximumResponseHeadersLength = m_CurrentRequest.MaximumResponseHeadersLength * 1024;
2450                         m_ResponseData = new CoreResponseData();
2451                         m_ReadState = ReadState.StatusLine;
2452                         m_TotalResponseHeadersLength = 0;
2453
2454                         InitializeParseStatusLine();
2455                         goto case ReadState.StatusLine;
2456
2457                     case ReadState.StatusLine:
2458                         //
2459                         // Reads HTTP status response line
2460                         //
2461                         if (SettingsSectionInternal.Section.UseUnsafeHeaderParsing)
2462                         {
2463                             // This one uses an array to store the parsed values in.  Marshal between this legacy way.
2464                             int[] statusInts = new int[] { 0, m_StatusLineValues.MajorVersion, m_StatusLineValues.MinorVersion, m_StatusLineValues.StatusCode };
2465                             if (m_StatusLineValues.StatusDescription == null)
2466                                 m_StatusLineValues.StatusDescription = "";
2467
2468                             parseSubStatus = ParseStatusLine(
2469                                 m_ReadBuffer, // buffer we're working with
2470                                 m_BytesRead,  // total bytes read so far
2471                                 ref m_BytesScanned, // index off of what we've scanned
2472                                 ref statusInts,
2473                                 ref m_StatusLineValues.StatusDescription,
2474                                 ref m_StatusState,
2475                                 ref m_ParseError);
2476
2477                             m_StatusLineValues.MajorVersion = statusInts[1];
2478                             m_StatusLineValues.MinorVersion = statusInts[2];
2479                             m_StatusLineValues.StatusCode = statusInts[3];
2480                         }
2481                         else
2482                         {
2483                             parseSubStatus = ParseStatusLineStrict(
2484                                 m_ReadBuffer, // buffer we're working with
2485                                 m_BytesRead,  // total bytes read so far
2486                                 ref m_BytesScanned, // index off of what we've scanned
2487                                 ref m_StatusState,
2488                                 m_StatusLineValues,
2489                                 m_MaximumResponseHeadersLength,
2490                                 ref m_TotalResponseHeadersLength,
2491                                 ref m_ParseError);
2492                         }
2493
2494                         if (parseSubStatus == DataParseStatus.Done)
2495                         {
2496                             if (Logging.On) Logging.PrintInfo(Logging.Web, this, SR.GetString(SR.net_log_received_status_line, m_StatusLineValues.MajorVersion+"."+m_StatusLineValues.MinorVersion, m_StatusLineValues.StatusCode, m_StatusLineValues.StatusDescription));
2497                             SetStatusLineParsed();
2498                             m_ReadState = ReadState.Headers;
2499                             m_ResponseData.m_ResponseHeaders = new WebHeaderCollection(WebHeaderCollectionType.HttpWebResponse);
2500                             goto case ReadState.Headers;
2501                         }
2502                         else if (parseSubStatus != DataParseStatus.NeedMoreData)
2503                         {
2504                             //
2505                             // report error - either Invalid or DataTooBig
2506                             //
2507                             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData() ParseStatusLine() parseSubStatus:" + parseSubStatus.ToString());
2508                             parseStatus = parseSubStatus;
2509                             break;
2510                         }
2511
2512                         break; // read more data
2513
2514                     case ReadState.Headers:
2515                         //
2516                         // Parse additional lines of header-name: value pairs
2517                         //
2518                         if (m_BytesScanned >= m_BytesRead) {
2519                             //
2520                             // we already can tell we need more data
2521                             //
2522                             break;
2523                         }
2524
2525                         if (SettingsSectionInternal.Section.UseUnsafeHeaderParsing)
2526                         {
2527                             parseSubStatus = m_ResponseData.m_ResponseHeaders.ParseHeaders(
2528                                 m_ReadBuffer,
2529                                 m_BytesRead,
2530                                 ref m_BytesScanned,
2531                                 ref m_TotalResponseHeadersLength,
2532                                 m_MaximumResponseHeadersLength,
2533                                 ref m_ParseError);
2534                         }
2535                         else
2536                         {
2537                             parseSubStatus = m_ResponseData.m_ResponseHeaders.ParseHeadersStrict(
2538                                 m_ReadBuffer,
2539                                 m_BytesRead,
2540                                 ref m_BytesScanned,
2541                                 ref m_TotalResponseHeadersLength,
2542                                 m_MaximumResponseHeadersLength,
2543                                 ref m_ParseError);
2544                         }
2545
2546                         if (parseSubStatus == DataParseStatus.Invalid || parseSubStatus == DataParseStatus.DataTooBig)
2547                         {
2548                             //
2549                             // report error
2550                             //
2551                             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData() ParseHeaders() parseSubStatus:" + parseSubStatus.ToString());
2552                             parseStatus = parseSubStatus;
2553                             break;
2554                         }
2555                         else if (parseSubStatus == DataParseStatus.Done)
2556                         {
2557                             if (Logging.On) Logging.PrintInfo(Logging.Web, this, SR.GetString(SR.net_log_received_headers, m_ResponseData.m_ResponseHeaders.ToString(true)));
2558
2559                             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData() DataParseStatus.Done StatusCode:" + (int)m_ResponseData.m_StatusCode + " m_BytesRead:" + m_BytesRead + " m_BytesScanned:" + m_BytesScanned);
2560
2561                             //get the IIS server version
2562                             if(m_IISVersion == -1){
2563                                 string server = m_ResponseData.m_ResponseHeaders.Server;
2564                                 if (server != null && server.ToLower(CultureInfo.InvariantCulture).Contains("microsoft-iis")){
2565                                     int i = server.IndexOf("/");
2566                                     if(i++>0 && i <server.Length){
2567                                         m_IISVersion = server[i++] - '0';
2568                                         while(i < server.Length && Char.IsDigit(server[i])) {
2569                                             m_IISVersion = m_IISVersion*10 + server[i++] - '0';
2570                                         }
2571                                     }
2572                                 }
2573                                 //we got a response,so if we don't know the server by now and it wasn't a 100 continue,
2574                                 //we can't assume we will ever know it.  IIS5 sends its server header w/ the continue
2575
2576                                 if(m_IISVersion == -1 && m_ResponseData.m_StatusCode != HttpStatusCode.Continue){
2577                                     m_IISVersion = 0;
2578                                 }
2579                             }
2580
2581                             if (m_ResponseData.m_StatusCode == HttpStatusCode.Continue || m_ResponseData.m_StatusCode == HttpStatusCode.BadRequest) {
2582
2583                                 GlobalLog.Assert(m_CurrentRequest != null, "Connection#{0}::ParseResponseData()|m_CurrentRequest == null", ValidationHelper.HashString(this));
2584                                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData() HttpWebRequest#" + ValidationHelper.HashString(m_CurrentRequest));
2585
2586                                 if (m_ResponseData.m_StatusCode == HttpStatusCode.BadRequest) {
2587                                     // If we have a 400 and we were sending a chunked request going through to a proxy with a chunked upload,
2588                                     // this proxy is a partially compliant so shut off fancy features (pipelining and chunked uploads)
2589                                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData() got a 400 StatusDescription:" + m_ResponseData.m_StatusDescription);
2590                                     if (ServicePoint.HttpBehaviour == HttpBehaviour.HTTP11
2591                                         && m_CurrentRequest.HttpWriteMode==HttpWriteMode.Chunked
2592                                         && m_ResponseData.m_ResponseHeaders.Via != null
2593                                         && string.Compare(m_ResponseData.m_StatusDescription, "Bad Request ( The HTTP request includes a non-supported header. Contact the Server administrator.  )", StringComparison.OrdinalIgnoreCase)==0) {
2594                                         GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData() downgrading server to HTTP11PartiallyCompliant.");
2595                                         ServicePoint.HttpBehaviour = HttpBehaviour.HTTP11PartiallyCompliant;
2596                                     }
2597                                 }
2598                                 else {
2599                                     // If we have an HTTP continue, eat these headers and look
2600                                     //  for the 200 OK
2601                                     //
2602                                     // we got a 100 Continue. set this on the HttpWebRequest
2603                                     //
2604                                     m_CurrentRequest.Saw100Continue = true;
2605                                     if (!ServicePoint.Understands100Continue) {
2606                                         //
2607                                         // and start expecting it again if this behaviour was turned off
2608                                         //
2609                                         GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData() HttpWebRequest#" + ValidationHelper.HashString(m_CurrentRequest) + " ServicePoint#" + ValidationHelper.HashString(ServicePoint) + " sent UNexpected 100 Continue");
2610                                         ServicePoint.Understands100Continue = true;
2611                                     }
2612
2613                                     //
2614                                     // set Continue Ack on request.
2615                                     //
2616                                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData() calling SetRequestContinue()");
2617                                     continueResponseData = m_ResponseData;
2618
2619                                     //if we got a 100continue we ---- it and start looking for a final response
2620                                     goto case ReadState.Start;
2621                                 }
2622                             }
2623
2624                             m_ReadState = ReadState.Data;
2625                             goto case ReadState.Data;
2626                         }
2627
2628                         // need more data,
2629                         break;
2630
2631                     case ReadState.Data:
2632
2633                         // (check status code for continue handling)
2634                         // 1. Figure out if its Chunked, content-length, or encoded
2635                         // 2. Takes extra data, place in stream(s)
2636                         // 3. Wake up blocked stream requests
2637                         //
2638
2639                         // Got through one entire response
2640                         requestDone = true;
2641
2642                         // parse and create a stream if needed
2643                         parseStatus = ParseStreamData(ref returnResult);
2644
2645                         GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData() result:" + parseStatus);
2646                         GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData()" + " WriteDone:" + m_WriteDone + " ReadDone:" + m_ReadDone + " WaitList:" + m_WaitList.Count + " WriteList:" + m_WriteList.Count);
2647                         break;
2648                 }
2649
2650             if (m_BytesScanned == m_BytesRead)
2651                 ClearReaderState();
2652
2653             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData() m_ReadState:" + m_ReadState);
2654             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData()", parseStatus.ToString());
2655             return parseStatus;
2656         }
2657
2658         /// <devdoc>
2659         ///    <para>
2660         ///       Cause the Connection to Close and Abort its socket,
2661         ///         after the next request is completed.  If the Connection
2662         ///         is already idle, then Aborts the socket immediately.
2663         ///    </para>
2664         /// </devdoc>
2665         internal void CloseOnIdle() {
2666             // The timer thread is allowed to call this.  (It doesn't call user code and doesn't block.)
2667             GlobalLog.ThreadContract(ThreadKinds.Unknown, ThreadKinds.SafeSources | ThreadKinds.Timer, "Connection#" + ValidationHelper.HashString(this) + "::CloseOnIdle");
2668
2669             lock(this){
2670                 m_KeepAlive = false;
2671                 m_RemovedFromConnectionList = true;
2672                 if (!m_Idle)
2673                 {
2674                     CheckIdle();
2675                 }
2676                 if (m_Idle)
2677                 {
2678                     AbortSocket(false);
2679                     GC.SuppressFinalize(this);
2680                 }
2681             }
2682         }
2683
2684         internal bool AbortOrDisassociate(HttpWebRequest request, WebException webException)
2685         {
2686             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::AbortOrDisassociate", "request#" + ValidationHelper.HashString(request));
2687             GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection#" + ValidationHelper.HashString(this) + "::AbortOrDisassociate()");
2688
2689             ConnectionReturnResult result = null;
2690             lock(this)
2691             {
2692                 int idx = m_WriteList.IndexOf(request);
2693                 // If the request is in the submission AND this is the first request we have to abort the connection,
2694                 // Otheriwse we simply disassociate it from the current connection.
2695                 if (idx == -1)
2696                 {
2697                     WaitListItem foundItem = null;
2698
2699                     if (m_WaitList.Count > 0)
2700                     {
2701                         foundItem = m_WaitList.Find(o => object.ReferenceEquals(o.Request, request));
2702                     }
2703
2704                     // If not found then the request must be already dispatched and the response stream is drained
2705                     // If so then we let request.Abort() to deal with this situation.
2706                     if (foundItem != null)
2707                     {
2708                         NetworkingPerfCounters.Instance.IncrementAverage(NetworkingPerfCounterName.HttpWebRequestAvgQueueTime,
2709                             foundItem.QueueStartTime);
2710                         m_WaitList.Remove(foundItem);
2711                         UnlockIfNeeded(foundItem.Request);
2712                     }
2713
2714                     GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::AbortOrDisassociate()", "Request was wisassociated");
2715                     return true;
2716                 }
2717                 else if (idx != 0)
2718                 {
2719                     // Make this connection Keep-Alive=false, remove the request and do not close the connection
2720                     // When the active request completes, the rest of the pipeline (minus aborted request) will be resubmitted.
2721                     m_WriteList.RemoveAt(idx);
2722                     m_KeepAlive = false;
2723                     GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::AbortOrDisassociate()", "Request was Disassociated from the Write List, idx = " + idx);
2724                     return true;
2725                 }
2726
2727 #if DEBUG
2728                 try
2729                 {
2730 #endif
2731                 m_KeepAlive = false;
2732                 if (webException != null && m_InnerException == null)
2733                 {
2734                     m_InnerException = webException;
2735                     m_Error = webException.Status;
2736                 }
2737                 else
2738                 {
2739                     m_Error = WebExceptionStatus.RequestCanceled;
2740                 }
2741
2742                 PrepareCloseConnectionSocket(ref result);
2743                 // Hard Close the socket.
2744                 Close(0);
2745                 FreeReadBuffer();       // Do it after close completes to insure buffer not in use
2746 #if DEBUG
2747                 }
2748                 catch (Exception exception)
2749                 {
2750                     t_LastStressException = exception;
2751                     if (!NclUtilities.IsFatal(exception)){
2752                         GlobalLog.Assert("Connection#" + ValidationHelper.HashString(this) + "::AbortOrDisassociate()", exception.Message);
2753                     }
2754                 }
2755 #endif
2756             }
2757             ConnectionReturnResult.SetResponses(result);
2758             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::AbortOrDisassociate()", "Connection Aborted");
2759             return false;
2760         }
2761
2762 #if DEBUG
2763         [ThreadStatic]
2764         private static Exception t_LastStressException;
2765 #endif
2766
2767         internal void AbortSocket(bool isAbortState)
2768         {
2769             // The timer/finalization thread is allowed to call this.  (It doesn't call user code and doesn't block.)
2770             GlobalLog.ThreadContract(ThreadKinds.Unknown, ThreadKinds.SafeSources | ThreadKinds.Timer | ThreadKinds.Finalization, "Connection#" + ValidationHelper.HashString(this) + "::AbortSocket");
2771             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::Abort", "isAbortState:" + isAbortState.ToString());
2772
2773             if (isAbortState) {
2774                 UnlockRequest();
2775                 CheckIdle();
2776             }
2777             else {
2778                 // This one is recoverable, set it to keep  Read/Write StartNextRequest happy.
2779                 m_Error = WebExceptionStatus.KeepAliveFailure;
2780             }
2781
2782             // Stream close isn't threadsafe.
2783             lock (this)
2784             {
2785                 Close(0);
2786                 FreeReadBuffer();       // Do it after close completes to insure buffer not in use
2787             }
2788
2789             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::Abort", "isAbortState:" + isAbortState.ToString());
2790         }
2791
2792
2793         /*++
2794
2795             PrepareCloseConnectionSocket - reset the connection requests list.
2796
2797             This method is called when we want to close the conection.
2798             It must be called with the critical section held.
2799             The caller must call this.Close if decided to call this method.
2800
2801             All connection closes (either ours or server initiated) eventually go through here.
2802
2803             As to what we do: we loop through our write and wait list and pull requests
2804             off it, and give each request an error failure. Then the caller will
2805             dispatch the responses.
2806
2807         --*/
2808
2809         private void PrepareCloseConnectionSocket(ref ConnectionReturnResult returnResult)
2810         {
2811             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::PrepareCloseConnectionSocket", m_Error.ToString());
2812
2813             // Effectivelly, closing a connection makes it exempted from the "Idling" logic
2814             m_IdleSinceUtc = DateTime.MinValue;
2815             CanBePooled = false;
2816
2817             if (m_WriteList.Count != 0 || m_WaitList.Count != 0)
2818             {
2819
2820                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::PrepareCloseConnectionSocket() m_WriteList.Count:" + m_WriteList.Count);
2821                 DebugDumpWriteListEntries();
2822                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::PrepareCloseConnectionSocket() m_WaitList.Count:" + m_WaitList.Count);
2823                 DebugDumpWaitListEntries();
2824
2825                 HttpWebRequest lockedRequest = LockedRequest;
2826
2827                 if (lockedRequest != null)
2828                 {
2829                     bool callUnlockRequest = false;
2830                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::PrepareCloseConnectionSocket() looking for HttpWebRequest#" + ValidationHelper.HashString(lockedRequest));
2831
2832                     foreach (HttpWebRequest request in m_WriteList)
2833                     {
2834                         if (request == lockedRequest) {
2835                             callUnlockRequest = true;
2836                         }
2837                     }
2838
2839                     if (!callUnlockRequest) {
2840                         foreach (WaitListItem item in m_WaitList) {
2841                             if (item.Request == lockedRequest) {
2842                                 callUnlockRequest = true;
2843                                 break;
2844                             }
2845                         }
2846                     }
2847                     if (callUnlockRequest) {
2848                         UnlockRequest();
2849                     }
2850                 }
2851
2852                 HttpWebRequest[] requestArray = null;
2853
2854                 // WaitList gets Isolated exception status, free to retry multiple times
2855                 if (m_WaitList.Count != 0)
2856                 {
2857                     requestArray = new HttpWebRequest[m_WaitList.Count];
2858                     for (int i = 0; i < m_WaitList.Count; i++)
2859                     {
2860                         requestArray[i] = m_WaitList[i].Request;
2861                     }
2862                     ConnectionReturnResult.AddExceptionRange(ref returnResult, requestArray, ExceptionHelper.IsolatedException);
2863                 }
2864
2865                 //
2866                 // WriteList (except for single request list) gets Recoverable exception status, may be retired if not failed once
2867                 // For a single request list the exception is computed here
2868                 // InnerExeption if any may tell more details in both cases
2869                 //
2870                 if (m_WriteList.Count != 0)
2871                 {
2872                     Exception theException = m_InnerException;
2873
2874
2875                     if(theException != null)
2876                        GlobalLog.Print(theException.ToString());
2877
2878                     GlobalLog.Print("m_Error = "+ m_Error.ToString());
2879
2880                     if (!(theException is WebException) && !(theException is SecurityException))
2881                     {
2882                         if (m_Error == WebExceptionStatus.ServerProtocolViolation)
2883                         {
2884                             string errorString = NetRes.GetWebStatusString(m_Error);
2885
2886                             string detailedInfo = "";
2887                             if (m_ParseError.Section != WebParseErrorSection.Generic)
2888                                 detailedInfo += " Section=" + m_ParseError.Section.ToString();
2889                             if (m_ParseError.Code != WebParseErrorCode.Generic) {
2890                                 detailedInfo += " Detail=" + SR.GetString("net_WebResponseParseError_" + m_ParseError.Code.ToString());
2891                             }
2892                             if (detailedInfo.Length != 0)
2893                                 errorString += "." + detailedInfo;
2894
2895                             theException = new WebException(errorString,
2896                                                             theException,
2897                                                             m_Error,
2898                                                             null,
2899                                                             WebExceptionInternalStatus.RequestFatal);
2900                         }
2901                         else if (m_Error == WebExceptionStatus.SecureChannelFailure)
2902                         {
2903                             theException = new WebException(NetRes.GetWebStatusString("net_requestaborted", WebExceptionStatus.SecureChannelFailure),
2904                                                             WebExceptionStatus.SecureChannelFailure);
2905                         }
2906
2907                         else if (m_Error == WebExceptionStatus.Timeout)
2908                         {
2909                             theException = new WebException(NetRes.GetWebStatusString("net_requestaborted", WebExceptionStatus.Timeout),
2910                                                             WebExceptionStatus.Timeout);
2911                         }
2912                         else if(m_Error == WebExceptionStatus.RequestCanceled)
2913                         {
2914                             theException = new WebException(NetRes.GetWebStatusString("net_requestaborted", WebExceptionStatus.RequestCanceled),
2915                                                             WebExceptionStatus.RequestCanceled,
2916                                                             WebExceptionInternalStatus.RequestFatal,
2917                                                             theException);
2918                         }
2919                         else if(m_Error == WebExceptionStatus.MessageLengthLimitExceeded ||
2920                                 m_Error == WebExceptionStatus.TrustFailure)
2921                         {
2922                             theException = new WebException(NetRes.GetWebStatusString("net_connclosed", m_Error),
2923                                                             m_Error,
2924                                                             WebExceptionInternalStatus.RequestFatal,
2925                                                             theException);
2926                         }
2927                         else
2928                         {
2929                             if (m_Error == WebExceptionStatus.Success)
2930                             {
2931                                 throw new InternalException();              // TODO: replace it with a generic error for the product bits
2932                                 //m_Error = WebExceptionStatus.UnknownError;
2933                             }
2934
2935                             bool retry = false;
2936                             bool isolatedKeepAliveFailure = false;
2937
2938                             if (m_WriteList.Count != 1)
2939                             {
2940                                 // Real scenario: SSL against IIS-5 would fail if pipelinning.
2941                                 // retry = true will cover a general case when >>the server<< aborts a pipeline
2942                                 // Basically all pipelined requests are marked with recoverable error including the very active request.
2943                                 retry = true;
2944                             }
2945                             else if (m_Error == WebExceptionStatus.KeepAliveFailure)
2946                             {
2947                                 HttpWebRequest request = (HttpWebRequest) m_WriteList[0];
2948                                 // Check that the active request did not start the body yet
2949                                 if (!request.BodyStarted)
2950                                     isolatedKeepAliveFailure = true;
2951                             }
2952                             else{
2953                                 retry = (!AtLeastOneResponseReceived && !((HttpWebRequest) m_WriteList[0]).BodyStarted);
2954                             }
2955                                 theException = new WebException(NetRes.GetWebStatusString("net_connclosed", m_Error),
2956                                                             m_Error,
2957                                                             (isolatedKeepAliveFailure? WebExceptionInternalStatus.Isolated:
2958                                                                 retry? WebExceptionInternalStatus.Recoverable:
2959                                                                 WebExceptionInternalStatus.RequestFatal),
2960                                                             theException);
2961                         }
2962                     }
2963
2964                     WebException pipelineException = new WebException(NetRes.GetWebStatusString("net_connclosed", WebExceptionStatus.PipelineFailure),
2965                                                                       WebExceptionStatus.PipelineFailure,
2966                                                                       WebExceptionInternalStatus.Recoverable,
2967                                                                       theException);
2968
2969                     requestArray = new HttpWebRequest[m_WriteList.Count];
2970                     m_WriteList.CopyTo(requestArray, 0);
2971                     ConnectionReturnResult.AddExceptionRange(ref returnResult, requestArray, pipelineException, theException);
2972                 }
2973
2974 #if TRAVE
2975                 foreach (WaitListItem item in m_WaitList) {
2976                     GlobalLog.Print("Request removed from WaitList#"+ValidationHelper.HashString(item.Request));
2977                 }
2978
2979                 foreach (HttpWebRequest request in m_WriteList) {
2980                     GlobalLog.Print("Request removed from m_WriteList#"+ValidationHelper.HashString(request));
2981                 }
2982 #endif
2983
2984                 m_WriteList.Clear();
2985
2986                 foreach (WaitListItem item in m_WaitList)
2987                 {
2988                     NetworkingPerfCounters.Instance.IncrementAverage(NetworkingPerfCounterName.HttpWebRequestAvgQueueTime,
2989                         item.QueueStartTime);
2990                 }
2991                 m_WaitList.Clear();
2992             }
2993
2994             CheckIdle();
2995
2996             if (m_Idle)
2997             {
2998                 GC.SuppressFinalize(this);
2999             }
3000             if (!m_RemovedFromConnectionList && ConnectionGroup != null)
3001             {
3002                 m_RemovedFromConnectionList = true;
3003                 ConnectionGroup.Disassociate(this);
3004             }
3005
3006             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::PrepareCloseConnectionSocket");
3007         }
3008
3009
3010         /*++
3011
3012             HandleError - Handle a protocol error from the server.
3013
3014             This method is called when we've detected some sort of fatal protocol
3015             violation while parsing a response, receiving data from the server,
3016             or failing to connect to the server. We'll fabricate
3017             a WebException and then call CloseConnection which closes the
3018             connection as well as informs the request through a callback.
3019
3020             Input:
3021                     webExceptionStatus -
3022                     connectFailure -
3023                     readFailure -
3024
3025             Returns: Nothing
3026
3027         --*/
3028         internal void HandleConnectStreamException(bool writeDone, bool readDone, WebExceptionStatus webExceptionStatus, ref ConnectionReturnResult returnResult, Exception e)
3029         {
3030             if (m_InnerException == null)
3031             {
3032                 m_InnerException = e;
3033                 if (!(e is WebException) && NetworkStream is TlsStream)
3034                 {
3035                     // Unless a WebException is passed the Connection knows better the error code if the transport is TlsStream
3036                     webExceptionStatus = ((TlsStream) NetworkStream).ExceptionStatus;
3037                 }
3038                 else if (e is ObjectDisposedException)
3039                 {
3040                     webExceptionStatus = WebExceptionStatus.RequestCanceled;
3041                 }
3042             }
3043             HandleError(writeDone, readDone, webExceptionStatus, ref returnResult);
3044         }
3045         //
3046         private void HandleErrorWithReadDone(WebExceptionStatus webExceptionStatus, ref ConnectionReturnResult returnResult)
3047         {
3048             HandleError(false, true, webExceptionStatus, ref returnResult);
3049         }
3050         //
3051         private void HandleError(bool writeDone, bool readDone, WebExceptionStatus webExceptionStatus, ref ConnectionReturnResult returnResult)
3052         {
3053             lock(this)
3054             {
3055                 if (writeDone)
3056                     m_WriteDone = true;
3057                 if (readDone)
3058                     m_ReadDone = true;
3059
3060                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::HandleError() m_WriteList.Count:" + m_WriteList.Count +
3061                                 " m_WaitList.Count:" + m_WaitList.Count +
3062                                 " new WriteDone:" + m_WriteDone + " new ReadDone:" + m_ReadDone);
3063                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::HandleError() current HttpWebRequest#" + ValidationHelper.HashString(m_CurrentRequest));
3064
3065                 if(webExceptionStatus == WebExceptionStatus.Success)
3066                     throw new InternalException(); //consider making an assert later.
3067
3068                 m_Error = webExceptionStatus;
3069
3070                 PrepareCloseConnectionSocket(ref returnResult);
3071                 // This will kill the socket
3072                 // Must be done inside the lock.  (Stream Close() isn't threadsafe.)
3073                 Close(0);
3074                 FreeReadBuffer();       // Do it after close completes to insure buffer not in use
3075             }
3076         }
3077
3078         private static void ReadCallbackWrapper(IAsyncResult asyncResult)
3079         {
3080             if (asyncResult.CompletedSynchronously)
3081             {
3082                 return;
3083             }
3084
3085             ((Connection) asyncResult.AsyncState).ReadCallback(asyncResult);
3086         }
3087
3088         /// <devdoc>
3089         ///    <para>Performs read callback processing on connection
3090         ///     handles getting headers parsed and streams created</para>
3091         /// </devdoc>
3092         private void ReadCallback(IAsyncResult asyncResult)
3093         {
3094             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::ReadCallback", ValidationHelper.HashString(asyncResult));
3095
3096             int bytesRead = -1;
3097             WebExceptionStatus errorStatus = WebExceptionStatus.ReceiveFailure;
3098
3099             //
3100             // parameter validation
3101             //
3102             GlobalLog.Assert(asyncResult != null, "Connection#{0}::ReadCallback()|asyncResult == null", ValidationHelper.HashString(this));
3103             GlobalLog.Assert((asyncResult is OverlappedAsyncResult || asyncResult is LazyAsyncResult), "Connection#{0}::ReadCallback()|asyncResult is not OverlappedAsyncResult.", ValidationHelper.HashString(this));
3104
3105             try {
3106                 bytesRead = EndRead(asyncResult);
3107                 if (bytesRead == 0)
3108                     bytesRead = -1; // 0 is reserved for re-entry on already buffered data
3109
3110                 errorStatus = WebExceptionStatus.Success;
3111             }
3112             catch (Exception exception) {
3113                 // Notify request's SubmitWriteStream that a socket error happened.  This will cause future writes to
3114                    // throw an IOException.
3115                    HttpWebRequest curRequest = m_CurrentRequest;
3116                    if (curRequest != null)
3117                    {
3118                        curRequest.ErrorStatusCodeNotify(this, false, true);
3119                    }
3120
3121
3122                 if (m_InnerException == null)
3123                     m_InnerException = exception;
3124
3125                 if (exception.GetType() == typeof(ObjectDisposedException))
3126                     errorStatus = WebExceptionStatus.RequestCanceled;
3127
3128 #if !FEATURE_PAL
3129                 //ASYNCISSUE
3130                 // Consider: In case of a async exception we should do minimal cleanup here trying the appDomain
3131                 // to survive or to force unloading of the appDomain .
3132                 // need to handle SSL errors too
3133                 if (NetworkStream is TlsStream)  {
3134                     errorStatus = ((TlsStream) NetworkStream).ExceptionStatus;
3135                 }
3136                 else {
3137                     errorStatus = WebExceptionStatus.ReceiveFailure;
3138                 }
3139 #else
3140                 errorStatus = WebExceptionStatus.ReceiveFailure;
3141 #endif // !FEATURE_PAL
3142                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadCallback() EndRead() errorStatus:" + errorStatus.ToString() + " caught exception:" + exception);
3143             }
3144
3145             ReadComplete(bytesRead, errorStatus);
3146             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::ReadCallback");
3147         }
3148
3149
3150         /// <devdoc>
3151         ///    <para>Attempts to poll the socket, to see if data is waiting to be read,
3152         ///     if there is data there, then a read is started</para>
3153         /// </devdoc>
3154         internal void PollAndRead(HttpWebRequest request, bool userRetrievedStream) {
3155             GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection#" + ValidationHelper.HashString(this) + "::PollAndRead");
3156
3157             // Ensure that we don't already have a response for this request, before we attempt to read the socket.
3158             request.NeedsToReadForResponse = true;
3159             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::PollAndRead() InternalPeekCompleted:" + request.ConnectionReaderAsyncResult.InternalPeekCompleted.ToString() + " Result:" + ValidationHelper.ToString(request.ConnectionReaderAsyncResult.Result));
3160             if (request.ConnectionReaderAsyncResult.InternalPeekCompleted && request.ConnectionReaderAsyncResult.Result == null && CanBePooled)
3161             {
3162                 SyncRead(request, userRetrievedStream, true);
3163             }
3164         }
3165         //
3166         //    Peforms a [....] Read and calls the ReadComplete to process the result
3167         //    The reads are done iteratively, until the Request has received enough
3168         //    data to contruct a response, or a 100-Continue is read, allowing the HttpWebRequest
3169         //    to return a write stream
3170         //
3171         //    probeRead = true only for POST request and when the caller needs to wait for 100-continue
3172         //
3173         internal void SyncRead(HttpWebRequest request, bool userRetrievedStream, bool probeRead)
3174         {
3175             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::SyncRead(byte[]) request#" + ValidationHelper.HashString(request) + (probeRead? ", Probe read = TRUE":string.Empty));
3176             GlobalLog.ThreadContract(ThreadKinds.Sync, "Connection#" + ValidationHelper.HashString(this) + "::SyncRead");
3177
3178             // prevent recursive calls to this function
3179             if (t_SyncReadNesting > 0) {
3180                 GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::SyncRead() - nesting");
3181                 return;
3182             }
3183
3184             bool pollSuccess = probeRead? false: true;
3185
3186             try {
3187                 t_SyncReadNesting++;
3188
3189                 // grab a counter to tell us whenever the SetRequestContinue is called
3190                 int requestContinueCount = probeRead ? request.RequestContinueCount : 0;
3191
3192                 bool requestDone;
3193                 int bytesRead = -1;
3194                 WebExceptionStatus errorStatus = WebExceptionStatus.ReceiveFailure;
3195
3196
3197                 if (m_BytesScanned < m_BytesRead)
3198                 {
3199                     // left over from previous read
3200                     pollSuccess = true;
3201                     bytesRead = 0; //tell it we want to use buffered data on the first iteration
3202                     errorStatus = WebExceptionStatus.Success;
3203                 }
3204
3205                 do {
3206                     requestDone = true;
3207
3208                     try {
3209                         if (bytesRead != 0)
3210                         {
3211                             errorStatus = WebExceptionStatus.ReceiveFailure;
3212
3213                             if (!pollSuccess)
3214                             {
3215                                 pollSuccess = Poll(request.ContinueTimeout * 1000, SelectMode.SelectRead);  // Timeout is in microseconds
3216                                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::SyncRead() PollSuccess : " + pollSuccess);
3217                             }
3218
3219                             if (pollSuccess)
3220                             {
3221                                 //Ensures that we'll timeout eventually on an appdomain unload.
3222                                 //Will be a no-op if the timeout doesn't change from request to request.
3223                                 ReadTimeout = request.Timeout;
3224
3225                                 bytesRead = Read(m_ReadBuffer, m_BytesRead, m_ReadBuffer.Length - m_BytesRead);
3226                                 errorStatus = WebExceptionStatus.Success;
3227                                 if (bytesRead == 0)
3228                                     bytesRead = -1; // 0 is reserved for re-entry on already buffered data
3229                             }
3230                         }
3231                     }
3232                     catch (Exception exception)
3233                     {
3234                         if (NclUtilities.IsFatal(exception)) throw;
3235
3236                         if (m_InnerException == null)
3237                             m_InnerException = exception;
3238
3239                         if (exception.GetType() == typeof(ObjectDisposedException))
3240                             errorStatus = WebExceptionStatus.RequestCanceled;
3241
3242                         // need to handle SSL errors too
3243 #if !FEATURE_PAL
3244                         else if (NetworkStream is TlsStream)  {
3245                             errorStatus = ((TlsStream)NetworkStream).ExceptionStatus;
3246                         }
3247 #endif // !FEATURE_PAL
3248                         else
3249                         {
3250                             SocketException socketException = exception.InnerException as SocketException;
3251                             if (socketException != null)
3252                             {
3253                                  if (socketException.ErrorCode == (int) SocketError.TimedOut)
3254                                     errorStatus = WebExceptionStatus.Timeout;
3255                                 else
3256                                     errorStatus = WebExceptionStatus.ReceiveFailure;
3257                             }
3258                         }
3259
3260                         GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::SyncRead() Read() threw errorStatus:" + errorStatus.ToString() + " bytesRead:" + bytesRead.ToString());
3261                     }
3262
3263                     if (pollSuccess)
3264                         requestDone = ReadComplete(bytesRead, errorStatus);
3265
3266                     bytesRead = -1;
3267                 } while (!requestDone && (userRetrievedStream || requestContinueCount == request.RequestContinueCount));
3268             }
3269             finally {
3270                 t_SyncReadNesting--;
3271             }
3272
3273             if (probeRead)
3274             {
3275                 // [....] 100-Continue wait only
3276                 request.FinishContinueWait();
3277                 if (pollSuccess)
3278                 {
3279                     if (!request.Saw100Continue && !userRetrievedStream)
3280                     {
3281                         //During polling, we got a response that wasn't a 100 continue.
3282                         request.NeedsToReadForResponse = false;
3283                     }
3284                 }
3285                 else
3286                 {
3287                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::SyncRead() Poll has timed out, calling SetRequestContinue().");
3288                     request.SetRequestContinue();
3289                 }
3290             }
3291             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::SyncRead()");
3292         }
3293
3294
3295         //
3296         //    Performs read callback processing on connection
3297         //    handles getting headers parsed and streams created
3298         //
3299         //    bytesRead == 0  when  we re-enter on buffered data without doing actual read
3300         //    bytesRead == -1 when  we got a connection close plus when errorStatus == sucess we got a g----ful close.
3301         //    Otheriwse bytesRead is read byted to add to m_BytesRead i.e. to previously buffered data
3302         //
3303         private bool ReadComplete(int bytesRead, WebExceptionStatus errorStatus)
3304         {
3305             bool requestDone = true;
3306             CoreResponseData continueResponseData = null;
3307             ConnectionReturnResult returnResult = null;
3308             HttpWebRequest currentRequest = null;
3309
3310             try
3311             {
3312                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadComplete() m_BytesRead:" + m_BytesRead.ToString() + " m_BytesScanned:" + m_BytesScanned + " (+= new bytesRead:" + bytesRead.ToString() + ")");
3313
3314                 if (bytesRead < 0)
3315                 {
3316                     // Means we might have gotten g----full or hard connection close.
3317
3318                     // If this is the first thing we read for a request then it
3319                     // could be an idle connection closed by the server (isolated error)
3320                     if (m_ReadState == ReadState.Start && m_AtLeastOneResponseReceived)
3321                     {
3322                         // Note that KeepAliveFailure will be checked against POST-type requests
3323                         // and it's fatal if the body was already started.
3324                         if (errorStatus == WebExceptionStatus.Success || errorStatus == WebExceptionStatus.ReceiveFailure)
3325                             errorStatus = WebExceptionStatus.KeepAliveFailure;
3326                     }
3327                     else if (errorStatus == WebExceptionStatus.Success)
3328                     {
3329                         // we got unexpected FIN in the middle of the response, or on a fresh connection, that's fatal
3330                         errorStatus = WebExceptionStatus.ConnectionClosed;
3331                     }
3332
3333                     // Notify request's SubmitWriteStream that a socket error happened.  This will cause future writes to
3334                     // throw an IOException.
3335                     HttpWebRequest curRequest = m_CurrentRequest;
3336                     if (curRequest != null)
3337                     {
3338                         curRequest.ErrorStatusCodeNotify(this, false, true);
3339                     }
3340
3341                     HandleErrorWithReadDone(errorStatus, ref returnResult);
3342                     goto done;
3343                 }
3344
3345                 // Otherwise, we've got data.
3346                 GlobalLog.Dump(m_ReadBuffer, m_BytesScanned, m_BytesRead - m_BytesScanned);
3347                 GlobalLog.Dump(m_ReadBuffer, m_BytesRead, bytesRead);
3348
3349
3350                 bytesRead += m_BytesRead;
3351                 if (bytesRead  > m_ReadBuffer.Length)
3352                     throw new InternalException();  //in case we posted two receives at once
3353                 m_BytesRead = bytesRead;
3354
3355                 // We have the parsing code seperated out in ParseResponseData
3356                 //
3357                 // If we don't have all the headers yet. Resubmit the receive,
3358                 // passing in the bytes read total as our index. When we get
3359                 // back here we'll end up reparsing from the beginning, which is
3360                 // OK. because this shouldn't be a performance case.
3361
3362                 //if we're back here, we need to reset the scanned bytes to 0.
3363
3364                 DataParseStatus parseStatus = ParseResponseData(ref returnResult, out requestDone, out continueResponseData);
3365
3366                 // copy off m_CurrentRequest as we might start processing a next request before exiting this method
3367                 currentRequest = m_CurrentRequest;
3368
3369                 if (parseStatus != DataParseStatus.NeedMoreData)
3370                     m_CurrentRequest = null;
3371
3372                 if (parseStatus == DataParseStatus.Invalid || parseStatus == DataParseStatus.DataTooBig)
3373                 {
3374                     // Tell the request's SubmitWriteStream that the connection will be closed.  It should ---- any
3375                     // future writes so that the appropriate exception will be received in GetResponse().
3376                     if (currentRequest != null)
3377                     {
3378                         currentRequest.ErrorStatusCodeNotify(this, false, false);
3379                     }
3380
3381                     //
3382                     // report error
3383                     //
3384                     if (parseStatus == DataParseStatus.Invalid) {
3385                         HandleErrorWithReadDone(WebExceptionStatus.ServerProtocolViolation, ref returnResult);
3386                     }
3387                     else {
3388                         HandleErrorWithReadDone(WebExceptionStatus.MessageLengthLimitExceeded, ref returnResult);
3389                     }
3390                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadComplete() parseStatus:" + parseStatus + " returnResult:" + returnResult);
3391                     goto done;
3392                 }
3393
3394                 //Done means the ConnectStream take care of this connection until ConnectStream.CallDone()
3395                 else if (parseStatus == DataParseStatus.Done)
3396                 {
3397                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadComplete() [The response stream is ready] parseStatus = DataParseStatus.Done");
3398                     goto done;
3399                 }
3400
3401                 //
3402                 // we may reach the end of our buffer only when parsing headers.
3403                 // this can happen when the header section is bigger than our initial 4k guess
3404                 // which should be a good assumption in 99.9% of the cases. what we do here is:
3405                 // 1) if there's a single BIG header (bigger than the current size) we will need to
3406                 //    grow the buffer before we move data over and read more data.
3407                 // 2) move unparsed data to the beginning of the buffer and read more data in the
3408                 //    remaining part of the data.
3409                 //
3410                 if (parseStatus == DataParseStatus.NeedMoreData)
3411                 {
3412                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadComplete() OLD buffer. m_ReadBuffer.Length:" + m_ReadBuffer.Length.ToString() + " m_BytesRead:" + m_BytesRead.ToString() + " m_BytesScanned:" + m_BytesScanned.ToString());
3413                     int unparsedDataSize = m_BytesRead - m_BytesScanned;
3414                     if (unparsedDataSize != 0)
3415                     {
3416                         if (m_BytesScanned == 0 && m_BytesRead == m_ReadBuffer.Length)
3417                         {
3418                             //
3419                             // 1) we need to grow the buffer, move the unparsed data to the beginning of the buffer before reading more data.
3420                             // since the buffer size is 4k, should we just double
3421                             //
3422                             byte[] newReadBuffer = new byte[m_ReadBuffer.Length * 2 /*+ ReadBufferSize*/];
3423                             Buffer.BlockCopy(m_ReadBuffer, 0, newReadBuffer, 0, m_BytesRead);
3424
3425                             // if m_ReadBuffer is from the pinnable cache, give it back
3426                             FreeReadBuffer();
3427                             m_ReadBuffer = newReadBuffer;
3428                         }
3429                         else
3430                         {
3431                             //
3432                             // just move data around in the same buffer.
3433                             //
3434                             Buffer.BlockCopy(m_ReadBuffer, m_BytesScanned, m_ReadBuffer, 0, unparsedDataSize);
3435                         }
3436                     }
3437                     //
3438                     // update indexes and offsets in the new buffer
3439                     //
3440                     m_BytesRead = unparsedDataSize;
3441                     m_BytesScanned = 0;
3442                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadComplete() NEW or shifted buffer. m_ReadBuffer.Length:" + m_ReadBuffer.Length.ToString() + " m_BytesRead:" + m_BytesRead.ToString() + " m_BytesScanned:" + m_BytesScanned.ToString());
3443
3444                     if (currentRequest != null)
3445                     {
3446                         //
3447                         // This case means that we still parsing the headers, so need to post another read in the async case
3448
3449                         if (currentRequest.Async)
3450                         {
3451                             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadComplete() Reposting Async Read.  Buffer:" + ValidationHelper.HashString(m_ReadBuffer) + " BytesScanned:" + m_BytesScanned.ToString());
3452
3453                             if (Thread.CurrentThread.IsThreadPoolThread)
3454                             {
3455                                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadComplete() Calling PostReceive().");
3456                                 PostReceive();
3457                             }
3458                             else
3459                             {
3460                                 // Offload to the threadpool to protect against the case where one request's thread posts IO that another request
3461                                 // depends on, but the first thread dies in the mean time.
3462                                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadComplete() ThreadPool.UnsafeQueueUserWorkItem(m_PostReceiveDelegate, this)");
3463                                 ThreadPool.UnsafeQueueUserWorkItem(m_PostReceiveDelegate, this);
3464                             }
3465                         }
3466                     }
3467                 }
3468             }
3469             //
3470             // Any exception is processed by HandleError() and ----ed to avoid throwing on a thread pool
3471             // In the [....] case the HandleError() will abort the request so the caller will pick up the result.
3472             //
3473             catch (Exception exception) {
3474                 if (NclUtilities.IsFatal(exception)) throw;
3475
3476                 requestDone = true;
3477
3478                 if (m_InnerException == null)
3479                     m_InnerException = exception;
3480
3481                 // Notify request's SubmitWriteStream that a socket error happened.  This will cause future writes to
3482                 // throw an IOException.
3483                 HttpWebRequest curRequest = m_CurrentRequest;
3484                 if (curRequest != null)
3485                 {
3486                     curRequest.ErrorStatusCodeNotify(this, false, true);
3487                 }
3488
3489                 HandleErrorWithReadDone(WebExceptionStatus.ReceiveFailure, ref returnResult);
3490             }
3491
3492 done:
3493             try {
3494                 // It is only safe to continue if there was a 100 continue OR buffering is supported.
3495                 if (currentRequest != null && currentRequest.HttpWriteMode != HttpWriteMode.None &&
3496                     (continueResponseData != null
3497                         // not a 100 continue, but we have buffering so we don't care what it was.
3498                         || (returnResult != null && returnResult.IsNotEmpty && currentRequest.AllowWriteStreamBuffering)
3499                     )
3500                    )
3501                 {
3502                     // if returnResult is not empty it must also contain some result for the currently active request
3503                     // Since it could be a POST request waiting on the body submit, signal the body here
3504                     if (currentRequest.FinishContinueWait())
3505                     {
3506                         currentRequest.SetRequestContinue(continueResponseData);
3507                     }
3508                 }
3509             }
3510             finally {
3511                 ConnectionReturnResult.SetResponses(returnResult);
3512             }
3513
3514             return requestDone;
3515         }
3516
3517
3518         //     This method is called by ConnectStream, only when resubmitting
3519         //     We have sent the headers already in HttpWebRequest.EndSubmitRequest()
3520         //     which calls ConnectStream.WriteHeaders() which calls to HttpWebRequest.EndWriteHeaders()
3521         //     which calls ConnectStream.ResubmitWrite() which calls in here
3522         internal void Write(ScatterGatherBuffers writeBuffer) {
3523             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::Write(ScatterGatherBuffers) networkstream#" + ValidationHelper.HashString(NetworkStream));
3524             //
3525             // parameter validation
3526             //
3527             GlobalLog.Assert(writeBuffer != null, "Connection#{0}::Write(ScatterGatherBuffers)|writeBuffer == null", ValidationHelper.HashString(this));
3528             //
3529             // set up array for MultipleWrite call
3530             // note that GetBuffers returns null if we never wrote to it.
3531             //
3532             BufferOffsetSize[] buffers = writeBuffer.GetBuffers();
3533             if (buffers!=null) {
3534                 //
3535                 // This will block writing the buffers out.
3536                 //
3537                 MultipleWrite(buffers);
3538             }
3539             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::Write(ScatterGatherBuffers) this:" + ValidationHelper.HashString(this) + " writeBuffer.Size:" + writeBuffer.Length.ToString());
3540         }
3541
3542
3543         /*++
3544
3545             PostReceiveWrapper - Post a receive from a worker thread.
3546
3547             This is our delegate, used for posting receives from a worker thread.
3548             We do this when we can't be sure that we're already on a worker thread,
3549             and we don't want to post from a client thread because if it goes away
3550             I/O gets cancelled.
3551
3552             Input:
3553
3554             state           - a null object
3555
3556             Returns:
3557
3558         --*/
3559         private static void PostReceiveWrapper(object state) {
3560             Connection thisConnection = state as Connection;
3561             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(thisConnection) + "::PostReceiveWrapper", "Cnt#" + ValidationHelper.HashString(thisConnection));
3562             GlobalLog.Assert(thisConnection != null, "Connection#{0}::PostReceiveWrapper()|thisConnection == null", ValidationHelper.HashString(thisConnection));
3563
3564             thisConnection.PostReceive();
3565
3566             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(thisConnection) + "::PostReceiveWrapper");
3567         }
3568
3569         private void PostReceive()
3570         {
3571             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::PostReceive", "");
3572
3573             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::PostReceive() m_ReadBuffer:" + ValidationHelper.HashString(m_ReadBuffer) + " Length:" + m_ReadBuffer.Length.ToString());
3574
3575             try
3576             {
3577                 GlobalLog.Assert(m_BytesScanned == 0, "PostReceive()|A receive should not be posted when m_BytesScanned != 0 (the data should be moved to offset 0).");
3578
3579                 if (m_LastAsyncResult != null && !m_LastAsyncResult.IsCompleted)
3580                     throw new InternalException();  //This may cause duplicate requests if we let it through in retail
3581
3582
3583                 m_LastAsyncResult = UnsafeBeginRead(m_ReadBuffer, m_BytesRead, m_ReadBuffer.Length - m_BytesRead, m_ReadCallback, this);
3584                 if (m_LastAsyncResult.CompletedSynchronously)
3585                 {
3586                     //
3587                     ReadCallback(m_LastAsyncResult);
3588                 }
3589             }
3590             catch (Exception exception) {
3591                 // Notify request's SubmitWriteStream that a socket error happened.  This will cause future writes to
3592                 // throw an IOException.
3593                 HttpWebRequest curRequest = m_CurrentRequest;
3594                 if (curRequest != null)
3595                 {
3596                     curRequest.ErrorStatusCodeNotify(this, false, true);
3597                 }
3598
3599                 //ASYNCISSUE
3600                 ConnectionReturnResult returnResult = null;
3601                 HandleErrorWithReadDone(WebExceptionStatus.ReceiveFailure, ref returnResult);
3602                 ConnectionReturnResult.SetResponses(returnResult);
3603                 GlobalLog.LeaveException("Connection#" + ValidationHelper.HashString(this) + "::PostReceive", exception);
3604             }
3605
3606             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::PostReceive");
3607         }
3608
3609
3610
3611         private static void TunnelThroughProxyWrapper(IAsyncResult result){
3612             if(result.CompletedSynchronously){
3613                 return;
3614             }
3615
3616             bool success = false;
3617             WebExceptionStatus ws = WebExceptionStatus.ConnectFailure;
3618             HttpWebRequest req = (HttpWebRequest)((LazyAsyncResult)result).AsyncObject;
3619             Connection conn = (Connection)((TunnelStateObject)result.AsyncState).Connection;
3620             HttpWebRequest originalReq = (HttpWebRequest)((TunnelStateObject)result.AsyncState).OriginalRequest;
3621
3622             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(conn) + "::TunnelThroughProxyCallback");
3623
3624             try{
3625                 req.EndGetResponse(result);
3626                 HttpWebResponse connectResponse = (HttpWebResponse)req.GetResponse();
3627                 ConnectStream connectStream = (ConnectStream)connectResponse.GetResponseStream();
3628
3629                 // this stream will be used as the real stream for TlsStream
3630                 conn.NetworkStream = new NetworkStream(connectStream.Connection.NetworkStream, true);
3631                 // This will orphan the original connect stream now owned by tunnelStream
3632                 connectStream.Connection.NetworkStream.ConvertToNotSocketOwner();
3633                 success = true;
3634             }
3635
3636             catch (Exception exception) {
3637                 if (conn.m_InnerException == null)
3638                     conn.m_InnerException = exception;
3639
3640                 if (exception is WebException) {
3641                     ws = ((WebException)exception).Status;
3642                 }
3643
3644                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(conn) + "::TunnelThroughProxyCallback() exception occurred: " + exception);
3645             }
3646             if(!success)
3647             {
3648                 ConnectionReturnResult returnResult = null;
3649                 conn.HandleError(false, false, ws, ref returnResult);
3650                 ConnectionReturnResult.SetResponses(returnResult);
3651                 return;
3652             }
3653
3654             conn.CompleteConnection(true, originalReq);
3655             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(conn) + "::TunnelThroughProxyCallback");
3656         }
3657
3658
3659
3660
3661         //
3662         private bool TunnelThroughProxy(Uri proxy, HttpWebRequest originalRequest, bool async) {
3663             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::TunnelThroughProxy", "proxy="+proxy+", async="+async+", originalRequest #"+ValidationHelper.HashString(originalRequest));
3664
3665             bool result = false;
3666             HttpWebRequest connectRequest = null;
3667             HttpWebResponse connectResponse = null;
3668
3669             try {
3670                 (new WebPermission(NetworkAccess.Connect, proxy)).Assert();
3671                 try {
3672                     connectRequest = new HttpWebRequest(
3673                         proxy,
3674                         originalRequest.Address,
3675                        // new Uri("https://" + originalRequest.Address.GetParts(UriComponents.HostAndPort, UriFormat.UriEscaped)),
3676                         originalRequest
3677                         );
3678                 }
3679                 finally {
3680                     WebPermission.RevertAssert();
3681                 }
3682
3683                 connectRequest.Credentials = originalRequest.InternalProxy == null ? null : originalRequest.InternalProxy.Credentials;
3684                 connectRequest.InternalProxy = null;
3685                 connectRequest.PreAuthenticate = true;
3686
3687                 if(async){
3688                     TunnelStateObject o = new TunnelStateObject(originalRequest, this);
3689                     IAsyncResult asyncResult = connectRequest.BeginGetResponse(m_TunnelCallback, o);
3690                     if(!asyncResult.CompletedSynchronously){
3691                         GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::TunnelThroughProxy completed asynchronously", true);
3692                         return true;
3693                     }
3694                     connectResponse = (HttpWebResponse)connectRequest.EndGetResponse(asyncResult);
3695                 }
3696                 else{
3697                     connectResponse = (HttpWebResponse)connectRequest.GetResponse();
3698                 }
3699
3700                 ConnectStream connectStream = (ConnectStream)connectResponse.GetResponseStream();
3701
3702                 // this stream will be used as the real stream for TlsStream
3703                 NetworkStream = new NetworkStream(connectStream.Connection.NetworkStream, true);
3704                 // This will orphan the original connect stream now owned by tunnelStream
3705                 connectStream.Connection.NetworkStream.ConvertToNotSocketOwner();
3706                 result = true;
3707             }
3708             catch (Exception exception) {
3709                 if (m_InnerException == null)
3710                     m_InnerException = exception;
3711                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::TunnelThroughProxy() exception occurred: " + exception);
3712             }
3713
3714             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::TunnelThroughProxy", result);
3715
3716             return result;
3717         }
3718
3719         //
3720         // CheckNonIdle - called after update of the WriteList/WaitList,
3721         //   upon the departure of our Idle state our, BusyCount will
3722         //   go to non-0, then we need to mark this transition
3723         //
3724
3725         private void CheckNonIdle() {
3726             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CheckNonIdle()");
3727             if (m_Idle && this.BusyCount != 0) {
3728                 m_Idle = false;
3729                 ServicePoint.IncrementConnection();
3730                 ConnectionGroup.IncrementConnection();
3731             }
3732         }
3733
3734         //
3735         // CheckIdle - called after update of the WriteList/WaitList,
3736         //    specifically called after we remove entries
3737         //
3738
3739         private void CheckIdle() {
3740             // The timer thread is allowed to call this.  (It doesn't call user code and doesn't block.)
3741             GlobalLog.ThreadContract(ThreadKinds.Unknown, ThreadKinds.SafeSources | ThreadKinds.Finalization | ThreadKinds.Timer, "Connection#" + ValidationHelper.HashString(this) + "::CheckIdle");
3742
3743             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CheckIdle() m_Idle = " + m_Idle + ", BusyCount = " + BusyCount);
3744             if (!m_Idle && this.BusyCount == 0)  {
3745                 m_Idle = true;
3746                 ServicePoint.DecrementConnection();
3747                 if (ConnectionGroup != null) {
3748                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CheckIdle() - calling ConnectionGoneIdle()");
3749                     ConnectionGroup.DecrementConnection();
3750                     ConnectionGroup.ConnectionGoneIdle();
3751                 }
3752                 // Remember the moment when this connection went idle.
3753                 m_IdleSinceUtc = DateTime.UtcNow;
3754             }
3755         }
3756
3757         //
3758         // DebugDumpArrayListEntries - debug goop
3759         //
3760         [Conditional("TRAVE")]
3761         private void DebugDumpWriteListEntries() {
3762             for (int i = 0; i < m_WriteList.Count; i++)
3763             {
3764                 DebugDumpListEntry(i, m_WriteList[i] as HttpWebRequest, "WriteList");
3765             }
3766         }
3767
3768         [Conditional("TRAVE")]
3769         private void DebugDumpWaitListEntries() {
3770             for (int i = 0; i < m_WaitList.Count; i++)
3771             {
3772                 DebugDumpListEntry(i, m_WaitList[i].Request, "WaitList");
3773             }
3774         }
3775
3776         [Conditional("TRAVE")]
3777         private void DebugDumpListEntry(int currentPos, HttpWebRequest req, string listType) {
3778             GlobalLog.Print("WaitList[" + currentPos.ToString() + "] Req: #" +
3779                 ValidationHelper.HashString(req));
3780         }
3781
3782         //
3783         // Validation & debugging
3784         //
3785         [System.Diagnostics.Conditional("DEBUG")]
3786         internal void DebugMembers(int requestHash) {
3787 #if TRAVE
3788             bool dump = requestHash==0;
3789             GlobalLog.Print("Cnt#" + this.GetHashCode());
3790             if (!dump) {
3791                 foreach(HttpWebRequest request in  m_WriteList) {
3792                     if (request.GetHashCode() == requestHash) {
3793                         GlobalLog.Print("Found-WriteList");
3794                         Dump();
3795                         return;
3796                     }
3797                 }
3798                 foreach(WaitListItem item in m_WaitList) {
3799                     if (item.Request.GetHashCode() == requestHash) {
3800                         GlobalLog.Print("Found-WaitList");
3801                         Dump();
3802                         return;
3803                     }
3804                 }
3805             }
3806             else {
3807                 Dump();
3808                 DebugDumpWriteListEntries();
3809                 DebugDumpWaitListEntries();
3810             }
3811 #endif
3812         }
3813
3814 #if TRAVE
3815         [System.Diagnostics.Conditional("TRAVE")]
3816         internal void Dump() {
3817             GlobalLog.Print("CanPipeline:" + m_CanPipeline);
3818             GlobalLog.Print("Pipelining:" + m_Pipelining);
3819             GlobalLog.Print("KeepAlive:" + m_KeepAlive);
3820             GlobalLog.Print("m_Error:" + m_Error);
3821             GlobalLog.Print("m_ReadBuffer:" + m_ReadBuffer);
3822             GlobalLog.Print("m_BytesRead:" + m_BytesRead);
3823             GlobalLog.Print("m_BytesScanned:" + m_BytesScanned);
3824             GlobalLog.Print("m_ResponseData:" + m_ResponseData);
3825             GlobalLog.Print("m_ReadState:" + m_ReadState);
3826             GlobalLog.Print("m_StatusState:" + m_StatusState);
3827             GlobalLog.Print("ConnectionGroup:" + ConnectionGroup);
3828             GlobalLog.Print("Idle:" + m_Idle);
3829             GlobalLog.Print("ServicePoint:" + ServicePoint);
3830             GlobalLog.Print("m_Version:" + ServicePoint.ProtocolVersion);
3831             GlobalLog.Print("NetworkStream:" + NetworkStream);
3832 #if !FEATURE_PAL
3833             if ( NetworkStream is TlsStream) {
3834                 TlsStream tlsStream = NetworkStream as TlsStream;
3835                 tlsStream.DebugMembers();
3836             }
3837             else
3838 #endif // !FEATURE_PAL
3839             if (NetworkStream != null) {
3840                 NetworkStream.DebugMembers();
3841             }
3842             GlobalLog.Print("m_AbortDelegate:" + m_AbortDelegate);
3843             GlobalLog.Print("ReadDone:" + m_ReadDone);
3844             GlobalLog.Print("WriteDone:" + m_WriteDone);
3845             GlobalLog.Print("Free:" + m_Free);
3846         }
3847 #endif
3848     }
3849 }