mcs/class/referencesource/System/net/System/Net/_Connection.cs

   1 //------------------------------------------------------------------------------
   2 // <copyright file="_Connection.cs" company="Microsoft">
   3 //     Copyright (c) Microsoft Corporation.  All rights reserved.
   4 // </copyright>
   5 //------------------------------------------------------------------------------
   6
   7 namespace System.Net {
   8
   9     using System;
  10     using System.Collections;
  11     using System.Collections.Generic;
  12     using System.Diagnostics;
  13     using System.Net.Sockets;
  14     using System.Threading;
  15     using System.Security;
  16     using System.Globalization;
  17     using System.Net.Configuration;
  18     using System.Diagnostics.CodeAnalysis;
  19
  20 #if !MONO
  21     internal enum ReadState {
  22         Start,
  23         StatusLine, // about to parse status line
  24         Headers,    // reading headers
  25         Data        // now read data
  26     }
  27 #endif
  28
  29     internal enum DataParseStatus {
  30         NeedMoreData = 0,   // need more data
  31         ContinueParsing,    // continue parsing
  32         Done,               // done
  33         Invalid,            // bad data format
  34         DataTooBig,         // data exceeds the allowed size
  35     }
  36
  37     internal enum WriteBufferState {
  38         Disabled,
  39         Headers,
  40         Buffer,
  41         Playback,
  42     }
  43
  44     // The enum lietrals will be displayed to the user in the exception message
  45     internal enum WebParseErrorSection {
  46         Generic,
  47         ResponseHeader,
  48         ResponseStatusLine,
  49         ResponseBody
  50     }
  51
  52     // The enum literal will be used to look up an error string in the resource file
  53     internal enum WebParseErrorCode {
  54         Generic,
  55         InvalidHeaderName,
  56         InvalidContentLength,
  57         IncompleteHeaderLine,
  58         CrLfError,
  59         InvalidChunkFormat,
  60         UnexpectedServerResponse
  61     }
  62
  63     // Only defined for DataParseStatus.Invalid
  64     struct WebParseError {
  65         public WebParseErrorSection  Section;
  66         public WebParseErrorCode     Code;
  67     }
  68
  69 #if !MONO
  70     struct TunnelStateObject {
  71         internal TunnelStateObject(HttpWebRequest r, Connection c){
  72             Connection = c;
  73             OriginalRequest = r;
  74         }
  75
  76         internal Connection Connection;
  77         internal HttpWebRequest OriginalRequest;
  78     }
  79
  80     //
  81     // ConnectionReturnResult - used to spool requests that have been completed,
  82     //  and need to be notified.
  83     //
  84
  85     internal class ConnectionReturnResult {
  86
  87         private static readonly WaitCallback s_InvokeConnectionCallback = new WaitCallback(InvokeConnectionCallback);
  88
  89         private struct RequestContext {
  90             internal HttpWebRequest      Request;
  91             internal object              CoreResponse;
  92
  93             internal RequestContext(HttpWebRequest request, object coreResponse)
  94             {
  95                 Request = request;
  96                 CoreResponse = coreResponse;
  97             }
  98         }
  99
 100         private List<RequestContext>  m_Context;
 101
 102         internal ConnectionReturnResult()
 103         {
 104             m_Context = new List<RequestContext>(5);
 105         }
 106
 107         internal ConnectionReturnResult(int capacity)
 108         {
 109             m_Context = new List<RequestContext>(capacity);
 110         }
 111
 112         internal bool IsNotEmpty {
 113             get {
 114                 return m_Context.Count != 0;
 115             }
 116         }
 117
 118         internal static void Add(ref ConnectionReturnResult returnResult, HttpWebRequest request, CoreResponseData coreResponseData)
 119         {
 120             if (coreResponseData == null)
 121                 throw new InternalException(); //This may cause duplicate requests if we let it through in retail
 122
 123             if (returnResult == null) {
 124                 returnResult = new ConnectionReturnResult();
 125             }
 126
 127 #if DEBUG
 128             //This may cause duplicate requests if we let it through in retail but it's may be expensive to catch here
 129             for (int j = 0; j <  returnResult.m_Context.Count; ++j)
 130                 if ((object)returnResult.m_Context[j].Request == (object) request)
 131                     throw new InternalException();
 132 #endif
 133
 134             returnResult.m_Context.Add(new RequestContext(request, coreResponseData));
 135         }
 136
 137         internal static void AddExceptionRange(ref ConnectionReturnResult returnResult, HttpWebRequest [] requests, Exception exception)
 138         {
 139             AddExceptionRange(ref returnResult, requests, exception, exception);
 140         }
 141         internal static void AddExceptionRange(ref ConnectionReturnResult returnResult, HttpWebRequest [] requests, Exception exception, Exception firstRequestException)
 142         {
 143
 144             //This may cause duplicate requests if we let it through in retail
 145             if (exception == null)
 146                 throw new InternalException();
 147
 148             if (returnResult == null) {
 149                 returnResult = new ConnectionReturnResult(requests.Length);
 150             }
 151             // "abortedRequestExeption" is assigned to the "abortedRequest" or to the very first request if the latest is null
 152             // Everyone else will get "exception"
 153             for (int i = 0; i < requests.Length; ++i)
 154             {
 155 #if DEBUG
 156                 //This may cause duplicate requests if we let it through in retail but it's may be expensive to catch here
 157                 for (int j = 0; j <  returnResult.m_Context.Count; ++j)
 158                     if ((object)returnResult.m_Context[j].Request == (object) requests[i])
 159                         throw new InternalException();
 160 #endif
 161
 162                 if (i == 0)
 163                     returnResult.m_Context.Add(new RequestContext(requests[i], firstRequestException));
 164                 else
 165                     returnResult.m_Context.Add(new RequestContext(requests[i], exception));
 166             }
 167         }
 168
 169         internal static void SetResponses(ConnectionReturnResult returnResult) {
 170             if (returnResult==null){
 171                 return;
 172             }
 173
 174             GlobalLog.Print("ConnectionReturnResult#" + ValidationHelper.HashString(returnResult) + "::SetResponses() count=" + returnResult.m_Context.Count.ToString());
 175             for (int i = 0; i < returnResult.m_Context.Count; i++)
 176             {
 177                 try {
 178                     HttpWebRequest request = returnResult.m_Context[i].Request;
 179 #if DEBUG
 180                     CoreResponseData coreResponseData = returnResult.m_Context[i].CoreResponse as CoreResponseData;
 181                     if (coreResponseData == null)
 182                         GlobalLog.DebugRemoveRequest(request);
 183 #endif
 184                     request.SetAndOrProcessResponse(returnResult.m_Context[i].CoreResponse);
 185                 }
 186                 catch(Exception e) {
 187                     //ASYNCISSUE
 188                     // on error, with more than one callback need to queue others off to another thread
 189
 190                     GlobalLog.Print("ConnectionReturnResult#" + ValidationHelper.HashString(returnResult) + "::Exception"+e);
 191                     returnResult.m_Context.RemoveRange(0,(i+1));
 192                     if (returnResult.m_Context.Count > 0)
 193                     {
 194                         ThreadPool.UnsafeQueueUserWorkItem(s_InvokeConnectionCallback, returnResult);
 195                     }
 196                     throw;
 197                 }
 198             }
 199
 200             returnResult.m_Context.Clear();
 201         }
 202
 203         private static void InvokeConnectionCallback(object objectReturnResult)
 204         {
 205             ConnectionReturnResult returnResult = (ConnectionReturnResult)objectReturnResult;
 206             SetResponses(returnResult);
 207         }
 208     }
 209
 210     //
 211     // Connection - this is the Connection used to parse
 212     //   server responses, queue requests, and pipeline requests
 213     //
 214     internal class Connection : PooledStream {
 215
 216         //
 217         // thread statics - these values must be per thread, because
 218         //  other requests and operations can take place concurrently on this Connection.
 219         //  Our concern is to make sure that a nested call does not get confused with an
 220         //  operation on another thread.   Parameter passing cannot be used, because
 221         //  the call stack may exit and then reenter the same Connection object.
 222         //
 223         [ThreadStatic]
 224         private static int t_SyncReadNesting;
 225
 226         private const int CRLFSize = 2;
 227         private const long c_InvalidContentLength = -2L;
 228
 229         //
 230         // Buffer manager that allocates and reuses 4k buffers.
 231         //
 232         private const int CachedBufferSize = 4096;
 233         private static PinnableBufferCache s_PinnableBufferCache = new PinnableBufferCache("System.Net.Connection", CachedBufferSize);
 234
 235         //
 236         // Little status line holder.
 237         //
 238         private class StatusLineValues
 239         {
 240             internal int MajorVersion;
 241             internal int MinorVersion;
 242             internal int StatusCode;
 243             internal string StatusDescription;
 244         }
 245
 246         private class WaitListItem
 247         {
 248             private HttpWebRequest request;
 249             private long queueStartTime;
 250
 251             public HttpWebRequest Request
 252             {
 253                 get { return request; }
 254             }
 255
 256             public long QueueStartTime
 257             {
 258                 get { return queueStartTime; }
 259             }
 260
 261             public WaitListItem(HttpWebRequest request, long queueStartTime)
 262             {
 263                 this.request = request;
 264                 this.queueStartTime = queueStartTime;
 265             }
 266         }
 267
 268         //
 269         // class members
 270         //
 271         private WebExceptionStatus  m_Error;
 272         internal Exception           m_InnerException;
 273
 274
 275         internal int                m_IISVersion = -1; //-1 means unread
 276         private byte[]              m_ReadBuffer;
 277         private bool                m_ReadBufferFromPinnableCache; // If we get our m_readBuffer from the Pinnable cache we have to explicitly free it
 278         private int                 m_BytesRead;
 279         private int                 m_BytesScanned;
 280         private int                 m_TotalResponseHeadersLength;
 281         private int                 m_MaximumResponseHeadersLength;
 282         private long                m_MaximumUnauthorizedUploadLength;
 283         private CoreResponseData    m_ResponseData;
 284         private ReadState           m_ReadState;
 285         private StatusLineValues    m_StatusLineValues;
 286         private int                 m_StatusState;
 287         private List<WaitListItem>  m_WaitList;
 288         private ArrayList           m_WriteList;
 289         private IAsyncResult        m_LastAsyncResult;
 290         private TimerThread.Timer   m_RecycleTimer;
 291         private WebParseError       m_ParseError;
 292         private bool                m_AtLeastOneResponseReceived;
 293
 294         private static readonly WaitCallback m_PostReceiveDelegate = new WaitCallback(PostReceiveWrapper);
 295         private static readonly AsyncCallback m_ReadCallback = new AsyncCallback(ReadCallbackWrapper);
 296         private static readonly AsyncCallback m_TunnelCallback = new AsyncCallback(TunnelThroughProxyWrapper);
 297         private static byte[] s_NullBuffer = new byte[0];
 298
 299         //
 300         // Abort handling variables. When trying to abort the
 301         // connection, we set Aborted = true, and close m_AbortSocket
 302         // if its non-null. m_AbortDelegate, is returned to every
 303         // request from our SubmitRequest method.  Calling m_AbortDelegate
 304         // drives us into Abort mode.
 305         //
 306         private HttpAbortDelegate m_AbortDelegate;
 307         private ConnectionGroup   m_ConnectionGroup;
 308
 309         private UnlockConnectionDelegate m_ConnectionUnlock;
 310
 311         //
 312         // ReadDone and m_Write - no two vars are so complicated,
 313         //  as these two. Used for m_WriteList managment, most be under crit
 314         //  section when accessing.
 315         //
 316         // ReadDone tracks the item at the end or
 317         //  just recenlty removed from the m_WriteList. While a
 318         //  pending BeginRead is in place, we need this to be false, in
 319         //  order to indicate to tell the WriteDone callback, that we can
 320         //  handle errors/resets.  The only exception is when the m_WriteList
 321         //  is empty, and there are no outstanding requests, then all it can
 322         //  be true.
 323         //
 324         // WriteDone tracks the item just added at the begining of the m_WriteList.
 325         //  this needs to be false while we about to write something, but have not
 326         //  yet begin or finished the write.  Upon completion, its set to true,
 327         //  so that DoneReading/ReadStartNextRequest can close the socket, without fear
 328         //  of a errand writer still banging away on another thread.
 329         //
 330
 331         private DateTime        m_IdleSinceUtc;
 332         private HttpWebRequest  m_LockedRequest;
 333         private HttpWebRequest  m_CurrentRequest; // This is the request whose response is being parsed, same as WriteList[0] but could be different if request was aborted.
 334         private bool m_CanPipeline;
 335         private bool m_Free = true;
 336         private bool m_Idle = true;
 337         private bool m_KeepAlive = true;
 338         private bool m_Pipelining;
 339         private int m_ReservedCount;
 340         private bool m_ReadDone;
 341         private bool m_WriteDone;
 342         private bool m_RemovedFromConnectionList;
 343         private bool m_NonKeepAliveRequestPipelined;
 344
 345         // Pipeline Throttling: m_IsPipelinePaused==true when we stopped and false when it's ok to add to the pipeline.
 346         private bool                m_IsPipelinePaused;
 347         private static int          s_MaxPipelinedCount = 10;
 348         private static int          s_MinPipelinedCount = 5;
 349
 350 #if TRAVE
 351         private bool q_Tunnelling;
 352 #endif
 353
 354         internal override ServicePoint ServicePoint {
 355             get {
 356                 return ConnectionGroup.ServicePoint;
 357             }
 358         }
 359
 360         private ConnectionGroup ConnectionGroup {
 361             get {
 362                 return m_ConnectionGroup;
 363             }
 364         }
 365
 366         //
 367         // LockedRequest is the request that needs exclusive access to this connection
 368         // the ConnectionGroup should proctect the Connection object from any new
 369         // Requests being queued, until this m_LockedRequest is finished.
 370         //
 371         internal HttpWebRequest LockedRequest {
 372             get {
 373                 return m_LockedRequest;
 374             }
 375             set {
 376                 HttpWebRequest myLock = m_LockedRequest;
 377
 378                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::LockedRequest_set() old#"+ ((myLock!=null)?myLock.GetHashCode().ToString():"null") +  " new#" + ((value!=null)?value.GetHashCode().ToString():"null"));
 379
 380                 if ((object)value == (object)myLock)
 381                 {
 382                     if (value != null && (object)value.UnlockConnectionDelegate != (object) m_ConnectionUnlock)
 383                     {
 384                         throw new InternalException();
 385                     }
 386                     return;
 387                 }
 388
 389                 object myDelegate = myLock == null? null: myLock.UnlockConnectionDelegate;
 390                 if (myDelegate != null && (value != null || (object)m_ConnectionUnlock != (object)myDelegate))
 391                     throw new InternalException();
 392
 393                 if (value == null)
 394                 {
 395                     m_LockedRequest = null;
 396                     myLock.UnlockConnectionDelegate = null;
 397                     return;
 398                 }
 399
 400                 UnlockConnectionDelegate chkDelegate = value.UnlockConnectionDelegate;
 401                 //
 402                 // If "value" request was already locking a connection that is not "this", unlock that other connection
 403                 //
 404                 if ((object)chkDelegate != null)
 405                 {
 406                     if ((object)chkDelegate == (object)m_ConnectionUnlock)
 407                         throw new InternalException();
 408
 409                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::LockedRequest_set() Unlocking old request Connection");
 410                     chkDelegate();
 411                 }
 412
 413                 value.UnlockConnectionDelegate = m_ConnectionUnlock;
 414                 m_LockedRequest = value;
 415             }
 416         }
 417
 418
 419         /// <devdoc>
 420         ///    <para>
 421         ///       Delegate called when the request is finished using this Connection
 422         ///         exclusively.  Called in Abort cases and after NTLM authenticaiton completes.
 423         ///    </para>
 424         /// </devdoc>
 425         private void UnlockRequest() {
 426             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::UnlockRequest() LockedRequest#" + ValidationHelper.HashString(LockedRequest));
 427
 428             LockedRequest = null;
 429
 430             if (ConnectionGroup != null) {
 431                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::UnlockRequest() - forcing call to ConnectionGoneIdle()");
 432                 ConnectionGroup.ConnectionGoneIdle();
 433             }
 434         }
 435
 436
 437 #if TRAVE
 438         /*
 439         private string MyLocalEndPoint {
 440             get {
 441                 try {
 442                     return NetworkStream.InternalSocket.LocalEndPoint.ToString();
 443                 }
 444                 catch {
 445                     return "no connection";
 446                 }
 447             }
 448         }
 449         */
 450
 451         private string MyLocalPort {
 452             get {
 453                 try {
 454                     if (NetworkStream == null || !NetworkStream.Connected) {
 455                         return "no connection";
 456                     }
 457                     return ((IPEndPoint)NetworkStream.InternalSocket.LocalEndPoint).Port.ToString();
 458                 }
 459                 catch {
 460                     return "no connection";
 461                 }
 462             }
 463         }
 464 #endif
 465
 466         internal Connection(ConnectionGroup connectionGroup) : base(null) {
 467             //
 468             // add this Connection to the pool in the connection group,
 469             //  keep a weak reference to it
 470             //
 471             m_MaximumUnauthorizedUploadLength = SettingsSectionInternal.Section.MaximumUnauthorizedUploadLength;
 472             if(m_MaximumUnauthorizedUploadLength > 0){
 473                 m_MaximumUnauthorizedUploadLength*=1024;
 474             }
 475             m_ResponseData = new CoreResponseData();
 476             m_ConnectionGroup = connectionGroup;
 477             m_ReadBuffer = s_PinnableBufferCache.AllocateBuffer();
 478             m_ReadBufferFromPinnableCache = true;
 479             m_ReadState = ReadState.Start;
 480             m_WaitList = new List<WaitListItem>();
 481             m_WriteList = new ArrayList();
 482             m_AbortDelegate = new HttpAbortDelegate(AbortOrDisassociate);
 483             m_ConnectionUnlock = new UnlockConnectionDelegate(UnlockRequest);
 484
 485             // for status line parsing
 486             m_StatusLineValues = new StatusLineValues();
 487             m_RecycleTimer = ConnectionGroup.ServicePoint.ConnectionLeaseTimerQueue.CreateTimer();
 488             // the following line must be the last line of the constructor
 489             ConnectionGroup.Associate(this);
 490             m_ReadDone = true;
 491             m_WriteDone = true;
 492             m_Error = WebExceptionStatus.Success;
 493             if (PinnableBufferCacheEventSource.Log.IsEnabled())
 494             {
 495                 PinnableBufferCacheEventSource.Log.DebugMessage1("CTOR: In System.Net.Connection.Connnection", this.GetHashCode());
 496             }
 497         }
 498
 499         ~Connection() {
 500             if (m_ReadBufferFromPinnableCache)
 501             {
 502                 if (PinnableBufferCacheEventSource.Log.IsEnabled())
 503                 {
 504                     PinnableBufferCacheEventSource.Log.DebugMessage1("DTOR: ERROR Needing to Free m_ReadBuffer in Connection Destructor", m_ReadBuffer.GetHashCode());
 505                 }
 506             }
 507             FreeReadBuffer();
 508         }
 509
 510         // If the buffer came from the the pinnable cache, return it to the cache.
 511         // NOTE: This method is called from this object's finalizer and should not access any member objects.
 512         void FreeReadBuffer() {
 513             if (m_ReadBufferFromPinnableCache) {
 514                 s_PinnableBufferCache.FreeBuffer(m_ReadBuffer);
 515                 m_ReadBufferFromPinnableCache = false;
 516             }
 517             m_ReadBuffer = null;
 518         }
 519
 520         protected override void Dispose(bool disposing) {
 521             if (PinnableBufferCacheEventSource.Log.IsEnabled()) {
 522                 PinnableBufferCacheEventSource.Log.DebugMessage1("In System.Net.Connection.Dispose()", this.GetHashCode());
 523             }
 524             FreeReadBuffer();
 525             base.Dispose(disposing);
 526         }
 527
 528         internal int BusyCount {
 529             get {
 530                 return (m_ReadDone?0:1) + 2 * (m_WaitList.Count + m_WriteList.Count) + m_ReservedCount;
 531             }
 532         }
 533
 534         internal int IISVersion{
 535             get{
 536                 return m_IISVersion;
 537             }
 538         }
 539
 540         internal bool AtLeastOneResponseReceived {
 541             get {
 542                 return m_AtLeastOneResponseReceived;
 543             }
 544         }
 545
 546         /*++
 547
 548             SubmitRequest       - Submit a request for sending.
 549
 550             The core submit handler. This is called when a request needs to be
 551             submitted to the network. This routine is asynchronous; the caller
 552             passes in an HttpSubmitDelegate that we invoke when the caller
 553             can use the underlying network. The delegate is invoked with the
 554             stream that it can right to.
 555
 556             On the Sync path, we work by attempting to gain control of the Connection
 557             for writing and reading.  If some other thread is using the Connection,
 558             We wait inside of a LazyAsyncResult until it is availble.
 559
 560
 561             Input:
 562                     request                 - request that's being submitted.
 563                     SubmitDelegate          - Delegate to be invoked.
 564                     forcedsubmit            - Queue the request even if connection is going to close.
 565
 566             Returns:
 567                     true when the request was correctly submitted
 568
 569         --*/
 570         // userReqeustThread says whether we can post IO from this thread or not.
 571         [SuppressMessage("Microsoft.Reliability","CA2002:DoNotLockOnObjectsWithWeakIdentity", Justification="Re-Baseline System violations from 3.5 SP1 due to added parameter")]
 572         internal bool SubmitRequest(HttpWebRequest request, bool forcedsubmit)
 573         {
 574             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest", "request#" + ValidationHelper.HashString(request));
 575             GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest");
 576             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest() Free:" + m_Free.ToString() + " m_WaitList.Count:" + m_WaitList.Count.ToString());
 577
 578             TriState startRequestResult = TriState.Unspecified;
 579             ConnectionReturnResult returnResult = null;
 580             bool expiredIdleConnection = false;
 581
 582             // See if the connection is free, and if the underlying socket or
 583             // stream is set up. If it is, we can assign this connection to the
 584             // request right now. Otherwise we'll have to put this request on
 585             // on the wait list until it its turn.
 586
 587             lock(this)
 588             {
 589                 request.AbortDelegate = m_AbortDelegate;
 590
 591                 if (request.Aborted)
 592                 {
 593                     // Note that request is not on the connection list yet and Abort() will push the response on the request
 594                     GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest - (Request was aborted before being submitted)", true);
 595                     UnlockIfNeeded(request);
 596                     return true;
 597                 }
 598                 //
 599                 // There is a race condition between FindConnection and PrepareCloseConnectionSocket
 600                 // Some request may already try to submit themselves while the connection is dying.
 601                 //
 602                 // Retry if that's the case
 603                 //
 604                 if (!CanBePooled)
 605                 {
 606                     GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest", "false - can't be pooled");
 607                     UnlockIfNeeded(request);
 608                     return false;
 609                 }
 610
 611                 //
 612                 // There is a race condition between SubmitRequest and FindConnection. A non keep-alive request may
 613                 // get submitted on this connection just after we check for it. So make sure that if we are queueing
 614                 // behind non keep-alive request then its a forced submit.
 615                 // Retry if that's not the case.
 616                 //
 617                 if (!forcedsubmit && NonKeepAliveRequestPipelined)
 618                 {
 619                     GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest", "false - behind non keep-alive request");
 620                     UnlockIfNeeded(request);
 621                     return false;
 622                 }
 623
 624                 // See if our timer still matches the SerivcePoint.  If not, get rid of it.
 625                 if (m_RecycleTimer.Duration != ServicePoint.ConnectionLeaseTimerQueue.Duration) {
 626                     m_RecycleTimer.Cancel();
 627                     m_RecycleTimer = ServicePoint.ConnectionLeaseTimerQueue.CreateTimer();
 628                 }
 629
 630                 if (m_RecycleTimer.HasExpired) {
 631                     request.KeepAlive = false;
 632                 }
 633
 634                 //
 635                 // If the connection has already been locked by another request, then
 636                 // we fail the submission on this Connection.
 637                 //
 638
 639                 if (LockedRequest != null && LockedRequest != request) {
 640                     GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest", "false");
 641                     return false;
 642                 }
 643
 644
 645                 //free means no one in the wait list.  We should only add a request
 646                 //if the request can pipeline, or pipelining isn't available
 647
 648                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest WriteDone:" + m_WriteDone.ToString() + ", ReadDone:" + m_ReadDone.ToString() + ", m_WriteList.Count:" + m_WriteList.Count.ToString());
 649
 650                 //
 651                 // If this request is marked as non keep-alive, we should stop pipelining more requests on this
 652                 // connection. The keep-alive context is transfered to the connection from request only after we start
 653                 // receiving response for the request.
 654                 //
 655                 if (!forcedsubmit && !m_NonKeepAliveRequestPipelined) {
 656                     m_NonKeepAliveRequestPipelined = (!request.KeepAlive && !request.NtlmKeepAlive);
 657                 }
 658
 659                 if (m_Free && m_WriteDone && !forcedsubmit && (m_WriteList.Count == 0 || (request.Pipelined && !request.HasEntityBody && m_CanPipeline && m_Pipelining && !m_IsPipelinePaused))) {
 660
 661                     // Connection is free. Mark it as busy and see if underlying
 662                     // socket is up.
 663                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest - Free ");
 664                     m_Free = false;
 665
 666                     // This codepath handles the case where the server has closed the Connection by
 667                     // returning false below: the request will be resubmitted on a different Connection.
 668                     startRequestResult = StartRequest(request, true);
 669                     if (startRequestResult == TriState.Unspecified)
 670                     {
 671                         expiredIdleConnection = true;
 672                         PrepareCloseConnectionSocket(ref returnResult);
 673                         // Hard Close the socket.
 674                         Close(0);
 675                         FreeReadBuffer();       // Do it after close completes to insure buffer not in use
 676                     }
 677                 }
 678                 else {
 679                     m_WaitList.Add(new WaitListItem(request, NetworkingPerfCounters.GetTimestamp()));
 680                     NetworkingPerfCounters.Instance.Increment(NetworkingPerfCounterName.HttpWebRequestQueued);
 681                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest - Request added to WaitList#"+ValidationHelper.HashString(request));
 682 #if TRAVE
 683                     if (q_Tunnelling) {
 684                         GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest() MyLocalPort:" + MyLocalPort + " ERROR: adding HttpWebRequest#" + ValidationHelper.HashString(request) +" to tunnelling WaitList");
 685                     }
 686                     else {
 687                         GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest() MyLocalPort:" + MyLocalPort + " adding HttpWebRequest#" + ValidationHelper.HashString(request) +" to non-tunnelling WaitList m_WaitList.Count:" + m_WaitList.Count);
 688                     }
 689 #endif
 690                     CheckNonIdle();
 691                 }
 692             }
 693
 694             if (expiredIdleConnection)
 695             {
 696                 GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest(), expired idle connection", false);
 697                 ConnectionReturnResult.SetResponses(returnResult);
 698                 return false;
 699             }
 700
 701             GlobalLog.DebugAddRequest(request, this, 0);
 702             if(Logging.On)Logging.Associate(Logging.Web, this, request);
 703
 704             if (startRequestResult != TriState.Unspecified) {
 705                 CompleteStartRequest(true, request, startRequestResult);
 706             }
 707             // On [....], we wait for the Connection to be come availble here,
 708             if (!request.Async)
 709             {
 710                 object responseObject = request.ConnectionAsyncResult.InternalWaitForCompletion();
 711                 ConnectStream writeStream = responseObject as ConnectStream;
 712                 AsyncTriState triStateAsync = null;
 713                 if (writeStream == null)
 714                     triStateAsync = responseObject as AsyncTriState;
 715
 716                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest() Pipelining:"+m_Pipelining);
 717
 718                 if (startRequestResult == TriState.Unspecified && triStateAsync != null) {
 719                     // May need to recreate Connection here (i.e. call Socket.Connect)
 720                     CompleteStartRequest(true, request, triStateAsync.Value);
 721                 }
 722                 else if (writeStream != null)
 723                 {
 724                     // return the Stream to the Request
 725                     request.SetRequestSubmitDone(writeStream);
 726                 }
 727 #if DEBUG
 728                 else if (responseObject is Exception)
 729                 {
 730                     Exception exception = responseObject as Exception;
 731                     WebException webException = responseObject as WebException;
 732                     GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest (SYNC) - Error waiting for a connection: " + exception.Message,
 733                                     "Status:" + (webException == null? exception.GetType().FullName: (webException.Status.ToString() +  " Internal Status: " + webException.InternalStatus.ToString())));
 734                     return true;
 735                 }
 736 #endif
 737             }
 738
 739             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::SubmitRequest", true);
 740             return true;
 741         }
 742
 743         private void UnlockIfNeeded(HttpWebRequest request) {
 744             if (LockedRequest == request) {
 745                 UnlockRequest();
 746             }
 747         }
 748
 749         // Wrapper for TriState for marhshalling across Thread boundaries
 750         private class AsyncTriState {
 751             public TriState Value;
 752             public AsyncTriState(TriState newValue) {
 753                 Value = newValue;
 754             }
 755         }
 756
 757         /*++
 758
 759             StartRequest       - Start a request going.
 760
 761             Routine to start a request. Called when we know the connection is
 762             free and we want to get a request going. This routine initializes
 763             some state, adds the request to the write queue, and checks to
 764             see whether or not the underlying connection is alive. If it's
 765             not, it queues a request to get it going. If the connection
 766             was alive we call the callback delegate of the request.
 767
 768             This routine MUST BE called with the critcal section held.
 769
 770             Input:
 771                     request                 - request that's being started.
 772                     canPollRead             - whether the calling code handles
 773                                               Unspecified due to the Connection
 774                                               being closed by the server.
 775
 776             Returns:
 777                     True if request was started, false otherwise.
 778
 779         --*/
 780
 781         private TriState StartRequest(HttpWebRequest request, bool canPollRead)
 782         {
 783             GlobalLog.Enter(
 784                 "Connection#" + ValidationHelper.HashString(this) +
 785                 "::StartRequest",
 786                 "HttpWebRequest#" + ValidationHelper.HashString(request) +
 787                 " WriteDone:"+ m_WriteDone +
 788                 " ReadDone:" + m_ReadDone +
 789                 " WaitList:" + m_WaitList.Count +
 790                 " WriteList:" + m_WriteList.Count);
 791             GlobalLog.ThreadContract(ThreadKinds.Unknown,
 792                 "Connection#" + ValidationHelper.HashString(this) +
 793                 "::StartRequest");
 794
 795             if (m_WriteList.Count == 0)
 796             {
 797                 // check if we consider connection timed out
 798                 if (ServicePoint.MaxIdleTime != -1 &&
 799                     m_IdleSinceUtc != DateTime.MinValue &&
 800                     m_IdleSinceUtc +
 801                         TimeSpan.FromMilliseconds(ServicePoint.MaxIdleTime) <
 802                         DateTime.UtcNow)
 803                 {
 804                     // This idle keep-alive connection timed out.
 805                     GlobalLog.Leave(
 806                         "Connection#" + ValidationHelper.HashString(this) +
 807                         "::StartRequest()" +
 808                         " Expired connection was idle for "
 809                         + (int)
 810                             ((DateTime.UtcNow - m_IdleSinceUtc).TotalSeconds) +
 811                         " secs, request will be retried: #" +
 812                         ValidationHelper.HashString(request));
 813                     return TriState.Unspecified; // don't use it
 814                 } else if (canPollRead) {
 815                     // Not timed out from our perspective but...
 816                     // Check if remote has:
 817                     //   1) closed an idle connection (TCP FIN)
 818                     // or
 819                     //   2) sent some errant data on an idle connection.
 820                     bool pollRead = PollRead();
 821                     if (pollRead) {
 822                         GlobalLog.Leave(
 823                             "Connection#" + ValidationHelper.HashString(this) +
 824                             "::StartRequest() " +
 825                             "Idle connection remotely closed, " +
 826                             "request will be retried: #" +
 827                             ValidationHelper.HashString(request));
 828                         return TriState.Unspecified; // don't use it
 829                     }
 830                 }
 831             }
 832
 833             TriState needReConnect = TriState.False;
 834              // Starting a request means the connection is not idle anymore
 835             m_IdleSinceUtc = DateTime.MinValue;
 836
 837              // Initialze state, and add the request to the write queue.
 838
 839              //
 840              // Note that m_Pipelining shold be only set here but the sanity check is made by the caller
 841              // means if the caller has found that it is safe to pipeline the below result must be true as well
 842              //
 843             if (!m_IsPipelinePaused)
 844                 m_IsPipelinePaused = m_WriteList.Count >= s_MaxPipelinedCount;
 845
 846             m_Pipelining = m_CanPipeline && request.Pipelined && (!request.HasEntityBody);
 847
 848             // start of write process, disable done-ness flag
 849             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::StartRequest() setting WriteDone:" + m_WriteDone.ToString() + " to false");
 850             m_WriteDone = false;
 851             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::StartRequest() m_WriteList adding HttpWebRequest#" + ValidationHelper.HashString(request));
 852             m_WriteList.Add(request);
 853
 854             GlobalLog.Print(m_WriteList.Count+" requests queued");
 855             CheckNonIdle();
 856
 857             // with no network stream around, we will have to create one, therefore, we can't have
 858             //  the possiblity to even have a DoneReading().
 859
 860             if (IsInitalizing)
 861                 needReConnect = TriState.True;
 862
 863 #if TRAVE
 864             if (request.IsTunnelRequest) {
 865                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::StartRequest() MyLocalPort:" + MyLocalPort + " setting Tunnelling to true HttpWebRequest#" + ValidationHelper.HashString(request));
 866                 q_Tunnelling = true;
 867             }
 868             else {
 869                 GlobalLog.Assert(!q_Tunnelling, "Connection#{0}::StartRequest()|MyLocalPort:{1} ERROR: Already tunnelling during non-tunnel request HttpWebRequest#{2}.", ValidationHelper.HashString(this), MyLocalPort, ValidationHelper.HashString(request));
 870             }
 871 #endif
 872
 873             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::StartRequest", needReConnect.ToString());
 874             return needReConnect;
 875         }
 876
 877         private void CompleteStartRequest(bool onSubmitThread, HttpWebRequest request, TriState needReConnect) {
 878             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::CompleteStartRequest", ValidationHelper.HashString(request));
 879             GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection#" + ValidationHelper.HashString(this) + "::CompleteStartRequest");
 880
 881             if (needReConnect == TriState.True) {
 882                 // Socket is not alive.
 883
 884                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CompleteStartRequest() Queue StartConnection Delegate ");
 885                 try {
 886                     if (request.Async) {
 887                         CompleteStartConnection(true, request);
 888                     }
 889                     else if (onSubmitThread) {
 890                         CompleteStartConnection(false, request);
 891                     }
 892                     // else - fall through and wake up other thread
 893                 }
 894                 catch (Exception exception) {
 895                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CompleteStartRequest(): exception: " + exception.ToString());
 896                     if (NclUtilities.IsFatal(exception)) throw;
 897                     //
 898                     // Should not be here because CompleteStartConnection and below tries to catch everything
 899                     //
 900                     GlobalLog.Assert(exception.ToString());
 901                 }
 902
 903                 // If neeeded wake up other thread where SubmitRequest was called
 904                 if (!request.Async) {
 905                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CompleteStartRequest() Invoking Async Result");
 906                     request.ConnectionAsyncResult.InvokeCallback(new AsyncTriState(needReConnect));
 907                 }
 908
 909
 910                 GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::CompleteStartRequest", "needReConnect");
 911                 return;
 912             }
 913
 914
 915             //
 916             // From now on the request.SetRequestSubmitDone must be called or it may hang
 917             // For a [....] request the write side reponse windowwas opened in HttpWebRequest.SubmitRequest
 918             if (request.Async)
 919                 request.OpenWriteSideResponseWindow();
 920
 921
 922             ConnectStream writeStream = new ConnectStream(this, request);
 923
 924             // Call the request to let them know that we have a write-stream, this might invoke Send() call
 925             if (request.Async || onSubmitThread) {
 926                 request.SetRequestSubmitDone(writeStream);
 927             }
 928             else {
 929                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CompleteStartRequest() Invoking Async Result");
 930                 request.ConnectionAsyncResult.InvokeCallback(writeStream);
 931             }
 932             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::CompleteStartRequest");
 933         }
 934
 935         /*++
 936
 937             CheckNextRequest
 938
 939             Gets the next request from the wait queue, if there is one.
 940
 941             Must be called with the crit sec held.
 942
 943
 944         --*/
 945         private HttpWebRequest CheckNextRequest()
 946         {
 947         GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection#" + ValidationHelper.HashString(this) + "::CheckNextRequest");
 948
 949             if (m_WaitList.Count == 0) {
 950                 // We're free now, if we're not going to close the connection soon.
 951                 m_Free = m_KeepAlive;
 952                 return null;
 953             }
 954             if (!CanBePooled) {
 955                 return null;
 956             }
 957
 958             WaitListItem item = m_WaitList[0];
 959             HttpWebRequest nextRequest = item.Request;
 960
 961             if (m_IsPipelinePaused)
 962                 m_IsPipelinePaused = m_WriteList.Count > s_MinPipelinedCount;
 963
 964             if (!nextRequest.Pipelined || nextRequest.HasEntityBody || !m_CanPipeline || !m_Pipelining || m_IsPipelinePaused) {
 965                 if (m_WriteList.Count != 0) {
 966                     nextRequest = null;
 967                 }
 968             }
 969             if (nextRequest != null) {
 970                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CheckNextRequest() Removing request#" + ValidationHelper.HashString(nextRequest) + " from m_WaitList. New Count:" + (m_WaitList.Count - 1).ToString());
 971
 972                 NetworkingPerfCounters.Instance.IncrementAverage(NetworkingPerfCounterName.HttpWebRequestAvgQueueTime,
 973                     item.QueueStartTime);
 974                 m_WaitList.RemoveAt(0);
 975                 CheckIdle();
 976             }
 977             return nextRequest;
 978         }
 979
 980         private void CompleteStartConnection(bool async, HttpWebRequest httpWebRequest)
 981         {
 982             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::CompleteStartConnection",  "async:" + async.ToString() + " httpWebRequest:" + ValidationHelper.HashString(httpWebRequest));
 983             GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection#" + ValidationHelper.HashString(this) + "::CompleteStartConnection");
 984
 985             WebExceptionStatus ws = WebExceptionStatus.ConnectFailure;
 986             m_InnerException = null;
 987             bool success = true;
 988
 989             try {
 990 #if DEBUG
 991                 lock (this)
 992                 {
 993                     // m_WriteList can be empty if request got aborted.  In that case no new requests can come in so it should remain zero.
 994                     if (m_WriteList.Count != 0)
 995                     {
 996                         GlobalLog.Assert(m_WriteList.Count == 1, "Connection#{0}::CompleteStartConnection()|WriteList is not sized 1.", ValidationHelper.HashString(this));
 997                         GlobalLog.Assert((m_WriteList[0] as HttpWebRequest) == httpWebRequest, "Connection#{0}::CompleteStartConnection()|Last request on write list does not match.", ValidationHelper.HashString(this));
 998                     }
 999                 }
1000 #endif
1001
1002                 //
1003                 // we will create a tunnel through a proxy then create
1004                 // and connect the socket we will use for the connection
1005                 // otherwise we will just create a socket and use it
1006                 //
1007                 if ((httpWebRequest.IsWebSocketRequest || httpWebRequest.Address.Scheme == Uri.UriSchemeHttps) &&
1008                     ServicePoint.InternalProxyServicePoint)
1009                 {
1010                     if(!TunnelThroughProxy(ServicePoint.InternalAddress, httpWebRequest,async)) {
1011                         ws = WebExceptionStatus.ConnectFailure;
1012                         success = false;
1013                     }
1014                     if (async && success) {
1015                         return;
1016                     }
1017                 } else {
1018                     if (!Activate(httpWebRequest, async, new GeneralAsyncDelegate(CompleteConnectionWrapper)))
1019                     {
1020                         return;
1021                     }
1022                 }
1023             }
1024             catch (Exception exception) {
1025                 if (m_InnerException == null)
1026                     m_InnerException = exception;
1027
1028                 if (exception is WebException) {
1029                     ws = ((WebException)exception).Status;
1030                 }
1031                 success = false;
1032             }
1033             if(!success)
1034             {
1035                 ConnectionReturnResult returnResult = null;
1036                 HandleError(false, false, ws, ref returnResult);
1037                 ConnectionReturnResult.SetResponses(returnResult);
1038                 GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::CompleteStartConnection Failed to connect.");
1039                 return;
1040             }
1041
1042             // Getting here means we connected synchronously.  Continue with the next step.
1043
1044             CompleteConnection(async, httpWebRequest);
1045             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::CompleteStartConnection");
1046         }
1047
1048         private void CompleteConnectionWrapper(object request, object state)
1049         {
1050 #if DEBUG
1051             using (GlobalLog.SetThreadKind(ThreadKinds.System | ThreadKinds.Async)) {
1052 #endif
1053             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(state) + "::CompleteConnectionWrapper", "request:" + ValidationHelper.HashString(request));
1054
1055             Exception stateException = state as Exception;
1056             if (stateException != null)
1057             {
1058                 GlobalLog.Print("CompleteConnectionWrapper() Request#" + ValidationHelper.HashString(request) + " Connection is in error: " + stateException.ToString());
1059                 ConnectionReturnResult returnResult = null;
1060
1061                 if (m_InnerException == null)
1062                     m_InnerException = stateException;
1063
1064                 HandleError(false, false, WebExceptionStatus.ConnectFailure, ref returnResult);
1065                 ConnectionReturnResult.SetResponses(returnResult);
1066             }
1067             CompleteConnection(true, (HttpWebRequest) request);
1068
1069             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(state) + "::CompleteConnectionWrapper" + (stateException == null? string.Empty: " failed"));
1070 #if DEBUG
1071             }
1072 #endif
1073         }
1074
1075         private void CompleteConnection(bool async, HttpWebRequest request)
1076         {
1077             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::CompleteConnection", "async:" + async.ToString() + " request:" + ValidationHelper.HashString(request));
1078             GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection#" + ValidationHelper.HashString(this) + "::CompleteConnection");
1079
1080             WebExceptionStatus ws = WebExceptionStatus.ConnectFailure;
1081             //
1082             // From now on the request.SetRequestSubmitDone must be called or it may hang
1083             // For a [....] request the write side reponse windowwas opened in HttpWebRequest.SubmitRequest
1084             if (request.Async)
1085                 request.OpenWriteSideResponseWindow();
1086
1087             try
1088             {
1089                 try {
1090 #if !FEATURE_PAL
1091                     if (request.Address.Scheme == Uri.UriSchemeHttps) {
1092                         TlsStream tlsStream = new TlsStream(request.GetRemoteResourceUri().IdnHost,
1093                             NetworkStream, request.ClientCertificates, ServicePoint, request,
1094                             request.Async ? request.GetConnectingContext().ContextCopy : null);
1095                         NetworkStream = tlsStream;
1096                     }
1097 #endif
1098                     ws = WebExceptionStatus.Success;
1099                 }
1100                 catch {
1101                     // The TLS stream could not be created.  Close the current non-TLS stream immediately
1102                     // to prevent any future use of it.  Due to race conditions, the error handling will sometimes
1103                     // try to write (flush) out some of the HTTP headers to the stream as it is closing down the failed
1104                     // HttpWebRequest. This would cause plain text to go on the wire even though the stream should
1105                     // have been TLS encrypted.
1106                     NetworkStream.Close();
1107                     throw;
1108                 }
1109                 finally {
1110                     //
1111                     // There is a ---- with Abort so TlsStream ctor may throw.
1112                     // SetRequestSubmitDone will deal with this kind of errors.
1113                     //
1114
1115                     m_ReadState = ReadState.Start;
1116                     ClearReaderState();
1117
1118                     request.SetRequestSubmitDone(new ConnectStream(this, request));
1119                 }
1120             }
1121             catch (Exception exception)
1122             {
1123                 if (m_InnerException == null)
1124                     m_InnerException = exception;
1125                 WebException webException = exception as WebException;
1126                 if (webException != null)
1127                 {
1128                     ws = webException.Status;
1129                 }
1130             }
1131
1132             if (ws != WebExceptionStatus.Success)
1133             {
1134                 ConnectionReturnResult returnResult = null;
1135                 HandleError(false, false, ws, ref returnResult);
1136                 ConnectionReturnResult.SetResponses(returnResult);
1137
1138                 if (Logging.On) Logging.PrintError(Logging.Web, this, "CompleteConnection", "on error");
1139                 GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::CompleteConnection", "on error");
1140             }
1141             else
1142             {
1143                 GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::CompleteConnection");
1144             }
1145         }
1146
1147         private void InternalWriteStartNextRequest(HttpWebRequest request, ref bool calledCloseConnection, ref TriState startRequestResult, ref HttpWebRequest nextRequest, ref ConnectionReturnResult returnResult) {
1148             GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection#" + ValidationHelper.HashString(this) + "::InternalWriteStartNextRequest");
1149
1150             lock(this) {
1151
1152                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::WriteStartNextRequest() setting WriteDone:" + m_WriteDone.ToString() + " to true");
1153                 m_WriteDone = true;
1154
1155                 //
1156                 // If we're not doing keep alive, and the read on this connection
1157                 // has already completed, now is the time to close the
1158                 // connection.
1159                 //
1160                 //need to wait for read to set the error
1161                 if (!m_KeepAlive || m_Error != WebExceptionStatus.Success || !CanBePooled) {
1162                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::WriteStartNextRequest() m_WriteList.Count:" + m_WriteList.Count);
1163                     if (m_ReadDone) {
1164                         // We could be closing because of an unexpected keep-alive
1165                         // failure, ie we pipelined a few requests and in the middle
1166                         // the remote server stopped doing keep alive. In this
1167                         // case m_Error could be success, which would be misleading.
1168                         // So in that case we'll set it to connection closed.
1169
1170                         if (m_Error == WebExceptionStatus.Success) {
1171                             // Only reason we could have gotten here is because
1172                             // we're not keeping the connection alive.
1173                             m_Error = WebExceptionStatus.KeepAliveFailure;
1174                         }
1175
1176                         // PrepareCloseConnectionSocket is called with the critical section
1177                         // held. Note that we know since it's not a keep-alive
1178                         // connection the read half wouldn't have posted a receive
1179                         // for this connection, so it's OK to call PrepareCloseConnectionSocket now.
1180                         PrepareCloseConnectionSocket(ref returnResult);
1181                         calledCloseConnection = true;
1182                         Close();
1183                     }
1184                     else {
1185                         if (m_Error!=WebExceptionStatus.Success) {
1186                             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::WriteStartNextRequest() a Failure, m_Error = " + m_Error.ToString());
1187                         }
1188                     }
1189                 }
1190                 else {
1191                     // If we're pipelining, we get get the next request going
1192                     // as soon as the write is done. Otherwise we have to wait
1193                     // until both read and write are done.
1194
1195
1196                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::WriteStartNextRequest() Non-Error m_WriteList.Count:" + m_WriteList.Count + " m_WaitList.Count:" + m_WaitList.Count);
1197
1198                     if (m_Pipelining || m_ReadDone)
1199                     {
1200                         nextRequest = CheckNextRequest();
1201                     }
1202                     if (nextRequest != null)
1203                     {
1204                         // This codepath doesn't handle the case where the server has closed the Connection because we
1205                         // just finished using it and didn't get a Connection: close header.
1206                         startRequestResult = StartRequest(nextRequest, false);
1207                         GlobalLog.Assert(startRequestResult != TriState.Unspecified, "WriteStartNextRequest got TriState.Unspecified from StartRequest, things are about to hang!");
1208                     }
1209                 }
1210             } // lock
1211         }
1212
1213         internal void WriteStartNextRequest(HttpWebRequest request, ref ConnectionReturnResult returnResult) {
1214             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::WriteStartNextRequest" + " WriteDone:" + m_WriteDone + " ReadDone:" + m_ReadDone + " WaitList:" + m_WaitList.Count + " WriteList:" + m_WriteList.Count);
1215             GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection#" + ValidationHelper.HashString(this) + "::WriteStartNextRequest");
1216
1217             TriState startRequestResult = TriState.Unspecified;
1218             HttpWebRequest nextRequest = null;
1219             bool calledCloseConnection = false;
1220
1221             InternalWriteStartNextRequest(request, ref calledCloseConnection, ref startRequestResult, ref nextRequest, ref returnResult);
1222
1223             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::WriteStartNextRequest: Pipelining:" + m_Pipelining + " nextRequest#"+ValidationHelper.HashString(nextRequest));
1224
1225             if (!calledCloseConnection && startRequestResult != TriState.Unspecified)
1226             {
1227                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::WriteStartNextRequest calling CompleteStartRequest");
1228                 CompleteStartRequest(false, nextRequest, startRequestResult);
1229             }
1230
1231             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::WriteStartNextRequest");
1232         }
1233
1234
1235         internal void SetLeftoverBytes(byte[] buffer, int bufferOffset, int bufferCount)
1236         {
1237             // The ConnectStream read past the response of its HTTP response (can happen in chunked scenarios).
1238             // Get the buffer containing bytes belonging to the next request and use them for the next request.
1239
1240             if (bufferOffset > 0)
1241             {
1242                 // We need to move leftover bytes to the beginning of the buffer.
1243                 Buffer.BlockCopy(buffer, bufferOffset, buffer, 0, bufferCount);
1244             }
1245
1246             // If we had to reallocate the buffer, we are going to clobber the one that was allocated from the pin friendly cache.
1247             // give it back
1248             if (m_ReadBuffer != buffer)
1249             {
1250                 // if m_ReadBuffer is from the pinnable cache, give it back
1251                 FreeReadBuffer();
1252                 m_ReadBuffer = buffer;
1253             }
1254
1255             m_BytesScanned = 0;
1256             m_BytesRead = bufferCount;
1257         }
1258
1259         /*++
1260
1261             ReadStartNextRequest
1262
1263             This method is called by a stream interface when it's done reading.
1264             We might possible free up the connection for another request here.
1265
1266             Called when we think we might need to start another request because
1267             a read completed.
1268
1269         --*/
1270         internal void ReadStartNextRequest(WebRequest currentRequest, ref ConnectionReturnResult returnResult)
1271         {
1272             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::ReadStartNextRequest" + " WriteDone:" + m_WriteDone + " ReadDone:" + m_ReadDone + " WaitList:" + m_WaitList.Count + " WriteList:" + m_WriteList.Count);
1273             GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection#" + ValidationHelper.HashString(this) + "::ReadStartNextRequest");
1274
1275             HttpWebRequest nextRequest = null;
1276             TriState startRequestResult = TriState.Unspecified;
1277             bool calledCloseConnection = false;
1278             bool mustExit = false;
1279
1280             // ReadStartNextRequest is called by ConnectStream.CallDone: This guarantees that the request
1281             // is done and the response (response stream) was closed. Remove the reservation for the request.
1282             int currentCount = Interlocked.Decrement(ref m_ReservedCount);
1283             GlobalLog.Assert(currentCount >= 0, "m_ReservedCount must not be < 0 when decremented.");
1284
1285             try {
1286                 lock(this) {
1287                     if (m_WriteList.Count > 0 && (object)currentRequest == m_WriteList[0])
1288                     {
1289                         // advance back to state 0
1290                         m_ReadState = ReadState.Start;
1291                         m_WriteList.RemoveAt(0);
1292
1293                         // Must reset ConnectStream here to prevent a leak through the stream of the last request on each connection.
1294                         m_ResponseData.m_ConnectStream = null;
1295
1296                         GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadStartNextRequest() Removed request#" + ValidationHelper.HashString(currentRequest) + " from m_WriteList. New m_WriteList.Count:" + m_WriteList.Count.ToString());
1297                     }
1298                     else
1299                     {
1300                         GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadStartNextRequest() The request#" + ValidationHelper.HashString(currentRequest) + " was disassociated so do nothing.  m_WriteList.Count:" + m_WriteList.Count.ToString());
1301                         mustExit = true;;
1302                     }
1303
1304                     //
1305                     // Since this is called after we're done reading the current
1306                     // request, if we're not doing keepalive and we're done
1307                     // writing we can close the connection now.
1308                     //
1309                     if(!mustExit)
1310                     {
1311                         //
1312                         // m_ReadDone==true is implied because we just finished a request but really the value must still be false here
1313                         //
1314                         if (m_ReadDone)
1315                             throw new InternalException();  // other requests may already started reading on this connection, need a QFE
1316
1317                         if (!m_KeepAlive || m_Error != WebExceptionStatus.Success || !CanBePooled)
1318                         {
1319                             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadStartNextRequest() KeepAlive:" + m_KeepAlive + " WriteDone:" + m_WriteDone);
1320                             // Finished one request and connection is closing.
1321                             // We will not read from this connection so set readDone = true
1322                             m_ReadDone = true;
1323
1324                             if (m_WriteDone)
1325                             {
1326
1327                                 // We could be closing because of an unexpected keep-alive
1328                                 // failure, ie we pipelined a few requests and in the middle
1329                                 // the remote server stopped doing keep alive. In this
1330                                 // case m_Error could be success, which would be misleading.
1331                                 // So in that case we'll set it to KeepAliveFailure.
1332
1333                                 if (m_Error == WebExceptionStatus.Success) {
1334                                     // Only reason we could have gotten here is because
1335                                     // we're not keeping the connection alive.
1336                                     m_Error = WebExceptionStatus.KeepAliveFailure;
1337                                 }
1338
1339                                 // PrepareCloseConnectionSocket has to be called with the critical section held.
1340                                 PrepareCloseConnectionSocket(ref returnResult);
1341                                 calledCloseConnection = true;
1342                                 Close();
1343                             }
1344                         }
1345                         else
1346                         {
1347                             // We try to sort out KeepAliveFailure thing (search by context)
1348                             m_AtLeastOneResponseReceived = true;
1349
1350                             if (m_WriteList.Count != 0)
1351                             {
1352                                 // If a *pipelined* request that is being submitted has finished with the headers, post a receive
1353                                 nextRequest = m_WriteList[0] as HttpWebRequest;
1354                                 // If the active request has not finished its headers we can set m_ReadDone = true
1355                                 // and that will be changed when said request will call CheckStartReceive
1356                                 if (!nextRequest.HeadersCompleted)
1357                                 {
1358                                     nextRequest = null;
1359                                     m_ReadDone = true;
1360                                 }
1361                             }
1362                             // If there are no requests left to write (means pipeline),
1363                             // we can get the next request from wait list going now.
1364                             else
1365                             {
1366                                 m_ReadDone = true;
1367
1368                                 // Sometime we get a response before completing the body in which case
1369                                 // we defer next request to WriteStartNextRequest
1370                                 if (m_WriteDone)
1371                                 {
1372                                     nextRequest = CheckNextRequest();
1373
1374                                     if (nextRequest != null )
1375                                     {
1376                                         // We cannot have HeadersCompleted on the request that was not placed yet on the write list
1377                                         if(nextRequest.HeadersCompleted) // TODO: change to be Assert but only when stress got stable-stable
1378                                             throw new InternalException();
1379
1380                                         // This codepath doesn't handle the case where the server has closed the
1381                                         // Connection because we just finished using it and didn't get a
1382                                         // Connection: close header.
1383                                         startRequestResult = StartRequest(nextRequest, false);
1384                                         GlobalLog.Assert(startRequestResult != TriState.Unspecified, "ReadStartNextRequest got TriState.Unspecified from StartRequest, things are about to hang!");
1385                                     }
1386                                     else
1387                                     {
1388                                         //There are no other requests to process, so make connection avaliable for all
1389                                         m_Free = true;
1390                                     }
1391                                 }
1392                             }
1393                         }
1394                     }
1395                 }
1396             }
1397             finally
1398             {
1399                 CheckIdle();
1400                 //set result here to prevent nesting of readstartnextrequest.
1401                 if(returnResult != null){
1402                     ConnectionReturnResult.SetResponses(returnResult);
1403                 }
1404             }
1405
1406             if(!mustExit && !calledCloseConnection)
1407             {
1408                 if (startRequestResult != TriState.Unspecified)
1409                 {
1410                     CompleteStartRequest(false, nextRequest, startRequestResult);
1411                 }
1412                 else if (nextRequest != null)
1413                 {
1414                     // Handling receive, note that is for pipelinning case only !
1415                     if (!nextRequest.Async)
1416                     {
1417                         nextRequest.ConnectionReaderAsyncResult.InvokeCallback();
1418                     }
1419                     else
1420                     {
1421                         if (m_BytesScanned < m_BytesRead)
1422                         {
1423                             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadStartNextRequest() Calling ReadComplete, bytes unparsed = " + (m_BytesRead - m_BytesScanned));
1424                             ReadComplete(0, WebExceptionStatus.Success);
1425                         }
1426                         else if (Thread.CurrentThread.IsThreadPoolThread)
1427                         {
1428                             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadStartNextRequest() Calling PostReceive().");
1429                             PostReceive();
1430                         }
1431                         else
1432                         {
1433                             // Offload to the threadpool to protect against the case where one request's thread posts IO that another request
1434                             // depends on, but the first thread dies in the mean time.
1435                             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadStartNextRequest() ThreadPool.UnsafeQueueUserWorkItem(m_PostReceiveDelegate, this)");
1436                             ThreadPool.UnsafeQueueUserWorkItem(m_PostReceiveDelegate, this);
1437                         }
1438                     }
1439                 }
1440             }
1441             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::ReadStartNextRequest");
1442         }
1443
1444         internal void MarkAsReserved()
1445         {
1446             // We use an interlock here rather than a lock() to avoid deadlocks in the following situation:
1447             // - ConnectionGroup is holding lock(obj1), calls into Connection which is waiting for lock(obj2)
1448             // - on another thread Connection is holding lock(obj2) and calls into ConnectionGroup which will wait
1449             //   for lock(obj1).
1450             int currentCount = Interlocked.Increment(ref m_ReservedCount);
1451             GlobalLog.Assert(currentCount > 0, "m_ReservedCount must not be less or equal zero after incrementing.");
1452         }
1453
1454         //
1455         //
1456         //
1457         internal void CheckStartReceive(HttpWebRequest request)
1458         {
1459             lock (this)
1460             {
1461                 request.HeadersCompleted = true;
1462                 if (m_WriteList.Count == 0)
1463                 {
1464                     // aborted request, was already dispatched.
1465                     // Note it could have been aborted softly if not the first one in the pipeline
1466                     return;
1467                 }
1468
1469                 // Note we do NOT allow receive if pipelining and the passed request is not the first one on the write queue
1470                 if (!m_ReadDone || m_WriteList[0] != (object)request)
1471                 {
1472                     // ReadStartNextRequest should take care of these cases
1473                     return;
1474                 }
1475                 // Start a receive
1476                 m_ReadDone = false;
1477                 m_CurrentRequest = (HttpWebRequest)m_WriteList[0];
1478             }
1479
1480             if (!request.Async)
1481             {
1482                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CheckStartReceive() SYNC request, calling ConnectionReaderAsyncResult.InvokeCallback()");
1483                 request.ConnectionReaderAsyncResult.InvokeCallback();
1484             }
1485             else if (m_BytesScanned < m_BytesRead)
1486             {
1487                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CheckStartReceive() Calling ReadComplete, bytes unparsed = " + (m_BytesRead - m_BytesScanned));
1488                 ReadComplete(0, WebExceptionStatus.Success);
1489             }
1490             else if (Thread.CurrentThread.IsThreadPoolThread)
1491             {
1492                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CheckStartReceive() Calling PostReceive().");
1493                 PostReceive();
1494             }
1495             else
1496             {
1497                 // Offload to the threadpool to protect against the case where one request's thread posts IO that another request
1498                 // depends on, but the first thread dies in the mean time.
1499                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CheckStartReceive() ThreadPool.UnsafeQueueUserWorkItem(m_PostReceiveDelegate, this)");
1500                 ThreadPool.UnsafeQueueUserWorkItem(m_PostReceiveDelegate, this);
1501             }
1502         }
1503
1504         /*++
1505
1506         Routine Description:
1507
1508            Clears out common member vars used for Status Line parsing
1509
1510         Arguments:
1511
1512            None.
1513
1514         Return Value:
1515
1516            None.
1517
1518         --*/
1519
1520         private void InitializeParseStatusLine() {
1521             m_StatusState = BeforeVersionNumbers;
1522             m_StatusLineValues.MajorVersion = 0;
1523             m_StatusLineValues.MinorVersion = 0;
1524             m_StatusLineValues.StatusCode = 0;
1525             m_StatusLineValues.StatusDescription = null;
1526         }
1527
1528         /*++
1529
1530         Routine Description:
1531
1532            Performs status line parsing on incomming server responses
1533
1534         Arguments:
1535
1536            statusLine - status line that we wish to parse
1537            statusLineLength - length of the array
1538            statusLineInts - array of ints contanes result
1539            statusDescription - string with discription
1540            statusStatus     - state stored between parse attempts
1541
1542         Return Value:
1543
1544            bool - Success true/false
1545
1546         --*/
1547
1548         private const int BeforeVersionNumbers = 0;
1549         private const int MajorVersionNumber   = 1;
1550         private const int MinorVersionNumber   = 2;
1551         private const int StatusCodeNumber     = 3;
1552         private const int AfterStatusCode      = 4;
1553         private const int AfterCarriageReturn  = 5;
1554
1555         private const string BeforeVersionNumberBytes = "HTTP/";
1556
1557         private DataParseStatus ParseStatusLine(
1558                 byte [] statusLine,
1559                 int statusLineLength,
1560                 ref int bytesParsed,
1561                 ref int [] statusLineInts,
1562                 ref string statusDescription,
1563                 ref int statusState,
1564                 ref WebParseError parseError) {
1565             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::ParseStatusLine", statusLineLength.ToString(NumberFormatInfo.InvariantInfo) + ", " + bytesParsed.ToString(NumberFormatInfo.InvariantInfo) +", " +statusState.ToString(NumberFormatInfo.InvariantInfo));
1566             GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection#" + ValidationHelper.HashString(this) + "::ParseStatusLine");
1567             GlobalLog.Assert((statusLineLength - bytesParsed) >= 0, "Connection#{0}::ParseStatusLine()|(statusLineLength - bytesParsed) < 0", ValidationHelper.HashString(this));
1568             //GlobalLog.Dump(statusLine, bytesParsed, statusLineLength);
1569
1570             DataParseStatus parseStatus = DataParseStatus.Done;
1571             int statusLineSize = 0;
1572             int startIndexStatusDescription = -1;
1573             int lastUnSpaceIndex = 0;
1574
1575             //
1576             // While walking the Status Line looking for terminating \r\n,
1577             //   we extract the Major.Minor Versions and Status Code in that order.
1578             //   text and spaces will lie between/before/after the three numbers
1579             //   but the idea is to remember which number we're calculating based on a numeric state
1580             //   If all goes well the loop will churn out an array with the 3 numbers plugged in as DWORDs
1581             //
1582
1583             while ((bytesParsed < statusLineLength) && (statusLine[bytesParsed] != '\r') && (statusLine[bytesParsed] != '\n')) {
1584
1585                 // below should be wrapped in while (response[i] != ' ') to be more robust???
1586                 switch (statusState) {
1587                     case BeforeVersionNumbers:
1588                         if (statusLine[bytesParsed] == '/') {
1589                             //INET_ASSERT(statusState == BeforeVersionNumbers);
1590                             statusState++; // = MajorVersionNumber
1591                         }
1592                         else if (statusLine[bytesParsed] == ' ') {
1593                             statusState = StatusCodeNumber;
1594                         }
1595
1596                         break;
1597
1598                     case MajorVersionNumber:
1599
1600                         if (statusLine[bytesParsed] == '.') {
1601                             //INET_ASSERT(statusState == MajorVersionNumber);
1602                             statusState++; // = MinorVersionNumber
1603                             break;
1604                         }
1605                         // fall through
1606                         goto case MinorVersionNumber;
1607
1608                     case MinorVersionNumber:
1609
1610                         if (statusLine[bytesParsed] == ' ') {
1611                             //INET_ASSERT(statusState == MinorVersionNumber);
1612                             statusState++; // = StatusCodeNumber
1613                             break;
1614                         }
1615                         // fall through
1616                         goto case StatusCodeNumber;
1617
1618                     case StatusCodeNumber:
1619
1620                         if (Char.IsDigit((char)statusLine[bytesParsed])) {
1621                             int val = statusLine[bytesParsed] - '0';
1622                             statusLineInts[statusState] = statusLineInts[statusState] * 10 + val;
1623                         }
1624                         else if (statusLineInts[StatusCodeNumber] > 0) {
1625                             //
1626                             // we eat spaces before status code is found,
1627                             //  once we have the status code we can go on to the next
1628                             //  state on the next non-digit. This is done
1629                             //  to cover cases with several spaces between version
1630                             //  and the status code number.
1631                             //
1632
1633                             statusState++; // = AfterStatusCode
1634                             break;
1635                         }
1636                         else if (!Char.IsWhiteSpace((char) statusLine[bytesParsed])) {
1637                             statusLineInts[statusState] = (int)-1;
1638                         }
1639
1640                         break;
1641
1642                     case AfterStatusCode:
1643                         if (statusLine[bytesParsed] != ' ') {
1644                             lastUnSpaceIndex = bytesParsed;
1645                             if (startIndexStatusDescription == -1) {
1646                                 startIndexStatusDescription = bytesParsed;
1647                             }
1648                         }
1649                         break;
1650
1651                 }
1652                 ++bytesParsed;
1653                 if (m_MaximumResponseHeadersLength>=0 && ++m_TotalResponseHeadersLength>=m_MaximumResponseHeadersLength) {
1654                     parseStatus = DataParseStatus.DataTooBig;
1655                     goto quit;
1656                 }
1657             }
1658
1659             statusLineSize = bytesParsed;
1660
1661             // add to Description if already partialy parsed
1662             if (startIndexStatusDescription != -1) {
1663                 statusDescription +=
1664                     WebHeaderCollection.HeaderEncoding.GetString(
1665                         statusLine,
1666                         startIndexStatusDescription,
1667                         lastUnSpaceIndex - startIndexStatusDescription + 1 );
1668             }
1669
1670             if (bytesParsed == statusLineLength) {
1671                 //
1672                 // response now points one past the end of the buffer. We may be looking
1673                 // over the edge...
1674                 //
1675                 // if we're at the end of the connection then the server sent us an
1676                 // incorrectly formatted response. Probably an error.
1677                 //
1678                 // Otherwise its a partial response. We need more
1679                 //
1680                 parseStatus = DataParseStatus.NeedMoreData;
1681                 //
1682                 // if we really hit the end of the response then update the amount of
1683                 // headers scanned
1684                 //
1685                 GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::ParseStatusLine", parseStatus.ToString());
1686                 return parseStatus;
1687             }
1688
1689             while ((bytesParsed < statusLineLength)
1690                    && ((statusLine[bytesParsed] == '\r') || (statusLine[bytesParsed] == ' '))) {
1691                 ++bytesParsed;
1692                 if (m_MaximumResponseHeadersLength>=0 && ++m_TotalResponseHeadersLength>=m_MaximumResponseHeadersLength) {
1693                     parseStatus = DataParseStatus.DataTooBig;
1694                     goto quit;
1695                 }
1696             }
1697
1698             if (bytesParsed == statusLineLength) {
1699
1700                 //
1701                 // hit end of buffer without finding LF
1702                 //
1703
1704                 parseStatus = DataParseStatus.NeedMoreData;
1705                 goto quit;
1706
1707             }
1708             else if (statusLine[bytesParsed] == '\n') {
1709                 ++bytesParsed;
1710                 if (m_MaximumResponseHeadersLength>=0 && ++m_TotalResponseHeadersLength>=m_MaximumResponseHeadersLength) {
1711                     parseStatus = DataParseStatus.DataTooBig;
1712                     goto quit;
1713                 }
1714                 //
1715                 // if we found the empty line then we are done
1716                 //
1717                 parseStatus = DataParseStatus.Done;
1718             }
1719
1720
1721             //
1722             // Now we have our parsed header to add to the array
1723             //
1724
1725 quit:
1726             if (parseStatus == DataParseStatus.Done && statusState != AfterStatusCode) {
1727                 // need to handle the case where we parse the StatusCode,
1728                 //  but didn't get a status Line, and there was no space afer it.
1729                 if (statusState != StatusCodeNumber || statusLineInts[StatusCodeNumber] <= 0) {
1730                     //
1731                     // we're done with the status line, if we didn't parse all the
1732                     // numbers needed this is invalid protocol on the server
1733                     //
1734                     parseStatus = DataParseStatus.Invalid;
1735                 }
1736             }
1737
1738             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseStatusLine() StatusCode:" + statusLineInts[StatusCodeNumber] + " MajorVersionNumber:" + statusLineInts[MajorVersionNumber] + " MinorVersionNumber:" + statusLineInts[MinorVersionNumber] + " StatusDescription:" + ValidationHelper.ToString(statusDescription));
1739             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::ParseStatusLine", parseStatus.ToString());
1740
1741             if (parseStatus == DataParseStatus.Invalid) {
1742                 parseError.Section = WebParseErrorSection.ResponseStatusLine;
1743                 parseError.Code = WebParseErrorCode.Generic;
1744             }
1745
1746             return parseStatus;
1747         }
1748
1749         // Must all start with a different first character.
1750         private static readonly string[] s_ShortcutStatusDescriptions = new string[] { "OK", "Continue", "Unauthorized" };
1751
1752         //
1753         // Updated version of ParseStatusLine() - secure and fast
1754         //
1755         private static unsafe DataParseStatus ParseStatusLineStrict(
1756                 byte[] statusLine,
1757                 int statusLineLength,
1758                 ref int bytesParsed,
1759                 ref int statusState,
1760                 StatusLineValues statusLineValues,
1761                 int maximumHeaderLength,
1762                 ref int totalBytesParsed,
1763                 ref WebParseError parseError)
1764         {
1765             GlobalLog.Enter("Connection::ParseStatusLineStrict", statusLineLength.ToString(NumberFormatInfo.InvariantInfo) + ", " + bytesParsed.ToString(NumberFormatInfo.InvariantInfo) + ", " + statusState.ToString(NumberFormatInfo.InvariantInfo));
1766             GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection::ParseStatusLineStrict");
1767             GlobalLog.Assert((statusLineLength - bytesParsed) >= 0, "Connection::ParseStatusLineStrict()|(statusLineLength - bytesParsed) < 0");
1768             GlobalLog.Assert(maximumHeaderLength <= 0 || totalBytesParsed <= maximumHeaderLength, "Connection::ParseStatusLineStrict()|Headers already read exceeds limit.");
1769
1770             // Remember where we started.
1771             int initialBytesParsed = bytesParsed;
1772
1773             // Set up parsing status with what will happen if we exceed the buffer.
1774             DataParseStatus parseStatus = DataParseStatus.DataTooBig;
1775             int effectiveMax = maximumHeaderLength <= 0 ? int.MaxValue : (maximumHeaderLength - totalBytesParsed + bytesParsed);
1776             if (statusLineLength < effectiveMax)
1777             {
1778                 parseStatus = DataParseStatus.NeedMoreData;
1779                 effectiveMax = statusLineLength;
1780             }
1781
1782             // sanity check
1783             if (bytesParsed >= effectiveMax)
1784                 goto quit;
1785
1786             fixed (byte* byteBuffer = statusLine)
1787             {
1788                 // Use this switch to jump midway into the action.  They all fall through until the end of the buffer is reached or
1789                 // the status line is fully parsed.
1790                 switch (statusState)
1791                 {
1792                     case BeforeVersionNumbers:
1793                         // This takes advantage of the fact that this token must be the very first thing in the response.
1794                         while (totalBytesParsed - initialBytesParsed + bytesParsed < BeforeVersionNumberBytes.Length)
1795                         {
1796                             if ((byte)BeforeVersionNumberBytes[totalBytesParsed - initialBytesParsed + bytesParsed] != byteBuffer[bytesParsed])
1797                             {
1798                                 parseStatus = DataParseStatus.Invalid;
1799                                 goto quit;
1800                             }
1801
1802                             if(++bytesParsed == effectiveMax)
1803                                 goto quit;
1804                         }
1805
1806                         // When entering the MajorVersionNumber phase, make sure at least one digit is present.
1807                         if (byteBuffer[bytesParsed] == '.')
1808                         {
1809                             parseStatus = DataParseStatus.Invalid;
1810                             goto quit;
1811                         }
1812
1813                         statusState = MajorVersionNumber;
1814                         goto case MajorVersionNumber;
1815
1816                     case MajorVersionNumber:
1817                         while (byteBuffer[bytesParsed] != '.')
1818                         {
1819                             if (byteBuffer[bytesParsed] < '0' || byteBuffer[bytesParsed] > '9')
1820                             {
1821                                 parseStatus = DataParseStatus.Invalid;
1822                                 goto quit;
1823                             }
1824
1825                             statusLineValues.MajorVersion = statusLineValues.MajorVersion * 10 + byteBuffer[bytesParsed] - '0';
1826
1827                             if (++bytesParsed == effectiveMax)
1828                                 goto quit;
1829                         }
1830
1831                         // Need visibility past the dot.
1832                         if (bytesParsed + 1 == effectiveMax)
1833                             goto quit;
1834                         bytesParsed++;
1835
1836                         // When entering the MinorVersionNumber phase, make sure at least one digit is present.
1837                         if (byteBuffer[bytesParsed] == ' ')
1838                         {
1839                             parseStatus = DataParseStatus.Invalid;
1840                             goto quit;
1841                         }
1842
1843                         statusState = MinorVersionNumber;
1844                         goto case MinorVersionNumber;
1845
1846                     case MinorVersionNumber:
1847                         // Only a single SP character is allowed to delimit fields in the status line.
1848                         while (byteBuffer[bytesParsed] != ' ')
1849                         {
1850                             if (byteBuffer[bytesParsed] < '0' || byteBuffer[bytesParsed] > '9')
1851                             {
1852                                 parseStatus = DataParseStatus.Invalid;
1853                                 goto quit;
1854                             }
1855
1856                             statusLineValues.MinorVersion = statusLineValues.MinorVersion * 10 + byteBuffer[bytesParsed] - '0';
1857
1858                             if (++bytesParsed == effectiveMax)
1859                                 goto quit;
1860                         }
1861
1862                         statusState = StatusCodeNumber;
1863
1864                         // Start the status code out as "1".  This will effectively add 1000 to the code.  It's used to count
1865                         // the number of digits to make sure it's three.  At the end, subtract 1000.
1866                         statusLineValues.StatusCode = 1;
1867
1868                         // Move past the space.
1869                         if (++bytesParsed == effectiveMax)
1870                             goto quit;
1871
1872                         goto case StatusCodeNumber;
1873
1874                     case StatusCodeNumber:
1875                         // RFC2616 says codes with an unrecognized first digit
1876                         // should be rejected.  We're allowing the application to define their own "understanding" of
1877                         // 0, 6, 7, 8, and 9xx codes.
1878                         while (byteBuffer[bytesParsed] >= '0' && byteBuffer[bytesParsed] <= '9')
1879                         {
1880                             // Make sure it isn't too big.  The leading '1' will be removed after three digits are read.
1881                             if (statusLineValues.StatusCode >= 1000)
1882                             {
1883                                 parseStatus = DataParseStatus.Invalid;
1884                                 goto quit;
1885                             }
1886
1887                             statusLineValues.StatusCode = statusLineValues.StatusCode * 10 + byteBuffer[bytesParsed] - '0';
1888
1889                             if (++bytesParsed == effectiveMax)
1890                                 goto quit;
1891                         }
1892
1893                         // Make sure there was enough, and exactly one space.
1894                         if (byteBuffer[bytesParsed] != ' ' || statusLineValues.StatusCode < 1000)
1895                         {
1896                             if(byteBuffer[bytesParsed] == '\r' && statusLineValues.StatusCode >= 1000){
1897                                 statusLineValues.StatusCode -= 1000;
1898                                 statusState = AfterCarriageReturn;
1899                                 if (++bytesParsed == effectiveMax)
1900                                     goto quit;
1901                                 goto case AfterCarriageReturn;
1902                             }
1903                             parseStatus = DataParseStatus.Invalid;
1904                             goto quit;
1905                         }
1906
1907                         // Remove the extra leading 1.
1908                         statusLineValues.StatusCode -= 1000;
1909
1910                         statusState = AfterStatusCode;
1911
1912                         // Move past the space.
1913                         if (++bytesParsed == effectiveMax)
1914                             goto quit;
1915
1916                         goto case AfterStatusCode;
1917
1918                     case AfterStatusCode:
1919                     {
1920                         // Check for shortcuts.
1921                         if (statusLineValues.StatusDescription == null)
1922                         {
1923                             foreach (string s in s_ShortcutStatusDescriptions)
1924                             {
1925                                 if (bytesParsed < effectiveMax - s.Length && byteBuffer[bytesParsed] == (byte) s[0])
1926                                 {
1927                                     int i;
1928                                     byte *pBuffer = byteBuffer + bytesParsed + 1;
1929                                     for(i = 1; i < s.Length; i++)
1930                                         if (*(pBuffer++) != (byte) s[i])
1931                                             break;
1932                                     if (i == s.Length)
1933                                     {
1934                                         statusLineValues.StatusDescription = s;
1935                                         bytesParsed += s.Length;
1936                                     }
1937                                     break;
1938                                 }
1939                             }
1940                         }
1941
1942                         int beginning = bytesParsed;
1943
1944                         while (byteBuffer[bytesParsed] != '\r')
1945                         {
1946                             if (byteBuffer[bytesParsed] < ' ' || byteBuffer[bytesParsed] == 127)
1947                             {
1948                                 parseStatus = DataParseStatus.Invalid;
1949                                 goto quit;
1950                             }
1951
1952                             if (++bytesParsed == effectiveMax)
1953                             {
1954                                 string s = WebHeaderCollection.HeaderEncoding.GetString(byteBuffer + beginning, bytesParsed - beginning);
1955                                 if (statusLineValues.StatusDescription == null)
1956                                     statusLineValues.StatusDescription = s;
1957                                 else
1958                                     statusLineValues.StatusDescription += s;
1959
1960                                 goto quit;
1961                             }
1962                         }
1963
1964                         if (bytesParsed > beginning)
1965                         {
1966                             string s = WebHeaderCollection.HeaderEncoding.GetString(byteBuffer + beginning, bytesParsed - beginning);
1967                             if (statusLineValues.StatusDescription == null)
1968                                 statusLineValues.StatusDescription = s;
1969                             else
1970                                 statusLineValues.StatusDescription += s;
1971                         }
1972                         else if (statusLineValues.StatusDescription == null)
1973                         {
1974                             statusLineValues.StatusDescription = "";
1975                         }
1976
1977                         statusState = AfterCarriageReturn;
1978
1979                         // Move past the CR.
1980                         if (++bytesParsed == effectiveMax)
1981                             goto quit;
1982
1983                         goto case AfterCarriageReturn;
1984                     }
1985
1986                     case AfterCarriageReturn:
1987                         if (byteBuffer[bytesParsed] != '\n')
1988                         {
1989                             parseStatus = DataParseStatus.Invalid;
1990                             goto quit;
1991                         }
1992
1993                         parseStatus = DataParseStatus.Done;
1994                         bytesParsed++;
1995                         break;
1996                 }
1997             }
1998
1999 quit:
2000             totalBytesParsed += bytesParsed - initialBytesParsed;
2001
2002             GlobalLog.Print("Connection::ParseStatusLineStrict() StatusCode:" + statusLineValues.StatusCode + " MajorVersionNumber:" + statusLineValues.MajorVersion + " MinorVersionNumber:" + statusLineValues.MinorVersion + " StatusDescription:" + ValidationHelper.ToString(statusLineValues.StatusDescription));
2003             GlobalLog.Leave("Connection::ParseStatusLineStrict", parseStatus.ToString());
2004
2005             if (parseStatus == DataParseStatus.Invalid) {
2006                 parseError.Section = WebParseErrorSection.ResponseStatusLine;
2007                 parseError.Code = WebParseErrorCode.Generic;
2008             }
2009
2010             return parseStatus;
2011         }
2012
2013
2014         /*++
2015
2016         Routine Description:
2017
2018            SetStatusLineParsed - processes the result of status line,
2019              after it has been parsed, reads vars and formats result of parsing
2020
2021         Arguments:
2022
2023            None - uses member vars
2024
2025         Return Value:
2026
2027            None
2028
2029         --*/
2030
2031         private void SetStatusLineParsed() {
2032             // transfer this to response data
2033             m_ResponseData.m_StatusCode = (HttpStatusCode) m_StatusLineValues.StatusCode;
2034             m_ResponseData.m_StatusDescription = m_StatusLineValues.StatusDescription;
2035             m_ResponseData.m_IsVersionHttp11 = m_StatusLineValues.MajorVersion >= 1 && m_StatusLineValues.MinorVersion >= 1;
2036             if (ServicePoint.HttpBehaviour==HttpBehaviour.Unknown || ServicePoint.HttpBehaviour==HttpBehaviour.HTTP11 && !m_ResponseData.m_IsVersionHttp11) {
2037                 // it's only safe to start doing HTTP/1.1 behaviour if the server's version was unknown
2038                 // or if we need to downgrade
2039                 ServicePoint.HttpBehaviour = m_ResponseData.m_IsVersionHttp11 ? HttpBehaviour.HTTP11 : HttpBehaviour.HTTP10;
2040             }
2041
2042             m_CanPipeline = ServicePoint.SupportsPipelining;
2043         }
2044
2045         /*++
2046
2047             ProcessHeaderData - Pulls out Content-length, and other critical
2048                 data from the newly parsed headers
2049
2050             Input:
2051
2052                 Nothing.
2053
2054             Returns:
2055
2056                 long - size of contentLength that we are to use
2057
2058         --*/
2059         private long ProcessHeaderData(ref bool fHaveChunked, HttpWebRequest request, out bool dummyResponseStream)
2060         {
2061             long contentLength = -1;
2062             fHaveChunked = false;
2063             //
2064             // Check for the "Transfer-Encoding" header to contain the "chunked" string
2065             //
2066             string transferEncodingString = m_ResponseData.m_ResponseHeaders[HttpKnownHeaderNames.TransferEncoding];
2067             if (transferEncodingString!=null) {
2068                 transferEncodingString = transferEncodingString.ToLower(CultureInfo.InvariantCulture);
2069                 fHaveChunked = transferEncodingString.IndexOf(HttpWebRequest.ChunkedHeader) != -1;
2070             }
2071
2072             if (!fHaveChunked) {
2073                 //
2074                 // If the response is not chunked, parse the "Content-Length" into a long for data size.
2075                 //
2076                 string contentLengthString = m_ResponseData.m_ResponseHeaders.ContentLength;
2077                 if (contentLengthString!=null) {
2078                     int index = contentLengthString.IndexOf(':');
2079                     if (index!=-1) {
2080                         contentLengthString = contentLengthString.Substring(index + 1);
2081                     }
2082                     bool success = long.TryParse(contentLengthString, NumberStyles.None, CultureInfo.InvariantCulture.NumberFormat, out contentLength);
2083                     if (!success) {
2084                         contentLength = -1;
2085                         //   in some very rare cases, a proxy server may
2086                         //   send us a pair of numbers in comma delimated
2087                         //   fashion, so we need to handle this case
2088                         index = contentLengthString.LastIndexOf(',');
2089                         if (index!=-1) {
2090                             contentLengthString = contentLengthString.Substring(index + 1);
2091                             success = long.TryParse(contentLengthString, NumberStyles.None, CultureInfo.InvariantCulture.NumberFormat, out contentLength);
2092                             if (!success) {
2093                                 contentLength = -1;
2094                             }
2095                         }
2096                     }
2097                     if (contentLength < 0)
2098                     {
2099                         GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::ProcessHeaderData - ContentLength value in header: " + contentLengthString + ", HttpWebRequest#"+ValidationHelper.HashString(m_CurrentRequest));
2100                         contentLength = c_InvalidContentLength; // This will indicate a CL error to the caller
2101                     }
2102                 }
2103             }
2104
2105             // ** else ** signal no content-length present??? or error out?
2106             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ProcessHeaderData() Content-Length parsed:" + contentLength.ToString(NumberFormatInfo.InvariantInfo));
2107
2108             dummyResponseStream = !request.CanGetResponseStream || m_ResponseData.m_StatusCode < HttpStatusCode.OK ||
2109                                   m_ResponseData.m_StatusCode == HttpStatusCode.NoContent || (m_ResponseData.m_StatusCode == HttpStatusCode.NotModified && contentLength < 0) ;
2110
2111             if (m_KeepAlive)
2112             {
2113                 //
2114                 // Deciding on  KEEP ALIVE
2115                 //
2116                 bool resetKeepAlive = false;
2117
2118                 //(1) if no content-length and no chunked, then turn off keep-alive
2119                 //    In some cases, though, Content-Length should be assumed to be 0 based on HTTP RFC 2616
2120                 if (!dummyResponseStream && contentLength < 0 && !fHaveChunked)
2121                 {
2122                     resetKeepAlive = true;
2123                 }
2124                 //(2) A workaround for a failed client ssl session on IIS6
2125                 //    The problem is that we cannot change the connection group name after it gets created.
2126                 //    IIS6 does not close the connection on 403 so all subsequent requests will fail to be authorized on THAT connection.
2127                 //-----------------------------------------------------------------------------------------------
2128                 //5/15/2006
2129                 //[....]
2130                 //The DTS Issue 595216 claims that we are unnecessarily closing the
2131                 //connection on 403 - even if it is a non SSL request. It seems
2132                 //that the original intention is to close the request for SSL requests
2133                 //The following code change would enforce closing onl fo SSL requests.
2134                 //-----------------------------------------------------------------------------------------------
2135                 else if (m_ResponseData.m_StatusCode == HttpStatusCode.Forbidden && NetworkStream is TlsStream)
2136                 {
2137                     resetKeepAlive = true;
2138                 }
2139                 // (3) Possibly cease posting a big body on the connection, was invented mainly for the very first 401 response
2140                 //
2141                 //     This optimization is for the discovery legs only.  For ntlm this is fine, because the 1st actual authleg
2142                 //     is always sent w/ content-length = 0.
2143                 //     For Kerberos preauth, it there could be 1 or 2 auth legs, but we don't know how many there are in advance,
2144                 //     so we don't have a way of eliminating the 1st auth leg.
2145                 else if (m_ResponseData.m_StatusCode > HttpWebRequest.MaxOkStatus &&
2146                          ((request.CurrentMethod == KnownHttpVerb.Post || request.CurrentMethod == KnownHttpVerb.Put) &&
2147                             m_MaximumUnauthorizedUploadLength >= 0 && request.ContentLength > m_MaximumUnauthorizedUploadLength
2148                             && (request.CurrentAuthenticationState == null || request.CurrentAuthenticationState.Module == null)))
2149                 {
2150                     resetKeepAlive = true;
2151                 }
2152                 //(4) for Http/1.0 servers, we can't be sure what their behavior
2153                 //    in this case, so the best thing is to disable KeepAlive unless explicitly set
2154                 //
2155                 else
2156                 {
2157                     //QFE: 4599.
2158                     //Author: [....]
2159                     //in v2.0, in case of SSL Requests through proxy that require NTLM authentication,
2160                     //we are not honoring the Proxy-Connection: Keep-Alive header and
2161                     //closing the connection.
2162                     //
2163                     //In v1.1 we did not have this issue because in v1.1, we would have set an
2164                     //EmptyProxy on the CONNECT request which kind of made it look like the
2165                     //service point is a proxy service point
2166                     //
2167                     //In v2.0, we don't use the GlobalProxySelection.GetEmptyWebProxy we use null
2168                     //to indicate we are not using a proxy.
2169                     //The CONNECT request is a proxy request and the service point is to the
2170                     //proxy.
2171                     //Design Notes
2172                     //------------
2173                     //This is a surgical fix. The "UsesProxySemantics is defined as
2174                     //ServicePoint is a Proxy Service point && (scheme is != https || the request is a tunnel request)
2175                     //Ideally we use one definition of whether we are going trough a proxy or not.
2176                     //The fact is that if you are connecting to a proxy, it is a proxy request and
2177                     //you should honor the Proxy-Connection header.
2178                     //
2179                     //For the purpose of this QFE, when we receive a header we test
2180                     //if this is a Proxy Service Point OR if this is a TUNNEL request
2181
2182
2183
2184                     bool haveClose = false;
2185                     bool haveKeepAlive = false;
2186                     string connection = m_ResponseData.m_ResponseHeaders[HttpKnownHeaderNames.Connection];
2187                     if (connection == null && (
2188                         (ServicePoint.InternalProxyServicePoint) ||
2189                         (request.IsTunnelRequest)))
2190                     {
2191                         connection = m_ResponseData.m_ResponseHeaders[HttpKnownHeaderNames.ProxyConnection];
2192                     }
2193
2194                     if (connection != null) {
2195                         connection = connection.ToLower(CultureInfo.InvariantCulture);
2196                         if (connection.IndexOf("keep-alive") != -1) {
2197                             haveKeepAlive = true;
2198                         }
2199                         else if (connection.IndexOf("close") != -1) {
2200                             haveClose = true;
2201                         }
2202                     }
2203
2204                     if ((haveClose && ServicePoint.HttpBehaviour==HttpBehaviour.HTTP11) ||
2205                         (!haveKeepAlive && ServicePoint.HttpBehaviour<=HttpBehaviour.HTTP10))
2206                     {
2207                         resetKeepAlive = true;
2208                     }
2209                 }
2210
2211
2212                 if (resetKeepAlive)
2213                 {
2214                     lock (this) {
2215                         m_KeepAlive = false;
2216                         m_Free = false;
2217                     }
2218                 }
2219             }
2220
2221             return contentLength;
2222         }
2223
2224         internal bool KeepAlive
2225         {
2226             get
2227             {
2228                 return m_KeepAlive;
2229             }
2230         }
2231
2232         internal bool NonKeepAliveRequestPipelined
2233         {
2234             get
2235             {
2236                 return m_NonKeepAliveRequestPipelined;
2237             }
2238         }
2239
2240         /*++
2241
2242             ParseStreamData
2243
2244             Handles parsing of the blocks of data received after buffer,
2245              distributes the data to stream constructors as needed
2246
2247             returnResult - contains a object containing Requests
2248                 that must be notified upon return from callback
2249
2250         --*/
2251         private DataParseStatus ParseStreamData(ref ConnectionReturnResult returnResult)
2252         {
2253             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::ParseStreamData");
2254
2255             if (m_CurrentRequest == null)
2256             {
2257                 GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::ParseStreamData - Aborted Request, return DataParseStatus.Invalid");
2258                 m_ParseError.Section = WebParseErrorSection.Generic;
2259                 m_ParseError.Code    = WebParseErrorCode.UnexpectedServerResponse;
2260                 return DataParseStatus.Invalid;
2261             }
2262
2263             bool fHaveChunked = false;
2264             bool dummyResponseStream;
2265             // content-length if there is one
2266             long contentLength = ProcessHeaderData(ref fHaveChunked, m_CurrentRequest, out dummyResponseStream);
2267
2268             GlobalLog.Assert(!fHaveChunked || contentLength == -1, "Connection#{0}::ParseStreamData()|fHaveChunked but contentLength != -1", ValidationHelper.HashString(this));
2269
2270             if (contentLength == c_InvalidContentLength)
2271             {
2272                 m_ParseError.Section = WebParseErrorSection.ResponseHeader;
2273                 m_ParseError.Code    = WebParseErrorCode.InvalidContentLength;
2274                 return DataParseStatus.Invalid;
2275             }
2276
2277             // bytes left over that have not been parsed
2278             int bufferLeft = (m_BytesRead - m_BytesScanned);
2279
2280             if (m_ResponseData.m_StatusCode > HttpWebRequest.MaxOkStatus)
2281             {
2282                 // This will tell the request to be prepared for possible connection drop
2283                 // Also that will stop writing on the wire if the connection is not kept alive
2284                 m_CurrentRequest.ErrorStatusCodeNotify(this, m_KeepAlive, false);
2285             }
2286
2287             int bytesToCopy;
2288             //
2289             //  If pipelining, then look for extra data that could
2290             //  be part of of another stream, if its there,
2291             //  then we need to copy it, add it to a stream,
2292             //  and then continue with the next headers
2293             //
2294
2295             if (dummyResponseStream)
2296             {
2297                 bytesToCopy = 0;
2298                 fHaveChunked = false;
2299             }
2300             else
2301             {
2302                 bytesToCopy = -1;
2303
2304                 if (!fHaveChunked && (contentLength <= (long)Int32.MaxValue))
2305                 {
2306                     bytesToCopy = (int)contentLength;
2307                 }
2308             }
2309
2310             DataParseStatus result;
2311
2312             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseStreamData() bytesToCopy:" + bytesToCopy + " bufferLeft:" + bufferLeft);
2313
2314             if (m_CurrentRequest.IsWebSocketRequest && m_ResponseData.m_StatusCode == HttpStatusCode.SwitchingProtocols)
2315             {
2316                 m_ResponseData.m_ConnectStream = new ConnectStream(this, m_ReadBuffer, m_BytesScanned, bufferLeft, bufferLeft, fHaveChunked, m_CurrentRequest);
2317
2318                 // The parsing will be resumed from m_BytesScanned when response stream is closed.
2319                 result = DataParseStatus.Done;
2320                 ClearReaderState();
2321             }
2322             else if (bytesToCopy != -1 && bytesToCopy <= bufferLeft)
2323             {
2324                 m_ResponseData.m_ConnectStream = new ConnectStream(this, m_ReadBuffer, m_BytesScanned, bytesToCopy, dummyResponseStream? 0: contentLength, fHaveChunked, m_CurrentRequest);
2325
2326                 // The parsing will be resumed from m_BytesScanned when response stream is closed.
2327                 result = DataParseStatus.ContinueParsing;
2328                 m_BytesScanned += bytesToCopy;
2329             }
2330             else
2331             {
2332                 m_ResponseData.m_ConnectStream = new ConnectStream(this, m_ReadBuffer, m_BytesScanned, bufferLeft, dummyResponseStream? 0: contentLength, fHaveChunked, m_CurrentRequest);
2333
2334                 // This is the default case where we have a buffer with no more streams except the last one to create so we create it.
2335                 // Note the buffer is fully consumed so we can reset the buffer offests.
2336                 result = DataParseStatus.Done;
2337                 ClearReaderState();
2338             }
2339
2340             m_ResponseData.m_ContentLength = contentLength;
2341             ConnectionReturnResult.Add(ref returnResult, m_CurrentRequest, m_ResponseData.Clone());
2342
2343 #if DEBUG
2344             GlobalLog.DebugUpdateRequest(m_CurrentRequest, this, GlobalLog.WaitingForReadDoneFlag);
2345 #endif
2346
2347             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::ParseStreamData");
2348             return result; // response stream is taking over the reading
2349         }
2350
2351         // Called before restarting Read operations
2352         private void ClearReaderState() {
2353             m_BytesRead    = 0;
2354             m_BytesScanned = 0;
2355         }
2356
2357         /*++
2358
2359             ParseResponseData - Parses the incomming headers, and handles
2360               creation of new streams that are found while parsing, and passes
2361               extra data the new streams
2362
2363             Input:
2364
2365                 returnResult - returns an object containing items that need to be called
2366                     at the end of the read callback
2367
2368             Returns:
2369
2370                 bool - true if one should continue reading more data
2371
2372         --*/
2373         private DataParseStatus ParseResponseData(ref ConnectionReturnResult returnResult, out bool requestDone, out CoreResponseData continueResponseData)
2374         {
2375             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData()");
2376
2377             DataParseStatus parseStatus = DataParseStatus.NeedMoreData;
2378             DataParseStatus parseSubStatus;
2379
2380             // Indicates whether or not at least one whole request was processed in this loop.
2381             // (i.e. Whether ParseStreamData() was called.
2382             requestDone = false;
2383             continueResponseData = null;
2384
2385             // loop in case of multiple sets of headers or streams,
2386             //  that may be generated due to a pipelined response
2387
2388                 // Invariants: at the start of this loop, m_BytesRead
2389                 // is the number of bytes in the buffer, and m_BytesScanned
2390                 // is how many bytes of the buffer we've consumed so far.
2391                 // and the m_ReadState var will be updated at end of
2392                 // each code path, call to this function to reflect,
2393                 // the state, or error condition of the parsing of data
2394                 //
2395                 // We use the following variables in the code below:
2396                 //
2397                 //  m_ReadState - tracks the current state of our Parsing in a
2398                 //      response. z.B.
2399                 //      Start - initial start state and begining of response
2400                 //      StatusLine - the first line sent in response, include status code
2401                 //      Headers - \r\n delimiated Header parsing until we find entity body
2402                 //      Data - Entity Body parsing, if we have all data, we create stream directly
2403                 //
2404                 //  m_ResponseData - An object used to gather Stream, Headers, and other
2405                 //      tidbits so that a request/Response can receive this data when
2406                 //      this code is finished processing
2407                 //
2408                 //  m_ReadBuffer - Of course the buffer of data we are parsing from
2409                 //
2410                 //  m_BytesScanned - The bytes scanned in this buffer so far,
2411                 //      since its always assumed that parse to completion, this
2412                 //      var becomes ended of known data at the end of this function,
2413                 //      based off of 0
2414                 //
2415                 //  m_BytesRead - The total bytes read in buffer, should be const,
2416                 //      till its updated at end of function.
2417                 //
2418
2419                 //
2420                 // Now attempt to parse the data,
2421                 //   we first parse status line,
2422                 //   then read headers,
2423                 //   and finally transfer results to a new stream, and tell request
2424                 //
2425
2426                 switch (m_ReadState) {
2427
2428                     case ReadState.Start:
2429
2430
2431                         if (m_CurrentRequest == null)
2432                         {
2433                             lock (this)
2434                             {
2435                                 if (m_WriteList.Count == 0 || ((m_CurrentRequest = m_WriteList[0] as HttpWebRequest) == null))
2436                                 {
2437                                     m_ParseError.Section = WebParseErrorSection.Generic;
2438                                     m_ParseError.Code    = WebParseErrorCode.Generic;
2439                                     parseStatus = DataParseStatus.Invalid;
2440                                     break;
2441                                 }
2442                             }
2443                         }
2444
2445                         //
2446                         // Start of new response. Transfer the keep-alive context from the corresponding request to
2447                         // the connection
2448                         //
2449                         m_KeepAlive &= (m_CurrentRequest.KeepAlive || m_CurrentRequest.NtlmKeepAlive);
2450
2451                         m_MaximumResponseHeadersLength = m_CurrentRequest.MaximumResponseHeadersLength * 1024;
2452                         m_ResponseData = new CoreResponseData();
2453                         m_ReadState = ReadState.StatusLine;
2454                         m_TotalResponseHeadersLength = 0;
2455
2456                         InitializeParseStatusLine();
2457                         goto case ReadState.StatusLine;
2458
2459                     case ReadState.StatusLine:
2460                         //
2461                         // Reads HTTP status response line
2462                         //
2463                         if (SettingsSectionInternal.Section.UseUnsafeHeaderParsing)
2464                         {
2465                             // This one uses an array to store the parsed values in.  Marshal between this legacy way.
2466                             int[] statusInts = new int[] { 0, m_StatusLineValues.MajorVersion, m_StatusLineValues.MinorVersion, m_StatusLineValues.StatusCode };
2467                             if (m_StatusLineValues.StatusDescription == null)
2468                                 m_StatusLineValues.StatusDescription = "";
2469
2470                             parseSubStatus = ParseStatusLine(
2471                                 m_ReadBuffer, // buffer we're working with
2472                                 m_BytesRead,  // total bytes read so far
2473                                 ref m_BytesScanned, // index off of what we've scanned
2474                                 ref statusInts,
2475                                 ref m_StatusLineValues.StatusDescription,
2476                                 ref m_StatusState,
2477                                 ref m_ParseError);
2478
2479                             m_StatusLineValues.MajorVersion = statusInts[1];
2480                             m_StatusLineValues.MinorVersion = statusInts[2];
2481                             m_StatusLineValues.StatusCode = statusInts[3];
2482                         }
2483                         else
2484                         {
2485                             parseSubStatus = ParseStatusLineStrict(
2486                                 m_ReadBuffer, // buffer we're working with
2487                                 m_BytesRead,  // total bytes read so far
2488                                 ref m_BytesScanned, // index off of what we've scanned
2489                                 ref m_StatusState,
2490                                 m_StatusLineValues,
2491                                 m_MaximumResponseHeadersLength,
2492                                 ref m_TotalResponseHeadersLength,
2493                                 ref m_ParseError);
2494                         }
2495
2496                         if (parseSubStatus == DataParseStatus.Done)
2497                         {
2498                             if (Logging.On) Logging.PrintInfo(Logging.Web, this, SR.GetString(SR.net_log_received_status_line, m_StatusLineValues.MajorVersion+"."+m_StatusLineValues.MinorVersion, m_StatusLineValues.StatusCode, m_StatusLineValues.StatusDescription));
2499                             SetStatusLineParsed();
2500                             m_ReadState = ReadState.Headers;
2501                             m_ResponseData.m_ResponseHeaders = new WebHeaderCollection(WebHeaderCollectionType.HttpWebResponse);
2502                             goto case ReadState.Headers;
2503                         }
2504                         else if (parseSubStatus != DataParseStatus.NeedMoreData)
2505                         {
2506                             //
2507                             // report error - either Invalid or DataTooBig
2508                             //
2509                             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData() ParseStatusLine() parseSubStatus:" + parseSubStatus.ToString());
2510                             parseStatus = parseSubStatus;
2511                             break;
2512                         }
2513
2514                         break; // read more data
2515
2516                     case ReadState.Headers:
2517                         //
2518                         // Parse additional lines of header-name: value pairs
2519                         //
2520                         if (m_BytesScanned >= m_BytesRead) {
2521                             //
2522                             // we already can tell we need more data
2523                             //
2524                             break;
2525                         }
2526
2527                         if (SettingsSectionInternal.Section.UseUnsafeHeaderParsing)
2528                         {
2529                             parseSubStatus = m_ResponseData.m_ResponseHeaders.ParseHeaders(
2530                                 m_ReadBuffer,
2531                                 m_BytesRead,
2532                                 ref m_BytesScanned,
2533                                 ref m_TotalResponseHeadersLength,
2534                                 m_MaximumResponseHeadersLength,
2535                                 ref m_ParseError);
2536                         }
2537                         else
2538                         {
2539                             parseSubStatus = m_ResponseData.m_ResponseHeaders.ParseHeadersStrict(
2540                                 m_ReadBuffer,
2541                                 m_BytesRead,
2542                                 ref m_BytesScanned,
2543                                 ref m_TotalResponseHeadersLength,
2544                                 m_MaximumResponseHeadersLength,
2545                                 ref m_ParseError);
2546                         }
2547
2548                         if (parseSubStatus == DataParseStatus.Invalid || parseSubStatus == DataParseStatus.DataTooBig)
2549                         {
2550                             //
2551                             // report error
2552                             //
2553                             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData() ParseHeaders() parseSubStatus:" + parseSubStatus.ToString());
2554                             parseStatus = parseSubStatus;
2555                             break;
2556                         }
2557                         else if (parseSubStatus == DataParseStatus.Done)
2558                         {
2559                             if (Logging.On) Logging.PrintInfo(Logging.Web, this, SR.GetString(SR.net_log_received_headers, m_ResponseData.m_ResponseHeaders.ToString(true)));
2560
2561                             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData() DataParseStatus.Done StatusCode:" + (int)m_ResponseData.m_StatusCode + " m_BytesRead:" + m_BytesRead + " m_BytesScanned:" + m_BytesScanned);
2562
2563                             //get the IIS server version
2564                             if(m_IISVersion == -1){
2565                                 string server = m_ResponseData.m_ResponseHeaders.Server;
2566                                 if (server != null && server.ToLower(CultureInfo.InvariantCulture).Contains("microsoft-iis")){
2567                                     int i = server.IndexOf("/");
2568                                     if(i++>0 && i <server.Length){
2569                                         m_IISVersion = server[i++] - '0';
2570                                         while(i < server.Length && Char.IsDigit(server[i])) {
2571                                             m_IISVersion = m_IISVersion*10 + server[i++] - '0';
2572                                         }
2573                                     }
2574                                 }
2575                                 //we got a response,so if we don't know the server by now and it wasn't a 100 continue,
2576                                 //we can't assume we will ever know it.  IIS5 sends its server header w/ the continue
2577
2578                                 if(m_IISVersion == -1 && m_ResponseData.m_StatusCode != HttpStatusCode.Continue){
2579                                     m_IISVersion = 0;
2580                                 }
2581                             }
2582
2583                             if (m_ResponseData.m_StatusCode == HttpStatusCode.Continue || m_ResponseData.m_StatusCode == HttpStatusCode.BadRequest) {
2584
2585                                 GlobalLog.Assert(m_CurrentRequest != null, "Connection#{0}::ParseResponseData()|m_CurrentRequest == null", ValidationHelper.HashString(this));
2586                                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData() HttpWebRequest#" + ValidationHelper.HashString(m_CurrentRequest));
2587
2588                                 if (m_ResponseData.m_StatusCode == HttpStatusCode.BadRequest) {
2589                                     // If we have a 400 and we were sending a chunked request going through to a proxy with a chunked upload,
2590                                     // this proxy is a partially compliant so shut off fancy features (pipelining and chunked uploads)
2591                                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData() got a 400 StatusDescription:" + m_ResponseData.m_StatusDescription);
2592                                     if (ServicePoint.HttpBehaviour == HttpBehaviour.HTTP11
2593                                         && m_CurrentRequest.HttpWriteMode==HttpWriteMode.Chunked
2594                                         && m_ResponseData.m_ResponseHeaders.Via != null
2595                                         && string.Compare(m_ResponseData.m_StatusDescription, "Bad Request ( The HTTP request includes a non-supported header. Contact the Server administrator.  )", StringComparison.OrdinalIgnoreCase)==0) {
2596                                         GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData() downgrading server to HTTP11PartiallyCompliant.");
2597                                         ServicePoint.HttpBehaviour = HttpBehaviour.HTTP11PartiallyCompliant;
2598                                     }
2599                                 }
2600                                 else {
2601                                     // If we have an HTTP continue, eat these headers and look
2602                                     //  for the 200 OK
2603                                     //
2604                                     // we got a 100 Continue. set this on the HttpWebRequest
2605                                     //
2606                                     m_CurrentRequest.Saw100Continue = true;
2607                                     if (!ServicePoint.Understands100Continue) {
2608                                         //
2609                                         // and start expecting it again if this behaviour was turned off
2610                                         //
2611                                         GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData() HttpWebRequest#" + ValidationHelper.HashString(m_CurrentRequest) + " ServicePoint#" + ValidationHelper.HashString(ServicePoint) + " sent UNexpected 100 Continue");
2612                                         ServicePoint.Understands100Continue = true;
2613                                     }
2614
2615                                     //
2616                                     // set Continue Ack on request.
2617                                     //
2618                                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData() calling SetRequestContinue()");
2619                                     continueResponseData = m_ResponseData;
2620
2621                                     //if we got a 100continue we ---- it and start looking for a final response
2622                                     goto case ReadState.Start;
2623                                 }
2624                             }
2625
2626                             m_ReadState = ReadState.Data;
2627                             goto case ReadState.Data;
2628                         }
2629
2630                         // need more data,
2631                         break;
2632
2633                     case ReadState.Data:
2634
2635                         // (check status code for continue handling)
2636                         // 1. Figure out if its Chunked, content-length, or encoded
2637                         // 2. Takes extra data, place in stream(s)
2638                         // 3. Wake up blocked stream requests
2639                         //
2640
2641                         // Got through one entire response
2642                         requestDone = true;
2643
2644                         // parse and create a stream if needed
2645                         parseStatus = ParseStreamData(ref returnResult);
2646
2647                         GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData() result:" + parseStatus);
2648                         GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData()" + " WriteDone:" + m_WriteDone + " ReadDone:" + m_ReadDone + " WaitList:" + m_WaitList.Count + " WriteList:" + m_WriteList.Count);
2649                         break;
2650                 }
2651
2652             if (m_BytesScanned == m_BytesRead)
2653                 ClearReaderState();
2654
2655             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData() m_ReadState:" + m_ReadState);
2656             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::ParseResponseData()", parseStatus.ToString());
2657             return parseStatus;
2658         }
2659
2660         /// <devdoc>
2661         ///    <para>
2662         ///       Cause the Connection to Close and Abort its socket,
2663         ///         after the next request is completed.  If the Connection
2664         ///         is already idle, then Aborts the socket immediately.
2665         ///    </para>
2666         /// </devdoc>
2667         internal void CloseOnIdle() {
2668             // The timer thread is allowed to call this.  (It doesn't call user code and doesn't block.)
2669             GlobalLog.ThreadContract(ThreadKinds.Unknown, ThreadKinds.SafeSources | ThreadKinds.Timer, "Connection#" + ValidationHelper.HashString(this) + "::CloseOnIdle");
2670
2671             lock(this){
2672                 m_KeepAlive = false;
2673                 m_RemovedFromConnectionList = true;
2674                 if (!m_Idle)
2675                 {
2676                     CheckIdle();
2677                 }
2678                 if (m_Idle)
2679                 {
2680                     AbortSocket(false);
2681                     GC.SuppressFinalize(this);
2682                 }
2683             }
2684         }
2685
2686         internal bool AbortOrDisassociate(HttpWebRequest request, WebException webException)
2687         {
2688             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::AbortOrDisassociate", "request#" + ValidationHelper.HashString(request));
2689             GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection#" + ValidationHelper.HashString(this) + "::AbortOrDisassociate()");
2690
2691             ConnectionReturnResult result = null;
2692             lock(this)
2693             {
2694                 int idx = m_WriteList.IndexOf(request);
2695                 // If the request is in the submission AND this is the first request we have to abort the connection,
2696                 // Otheriwse we simply disassociate it from the current connection.
2697                 if (idx == -1)
2698                 {
2699                     WaitListItem foundItem = null;
2700
2701                     if (m_WaitList.Count > 0)
2702                     {
2703                         foundItem = m_WaitList.Find(o => object.ReferenceEquals(o.Request, request));
2704                     }
2705
2706                     // If not found then the request must be already dispatched and the response stream is drained
2707                     // If so then we let request.Abort() to deal with this situation.
2708                     if (foundItem != null)
2709                     {
2710                         NetworkingPerfCounters.Instance.IncrementAverage(NetworkingPerfCounterName.HttpWebRequestAvgQueueTime,
2711                             foundItem.QueueStartTime);
2712                         m_WaitList.Remove(foundItem);
2713                         UnlockIfNeeded(foundItem.Request);
2714                     }
2715
2716                     GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::AbortOrDisassociate()", "Request was wisassociated");
2717                     return true;
2718                 }
2719                 else if (idx != 0)
2720                 {
2721                     // Make this connection Keep-Alive=false, remove the request and do not close the connection
2722                     // When the active request completes, the rest of the pipeline (minus aborted request) will be resubmitted.
2723                     m_WriteList.RemoveAt(idx);
2724                     m_KeepAlive = false;
2725                     GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::AbortOrDisassociate()", "Request was Disassociated from the Write List, idx = " + idx);
2726                     return true;
2727                 }
2728
2729 #if DEBUG
2730                 try
2731                 {
2732 #endif
2733                 m_KeepAlive = false;
2734                 if (webException != null && m_InnerException == null)
2735                 {
2736                     m_InnerException = webException;
2737                     m_Error = webException.Status;
2738                 }
2739                 else
2740                 {
2741                     m_Error = WebExceptionStatus.RequestCanceled;
2742                 }
2743
2744                 PrepareCloseConnectionSocket(ref result);
2745                 // Hard Close the socket.
2746                 Close(0);
2747                 FreeReadBuffer();       // Do it after close completes to insure buffer not in use
2748 #if DEBUG
2749                 }
2750                 catch (Exception exception)
2751                 {
2752                     t_LastStressException = exception;
2753                     if (!NclUtilities.IsFatal(exception)){
2754                         GlobalLog.Assert("Connection#" + ValidationHelper.HashString(this) + "::AbortOrDisassociate()", exception.Message);
2755                     }
2756                 }
2757 #endif
2758             }
2759             ConnectionReturnResult.SetResponses(result);
2760             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::AbortOrDisassociate()", "Connection Aborted");
2761             return false;
2762         }
2763
2764 #if DEBUG
2765         [ThreadStatic]
2766         private static Exception t_LastStressException;
2767 #endif
2768
2769         internal void AbortSocket(bool isAbortState)
2770         {
2771             // The timer/finalization thread is allowed to call this.  (It doesn't call user code and doesn't block.)
2772             GlobalLog.ThreadContract(ThreadKinds.Unknown, ThreadKinds.SafeSources | ThreadKinds.Timer | ThreadKinds.Finalization, "Connection#" + ValidationHelper.HashString(this) + "::AbortSocket");
2773             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::Abort", "isAbortState:" + isAbortState.ToString());
2774
2775             if (isAbortState) {
2776                 UnlockRequest();
2777                 CheckIdle();
2778             }
2779             else {
2780                 // This one is recoverable, set it to keep  Read/Write StartNextRequest happy.
2781                 m_Error = WebExceptionStatus.KeepAliveFailure;
2782             }
2783
2784             // Stream close isn't threadsafe.
2785             lock (this)
2786             {
2787                 Close(0);
2788                 FreeReadBuffer();       // Do it after close completes to insure buffer not in use
2789             }
2790
2791             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::Abort", "isAbortState:" + isAbortState.ToString());
2792         }
2793
2794
2795         /*++
2796
2797             PrepareCloseConnectionSocket - reset the connection requests list.
2798
2799             This method is called when we want to close the conection.
2800             It must be called with the critical section held.
2801             The caller must call this.Close if decided to call this method.
2802
2803             All connection closes (either ours or server initiated) eventually go through here.
2804
2805             As to what we do: we loop through our write and wait list and pull requests
2806             off it, and give each request an error failure. Then the caller will
2807             dispatch the responses.
2808
2809         --*/
2810
2811         private void PrepareCloseConnectionSocket(ref ConnectionReturnResult returnResult)
2812         {
2813             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::PrepareCloseConnectionSocket", m_Error.ToString());
2814
2815             // Effectivelly, closing a connection makes it exempted from the "Idling" logic
2816             m_IdleSinceUtc = DateTime.MinValue;
2817             CanBePooled = false;
2818
2819             if (m_WriteList.Count != 0 || m_WaitList.Count != 0)
2820             {
2821
2822                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::PrepareCloseConnectionSocket() m_WriteList.Count:" + m_WriteList.Count);
2823                 DebugDumpWriteListEntries();
2824                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::PrepareCloseConnectionSocket() m_WaitList.Count:" + m_WaitList.Count);
2825                 DebugDumpWaitListEntries();
2826
2827                 HttpWebRequest lockedRequest = LockedRequest;
2828
2829                 if (lockedRequest != null)
2830                 {
2831                     bool callUnlockRequest = false;
2832                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::PrepareCloseConnectionSocket() looking for HttpWebRequest#" + ValidationHelper.HashString(lockedRequest));
2833
2834                     foreach (HttpWebRequest request in m_WriteList)
2835                     {
2836                         if (request == lockedRequest) {
2837                             callUnlockRequest = true;
2838                         }
2839                     }
2840
2841                     if (!callUnlockRequest) {
2842                         foreach (WaitListItem item in m_WaitList) {
2843                             if (item.Request == lockedRequest) {
2844                                 callUnlockRequest = true;
2845                                 break;
2846                             }
2847                         }
2848                     }
2849                     if (callUnlockRequest) {
2850                         UnlockRequest();
2851                     }
2852                 }
2853
2854                 HttpWebRequest[] requestArray = null;
2855
2856                 // WaitList gets Isolated exception status, free to retry multiple times
2857                 if (m_WaitList.Count != 0)
2858                 {
2859                     requestArray = new HttpWebRequest[m_WaitList.Count];
2860                     for (int i = 0; i < m_WaitList.Count; i++)
2861                     {
2862                         requestArray[i] = m_WaitList[i].Request;
2863                     }
2864                     ConnectionReturnResult.AddExceptionRange(ref returnResult, requestArray, ExceptionHelper.IsolatedException);
2865                 }
2866
2867                 //
2868                 // WriteList (except for single request list) gets Recoverable exception status, may be retired if not failed once
2869                 // For a single request list the exception is computed here
2870                 // InnerExeption if any may tell more details in both cases
2871                 //
2872                 if (m_WriteList.Count != 0)
2873                 {
2874                     Exception theException = m_InnerException;
2875
2876
2877                     if(theException != null)
2878                        GlobalLog.Print(theException.ToString());
2879
2880                     GlobalLog.Print("m_Error = "+ m_Error.ToString());
2881
2882                     if (!(theException is WebException) && !(theException is SecurityException))
2883                     {
2884                         if (m_Error == WebExceptionStatus.ServerProtocolViolation)
2885                         {
2886                             string errorString = NetRes.GetWebStatusString(m_Error);
2887
2888                             string detailedInfo = "";
2889                             if (m_ParseError.Section != WebParseErrorSection.Generic)
2890                                 detailedInfo += " Section=" + m_ParseError.Section.ToString();
2891                             if (m_ParseError.Code != WebParseErrorCode.Generic) {
2892                                 detailedInfo += " Detail=" + SR.GetString("net_WebResponseParseError_" + m_ParseError.Code.ToString());
2893                             }
2894                             if (detailedInfo.Length != 0)
2895                                 errorString += "." + detailedInfo;
2896
2897                             theException = new WebException(errorString,
2898                                                             theException,
2899                                                             m_Error,
2900                                                             null,
2901                                                             WebExceptionInternalStatus.RequestFatal);
2902                         }
2903                         else if (m_Error == WebExceptionStatus.SecureChannelFailure)
2904                         {
2905                             theException = new WebException(NetRes.GetWebStatusString("net_requestaborted", WebExceptionStatus.SecureChannelFailure),
2906                                                             WebExceptionStatus.SecureChannelFailure);
2907                         }
2908
2909                         else if (m_Error == WebExceptionStatus.Timeout)
2910                         {
2911                             theException = new WebException(NetRes.GetWebStatusString("net_requestaborted", WebExceptionStatus.Timeout),
2912                                                             WebExceptionStatus.Timeout);
2913                         }
2914                         else if(m_Error == WebExceptionStatus.RequestCanceled)
2915                         {
2916                             theException = new WebException(NetRes.GetWebStatusString("net_requestaborted", WebExceptionStatus.RequestCanceled),
2917                                                             WebExceptionStatus.RequestCanceled,
2918                                                             WebExceptionInternalStatus.RequestFatal,
2919                                                             theException);
2920                         }
2921                         else if(m_Error == WebExceptionStatus.MessageLengthLimitExceeded ||
2922                                 m_Error == WebExceptionStatus.TrustFailure)
2923                         {
2924                             theException = new WebException(NetRes.GetWebStatusString("net_connclosed", m_Error),
2925                                                             m_Error,
2926                                                             WebExceptionInternalStatus.RequestFatal,
2927                                                             theException);
2928                         }
2929                         else
2930                         {
2931                             if (m_Error == WebExceptionStatus.Success)
2932                             {
2933                                 throw new InternalException();              // TODO: replace it with a generic error for the product bits
2934                                 //m_Error = WebExceptionStatus.UnknownError;
2935                             }
2936
2937                             bool retry = false;
2938                             bool isolatedKeepAliveFailure = false;
2939
2940                             if (m_WriteList.Count != 1)
2941                             {
2942                                 // Real scenario: SSL against IIS-5 would fail if pipelinning.
2943                                 // retry = true will cover a general case when >>the server<< aborts a pipeline
2944                                 // Basically all pipelined requests are marked with recoverable error including the very active request.
2945                                 retry = true;
2946                             }
2947                             else if (m_Error == WebExceptionStatus.KeepAliveFailure)
2948                             {
2949                                 HttpWebRequest request = (HttpWebRequest) m_WriteList[0];
2950                                 // Check that the active request did not start the body yet
2951                                 if (!request.BodyStarted)
2952                                     isolatedKeepAliveFailure = true;
2953                             }
2954                             else{
2955                                 retry = (!AtLeastOneResponseReceived && !((HttpWebRequest) m_WriteList[0]).BodyStarted);
2956                             }
2957                                 theException = new WebException(NetRes.GetWebStatusString("net_connclosed", m_Error),
2958                                                             m_Error,
2959                                                             (isolatedKeepAliveFailure? WebExceptionInternalStatus.Isolated:
2960                                                                 retry? WebExceptionInternalStatus.Recoverable:
2961                                                                 WebExceptionInternalStatus.RequestFatal),
2962                                                             theException);
2963                         }
2964                     }
2965
2966                     WebException pipelineException = new WebException(NetRes.GetWebStatusString("net_connclosed", WebExceptionStatus.PipelineFailure),
2967                                                                       WebExceptionStatus.PipelineFailure,
2968                                                                       WebExceptionInternalStatus.Recoverable,
2969                                                                       theException);
2970
2971                     requestArray = new HttpWebRequest[m_WriteList.Count];
2972                     m_WriteList.CopyTo(requestArray, 0);
2973                     ConnectionReturnResult.AddExceptionRange(ref returnResult, requestArray, pipelineException, theException);
2974                 }
2975
2976 #if TRAVE
2977                 foreach (WaitListItem item in m_WaitList) {
2978                     GlobalLog.Print("Request removed from WaitList#"+ValidationHelper.HashString(item.Request));
2979                 }
2980
2981                 foreach (HttpWebRequest request in m_WriteList) {
2982                     GlobalLog.Print("Request removed from m_WriteList#"+ValidationHelper.HashString(request));
2983                 }
2984 #endif
2985
2986                 m_WriteList.Clear();
2987
2988                 foreach (WaitListItem item in m_WaitList)
2989                 {
2990                     NetworkingPerfCounters.Instance.IncrementAverage(NetworkingPerfCounterName.HttpWebRequestAvgQueueTime,
2991                         item.QueueStartTime);
2992                 }
2993                 m_WaitList.Clear();
2994             }
2995
2996             CheckIdle();
2997
2998             if (m_Idle)
2999             {
3000                 GC.SuppressFinalize(this);
3001             }
3002             if (!m_RemovedFromConnectionList && ConnectionGroup != null)
3003             {
3004                 m_RemovedFromConnectionList = true;
3005                 ConnectionGroup.Disassociate(this);
3006             }
3007
3008             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::PrepareCloseConnectionSocket");
3009         }
3010
3011
3012         /*++
3013
3014             HandleError - Handle a protocol error from the server.
3015
3016             This method is called when we've detected some sort of fatal protocol
3017             violation while parsing a response, receiving data from the server,
3018             or failing to connect to the server. We'll fabricate
3019             a WebException and then call CloseConnection which closes the
3020             connection as well as informs the request through a callback.
3021
3022             Input:
3023                     webExceptionStatus -
3024                     connectFailure -
3025                     readFailure -
3026
3027             Returns: Nothing
3028
3029         --*/
3030         internal void HandleConnectStreamException(bool writeDone, bool readDone, WebExceptionStatus webExceptionStatus, ref ConnectionReturnResult returnResult, Exception e)
3031         {
3032             if (m_InnerException == null)
3033             {
3034                 m_InnerException = e;
3035                 if (!(e is WebException) && NetworkStream is TlsStream)
3036                 {
3037                     // Unless a WebException is passed the Connection knows better the error code if the transport is TlsStream
3038                     webExceptionStatus = ((TlsStream) NetworkStream).ExceptionStatus;
3039                 }
3040                 else if (e is ObjectDisposedException)
3041                 {
3042                     webExceptionStatus = WebExceptionStatus.RequestCanceled;
3043                 }
3044             }
3045             HandleError(writeDone, readDone, webExceptionStatus, ref returnResult);
3046         }
3047         //
3048         private void HandleErrorWithReadDone(WebExceptionStatus webExceptionStatus, ref ConnectionReturnResult returnResult)
3049         {
3050             HandleError(false, true, webExceptionStatus, ref returnResult);
3051         }
3052         //
3053         private void HandleError(bool writeDone, bool readDone, WebExceptionStatus webExceptionStatus, ref ConnectionReturnResult returnResult)
3054         {
3055             lock(this)
3056             {
3057                 if (writeDone)
3058                     m_WriteDone = true;
3059                 if (readDone)
3060                     m_ReadDone = true;
3061
3062                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::HandleError() m_WriteList.Count:" + m_WriteList.Count +
3063                                 " m_WaitList.Count:" + m_WaitList.Count +
3064                                 " new WriteDone:" + m_WriteDone + " new ReadDone:" + m_ReadDone);
3065                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::HandleError() current HttpWebRequest#" + ValidationHelper.HashString(m_CurrentRequest));
3066
3067                 if(webExceptionStatus == WebExceptionStatus.Success)
3068                     throw new InternalException(); //consider making an assert later.
3069
3070                 m_Error = webExceptionStatus;
3071
3072                 PrepareCloseConnectionSocket(ref returnResult);
3073                 // This will kill the socket
3074                 // Must be done inside the lock.  (Stream Close() isn't threadsafe.)
3075                 Close(0);
3076                 FreeReadBuffer();       // Do it after close completes to insure buffer not in use
3077             }
3078         }
3079
3080         private static void ReadCallbackWrapper(IAsyncResult asyncResult)
3081         {
3082             if (asyncResult.CompletedSynchronously)
3083             {
3084                 return;
3085             }
3086
3087             ((Connection) asyncResult.AsyncState).ReadCallback(asyncResult);
3088         }
3089
3090         /// <devdoc>
3091         ///    <para>Performs read callback processing on connection
3092         ///     handles getting headers parsed and streams created</para>
3093         /// </devdoc>
3094         private void ReadCallback(IAsyncResult asyncResult)
3095         {
3096             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::ReadCallback", ValidationHelper.HashString(asyncResult));
3097
3098             int bytesRead = -1;
3099             WebExceptionStatus errorStatus = WebExceptionStatus.ReceiveFailure;
3100
3101             //
3102             // parameter validation
3103             //
3104             GlobalLog.Assert(asyncResult != null, "Connection#{0}::ReadCallback()|asyncResult == null", ValidationHelper.HashString(this));
3105             GlobalLog.Assert((asyncResult is OverlappedAsyncResult || asyncResult is LazyAsyncResult), "Connection#{0}::ReadCallback()|asyncResult is not OverlappedAsyncResult.", ValidationHelper.HashString(this));
3106
3107             try {
3108                 bytesRead = EndRead(asyncResult);
3109                 if (bytesRead == 0)
3110                     bytesRead = -1; // 0 is reserved for re-entry on already buffered data
3111
3112                 errorStatus = WebExceptionStatus.Success;
3113             }
3114             catch (Exception exception) {
3115                 // Notify request's SubmitWriteStream that a socket error happened.  This will cause future writes to
3116                    // throw an IOException.
3117                    HttpWebRequest curRequest = m_CurrentRequest;
3118                    if (curRequest != null)
3119                    {
3120                        curRequest.ErrorStatusCodeNotify(this, false, true);
3121                    }
3122
3123
3124                 if (m_InnerException == null)
3125                     m_InnerException = exception;
3126
3127                 if (exception.GetType() == typeof(ObjectDisposedException))
3128                     errorStatus = WebExceptionStatus.RequestCanceled;
3129
3130 #if !FEATURE_PAL
3131                 //ASYNCISSUE
3132                 // Consider: In case of a async exception we should do minimal cleanup here trying the appDomain
3133                 // to survive or to force unloading of the appDomain .
3134                 // need to handle SSL errors too
3135                 if (NetworkStream is TlsStream)  {
3136                     errorStatus = ((TlsStream) NetworkStream).ExceptionStatus;
3137                 }
3138                 else {
3139                     errorStatus = WebExceptionStatus.ReceiveFailure;
3140                 }
3141 #else
3142                 errorStatus = WebExceptionStatus.ReceiveFailure;
3143 #endif // !FEATURE_PAL
3144                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadCallback() EndRead() errorStatus:" + errorStatus.ToString() + " caught exception:" + exception);
3145             }
3146
3147             ReadComplete(bytesRead, errorStatus);
3148             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::ReadCallback");
3149         }
3150
3151
3152         /// <devdoc>
3153         ///    <para>Attempts to poll the socket, to see if data is waiting to be read,
3154         ///     if there is data there, then a read is started</para>
3155         /// </devdoc>
3156         internal void PollAndRead(HttpWebRequest request, bool userRetrievedStream) {
3157             GlobalLog.ThreadContract(ThreadKinds.Unknown, "Connection#" + ValidationHelper.HashString(this) + "::PollAndRead");
3158
3159             // Ensure that we don't already have a response for this request, before we attempt to read the socket.
3160             request.NeedsToReadForResponse = true;
3161             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::PollAndRead() InternalPeekCompleted:" + request.ConnectionReaderAsyncResult.InternalPeekCompleted.ToString() + " Result:" + ValidationHelper.ToString(request.ConnectionReaderAsyncResult.Result));
3162             if (request.ConnectionReaderAsyncResult.InternalPeekCompleted && request.ConnectionReaderAsyncResult.Result == null && CanBePooled)
3163             {
3164                 SyncRead(request, userRetrievedStream, true);
3165             }
3166         }
3167         //
3168         //    Peforms a [....] Read and calls the ReadComplete to process the result
3169         //    The reads are done iteratively, until the Request has received enough
3170         //    data to contruct a response, or a 100-Continue is read, allowing the HttpWebRequest
3171         //    to return a write stream
3172         //
3173         //    probeRead = true only for POST request and when the caller needs to wait for 100-continue
3174         //
3175         internal void SyncRead(HttpWebRequest request, bool userRetrievedStream, bool probeRead)
3176         {
3177             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::SyncRead(byte[]) request#" + ValidationHelper.HashString(request) + (probeRead? ", Probe read = TRUE":string.Empty));
3178             GlobalLog.ThreadContract(ThreadKinds.Sync, "Connection#" + ValidationHelper.HashString(this) + "::SyncRead");
3179
3180             // prevent recursive calls to this function
3181             if (t_SyncReadNesting > 0) {
3182                 GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::SyncRead() - nesting");
3183                 return;
3184             }
3185
3186             bool pollSuccess = probeRead? false: true;
3187
3188             try {
3189                 t_SyncReadNesting++;
3190
3191                 // grab a counter to tell us whenever the SetRequestContinue is called
3192                 int requestContinueCount = probeRead ? request.RequestContinueCount : 0;
3193
3194                 bool requestDone;
3195                 int bytesRead = -1;
3196                 WebExceptionStatus errorStatus = WebExceptionStatus.ReceiveFailure;
3197
3198
3199                 if (m_BytesScanned < m_BytesRead)
3200                 {
3201                     // left over from previous read
3202                     pollSuccess = true;
3203                     bytesRead = 0; //tell it we want to use buffered data on the first iteration
3204                     errorStatus = WebExceptionStatus.Success;
3205                 }
3206
3207                 do {
3208                     requestDone = true;
3209
3210                     try {
3211                         if (bytesRead != 0)
3212                         {
3213                             errorStatus = WebExceptionStatus.ReceiveFailure;
3214
3215                             if (!pollSuccess)
3216                             {
3217                                 pollSuccess = Poll(request.ContinueTimeout * 1000, SelectMode.SelectRead);  // Timeout is in microseconds
3218                                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::SyncRead() PollSuccess : " + pollSuccess);
3219                             }
3220
3221                             if (pollSuccess)
3222                             {
3223                                 //Ensures that we'll timeout eventually on an appdomain unload.
3224                                 //Will be a no-op if the timeout doesn't change from request to request.
3225                                 ReadTimeout = request.Timeout;
3226
3227                                 bytesRead = Read(m_ReadBuffer, m_BytesRead, m_ReadBuffer.Length - m_BytesRead);
3228                                 errorStatus = WebExceptionStatus.Success;
3229                                 if (bytesRead == 0)
3230                                     bytesRead = -1; // 0 is reserved for re-entry on already buffered data
3231                             }
3232                         }
3233                     }
3234                     catch (Exception exception)
3235                     {
3236                         if (NclUtilities.IsFatal(exception)) throw;
3237
3238                         if (m_InnerException == null)
3239                             m_InnerException = exception;
3240
3241                         if (exception.GetType() == typeof(ObjectDisposedException))
3242                             errorStatus = WebExceptionStatus.RequestCanceled;
3243
3244                         // need to handle SSL errors too
3245 #if !FEATURE_PAL
3246                         else if (NetworkStream is TlsStream)  {
3247                             errorStatus = ((TlsStream)NetworkStream).ExceptionStatus;
3248                         }
3249 #endif // !FEATURE_PAL
3250                         else
3251                         {
3252                             SocketException socketException = exception.InnerException as SocketException;
3253                             if (socketException != null)
3254                             {
3255                                  if (socketException.ErrorCode == (int) SocketError.TimedOut)
3256                                     errorStatus = WebExceptionStatus.Timeout;
3257                                 else
3258                                     errorStatus = WebExceptionStatus.ReceiveFailure;
3259                             }
3260                         }
3261
3262                         GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::SyncRead() Read() threw errorStatus:" + errorStatus.ToString() + " bytesRead:" + bytesRead.ToString());
3263                     }
3264
3265                     if (pollSuccess)
3266                         requestDone = ReadComplete(bytesRead, errorStatus);
3267
3268                     bytesRead = -1;
3269                 } while (!requestDone && (userRetrievedStream || requestContinueCount == request.RequestContinueCount));
3270             }
3271             finally {
3272                 t_SyncReadNesting--;
3273             }
3274
3275             if (probeRead)
3276             {
3277                 // [....] 100-Continue wait only
3278                 request.FinishContinueWait();
3279                 if (pollSuccess)
3280                 {
3281                     if (!request.Saw100Continue && !userRetrievedStream)
3282                     {
3283                         //During polling, we got a response that wasn't a 100 continue.
3284                         request.NeedsToReadForResponse = false;
3285                     }
3286                 }
3287                 else
3288                 {
3289                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::SyncRead() Poll has timed out, calling SetRequestContinue().");
3290                     request.SetRequestContinue();
3291                 }
3292             }
3293             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::SyncRead()");
3294         }
3295
3296
3297         //
3298         //    Performs read callback processing on connection
3299         //    handles getting headers parsed and streams created
3300         //
3301         //    bytesRead == 0  when  we re-enter on buffered data without doing actual read
3302         //    bytesRead == -1 when  we got a connection close plus when errorStatus == sucess we got a g----ful close.
3303         //    Otheriwse bytesRead is read byted to add to m_BytesRead i.e. to previously buffered data
3304         //
3305         private bool ReadComplete(int bytesRead, WebExceptionStatus errorStatus)
3306         {
3307             bool requestDone = true;
3308             CoreResponseData continueResponseData = null;
3309             ConnectionReturnResult returnResult = null;
3310             HttpWebRequest currentRequest = null;
3311
3312             try
3313             {
3314                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadComplete() m_BytesRead:" + m_BytesRead.ToString() + " m_BytesScanned:" + m_BytesScanned + " (+= new bytesRead:" + bytesRead.ToString() + ")");
3315
3316                 if (bytesRead < 0)
3317                 {
3318                     // Means we might have gotten g----full or hard connection close.
3319
3320                     // If this is the first thing we read for a request then it
3321                     // could be an idle connection closed by the server (isolated error)
3322                     if (m_ReadState == ReadState.Start && m_AtLeastOneResponseReceived)
3323                     {
3324                         // Note that KeepAliveFailure will be checked against POST-type requests
3325                         // and it's fatal if the body was already started.
3326                         if (errorStatus == WebExceptionStatus.Success || errorStatus == WebExceptionStatus.ReceiveFailure)
3327                             errorStatus = WebExceptionStatus.KeepAliveFailure;
3328                     }
3329                     else if (errorStatus == WebExceptionStatus.Success)
3330                     {
3331                         // we got unexpected FIN in the middle of the response, or on a fresh connection, that's fatal
3332                         errorStatus = WebExceptionStatus.ConnectionClosed;
3333                     }
3334
3335                     // Notify request's SubmitWriteStream that a socket error happened.  This will cause future writes to
3336                     // throw an IOException.
3337                     HttpWebRequest curRequest = m_CurrentRequest;
3338                     if (curRequest != null)
3339                     {
3340                         curRequest.ErrorStatusCodeNotify(this, false, true);
3341                     }
3342
3343                     HandleErrorWithReadDone(errorStatus, ref returnResult);
3344                     goto done;
3345                 }
3346
3347                 // Otherwise, we've got data.
3348                 GlobalLog.Dump(m_ReadBuffer, m_BytesScanned, m_BytesRead - m_BytesScanned);
3349                 GlobalLog.Dump(m_ReadBuffer, m_BytesRead, bytesRead);
3350
3351
3352                 bytesRead += m_BytesRead;
3353                 if (bytesRead  > m_ReadBuffer.Length)
3354                     throw new InternalException();  //in case we posted two receives at once
3355                 m_BytesRead = bytesRead;
3356
3357                 // We have the parsing code seperated out in ParseResponseData
3358                 //
3359                 // If we don't have all the headers yet. Resubmit the receive,
3360                 // passing in the bytes read total as our index. When we get
3361                 // back here we'll end up reparsing from the beginning, which is
3362                 // OK. because this shouldn't be a performance case.
3363
3364                 //if we're back here, we need to reset the scanned bytes to 0.
3365
3366                 DataParseStatus parseStatus = ParseResponseData(ref returnResult, out requestDone, out continueResponseData);
3367
3368                 // copy off m_CurrentRequest as we might start processing a next request before exiting this method
3369                 currentRequest = m_CurrentRequest;
3370
3371                 if (parseStatus != DataParseStatus.NeedMoreData)
3372                     m_CurrentRequest = null;
3373
3374                 if (parseStatus == DataParseStatus.Invalid || parseStatus == DataParseStatus.DataTooBig)
3375                 {
3376                     // Tell the request's SubmitWriteStream that the connection will be closed.  It should ---- any
3377                     // future writes so that the appropriate exception will be received in GetResponse().
3378                     if (currentRequest != null)
3379                     {
3380                         currentRequest.ErrorStatusCodeNotify(this, false, false);
3381                     }
3382
3383                     //
3384                     // report error
3385                     //
3386                     if (parseStatus == DataParseStatus.Invalid) {
3387                         HandleErrorWithReadDone(WebExceptionStatus.ServerProtocolViolation, ref returnResult);
3388                     }
3389                     else {
3390                         HandleErrorWithReadDone(WebExceptionStatus.MessageLengthLimitExceeded, ref returnResult);
3391                     }
3392                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadComplete() parseStatus:" + parseStatus + " returnResult:" + returnResult);
3393                     goto done;
3394                 }
3395
3396                 //Done means the ConnectStream take care of this connection until ConnectStream.CallDone()
3397                 else if (parseStatus == DataParseStatus.Done)
3398                 {
3399                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadComplete() [The response stream is ready] parseStatus = DataParseStatus.Done");
3400                     goto done;
3401                 }
3402
3403                 //
3404                 // we may reach the end of our buffer only when parsing headers.
3405                 // this can happen when the header section is bigger than our initial 4k guess
3406                 // which should be a good assumption in 99.9% of the cases. what we do here is:
3407                 // 1) if there's a single BIG header (bigger than the current size) we will need to
3408                 //    grow the buffer before we move data over and read more data.
3409                 // 2) move unparsed data to the beginning of the buffer and read more data in the
3410                 //    remaining part of the data.
3411                 //
3412                 if (parseStatus == DataParseStatus.NeedMoreData)
3413                 {
3414                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadComplete() OLD buffer. m_ReadBuffer.Length:" + m_ReadBuffer.Length.ToString() + " m_BytesRead:" + m_BytesRead.ToString() + " m_BytesScanned:" + m_BytesScanned.ToString());
3415                     int unparsedDataSize = m_BytesRead - m_BytesScanned;
3416                     if (unparsedDataSize != 0)
3417                     {
3418                         if (m_BytesScanned == 0 && m_BytesRead == m_ReadBuffer.Length)
3419                         {
3420                             //
3421                             // 1) we need to grow the buffer, move the unparsed data to the beginning of the buffer before reading more data.
3422                             // since the buffer size is 4k, should we just double
3423                             //
3424                             byte[] newReadBuffer = new byte[m_ReadBuffer.Length * 2 /*+ ReadBufferSize*/];
3425                             Buffer.BlockCopy(m_ReadBuffer, 0, newReadBuffer, 0, m_BytesRead);
3426
3427                             // if m_ReadBuffer is from the pinnable cache, give it back
3428                             FreeReadBuffer();
3429                             m_ReadBuffer = newReadBuffer;
3430                         }
3431                         else
3432                         {
3433                             //
3434                             // just move data around in the same buffer.
3435                             //
3436                             Buffer.BlockCopy(m_ReadBuffer, m_BytesScanned, m_ReadBuffer, 0, unparsedDataSize);
3437                         }
3438                     }
3439                     //
3440                     // update indexes and offsets in the new buffer
3441                     //
3442                     m_BytesRead = unparsedDataSize;
3443                     m_BytesScanned = 0;
3444                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadComplete() NEW or shifted buffer. m_ReadBuffer.Length:" + m_ReadBuffer.Length.ToString() + " m_BytesRead:" + m_BytesRead.ToString() + " m_BytesScanned:" + m_BytesScanned.ToString());
3445
3446                     if (currentRequest != null)
3447                     {
3448                         //
3449                         // This case means that we still parsing the headers, so need to post another read in the async case
3450
3451                         if (currentRequest.Async)
3452                         {
3453                             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadComplete() Reposting Async Read.  Buffer:" + ValidationHelper.HashString(m_ReadBuffer) + " BytesScanned:" + m_BytesScanned.ToString());
3454
3455                             if (Thread.CurrentThread.IsThreadPoolThread)
3456                             {
3457                                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadComplete() Calling PostReceive().");
3458                                 PostReceive();
3459                             }
3460                             else
3461                             {
3462                                 // Offload to the threadpool to protect against the case where one request's thread posts IO that another request
3463                                 // depends on, but the first thread dies in the mean time.
3464                                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::ReadComplete() ThreadPool.UnsafeQueueUserWorkItem(m_PostReceiveDelegate, this)");
3465                                 ThreadPool.UnsafeQueueUserWorkItem(m_PostReceiveDelegate, this);
3466                             }
3467                         }
3468                     }
3469                 }
3470             }
3471             //
3472             // Any exception is processed by HandleError() and ----ed to avoid throwing on a thread pool
3473             // In the [....] case the HandleError() will abort the request so the caller will pick up the result.
3474             //
3475             catch (Exception exception) {
3476                 if (NclUtilities.IsFatal(exception)) throw;
3477
3478                 requestDone = true;
3479
3480                 if (m_InnerException == null)
3481                     m_InnerException = exception;
3482
3483                 // Notify request's SubmitWriteStream that a socket error happened.  This will cause future writes to
3484                 // throw an IOException.
3485                 HttpWebRequest curRequest = m_CurrentRequest;
3486                 if (curRequest != null)
3487                 {
3488                     curRequest.ErrorStatusCodeNotify(this, false, true);
3489                 }
3490
3491                 HandleErrorWithReadDone(WebExceptionStatus.ReceiveFailure, ref returnResult);
3492             }
3493
3494 done:
3495             try {
3496                 // It is only safe to continue if there was a 100 continue OR buffering is supported.
3497                 if (currentRequest != null && currentRequest.HttpWriteMode != HttpWriteMode.None &&
3498                     (continueResponseData != null
3499                         // not a 100 continue, but we have buffering so we don't care what it was.
3500                         || (returnResult != null && returnResult.IsNotEmpty && currentRequest.AllowWriteStreamBuffering)
3501                     )
3502                    )
3503                 {
3504                     // if returnResult is not empty it must also contain some result for the currently active request
3505                     // Since it could be a POST request waiting on the body submit, signal the body here
3506                     if (currentRequest.FinishContinueWait())
3507                     {
3508                         currentRequest.SetRequestContinue(continueResponseData);
3509                     }
3510                 }
3511             }
3512             finally {
3513                 ConnectionReturnResult.SetResponses(returnResult);
3514             }
3515
3516             return requestDone;
3517         }
3518
3519
3520         //     This method is called by ConnectStream, only when resubmitting
3521         //     We have sent the headers already in HttpWebRequest.EndSubmitRequest()
3522         //     which calls ConnectStream.WriteHeaders() which calls to HttpWebRequest.EndWriteHeaders()
3523         //     which calls ConnectStream.ResubmitWrite() which calls in here
3524         internal void Write(ScatterGatherBuffers writeBuffer) {
3525             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::Write(ScatterGatherBuffers) networkstream#" + ValidationHelper.HashString(NetworkStream));
3526             //
3527             // parameter validation
3528             //
3529             GlobalLog.Assert(writeBuffer != null, "Connection#{0}::Write(ScatterGatherBuffers)|writeBuffer == null", ValidationHelper.HashString(this));
3530             //
3531             // set up array for MultipleWrite call
3532             // note that GetBuffers returns null if we never wrote to it.
3533             //
3534             BufferOffsetSize[] buffers = writeBuffer.GetBuffers();
3535             if (buffers!=null) {
3536                 //
3537                 // This will block writing the buffers out.
3538                 //
3539                 MultipleWrite(buffers);
3540             }
3541             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::Write(ScatterGatherBuffers) this:" + ValidationHelper.HashString(this) + " writeBuffer.Size:" + writeBuffer.Length.ToString());
3542         }
3543
3544
3545         /*++
3546
3547             PostReceiveWrapper - Post a receive from a worker thread.
3548
3549             This is our delegate, used for posting receives from a worker thread.
3550             We do this when we can't be sure that we're already on a worker thread,
3551             and we don't want to post from a client thread because if it goes away
3552             I/O gets cancelled.
3553
3554             Input:
3555
3556             state           - a null object
3557
3558             Returns:
3559
3560         --*/
3561         private static void PostReceiveWrapper(object state) {
3562             Connection thisConnection = state as Connection;
3563             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(thisConnection) + "::PostReceiveWrapper", "Cnt#" + ValidationHelper.HashString(thisConnection));
3564             GlobalLog.Assert(thisConnection != null, "Connection#{0}::PostReceiveWrapper()|thisConnection == null", ValidationHelper.HashString(thisConnection));
3565
3566             thisConnection.PostReceive();
3567
3568             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(thisConnection) + "::PostReceiveWrapper");
3569         }
3570
3571         private void PostReceive()
3572         {
3573             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::PostReceive", "");
3574
3575             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::PostReceive() m_ReadBuffer:" + ValidationHelper.HashString(m_ReadBuffer) + " Length:" + m_ReadBuffer.Length.ToString());
3576
3577             try
3578             {
3579                 GlobalLog.Assert(m_BytesScanned == 0, "PostReceive()|A receive should not be posted when m_BytesScanned != 0 (the data should be moved to offset 0).");
3580
3581                 if (m_LastAsyncResult != null && !m_LastAsyncResult.IsCompleted)
3582                     throw new InternalException();  //This may cause duplicate requests if we let it through in retail
3583
3584
3585                 m_LastAsyncResult = UnsafeBeginRead(m_ReadBuffer, m_BytesRead, m_ReadBuffer.Length - m_BytesRead, m_ReadCallback, this);
3586                 if (m_LastAsyncResult.CompletedSynchronously)
3587                 {
3588                     //
3589                     ReadCallback(m_LastAsyncResult);
3590                 }
3591             }
3592             catch (Exception exception) {
3593                 // Notify request's SubmitWriteStream that a socket error happened.  This will cause future writes to
3594                 // throw an IOException.
3595                 HttpWebRequest curRequest = m_CurrentRequest;
3596                 if (curRequest != null)
3597                 {
3598                     curRequest.ErrorStatusCodeNotify(this, false, true);
3599                 }
3600
3601                 //ASYNCISSUE
3602                 ConnectionReturnResult returnResult = null;
3603                 HandleErrorWithReadDone(WebExceptionStatus.ReceiveFailure, ref returnResult);
3604                 ConnectionReturnResult.SetResponses(returnResult);
3605                 GlobalLog.LeaveException("Connection#" + ValidationHelper.HashString(this) + "::PostReceive", exception);
3606             }
3607
3608             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::PostReceive");
3609         }
3610
3611
3612
3613         private static void TunnelThroughProxyWrapper(IAsyncResult result){
3614             if(result.CompletedSynchronously){
3615                 return;
3616             }
3617
3618             bool success = false;
3619             WebExceptionStatus ws = WebExceptionStatus.ConnectFailure;
3620             HttpWebRequest req = (HttpWebRequest)((LazyAsyncResult)result).AsyncObject;
3621             Connection conn = (Connection)((TunnelStateObject)result.AsyncState).Connection;
3622             HttpWebRequest originalReq = (HttpWebRequest)((TunnelStateObject)result.AsyncState).OriginalRequest;
3623
3624             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(conn) + "::TunnelThroughProxyCallback");
3625
3626             try{
3627                 req.EndGetResponse(result);
3628                 HttpWebResponse connectResponse = (HttpWebResponse)req.GetResponse();
3629                 ConnectStream connectStream = (ConnectStream)connectResponse.GetResponseStream();
3630
3631                 // this stream will be used as the real stream for TlsStream
3632                 conn.NetworkStream = new NetworkStream(connectStream.Connection.NetworkStream, true);
3633                 // This will orphan the original connect stream now owned by tunnelStream
3634                 connectStream.Connection.NetworkStream.ConvertToNotSocketOwner();
3635                 success = true;
3636             }
3637
3638             catch (Exception exception) {
3639                 if (conn.m_InnerException == null)
3640                     conn.m_InnerException = exception;
3641
3642                 if (exception is WebException) {
3643                     ws = ((WebException)exception).Status;
3644                 }
3645
3646                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(conn) + "::TunnelThroughProxyCallback() exception occurred: " + exception);
3647             }
3648             if(!success)
3649             {
3650                 ConnectionReturnResult returnResult = null;
3651                 conn.HandleError(false, false, ws, ref returnResult);
3652                 ConnectionReturnResult.SetResponses(returnResult);
3653                 return;
3654             }
3655
3656             conn.CompleteConnection(true, originalReq);
3657             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(conn) + "::TunnelThroughProxyCallback");
3658         }
3659
3660
3661
3662
3663         //
3664         private bool TunnelThroughProxy(Uri proxy, HttpWebRequest originalRequest, bool async) {
3665             GlobalLog.Enter("Connection#" + ValidationHelper.HashString(this) + "::TunnelThroughProxy", "proxy="+proxy+", async="+async+", originalRequest #"+ValidationHelper.HashString(originalRequest));
3666
3667             bool result = false;
3668             HttpWebRequest connectRequest = null;
3669             HttpWebResponse connectResponse = null;
3670
3671             try {
3672                 (new WebPermission(NetworkAccess.Connect, proxy)).Assert();
3673                 try {
3674                     connectRequest = new HttpWebRequest(
3675                         proxy,
3676                         originalRequest.Address,
3677                        // new Uri("https://" + originalRequest.Address.GetParts(UriComponents.HostAndPort, UriFormat.UriEscaped)),
3678                         originalRequest
3679                         );
3680                 }
3681                 finally {
3682                     WebPermission.RevertAssert();
3683                 }
3684
3685                 connectRequest.Credentials = originalRequest.InternalProxy == null ? null : originalRequest.InternalProxy.Credentials;
3686                 connectRequest.InternalProxy = null;
3687                 connectRequest.PreAuthenticate = true;
3688
3689                 if(async){
3690                     TunnelStateObject o = new TunnelStateObject(originalRequest, this);
3691                     IAsyncResult asyncResult = connectRequest.BeginGetResponse(m_TunnelCallback, o);
3692                     if(!asyncResult.CompletedSynchronously){
3693                         GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::TunnelThroughProxy completed asynchronously", true);
3694                         return true;
3695                     }
3696                     connectResponse = (HttpWebResponse)connectRequest.EndGetResponse(asyncResult);
3697                 }
3698                 else{
3699                     connectResponse = (HttpWebResponse)connectRequest.GetResponse();
3700                 }
3701
3702                 ConnectStream connectStream = (ConnectStream)connectResponse.GetResponseStream();
3703
3704                 // this stream will be used as the real stream for TlsStream
3705                 NetworkStream = new NetworkStream(connectStream.Connection.NetworkStream, true);
3706                 // This will orphan the original connect stream now owned by tunnelStream
3707                 connectStream.Connection.NetworkStream.ConvertToNotSocketOwner();
3708                 result = true;
3709             }
3710             catch (Exception exception) {
3711                 if (m_InnerException == null)
3712                     m_InnerException = exception;
3713                 GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::TunnelThroughProxy() exception occurred: " + exception);
3714             }
3715
3716             GlobalLog.Leave("Connection#" + ValidationHelper.HashString(this) + "::TunnelThroughProxy", result);
3717
3718             return result;
3719         }
3720
3721         //
3722         // CheckNonIdle - called after update of the WriteList/WaitList,
3723         //   upon the departure of our Idle state our, BusyCount will
3724         //   go to non-0, then we need to mark this transition
3725         //
3726
3727         private void CheckNonIdle() {
3728             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CheckNonIdle()");
3729             if (m_Idle && this.BusyCount != 0) {
3730                 m_Idle = false;
3731                 ServicePoint.IncrementConnection();
3732                 ConnectionGroup.IncrementConnection();
3733             }
3734         }
3735
3736         //
3737         // CheckIdle - called after update of the WriteList/WaitList,
3738         //    specifically called after we remove entries
3739         //
3740
3741         private void CheckIdle() {
3742             // The timer thread is allowed to call this.  (It doesn't call user code and doesn't block.)
3743             GlobalLog.ThreadContract(ThreadKinds.Unknown, ThreadKinds.SafeSources | ThreadKinds.Finalization | ThreadKinds.Timer, "Connection#" + ValidationHelper.HashString(this) + "::CheckIdle");
3744
3745             GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CheckIdle() m_Idle = " + m_Idle + ", BusyCount = " + BusyCount);
3746             if (!m_Idle && this.BusyCount == 0)  {
3747                 m_Idle = true;
3748                 ServicePoint.DecrementConnection();
3749                 if (ConnectionGroup != null) {
3750                     GlobalLog.Print("Connection#" + ValidationHelper.HashString(this) + "::CheckIdle() - calling ConnectionGoneIdle()");
3751                     ConnectionGroup.DecrementConnection();
3752                     ConnectionGroup.ConnectionGoneIdle();
3753                 }
3754                 // Remember the moment when this connection went idle.
3755                 m_IdleSinceUtc = DateTime.UtcNow;
3756             }
3757         }
3758
3759         //
3760         // DebugDumpArrayListEntries - debug goop
3761         //
3762         [Conditional("TRAVE")]
3763         private void DebugDumpWriteListEntries() {
3764             for (int i = 0; i < m_WriteList.Count; i++)
3765             {
3766                 DebugDumpListEntry(i, m_WriteList[i] as HttpWebRequest, "WriteList");
3767             }
3768         }
3769
3770         [Conditional("TRAVE")]
3771         private void DebugDumpWaitListEntries() {
3772             for (int i = 0; i < m_WaitList.Count; i++)
3773             {
3774                 DebugDumpListEntry(i, m_WaitList[i].Request, "WaitList");
3775             }
3776         }
3777
3778         [Conditional("TRAVE")]
3779         private void DebugDumpListEntry(int currentPos, HttpWebRequest req, string listType) {
3780             GlobalLog.Print("WaitList[" + currentPos.ToString() + "] Req: #" +
3781                 ValidationHelper.HashString(req));
3782         }
3783
3784         //
3785         // Validation & debugging
3786         //
3787         [System.Diagnostics.Conditional("DEBUG")]
3788         internal void DebugMembers(int requestHash) {
3789 #if TRAVE
3790             bool dump = requestHash==0;
3791             GlobalLog.Print("Cnt#" + this.GetHashCode());
3792             if (!dump) {
3793                 foreach(HttpWebRequest request in  m_WriteList) {
3794                     if (request.GetHashCode() == requestHash) {
3795                         GlobalLog.Print("Found-WriteList");
3796                         Dump();
3797                         return;
3798                     }
3799                 }
3800                 foreach(WaitListItem item in m_WaitList) {
3801                     if (item.Request.GetHashCode() == requestHash) {
3802                         GlobalLog.Print("Found-WaitList");
3803                         Dump();
3804                         return;
3805                     }
3806                 }
3807             }
3808             else {
3809                 Dump();
3810                 DebugDumpWriteListEntries();
3811                 DebugDumpWaitListEntries();
3812             }
3813 #endif
3814         }
3815
3816 #if TRAVE
3817         [System.Diagnostics.Conditional("TRAVE")]
3818         internal void Dump() {
3819             GlobalLog.Print("CanPipeline:" + m_CanPipeline);
3820             GlobalLog.Print("Pipelining:" + m_Pipelining);
3821             GlobalLog.Print("KeepAlive:" + m_KeepAlive);
3822             GlobalLog.Print("m_Error:" + m_Error);
3823             GlobalLog.Print("m_ReadBuffer:" + m_ReadBuffer);
3824             GlobalLog.Print("m_BytesRead:" + m_BytesRead);
3825             GlobalLog.Print("m_BytesScanned:" + m_BytesScanned);
3826             GlobalLog.Print("m_ResponseData:" + m_ResponseData);
3827             GlobalLog.Print("m_ReadState:" + m_ReadState);
3828             GlobalLog.Print("m_StatusState:" + m_StatusState);
3829             GlobalLog.Print("ConnectionGroup:" + ConnectionGroup);
3830             GlobalLog.Print("Idle:" + m_Idle);
3831             GlobalLog.Print("ServicePoint:" + ServicePoint);
3832             GlobalLog.Print("m_Version:" + ServicePoint.ProtocolVersion);
3833             GlobalLog.Print("NetworkStream:" + NetworkStream);
3834 #if !FEATURE_PAL
3835             if ( NetworkStream is TlsStream) {
3836                 TlsStream tlsStream = NetworkStream as TlsStream;
3837                 tlsStream.DebugMembers();
3838             }
3839             else
3840 #endif // !FEATURE_PAL
3841             if (NetworkStream != null) {
3842                 NetworkStream.DebugMembers();
3843             }
3844             GlobalLog.Print("m_AbortDelegate:" + m_AbortDelegate);
3845             GlobalLog.Print("ReadDone:" + m_ReadDone);
3846             GlobalLog.Print("WriteDone:" + m_WriteDone);
3847             GlobalLog.Print("Free:" + m_Free);
3848         }
3849 #endif
3850     }
3851 #endif
3852 }