mcs/class/referencesource/System.Data/System/Data/ProviderBase/DbConnectionPool.cs

   1 //------------------------------------------------------------------------------
   2 // <copyright file="DbConnectionPool.cs" company="Microsoft">
   3 //      Copyright (c) Microsoft Corporation.  All rights reserved.
   4 // </copyright>
   5 // <owner current="true" primary="true">[....]</owner>
   6 // <owner current="true" primary="false">[....]</owner>
   7 //------------------------------------------------------------------------------
   8
   9 namespace System.Data.ProviderBase {
  10
  11     using System;
  12     using System.Collections;
  13     using System.Collections.Generic;
  14     using System.Data.Common;
  15     using System.Data.SqlClient;
  16     using System.Diagnostics;
  17     using System.Globalization;
  18     using System.Runtime.CompilerServices;
  19     using System.Runtime.ConstrainedExecution;
  20     using System.Runtime.InteropServices;
  21     using System.Security;
  22     using System.Security.Permissions;
  23     using System.Security.Principal;
  24     using System.Threading;
  25     using System.Threading.Tasks;
  26     using SysTx = System.Transactions;
  27     using System.Runtime.Versioning;
  28     using System.Diagnostics.CodeAnalysis;
  29     using System.Collections.Concurrent;
  30
  31     sealed internal class DbConnectionPool {
  32         private enum State {
  33             Initializing,
  34             Running,
  35             ShuttingDown,
  36         }
  37
  38         internal const Bid.ApiGroup PoolerTracePoints = Bid.ApiGroup.Pooling;
  39
  40         // This class is a way to stash our cloned Tx key for later disposal when it's no longer needed.
  41         // We can't get at the key in the dictionary without enumerating entries, so we stash an extra
  42         // copy as part of the value.
  43         sealed private class TransactedConnectionList : List<DbConnectionInternal> {
  44             private SysTx.Transaction _transaction;
  45             internal TransactedConnectionList(int initialAllocation, SysTx.Transaction tx) : base(initialAllocation) {
  46                 _transaction = tx;
  47             }
  48
  49             internal void Dispose() {
  50                 if (null != _transaction) {
  51                     _transaction.Dispose();
  52                 }
  53             }
  54         }
  55
  56         sealed class PendingGetConnection {
  57             public PendingGetConnection(long dueTime, DbConnection owner, TaskCompletionSource<DbConnectionInternal> completion, DbConnectionOptions userOptions) {
  58                 DueTime = dueTime;
  59                 Owner = owner;
  60                 Completion = completion;
  61             }
  62             public long DueTime { get; private set; }
  63             public DbConnection Owner { get; private set; }
  64             public TaskCompletionSource<DbConnectionInternal> Completion { get; private set; }
  65             public DbConnectionOptions UserOptions { get; private set; }
  66         }
  67
  68         sealed private class TransactedConnectionPool
  69         {
  70
  71             Dictionary<SysTx.Transaction, TransactedConnectionList> _transactedCxns;
  72
  73             DbConnectionPool _pool;
  74
  75             private static int _objectTypeCount; // Bid counter
  76             internal readonly int _objectID = System.Threading.Interlocked.Increment(ref _objectTypeCount);
  77
  78             internal TransactedConnectionPool(DbConnectionPool pool)
  79             {
  80                 Debug.Assert(null != pool, "null pool?");
  81
  82                 _pool = pool;
  83                 _transactedCxns = new Dictionary<SysTx.Transaction, TransactedConnectionList> ();
  84
  85                 Bid.PoolerTrace("<prov.DbConnectionPool.TransactedConnectionPool.TransactedConnectionPool|RES|CPOOL> %d#, Constructed for connection pool %d#\n", ObjectID, _pool.ObjectID);
  86             }
  87
  88             internal int ObjectID {
  89                 get {
  90                     return _objectID;
  91                 }
  92             }
  93
  94             internal DbConnectionPool Pool {
  95                 get {
  96                     return _pool;
  97                 }
  98             }
  99
 100             internal DbConnectionInternal GetTransactedObject(SysTx.Transaction transaction)
 101             {
 102                 Debug.Assert(null != transaction, "null transaction?");
 103
 104                 DbConnectionInternal transactedObject = null;
 105
 106                 TransactedConnectionList connections;
 107                 bool txnFound = false;
 108
 109                 lock (_transactedCxns)
 110                 {
 111                     txnFound = _transactedCxns.TryGetValue ( transaction, out connections );
 112                 }
 113
 114                 // NOTE: GetTransactedObject is only used when AutoEnlist = True and the ambient transaction
 115                 //   (Sys.Txns.Txn.Current) is still valid/non-null. This, in turn, means that we don't need
 116                 //   to worry about a pending asynchronous TransactionCompletedEvent to trigger processing in
 117                 //   TransactionEnded below and potentially wipe out the connections list underneath us. It
 118                 //   is similarly alright if a pending addition to the connections list in PutTransactedObject
 119                 //   below is not completed prior to the lock on the connections object here...getting a new
 120                 //   connection is probably better than unnecessarily locking
 121                 if (txnFound)
 122                 {
 123                     Debug.Assert ( connections != null );
 124
 125                     // synchronize multi-threaded access with PutTransactedObject (TransactionEnded should
 126                     //   not be a concern, see comments above)
 127                     lock ( connections )
 128                     {
 129                         int i = connections.Count - 1;
 130                         if (0 <= i)
 131                         {
 132                             transactedObject = connections[i];
 133                             connections.RemoveAt(i);
 134                         }
 135                     }
 136                 }
 137
 138                 if (null != transactedObject) {
 139                     Bid.PoolerTrace("<prov.DbConnectionPool.TransactedConnectionPool.GetTransactedObject|RES|CPOOL> %d#, Transaction %d#, Connection %d#, Popped.\n", ObjectID, transaction.GetHashCode(), transactedObject.ObjectID);
 140                 }
 141                 return transactedObject;
 142             }
 143
 144             internal void PutTransactedObject(SysTx.Transaction transaction, DbConnectionInternal transactedObject) {
 145                 Debug.Assert(null != transaction, "null transaction?");
 146                 Debug.Assert(null != transactedObject, "null transactedObject?");
 147
 148                 TransactedConnectionList connections;
 149                 bool txnFound = false;
 150
 151                 // NOTE: because TransactionEnded is an asynchronous notification, there's no guarantee
 152                 //   around the order in which PutTransactionObject and TransactionEnded are called.
 153
 154                 lock ( _transactedCxns )
 155                 {
 156                     // Check if a transacted pool has been created for this transaction
 157                     if ( txnFound = _transactedCxns.TryGetValue ( transaction, out connections ) )
 158                     {
 159                         Debug.Assert ( connections != null );
 160
 161                         // synchronize multi-threaded access with GetTransactedObject
 162                         lock ( connections )
 163                         {
 164                             Debug.Assert(0 > connections.IndexOf(transactedObject), "adding to pool a second time?");
 165                             Bid.PoolerTrace("<prov.DbConnectionPool.TransactedConnectionPool.PutTransactedObject|RES|CPOOL> %d#, Transaction %d#, Connection %d#, Pushing.\n", ObjectID, transaction.GetHashCode(), transactedObject.ObjectID);
 166
 167                             connections.Add(transactedObject);
 168                         }
 169                     }
 170                 }
 171
 172                 //
 173
 174                 if ( !txnFound )
 175                 {
 176                     // create the transacted pool, making sure to clone the associated transaction
 177                     //   for use as a key in our internal dictionary of transactions and connections
 178                     SysTx.Transaction transactionClone = null;
 179                     TransactedConnectionList newConnections = null;
 180
 181                     try
 182                     {
 183                         transactionClone = transaction.Clone();
 184                         newConnections = new TransactedConnectionList(2, transactionClone); // start with only two connections in the list; most times we won't need that many.
 185
 186                         lock ( _transactedCxns )
 187                         {
 188                             // NOTE: in the interim between the locks on the transacted pool (this) during
 189                             //   execution of this method, another thread (threadB) may have attempted to
 190                             //   add a different connection to the transacted pool under the same
 191                             //   transaction. As a result, threadB may have completed creating the
 192                             //   transacted pool while threadA was processing the above instructions.
 193                             if (txnFound = _transactedCxns.TryGetValue(transaction, out connections))
 194                             {
 195                                 Debug.Assert ( connections != null );
 196
 197                                 // synchronize multi-threaded access with GetTransactedObject
 198                                 lock ( connections )
 199                                 {
 200                                     Debug.Assert(0 > connections.IndexOf(transactedObject), "adding to pool a second time?");
 201                                     Bid.PoolerTrace("<prov.DbConnectionPool.TransactedConnectionPool.PutTransactedObject|RES|CPOOL> %d#, Transaction %d#, Connection %d#, Pushing.\n", ObjectID, transaction.GetHashCode(), transactedObject.ObjectID);
 202
 203                                     connections.Add(transactedObject);
 204                                 }
 205                             }
 206                             else
 207                             {
 208                                 Bid.PoolerTrace("<prov.DbConnectionPool.TransactedConnectionPool.PutTransactedObject|RES|CPOOL> %d#, Transaction %d#, Connection %d#, Adding List to transacted pool.\n", ObjectID, transaction.GetHashCode(), transactedObject.ObjectID);
 209
 210                                 // add the connection/transacted object to the list
 211                                 newConnections.Add ( transactedObject );
 212
 213                                 _transactedCxns.Add(transactionClone, newConnections);
 214                                 transactionClone = null; // we've used it -- don't throw it or the TransactedConnectionList that references it away.
 215                             }
 216                         }
 217                     }
 218                     finally
 219                     {
 220                         if (null != transactionClone)
 221                         {
 222                             if ( newConnections != null )
 223                             {
 224                                 // another thread created the transaction pool and thus the new
 225                                 //   TransactedConnectionList was not used, so dispose of it and
 226                                 //   the transaction clone that it incorporates.
 227                                 newConnections.Dispose();
 228                             }
 229                             else
 230                             {
 231                                 // memory allocation for newConnections failed...clean up unused transactionClone
 232                                 transactionClone.Dispose();
 233                             }
 234                         }
 235                     }
 236                     Bid.PoolerTrace("<prov.DbConnectionPool.TransactedConnectionPool.PutTransactedObject|RES|CPOOL> %d#, Transaction %d#, Connection %d#, Added.\n", ObjectID, transaction.GetHashCode(), transactedObject.ObjectID );
 237                 }
 238
 239 #if !MOBILE
 240                 Pool.PerformanceCounters.NumberOfFreeConnections.Increment();
 241 #endif
 242             }
 243
 244             internal void TransactionEnded(SysTx.Transaction transaction, DbConnectionInternal transactedObject)
 245             {
 246                 Bid.PoolerTrace("<prov.DbConnectionPool.TransactedConnectionPool.TransactionEnded|RES|CPOOL> %d#, Transaction %d#, Connection %d#, Transaction Completed\n", ObjectID, transaction.GetHashCode(), transactedObject.ObjectID);
 247
 248                 TransactedConnectionList connections;
 249                 int entry = -1;
 250
 251                 // NOTE: because TransactionEnded is an asynchronous notification, there's no guarantee
 252                 //   around the order in which PutTransactionObject and TransactionEnded are called. As
 253                 //   such, it is possible that the transaction does not yet have a pool created.
 254
 255                 //
 256
 257
 258
 259                 lock ( _transactedCxns )
 260                 {
 261                     if (_transactedCxns.TryGetValue(transaction, out connections))
 262                     {
 263                         Debug.Assert ( connections != null );
 264
 265                         bool shouldDisposeConnections = false;
 266
 267                         // Lock connections to avoid conflict with GetTransactionObject
 268                         lock (connections)
 269                         {
 270                             entry = connections.IndexOf(transactedObject);
 271
 272                             if ( entry >= 0 )
 273                             {
 274                                 connections.RemoveAt(entry);
 275                             }
 276
 277                             // Once we've completed all the ended notifications, we can
 278                             // safely remove the list from the transacted pool.
 279                             if (0 >= connections.Count)
 280                             {
 281                                 Bid.PoolerTrace("<prov.DbConnectionPool.TransactedConnectionPool.TransactionEnded|RES|CPOOL> %d#, Transaction %d#, Removing List from transacted pool.\n", ObjectID, transaction.GetHashCode());
 282                                 _transactedCxns.Remove(transaction);
 283
 284                                 // we really need to dispose our connection list; it may have
 285                                 // native resources via the tx and GC may not happen soon enough.
 286                                 shouldDisposeConnections = true;
 287                             }
 288                         }
 289
 290                         if (shouldDisposeConnections) {
 291                             connections.Dispose();
 292                         }
 293                     }
 294                     else
 295                     {
 296                         //Debug.Assert ( false, "TransactionCompletedEvent fired before PutTransactedObject put the connection in the transacted pool." );
 297                         Bid.PoolerTrace("<prov.DbConnectionPool.TransactedConnectionPool.TransactionEnded|RES|CPOOL> %d#, Transaction %d#, Connection %d#, Transacted pool not yet created prior to transaction completing. Connection may be leaked.\n", ObjectID, transaction.GetHashCode(), transactedObject.ObjectID );
 298                     }
 299                 }
 300
 301                 // If (and only if) we found the connection in the list of
 302                 // connections, we'll put it back...
 303                 if (0 <= entry)
 304                 {
 305 #if !MOBILE
 306                     Pool.PerformanceCounters.NumberOfFreeConnections.Decrement();
 307 #endif
 308                     Pool.PutObjectFromTransactedPool(transactedObject);
 309                 }
 310             }
 311
 312         }
 313
 314         private sealed class PoolWaitHandles : DbBuffer {
 315
 316             private readonly Semaphore _poolSemaphore;
 317             private readonly ManualResetEvent _errorEvent;
 318
 319             // Using a Mutex requires ThreadAffinity because SQL CLR can swap
 320             // the underlying Win32 thread associated with a managed thread in preemptive mode.
 321             // Using an AutoResetEvent does not have that complication.
 322             private readonly Semaphore _creationSemaphore;
 323
 324             private readonly SafeHandle _poolHandle;
 325             private readonly SafeHandle _errorHandle;
 326             private readonly SafeHandle _creationHandle;
 327
 328             private readonly int _releaseFlags;
 329
 330             [ResourceExposure(ResourceScope.None)] // SxS: this method does not create named objects
 331             [ResourceConsumption(ResourceScope.Machine, ResourceScope.Machine)]
 332             internal PoolWaitHandles() : base(3*IntPtr.Size) {
 333                 bool mustRelease1 = false, mustRelease2 = false, mustRelease3 = false;
 334
 335                 _poolSemaphore = new Semaphore(0, MAX_Q_SIZE);
 336                 _errorEvent = new ManualResetEvent(false);
 337                 _creationSemaphore = new Semaphore(1, 1);
 338
 339                 RuntimeHelpers.PrepareConstrainedRegions();
 340                 try {
 341                     // because SafeWaitHandle doesn't have reliability contract
 342                     _poolHandle     = _poolSemaphore.SafeWaitHandle;
 343                     _errorHandle    = _errorEvent.SafeWaitHandle;
 344                     _creationHandle = _creationSemaphore.SafeWaitHandle;
 345
 346                     _poolHandle.DangerousAddRef(ref mustRelease1);
 347                     _errorHandle.DangerousAddRef(ref mustRelease2);
 348                     _creationHandle.DangerousAddRef(ref mustRelease3);
 349
 350                     Debug.Assert(0 == SEMAPHORE_HANDLE, "SEMAPHORE_HANDLE");
 351                     Debug.Assert(1 == ERROR_HANDLE, "ERROR_HANDLE");
 352                     Debug.Assert(2 == CREATION_HANDLE, "CREATION_HANDLE");
 353
 354                     WriteIntPtr(SEMAPHORE_HANDLE*IntPtr.Size, _poolHandle.DangerousGetHandle());
 355                     WriteIntPtr(ERROR_HANDLE*IntPtr.Size,     _errorHandle.DangerousGetHandle());
 356                     WriteIntPtr(CREATION_HANDLE*IntPtr.Size,  _creationHandle.DangerousGetHandle());
 357                 }
 358                 finally {
 359                     if (mustRelease1) {
 360                         _releaseFlags |= 1;
 361                     }
 362                     if (mustRelease2) {
 363                         _releaseFlags |= 2;
 364                     }
 365                     if (mustRelease3) {
 366                         _releaseFlags |= 4;
 367                     }
 368                 }
 369             }
 370
 371             internal SafeHandle CreationHandle {
 372                 [ReliabilityContract(Consistency.WillNotCorruptState, Cer.Success)]
 373                 get { return _creationHandle; }
 374             }
 375
 376             internal Semaphore CreationSemaphore {
 377                 get { return _creationSemaphore; }
 378             }
 379
 380             internal ManualResetEvent ErrorEvent {
 381                 get { return _errorEvent; }
 382             }
 383
 384             internal Semaphore PoolSemaphore {
 385                 get { return _poolSemaphore; }
 386             }
 387
 388             protected override bool ReleaseHandle() {
 389                 // NOTE: The SafeHandle class guarantees this will be called exactly once.
 390                 // we know we can touch these other managed objects because of our original DangerousAddRef
 391                 if (0 != (1 & _releaseFlags)) {
 392                     _poolHandle.DangerousRelease();
 393                 }
 394                 if (0 != (2 & _releaseFlags)) {
 395                     _errorHandle.DangerousRelease();
 396                 }
 397                 if (0 != (4 & _releaseFlags)) {
 398                     _creationHandle.DangerousRelease();
 399                 }
 400                 return base.ReleaseHandle();
 401             }
 402         }
 403
 404         private const int MAX_Q_SIZE    = (int)0x00100000;
 405
 406         // The order of these is important; we want the WaitAny call to be signaled
 407         // for a free object before a creation signal.  Only the index first signaled
 408         // object is returned from the WaitAny call.
 409         private const int SEMAPHORE_HANDLE = (int)0x0;
 410         private const int ERROR_HANDLE     = (int)0x1;
 411         private const int CREATION_HANDLE  = (int)0x2;
 412         private const int BOGUS_HANDLE     = (int)0x3;
 413
 414         private const int WAIT_OBJECT_0 = 0;
 415         private const int WAIT_TIMEOUT   = (int)0x102;
 416         private const int WAIT_ABANDONED = (int)0x80;
 417         private const int WAIT_FAILED    = -1;
 418
 419         private const int ERROR_WAIT_DEFAULT = 5 * 1000; // 5 seconds
 420
 421         // we do want a testable, repeatable set of generated random numbers
 422         private static readonly Random _random = new Random(5101977); // Value obtained from Dave Driver
 423
 424         private readonly int              _cleanupWait;
 425         private readonly DbConnectionPoolIdentity _identity;
 426
 427         private readonly DbConnectionFactory          _connectionFactory;
 428         private readonly DbConnectionPoolGroup        _connectionPoolGroup;
 429         private readonly DbConnectionPoolGroupOptions _connectionPoolGroupOptions;
 430         private          DbConnectionPoolProviderInfo _connectionPoolProviderInfo;
 431
 432         /// <summary>
 433         /// The private member which carries the set of authenticationcontexts for this pool (based on the user's identity).
 434         /// </summary>
 435         private readonly ConcurrentDictionary<DbConnectionPoolAuthenticationContextKey, DbConnectionPoolAuthenticationContext> _pooledDbAuthenticationContexts;
 436
 437         private State                     _state;
 438
 439         private readonly ConcurrentStack<DbConnectionInternal> _stackOld = new ConcurrentStack<DbConnectionInternal>();
 440         private readonly ConcurrentStack<DbConnectionInternal> _stackNew = new ConcurrentStack<DbConnectionInternal>();
 441
 442         private readonly ConcurrentQueue<PendingGetConnection> _pendingOpens = new ConcurrentQueue<PendingGetConnection>();
 443         private int _pendingOpensWaiting = 0;
 444
 445         private readonly WaitCallback     _poolCreateRequest;
 446
 447         private int                       _waitCount;
 448         private readonly PoolWaitHandles  _waitHandles;
 449
 450         private Exception                 _resError;
 451         private volatile bool             _errorOccurred;
 452
 453         private int                       _errorWait;
 454         private Timer                     _errorTimer;
 455
 456         private Timer                     _cleanupTimer;
 457
 458         private readonly TransactedConnectionPool _transactedConnectionPool;
 459
 460         private readonly List<DbConnectionInternal> _objectList;
 461         private int                       _totalObjects;
 462
 463         private static int _objectTypeCount; // Bid counter
 464         internal readonly int _objectID = System.Threading.Interlocked.Increment(ref _objectTypeCount);
 465
 466         // only created by DbConnectionPoolGroup.GetConnectionPool
 467         internal DbConnectionPool(
 468                             DbConnectionFactory connectionFactory,
 469                             DbConnectionPoolGroup connectionPoolGroup,
 470                             DbConnectionPoolIdentity identity,
 471                             DbConnectionPoolProviderInfo connectionPoolProviderInfo ) {
 472             Debug.Assert(ADP.IsWindowsNT, "Attempting to construct a connection pool on Win9x?");
 473             Debug.Assert(null != connectionPoolGroup, "null connectionPoolGroup");
 474
 475             if ((null != identity) && identity.IsRestricted) {
 476                 throw ADP.InternalError(ADP.InternalErrorCode.AttemptingToPoolOnRestrictedToken);
 477             }
 478
 479             _state= State.Initializing;
 480
 481             lock(_random) { // Random.Next is not thread-safe
 482                 _cleanupWait = _random.Next(12, 24)*10*1000; // 2-4 minutes in 10 sec intervals, WebData 103603
 483             }
 484
 485             _connectionFactory = connectionFactory;
 486             _connectionPoolGroup = connectionPoolGroup;
 487             _connectionPoolGroupOptions = connectionPoolGroup.PoolGroupOptions;
 488             _connectionPoolProviderInfo = connectionPoolProviderInfo;
 489             _identity = identity;
 490
 491             _waitHandles = new PoolWaitHandles();
 492
 493             _errorWait      = ERROR_WAIT_DEFAULT;
 494             _errorTimer     = null;  // No error yet.
 495
 496             _objectList     = new List<DbConnectionInternal>(MaxPoolSize);
 497
 498             _pooledDbAuthenticationContexts = new ConcurrentDictionary<DbConnectionPoolAuthenticationContextKey, DbConnectionPoolAuthenticationContext>(concurrencyLevel: 4 * Environment.ProcessorCount /* default value in ConcurrentDictionary*/,
 499                                                                                                                                                         capacity: 2);
 500
 501             if(ADP.IsPlatformNT5) {
 502                 _transactedConnectionPool = new TransactedConnectionPool(this);
 503             }
 504
 505             _poolCreateRequest = new WaitCallback(PoolCreateRequest); // used by CleanupCallback
 506             _state = State.Running;
 507
 508             Bid.PoolerTrace("<prov.DbConnectionPool.DbConnectionPool|RES|CPOOL> %d#, Constructed.\n", ObjectID);
 509
 510             //_cleanupTimer & QueuePoolCreateRequest is delayed until DbConnectionPoolGroup calls
 511             // StartBackgroundCallbacks after pool is actually in the collection
 512         }
 513
 514         private int CreationTimeout {
 515             get { return PoolGroupOptions.CreationTimeout; }
 516         }
 517
 518         internal int Count {
 519             get { return _totalObjects; }
 520         }
 521
 522         internal DbConnectionFactory ConnectionFactory {
 523             get { return _connectionFactory; }
 524         }
 525
 526         internal bool ErrorOccurred {
 527             get { return _errorOccurred; }
 528         }
 529
 530         private bool HasTransactionAffinity {
 531             get { return PoolGroupOptions.HasTransactionAffinity; }
 532         }
 533
 534         internal TimeSpan LoadBalanceTimeout {
 535             get { return PoolGroupOptions.LoadBalanceTimeout; }
 536         }
 537
 538         private bool NeedToReplenish {
 539             get {
 540                 if (State.Running != _state) // SQL BU DT 364595 - don't allow connection create when not running.
 541                     return false;
 542
 543                 int totalObjects = Count;
 544
 545                 if (totalObjects >= MaxPoolSize)
 546                     return false;
 547
 548                 if (totalObjects < MinPoolSize)
 549                     return true;
 550
 551                 int freeObjects        = (_stackNew.Count + _stackOld.Count);
 552                 int waitingRequests    = _waitCount;
 553                 bool needToReplenish = (freeObjects < waitingRequests) || ((freeObjects == waitingRequests) && (totalObjects > 1));
 554
 555                 return needToReplenish;
 556             }
 557         }
 558
 559         internal DbConnectionPoolIdentity Identity {
 560             get { return _identity; }
 561         }
 562
 563         internal bool IsRunning {
 564             get { return State.Running == _state; }
 565         }
 566
 567         private int MaxPoolSize {
 568             get { return PoolGroupOptions.MaxPoolSize; }
 569         }
 570
 571         private int MinPoolSize {
 572             get { return PoolGroupOptions.MinPoolSize; }
 573         }
 574
 575         internal int ObjectID {
 576             get {
 577                 return _objectID;
 578             }
 579         }
 580
 581         internal DbConnectionPoolCounters PerformanceCounters {
 582             get { return _connectionFactory.PerformanceCounters; }
 583         }
 584
 585         internal DbConnectionPoolGroup PoolGroup {
 586             get { return _connectionPoolGroup; }
 587         }
 588
 589         internal DbConnectionPoolGroupOptions PoolGroupOptions {
 590             get { return _connectionPoolGroupOptions; }
 591         }
 592
 593         internal DbConnectionPoolProviderInfo ProviderInfo {
 594             get { return _connectionPoolProviderInfo; }
 595         }
 596
 597         /// <summary>
 598         /// Return the pooled authentication contexts.
 599         /// </summary>
 600         internal ConcurrentDictionary<DbConnectionPoolAuthenticationContextKey, DbConnectionPoolAuthenticationContext> AuthenticationContexts
 601         {
 602             get
 603             {
 604                 return _pooledDbAuthenticationContexts;
 605             }
 606         }
 607
 608         internal bool UseLoadBalancing {
 609             get { return PoolGroupOptions.UseLoadBalancing; }
 610         }
 611
 612         private bool UsingIntegrateSecurity {
 613             get { return (null != _identity && DbConnectionPoolIdentity.NoIdentity != _identity); }
 614         }
 615
 616         private void CleanupCallback(Object state) {
 617             // Called when the cleanup-timer ticks over.
 618
 619             // This is the automatic prunning method.  Every period, we will
 620             // perform a two-step process:
 621             //
 622             // First, for each free object above MinPoolSize, we will obtain a
 623             // semaphore representing one object and destroy one from old stack.
 624             // We will continue this until we either reach MinPoolSize, we are
 625             // unable to obtain a free object, or we have exhausted all the
 626             // objects on the old stack.
 627             //
 628             // Second we move all free objects on the new stack to the old stack.
 629             // So, every period the objects on the old stack are destroyed and
 630             // the objects on the new stack are pushed to the old stack.  All
 631             // objects that are currently out and in use are not on either stack.
 632             //
 633             // With this logic, objects are pruned from the pool if unused for
 634             // at least one period but not more than two periods.
 635
 636             Bid.PoolerTrace("<prov.DbConnectionPool.CleanupCallback|RES|INFO|CPOOL> %d#\n", ObjectID);
 637
 638             // Destroy free objects that put us above MinPoolSize from old stack.
 639             while(Count > MinPoolSize) { // While above MinPoolSize...
 640
 641                 if (_waitHandles.PoolSemaphore.WaitOne(0, false) /* != WAIT_TIMEOUT */) {
 642                     // We obtained a objects from the semaphore.
 643                     DbConnectionInternal obj;
 644
 645                     if (_stackOld.TryPop(out obj)) {
 646                         Debug.Assert(obj != null, "null connection is not expected");
 647                         // If we obtained one from the old stack, destroy it.
 648 #if !MOBILE
 649                         PerformanceCounters.NumberOfFreeConnections.Decrement();
 650 #endif
 651
 652                         // Transaction roots must survive even aging out (TxEnd event will clean them up).
 653                         bool shouldDestroy = true;
 654                         lock (obj) {    // Lock to prevent race condition window between IsTransactionRoot and shouldDestroy assignment
 655                             if (obj.IsTransactionRoot) {
 656                                 shouldDestroy = false;
 657                             }
 658                         }
 659
 660                         // !!!!!!!!!! WARNING !!!!!!!!!!!!!
 661                         //   ONLY touch obj after lock release if shouldDestroy is false!!!  Otherwise, it may be destroyed
 662                         //   by transaction-end thread!
 663
 664                         // Note that there is a minor race condition between this task and the transaction end event, if the latter runs
 665                         //  between the lock above and the SetInStasis call below. The reslult is that the stasis counter may be
 666                         //  incremented without a corresponding decrement (the transaction end task is normally expected
 667                         //  to decrement, but will only do so if the stasis flag is set when it runs). I've minimized the size
 668                         //  of the window, but we aren't totally eliminating it due to SetInStasis needing to do bid tracing, which
 669                         //  we don't want to do under this lock, if possible. It should be possible to eliminate this race condition with
 670                         //  more substantial re-architecture of the pool, but we don't have the time to do that work for the current release.
 671
 672                         if (shouldDestroy) {
 673                             DestroyObject(obj);
 674                         }
 675                         else {
 676                             obj.SetInStasis();
 677                         }
 678                     }
 679                     else {
 680                         // Else we exhausted the old stack (the object the
 681                         // semaphore represents is on the new stack), so break.
 682                         _waitHandles.PoolSemaphore.Release(1);
 683                         break;
 684                     }
 685                 }
 686                 else {
 687                     break;
 688                 }
 689             }
 690
 691             // Push to the old-stack.  For each free object, move object from
 692             // new stack to old stack.
 693             if(_waitHandles.PoolSemaphore.WaitOne(0, false) /* != WAIT_TIMEOUT */) {
 694                 for(;;) {
 695                     DbConnectionInternal obj;
 696
 697                     if (!_stackNew.TryPop(out obj))
 698                         break;
 699
 700                     Debug.Assert(obj != null, "null connection is not expected");
 701
 702                     Bid.PoolerTrace("<prov.DbConnectionPool.CleanupCallback|RES|INFO|CPOOL> %d#, ChangeStacks=%d#\n", ObjectID, obj.ObjectID);
 703
 704                     Debug.Assert(!obj.IsEmancipated, "pooled object not in pool");
 705                     Debug.Assert(obj.CanBePooled,     "pooled object is not poolable");
 706
 707                     _stackOld.Push(obj);
 708                 }
 709                 _waitHandles.PoolSemaphore.Release(1);
 710             }
 711
 712             // Queue up a request to bring us up to MinPoolSize
 713             QueuePoolCreateRequest();
 714         }
 715
 716         internal void Clear() {
 717             Bid.PoolerTrace("<prov.DbConnectionPool.Clear|RES|CPOOL> %d#, Clearing.\n", ObjectID);
 718
 719             DbConnectionInternal obj;
 720
 721             // First, quickly doom everything.
 722             lock(_objectList) {
 723                 int count = _objectList.Count;
 724
 725                 for (int i = 0; i < count; ++i) {
 726                     obj = _objectList[i];
 727
 728                     if (null != obj) {
 729                         obj.DoNotPoolThisConnection();
 730                     }
 731                 }
 732             }
 733
 734             // Second, dispose of all the free connections.
 735             while (_stackNew.TryPop(out obj)) {
 736                 Debug.Assert(obj != null, "null connection is not expected");
 737 #if !MOBILE
 738                 PerformanceCounters.NumberOfFreeConnections.Decrement();
 739 #endif
 740                 DestroyObject(obj);
 741             }
 742             while (_stackOld.TryPop(out obj)) {
 743                 Debug.Assert(obj != null, "null connection is not expected");
 744 #if !MOBILE
 745                 PerformanceCounters.NumberOfFreeConnections.Decrement();
 746 #endif
 747                 DestroyObject(obj);
 748             }
 749
 750             // Finally, reclaim everything that's emancipated (which, because
 751             // it's been doomed, will cause it to be disposed of as well)
 752             ReclaimEmancipatedObjects();
 753
 754             Bid.PoolerTrace("<prov.DbConnectionPool.Clear|RES|CPOOL> %d#, Cleared.\n", ObjectID);
 755         }
 756
 757         private Timer CreateCleanupTimer() {
 758             return (new Timer(new TimerCallback(this.CleanupCallback), null, _cleanupWait, _cleanupWait));
 759         }
 760
 761         private static readonly string[] AzureSqlServerEndpoints = {Res.GetString(Res.AZURESQL_GenericEndpoint),
 762                                                                     Res.GetString(Res.AZURESQL_GermanEndpoint),
 763                                                                     Res.GetString(Res.AZURESQL_UsGovEndpoint),
 764                                                                     Res.GetString(Res.AZURESQL_ChinaEndpoint) };
 765         private static bool IsAzureSqlServerEndpoint(string dataSource)
 766         {
 767             // remove server port
 768             var i = dataSource.LastIndexOf(',');
 769             if (i >= 0)
 770             {
 771                 dataSource = dataSource.Substring(0, i);
 772             }
 773
 774             // check for the instance name
 775             i = dataSource.LastIndexOf('\\');
 776             if (i >= 0)
 777             {
 778                 dataSource = dataSource.Substring(0, i);
 779             }
 780
 781             // trim redundant whitespaces
 782             dataSource = dataSource.Trim();
 783
 784             // check if servername end with any azure endpoints
 785             for (i = 0; i < AzureSqlServerEndpoints.Length; i++)
 786             {
 787                 if (dataSource.EndsWith(AzureSqlServerEndpoints[i], StringComparison.OrdinalIgnoreCase))
 788                 {
 789                     return true;
 790                 }
 791             }
 792             return false;
 793         }
 794
 795         private bool IsBlockingPeriodEnabled()
 796         {
 797             var poolGroupConnectionOptions = _connectionPoolGroup.ConnectionOptions as SqlConnectionString;
 798             if (poolGroupConnectionOptions == null)
 799             {
 800                 return true;
 801             }
 802
 803             var policy = poolGroupConnectionOptions.PoolBlockingPeriod;
 804
 805             switch (policy)
 806             {
 807                 case PoolBlockingPeriod.Auto:
 808                 {
 809                     if (IsAzureSqlServerEndpoint(poolGroupConnectionOptions.DataSource))
 810                     {
 811                         return false; // in Azure it will be Disabled
 812                     }
 813                     else
 814                     {
 815                         return true; // in Non Azure, it will be Enabled
 816                     }
 817                 }
 818                 case PoolBlockingPeriod.AlwaysBlock:
 819                 {
 820                     return true; //Enabled
 821                 }
 822                 case PoolBlockingPeriod.NeverBlock:
 823                 {
 824                     return false; //Disabled
 825                 }
 826                 default:
 827                 {
 828                     //we should never get into this path.
 829                     Debug.Fail("Unknown PoolBlockingPeriod. Please specify explicit results in above switch case statement.");
 830                     return true;
 831                 }
 832             }
 833         }
 834
 835         private DbConnectionInternal CreateObject(DbConnection owningObject, DbConnectionOptions userOptions, DbConnectionInternal oldConnection) {
 836             DbConnectionInternal newObj = null;
 837
 838             try {
 839                 newObj = _connectionFactory.CreatePooledConnection(this, owningObject, _connectionPoolGroup.ConnectionOptions, _connectionPoolGroup.PoolKey, userOptions);
 840                 if (null == newObj) {
 841                     throw ADP.InternalError(ADP.InternalErrorCode.CreateObjectReturnedNull);    // CreateObject succeeded, but null object
 842                 }
 843                 if (!newObj.CanBePooled) {
 844                     throw ADP.InternalError(ADP.InternalErrorCode.NewObjectCannotBePooled);        // CreateObject succeeded, but non-poolable object
 845                 }
 846                 newObj.PrePush(null);
 847
 848                 lock (_objectList) {
 849                     if ((oldConnection != null) && (oldConnection.Pool == this)) {
 850                         _objectList.Remove(oldConnection);
 851                     }
 852                     _objectList.Add(newObj);
 853                     _totalObjects = _objectList.Count;
 854 #if !MOBILE
 855                     PerformanceCounters.NumberOfPooledConnections.Increment();   //
 856 #endif
 857                 }
 858
 859                 // If the old connection belonged to another pool, we need to remove it from that
 860                 if (oldConnection != null) {
 861                     var oldConnectionPool = oldConnection.Pool;
 862                     if (oldConnectionPool != null && oldConnectionPool != this) {
 863                         Debug.Assert(oldConnectionPool._state == State.ShuttingDown, "Old connections pool should be shutting down");
 864                         lock (oldConnectionPool._objectList) {
 865                             oldConnectionPool._objectList.Remove(oldConnection);
 866                             oldConnectionPool._totalObjects = oldConnectionPool._objectList.Count;
 867                         }
 868                     }
 869                 }
 870                 Bid.PoolerTrace("<prov.DbConnectionPool.CreateObject|RES|CPOOL> %d#, Connection %d#, Added to pool.\n", ObjectID, newObj.ObjectID);
 871
 872                 // Reset the error wait:
 873                 _errorWait = ERROR_WAIT_DEFAULT;
 874             }
 875             catch(Exception e)   {
 876                 //
 877                 if (!ADP.IsCatchableExceptionType(e)) {
 878                     throw;
 879                 }
 880
 881                 ADP.TraceExceptionForCapture(e);
 882
 883                 if (!IsBlockingPeriodEnabled())
 884                 {
 885                     throw;
 886                 }
 887
 888                 newObj = null; // set to null, so we do not return bad new object
 889                 // Failed to create instance
 890                 _resError = e;
 891
 892                 // VSTFDEVDIV 479561: Make sure the timer starts even if ThreadAbort occurs after setting the ErrorEvent.
 893
 894                 // timer allocation has to be done out of CER block
 895                 Timer t = new Timer(new TimerCallback(this.ErrorCallback), null, Timeout.Infinite, Timeout.Infinite);
 896                 bool timerIsNotDisposed;
 897                 RuntimeHelpers.PrepareConstrainedRegions();
 898                 try{} finally {
 899                     _waitHandles.ErrorEvent.Set();
 900                     _errorOccurred = true;
 901
 902                     // Enable the timer.
 903                     // Note that the timer is created to allow periodic invocation. If ThreadAbort occurs in the middle of ErrorCallback,
 904                     // the timer will restart. Otherwise, the timer callback (ErrorCallback) destroys the timer after resetting the error to avoid second callback.
 905                     _errorTimer = t;
 906                     timerIsNotDisposed = t.Change(_errorWait, _errorWait);
 907                 }
 908
 909                 Debug.Assert(timerIsNotDisposed, "ErrorCallback timer has been disposed");
 910
 911                 if (30000 < _errorWait) {
 912                     _errorWait = 60000;
 913                 }
 914                 else {
 915                     _errorWait *= 2;
 916                 }
 917                 throw;
 918             }
 919             return newObj;
 920         }
 921
 922         private void DeactivateObject(DbConnectionInternal obj)
 923         {
 924             Bid.PoolerTrace("<prov.DbConnectionPool.DeactivateObject|RES|CPOOL> %d#, Connection %d#, Deactivating.\n", ObjectID, obj.ObjectID);
 925
 926             obj.DeactivateConnection(); // we presume this operation is safe outside of a lock...
 927
 928             bool returnToGeneralPool = false;
 929             bool destroyObject = false;
 930             bool rootTxn = false;
 931
 932             if ( obj.IsConnectionDoomed )
 933             {
 934                 // the object is not fit for reuse -- just dispose of it.
 935                 destroyObject = true;
 936             }
 937             else
 938             {
 939                 // NOTE: constructor should ensure that current state cannot be State.Initializing, so it can only
 940                 //   be State.Running or State.ShuttingDown
 941                 Debug.Assert ( _state == State.Running || _state == State.ShuttingDown );
 942
 943                 lock (obj)
 944                 {
 945                     // A connection with a delegated transaction cannot currently
 946                     // be returned to a different customer until the transaction
 947                     // actually completes, so we send it into Stasis -- the SysTx
 948                     // transaction object will ensure that it is owned (not lost),
 949                     // and it will be certain to put it back into the pool.
 950
 951                     if ( _state == State.ShuttingDown )
 952                     {
 953                         if ( obj.IsTransactionRoot )
 954                         {
 955                             // SQLHotfix# 50003503 - connections that are affiliated with a
 956                             //   root transaction and that also happen to be in a connection
 957                             //   pool that is being shutdown need to be put in stasis so that
 958                             //   the root transaction isn't effectively orphaned with no
 959                             //   means to promote itself to a full delegated transaction or
 960                             //   Commit or Rollback
 961                             obj.SetInStasis();
 962                             rootTxn = true;
 963                         }
 964                         else
 965                         {
 966                             // connection is being closed and the pool has been marked as shutting
 967                             //   down, so destroy this object.
 968                             destroyObject = true;
 969                         }
 970                     }
 971                     else
 972                     {
 973                         if ( obj.IsNonPoolableTransactionRoot )
 974                         {
 975                             obj.SetInStasis();
 976                             rootTxn = true;
 977                         }
 978                         else if ( obj.CanBePooled )
 979                         {
 980                             // We must put this connection into the transacted pool
 981                             // while inside a lock to prevent a race condition with
 982                             // the transaction asyncronously completing on a second
 983                             // thread.
 984
 985                             SysTx.Transaction transaction = obj.EnlistedTransaction;
 986                             if (null != transaction)
 987                             {
 988                                 // NOTE: we're not locking on _state, so it's possible that its
 989                                 //   value could change between the conditional check and here.
 990                                 //   Although perhaps not ideal, this is OK because the
 991                                 //   DelegatedTransactionEnded event will clean up the
 992                                 //   connection appropriately regardless of the pool state.
 993                                 Debug.Assert ( _transactedConnectionPool != null, "Transacted connection pool was not expected to be null.");
 994                                 _transactedConnectionPool.PutTransactedObject(transaction, obj);
 995                                 rootTxn = true;
 996                             }
 997                             else
 998                             {
 999                                 // return to general pool
1000                                 returnToGeneralPool = true;
1001                             }
1002                         }
1003                         else
1004                         {
1005                             if ( obj.IsTransactionRoot && !obj.IsConnectionDoomed )
1006                             {
1007                                 // SQLHotfix# 50003503 - if the object cannot be pooled but is a transaction
1008                                 //   root, then we must have hit one of two race conditions:
1009                                 //       1) PruneConnectionPoolGroups shutdown the pool and marked this connection
1010                                 //          as non-poolable while we were processing within this lock
1011                                 //       2) The LoadBalancingTimeout expired on this connection and marked this
1012                                 //          connection as DoNotPool.
1013                                 //
1014                                 //   This connection needs to be put in stasis so that the root transaction isn't
1015                                 //   effectively orphaned with no means to promote itself to a full delegated
1016                                 //   transaction or Commit or Rollback
1017                                 obj.SetInStasis();
1018                                 rootTxn = true;
1019                             }
1020                             else
1021                             {
1022                                 // object is not fit for reuse -- just dispose of it
1023                                 destroyObject = true;
1024                             }
1025                         }
1026                     }
1027                 }
1028             }
1029
1030             if (returnToGeneralPool)
1031             {
1032                 // Only push the connection into the general pool if we didn't
1033                 //   already push it onto the transacted pool, put it into stasis,
1034                 //   or want to destroy it.
1035                 Debug.Assert ( destroyObject == false );
1036                 PutNewObject(obj);
1037             }
1038             else if ( destroyObject )
1039             {
1040                 // VSTFDEVDIV# 479556 - connections that have been marked as no longer
1041                 //   poolable (e.g. exceeded their connection lifetime) are not, in fact,
1042                 //   returned to the general pool
1043                 DestroyObject(obj);
1044                 QueuePoolCreateRequest();
1045             }
1046
1047             //-------------------------------------------------------------------------------------
1048             // postcondition
1049
1050             // ensure that the connection was processed
1051             Debug.Assert ( rootTxn == true || returnToGeneralPool == true || destroyObject == true );
1052
1053             //
1054         }
1055
1056         internal void DestroyObject(DbConnectionInternal obj) {
1057             // A connection with a delegated transaction cannot be disposed of
1058             // until the delegated transaction has actually completed.  Instead,
1059             // we simply leave it alone; when the transaction completes, it will
1060             // come back through PutObjectFromTransactedPool, which will call us
1061             // again.
1062             if (obj.IsTxRootWaitingForTxEnd) {
1063                 Bid.PoolerTrace("<prov.DbConnectionPool.DestroyObject|RES|CPOOL> %d#, Connection %d#, Has Delegated Transaction, waiting to Dispose.\n", ObjectID, obj.ObjectID);
1064             }
1065             else {
1066                 Bid.PoolerTrace("<prov.DbConnectionPool.DestroyObject|RES|CPOOL> %d#, Connection %d#, Removing from pool.\n", ObjectID, obj.ObjectID);
1067
1068                 bool removed = false;
1069                 lock (_objectList) {
1070                     removed = _objectList.Remove(obj);
1071                     Debug.Assert(removed, "attempt to DestroyObject not in list");
1072                     _totalObjects = _objectList.Count;
1073                 }
1074
1075                 if (removed) {
1076                     Bid.PoolerTrace("<prov.DbConnectionPool.DestroyObject|RES|CPOOL> %d#, Connection %d#, Removed from pool.\n", ObjectID, obj.ObjectID);
1077 #if !MOBILE
1078                     PerformanceCounters.NumberOfPooledConnections.Decrement();
1079 #endif
1080                 }
1081                 obj.Dispose();
1082
1083                 Bid.PoolerTrace("<prov.DbConnectionPool.DestroyObject|RES|CPOOL> %d#, Connection %d#, Disposed.\n", ObjectID, obj.ObjectID);
1084 #if !MOBILE
1085                 PerformanceCounters.HardDisconnectsPerSecond.Increment();
1086 #endif
1087             }
1088         }
1089
1090         private void ErrorCallback(Object state) {
1091             Bid.PoolerTrace("<prov.DbConnectionPool.ErrorCallback|RES|CPOOL> %d#, Resetting Error handling.\n", ObjectID);
1092
1093             _errorOccurred = false;
1094             _waitHandles.ErrorEvent.Reset();
1095
1096             // the error state is cleaned, destroy the timer to avoid periodic invocation
1097             Timer t     = _errorTimer;
1098             _errorTimer = null;
1099             if (t != null) {
1100                 t.Dispose(); // Cancel timer request.
1101             }
1102         }
1103
1104
1105         private Exception TryCloneCachedException()
1106         // Cached exception can be of any type, so is not always cloneable.
1107         // This functions clones SqlException
1108         // OleDb and Odbc connections are not passing throw this code
1109         {
1110             if (_resError==null)
1111                 return null;
1112             if (_resError.GetType()==typeof(SqlClient.SqlException))
1113                 return ((SqlClient.SqlException)_resError).InternalClone();
1114             return _resError;
1115         }
1116
1117         void WaitForPendingOpen() {
1118
1119             Debug.Assert(!Thread.CurrentThread.IsThreadPoolThread, "This thread may block for a long time.  Threadpool threads should not be used.");
1120
1121             PendingGetConnection next;
1122
1123             do {
1124                 bool started = false;
1125
1126                 RuntimeHelpers.PrepareConstrainedRegions();
1127                 try {
1128
1129                     RuntimeHelpers.PrepareConstrainedRegions();
1130                     try { }
1131                     finally {
1132                         started = Interlocked.CompareExchange(ref _pendingOpensWaiting, 1, 0) == 0;
1133                     }
1134
1135                     if (!started) {
1136                         return;
1137                     }
1138
1139                     while (_pendingOpens.TryDequeue(out next)) {
1140
1141                         if (next.Completion.Task.IsCompleted) {
1142                             continue;
1143                         }
1144
1145                         uint delay;
1146                         if (next.DueTime == Timeout.Infinite) {
1147                             delay = unchecked((uint)Timeout.Infinite);
1148                         }
1149                         else {
1150                             delay = (uint)Math.Max(ADP.TimerRemainingMilliseconds(next.DueTime), 0);
1151                         }
1152
1153                         DbConnectionInternal connection = null;
1154                         bool timeout = false;
1155                         Exception caughtException = null;
1156
1157                         RuntimeHelpers.PrepareConstrainedRegions();
1158                         try {
1159 #if DEBUG
1160                             System.Data.SqlClient.TdsParser.ReliabilitySection tdsReliabilitySection = new System.Data.SqlClient.TdsParser.ReliabilitySection();
1161
1162                             RuntimeHelpers.PrepareConstrainedRegions();
1163                             try {
1164                                 tdsReliabilitySection.Start();
1165 #else
1166                             {
1167 #endif //DEBUG
1168
1169                                 bool allowCreate = true;
1170                                 bool onlyOneCheckConnection = false;
1171                                 ADP.SetCurrentTransaction(next.Completion.Task.AsyncState as Transactions.Transaction);
1172                                 timeout = !TryGetConnection(next.Owner, delay, allowCreate, onlyOneCheckConnection, next.UserOptions, out connection);
1173                             }
1174 #if DEBUG
1175                             finally {
1176                                 tdsReliabilitySection.Stop();
1177                             }
1178 #endif //DEBUG
1179                         }
1180                         catch (System.OutOfMemoryException) {
1181                             if (connection != null) { connection.DoomThisConnection(); }
1182                             throw;
1183                         }
1184                         catch (System.StackOverflowException) {
1185                             if (connection != null) { connection.DoomThisConnection(); }
1186                             throw;
1187                         }
1188                         catch (System.Threading.ThreadAbortException) {
1189                             if (connection != null) { connection.DoomThisConnection(); }
1190                             throw;
1191                         }
1192                         catch (Exception e) {
1193                             caughtException = e;
1194                         }
1195
1196                         if (caughtException != null) {
1197                             next.Completion.TrySetException(caughtException);
1198                         }
1199                         else if (timeout) {
1200                             next.Completion.TrySetException(ADP.ExceptionWithStackTrace(ADP.PooledOpenTimeout()));
1201                         }
1202                         else {
1203                             Debug.Assert(connection != null, "connection should never be null in success case");
1204                             if (!next.Completion.TrySetResult(connection)) {
1205                                 // if the completion was cancelled, lets try and get this connection back for the next try
1206                                 PutObject(connection, next.Owner);
1207                             }
1208                         }
1209                     }
1210                 }
1211                 finally {
1212                     if (started) {
1213                         Interlocked.Exchange(ref _pendingOpensWaiting, 0);
1214                     }
1215                 }
1216
1217             } while (_pendingOpens.TryPeek(out next));
1218         }
1219
1220         internal bool TryGetConnection(DbConnection owningObject, TaskCompletionSource<DbConnectionInternal> retry, DbConnectionOptions userOptions, out DbConnectionInternal connection) {
1221
1222             uint waitForMultipleObjectsTimeout = 0;
1223             bool allowCreate = false;
1224
1225             if (retry == null) {
1226                 waitForMultipleObjectsTimeout = (uint)CreationTimeout;
1227
1228                 // VSTFDEVDIV 445531: set the wait timeout to INFINITE (-1) if the SQL connection timeout is 0 (== infinite)
1229                 if (waitForMultipleObjectsTimeout == 0)
1230                     waitForMultipleObjectsTimeout = unchecked((uint)Timeout.Infinite);
1231
1232                 allowCreate = true;
1233             }
1234
1235             if (_state != State.Running) {
1236                 Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, DbConnectionInternal State != Running.\n", ObjectID);
1237                 connection = null;
1238                 return true;
1239             }
1240
1241             bool onlyOneCheckConnection = true;
1242             if (TryGetConnection(owningObject, waitForMultipleObjectsTimeout, allowCreate, onlyOneCheckConnection, userOptions, out connection)) {
1243                 return true;
1244             }
1245             else if (retry == null) {
1246                 // timed out on a [....] call
1247                 return true;
1248             }
1249
1250             var pendingGetConnection =
1251                 new PendingGetConnection(
1252                     CreationTimeout == 0 ? Timeout.Infinite : ADP.TimerCurrent() + ADP.TimerFromSeconds(CreationTimeout/1000),
1253                     owningObject,
1254                     retry,
1255                     userOptions);
1256             _pendingOpens.Enqueue(pendingGetConnection);
1257
1258             // it is better to StartNew too many times than not enough
1259             if (_pendingOpensWaiting == 0) {
1260                 Thread waitOpenThread = new Thread(WaitForPendingOpen);
1261                 waitOpenThread.IsBackground = true;
1262                 waitOpenThread.Start();
1263             }
1264
1265             connection = null;
1266             return false;
1267         }
1268
1269         [SuppressMessage("Microsoft.Reliability", "CA2001:AvoidCallingProblematicMethods")] // copied from Triaged.cs
1270         [ResourceExposure(ResourceScope.None)] // SxS: this method does not expose resources
1271         [ResourceConsumption(ResourceScope.Machine, ResourceScope.Machine)]
1272         private bool TryGetConnection(DbConnection owningObject, uint waitForMultipleObjectsTimeout, bool allowCreate, bool onlyOneCheckConnection, DbConnectionOptions userOptions, out DbConnectionInternal connection) {
1273             DbConnectionInternal obj = null;
1274             SysTx.Transaction transaction = null;
1275
1276 #if !MOBILE
1277             PerformanceCounters.SoftConnectsPerSecond.Increment();
1278 #endif
1279
1280             Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, Getting connection.\n", ObjectID);
1281
1282             // If automatic transaction enlistment is required, then we try to
1283             // get the connection from the transacted connection pool first.
1284             if (HasTransactionAffinity) {
1285                 obj = GetFromTransactedPool(out transaction);
1286             }
1287
1288             if (null == obj) {
1289                 Interlocked.Increment(ref _waitCount);
1290                 uint waitHandleCount = allowCreate ? 3u : 2u;
1291
1292                 do {
1293                     int waitResult = BOGUS_HANDLE;
1294                     int releaseSemaphoreResult = 0;
1295
1296                     bool mustRelease = false;
1297                     int waitForMultipleObjectsExHR = 0;
1298                     RuntimeHelpers.PrepareConstrainedRegions();
1299                     try {
1300                         _waitHandles.DangerousAddRef(ref mustRelease);
1301
1302                         // We absolutely must have the value of waitResult set,
1303                         // or we may leak the mutex in async abort cases.
1304                         RuntimeHelpers.PrepareConstrainedRegions();
1305                         try {
1306                             Debug.Assert(2 == waitHandleCount || 3 == waitHandleCount, "unexpected waithandle count");
1307                         }
1308                         finally {
1309                             waitResult = SafeNativeMethods.WaitForMultipleObjectsEx(waitHandleCount, _waitHandles.DangerousGetHandle(), false, waitForMultipleObjectsTimeout, false);
1310
1311 #if !FULL_AOT_RUNTIME
1312                             // VSTFDEVDIV 479551 - call GetHRForLastWin32Error immediately after after the native call
1313                             if (waitResult == WAIT_FAILED) {
1314                                 waitForMultipleObjectsExHR = Marshal.GetHRForLastWin32Error();
1315                             }
1316 #endif
1317                         }
1318
1319                         // From the WaitAny docs: "If more than one object became signaled during
1320                         // the call, this is the array index of the signaled object with the
1321                         // smallest index value of all the signaled objects."  This is important
1322                         // so that the free object signal will be returned before a creation
1323                         // signal.
1324
1325                         switch (waitResult) {
1326                         case WAIT_TIMEOUT:
1327                             Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, Wait timed out.\n", ObjectID);
1328                             Interlocked.Decrement(ref _waitCount);
1329                             connection = null;
1330                             return false;
1331
1332                         case ERROR_HANDLE:
1333                             // Throw the error that PoolCreateRequest stashed.
1334                             Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, Errors are set.\n", ObjectID);
1335                             Interlocked.Decrement(ref _waitCount);
1336                             throw TryCloneCachedException();
1337
1338                         case CREATION_HANDLE:
1339                             Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, Creating new connection.\n", ObjectID);
1340
1341                             try {
1342                                 obj = UserCreateRequest(owningObject, userOptions);
1343                             }
1344                             catch {
1345                                 if (null == obj) {
1346                                     Interlocked.Decrement(ref _waitCount);
1347                                 }
1348                                 throw;
1349                             }
1350                             finally {
1351                                 // SQLBUDT #386664 - ensure that we release this waiter, regardless
1352                                 // of any exceptions that may be thrown.
1353                                 if (null != obj) {
1354                                     Interlocked.Decrement(ref _waitCount);
1355                                 }
1356                             }
1357
1358                             if (null == obj) {
1359                                 // If we were not able to create an object, check to see if
1360                                 // we reached MaxPoolSize.  If so, we will no longer wait on
1361                                 // the CreationHandle, but instead wait for a free object or
1362                                 // the timeout.
1363
1364                                 //
1365
1366                                 if (Count >= MaxPoolSize && 0 != MaxPoolSize) {
1367                                     if (!ReclaimEmancipatedObjects()) {
1368                                         // modify handle array not to wait on creation mutex anymore
1369                                         Debug.Assert(2 == CREATION_HANDLE, "creation handle changed value");
1370                                         waitHandleCount = 2;
1371                                     }
1372                                 }
1373                             }
1374                             break;
1375
1376                         case SEMAPHORE_HANDLE:
1377                             //
1378                             //    guaranteed available inventory
1379                             //
1380                             Interlocked.Decrement(ref _waitCount);
1381                             obj = GetFromGeneralPool();
1382
1383                             if ((obj != null) && (!obj.IsConnectionAlive())) {
1384                                 Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, Connection %d#, found dead and removed.\n", ObjectID, obj.ObjectID);
1385                                 DestroyObject(obj);
1386                                 obj = null;     // Setting to null in case creating a new object fails
1387
1388                                 if (onlyOneCheckConnection) {
1389                                     if (_waitHandles.CreationSemaphore.WaitOne(unchecked((int)waitForMultipleObjectsTimeout))) {
1390                                         RuntimeHelpers.PrepareConstrainedRegions();
1391                                         try {
1392                                             Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, Creating new connection.\n", ObjectID);
1393                                             obj = UserCreateRequest(owningObject, userOptions);
1394                                         }
1395                                         finally {
1396                                             _waitHandles.CreationSemaphore.Release(1);
1397                                         }
1398                                     }
1399                                     else {
1400                                         // Timeout waiting for creation semaphore - return null
1401                                         Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, Wait timed out.\n", ObjectID);
1402                                         connection = null;
1403                                         return false;
1404                                     }
1405                                 }
1406                             }
1407                             break;
1408
1409                         case WAIT_FAILED:
1410                             Debug.Assert(waitForMultipleObjectsExHR != 0, "WaitForMultipleObjectsEx failed but waitForMultipleObjectsExHR remained 0");
1411                             Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, Wait failed.\n", ObjectID);
1412                             Interlocked.Decrement(ref _waitCount);
1413                             Marshal.ThrowExceptionForHR(waitForMultipleObjectsExHR);
1414                             goto default; // if ThrowExceptionForHR didn't throw for some reason
1415                         case (WAIT_ABANDONED+SEMAPHORE_HANDLE):
1416                             Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, Semaphore handle abandonded.\n", ObjectID);
1417                             Interlocked.Decrement(ref _waitCount);
1418                             throw new AbandonedMutexException(SEMAPHORE_HANDLE,_waitHandles.PoolSemaphore);
1419                         case (WAIT_ABANDONED+ERROR_HANDLE):
1420                             Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, Error handle abandonded.\n", ObjectID);
1421                             Interlocked.Decrement(ref _waitCount);
1422                             throw new AbandonedMutexException(ERROR_HANDLE,_waitHandles.ErrorEvent);
1423                         case (WAIT_ABANDONED+CREATION_HANDLE):
1424                             Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, Creation handle abandoned.\n", ObjectID);
1425                             Interlocked.Decrement(ref _waitCount);
1426                             throw new AbandonedMutexException(CREATION_HANDLE,_waitHandles.CreationSemaphore);
1427                         default:
1428                             Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, WaitForMultipleObjects=%d\n", ObjectID, waitResult);
1429                             Interlocked.Decrement(ref _waitCount);
1430                             throw ADP.InternalError(ADP.InternalErrorCode.UnexpectedWaitAnyResult);
1431                         }
1432                     }
1433                     finally {
1434                         if (CREATION_HANDLE == waitResult) {
1435                             int result = SafeNativeMethods.ReleaseSemaphore(_waitHandles.CreationHandle.DangerousGetHandle(), 1, IntPtr.Zero);
1436                             if (0 == result) { // failure case
1437 #if !FULL_AOT_RUNTIME
1438                                 releaseSemaphoreResult = Marshal.GetHRForLastWin32Error();
1439 #endif
1440                             }
1441                         }
1442                         if (mustRelease) {
1443                             _waitHandles.DangerousRelease();
1444                         }
1445                     }
1446                     if (0 != releaseSemaphoreResult) {
1447                         Marshal.ThrowExceptionForHR(releaseSemaphoreResult); // will only throw if (hresult < 0)
1448                     }
1449                 } while (null == obj);
1450             }
1451
1452             if (null != obj)
1453             {
1454                 PrepareConnection(owningObject, obj, transaction);
1455             }
1456
1457             connection = obj;
1458             return true;
1459         }
1460
1461         private void PrepareConnection(DbConnection owningObject, DbConnectionInternal obj, SysTx.Transaction transaction) {
1462             lock (obj)
1463             {   // Protect against Clear and ReclaimEmancipatedObjects, which call IsEmancipated, which is affected by PrePush and PostPop
1464                 obj.PostPop(owningObject);
1465             }
1466             try
1467             {
1468                 obj.ActivateConnection(transaction);
1469             }
1470             catch
1471             {
1472                 // if Activate throws an exception
1473                 // put it back in the pool or have it properly disposed of
1474                 this.PutObject(obj, owningObject);
1475                 throw;
1476             }
1477         }
1478
1479         /// <summary>
1480         /// Creates a new connection to replace an existing connection
1481         /// </summary>
1482         /// <param name="owningObject">Outer connection that currently owns <paramref name="oldConnection"/></param>
1483         /// <param name="userOptions">Options used to create the new connection</param>
1484         /// <param name="oldConnection">Inner connection that will be replaced</param>
1485         /// <returns>A new inner connection that is attached to the <paramref name="owningObject"/></returns>
1486         internal DbConnectionInternal ReplaceConnection(DbConnection owningObject, DbConnectionOptions userOptions, DbConnectionInternal oldConnection) {
1487 #if !MOBILE
1488             PerformanceCounters.SoftConnectsPerSecond.Increment();
1489 #endif
1490             Bid.PoolerTrace("<prov.DbConnectionPool.ReplaceConnection|RES|CPOOL> %d#, replacing connection.\n", ObjectID);
1491
1492             DbConnectionInternal newConnection = UserCreateRequest(owningObject, userOptions, oldConnection);
1493
1494             if (newConnection != null) {
1495                 PrepareConnection(owningObject, newConnection, oldConnection.EnlistedTransaction);
1496                 oldConnection.PrepareForReplaceConnection();
1497                 oldConnection.DeactivateConnection();
1498                 oldConnection.Dispose();
1499             }
1500
1501             return newConnection;
1502         }
1503
1504         private DbConnectionInternal GetFromGeneralPool() {
1505             DbConnectionInternal obj = null;
1506
1507             if (!_stackNew.TryPop(out obj)) {
1508                 if (!_stackOld.TryPop(out obj)) {
1509                     obj = null;
1510                 }
1511                 else {
1512                     Debug.Assert(obj != null, "null connection is not expected");
1513                 }
1514             }
1515             else {
1516                 Debug.Assert(obj != null, "null connection is not expected");
1517             }
1518
1519             // SQLBUDT #356870 -- When another thread is clearing this pool,
1520             // it will remove all connections in this pool which causes the
1521             // following assert to fire, which really mucks up stress against
1522             //  checked bits.  The assert is benign, so we're commenting it out.
1523             //Debug.Assert(obj != null, "GetFromGeneralPool called with nothing in the pool!");
1524
1525             if (null != obj) {
1526                 Bid.PoolerTrace("<prov.DbConnectionPool.GetFromGeneralPool|RES|CPOOL> %d#, Connection %d#, Popped from general pool.\n", ObjectID, obj.ObjectID);
1527 #if !MOBILE
1528                 PerformanceCounters.NumberOfFreeConnections.Decrement();
1529 #endif
1530             }
1531             return(obj);
1532         }
1533
1534         private DbConnectionInternal GetFromTransactedPool(out SysTx.Transaction transaction) {
1535             transaction = ADP.GetCurrentTransaction();
1536             DbConnectionInternal obj = null;
1537
1538             if (null != transaction && null != _transactedConnectionPool) {
1539                 obj = _transactedConnectionPool.GetTransactedObject(transaction);
1540
1541                 if (null != obj) {
1542                     Bid.PoolerTrace("<prov.DbConnectionPool.GetFromTransactedPool|RES|CPOOL> %d#, Connection %d#, Popped from transacted pool.\n", ObjectID, obj.ObjectID);
1543 #if !MOBILE
1544                     PerformanceCounters.NumberOfFreeConnections.Decrement();
1545 #endif
1546
1547                     if (obj.IsTransactionRoot) {
1548                         try {
1549                             obj.IsConnectionAlive(true);
1550                         }
1551                         catch {
1552                             Bid.PoolerTrace("<prov.DbConnectionPool.GetFromTransactedPool|RES|CPOOL> %d#, Connection %d#, found dead and removed.\n", ObjectID, obj.ObjectID);
1553                             DestroyObject(obj);
1554                             throw;
1555                         }
1556                     }
1557                     else if (!obj.IsConnectionAlive()) {
1558                         Bid.PoolerTrace("<prov.DbConnectionPool.GetFromTransactedPool|RES|CPOOL> %d#, Connection %d#, found dead and removed.\n", ObjectID, obj.ObjectID);
1559                         DestroyObject(obj);
1560                         obj = null;
1561                     }
1562                 }
1563             }
1564             return obj;
1565         }
1566
1567         [ResourceExposure(ResourceScope.None)] // SxS: this method does not expose resources
1568         [ResourceConsumption(ResourceScope.Machine, ResourceScope.Machine)]
1569         private void PoolCreateRequest(object state) {
1570             // called by pooler to ensure pool requests are currently being satisfied -
1571             // creation mutex has not been obtained
1572
1573             IntPtr hscp;
1574
1575             Bid.PoolerScopeEnter(out hscp, "<prov.DbConnectionPool.PoolCreateRequest|RES|INFO|CPOOL> %d#\n", ObjectID);
1576
1577             try {
1578                 if (State.Running == _state) {
1579
1580                     // in case WaitForPendingOpen ever failed with no subsequent OpenAsync calls,
1581                     // start it back up again
1582                     if (!_pendingOpens.IsEmpty && _pendingOpensWaiting == 0) {
1583                         Thread waitOpenThread = new Thread(WaitForPendingOpen);
1584                         waitOpenThread.IsBackground = true;
1585                         waitOpenThread.Start();
1586                     }
1587
1588                     // Before creating any new objects, reclaim any released objects that were
1589                     // not closed.
1590                     ReclaimEmancipatedObjects();
1591
1592                     if (!ErrorOccurred) {
1593                         if (NeedToReplenish) {
1594                             // Check to see if pool was created using integrated security and if so, make
1595                             // sure the identity of current user matches that of user that created pool.
1596                             // If it doesn't match, do not create any objects on the ThreadPool thread,
1597                             // since either Open will fail or we will open a object for this pool that does
1598                             // not belong in this pool.  The side effect of this is that if using integrated
1599                             // security min pool size cannot be guaranteed.
1600                             if (UsingIntegrateSecurity && !_identity.Equals(DbConnectionPoolIdentity.GetCurrent())) {
1601                                 return;
1602                             }
1603                             bool mustRelease = false;
1604                             int waitResult = BOGUS_HANDLE;
1605                             uint timeout = (uint)CreationTimeout;
1606
1607                             RuntimeHelpers.PrepareConstrainedRegions();
1608                             try {
1609                                 _waitHandles.DangerousAddRef(ref mustRelease);
1610
1611                                 // Obtain creation mutex so we're the only one creating objects
1612                                 // and we must have the wait result
1613                                 RuntimeHelpers.PrepareConstrainedRegions();
1614                                 try { } finally {
1615                                     waitResult = SafeNativeMethods.WaitForSingleObjectEx(_waitHandles.CreationHandle.DangerousGetHandle(), timeout, false);
1616                                 }
1617                                 if (WAIT_OBJECT_0 == waitResult) {
1618                                     DbConnectionInternal newObj;
1619
1620                                     // Check ErrorOccurred again after obtaining mutex
1621                                     if (!ErrorOccurred) {
1622                                         while (NeedToReplenish) {
1623                                             // Don't specify any user options because there is no outer connection associated with the new connection
1624                                             newObj = CreateObject(owningObject: null, userOptions: null, oldConnection: null);
1625
1626                                             // We do not need to check error flag here, since we know if
1627                                             // CreateObject returned null, we are in error case.
1628                                             if (null != newObj) {
1629                                                 PutNewObject(newObj);
1630                                             }
1631                                             else {
1632                                                 break;
1633                                             }
1634                                         }
1635                                     }
1636                                 }
1637                                 else if (WAIT_TIMEOUT == waitResult) {
1638                                     // do not wait forever and potential block this worker thread
1639                                     // instead wait for a period of time and just requeue to try again
1640                                     QueuePoolCreateRequest();
1641                                 }
1642                                 else {
1643                                     // trace waitResult and ignore the failure
1644                                     Bid.PoolerTrace("<prov.DbConnectionPool.PoolCreateRequest|RES|CPOOL> %d#, PoolCreateRequest called WaitForSingleObject failed %d", ObjectID, waitResult);
1645                                 }
1646                             }
1647                             catch (Exception e) {
1648                                 //
1649                                 if (!ADP.IsCatchableExceptionType(e)) {
1650                                     throw;
1651                                 }
1652
1653                                 // Now that CreateObject can throw, we need to catch the exception and discard it.
1654                                 // There is no further action we can take beyond tracing.  The error will be
1655                                 // thrown to the user the next time they request a connection.
1656                                 Bid.PoolerTrace("<prov.DbConnectionPool.PoolCreateRequest|RES|CPOOL> %d#, PoolCreateRequest called CreateConnection which threw an exception: %ls", ObjectID, e);
1657                             }
1658                             finally {
1659                                 if (WAIT_OBJECT_0 == waitResult) {
1660                                     // reuse waitResult and ignore its value
1661                                     waitResult = SafeNativeMethods.ReleaseSemaphore(_waitHandles.CreationHandle.DangerousGetHandle(), 1, IntPtr.Zero);
1662                                 }
1663                                 if (mustRelease) {
1664                                     _waitHandles.DangerousRelease();
1665                                 }
1666                             }
1667                         }
1668                     }
1669                 }
1670             }
1671             finally {
1672                 Bid.ScopeLeave(ref hscp);
1673             }
1674         }
1675
1676         internal void PutNewObject(DbConnectionInternal obj) {
1677             Debug.Assert(null != obj,        "why are we adding a null object to the pool?");
1678
1679             // VSTFDEVDIV 742887 - When another thread is clearing this pool, it
1680             // will set _cannotBePooled for all connections in this pool without prejudice which
1681             // causes the following assert to fire, which really mucks up stress
1682             // against checked bits.
1683             // Debug.Assert(obj.CanBePooled,    "non-poolable object in pool");
1684
1685             Bid.PoolerTrace("<prov.DbConnectionPool.PutNewObject|RES|CPOOL> %d#, Connection %d#, Pushing to general pool.\n", ObjectID, obj.ObjectID);
1686
1687             _stackNew.Push(obj);
1688             _waitHandles.PoolSemaphore.Release(1);
1689 #if !MOBILE
1690             PerformanceCounters.NumberOfFreeConnections.Increment();
1691 #endif
1692         }
1693
1694         internal void PutObject(DbConnectionInternal obj, object owningObject) {
1695             Debug.Assert(null != obj, "null obj?");
1696
1697 #if !MOBILE
1698             PerformanceCounters.SoftDisconnectsPerSecond.Increment();
1699 #endif
1700
1701             // Once a connection is closing (which is the state that we're in at
1702             // this point in time) you cannot delegate a transaction to or enlist
1703             // a transaction in it, so we can correctly presume that if there was
1704             // not a delegated or enlisted transaction to start with, that there
1705             // will not be a delegated or enlisted transaction once we leave the
1706             // lock.
1707
1708             lock (obj) {
1709                 // Calling PrePush prevents the object from being reclaimed
1710                 // once we leave the lock, because it sets _pooledCount such
1711                 // that it won't appear to be out of the pool.  What that
1712                 // means, is that we're now responsible for this connection:
1713                 // it won't get reclaimed if we drop the ball somewhere.
1714                 obj.PrePush(owningObject);
1715
1716                 //
1717             }
1718
1719             DeactivateObject(obj);
1720         }
1721
1722         internal void PutObjectFromTransactedPool(DbConnectionInternal obj) {
1723             Debug.Assert(null != obj, "null pooledObject?");
1724             Debug.Assert(obj.EnlistedTransaction == null, "pooledObject is still enlisted?");
1725
1726             // called by the transacted connection pool , once it's removed the
1727             // connection from it's list.  We put the connection back in general
1728             // circulation.
1729
1730             // NOTE: there is no locking required here because if we're in this
1731             // method, we can safely presume that the caller is the only person
1732             // that is using the connection, and that all pre-push logic has been
1733             // done and all transactions are ended.
1734
1735             Bid.PoolerTrace("<prov.DbConnectionPool.PutObjectFromTransactedPool|RES|CPOOL> %d#, Connection %d#, Transaction has ended.\n", ObjectID, obj.ObjectID);
1736
1737             if (_state == State.Running && obj.CanBePooled) {
1738                 PutNewObject(obj);
1739             }
1740             else {
1741                 DestroyObject(obj);
1742                 QueuePoolCreateRequest();
1743             }
1744         }
1745
1746         private void QueuePoolCreateRequest() {
1747             if (State.Running == _state) {
1748                 // Make sure we're at quota by posting a callback to the threadpool.
1749                 ThreadPool.QueueUserWorkItem(_poolCreateRequest);
1750             }
1751         }
1752
1753         private bool ReclaimEmancipatedObjects() {
1754             bool emancipatedObjectFound = false;
1755
1756             Bid.PoolerTrace("<prov.DbConnectionPool.ReclaimEmancipatedObjects|RES|CPOOL> %d#\n", ObjectID);
1757
1758             List<DbConnectionInternal> reclaimedObjects = new List<DbConnectionInternal>();
1759             int count;
1760
1761             lock(_objectList) {
1762                 count = _objectList.Count;
1763
1764                 for (int i = 0; i < count; ++i) {
1765                     DbConnectionInternal obj = _objectList[i];
1766
1767                     if (null != obj) {
1768                         bool locked = false;
1769
1770                         try {
1771                             Monitor.TryEnter(obj, ref locked);
1772
1773                             if (locked) { // avoid race condition with PrePush/PostPop and IsEmancipated
1774                                 if (obj.IsEmancipated) {
1775                                     // Inside the lock, we want to do as little
1776                                     // as possible, so we simply mark the object
1777                                     // as being in the pool, but hand it off to
1778                                     // an out of pool list to be deactivated,
1779                                     // etc.
1780                                     obj.PrePush(null);
1781                                     reclaimedObjects.Add(obj);
1782                                 }
1783                             }
1784                         }
1785                         finally {
1786                             if (locked)
1787                                 Monitor.Exit(obj);
1788                         }
1789                     }
1790                 }
1791             }
1792
1793             // NOTE: we don't want to call DeactivateObject while we're locked,
1794             // because it can make roundtrips to the server and this will block
1795             // object creation in the pooler.  Instead, we queue things we need
1796             // to do up, and process them outside the lock.
1797             count = reclaimedObjects.Count;
1798
1799             for (int i = 0; i < count; ++i) {
1800                 DbConnectionInternal obj = reclaimedObjects[i];
1801
1802                 Bid.PoolerTrace("<prov.DbConnectionPool.ReclaimEmancipatedObjects|RES|CPOOL> %d#, Connection %d#, Reclaiming.\n", ObjectID, obj.ObjectID);
1803 #if !MOBILE
1804                 PerformanceCounters.NumberOfReclaimedConnections.Increment();
1805 #endif
1806
1807                 emancipatedObjectFound = true;
1808
1809                 obj.DetachCurrentTransactionIfEnded();
1810                 DeactivateObject(obj);
1811             }
1812             return emancipatedObjectFound;
1813         }
1814
1815         internal void Startup() {
1816             Bid.PoolerTrace("<prov.DbConnectionPool.Startup|RES|INFO|CPOOL> %d#, CleanupWait=%d\n", ObjectID, _cleanupWait);
1817
1818             _cleanupTimer = CreateCleanupTimer();
1819             if (NeedToReplenish) {
1820                 QueuePoolCreateRequest();
1821             }
1822         }
1823
1824         internal void Shutdown() {
1825             Bid.PoolerTrace("<prov.DbConnectionPool.Shutdown|RES|INFO|CPOOL> %d#\n", ObjectID);
1826
1827             _state = State.ShuttingDown;
1828
1829             // deactivate timer callbacks
1830             Timer t = _cleanupTimer;
1831             _cleanupTimer = null;
1832             if (null != t) {
1833                 t.Dispose();
1834             }
1835         }
1836
1837         // TransactionEnded merely provides the plumbing for DbConnectionInternal to access the transacted pool
1838         //   that is implemented inside DbConnectionPool. This method's counterpart (PutTransactedObject) should
1839         //   only be called from DbConnectionPool.DeactivateObject and thus the plumbing to provide access to
1840         //   other objects is unnecessary (hence the asymmetry of Ended but no Begin)
1841         internal void TransactionEnded(SysTx.Transaction transaction, DbConnectionInternal transactedObject) {
1842             Debug.Assert(null != transaction, "null transaction?");
1843             Debug.Assert(null != transactedObject, "null transactedObject?");
1844             // Note: connection may still be associated with transaction due to Explicit Unbinding requirement.
1845
1846             Bid.PoolerTrace("<prov.DbConnectionPool.TransactionEnded|RES|CPOOL> %d#, Transaction %d#, Connection %d#, Transaction Completed\n", ObjectID, transaction.GetHashCode(), transactedObject.ObjectID);
1847
1848             // called by the internal connection when it get's told that the
1849             // transaction is completed.  We tell the transacted pool to remove
1850             // the connection from it's list, then we put the connection back in
1851             // general circulation.
1852
1853             TransactedConnectionPool transactedConnectionPool = _transactedConnectionPool;
1854             if (null != transactedConnectionPool) {
1855                 transactedConnectionPool.TransactionEnded(transaction, transactedObject);
1856             }
1857         }
1858
1859
1860         private DbConnectionInternal UserCreateRequest(DbConnection owningObject, DbConnectionOptions userOptions, DbConnectionInternal oldConnection = null) {
1861             // called by user when they were not able to obtain a free object but
1862             // instead obtained creation mutex
1863
1864             DbConnectionInternal obj = null;
1865             if (ErrorOccurred) {
1866                 throw TryCloneCachedException();
1867             }
1868             else {
1869                  if ((oldConnection != null) || (Count < MaxPoolSize) || (0 == MaxPoolSize)) {
1870                     // If we have an odd number of total objects, reclaim any dead objects.
1871                     // If we did not find any objects to reclaim, create a new one.
1872
1873                     //
1874                      if ((oldConnection != null) || (Count & 0x1) == 0x1 || !ReclaimEmancipatedObjects())
1875                         obj = CreateObject(owningObject, userOptions, oldConnection);
1876                 }
1877                 return obj;
1878             }
1879         }
1880     }
1881 }