mcs/class/referencesource/System.Data/System/Data/ProviderBase/DbConnectionPool.cs

   1 //------------------------------------------------------------------------------
   2 // <copyright file="DbConnectionPool.cs" company="Microsoft">
   3 //      Copyright (c) Microsoft Corporation.  All rights reserved.
   4 // </copyright>
   5 // <owner current="true" primary="true">[....]</owner>
   6 // <owner current="true" primary="false">[....]</owner>
   7 //------------------------------------------------------------------------------
   8
   9 namespace System.Data.ProviderBase {
  10
  11     using System;
  12     using System.Collections;
  13     using System.Collections.Generic;
  14     using System.Data.Common;
  15     using System.Diagnostics;
  16     using System.Globalization;
  17     using System.Runtime.CompilerServices;
  18     using System.Runtime.ConstrainedExecution;
  19     using System.Runtime.InteropServices;
  20     using System.Security;
  21     using System.Security.Permissions;
  22     using System.Security.Principal;
  23     using System.Threading;
  24     using System.Threading.Tasks;
  25     using SysTx = System.Transactions;
  26     using System.Runtime.Versioning;
  27     using System.Diagnostics.CodeAnalysis;
  28     using System.Collections.Concurrent;
  29
  30     sealed internal class DbConnectionPool {
  31         private enum State {
  32             Initializing,
  33             Running,
  34             ShuttingDown,
  35         }
  36
  37         internal const Bid.ApiGroup PoolerTracePoints = Bid.ApiGroup.Pooling;
  38
  39         // This class is a way to stash our cloned Tx key for later disposal when it's no longer needed.
  40         // We can't get at the key in the dictionary without enumerating entries, so we stash an extra
  41         // copy as part of the value.
  42         sealed private class TransactedConnectionList : List<DbConnectionInternal> {
  43             private SysTx.Transaction _transaction;
  44             internal TransactedConnectionList(int initialAllocation, SysTx.Transaction tx) : base(initialAllocation) {
  45                 _transaction = tx;
  46             }
  47
  48             internal void Dispose() {
  49                 if (null != _transaction) {
  50                     _transaction.Dispose();
  51                 }
  52             }
  53         }
  54
  55         sealed class PendingGetConnection {
  56             public PendingGetConnection(long dueTime, DbConnection owner, TaskCompletionSource<DbConnectionInternal> completion, DbConnectionOptions userOptions) {
  57                 DueTime = dueTime;
  58                 Owner = owner;
  59                 Completion = completion;
  60             }
  61             public long DueTime { get; private set; }
  62             public DbConnection Owner { get; private set; }
  63             public TaskCompletionSource<DbConnectionInternal> Completion { get; private set; }
  64             public DbConnectionOptions UserOptions { get; private set; }
  65         }
  66
  67         sealed private class TransactedConnectionPool
  68         {
  69
  70             Dictionary<SysTx.Transaction, TransactedConnectionList> _transactedCxns;
  71
  72             DbConnectionPool _pool;
  73
  74             private static int _objectTypeCount; // Bid counter
  75             internal readonly int _objectID = System.Threading.Interlocked.Increment(ref _objectTypeCount);
  76
  77             internal TransactedConnectionPool(DbConnectionPool pool)
  78             {
  79                 Debug.Assert(null != pool, "null pool?");
  80
  81                 _pool = pool;
  82                 _transactedCxns = new Dictionary<SysTx.Transaction, TransactedConnectionList> ();
  83
  84                 Bid.PoolerTrace("<prov.DbConnectionPool.TransactedConnectionPool.TransactedConnectionPool|RES|CPOOL> %d#, Constructed for connection pool %d#\n", ObjectID, _pool.ObjectID);
  85             }
  86
  87             internal int ObjectID {
  88                 get {
  89                     return _objectID;
  90                 }
  91             }
  92
  93             internal DbConnectionPool Pool {
  94                 get {
  95                     return _pool;
  96                 }
  97             }
  98
  99             internal DbConnectionInternal GetTransactedObject(SysTx.Transaction transaction)
 100             {
 101                 Debug.Assert(null != transaction, "null transaction?");
 102
 103                 DbConnectionInternal transactedObject = null;
 104
 105                 TransactedConnectionList connections;
 106                 bool txnFound = false;
 107
 108                 lock (_transactedCxns)
 109                 {
 110                     txnFound = _transactedCxns.TryGetValue ( transaction, out connections );
 111                 }
 112
 113                 // NOTE: GetTransactedObject is only used when AutoEnlist = True and the ambient transaction
 114                 //   (Sys.Txns.Txn.Current) is still valid/non-null. This, in turn, means that we don't need
 115                 //   to worry about a pending asynchronous TransactionCompletedEvent to trigger processing in
 116                 //   TransactionEnded below and potentially wipe out the connections list underneath us. It
 117                 //   is similarly alright if a pending addition to the connections list in PutTransactedObject
 118                 //   below is not completed prior to the lock on the connections object here...getting a new
 119                 //   connection is probably better than unnecessarily locking
 120                 if (txnFound)
 121                 {
 122                     Debug.Assert ( connections != null );
 123
 124                     // synchronize multi-threaded access with PutTransactedObject (TransactionEnded should
 125                     //   not be a concern, see comments above)
 126                     lock ( connections )
 127                     {
 128                         int i = connections.Count - 1;
 129                         if (0 <= i)
 130                         {
 131                             transactedObject = connections[i];
 132                             connections.RemoveAt(i);
 133                         }
 134                     }
 135                 }
 136
 137                 if (null != transactedObject) {
 138                     Bid.PoolerTrace("<prov.DbConnectionPool.TransactedConnectionPool.GetTransactedObject|RES|CPOOL> %d#, Transaction %d#, Connection %d#, Popped.\n", ObjectID, transaction.GetHashCode(), transactedObject.ObjectID);
 139                 }
 140                 return transactedObject;
 141             }
 142
 143             internal void PutTransactedObject(SysTx.Transaction transaction, DbConnectionInternal transactedObject) {
 144                 Debug.Assert(null != transaction, "null transaction?");
 145                 Debug.Assert(null != transactedObject, "null transactedObject?");
 146
 147                 TransactedConnectionList connections;
 148                 bool txnFound = false;
 149
 150                 // NOTE: because TransactionEnded is an asynchronous notification, there's no guarantee
 151                 //   around the order in which PutTransactionObject and TransactionEnded are called.
 152
 153                 lock ( _transactedCxns )
 154                 {
 155                     // Check if a transacted pool has been created for this transaction
 156                     if ( txnFound = _transactedCxns.TryGetValue ( transaction, out connections ) )
 157                     {
 158                         Debug.Assert ( connections != null );
 159
 160                         // synchronize multi-threaded access with GetTransactedObject
 161                         lock ( connections )
 162                         {
 163                             Debug.Assert(0 > connections.IndexOf(transactedObject), "adding to pool a second time?");
 164                             Bid.PoolerTrace("<prov.DbConnectionPool.TransactedConnectionPool.PutTransactedObject|RES|CPOOL> %d#, Transaction %d#, Connection %d#, Pushing.\n", ObjectID, transaction.GetHashCode(), transactedObject.ObjectID);
 165
 166                             connections.Add(transactedObject);
 167                         }
 168                     }
 169                 }
 170
 171                 //
 172
 173                 if ( !txnFound )
 174                 {
 175                     // create the transacted pool, making sure to clone the associated transaction
 176                     //   for use as a key in our internal dictionary of transactions and connections
 177                     SysTx.Transaction transactionClone = null;
 178                     TransactedConnectionList newConnections = null;
 179
 180                     try
 181                     {
 182                         transactionClone = transaction.Clone();
 183                         newConnections = new TransactedConnectionList(2, transactionClone); // start with only two connections in the list; most times we won't need that many.
 184
 185                         lock ( _transactedCxns )
 186                         {
 187                             // NOTE: in the interim between the locks on the transacted pool (this) during
 188                             //   execution of this method, another thread (threadB) may have attempted to
 189                             //   add a different connection to the transacted pool under the same
 190                             //   transaction. As a result, threadB may have completed creating the
 191                             //   transacted pool while threadA was processing the above instructions.
 192                             if (txnFound = _transactedCxns.TryGetValue(transaction, out connections))
 193                             {
 194                                 Debug.Assert ( connections != null );
 195
 196                                 // synchronize multi-threaded access with GetTransactedObject
 197                                 lock ( connections )
 198                                 {
 199                                     Debug.Assert(0 > connections.IndexOf(transactedObject), "adding to pool a second time?");
 200                                     Bid.PoolerTrace("<prov.DbConnectionPool.TransactedConnectionPool.PutTransactedObject|RES|CPOOL> %d#, Transaction %d#, Connection %d#, Pushing.\n", ObjectID, transaction.GetHashCode(), transactedObject.ObjectID);
 201
 202                                     connections.Add(transactedObject);
 203                                 }
 204                             }
 205                             else
 206                             {
 207                                 Bid.PoolerTrace("<prov.DbConnectionPool.TransactedConnectionPool.PutTransactedObject|RES|CPOOL> %d#, Transaction %d#, Connection %d#, Adding List to transacted pool.\n", ObjectID, transaction.GetHashCode(), transactedObject.ObjectID);
 208
 209                                 // add the connection/transacted object to the list
 210                                 newConnections.Add ( transactedObject );
 211
 212                                 _transactedCxns.Add(transactionClone, newConnections);
 213                                 transactionClone = null; // we've used it -- don't throw it or the TransactedConnectionList that references it away.
 214                             }
 215                         }
 216                     }
 217                     finally
 218                     {
 219                         if (null != transactionClone)
 220                         {
 221                             if ( newConnections != null )
 222                             {
 223                                 // another thread created the transaction pool and thus the new
 224                                 //   TransactedConnectionList was not used, so dispose of it and
 225                                 //   the transaction clone that it incorporates.
 226                                 newConnections.Dispose();
 227                             }
 228                             else
 229                             {
 230                                 // memory allocation for newConnections failed...clean up unused transactionClone
 231                                 transactionClone.Dispose();
 232                             }
 233                         }
 234                     }
 235                     Bid.PoolerTrace("<prov.DbConnectionPool.TransactedConnectionPool.PutTransactedObject|RES|CPOOL> %d#, Transaction %d#, Connection %d#, Added.\n", ObjectID, transaction.GetHashCode(), transactedObject.ObjectID );
 236                 }
 237
 238 #if !MOBILE
 239                 Pool.PerformanceCounters.NumberOfFreeConnections.Increment();
 240 #endif
 241             }
 242
 243             internal void TransactionEnded(SysTx.Transaction transaction, DbConnectionInternal transactedObject)
 244             {
 245                 Bid.PoolerTrace("<prov.DbConnectionPool.TransactedConnectionPool.TransactionEnded|RES|CPOOL> %d#, Transaction %d#, Connection %d#, Transaction Completed\n", ObjectID, transaction.GetHashCode(), transactedObject.ObjectID);
 246
 247                 TransactedConnectionList connections;
 248                 int entry = -1;
 249
 250                 // NOTE: because TransactionEnded is an asynchronous notification, there's no guarantee
 251                 //   around the order in which PutTransactionObject and TransactionEnded are called. As
 252                 //   such, it is possible that the transaction does not yet have a pool created.
 253
 254                 //
 255
 256
 257
 258                 lock ( _transactedCxns )
 259                 {
 260                     if (_transactedCxns.TryGetValue(transaction, out connections))
 261                     {
 262                         Debug.Assert ( connections != null );
 263
 264                         bool shouldDisposeConnections = false;
 265
 266                         // Lock connections to avoid conflict with GetTransactionObject
 267                         lock (connections)
 268                         {
 269                             entry = connections.IndexOf(transactedObject);
 270
 271                             if ( entry >= 0 )
 272                             {
 273                                 connections.RemoveAt(entry);
 274                             }
 275
 276                             // Once we've completed all the ended notifications, we can
 277                             // safely remove the list from the transacted pool.
 278                             if (0 >= connections.Count)
 279                             {
 280                                 Bid.PoolerTrace("<prov.DbConnectionPool.TransactedConnectionPool.TransactionEnded|RES|CPOOL> %d#, Transaction %d#, Removing List from transacted pool.\n", ObjectID, transaction.GetHashCode());
 281                                 _transactedCxns.Remove(transaction);
 282
 283                                 // we really need to dispose our connection list; it may have
 284                                 // native resources via the tx and GC may not happen soon enough.
 285                                 shouldDisposeConnections = true;
 286                             }
 287                         }
 288
 289                         if (shouldDisposeConnections) {
 290                             connections.Dispose();
 291                         }
 292                     }
 293                     else
 294                     {
 295                         //Debug.Assert ( false, "TransactionCompletedEvent fired before PutTransactedObject put the connection in the transacted pool." );
 296                         Bid.PoolerTrace("<prov.DbConnectionPool.TransactedConnectionPool.TransactionEnded|RES|CPOOL> %d#, Transaction %d#, Connection %d#, Transacted pool not yet created prior to transaction completing. Connection may be leaked.\n", ObjectID, transaction.GetHashCode(), transactedObject.ObjectID );
 297                     }
 298                 }
 299
 300                 // If (and only if) we found the connection in the list of
 301                 // connections, we'll put it back...
 302                 if (0 <= entry)
 303                 {
 304 #if !MOBILE
 305                     Pool.PerformanceCounters.NumberOfFreeConnections.Decrement();
 306 #endif
 307                     Pool.PutObjectFromTransactedPool(transactedObject);
 308                 }
 309             }
 310
 311         }
 312
 313         private sealed class PoolWaitHandles : DbBuffer {
 314
 315             private readonly Semaphore _poolSemaphore;
 316             private readonly ManualResetEvent _errorEvent;
 317
 318             // Using a Mutex requires ThreadAffinity because SQL CLR can swap
 319             // the underlying Win32 thread associated with a managed thread in preemptive mode.
 320             // Using an AutoResetEvent does not have that complication.
 321             private readonly Semaphore _creationSemaphore;
 322
 323             private readonly SafeHandle _poolHandle;
 324             private readonly SafeHandle _errorHandle;
 325             private readonly SafeHandle _creationHandle;
 326
 327             private readonly int _releaseFlags;
 328
 329             [ResourceExposure(ResourceScope.None)] // SxS: this method does not create named objects
 330             [ResourceConsumption(ResourceScope.Machine, ResourceScope.Machine)]
 331             internal PoolWaitHandles() : base(3*IntPtr.Size) {
 332                 bool mustRelease1 = false, mustRelease2 = false, mustRelease3 = false;
 333
 334                 _poolSemaphore = new Semaphore(0, MAX_Q_SIZE);
 335                 _errorEvent = new ManualResetEvent(false);
 336                 _creationSemaphore = new Semaphore(1, 1);
 337
 338                 RuntimeHelpers.PrepareConstrainedRegions();
 339                 try {
 340                     // because SafeWaitHandle doesn't have reliability contract
 341                     _poolHandle     = _poolSemaphore.SafeWaitHandle;
 342                     _errorHandle    = _errorEvent.SafeWaitHandle;
 343                     _creationHandle = _creationSemaphore.SafeWaitHandle;
 344
 345                     _poolHandle.DangerousAddRef(ref mustRelease1);
 346                     _errorHandle.DangerousAddRef(ref mustRelease2);
 347                     _creationHandle.DangerousAddRef(ref mustRelease3);
 348
 349                     Debug.Assert(0 == SEMAPHORE_HANDLE, "SEMAPHORE_HANDLE");
 350                     Debug.Assert(1 == ERROR_HANDLE, "ERROR_HANDLE");
 351                     Debug.Assert(2 == CREATION_HANDLE, "CREATION_HANDLE");
 352
 353                     WriteIntPtr(SEMAPHORE_HANDLE*IntPtr.Size, _poolHandle.DangerousGetHandle());
 354                     WriteIntPtr(ERROR_HANDLE*IntPtr.Size,     _errorHandle.DangerousGetHandle());
 355                     WriteIntPtr(CREATION_HANDLE*IntPtr.Size,  _creationHandle.DangerousGetHandle());
 356                 }
 357                 finally {
 358                     if (mustRelease1) {
 359                         _releaseFlags |= 1;
 360                     }
 361                     if (mustRelease2) {
 362                         _releaseFlags |= 2;
 363                     }
 364                     if (mustRelease3) {
 365                         _releaseFlags |= 4;
 366                     }
 367                 }
 368             }
 369
 370             internal SafeHandle CreationHandle {
 371                 [ReliabilityContract(Consistency.WillNotCorruptState, Cer.Success)]
 372                 get { return _creationHandle; }
 373             }
 374
 375             internal Semaphore CreationSemaphore {
 376                 get { return _creationSemaphore; }
 377             }
 378
 379             internal ManualResetEvent ErrorEvent {
 380                 get { return _errorEvent; }
 381             }
 382
 383             internal Semaphore PoolSemaphore {
 384                 get { return _poolSemaphore; }
 385             }
 386
 387             protected override bool ReleaseHandle() {
 388                 // NOTE: The SafeHandle class guarantees this will be called exactly once.
 389                 // we know we can touch these other managed objects because of our original DangerousAddRef
 390                 if (0 != (1 & _releaseFlags)) {
 391                     _poolHandle.DangerousRelease();
 392                 }
 393                 if (0 != (2 & _releaseFlags)) {
 394                     _errorHandle.DangerousRelease();
 395                 }
 396                 if (0 != (4 & _releaseFlags)) {
 397                     _creationHandle.DangerousRelease();
 398                 }
 399                 return base.ReleaseHandle();
 400             }
 401         }
 402
 403         private const int MAX_Q_SIZE    = (int)0x00100000;
 404
 405         // The order of these is important; we want the WaitAny call to be signaled
 406         // for a free object before a creation signal.  Only the index first signaled
 407         // object is returned from the WaitAny call.
 408         private const int SEMAPHORE_HANDLE = (int)0x0;
 409         private const int ERROR_HANDLE     = (int)0x1;
 410         private const int CREATION_HANDLE  = (int)0x2;
 411         private const int BOGUS_HANDLE     = (int)0x3;
 412
 413         private const int WAIT_OBJECT_0 = 0;
 414         private const int WAIT_TIMEOUT   = (int)0x102;
 415         private const int WAIT_ABANDONED = (int)0x80;
 416         private const int WAIT_FAILED    = -1;
 417
 418         private const int ERROR_WAIT_DEFAULT = 5 * 1000; // 5 seconds
 419
 420         // we do want a testable, repeatable set of generated random numbers
 421         private static readonly Random _random = new Random(5101977); // Value obtained from Dave Driver
 422
 423         private readonly int              _cleanupWait;
 424         private readonly DbConnectionPoolIdentity _identity;
 425
 426         private readonly DbConnectionFactory          _connectionFactory;
 427         private readonly DbConnectionPoolGroup        _connectionPoolGroup;
 428         private readonly DbConnectionPoolGroupOptions _connectionPoolGroupOptions;
 429         private          DbConnectionPoolProviderInfo _connectionPoolProviderInfo;
 430
 431         private State                     _state;
 432
 433         private readonly ConcurrentStack<DbConnectionInternal> _stackOld = new ConcurrentStack<DbConnectionInternal>();
 434         private readonly ConcurrentStack<DbConnectionInternal> _stackNew = new ConcurrentStack<DbConnectionInternal>();
 435
 436         private readonly ConcurrentQueue<PendingGetConnection> _pendingOpens = new ConcurrentQueue<PendingGetConnection>();
 437         private int _pendingOpensWaiting = 0;
 438
 439         private readonly WaitCallback     _poolCreateRequest;
 440
 441         private int                       _waitCount;
 442         private readonly PoolWaitHandles  _waitHandles;
 443
 444         private Exception                 _resError;
 445         private volatile bool             _errorOccurred;
 446
 447         private int                       _errorWait;
 448         private Timer                     _errorTimer;
 449
 450         private Timer                     _cleanupTimer;
 451
 452         private readonly TransactedConnectionPool _transactedConnectionPool;
 453
 454         private readonly List<DbConnectionInternal> _objectList;
 455         private int                       _totalObjects;
 456
 457         private static int _objectTypeCount; // Bid counter
 458         internal readonly int _objectID = System.Threading.Interlocked.Increment(ref _objectTypeCount);
 459
 460         // only created by DbConnectionPoolGroup.GetConnectionPool
 461         internal DbConnectionPool(
 462                             DbConnectionFactory connectionFactory,
 463                             DbConnectionPoolGroup connectionPoolGroup,
 464                             DbConnectionPoolIdentity identity,
 465                             DbConnectionPoolProviderInfo connectionPoolProviderInfo ) {
 466             Debug.Assert(ADP.IsWindowsNT, "Attempting to construct a connection pool on Win9x?");
 467             Debug.Assert(null != connectionPoolGroup, "null connectionPoolGroup");
 468
 469             if ((null != identity) && identity.IsRestricted) {
 470                 throw ADP.InternalError(ADP.InternalErrorCode.AttemptingToPoolOnRestrictedToken);
 471             }
 472
 473             _state= State.Initializing;
 474
 475             lock(_random) { // Random.Next is not thread-safe
 476                 _cleanupWait = _random.Next(12, 24)*10*1000; // 2-4 minutes in 10 sec intervals, WebData 103603
 477             }
 478
 479             _connectionFactory = connectionFactory;
 480             _connectionPoolGroup = connectionPoolGroup;
 481             _connectionPoolGroupOptions = connectionPoolGroup.PoolGroupOptions;
 482             _connectionPoolProviderInfo = connectionPoolProviderInfo;
 483             _identity = identity;
 484
 485             _waitHandles = new PoolWaitHandles();
 486
 487             _errorWait      = ERROR_WAIT_DEFAULT;
 488             _errorTimer     = null;  // No error yet.
 489
 490             _objectList     = new List<DbConnectionInternal>(MaxPoolSize);
 491
 492             if(ADP.IsPlatformNT5) {
 493                 _transactedConnectionPool = new TransactedConnectionPool(this);
 494             }
 495
 496             _poolCreateRequest = new WaitCallback(PoolCreateRequest); // used by CleanupCallback
 497             _state = State.Running;
 498
 499             Bid.PoolerTrace("<prov.DbConnectionPool.DbConnectionPool|RES|CPOOL> %d#, Constructed.\n", ObjectID);
 500
 501             //_cleanupTimer & QueuePoolCreateRequest is delayed until DbConnectionPoolGroup calls
 502             // StartBackgroundCallbacks after pool is actually in the collection
 503         }
 504
 505         private int CreationTimeout {
 506             get { return PoolGroupOptions.CreationTimeout; }
 507         }
 508
 509         internal int Count {
 510             get { return _totalObjects; }
 511         }
 512
 513         internal DbConnectionFactory ConnectionFactory {
 514             get { return _connectionFactory; }
 515         }
 516
 517         internal bool ErrorOccurred {
 518             get { return _errorOccurred; }
 519         }
 520
 521         private bool HasTransactionAffinity {
 522             get { return PoolGroupOptions.HasTransactionAffinity; }
 523         }
 524
 525         internal TimeSpan LoadBalanceTimeout {
 526             get { return PoolGroupOptions.LoadBalanceTimeout; }
 527         }
 528
 529         private bool NeedToReplenish {
 530             get {
 531                 if (State.Running != _state) // SQL BU DT 364595 - don't allow connection create when not running.
 532                     return false;
 533
 534                 int totalObjects = Count;
 535
 536                 if (totalObjects >= MaxPoolSize)
 537                     return false;
 538
 539                 if (totalObjects < MinPoolSize)
 540                     return true;
 541
 542                 int freeObjects        = (_stackNew.Count + _stackOld.Count);
 543                 int waitingRequests    = _waitCount;
 544                 bool needToReplenish = (freeObjects < waitingRequests) || ((freeObjects == waitingRequests) && (totalObjects > 1));
 545
 546                 return needToReplenish;
 547             }
 548         }
 549
 550         internal DbConnectionPoolIdentity Identity {
 551             get { return _identity; }
 552         }
 553
 554         internal bool IsRunning {
 555             get { return State.Running == _state; }
 556         }
 557
 558         private int MaxPoolSize {
 559             get { return PoolGroupOptions.MaxPoolSize; }
 560         }
 561
 562         private int MinPoolSize {
 563             get { return PoolGroupOptions.MinPoolSize; }
 564         }
 565
 566         internal int ObjectID {
 567             get {
 568                 return _objectID;
 569             }
 570         }
 571
 572         internal DbConnectionPoolCounters PerformanceCounters {
 573             get { return _connectionFactory.PerformanceCounters; }
 574         }
 575
 576         internal DbConnectionPoolGroup PoolGroup {
 577             get { return _connectionPoolGroup; }
 578         }
 579
 580         internal DbConnectionPoolGroupOptions PoolGroupOptions {
 581             get { return _connectionPoolGroupOptions; }
 582         }
 583
 584         internal DbConnectionPoolProviderInfo ProviderInfo {
 585             get { return _connectionPoolProviderInfo; }
 586         }
 587
 588         internal bool UseLoadBalancing {
 589             get { return PoolGroupOptions.UseLoadBalancing; }
 590         }
 591
 592         private bool UsingIntegrateSecurity {
 593             get { return (null != _identity && DbConnectionPoolIdentity.NoIdentity != _identity); }
 594         }
 595
 596         private void CleanupCallback(Object state) {
 597             // Called when the cleanup-timer ticks over.
 598
 599             // This is the automatic prunning method.  Every period, we will
 600             // perform a two-step process:
 601             //
 602             // First, for each free object above MinPoolSize, we will obtain a
 603             // semaphore representing one object and destroy one from old stack.
 604             // We will continue this until we either reach MinPoolSize, we are
 605             // unable to obtain a free object, or we have exhausted all the
 606             // objects on the old stack.
 607             //
 608             // Second we move all free objects on the new stack to the old stack.
 609             // So, every period the objects on the old stack are destroyed and
 610             // the objects on the new stack are pushed to the old stack.  All
 611             // objects that are currently out and in use are not on either stack.
 612             //
 613             // With this logic, objects are pruned from the pool if unused for
 614             // at least one period but not more than two periods.
 615
 616             Bid.PoolerTrace("<prov.DbConnectionPool.CleanupCallback|RES|INFO|CPOOL> %d#\n", ObjectID);
 617
 618             // Destroy free objects that put us above MinPoolSize from old stack.
 619             while(Count > MinPoolSize) { // While above MinPoolSize...
 620
 621                 if (_waitHandles.PoolSemaphore.WaitOne(0, false) /* != WAIT_TIMEOUT */) {
 622                     // We obtained a objects from the semaphore.
 623                     DbConnectionInternal obj;
 624
 625                     if (_stackOld.TryPop(out obj)) {
 626                         Debug.Assert(obj != null, "null connection is not expected");
 627                         // If we obtained one from the old stack, destroy it.
 628 #if !MOBILE
 629                         PerformanceCounters.NumberOfFreeConnections.Decrement();
 630 #endif
 631
 632                         // Transaction roots must survive even aging out (TxEnd event will clean them up).
 633                         bool shouldDestroy = true;
 634                         lock (obj) {    // Lock to prevent race condition window between IsTransactionRoot and shouldDestroy assignment
 635                             if (obj.IsTransactionRoot) {
 636                                 shouldDestroy = false;
 637                             }
 638                         }
 639
 640                         // !!!!!!!!!! WARNING !!!!!!!!!!!!!
 641                         //   ONLY touch obj after lock release if shouldDestroy is false!!!  Otherwise, it may be destroyed
 642                         //   by transaction-end thread!
 643
 644                         // Note that there is a minor race condition between this task and the transaction end event, if the latter runs
 645                         //  between the lock above and the SetInStasis call below. The reslult is that the stasis counter may be
 646                         //  incremented without a corresponding decrement (the transaction end task is normally expected
 647                         //  to decrement, but will only do so if the stasis flag is set when it runs). I've minimized the size
 648                         //  of the window, but we aren't totally eliminating it due to SetInStasis needing to do bid tracing, which
 649                         //  we don't want to do under this lock, if possible. It should be possible to eliminate this race condition with
 650                         //  more substantial re-architecture of the pool, but we don't have the time to do that work for the current release.
 651
 652                         if (shouldDestroy) {
 653                             DestroyObject(obj);
 654                         }
 655                         else {
 656                             obj.SetInStasis();
 657                         }
 658                     }
 659                     else {
 660                         // Else we exhausted the old stack (the object the
 661                         // semaphore represents is on the new stack), so break.
 662                         _waitHandles.PoolSemaphore.Release(1);
 663                         break;
 664                     }
 665                 }
 666                 else {
 667                     break;
 668                 }
 669             }
 670
 671             // Push to the old-stack.  For each free object, move object from
 672             // new stack to old stack.
 673             if(_waitHandles.PoolSemaphore.WaitOne(0, false) /* != WAIT_TIMEOUT */) {
 674                 for(;;) {
 675                     DbConnectionInternal obj;
 676
 677                     if (!_stackNew.TryPop(out obj))
 678                         break;
 679
 680                     Debug.Assert(obj != null, "null connection is not expected");
 681
 682                     Bid.PoolerTrace("<prov.DbConnectionPool.CleanupCallback|RES|INFO|CPOOL> %d#, ChangeStacks=%d#\n", ObjectID, obj.ObjectID);
 683
 684                     Debug.Assert(!obj.IsEmancipated, "pooled object not in pool");
 685                     Debug.Assert(obj.CanBePooled,     "pooled object is not poolable");
 686
 687                     _stackOld.Push(obj);
 688                 }
 689                 _waitHandles.PoolSemaphore.Release(1);
 690             }
 691
 692             // Queue up a request to bring us up to MinPoolSize
 693             QueuePoolCreateRequest();
 694         }
 695
 696         internal void Clear() {
 697             Bid.PoolerTrace("<prov.DbConnectionPool.Clear|RES|CPOOL> %d#, Clearing.\n", ObjectID);
 698
 699             DbConnectionInternal obj;
 700
 701             // First, quickly doom everything.
 702             lock(_objectList) {
 703                 int count = _objectList.Count;
 704
 705                 for (int i = 0; i < count; ++i) {
 706                     obj = _objectList[i];
 707
 708                     if (null != obj) {
 709                         obj.DoNotPoolThisConnection();
 710                     }
 711                 }
 712             }
 713
 714             // Second, dispose of all the free connections.
 715             while (_stackNew.TryPop(out obj)) {
 716                 Debug.Assert(obj != null, "null connection is not expected");
 717 #if !MOBILE
 718                 PerformanceCounters.NumberOfFreeConnections.Decrement();
 719 #endif
 720                 DestroyObject(obj);
 721             }
 722             while (_stackOld.TryPop(out obj)) {
 723                 Debug.Assert(obj != null, "null connection is not expected");
 724 #if !MOBILE
 725                 PerformanceCounters.NumberOfFreeConnections.Decrement();
 726 #endif
 727                 DestroyObject(obj);
 728             }
 729
 730             // Finally, reclaim everything that's emancipated (which, because
 731             // it's been doomed, will cause it to be disposed of as well)
 732             ReclaimEmancipatedObjects();
 733
 734             Bid.PoolerTrace("<prov.DbConnectionPool.Clear|RES|CPOOL> %d#, Cleared.\n", ObjectID);
 735         }
 736
 737         private Timer CreateCleanupTimer() {
 738             return (new Timer(new TimerCallback(this.CleanupCallback), null, _cleanupWait, _cleanupWait));
 739         }
 740
 741         private DbConnectionInternal CreateObject(DbConnection owningObject, DbConnectionOptions userOptions, DbConnectionInternal oldConnection) {
 742             DbConnectionInternal newObj = null;
 743
 744             try {
 745                 newObj = _connectionFactory.CreatePooledConnection(this, owningObject, _connectionPoolGroup.ConnectionOptions, _connectionPoolGroup.PoolKey, userOptions);
 746                 if (null == newObj) {
 747                     throw ADP.InternalError(ADP.InternalErrorCode.CreateObjectReturnedNull);    // CreateObject succeeded, but null object
 748                 }
 749                 if (!newObj.CanBePooled) {
 750                     throw ADP.InternalError(ADP.InternalErrorCode.NewObjectCannotBePooled);        // CreateObject succeeded, but non-poolable object
 751                 }
 752                 newObj.PrePush(null);
 753
 754                 lock (_objectList) {
 755                     if ((oldConnection != null) && (oldConnection.Pool == this)) {
 756                         _objectList.Remove(oldConnection);
 757                     }
 758                     _objectList.Add(newObj);
 759                     _totalObjects = _objectList.Count;
 760 #if !MOBILE
 761                     PerformanceCounters.NumberOfPooledConnections.Increment();   //
 762 #endif
 763                 }
 764
 765                 // If the old connection belonged to another pool, we need to remove it from that
 766                 if (oldConnection != null) {
 767                     var oldConnectionPool = oldConnection.Pool;
 768                     if (oldConnectionPool != null && oldConnectionPool != this) {
 769                         Debug.Assert(oldConnectionPool._state == State.ShuttingDown, "Old connections pool should be shutting down");
 770                         lock (oldConnectionPool._objectList) {
 771                             oldConnectionPool._objectList.Remove(oldConnection);
 772                             oldConnectionPool._totalObjects = oldConnectionPool._objectList.Count;
 773                         }
 774                     }
 775                 }
 776                 Bid.PoolerTrace("<prov.DbConnectionPool.CreateObject|RES|CPOOL> %d#, Connection %d#, Added to pool.\n", ObjectID, newObj.ObjectID);
 777
 778                 // Reset the error wait:
 779                 _errorWait = ERROR_WAIT_DEFAULT;
 780             }
 781             catch(Exception e)  {
 782                 //
 783                 if (!ADP.IsCatchableExceptionType(e)) {
 784                     throw;
 785                 }
 786
 787                 ADP.TraceExceptionForCapture(e);
 788
 789                 newObj = null; // set to null, so we do not return bad new object
 790                 // Failed to create instance
 791                 _resError = e;
 792
 793                 // VSTFDEVDIV 479561: Make sure the timer starts even if ThreadAbort occurs after setting the ErrorEvent.
 794
 795                 // timer allocation has to be done out of CER block
 796                 Timer t = new Timer(new TimerCallback(this.ErrorCallback), null, Timeout.Infinite, Timeout.Infinite);
 797                 bool timerIsNotDisposed;
 798                 RuntimeHelpers.PrepareConstrainedRegions();
 799                 try{} finally {
 800                     _waitHandles.ErrorEvent.Set();
 801                     _errorOccurred = true;
 802
 803                     // Enable the timer.
 804                     // Note that the timer is created to allow periodic invocation. If ThreadAbort occurs in the middle of ErrorCallback,
 805                     // the timer will restart. Otherwise, the timer callback (ErrorCallback) destroys the timer after resetting the error to avoid second callback.
 806                     _errorTimer = t;
 807                     timerIsNotDisposed = t.Change(_errorWait, _errorWait);
 808                 }
 809
 810                 Debug.Assert(timerIsNotDisposed, "ErrorCallback timer has been disposed");
 811
 812                 if (30000 < _errorWait) {
 813                     _errorWait = 60000;
 814                 }
 815                 else {
 816                     _errorWait *= 2;
 817                 }
 818                 throw;
 819             }
 820             return newObj;
 821         }
 822
 823         private void DeactivateObject(DbConnectionInternal obj)
 824         {
 825             Bid.PoolerTrace("<prov.DbConnectionPool.DeactivateObject|RES|CPOOL> %d#, Connection %d#, Deactivating.\n", ObjectID, obj.ObjectID);
 826
 827             obj.DeactivateConnection(); // we presume this operation is safe outside of a lock...
 828
 829             bool returnToGeneralPool = false;
 830             bool destroyObject = false;
 831             bool rootTxn = false;
 832
 833             if ( obj.IsConnectionDoomed )
 834             {
 835                 // the object is not fit for reuse -- just dispose of it.
 836                 destroyObject = true;
 837             }
 838             else
 839             {
 840                 // NOTE: constructor should ensure that current state cannot be State.Initializing, so it can only
 841                 //   be State.Running or State.ShuttingDown
 842                 Debug.Assert ( _state == State.Running || _state == State.ShuttingDown );
 843
 844                 lock (obj)
 845                 {
 846                     // A connection with a delegated transaction cannot currently
 847                     // be returned to a different customer until the transaction
 848                     // actually completes, so we send it into Stasis -- the SysTx
 849                     // transaction object will ensure that it is owned (not lost),
 850                     // and it will be certain to put it back into the pool.
 851
 852                     if ( _state == State.ShuttingDown )
 853                     {
 854                         if ( obj.IsTransactionRoot )
 855                         {
 856                             // SQLHotfix# 50003503 - connections that are affiliated with a
 857                             //   root transaction and that also happen to be in a connection
 858                             //   pool that is being shutdown need to be put in stasis so that
 859                             //   the root transaction isn't effectively orphaned with no
 860                             //   means to promote itself to a full delegated transaction or
 861                             //   Commit or Rollback
 862                             obj.SetInStasis();
 863                             rootTxn = true;
 864                         }
 865                         else
 866                         {
 867                             // connection is being closed and the pool has been marked as shutting
 868                             //   down, so destroy this object.
 869                             destroyObject = true;
 870                         }
 871                     }
 872                     else
 873                     {
 874                         if ( obj.IsNonPoolableTransactionRoot )
 875                         {
 876                             obj.SetInStasis();
 877                             rootTxn = true;
 878                         }
 879                         else if ( obj.CanBePooled )
 880                         {
 881                             // We must put this connection into the transacted pool
 882                             // while inside a lock to prevent a race condition with
 883                             // the transaction asyncronously completing on a second
 884                             // thread.
 885
 886                             SysTx.Transaction transaction = obj.EnlistedTransaction;
 887                             if (null != transaction)
 888                             {
 889                                 // NOTE: we're not locking on _state, so it's possible that its
 890                                 //   value could change between the conditional check and here.
 891                                 //   Although perhaps not ideal, this is OK because the
 892                                 //   DelegatedTransactionEnded event will clean up the
 893                                 //   connection appropriately regardless of the pool state.
 894                                 Debug.Assert ( _transactedConnectionPool != null, "Transacted connection pool was not expected to be null.");
 895                                 _transactedConnectionPool.PutTransactedObject(transaction, obj);
 896                                 rootTxn = true;
 897                             }
 898                             else
 899                             {
 900                                 // return to general pool
 901                                 returnToGeneralPool = true;
 902                             }
 903                         }
 904                         else
 905                         {
 906                             if ( obj.IsTransactionRoot && !obj.IsConnectionDoomed )
 907                             {
 908                                 // SQLHotfix# 50003503 - if the object cannot be pooled but is a transaction
 909                                 //   root, then we must have hit one of two race conditions:
 910                                 //       1) PruneConnectionPoolGroups shutdown the pool and marked this connection
 911                                 //          as non-poolable while we were processing within this lock
 912                                 //       2) The LoadBalancingTimeout expired on this connection and marked this
 913                                 //          connection as DoNotPool.
 914                                 //
 915                                 //   This connection needs to be put in stasis so that the root transaction isn't
 916                                 //   effectively orphaned with no means to promote itself to a full delegated
 917                                 //   transaction or Commit or Rollback
 918                                 obj.SetInStasis();
 919                                 rootTxn = true;
 920                             }
 921                             else
 922                             {
 923                                 // object is not fit for reuse -- just dispose of it
 924                                 destroyObject = true;
 925                             }
 926                         }
 927                     }
 928                 }
 929             }
 930
 931             if (returnToGeneralPool)
 932             {
 933                 // Only push the connection into the general pool if we didn't
 934                 //   already push it onto the transacted pool, put it into stasis,
 935                 //   or want to destroy it.
 936                 Debug.Assert ( destroyObject == false );
 937                 PutNewObject(obj);
 938             }
 939             else if ( destroyObject )
 940             {
 941                 // VSTFDEVDIV# 479556 - connections that have been marked as no longer
 942                 //   poolable (e.g. exceeded their connection lifetime) are not, in fact,
 943                 //   returned to the general pool
 944                 DestroyObject(obj);
 945                 QueuePoolCreateRequest();
 946             }
 947
 948             //-------------------------------------------------------------------------------------
 949             // postcondition
 950
 951             // ensure that the connection was processed
 952             Debug.Assert ( rootTxn == true || returnToGeneralPool == true || destroyObject == true );
 953
 954             //
 955         }
 956
 957         internal void DestroyObject(DbConnectionInternal obj) {
 958             // A connection with a delegated transaction cannot be disposed of
 959             // until the delegated transaction has actually completed.  Instead,
 960             // we simply leave it alone; when the transaction completes, it will
 961             // come back through PutObjectFromTransactedPool, which will call us
 962             // again.
 963             if (obj.IsTxRootWaitingForTxEnd) {
 964                 Bid.PoolerTrace("<prov.DbConnectionPool.DestroyObject|RES|CPOOL> %d#, Connection %d#, Has Delegated Transaction, waiting to Dispose.\n", ObjectID, obj.ObjectID);
 965             }
 966             else {
 967                 Bid.PoolerTrace("<prov.DbConnectionPool.DestroyObject|RES|CPOOL> %d#, Connection %d#, Removing from pool.\n", ObjectID, obj.ObjectID);
 968
 969                 bool removed = false;
 970                 lock (_objectList) {
 971                     removed = _objectList.Remove(obj);
 972                     Debug.Assert(removed, "attempt to DestroyObject not in list");
 973                     _totalObjects = _objectList.Count;
 974                 }
 975
 976                 if (removed) {
 977                     Bid.PoolerTrace("<prov.DbConnectionPool.DestroyObject|RES|CPOOL> %d#, Connection %d#, Removed from pool.\n", ObjectID, obj.ObjectID);
 978 #if !MOBILE
 979                     PerformanceCounters.NumberOfPooledConnections.Decrement();
 980 #endif
 981                 }
 982                 obj.Dispose();
 983
 984                 Bid.PoolerTrace("<prov.DbConnectionPool.DestroyObject|RES|CPOOL> %d#, Connection %d#, Disposed.\n", ObjectID, obj.ObjectID);
 985 #if !MOBILE
 986                 PerformanceCounters.HardDisconnectsPerSecond.Increment();
 987 #endif
 988             }
 989         }
 990
 991         private void ErrorCallback(Object state) {
 992             Bid.PoolerTrace("<prov.DbConnectionPool.ErrorCallback|RES|CPOOL> %d#, Resetting Error handling.\n", ObjectID);
 993
 994             _errorOccurred = false;
 995             _waitHandles.ErrorEvent.Reset();
 996
 997             // the error state is cleaned, destroy the timer to avoid periodic invocation
 998             Timer t     = _errorTimer;
 999             _errorTimer = null;
1000             if (t != null) {
1001                 t.Dispose(); // Cancel timer request.
1002             }
1003         }
1004
1005
1006         private Exception TryCloneCachedException()
1007         // Cached exception can be of any type, so is not always cloneable.
1008         // This functions clones SqlException
1009         // OleDb and Odbc connections are not passing throw this code
1010         {
1011             if (_resError==null)
1012                 return null;
1013             if (_resError.GetType()==typeof(SqlClient.SqlException))
1014                 return ((SqlClient.SqlException)_resError).InternalClone();
1015             return _resError;
1016         }
1017
1018         void WaitForPendingOpen() {
1019
1020             Debug.Assert(!Thread.CurrentThread.IsThreadPoolThread, "This thread may block for a long time.  Threadpool threads should not be used.");
1021
1022             PendingGetConnection next;
1023
1024             do {
1025                 bool started = false;
1026
1027                 RuntimeHelpers.PrepareConstrainedRegions();
1028                 try {
1029
1030                     RuntimeHelpers.PrepareConstrainedRegions();
1031                     try { }
1032                     finally {
1033                         started = Interlocked.CompareExchange(ref _pendingOpensWaiting, 1, 0) == 0;
1034                     }
1035
1036                     if (!started) {
1037                         return;
1038                     }
1039
1040                     while (_pendingOpens.TryDequeue(out next)) {
1041
1042                         if (next.Completion.Task.IsCompleted) {
1043                             continue;
1044                         }
1045
1046                         uint delay;
1047                         if (next.DueTime == Timeout.Infinite) {
1048                             delay = unchecked((uint)Timeout.Infinite);
1049                         }
1050                         else {
1051                             delay = (uint)Math.Max(ADP.TimerRemainingMilliseconds(next.DueTime), 0);
1052                         }
1053
1054                         DbConnectionInternal connection = null;
1055                         bool timeout = false;
1056                         Exception caughtException = null;
1057
1058                         RuntimeHelpers.PrepareConstrainedRegions();
1059                         try {
1060 #if DEBUG
1061                             System.Data.SqlClient.TdsParser.ReliabilitySection tdsReliabilitySection = new System.Data.SqlClient.TdsParser.ReliabilitySection();
1062
1063                             RuntimeHelpers.PrepareConstrainedRegions();
1064                             try {
1065                                 tdsReliabilitySection.Start();
1066 #else
1067                             {
1068 #endif //DEBUG
1069
1070                                 bool allowCreate = true;
1071                                 bool onlyOneCheckConnection = false;
1072                                 ADP.SetCurrentTransaction(next.Completion.Task.AsyncState as Transactions.Transaction);
1073                                 timeout = !TryGetConnection(next.Owner, delay, allowCreate, onlyOneCheckConnection, next.UserOptions, out connection);
1074                             }
1075 #if DEBUG
1076                             finally {
1077                                 tdsReliabilitySection.Stop();
1078                             }
1079 #endif //DEBUG
1080                         }
1081                         catch (System.OutOfMemoryException) {
1082                             if (connection != null) { connection.DoomThisConnection(); }
1083                             throw;
1084                         }
1085                         catch (System.StackOverflowException) {
1086                             if (connection != null) { connection.DoomThisConnection(); }
1087                             throw;
1088                         }
1089                         catch (System.Threading.ThreadAbortException) {
1090                             if (connection != null) { connection.DoomThisConnection(); }
1091                             throw;
1092                         }
1093                         catch (Exception e) {
1094                             caughtException = e;
1095                         }
1096
1097                         if (caughtException != null) {
1098                             next.Completion.TrySetException(caughtException);
1099                         }
1100                         else if (timeout) {
1101                             next.Completion.TrySetException(ADP.ExceptionWithStackTrace(ADP.PooledOpenTimeout()));
1102                         }
1103                         else {
1104                             Debug.Assert(connection != null, "connection should never be null in success case");
1105                             if (!next.Completion.TrySetResult(connection)) {
1106                                 // if the completion was cancelled, lets try and get this connection back for the next try
1107                                 PutObject(connection, next.Owner);
1108                             }
1109                         }
1110                     }
1111                 }
1112                 finally {
1113                     if (started) {
1114                         Interlocked.Exchange(ref _pendingOpensWaiting, 0);
1115                     }
1116                 }
1117
1118             } while (_pendingOpens.TryPeek(out next));
1119         }
1120
1121         internal bool TryGetConnection(DbConnection owningObject, TaskCompletionSource<DbConnectionInternal> retry, DbConnectionOptions userOptions, out DbConnectionInternal connection) {
1122
1123             uint waitForMultipleObjectsTimeout = 0;
1124             bool allowCreate = false;
1125
1126             if (retry == null) {
1127                 waitForMultipleObjectsTimeout = (uint)CreationTimeout;
1128
1129                 // VSTFDEVDIV 445531: set the wait timeout to INFINITE (-1) if the SQL connection timeout is 0 (== infinite)
1130                 if (waitForMultipleObjectsTimeout == 0)
1131                     waitForMultipleObjectsTimeout = unchecked((uint)Timeout.Infinite);
1132
1133                 allowCreate = true;
1134             }
1135
1136             if (_state != State.Running) {
1137                 Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, DbConnectionInternal State != Running.\n", ObjectID);
1138                 connection = null;
1139                 return true;
1140             }
1141
1142             bool onlyOneCheckConnection = true;
1143             if (TryGetConnection(owningObject, waitForMultipleObjectsTimeout, allowCreate, onlyOneCheckConnection, userOptions, out connection)) {
1144                 return true;
1145             }
1146             else if (retry == null) {
1147                 // timed out on a [....] call
1148                 return true;
1149             }
1150
1151             var pendingGetConnection =
1152                 new PendingGetConnection(
1153                     CreationTimeout == 0 ? Timeout.Infinite : ADP.TimerCurrent() + ADP.TimerFromSeconds(CreationTimeout/1000),
1154                     owningObject,
1155                     retry,
1156                     userOptions);
1157             _pendingOpens.Enqueue(pendingGetConnection);
1158
1159             // it is better to StartNew too many times than not enough
1160             if (_pendingOpensWaiting == 0) {
1161                 Thread waitOpenThread = new Thread(WaitForPendingOpen);
1162                 waitOpenThread.IsBackground = true;
1163                 waitOpenThread.Start();
1164             }
1165
1166             connection = null;
1167             return false;
1168         }
1169
1170         [SuppressMessage("Microsoft.Reliability", "CA2001:AvoidCallingProblematicMethods")] // copied from Triaged.cs
1171         [ResourceExposure(ResourceScope.None)] // SxS: this method does not expose resources
1172         [ResourceConsumption(ResourceScope.Machine, ResourceScope.Machine)]
1173         private bool TryGetConnection(DbConnection owningObject, uint waitForMultipleObjectsTimeout, bool allowCreate, bool onlyOneCheckConnection, DbConnectionOptions userOptions, out DbConnectionInternal connection) {
1174             DbConnectionInternal obj = null;
1175             SysTx.Transaction transaction = null;
1176
1177 #if !MOBILE
1178             PerformanceCounters.SoftConnectsPerSecond.Increment();
1179 #endif
1180
1181             Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, Getting connection.\n", ObjectID);
1182
1183             // If automatic transaction enlistment is required, then we try to
1184             // get the connection from the transacted connection pool first.
1185             if (HasTransactionAffinity) {
1186                 obj = GetFromTransactedPool(out transaction);
1187             }
1188
1189             if (null == obj) {
1190                 Interlocked.Increment(ref _waitCount);
1191                 uint waitHandleCount = allowCreate ? 3u : 2u;
1192
1193                 do {
1194                     int waitResult = BOGUS_HANDLE;
1195                     int releaseSemaphoreResult = 0;
1196
1197                     bool mustRelease = false;
1198                     int waitForMultipleObjectsExHR = 0;
1199                     RuntimeHelpers.PrepareConstrainedRegions();
1200                     try {
1201                         _waitHandles.DangerousAddRef(ref mustRelease);
1202
1203                         // We absolutely must have the value of waitResult set,
1204                         // or we may leak the mutex in async abort cases.
1205                         RuntimeHelpers.PrepareConstrainedRegions();
1206                         try {
1207                             Debug.Assert(2 == waitHandleCount || 3 == waitHandleCount, "unexpected waithandle count");
1208                         }
1209                         finally {
1210                             waitResult = SafeNativeMethods.WaitForMultipleObjectsEx(waitHandleCount, _waitHandles.DangerousGetHandle(), false, waitForMultipleObjectsTimeout, false);
1211
1212 #if !FULL_AOT_RUNTIME
1213                             // VSTFDEVDIV 479551 - call GetHRForLastWin32Error immediately after after the native call
1214                             if (waitResult == WAIT_FAILED) {
1215                                 waitForMultipleObjectsExHR = Marshal.GetHRForLastWin32Error();
1216                             }
1217 #endif
1218                         }
1219
1220                         // From the WaitAny docs: "If more than one object became signaled during
1221                         // the call, this is the array index of the signaled object with the
1222                         // smallest index value of all the signaled objects."  This is important
1223                         // so that the free object signal will be returned before a creation
1224                         // signal.
1225
1226                         switch (waitResult) {
1227                         case WAIT_TIMEOUT:
1228                             Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, Wait timed out.\n", ObjectID);
1229                             Interlocked.Decrement(ref _waitCount);
1230                             connection = null;
1231                             return false;
1232
1233                         case ERROR_HANDLE:
1234                             // Throw the error that PoolCreateRequest stashed.
1235                             Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, Errors are set.\n", ObjectID);
1236                             Interlocked.Decrement(ref _waitCount);
1237                             throw TryCloneCachedException();
1238
1239                         case CREATION_HANDLE:
1240                             Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, Creating new connection.\n", ObjectID);
1241
1242                             try {
1243                                 obj = UserCreateRequest(owningObject, userOptions);
1244                             }
1245                             catch {
1246                                 if (null == obj) {
1247                                     Interlocked.Decrement(ref _waitCount);
1248                                 }
1249                                 throw;
1250                             }
1251                             finally {
1252                                 // SQLBUDT #386664 - ensure that we release this waiter, regardless
1253                                 // of any exceptions that may be thrown.
1254                                 if (null != obj) {
1255                                     Interlocked.Decrement(ref _waitCount);
1256                                 }
1257                             }
1258
1259                             if (null == obj) {
1260                                 // If we were not able to create an object, check to see if
1261                                 // we reached MaxPoolSize.  If so, we will no longer wait on
1262                                 // the CreationHandle, but instead wait for a free object or
1263                                 // the timeout.
1264
1265                                 //
1266
1267                                 if (Count >= MaxPoolSize && 0 != MaxPoolSize) {
1268                                     if (!ReclaimEmancipatedObjects()) {
1269                                         // modify handle array not to wait on creation mutex anymore
1270                                         Debug.Assert(2 == CREATION_HANDLE, "creation handle changed value");
1271                                         waitHandleCount = 2;
1272                                     }
1273                                 }
1274                             }
1275                             break;
1276
1277                         case SEMAPHORE_HANDLE:
1278                             //
1279                             //    guaranteed available inventory
1280                             //
1281                             Interlocked.Decrement(ref _waitCount);
1282                             obj = GetFromGeneralPool();
1283
1284                             if ((obj != null) && (!obj.IsConnectionAlive())) {
1285                                 Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, Connection %d#, found dead and removed.\n", ObjectID, obj.ObjectID);
1286                                 DestroyObject(obj);
1287                                 obj = null;     // Setting to null in case creating a new object fails
1288
1289                                 if (onlyOneCheckConnection) {
1290                                     if (_waitHandles.CreationSemaphore.WaitOne(unchecked((int)waitForMultipleObjectsTimeout))) {
1291                                         RuntimeHelpers.PrepareConstrainedRegions();
1292                                         try {
1293                                             Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, Creating new connection.\n", ObjectID);
1294                                             obj = UserCreateRequest(owningObject, userOptions);
1295                                         }
1296                                         finally {
1297                                             _waitHandles.CreationSemaphore.Release(1);
1298                                         }
1299                                     }
1300                                     else {
1301                                         // Timeout waiting for creation semaphore - return null
1302                                         Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, Wait timed out.\n", ObjectID);
1303                                         connection = null;
1304                                         return false;
1305                                     }
1306                                 }
1307                             }
1308                             break;
1309
1310                         case WAIT_FAILED:
1311                             Debug.Assert(waitForMultipleObjectsExHR != 0, "WaitForMultipleObjectsEx failed but waitForMultipleObjectsExHR remained 0");
1312                             Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, Wait failed.\n", ObjectID);
1313                             Interlocked.Decrement(ref _waitCount);
1314                             Marshal.ThrowExceptionForHR(waitForMultipleObjectsExHR);
1315                             goto default; // if ThrowExceptionForHR didn't throw for some reason
1316                         case (WAIT_ABANDONED+SEMAPHORE_HANDLE):
1317                             Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, Semaphore handle abandonded.\n", ObjectID);
1318                             Interlocked.Decrement(ref _waitCount);
1319                             throw new AbandonedMutexException(SEMAPHORE_HANDLE,_waitHandles.PoolSemaphore);
1320                         case (WAIT_ABANDONED+ERROR_HANDLE):
1321                             Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, Error handle abandonded.\n", ObjectID);
1322                             Interlocked.Decrement(ref _waitCount);
1323                             throw new AbandonedMutexException(ERROR_HANDLE,_waitHandles.ErrorEvent);
1324                         case (WAIT_ABANDONED+CREATION_HANDLE):
1325                             Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, Creation handle abandoned.\n", ObjectID);
1326                             Interlocked.Decrement(ref _waitCount);
1327                             throw new AbandonedMutexException(CREATION_HANDLE,_waitHandles.CreationSemaphore);
1328                         default:
1329                             Bid.PoolerTrace("<prov.DbConnectionPool.GetConnection|RES|CPOOL> %d#, WaitForMultipleObjects=%d\n", ObjectID, waitResult);
1330                             Interlocked.Decrement(ref _waitCount);
1331                             throw ADP.InternalError(ADP.InternalErrorCode.UnexpectedWaitAnyResult);
1332                         }
1333                     }
1334                     finally {
1335                         if (CREATION_HANDLE == waitResult) {
1336                             int result = SafeNativeMethods.ReleaseSemaphore(_waitHandles.CreationHandle.DangerousGetHandle(), 1, IntPtr.Zero);
1337                             if (0 == result) { // failure case
1338 #if !FULL_AOT_RUNTIME
1339                                 releaseSemaphoreResult = Marshal.GetHRForLastWin32Error();
1340 #endif
1341                             }
1342                         }
1343                         if (mustRelease) {
1344                             _waitHandles.DangerousRelease();
1345                         }
1346                     }
1347                     if (0 != releaseSemaphoreResult) {
1348                         Marshal.ThrowExceptionForHR(releaseSemaphoreResult); // will only throw if (hresult < 0)
1349                     }
1350                 } while (null == obj);
1351             }
1352
1353             if (null != obj)
1354             {
1355                 PrepareConnection(owningObject, obj, transaction);
1356             }
1357
1358             connection = obj;
1359             return true;
1360         }
1361
1362         private void PrepareConnection(DbConnection owningObject, DbConnectionInternal obj, SysTx.Transaction transaction) {
1363             lock (obj)
1364             {   // Protect against Clear and ReclaimEmancipatedObjects, which call IsEmancipated, which is affected by PrePush and PostPop
1365                 obj.PostPop(owningObject);
1366             }
1367             try
1368             {
1369                 obj.ActivateConnection(transaction);
1370             }
1371             catch
1372             {
1373                 // if Activate throws an exception
1374                 // put it back in the pool or have it properly disposed of
1375                 this.PutObject(obj, owningObject);
1376                 throw;
1377             }
1378         }
1379
1380         /// <summary>
1381         /// Creates a new connection to replace an existing connection
1382         /// </summary>
1383         /// <param name="owningObject">Outer connection that currently owns <paramref name="oldConnection"/></param>
1384         /// <param name="userOptions">Options used to create the new connection</param>
1385         /// <param name="oldConnection">Inner connection that will be replaced</param>
1386         /// <returns>A new inner connection that is attached to the <paramref name="owningObject"/></returns>
1387         internal DbConnectionInternal ReplaceConnection(DbConnection owningObject, DbConnectionOptions userOptions, DbConnectionInternal oldConnection) {
1388 #if !MOBILE
1389             PerformanceCounters.SoftConnectsPerSecond.Increment();
1390 #endif
1391             Bid.PoolerTrace("<prov.DbConnectionPool.ReplaceConnection|RES|CPOOL> %d#, replacing connection.\n", ObjectID);
1392
1393             DbConnectionInternal newConnection = UserCreateRequest(owningObject, userOptions, oldConnection);
1394
1395             if (newConnection != null) {
1396                 PrepareConnection(owningObject, newConnection, oldConnection.EnlistedTransaction);
1397                 oldConnection.PrepareForReplaceConnection();
1398                 oldConnection.DeactivateConnection();
1399                 oldConnection.Dispose();
1400             }
1401
1402             return newConnection;
1403         }
1404
1405         private DbConnectionInternal GetFromGeneralPool() {
1406             DbConnectionInternal obj = null;
1407
1408             if (!_stackNew.TryPop(out obj)) {
1409                 if (!_stackOld.TryPop(out obj)) {
1410                     obj = null;
1411                 }
1412                 else {
1413                     Debug.Assert(obj != null, "null connection is not expected");
1414                 }
1415             }
1416             else {
1417                 Debug.Assert(obj != null, "null connection is not expected");
1418             }
1419
1420             // SQLBUDT #356870 -- When another thread is clearing this pool,
1421             // it will remove all connections in this pool which causes the
1422             // following assert to fire, which really mucks up stress against
1423             //  checked bits.  The assert is benign, so we're commenting it out.
1424             //Debug.Assert(obj != null, "GetFromGeneralPool called with nothing in the pool!");
1425
1426             if (null != obj) {
1427                 Bid.PoolerTrace("<prov.DbConnectionPool.GetFromGeneralPool|RES|CPOOL> %d#, Connection %d#, Popped from general pool.\n", ObjectID, obj.ObjectID);
1428 #if !MOBILE
1429                 PerformanceCounters.NumberOfFreeConnections.Decrement();
1430 #endif
1431             }
1432             return(obj);
1433         }
1434
1435         private DbConnectionInternal GetFromTransactedPool(out SysTx.Transaction transaction) {
1436             transaction = ADP.GetCurrentTransaction();
1437             DbConnectionInternal obj = null;
1438
1439             if (null != transaction && null != _transactedConnectionPool) {
1440                 obj = _transactedConnectionPool.GetTransactedObject(transaction);
1441
1442                 if (null != obj) {
1443                     Bid.PoolerTrace("<prov.DbConnectionPool.GetFromTransactedPool|RES|CPOOL> %d#, Connection %d#, Popped from transacted pool.\n", ObjectID, obj.ObjectID);
1444 #if !MOBILE
1445                     PerformanceCounters.NumberOfFreeConnections.Decrement();
1446 #endif
1447
1448                     if (obj.IsTransactionRoot) {
1449                         try {
1450                             obj.IsConnectionAlive(true);
1451                         }
1452                         catch {
1453                             Bid.PoolerTrace("<prov.DbConnectionPool.GetFromTransactedPool|RES|CPOOL> %d#, Connection %d#, found dead and removed.\n", ObjectID, obj.ObjectID);
1454                             DestroyObject(obj);
1455                             throw;
1456                         }
1457                     }
1458                     else if (!obj.IsConnectionAlive()) {
1459                         Bid.PoolerTrace("<prov.DbConnectionPool.GetFromTransactedPool|RES|CPOOL> %d#, Connection %d#, found dead and removed.\n", ObjectID, obj.ObjectID);
1460                         DestroyObject(obj);
1461                         obj = null;
1462                     }
1463                 }
1464             }
1465             return obj;
1466         }
1467
1468         [ResourceExposure(ResourceScope.None)] // SxS: this method does not expose resources
1469         [ResourceConsumption(ResourceScope.Machine, ResourceScope.Machine)]
1470         private void PoolCreateRequest(object state) {
1471             // called by pooler to ensure pool requests are currently being satisfied -
1472             // creation mutex has not been obtained
1473
1474             IntPtr hscp;
1475
1476             Bid.PoolerScopeEnter(out hscp, "<prov.DbConnectionPool.PoolCreateRequest|RES|INFO|CPOOL> %d#\n", ObjectID);
1477
1478             try {
1479                 if (State.Running == _state) {
1480
1481                     // in case WaitForPendingOpen ever failed with no subsequent OpenAsync calls,
1482                     // start it back up again
1483                     if (!_pendingOpens.IsEmpty && _pendingOpensWaiting == 0) {
1484                         Thread waitOpenThread = new Thread(WaitForPendingOpen);
1485                         waitOpenThread.IsBackground = true;
1486                         waitOpenThread.Start();
1487                     }
1488
1489                     // Before creating any new objects, reclaim any released objects that were
1490                     // not closed.
1491                     ReclaimEmancipatedObjects();
1492
1493                     if (!ErrorOccurred) {
1494                         if (NeedToReplenish) {
1495                             // Check to see if pool was created using integrated security and if so, make
1496                             // sure the identity of current user matches that of user that created pool.
1497                             // If it doesn't match, do not create any objects on the ThreadPool thread,
1498                             // since either Open will fail or we will open a object for this pool that does
1499                             // not belong in this pool.  The side effect of this is that if using integrated
1500                             // security min pool size cannot be guaranteed.
1501                             if (UsingIntegrateSecurity && !_identity.Equals(DbConnectionPoolIdentity.GetCurrent())) {
1502                                 return;
1503                             }
1504                             bool mustRelease = false;
1505                             int waitResult = BOGUS_HANDLE;
1506                             uint timeout = (uint)CreationTimeout;
1507
1508                             RuntimeHelpers.PrepareConstrainedRegions();
1509                             try {
1510                                 _waitHandles.DangerousAddRef(ref mustRelease);
1511
1512                                 // Obtain creation mutex so we're the only one creating objects
1513                                 // and we must have the wait result
1514                                 RuntimeHelpers.PrepareConstrainedRegions();
1515                                 try { } finally {
1516                                     waitResult = SafeNativeMethods.WaitForSingleObjectEx(_waitHandles.CreationHandle.DangerousGetHandle(), timeout, false);
1517                                 }
1518                                 if (WAIT_OBJECT_0 == waitResult) {
1519                                     DbConnectionInternal newObj;
1520
1521                                     // Check ErrorOccurred again after obtaining mutex
1522                                     if (!ErrorOccurred) {
1523                                         while (NeedToReplenish) {
1524                                             // Don't specify any user options because there is no outer connection associated with the new connection
1525                                             newObj = CreateObject(owningObject: null, userOptions: null, oldConnection: null);
1526
1527                                             // We do not need to check error flag here, since we know if
1528                                             // CreateObject returned null, we are in error case.
1529                                             if (null != newObj) {
1530                                                 PutNewObject(newObj);
1531                                             }
1532                                             else {
1533                                                 break;
1534                                             }
1535                                         }
1536                                     }
1537                                 }
1538                                 else if (WAIT_TIMEOUT == waitResult) {
1539                                     // do not wait forever and potential block this worker thread
1540                                     // instead wait for a period of time and just requeue to try again
1541                                     QueuePoolCreateRequest();
1542                                 }
1543                                 else {
1544                                     // trace waitResult and ignore the failure
1545                                     Bid.PoolerTrace("<prov.DbConnectionPool.PoolCreateRequest|RES|CPOOL> %d#, PoolCreateRequest called WaitForSingleObject failed %d", ObjectID, waitResult);
1546                                 }
1547                             }
1548                             catch (Exception e) {
1549                                 //
1550                                 if (!ADP.IsCatchableExceptionType(e)) {
1551                                     throw;
1552                                 }
1553
1554                                 // Now that CreateObject can throw, we need to catch the exception and discard it.
1555                                 // There is no further action we can take beyond tracing.  The error will be
1556                                 // thrown to the user the next time they request a connection.
1557                                 Bid.PoolerTrace("<prov.DbConnectionPool.PoolCreateRequest|RES|CPOOL> %d#, PoolCreateRequest called CreateConnection which threw an exception: %ls", ObjectID, e);
1558                             }
1559                             finally {
1560                                 if (WAIT_OBJECT_0 == waitResult) {
1561                                     // reuse waitResult and ignore its value
1562                                     waitResult = SafeNativeMethods.ReleaseSemaphore(_waitHandles.CreationHandle.DangerousGetHandle(), 1, IntPtr.Zero);
1563                                 }
1564                                 if (mustRelease) {
1565                                     _waitHandles.DangerousRelease();
1566                                 }
1567                             }
1568                         }
1569                     }
1570                 }
1571             }
1572             finally {
1573                 Bid.ScopeLeave(ref hscp);
1574             }
1575         }
1576
1577         internal void PutNewObject(DbConnectionInternal obj) {
1578             Debug.Assert(null != obj,        "why are we adding a null object to the pool?");
1579
1580             // VSTFDEVDIV 742887 - When another thread is clearing this pool, it
1581             // will set _cannotBePooled for all connections in this pool without prejudice which
1582             // causes the following assert to fire, which really mucks up stress
1583             // against checked bits.
1584             // Debug.Assert(obj.CanBePooled,    "non-poolable object in pool");
1585
1586             Bid.PoolerTrace("<prov.DbConnectionPool.PutNewObject|RES|CPOOL> %d#, Connection %d#, Pushing to general pool.\n", ObjectID, obj.ObjectID);
1587
1588             _stackNew.Push(obj);
1589             _waitHandles.PoolSemaphore.Release(1);
1590 #if !MOBILE
1591             PerformanceCounters.NumberOfFreeConnections.Increment();
1592 #endif
1593         }
1594
1595         internal void PutObject(DbConnectionInternal obj, object owningObject) {
1596             Debug.Assert(null != obj, "null obj?");
1597
1598 #if !MOBILE
1599             PerformanceCounters.SoftDisconnectsPerSecond.Increment();
1600 #endif
1601
1602             // Once a connection is closing (which is the state that we're in at
1603             // this point in time) you cannot delegate a transaction to or enlist
1604             // a transaction in it, so we can correctly presume that if there was
1605             // not a delegated or enlisted transaction to start with, that there
1606             // will not be a delegated or enlisted transaction once we leave the
1607             // lock.
1608
1609             lock (obj) {
1610                 // Calling PrePush prevents the object from being reclaimed
1611                 // once we leave the lock, because it sets _pooledCount such
1612                 // that it won't appear to be out of the pool.  What that
1613                 // means, is that we're now responsible for this connection:
1614                 // it won't get reclaimed if we drop the ball somewhere.
1615                 obj.PrePush(owningObject);
1616
1617                 //
1618             }
1619
1620             DeactivateObject(obj);
1621         }
1622
1623         internal void PutObjectFromTransactedPool(DbConnectionInternal obj) {
1624             Debug.Assert(null != obj, "null pooledObject?");
1625             Debug.Assert(obj.EnlistedTransaction == null, "pooledObject is still enlisted?");
1626
1627             // called by the transacted connection pool , once it's removed the
1628             // connection from it's list.  We put the connection back in general
1629             // circulation.
1630
1631             // NOTE: there is no locking required here because if we're in this
1632             // method, we can safely presume that the caller is the only person
1633             // that is using the connection, and that all pre-push logic has been
1634             // done and all transactions are ended.
1635
1636             Bid.PoolerTrace("<prov.DbConnectionPool.PutObjectFromTransactedPool|RES|CPOOL> %d#, Connection %d#, Transaction has ended.\n", ObjectID, obj.ObjectID);
1637
1638             if (_state == State.Running && obj.CanBePooled) {
1639                 PutNewObject(obj);
1640             }
1641             else {
1642                 DestroyObject(obj);
1643                 QueuePoolCreateRequest();
1644             }
1645         }
1646
1647         private void QueuePoolCreateRequest() {
1648             if (State.Running == _state) {
1649                 // Make sure we're at quota by posting a callback to the threadpool.
1650                 ThreadPool.QueueUserWorkItem(_poolCreateRequest);
1651             }
1652         }
1653
1654         private bool ReclaimEmancipatedObjects() {
1655             bool emancipatedObjectFound = false;
1656
1657             Bid.PoolerTrace("<prov.DbConnectionPool.ReclaimEmancipatedObjects|RES|CPOOL> %d#\n", ObjectID);
1658
1659             List<DbConnectionInternal> reclaimedObjects = new List<DbConnectionInternal>();
1660             int count;
1661
1662             lock(_objectList) {
1663                 count = _objectList.Count;
1664
1665                 for (int i = 0; i < count; ++i) {
1666                     DbConnectionInternal obj = _objectList[i];
1667
1668                     if (null != obj) {
1669                         bool locked = false;
1670
1671                         try {
1672                             Monitor.TryEnter(obj, ref locked);
1673
1674                             if (locked) { // avoid race condition with PrePush/PostPop and IsEmancipated
1675                                 if (obj.IsEmancipated) {
1676                                     // Inside the lock, we want to do as little
1677                                     // as possible, so we simply mark the object
1678                                     // as being in the pool, but hand it off to
1679                                     // an out of pool list to be deactivated,
1680                                     // etc.
1681                                     obj.PrePush(null);
1682                                     reclaimedObjects.Add(obj);
1683                                 }
1684                             }
1685                         }
1686                         finally {
1687                             if (locked)
1688                                 Monitor.Exit(obj);
1689                         }
1690                     }
1691                 }
1692             }
1693
1694             // NOTE: we don't want to call DeactivateObject while we're locked,
1695             // because it can make roundtrips to the server and this will block
1696             // object creation in the pooler.  Instead, we queue things we need
1697             // to do up, and process them outside the lock.
1698             count = reclaimedObjects.Count;
1699
1700             for (int i = 0; i < count; ++i) {
1701                 DbConnectionInternal obj = reclaimedObjects[i];
1702
1703                 Bid.PoolerTrace("<prov.DbConnectionPool.ReclaimEmancipatedObjects|RES|CPOOL> %d#, Connection %d#, Reclaiming.\n", ObjectID, obj.ObjectID);
1704 #if !MOBILE
1705                 PerformanceCounters.NumberOfReclaimedConnections.Increment();
1706 #endif
1707
1708                 emancipatedObjectFound = true;
1709
1710                 obj.DetachCurrentTransactionIfEnded();
1711                 DeactivateObject(obj);
1712             }
1713             return emancipatedObjectFound;
1714         }
1715
1716         internal void Startup() {
1717             Bid.PoolerTrace("<prov.DbConnectionPool.Startup|RES|INFO|CPOOL> %d#, CleanupWait=%d\n", ObjectID, _cleanupWait);
1718
1719             _cleanupTimer = CreateCleanupTimer();
1720             if (NeedToReplenish) {
1721                 QueuePoolCreateRequest();
1722             }
1723         }
1724
1725         internal void Shutdown() {
1726             Bid.PoolerTrace("<prov.DbConnectionPool.Shutdown|RES|INFO|CPOOL> %d#\n", ObjectID);
1727
1728             _state = State.ShuttingDown;
1729
1730             // deactivate timer callbacks
1731             Timer t = _cleanupTimer;
1732             _cleanupTimer = null;
1733             if (null != t) {
1734                 t.Dispose();
1735             }
1736         }
1737
1738         // TransactionEnded merely provides the plumbing for DbConnectionInternal to access the transacted pool
1739         //   that is implemented inside DbConnectionPool. This method's counterpart (PutTransactedObject) should
1740         //   only be called from DbConnectionPool.DeactivateObject and thus the plumbing to provide access to
1741         //   other objects is unnecessary (hence the asymmetry of Ended but no Begin)
1742         internal void TransactionEnded(SysTx.Transaction transaction, DbConnectionInternal transactedObject) {
1743             Debug.Assert(null != transaction, "null transaction?");
1744             Debug.Assert(null != transactedObject, "null transactedObject?");
1745             // Note: connection may still be associated with transaction due to Explicit Unbinding requirement.
1746
1747             Bid.PoolerTrace("<prov.DbConnectionPool.TransactionEnded|RES|CPOOL> %d#, Transaction %d#, Connection %d#, Transaction Completed\n", ObjectID, transaction.GetHashCode(), transactedObject.ObjectID);
1748
1749             // called by the internal connection when it get's told that the
1750             // transaction is completed.  We tell the transacted pool to remove
1751             // the connection from it's list, then we put the connection back in
1752             // general circulation.
1753
1754             TransactedConnectionPool transactedConnectionPool = _transactedConnectionPool;
1755             if (null != transactedConnectionPool) {
1756                 transactedConnectionPool.TransactionEnded(transaction, transactedObject);
1757             }
1758         }
1759
1760
1761         private DbConnectionInternal UserCreateRequest(DbConnection owningObject, DbConnectionOptions userOptions, DbConnectionInternal oldConnection = null) {
1762             // called by user when they were not able to obtain a free object but
1763             // instead obtained creation mutex
1764
1765             DbConnectionInternal obj = null;
1766             if (ErrorOccurred) {
1767                 throw TryCloneCachedException();
1768             }
1769             else {
1770                  if ((oldConnection != null) || (Count < MaxPoolSize) || (0 == MaxPoolSize)) {
1771                     // If we have an odd number of total objects, reclaim any dead objects.
1772                     // If we did not find any objects to reclaim, create a new one.
1773
1774                     //
1775                      if ((oldConnection != null) || (Count & 0x1) == 0x1 || !ReclaimEmancipatedObjects())
1776                         obj = CreateObject(owningObject, userOptions, oldConnection);
1777                 }
1778                 return obj;
1779             }
1780         }
1781     }
1782 }