func TestCreateAndRefresh(t *testing.T) { if true { t.Log("We don't currently use peerscanner, so this test is disabled. To reenable, we'll need to delete test distributions to avoid hitting our limit") return } _, counter, err := fdcount.Matching("TCP") if err != nil { t.Fatalf("Unable to get starting fdcount: %v", err) } cfr := getCfr() // Deleting cloudfront distributions is actually quite an involved process. // Fortunately, distributions per se cost us nothing. A separate service // will be implemented to delete test and otherwise unused distributions. name := uuid.NewV4().String() dist, err := CreateDistribution(cfr, name, name+"-grey.flashlightproxy.org", COMMENT) assert.NoError(t, err, "Should be able to create distribution") assert.Equal(t, "InProgress", dist.Status, "New distribution should have Status: \"InProgress\"") assert.Equal(t, dist.Comment, COMMENT, "New distribution should have the comment we've set for it") assert.Equal(t, name, dist.InstanceId, "New distribution should have the right InstanceId") assert.True(t, strings.HasSuffix(dist.Domain, ".cloudfront.net"), "Domain should be a .cloudfront.net subdomain, not '"+dist.Domain+"'") dist.Status = "modified to check it really gets overwritten" err = RefreshStatus(cfr, dist) assert.NoError(t, err, "Should be able to refresh status") // Just check that Status stays a valid one. Checking that it eventually // gets refreshed to "Deployed" would take a few minutes, and thus is out // of the scope of this unit test. assert.Equal(t, "InProgress", dist.Status, "New distribution should have Status: \"InProgress\" even after refreshing right away") assert.NoError(t, counter.AssertDelta(0), "All file descriptors should have been closed") }
func TestClose(t *testing.T) { _, fdc, err := fdcount.Matching("TCP") if err != nil { t.Fatal(err) } l, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatalf("Unable to listen: %s", err) } defer func() { if err := l.Close(); err != nil { t.Fatalf("Unable to close listener: %v", err) } time.Sleep(1 * time.Second) err = fdc.AssertDelta(0) if err != nil { t.Errorf("File descriptors didn't return to original: %s", err) } }() addr := l.Addr().String() conn, err := net.Dial("tcp", addr) if err != nil { t.Fatalf("Unable to dial %s: %s", addr, err) } c := Conn(conn, clientTimeout, func() {}) for i := 0; i < 100; i++ { _ = c.Close() } }
func doTestTLS(buffered bool, t *testing.T) { startServers(t, false) _, counter, err := fdcount.Matching("TCP") if err != nil { t.Fatalf("Unable to get fdcount: %v", err) } conn, err := prepareConn(httpsAddr, buffered, false, t, nil) if err != nil { t.Fatalf("Unable to prepareConn: %s", err) } tlsConn := tls.Client(conn, &tls.Config{ ServerName: "localhost", RootCAs: cert.PoolContainingCert(), }) defer func() { err := conn.Close() assert.Nil(t, err, "Closing conn should succeed") if !assert.NoError(t, counter.AssertDelta(2), "All file descriptors except the connection from proxy to destination site should have been closed") { DumpConnTrace() } }() err = tlsConn.Handshake() if err != nil { t.Fatalf("Unable to handshake: %s", err) } doRequests(tlsConn, t) assert.True(t, destsSent[httpsAddr], "https address wasn't recorded as sent destination") assert.True(t, destsReceived[httpsAddr], "https address wasn't recorded as received destination") }
func TestEnsureRegistered(t *testing.T) { _, counter, err := fdcount.Matching("TCP") if err != nil { t.Fatalf("Unable to get starting fdcount: %v", err) } u := getUtil() // Test with no existing record name, ip := "cfl-test-entry", "127.0.0.1" rec, proxying, err := u.EnsureRegistered(name, ip, nil) if assert.NoError(t, err, "Should be able to register with no record") { assert.NotNil(t, rec, "A new record should have been returned") assert.True(t, proxying, "Proxying (orange cloud) should be on") } // Test with existing record, but not passing it in rec, proxying, err = u.EnsureRegistered(name, ip, nil) if assert.NoError(t, err, "Should be able to register with unspecified existing record") { assert.NotNil(t, rec, "Existing record should have been returned") assert.True(t, proxying, "Proxying (orange cloud) should be on") // Test with existing record, passing it in rec, proxying, err = u.EnsureRegistered(name, ip, rec) if assert.NoError(t, err, "Should be able to register with specified existing record") { assert.NotNil(t, rec, "Existing record should have been returned") assert.True(t, proxying, "Proxying (orange cloud) should be on") } } if rec != nil { err := u.DestroyRecord(rec) assert.NoError(t, err, "Should be able to destroy record") } assert.NoError(t, counter.AssertDelta(0), "All file descriptors should have been closed") }
// This test stimulates a connection leak as seen in // https://github.com/getlantern/lantern/issues/2174. func TestHTTPRedirect(t *testing.T) { startProxy(t, false) client := &http.Client{ Transport: &http.Transport{ Dial: func(network, addr string) (net.Conn, error) { return Dial(addr, &Config{ DialProxy: func(addr string) (net.Conn, error) { return net.Dial("tcp", proxyAddr) }, NewRequest: newRequest, }) }, DisableKeepAlives: true, }, } _, counter, err := fdcount.Matching("TCP") if err != nil { t.Fatalf("Unable to get fdcount: %v", err) } resp, err := client.Head("http://www.facebook.com") if assert.NoError(t, err, "Head request to facebook should have succeeded") { resp.Body.Close() } assert.NoError(t, counter.AssertDelta(2), "All file descriptors except the connection from proxy to destination site should have been closed") }
func TestDialFailure(t *testing.T) { fail := int32(1) dialAttempts := int32(0) addr, err := startTestServer() if err != nil { t.Fatalf("Unable to start test server: %s", err) } _, fdc, err := fdcount.Matching("TCP") if err != nil { t.Fatal(err) } poolSize := 10 p := New(Config{ Size: poolSize, Dial: func() (net.Conn, error) { atomic.AddInt32(&dialAttempts, 1) if fail == int32(1) { return nil, fmt.Errorf("I'm failing intentionally!") } return net.DialTimeout("tcp", addr, 15*time.Millisecond) }, }) // Try to get connection, make sure it fails conn, err := p.Get() if !assert.Error(t, err, "Dialing should have failed") { if err := conn.Close(); err != nil { t.Fatalf("Unable to close connection: %v", err) } } // Wait for fill to run for a while with a failing connection time.Sleep(1 * time.Second) assert.Equal(t, 1, atomic.LoadInt32(&dialAttempts), fmt.Sprintf("There should have been only 1 dial attempt")) assert.NoError(t, fdc.AssertDelta(0), "There should be no additional file descriptors open") // Now make connection succeed and verify that it works atomic.StoreInt32(&fail, 0) time.Sleep(100 * time.Millisecond) connectAndRead(t, p, 1) time.Sleep(fillTime) log.Debug("Testing") assert.NoError(t, fdc.AssertDelta(10), "Pool should have filled") // Now make the connection fail again so that when we stop, we're stopping // while failing (tests a different code path for stopping) atomic.StoreInt32(&fail, 1) time.Sleep(100 * time.Millisecond) p.Close() assert.NoError(t, fdc.AssertDelta(0), "All connections should be closed") }
func TestOKWithInsecureSkipVerify(t *testing.T) { _, fdc, err := fdcount.Matching("TCP") if err != nil { t.Fatal(err) } conn, err := Dial("tcp", ADDR, false, &tls.Config{ InsecureSkipVerify: true, }) assert.NoError(t, err, "Unable to dial") <-receivedServerNames closeAndCountFDs(t, conn, err, fdc) }
func TestNotOKWithServerName(t *testing.T) { _, fdc, err := fdcount.Matching("TCP") if err != nil { t.Fatal(err) } conn, err := Dial("tcp", ADDR, true, nil) assert.Error(t, err, "There should have been a problem dialing") if err != nil { assert.Contains(t, err.Error(), CERTIFICATE_ERROR, "Wrong error on dial") } <-receivedServerNames closeAndCountFDs(t, conn, err, fdc) }
func TestList(t *testing.T) { _, counter, err := fdcount.Matching("TCP") if err != nil { t.Fatalf("Unable to get starting fdcount: %v", err) } cfr := getCfr() dists, err := ListDistributions(cfr) if assert.NoError(t, err, "Should be able to get all distributions") { for _, d := range dists { log.Tracef("%v : %v (%v)", d.InstanceId, d.Domain, d.Status) } assert.True(t, len(dists) > 0, "There should be some distributions") } assert.NoError(t, counter.AssertDelta(0), "All file descriptors should have been closed") }
func TestVariableTimeouts(t *testing.T) { // Timeouts can happen in different places, run a bunch of randomized trials // to try to cover all of them. _, fdc, err := fdcount.Matching("TCP") if err != nil { t.Fatal(err) } doTestTimeout := func(timeout time.Duration) (didTimeout bool) { _, err := DialWithDialer(&net.Dialer{ Timeout: timeout, }, "tcp", ADDR, false, &tls.Config{ RootCAs: cert.PoolContainingCert(), }) if err == nil { return false } else { if neterr, isNetError := err.(net.Error); isNetError { assert.True(t, neterr.Timeout(), "Dial error should be timeout", timeout) } else { t.Fatal(err) } return true } } // The 1000-5000 microseconds limits are arbitrary. In some systems this may be too low/high. // The algorithm will try to adapt if connections succeed and will lower the current limit, // but it won't be allowed to timeout below the established lower boundary. timeoutMin := 1000 timeoutMax := 5000 for i := 0; i < 500; i++ { timeout := rand.Intn(timeoutMax) + 1 didTimeout := doTestTimeout(time.Duration(timeout) * time.Microsecond) if !didTimeout { if timeout < timeoutMin { t.Fatalf("The connection succeeded in an unexpected short time: %d", timeout) } timeoutMax = int(float64(timeoutMax) * 0.75) i-- // repeat the test } } // Wait to give the sockets time to close time.Sleep(1 * time.Second) assert.NoError(t, fdc.AssertDelta(0), "Number of open files should be the same after test as before") }
func doTestPlainText(buffered bool, useHostFn bool, t *testing.T) { var counter *fdcount.Counter var err error startServers(t, useHostFn) err = fdcount.WaitUntilNoneMatch("CLOSE_WAIT", 5*time.Second) if err != nil { t.Fatalf("Unable to wait until no more connections are in CLOSE_WAIT: %v", err) } _, counter, err = fdcount.Matching("TCP") if err != nil { t.Fatalf("Unable to get fdcount: %v", err) } var reportedHost string var reportedHostMutex sync.Mutex onResponse := func(resp *http.Response) { reportedHostMutex.Lock() reportedHost = resp.Header.Get(X_ENPROXY_PROXY_HOST) reportedHostMutex.Unlock() } conn, err := prepareConn(httpAddr, buffered, false, t, onResponse) if err != nil { t.Fatalf("Unable to prepareConn: %s", err) } defer func() { err := conn.Close() assert.Nil(t, err, "Closing conn should succeed") if !assert.NoError(t, counter.AssertDelta(2), "All file descriptors except the connection from proxy to destination site should have been closed") { DumpConnTrace() } }() doRequests(conn, t) assert.Equal(t, 208, bytesReceived, "Wrong number of bytes received") assert.Equal(t, 284, bytesSent, "Wrong number of bytes sent") assert.True(t, destsSent[httpAddr], "http address wasn't recorded as sent destination") assert.True(t, destsReceived[httpAddr], "http address wasn't recorded as received destination") reportedHostMutex.Lock() rh := reportedHost reportedHostMutex.Unlock() assert.Equal(t, "localhost", rh, "Didn't get correct reported host") }
func TestAll(t *testing.T) { _, counter, err := fdcount.Matching("TCP") if err != nil { t.Fatalf("Unable to get starting fdcount: %v", err) } u := getUtil() recs, err := u.GetAllRecords() if assert.NoError(t, err, "Should be able to get all records") { for _, r := range recs { log.Tracef("%v : %v", r.Name, r.Content) } assert.True(t, len(recs) > 0, "There should be some records") } assert.NoError(t, counter.AssertDelta(0), "All file descriptors should have been closed") }
func TestVariableTimeouts(t *testing.T) { // Timeouts can happen in different places, run a bunch of randomized trials // to try to cover all of them. _, fdc, err := fdcount.Matching("TCP") if err != nil { t.Fatal(err) } for i := 0; i < 500; i++ { doTestTimeout(t, time.Duration(rand.Intn(5000)+1)*time.Microsecond) } // Wait to give the sockets time to close time.Sleep(1 * time.Second) assert.NoError(t, fdc.AssertDelta(0), "Number of open files should be the same after test as before") }
func TestDeadlineBeforeTimeout(t *testing.T) { _, fdc, err := fdcount.Matching("TCP") if err != nil { t.Fatal(err) } conn, err := DialWithDialer(&net.Dialer{ Timeout: 500 * time.Second, Deadline: time.Now().Add(5 * time.Microsecond), }, "tcp", ADDR, false, nil) assert.Error(t, err, "There should have been a problem dialing") if err != nil { assert.True(t, err.(net.Error).Timeout(), "Dial error should be timeout") } closeAndCountFDs(t, conn, err, fdc) }
func TestOKWithServerNameAndLongTimeout(t *testing.T) { _, fdc, err := fdcount.Matching("TCP") if err != nil { t.Fatal(err) } conn, err := DialWithDialer(&net.Dialer{ Timeout: 25 * time.Second, }, "tcp", ADDR, true, &tls.Config{ RootCAs: cert.PoolContainingCert(), }) assert.NoError(t, err, "Unable to dial") serverName := <-receivedServerNames assert.Equal(t, "localhost", serverName, "Unexpected ServerName on server") closeAndCountFDs(t, conn, err, fdc) }
func TestOKWithServerName(t *testing.T) { _, fdc, err := fdcount.Matching("TCP") if err != nil { t.Fatal(err) } cwt, err := DialForTimings(new(net.Dialer), "tcp", ADDR, true, &tls.Config{ RootCAs: cert.PoolContainingCert(), }) conn := cwt.Conn assert.NoError(t, err, "Unable to dial") serverName := <-receivedServerNames assert.Equal(t, "localhost", serverName, "Unexpected ServerName on server") assert.NotNil(t, cwt.ResolvedAddr, "Should have resolved addr") closeAndCountFDs(t, conn, err, fdc) }
func TestOKWithoutServerName(t *testing.T) { _, fdc, err := fdcount.Matching("TCP") if err != nil { t.Fatal(err) } config := &tls.Config{ RootCAs: cert.PoolContainingCert(), ServerName: "localhost", // we manually set a ServerName to make sure it doesn't get sent } conn, err := Dial("tcp", ADDR, false, config) assert.NoError(t, err, "Unable to dial") serverName := <-receivedServerNames assert.Empty(t, serverName, "Unexpected ServerName on server") assert.False(t, config.InsecureSkipVerify, "Original config shouldn't have been modified, but it was") closeAndCountFDs(t, conn, err, fdc) }
func TestNotOKWithoutServerName(t *testing.T) { _, fdc, err := fdcount.Matching("TCP") if err != nil { t.Fatal(err) } conn, err := Dial("tcp", ADDR, true, &tls.Config{ ServerName: "localhost", }) assert.Error(t, err, "There should have been a problem dialing") if err != nil { assert.Contains(t, err.Error(), CERTIFICATE_ERROR, "Wrong error on dial") } serverName := <-receivedServerNames assert.Empty(t, serverName, "Unexpected ServerName on server") closeAndCountFDs(t, conn, err, fdc) }
func TestList(t *testing.T) { if true { t.Log("We don't currently use peerscanner, so this test is disabled") return } _, counter, err := fdcount.Matching("TCP") if err != nil { t.Fatalf("Unable to get starting fdcount: %v", err) } cfr := getCfr() dists, err := ListDistributions(cfr) if assert.NoError(t, err, "Should be able to get all distributions") { for _, d := range dists { log.Tracef("%v : %v (%v)\t\t%v", d.InstanceId, d.Domain, d.Status, d.Comment) } assert.True(t, len(dists) > 0, "There should be some distributions") } assert.NoError(t, counter.AssertDelta(0), "All file descriptors should have been closed") }
func TestIdle(t *testing.T) { idleTimeout := 100 * time.Millisecond _, counter, err := fdcount.Matching("TCP") if err != nil { t.Fatalf("Unable to get fdcount: %v", err) } _, err = Dial(httpAddr, &Config{ DialProxy: func(addr string) (net.Conn, error) { return net.Dial("tcp", proxyAddr) }, NewRequest: newRequest, IdleTimeout: idleTimeout, }) if assert.NoError(t, err, "Dialing should have succeeded") { time.Sleep(idleTimeout * 2) assert.NoError(t, counter.AssertDelta(2), "All file descriptors except the connection from proxy to destination site should have been closed") } }
func TestRegister(t *testing.T) { _, counter, err := fdcount.Matching("TCP") if err != nil { t.Fatalf("Unable to get starting fdcount: %v", err) } u := getUtil() // Test with no existing record name, ip := "dsp-test-entry", "127.0.0.1" rec, err := u.Register(name, ip) if assert.NoError(t, err, "Should be able to register with no record") { assert.NotNil(t, rec, "A new record should have been returned") } if rec != nil { err := u.DestroyRecord(rec) assert.NoError(t, err, "Should be able to destroy record") } assert.NoError(t, counter.AssertDelta(0), "All file descriptors should have been closed") }
func TestNotOKWithBadRootCert(t *testing.T) { badCert, err := keyman.LoadCertificateFromPEMBytes([]byte(GoogleInternetAuthority)) if err != nil { t.Fatalf("Unable to load GoogleInternetAuthority cert: %s", err) } _, fdc, err := fdcount.Matching("TCP") if err != nil { t.Fatal(err) } conn, err := Dial("tcp", ADDR, true, &tls.Config{ RootCAs: badCert.PoolContainingCert(), }) assert.Error(t, err, "There should have been a problem dialing") if err != nil { assert.Contains(t, err.Error(), CERTIFICATE_ERROR, "Wrong error on dial") } <-receivedServerNames closeAndCountFDs(t, conn, err, fdc) }
func TestWrite(t *testing.T) { _, fdc, err := fdcount.Matching("TCP") if err != nil { t.Fatal(err) } listenerIdled := int32(0) connIdled := int32(0) l, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatalf("Unable to listen: %s", err) } addr := l.Addr().String() il := Listener(l, serverTimeout, func(conn net.Conn) { atomic.StoreInt32(&listenerIdled, 1) if err := conn.Close(); err != nil { t.Errorf("Unable to close connection: %v", err) } }) defer func() { if err := il.Close(); err != nil { t.Errorf("Unable to close listener: %v", err) } time.Sleep(1 * time.Second) err = fdc.AssertDelta(0) if err != nil { t.Errorf("File descriptors didn't return to original: %s", err) } }() go func() { conn, err := il.Accept() if err != nil { t.Fatalf("Unable to accept: %s", err) } go func() { // Discard data if _, err := io.Copy(ioutil.Discard, conn); err != nil { t.Fatalf("Unable to discard data: %v", err) } }() }() conn, err := net.Dial("tcp", addr) if err != nil { t.Fatalf("Unable to dial %s: %s", addr, err) } conn = &slowConn{orig: conn, targetDuration: slightlyLessThanClientTimeout} c := Conn(conn, clientTimeout, func() { atomic.StoreInt32(&connIdled, 1) if err := conn.Close(); err != nil { t.Fatalf("Unable to close connection: %v", err) } }) // Write messages for i := 0; i < dataLoops; i++ { n, err := c.Write(msg) if err != nil || n != len(msg) { t.Fatalf("Problem writing. n: %d err: %s", n, err) } } // Now write msg with a really short deadline if err := c.SetWriteDeadline(time.Now().Add(1 * time.Nanosecond)); err != nil { t.Fatalf("Unable to set write deadline: %v", err) } _, err = c.Write(msg) if netErr, ok := err.(net.Error); ok { if !netErr.Timeout() { t.Fatalf("Short deadline should have resulted in Timeout, but didn't: %s", err) } } else { t.Fatalf("Short deadline should have resulted in Timeout, but didn't: %s", err) } time.Sleep(slightlyMoreThanClientTimeout) if connIdled == 0 { t.Errorf("Conn failed to idle!") } connTimesOutIn := c.TimesOutIn() if connTimesOutIn > 0 { t.Errorf("TimesOutIn returned bad value, should have been negative, but was: %s", connTimesOutIn) } time.Sleep(9 * slightlyMoreThanClientTimeout) if listenerIdled == 0 { t.Errorf("Listener failed to idle!") } }
func TestRead(t *testing.T) { _, fdc, err := fdcount.Matching("TCP") if err != nil { t.Fatal(err) } listenerIdled := int32(0) connIdled := int32(0) l, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatalf("Unable to listen: %s", err) } il := Listener(l, serverTimeout, func(conn net.Conn) { atomic.StoreInt32(&listenerIdled, 1) if err := conn.Close(); err != nil { t.Fatalf("Unable to close connection: %v", err) } }) defer func() { if err := il.Close(); err != nil { t.Fatalf("Unable to close listener: %v", err) } time.Sleep(1 * time.Second) err = fdc.AssertDelta(0) if err != nil { t.Errorf("File descriptors didn't return to original: %s", err) } }() addr := l.Addr().String() go func() { conn, err := il.Accept() if err != nil { t.Fatalf("Unable to accept: %s", err) } go func() { // Feed data for i := 0; i < dataLoops; i++ { _, err := conn.Write(msg) if err != nil { return } } }() }() conn, err := net.Dial("tcp", addr) if err != nil { t.Fatalf("Unable to dial %s: %s", addr, err) } conn = &slowConn{orig: conn, targetDuration: slightlyLessThanClientTimeout} c := Conn(conn, clientTimeout, func() { atomic.StoreInt32(&connIdled, 1) if err := conn.Close(); err != nil { t.Fatalf("Unable to close connection: %v", err) } }) // Read messages (we use a buffer matching the message size to make sure // that each iterator of the below loop actually has something to read). b := make([]byte, len(msg)) totalN := 0 for i := 0; i < dataLoops; i++ { n, err := c.Read(b) if err != nil { t.Fatalf("Problem reading. Read %d bytes, err: %s", n, err) } totalN += n } if totalN == 0 { t.Fatal("Didn't read any data!") } // Now read with a really short deadline if err := c.SetReadDeadline(time.Now().Add(1 * time.Nanosecond)); err != nil { t.Fatalf("Unable to set read deadline: %v", err) } _, err = c.Read(msg) if netErr, ok := err.(net.Error); ok { if !netErr.Timeout() { t.Fatalf("Short deadline should have resulted in Timeout, but didn't: %s", err) } } else { t.Fatalf("Short deadline should have resulted in net.Error, but didn't: %s", err) } time.Sleep(slightlyMoreThanClientTimeout) if connIdled == 0 { t.Errorf("Conn failed to idle!") } time.Sleep(9 * slightlyMoreThanClientTimeout) if listenerIdled == 0 { t.Errorf("Listener failed to idle!") } }
func TestIt(t *testing.T) { poolSize := 20 addr, err := startTestServer() if err != nil { t.Fatalf("Unable to start test server: %s", err) } _, fdc, err := fdcount.Matching("TCP") if err != nil { t.Fatal(err) } randomlyClose := int32(0) p := New(Config{ Size: poolSize, ClaimTimeout: claimTimeout, Dial: func() (net.Conn, error) { conn, err := net.DialTimeout("tcp", addr, 15*time.Millisecond) if atomic.LoadInt32(&randomlyClose) == 1 { // Close about half of the connections immediately to test /// closed checking if err == nil && rand.Float32() > 0.5 { conn.Close() } } return conn, err }, }) time.Sleep(fillTime) assert.NoError(t, fdc.AssertDelta(0), "Pool should initially contain no conns") // Use more than the pooled connections connectAndRead(t, p, poolSize*2) time.Sleep(fillTime) assert.NoError(t, fdc.AssertDelta(poolSize), "Pool should fill itself back up to the right number of conns") // Wait for connections to time out time.Sleep(claimTimeout * 2) assert.NoError(t, fdc.AssertDelta(0), "After connections time out, but before dialing again, pool should be empty") // Test our connections again connectAndRead(t, p, poolSize*2) time.Sleep(fillTime) assert.NoError(t, fdc.AssertDelta(poolSize), "After pooled conns time out, pool should fill itself back up to the right number of conns") atomic.StoreInt32(&randomlyClose, 1) // Make sure we can still get connections and use them connectAndRead(t, p, poolSize) // Wait for pool to fill again time.Sleep(fillTime) p.Close() // Run another Close() concurrently just to make sure it doesn't muck things up go p.Close() assert.NoError(t, fdc.AssertDelta(0), "After stopping pool, there should be no more open conns") }