Example #1
0
func NewSingleHarness(t testing.TB) *SingleHarness {
	mgoserver := mgotest.NewStartedServer(t)
	return &SingleHarness{
		Harness:   newHarnessInternal(mgoserver.URL(), mgoserver, t),
		MgoServer: mgoserver,
	}
}
Example #2
0
// Basically tests manager.GetReplicaIDs
func TestGetReplicaIDs(t *testing.T) {
	skipIfOldMongo(t)

	s3 := s3storage.NewMockS3(t)
	defer s3.Stop()
	s, err := s3storage.NewStorageWithMockS3(s3)
	ensure.Nil(t, err)

	replicaID1 := "faux-replicaID1"
	replicaID2 := "faux-replicaID2"

	mongo := mgotest.NewStartedServer(t, "--storageEngine=rocksdb")
	defer mongo.Stop()

	r := lreplica.NewMockLocalReplica(mongo, 1)
	manager, err := strata.NewSnapshotManager(r, s)
	ensure.Nil(t, err)

	_, err = manager.CreateSnapshot(replicaID1)
	ensure.Nil(t, err)
	_, err = manager.CreateSnapshot(replicaID2)
	ensure.Nil(t, err)
	_, err = manager.CreateSnapshot(replicaID1)
	ensure.Nil(t, err)

	replicaIDs := manager.GetReplicaIDs()
	ensure.DeepEqual(t, len(replicaIDs), 2)
}
Example #3
0
func TestIgnoreMismatchingReplicaSets(t *testing.T) {
	t.Parallel()
	creator := ReplicaSetStateCreator{
		Log: nopLogger{},
	}
	replicaSet := mgotest.NewReplicaSet(2, t)
	singleMongo := mgotest.NewStartedServer(t)
	defer func() {
		replicaSet.Stop()
		singleMongo.Stop()
	}()

	urls := replicaSet.Addrs()
	urls = append(urls, singleMongo.URL())

	state, err := creator.FromAddrs("", "", urls, "rs")
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	if state.lastRS.Name != "rs" {
		t.Fatalf("unexpected replicaset: %s", state.lastRS.Name)
	}

	_, err = creator.FromAddrs("", "", urls, "")
	if err == nil {
		t.Fatalf("missing expected error: %s", err)
	}
}
Example #4
0
func TestNewReplicaSetStateFailure(t *testing.T) {
	t.Parallel()
	mgo := mgotest.NewStartedServer(t)
	mgo.Stop()
	_, err := NewReplicaSetState("", "", mgo.URL())
	const expected = "no reachable servers"
	if err == nil || err.Error() != expected {
		t.Fatalf("unexpected error: %s", err)
	}
}
Example #5
0
func TestSingleNodeNewReplicaSetState(t *testing.T) {
	t.Parallel()
	mgo := mgotest.NewStartedServer(t)
	defer mgo.Stop()
	rs, err := NewReplicaSetState("", "", mgo.URL())
	if err != nil {
		t.Fatal(err)
	}
	if rs.singleAddr != mgo.URL() {
		t.Fatalf("expected %s got %s", mgo.URL(), rs.singleAddr)
	}
}
Example #6
0
func TestSingleNodeNewReplicaSetStateAddrs(t *testing.T) {
	t.Parallel()
	mgo := mgotest.NewStartedServer(t)
	defer mgo.Stop()
	rs, err := NewReplicaSetState("", "", mgo.URL())
	if err != nil {
		t.Fatal(err)
	}
	addrs := rs.Addrs()
	if len(addrs) != 1 || addrs[0] != mgo.URL() {
		t.Fatalf("unexpected addrs %v", addrs)
	}
}
Example #7
0
func TestTestCommands(t *testing.T) {
	mongo := mgotest.NewStartedServer(t)
	defer mongo.Stop()
	session := mongo.Session()
	defer session.Close()
	command := bson.D{
		{Name: "sleep", Value: 1},
		{Name: "secs", Value: 1},
	}
	if err := session.DB("admin").Run(command, nil); err != nil {
		t.Fatal(err)
	}
}
Example #8
0
func test(t *testing.T, answer int) {
	t.Parallel()
	mongo := mgotest.NewStartedServer(t)
	defer mongo.Stop()
	const id = 1
	in := bson.M{"_id": id, "answer": answer}
	collection := mongo.Session().DB("tdb").C("tc")
	if err := collection.Insert(in); err != nil {
		t.Fatal(err)
	}
	out := bson.M{}
	if err := collection.FindId(id).One(out); err != nil {
		t.Fatal(err)
	}
	if out["answer"] != answer {
		t.Fatalf("did not find expected answer, got %v", out)
	}
}
Example #9
0
func TestChecksWithReplicaSets(t *testing.T) {
	t.Parallel()
	standalone := mgotest.NewStartedServer(t)
	defer standalone.Stop()
	rs := mgotest.NewReplicaSet(3, t)
	defer rs.Stop()

	if err := checkReplSetStatus(rs.Addrs(), "rs"); err != nil {
		t.Error("check should pass if all members are in the replica set:", err)
	}
	if err := checkReplSetStatus([]string{standalone.URL()}, "rs"); err == nil {
		t.Error("expected failure if single server running in standalone")
	}
	if err := checkReplSetStatus(append(rs.Addrs(), standalone.URL()), "rs"); err != nil {
		t.Error("check should ignore standalone if there are other healthy members:", err)
	}
	if err := checkReplSetStatus(rs.Addrs(), "rs-alt"); err == nil {
		t.Error("check should fail if members are in a different replica set")
	}
}
Example #10
0
func TestSingleNodeWhenExpectingRS(t *testing.T) {
	t.Parallel()
	mgoserver := mgotest.NewStartedServer(t)
	defer mgoserver.Stop()
	replicaSet := ReplicaSet{
		Addrs:          fmt.Sprintf("127.0.0.1:%d,127.0.0.1:%d", mgoserver.Port, mgoserver.Port+1),
		MaxConnections: 1,
	}
	var log nopLogger
	var graph inject.Graph
	err := graph.Provide(
		&inject.Object{Value: &log},
		&inject.Object{Value: &replicaSet},
		&inject.Object{Value: &stats.HookClient{}},
	)
	ensure.Nil(t, err)
	ensure.Nil(t, graph.Populate())
	objects := graph.Objects()
	err = startstop.Start(objects, &log)
	if err == nil || !strings.Contains(err.Error(), "was expecting it to be in a replica set") {
		t.Fatalf("did not get expected error, got: %s", err)
	}
}
Example #11
0
func TestRestoreToDifferentPath(t *testing.T) {
	skipIfOldMongo(t)

	dbpath1, err := ioutil.TempDir("", "lreplica_s3storage_test_dbpath_")
	ensure.Nil(t, err)
	fmt.Println("Using temporary database path", dbpath1)
	dbpath2, err := ioutil.TempDir("", "lreplica_s3storage_test_dbpath_")
	ensure.Nil(t, err)
	fmt.Println("Using temporary database path", dbpath2)

	s3 := s3storage.NewMockS3(t)
	defer s3.Stop()
	s, err := s3storage.NewStorageWithMockS3(s3)
	ensure.Nil(t, err)

	replicaID := "faux-replicaID"
	maxEntryID := 100

	// Populate database on dbpath1 and save a snapshot
	{
		mongo := mgotest.NewStartedServer(t, "--storageEngine=rocksdb", "--dbpath="+dbpath1)
		defer mongo.Stop()
		collection := mongo.Session().DB("tdb").C("tc")

		r := lreplica.NewMockLocalReplica(mongo, 1)
		manager, err := strata.NewSnapshotManager(r, s)
		ensure.Nil(t, err)

		// Snapshot 0 should only have entries with _id % 2 == 0
		for i := 0; i <= maxEntryID; i += 2 {
			if err := collection.Insert(bson.M{"_id": i, "answer": i}); err != nil {
				t.Fatal(err)
			}
		}
		_, err = manager.CreateSnapshot(replicaID)
		ensure.Nil(t, err)

		err = manager.SaveMetadataForReplica(replicaID)
		ensure.Nil(t, err)
	}

	// Restore and test snapshot
	_, err = restoreSnapshot(s, replicaID, dbpath2, "0")
	ensure.Nil(t, err)
	{
		mongo := mgotest.NewStartedServer(t, "--storageEngine=rocksdb", "--dbpath="+dbpath2)
		defer mongo.Stop()
		collection := mongo.Session().DB("tdb").C("tc")
		out := bson.M{}
		for i := 0; i <= maxEntryID; i++ {
			if i%2 == 0 {
				if err := collection.FindId(i).One(out); err != nil {
					t.Fatal(err)
				}
				if out["answer"] != i {
					t.Fatalf("did not find expected answer for id %d", i)
				}
			} else {
				if err := collection.FindId(i).One(out); err == nil {
					t.Fatalf("Did not expect %d to be in the collection", i)
				}
			}
		}
		mongo.Stop()
	}

	err = os.RemoveAll(dbpath1)
	ensure.Nil(t, err)
	err = os.RemoveAll(dbpath2)
	ensure.Nil(t, err)
}
Example #12
0
func TestIntegration(t *testing.T) {
	skipIfOldMongo(t)

	dbpath, err := ioutil.TempDir("", "lreplica_s3storage_test_dbpath_")
	ensure.Nil(t, err)
	fmt.Println("Using temporary database path", dbpath)

	s3 := s3storage.NewMockS3(t)
	defer s3.Stop()
	s, err := s3storage.NewStorageWithMockS3(s3)
	ensure.Nil(t, err)

	replicaID := "faux-replicaID"
	minEntryID := -100
	maxEntryID := int(math.Pow(2, 12))
	maxBackgroundCopies := 24

	// Populate database and save snapshots
	{
		mongo := mgotest.NewStartedServer(t, "--storageEngine=rocksdb", "--dbpath="+dbpath)
		defer mongo.Stop()
		collection := mongo.Session().DB("tdb").C("tc")

		r := lreplica.NewMockLocalReplica(mongo, maxBackgroundCopies)
		manager, err := strata.NewSnapshotManager(r, s)
		ensure.Nil(t, err)

		// Snapshot 0 should only have entries with _id % 3 == 0
		for i := 0; i <= maxEntryID; i += 3 {
			if err := collection.Insert(bson.M{"_id": i, "answer": i}); err != nil {
				t.Fatal(err)
			}
		}
		_, err = manager.CreateSnapshot(replicaID)
		ensure.Nil(t, err)

		// Snapshot 1 should have entries with _id % 3 == 0 or 1
		for i := 1; i <= maxEntryID; i += 3 {
			if err := collection.Insert(bson.M{"_id": i, "answer": i}); err != nil {
				t.Fatal(err)
			}
		}
		_, err = manager.CreateSnapshot(replicaID)
		ensure.Nil(t, err)

		// Restore attempt while database is in use should fail cleanly
		err = manager.SaveMetadataForReplica(replicaID)
		ensure.Nil(t, err)
		err = manager.RefreshMetadata()
		ensure.Nil(t, err)
		_, err = restoreSnapshot(s, replicaID, "", "0")
		ensure.True(t, fmt.Sprintf("%s", err) == "resource temporarily unavailable")

		// Snapshot 2 and subsequent snapshots shouldhave entries with negative _id
		// Snapshot 2 will be deleted
		for i := minEntryID; i < 0; i++ {
			if err := collection.Insert(bson.M{"_id": i, "answer": i}); err != nil {
				t.Fatal(err)
			}
		}
		_, err = manager.CreateSnapshot(replicaID)
		ensure.Nil(t, err)

		// Snapshot 3 does not have entries with _id % 4 == 0
		for i := 0; i <= maxEntryID; i += 4 {
			if err := collection.RemoveId(i); err != nil && err != mgo.ErrNotFound {
				t.Fatal(err)
			}
		}
		_, err = manager.CreateSnapshot(replicaID)
		ensure.Nil(t, err)

		// Remove local backups directory. We should see no ill effect.
		backuppath := dbpath + "/backup"
		_, err = os.Stat(backuppath)
		ensure.Nil(t, err)
		err = os.RemoveAll(backuppath)
		ensure.Nil(t, err)

		// Snapshot 4's entries with _id % 4 == 1 have a flipped value for "answer"
		for i := 1; i <= maxEntryID; i += 4 {
			err := collection.UpdateId(i, bson.M{"_id": i, "answer": -i})
			if err != nil && err != mgo.ErrNotFound {
				t.Fatal(err)
			}
		}
		_, err = manager.CreateSnapshot(replicaID)
		ensure.Nil(t, err)

		// Check that manager's metadata store has five snapshots
		lazyMetadatas, err := manager.GetLazyMetadata(replicaID)
		ensure.Nil(t, err)
		ensure.DeepEqual(t, len(lazyMetadatas), 5)

		// Persist to S3
		err = manager.SaveMetadataForReplica(replicaID)
		ensure.Nil(t, err)
		err = manager.RefreshMetadata()
		ensure.Nil(t, err)

		// Collect garbage. There shouldn't be any garbage files.
		gcStats, err := manager.CollectGarbage(replicaID)
		ensure.Nil(t, err)
		ensure.DeepEqual(t, gcStats.NumGarbage, 0)
		ensure.DeepEqual(t, gcStats.NumErrsDeleting, 0)

		// Delete snapshot 2, persist, and refresh metadata
		metadata1p5, err := manager.GetSnapshotMetadata(replicaID, "2")
		ensure.Nil(t, err)
		err = manager.DeleteSnapshot(*metadata1p5)
		ensure.Nil(t, err)
		err = manager.SaveMetadataForReplica(replicaID)
		ensure.Nil(t, err)
		err = manager.RefreshMetadata()
		ensure.Nil(t, err)

		// Check that manager's metadata store has four snapshots
		lazyMetadatas, err = manager.GetLazyMetadata(replicaID)
		ensure.Nil(t, err)
		ensure.DeepEqual(t, len(lazyMetadatas), 4)

		// Collect garbage. There should be garbage files.
		gcStats, err = manager.CollectGarbage(replicaID)
		ensure.Nil(t, err)
		ensure.True(t, gcStats.NumGarbage > 0)
		ensure.DeepEqual(t, gcStats.NumErrsDeleting, 0)

		mongo.Stop()
	}

	// Check snapshot 1
	_, err = restoreSnapshot(s, replicaID, "", "1")
	ensure.Nil(t, err)
	// Check that restore worked
	{
		mongo := mgotest.NewStartedServer(t, "--storageEngine=rocksdb", "--dbpath="+dbpath)
		defer mongo.Stop()
		collection := mongo.Session().DB("tdb").C("tc")
		out := bson.M{}
		for i := minEntryID; i <= maxEntryID; i++ {
			if i >= 0 && (i%3 == 0 || i%3 == 1) {
				if err := collection.FindId(i).One(out); err != nil {
					t.Fatal(err)
				}
				if out["answer"] != i {
					t.Fatalf("did not find expected answer for id %d", i)
				}
			} else {
				if err := collection.FindId(i).One(out); err == nil {
					t.Fatalf("Did not expect %d to be in the collection", i)
				}
			}
		}
		mongo.Stop()
	}

	// Check snapshot 0
	_, err = restoreSnapshot(s, replicaID, "", "0")
	ensure.Nil(t, err)
	// Check that restore worked
	{
		mongo := mgotest.NewStartedServer(t, "--storageEngine=rocksdb", "--dbpath="+dbpath)
		defer mongo.Stop()
		collection := mongo.Session().DB("tdb").C("tc")
		out := bson.M{}
		for i := minEntryID; i <= maxEntryID; i++ {
			if i >= 0 && i%3 == 0 {
				if err := collection.FindId(i).One(out); err != nil {
					t.Fatal(err)
				}
				if out["answer"] != i {
					t.Fatalf("did not find expected answer for id %d", i)
				}
			} else {
				if err := collection.FindId(i).One(out); err == nil {
					t.Fatalf("Did not expect %d to be in the collection", i)
				}
			}
		}
		mongo.Stop()
	}

	// Check snapshot 3
	_, err = restoreSnapshot(s, replicaID, "", "3")
	ensure.Nil(t, err)
	{
		mongo := mgotest.NewStartedServer(t, "--storageEngine=rocksdb", "--dbpath="+dbpath)
		defer mongo.Stop()
		collection := mongo.Session().DB("tdb").C("tc")
		out := bson.M{}
		for i := minEntryID; i <= maxEntryID; i++ {
			if i < 0 || (i%3 == 0 || i%3 == 1) && i%4 != 0 {
				if err := collection.FindId(i).One(out); err != nil {
					t.Fatal(err)
				}
				if out["answer"] != i {
					t.Fatalf("did not find expected answer for id %d", i)
				}
			} else {
				if err := collection.FindId(i).One(out); err == nil {
					t.Fatalf("Did not expect %d to be in the collection", i)
				}
			}
		}
		mongo.Stop()
	}

	// Check snapshot 4
	_, err = restoreSnapshot(s, replicaID, "", "4")
	ensure.Nil(t, err)
	{
		mongo := mgotest.NewStartedServer(t, "--storageEngine=rocksdb", "--dbpath="+dbpath)
		defer mongo.Stop()
		collection := mongo.Session().DB("tdb").C("tc")
		out := bson.M{}
		for i := minEntryID; i <= maxEntryID; i++ {
			if i < 0 || (i%3 == 0 || i%3 == 1) && i%4 != 0 {
				if err := collection.FindId(i).One(out); err != nil {
					t.Fatal(err)
				}
				correct := i
				if i >= 0 && i%4 == 1 {
					correct = -i
				}
				if out["answer"] != correct {
					t.Fatalf("did not find expected answer for id %d", i)
				}
			} else {
				if err := collection.FindId(i).One(out); err == nil {
					t.Fatalf("Did not expect %d to be in the collection", i)
				}
			}
		}
		mongo.Stop()
	}

	err = os.RemoveAll(dbpath)
	ensure.Nil(t, err)
}
Example #13
0
// TestCollectGarbage tests that the stats returned by CollectGarbage make sense
func TestCollectGarbage(t *testing.T) {
	skipIfOldMongo(t)

	dbpath, err := ioutil.TempDir("", "lreplica_s3storage_test_dbpath_")
	ensure.Nil(t, err)
	// Don't defer cleanup of dbpath because we want dbpath to be around for debugging if test fails
	fmt.Println("Using temporary database path", dbpath)

	s3 := s3storage.NewMockS3(t)
	defer s3.Stop()
	s, err := s3storage.NewStorageWithMockS3(s3)
	ensure.Nil(t, err)

	replicaID := "faux-replicaID"
	maxEntryID := 256

	mongo := mgotest.NewStartedServer(t, "--storageEngine=rocksdb", "--dbpath="+dbpath)
	defer mongo.Stop()
	collection := mongo.Session().DB("tdb").C("tc")

	r := lreplica.NewMockLocalReplica(mongo, 1)
	manager, err := strata.NewSnapshotManager(r, s)
	ensure.Nil(t, err)

	// There should not be any files yet
	gcStats, err := manager.CollectGarbage(replicaID)
	ensure.Nil(t, err)
	ensure.True(t, gcStats.NumNeeded == 0)
	ensure.True(t, gcStats.NumGarbage == 0)

	for i := 0; i <= maxEntryID; i += 3 {
		if err := collection.Insert(bson.M{"_id": i, "answer": i}); err != nil {
			t.Fatal(err)
		}
	}
	// Snapshot 0
	_, err = manager.CreateSnapshot(replicaID)
	ensure.Nil(t, err)
	err = manager.SaveMetadataForReplica(replicaID)
	ensure.Nil(t, err)
	err = manager.RefreshMetadata()
	ensure.Nil(t, err)
	gcStats0, err := manager.CollectGarbage(replicaID)
	ensure.Nil(t, err)
	ensure.True(t, gcStats0.NumNeeded > 0)
	ensure.True(t, gcStats0.NumNeeded == gcStats0.NumFMNeeded)
	ensure.True(t, gcStats0.NumStatsNeeded == 1)
	ensure.True(t, gcStats0.NumGarbage == 0)
	ensure.True(t, gcStats0.NumFMGarbage == 0)
	ensure.True(t, gcStats0.NumStatsGarbage == 0)
	ensure.True(t, gcStats0.SizeNeeded > 0)
	ensure.True(t, gcStats0.SizeGarbage == 0)

	// Snapshot 1 is a copy of snapshot 0
	_, err = manager.CreateSnapshot(replicaID)
	ensure.Nil(t, err)
	err = manager.SaveMetadataForReplica(replicaID)
	ensure.Nil(t, err)
	err = manager.RefreshMetadata()
	ensure.Nil(t, err)
	gcStats1, err := manager.CollectGarbage(replicaID)
	ensure.Nil(t, err)
	ensure.True(t, gcStats0.NumNeeded == gcStats1.NumNeeded)
	ensure.True(t, gcStats0.NumFMNeeded == gcStats1.NumFMNeeded)
	ensure.True(t, gcStats1.NumStatsNeeded == 2)
	ensure.True(t, gcStats1.NumGarbage == 0)
	ensure.True(t, gcStats1.NumFMGarbage == 0)
	ensure.True(t, gcStats1.NumStatsGarbage == 0)

	// Delete snapshot 1. This shouldn't cause any files to become garbage
	// because snapshot 0 is still around.
	metadata, err := manager.GetSnapshotMetadata(replicaID, "1")
	ensure.Nil(t, err)
	err = manager.DeleteSnapshot(*metadata)
	ensure.Nil(t, err)
	err = manager.SaveMetadataForReplica(replicaID)
	ensure.Nil(t, err)
	err = manager.RefreshMetadata()
	ensure.Nil(t, err)
	gcStats, err = manager.CollectGarbage(replicaID)
	ensure.Nil(t, err)
	ensure.True(t, gcStats0.NumNeeded == gcStats.NumNeeded)
	ensure.True(t, gcStats.NumStatsNeeded == 1)
	ensure.True(t, gcStats.NumGarbage == 0)
	ensure.True(t, gcStats.NumFMGarbage == 0)
	ensure.True(t, gcStats.NumStatsGarbage == 1)

	// Snapshot 2
	for i := 1; i <= maxEntryID; i += 3 {
		if err := collection.Insert(bson.M{"_id": i, "answer": i}); err != nil {
			t.Fatal(err)
		}
	}
	_, err = manager.CreateSnapshot(replicaID)
	ensure.Nil(t, err)
	err = manager.SaveMetadataForReplica(replicaID)
	ensure.Nil(t, err)
	err = manager.RefreshMetadata()
	ensure.Nil(t, err)

	// There should be new files but no garbage
	gcStats2, err := manager.CollectGarbage(replicaID)
	ensure.Nil(t, err)
	ensure.True(t, gcStats2.NumNeeded > gcStats0.NumNeeded)
	ensure.True(t, gcStats2.NumNeeded == gcStats2.NumFMNeeded)
	ensure.True(t, gcStats2.NumStatsNeeded == 2)
	ensure.True(t, gcStats2.NumGarbage == 0)
	ensure.True(t, gcStats2.NumFMGarbage == 0)
	ensure.True(t, gcStats2.NumStatsGarbage == 0)

	// Delete snapshot 2. This should cause a known number of files to become garbage.
	metadata, err = manager.GetSnapshotMetadata(replicaID, "1")
	ensure.Nil(t, err)
	err = manager.DeleteSnapshot(*metadata)
	ensure.Nil(t, err)
	err = manager.SaveMetadataForReplica(replicaID)
	ensure.Nil(t, err)
	err = manager.RefreshMetadata()
	ensure.Nil(t, err)
	gcStats, err = manager.CollectGarbage(replicaID)
	ensure.Nil(t, err)
	ensure.True(t, gcStats.NumNeeded == gcStats0.NumNeeded)
	ensure.True(t, gcStats.NumFMNeeded == gcStats.NumNeeded)
	ensure.True(t, gcStats.NumStatsNeeded == 1)
	ensure.True(t, gcStats.NumGarbage == gcStats2.NumNeeded-gcStats0.NumNeeded)
	ensure.True(t, gcStats.NumGarbage == gcStats.NumFMGarbage)
	ensure.True(t, gcStats.NumStatsGarbage == 1)
	ensure.True(t, gcStats.NumErrsDeleting == 0)

	// Delete snapshot 0
	metadata, err = manager.GetSnapshotMetadata(replicaID, "0")
	ensure.Nil(t, err)
	err = manager.DeleteSnapshot(*metadata)
	ensure.Nil(t, err)
	err = manager.SaveMetadataForReplica(replicaID)
	ensure.Nil(t, err)
	err = manager.RefreshMetadata()
	ensure.Nil(t, err)
	gcStats, err = manager.CollectGarbage(replicaID)
	ensure.Nil(t, err)
	ensure.True(t, gcStats.NumNeeded == 0)
	ensure.True(t, gcStats.NumFMNeeded == 0)
	ensure.True(t, gcStats.NumStatsNeeded == 0)
	ensure.True(t, gcStats.NumGarbage == gcStats0.NumNeeded)
	ensure.True(t, gcStats.NumFMGarbage == gcStats.NumGarbage)
	ensure.True(t, gcStats.NumStatsGarbage == 1)
	ensure.True(t, gcStats.NumErrsDeleting == 0)

	// Garbage collect again. There shouldn't be any files.
	gcStats, err = manager.CollectGarbage(replicaID)
	ensure.Nil(t, err)
	ensure.True(t, gcStats.NumNeeded == 0)
	ensure.True(t, gcStats.NumGarbage == 0)
	ensure.True(t, gcStats.NumFMNeeded == 0)
	ensure.True(t, gcStats.NumStatsNeeded == 0)

	// Stop mongo before we delete its dbpath
	mongo.Stop()
	err = os.RemoveAll(dbpath)
	ensure.Nil(t, err)
}