func NewSingleHarness(t testing.TB) *SingleHarness {
mgoserver := mgotest.NewStartedServer(t)
return &SingleHarness{
Harness: newHarnessInternal(mgoserver.URL(), mgoserver, t),
MgoServer: mgoserver,
}
}
// Basically tests manager.GetReplicaIDs
func TestGetReplicaIDs(t *testing.T) {
skipIfOldMongo(t)
s3 := s3storage.NewMockS3(t)
defer s3.Stop()
s, err := s3storage.NewStorageWithMockS3(s3)
ensure.Nil(t, err)
replicaID1 := "faux-replicaID1"
replicaID2 := "faux-replicaID2"
mongo := mgotest.NewStartedServer(t, "--storageEngine=rocksdb")
defer mongo.Stop()
r := lreplica.NewMockLocalReplica(mongo, 1)
manager, err := strata.NewSnapshotManager(r, s)
ensure.Nil(t, err)
_, err = manager.CreateSnapshot(replicaID1)
ensure.Nil(t, err)
_, err = manager.CreateSnapshot(replicaID2)
ensure.Nil(t, err)
_, err = manager.CreateSnapshot(replicaID1)
ensure.Nil(t, err)
replicaIDs := manager.GetReplicaIDs()
ensure.DeepEqual(t, len(replicaIDs), 2)
}
func TestIgnoreMismatchingReplicaSets(t *testing.T) {
t.Parallel()
creator := ReplicaSetStateCreator{
Log: nopLogger{},
}
replicaSet := mgotest.NewReplicaSet(2, t)
singleMongo := mgotest.NewStartedServer(t)
defer func() {
replicaSet.Stop()
singleMongo.Stop()
}()
urls := replicaSet.Addrs()
urls = append(urls, singleMongo.URL())
state, err := creator.FromAddrs("", "", urls, "rs")
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if state.lastRS.Name != "rs" {
t.Fatalf("unexpected replicaset: %s", state.lastRS.Name)
}
_, err = creator.FromAddrs("", "", urls, "")
if err == nil {
t.Fatalf("missing expected error: %s", err)
}
}
func TestNewReplicaSetStateFailure(t *testing.T) {
t.Parallel()
mgo := mgotest.NewStartedServer(t)
mgo.Stop()
_, err := NewReplicaSetState("", "", mgo.URL())
const expected = "no reachable servers"
if err == nil || err.Error() != expected {
t.Fatalf("unexpected error: %s", err)
}
}
func TestSingleNodeNewReplicaSetState(t *testing.T) {
t.Parallel()
mgo := mgotest.NewStartedServer(t)
defer mgo.Stop()
rs, err := NewReplicaSetState("", "", mgo.URL())
if err != nil {
t.Fatal(err)
}
if rs.singleAddr != mgo.URL() {
t.Fatalf("expected %s got %s", mgo.URL(), rs.singleAddr)
}
}
func TestSingleNodeNewReplicaSetStateAddrs(t *testing.T) {
t.Parallel()
mgo := mgotest.NewStartedServer(t)
defer mgo.Stop()
rs, err := NewReplicaSetState("", "", mgo.URL())
if err != nil {
t.Fatal(err)
}
addrs := rs.Addrs()
if len(addrs) != 1 || addrs[0] != mgo.URL() {
t.Fatalf("unexpected addrs %v", addrs)
}
}
func TestTestCommands(t *testing.T) {
mongo := mgotest.NewStartedServer(t)
defer mongo.Stop()
session := mongo.Session()
defer session.Close()
command := bson.D{
{Name: "sleep", Value: 1},
{Name: "secs", Value: 1},
}
if err := session.DB("admin").Run(command, nil); err != nil {
t.Fatal(err)
}
}
func test(t *testing.T, answer int) {
t.Parallel()
mongo := mgotest.NewStartedServer(t)
defer mongo.Stop()
const id = 1
in := bson.M{"_id": id, "answer": answer}
collection := mongo.Session().DB("tdb").C("tc")
if err := collection.Insert(in); err != nil {
t.Fatal(err)
}
out := bson.M{}
if err := collection.FindId(id).One(out); err != nil {
t.Fatal(err)
}
if out["answer"] != answer {
t.Fatalf("did not find expected answer, got %v", out)
}
}
func TestChecksWithReplicaSets(t *testing.T) {
t.Parallel()
standalone := mgotest.NewStartedServer(t)
defer standalone.Stop()
rs := mgotest.NewReplicaSet(3, t)
defer rs.Stop()
if err := checkReplSetStatus(rs.Addrs(), "rs"); err != nil {
t.Error("check should pass if all members are in the replica set:", err)
}
if err := checkReplSetStatus([]string{standalone.URL()}, "rs"); err == nil {
t.Error("expected failure if single server running in standalone")
}
if err := checkReplSetStatus(append(rs.Addrs(), standalone.URL()), "rs"); err != nil {
t.Error("check should ignore standalone if there are other healthy members:", err)
}
if err := checkReplSetStatus(rs.Addrs(), "rs-alt"); err == nil {
t.Error("check should fail if members are in a different replica set")
}
}
func TestSingleNodeWhenExpectingRS(t *testing.T) {
t.Parallel()
mgoserver := mgotest.NewStartedServer(t)
defer mgoserver.Stop()
replicaSet := ReplicaSet{
Addrs: fmt.Sprintf("127.0.0.1:%d,127.0.0.1:%d", mgoserver.Port, mgoserver.Port+1),
MaxConnections: 1,
}
var log nopLogger
var graph inject.Graph
err := graph.Provide(
&inject.Object{Value: &log},
&inject.Object{Value: &replicaSet},
&inject.Object{Value: &stats.HookClient{}},
)
ensure.Nil(t, err)
ensure.Nil(t, graph.Populate())
objects := graph.Objects()
err = startstop.Start(objects, &log)
if err == nil || !strings.Contains(err.Error(), "was expecting it to be in a replica set") {
t.Fatalf("did not get expected error, got: %s", err)
}
}
func TestRestoreToDifferentPath(t *testing.T) {
skipIfOldMongo(t)
dbpath1, err := ioutil.TempDir("", "lreplica_s3storage_test_dbpath_")
ensure.Nil(t, err)
fmt.Println("Using temporary database path", dbpath1)
dbpath2, err := ioutil.TempDir("", "lreplica_s3storage_test_dbpath_")
ensure.Nil(t, err)
fmt.Println("Using temporary database path", dbpath2)
s3 := s3storage.NewMockS3(t)
defer s3.Stop()
s, err := s3storage.NewStorageWithMockS3(s3)
ensure.Nil(t, err)
replicaID := "faux-replicaID"
maxEntryID := 100
// Populate database on dbpath1 and save a snapshot
{
mongo := mgotest.NewStartedServer(t, "--storageEngine=rocksdb", "--dbpath="+dbpath1)
defer mongo.Stop()
collection := mongo.Session().DB("tdb").C("tc")
r := lreplica.NewMockLocalReplica(mongo, 1)
manager, err := strata.NewSnapshotManager(r, s)
ensure.Nil(t, err)
// Snapshot 0 should only have entries with _id % 2 == 0
for i := 0; i <= maxEntryID; i += 2 {
if err := collection.Insert(bson.M{"_id": i, "answer": i}); err != nil {
t.Fatal(err)
}
}
_, err = manager.CreateSnapshot(replicaID)
ensure.Nil(t, err)
err = manager.SaveMetadataForReplica(replicaID)
ensure.Nil(t, err)
}
// Restore and test snapshot
_, err = restoreSnapshot(s, replicaID, dbpath2, "0")
ensure.Nil(t, err)
{
mongo := mgotest.NewStartedServer(t, "--storageEngine=rocksdb", "--dbpath="+dbpath2)
defer mongo.Stop()
collection := mongo.Session().DB("tdb").C("tc")
out := bson.M{}
for i := 0; i <= maxEntryID; i++ {
if i%2 == 0 {
if err := collection.FindId(i).One(out); err != nil {
t.Fatal(err)
}
if out["answer"] != i {
t.Fatalf("did not find expected answer for id %d", i)
}
} else {
if err := collection.FindId(i).One(out); err == nil {
t.Fatalf("Did not expect %d to be in the collection", i)
}
}
}
mongo.Stop()
}
err = os.RemoveAll(dbpath1)
ensure.Nil(t, err)
err = os.RemoveAll(dbpath2)
ensure.Nil(t, err)
}
func TestIntegration(t *testing.T) {
skipIfOldMongo(t)
dbpath, err := ioutil.TempDir("", "lreplica_s3storage_test_dbpath_")
ensure.Nil(t, err)
fmt.Println("Using temporary database path", dbpath)
s3 := s3storage.NewMockS3(t)
defer s3.Stop()
s, err := s3storage.NewStorageWithMockS3(s3)
ensure.Nil(t, err)
replicaID := "faux-replicaID"
minEntryID := -100
maxEntryID := int(math.Pow(2, 12))
maxBackgroundCopies := 24
// Populate database and save snapshots
{
mongo := mgotest.NewStartedServer(t, "--storageEngine=rocksdb", "--dbpath="+dbpath)
defer mongo.Stop()
collection := mongo.Session().DB("tdb").C("tc")
r := lreplica.NewMockLocalReplica(mongo, maxBackgroundCopies)
manager, err := strata.NewSnapshotManager(r, s)
ensure.Nil(t, err)
// Snapshot 0 should only have entries with _id % 3 == 0
for i := 0; i <= maxEntryID; i += 3 {
if err := collection.Insert(bson.M{"_id": i, "answer": i}); err != nil {
t.Fatal(err)
}
}
_, err = manager.CreateSnapshot(replicaID)
ensure.Nil(t, err)
// Snapshot 1 should have entries with _id % 3 == 0 or 1
for i := 1; i <= maxEntryID; i += 3 {
if err := collection.Insert(bson.M{"_id": i, "answer": i}); err != nil {
t.Fatal(err)
}
}
_, err = manager.CreateSnapshot(replicaID)
ensure.Nil(t, err)
// Restore attempt while database is in use should fail cleanly
err = manager.SaveMetadataForReplica(replicaID)
ensure.Nil(t, err)
err = manager.RefreshMetadata()
ensure.Nil(t, err)
_, err = restoreSnapshot(s, replicaID, "", "0")
ensure.True(t, fmt.Sprintf("%s", err) == "resource temporarily unavailable")
// Snapshot 2 and subsequent snapshots shouldhave entries with negative _id
// Snapshot 2 will be deleted
for i := minEntryID; i < 0; i++ {
if err := collection.Insert(bson.M{"_id": i, "answer": i}); err != nil {
t.Fatal(err)
}
}
_, err = manager.CreateSnapshot(replicaID)
ensure.Nil(t, err)
// Snapshot 3 does not have entries with _id % 4 == 0
for i := 0; i <= maxEntryID; i += 4 {
if err := collection.RemoveId(i); err != nil && err != mgo.ErrNotFound {
t.Fatal(err)
}
}
_, err = manager.CreateSnapshot(replicaID)
ensure.Nil(t, err)
// Remove local backups directory. We should see no ill effect.
backuppath := dbpath + "/backup"
_, err = os.Stat(backuppath)
ensure.Nil(t, err)
err = os.RemoveAll(backuppath)
ensure.Nil(t, err)
// Snapshot 4's entries with _id % 4 == 1 have a flipped value for "answer"
for i := 1; i <= maxEntryID; i += 4 {
err := collection.UpdateId(i, bson.M{"_id": i, "answer": -i})
if err != nil && err != mgo.ErrNotFound {
t.Fatal(err)
}
}
_, err = manager.CreateSnapshot(replicaID)
ensure.Nil(t, err)
// Check that manager's metadata store has five snapshots
lazyMetadatas, err := manager.GetLazyMetadata(replicaID)
ensure.Nil(t, err)
ensure.DeepEqual(t, len(lazyMetadatas), 5)
// Persist to S3
err = manager.SaveMetadataForReplica(replicaID)
ensure.Nil(t, err)
err = manager.RefreshMetadata()
ensure.Nil(t, err)
// Collect garbage. There shouldn't be any garbage files.
gcStats, err := manager.CollectGarbage(replicaID)
ensure.Nil(t, err)
ensure.DeepEqual(t, gcStats.NumGarbage, 0)
ensure.DeepEqual(t, gcStats.NumErrsDeleting, 0)
// Delete snapshot 2, persist, and refresh metadata
metadata1p5, err := manager.GetSnapshotMetadata(replicaID, "2")
ensure.Nil(t, err)
err = manager.DeleteSnapshot(*metadata1p5)
ensure.Nil(t, err)
err = manager.SaveMetadataForReplica(replicaID)
ensure.Nil(t, err)
err = manager.RefreshMetadata()
ensure.Nil(t, err)
// Check that manager's metadata store has four snapshots
lazyMetadatas, err = manager.GetLazyMetadata(replicaID)
ensure.Nil(t, err)
ensure.DeepEqual(t, len(lazyMetadatas), 4)
// Collect garbage. There should be garbage files.
gcStats, err = manager.CollectGarbage(replicaID)
ensure.Nil(t, err)
ensure.True(t, gcStats.NumGarbage > 0)
ensure.DeepEqual(t, gcStats.NumErrsDeleting, 0)
mongo.Stop()
}
// Check snapshot 1
_, err = restoreSnapshot(s, replicaID, "", "1")
ensure.Nil(t, err)
// Check that restore worked
{
mongo := mgotest.NewStartedServer(t, "--storageEngine=rocksdb", "--dbpath="+dbpath)
defer mongo.Stop()
collection := mongo.Session().DB("tdb").C("tc")
out := bson.M{}
for i := minEntryID; i <= maxEntryID; i++ {
if i >= 0 && (i%3 == 0 || i%3 == 1) {
if err := collection.FindId(i).One(out); err != nil {
t.Fatal(err)
}
if out["answer"] != i {
t.Fatalf("did not find expected answer for id %d", i)
}
} else {
if err := collection.FindId(i).One(out); err == nil {
t.Fatalf("Did not expect %d to be in the collection", i)
}
}
}
mongo.Stop()
}
// Check snapshot 0
_, err = restoreSnapshot(s, replicaID, "", "0")
ensure.Nil(t, err)
// Check that restore worked
{
mongo := mgotest.NewStartedServer(t, "--storageEngine=rocksdb", "--dbpath="+dbpath)
defer mongo.Stop()
collection := mongo.Session().DB("tdb").C("tc")
out := bson.M{}
for i := minEntryID; i <= maxEntryID; i++ {
if i >= 0 && i%3 == 0 {
if err := collection.FindId(i).One(out); err != nil {
t.Fatal(err)
}
if out["answer"] != i {
t.Fatalf("did not find expected answer for id %d", i)
}
} else {
if err := collection.FindId(i).One(out); err == nil {
t.Fatalf("Did not expect %d to be in the collection", i)
}
}
}
mongo.Stop()
}
// Check snapshot 3
_, err = restoreSnapshot(s, replicaID, "", "3")
ensure.Nil(t, err)
{
mongo := mgotest.NewStartedServer(t, "--storageEngine=rocksdb", "--dbpath="+dbpath)
defer mongo.Stop()
collection := mongo.Session().DB("tdb").C("tc")
out := bson.M{}
for i := minEntryID; i <= maxEntryID; i++ {
if i < 0 || (i%3 == 0 || i%3 == 1) && i%4 != 0 {
if err := collection.FindId(i).One(out); err != nil {
t.Fatal(err)
}
if out["answer"] != i {
t.Fatalf("did not find expected answer for id %d", i)
}
} else {
if err := collection.FindId(i).One(out); err == nil {
t.Fatalf("Did not expect %d to be in the collection", i)
}
}
}
mongo.Stop()
}
// Check snapshot 4
_, err = restoreSnapshot(s, replicaID, "", "4")
ensure.Nil(t, err)
{
mongo := mgotest.NewStartedServer(t, "--storageEngine=rocksdb", "--dbpath="+dbpath)
defer mongo.Stop()
collection := mongo.Session().DB("tdb").C("tc")
out := bson.M{}
for i := minEntryID; i <= maxEntryID; i++ {
if i < 0 || (i%3 == 0 || i%3 == 1) && i%4 != 0 {
if err := collection.FindId(i).One(out); err != nil {
t.Fatal(err)
}
correct := i
if i >= 0 && i%4 == 1 {
correct = -i
}
if out["answer"] != correct {
t.Fatalf("did not find expected answer for id %d", i)
}
} else {
if err := collection.FindId(i).One(out); err == nil {
t.Fatalf("Did not expect %d to be in the collection", i)
}
}
}
mongo.Stop()
}
err = os.RemoveAll(dbpath)
ensure.Nil(t, err)
}
// TestCollectGarbage tests that the stats returned by CollectGarbage make sense
func TestCollectGarbage(t *testing.T) {
skipIfOldMongo(t)
dbpath, err := ioutil.TempDir("", "lreplica_s3storage_test_dbpath_")
ensure.Nil(t, err)
// Don't defer cleanup of dbpath because we want dbpath to be around for debugging if test fails
fmt.Println("Using temporary database path", dbpath)
s3 := s3storage.NewMockS3(t)
defer s3.Stop()
s, err := s3storage.NewStorageWithMockS3(s3)
ensure.Nil(t, err)
replicaID := "faux-replicaID"
maxEntryID := 256
mongo := mgotest.NewStartedServer(t, "--storageEngine=rocksdb", "--dbpath="+dbpath)
defer mongo.Stop()
collection := mongo.Session().DB("tdb").C("tc")
r := lreplica.NewMockLocalReplica(mongo, 1)
manager, err := strata.NewSnapshotManager(r, s)
ensure.Nil(t, err)
// There should not be any files yet
gcStats, err := manager.CollectGarbage(replicaID)
ensure.Nil(t, err)
ensure.True(t, gcStats.NumNeeded == 0)
ensure.True(t, gcStats.NumGarbage == 0)
for i := 0; i <= maxEntryID; i += 3 {
if err := collection.Insert(bson.M{"_id": i, "answer": i}); err != nil {
t.Fatal(err)
}
}
// Snapshot 0
_, err = manager.CreateSnapshot(replicaID)
ensure.Nil(t, err)
err = manager.SaveMetadataForReplica(replicaID)
ensure.Nil(t, err)
err = manager.RefreshMetadata()
ensure.Nil(t, err)
gcStats0, err := manager.CollectGarbage(replicaID)
ensure.Nil(t, err)
ensure.True(t, gcStats0.NumNeeded > 0)
ensure.True(t, gcStats0.NumNeeded == gcStats0.NumFMNeeded)
ensure.True(t, gcStats0.NumStatsNeeded == 1)
ensure.True(t, gcStats0.NumGarbage == 0)
ensure.True(t, gcStats0.NumFMGarbage == 0)
ensure.True(t, gcStats0.NumStatsGarbage == 0)
ensure.True(t, gcStats0.SizeNeeded > 0)
ensure.True(t, gcStats0.SizeGarbage == 0)
// Snapshot 1 is a copy of snapshot 0
_, err = manager.CreateSnapshot(replicaID)
ensure.Nil(t, err)
err = manager.SaveMetadataForReplica(replicaID)
ensure.Nil(t, err)
err = manager.RefreshMetadata()
ensure.Nil(t, err)
gcStats1, err := manager.CollectGarbage(replicaID)
ensure.Nil(t, err)
ensure.True(t, gcStats0.NumNeeded == gcStats1.NumNeeded)
ensure.True(t, gcStats0.NumFMNeeded == gcStats1.NumFMNeeded)
ensure.True(t, gcStats1.NumStatsNeeded == 2)
ensure.True(t, gcStats1.NumGarbage == 0)
ensure.True(t, gcStats1.NumFMGarbage == 0)
ensure.True(t, gcStats1.NumStatsGarbage == 0)
// Delete snapshot 1. This shouldn't cause any files to become garbage
// because snapshot 0 is still around.
metadata, err := manager.GetSnapshotMetadata(replicaID, "1")
ensure.Nil(t, err)
err = manager.DeleteSnapshot(*metadata)
ensure.Nil(t, err)
err = manager.SaveMetadataForReplica(replicaID)
ensure.Nil(t, err)
err = manager.RefreshMetadata()
ensure.Nil(t, err)
gcStats, err = manager.CollectGarbage(replicaID)
ensure.Nil(t, err)
ensure.True(t, gcStats0.NumNeeded == gcStats.NumNeeded)
ensure.True(t, gcStats.NumStatsNeeded == 1)
ensure.True(t, gcStats.NumGarbage == 0)
ensure.True(t, gcStats.NumFMGarbage == 0)
ensure.True(t, gcStats.NumStatsGarbage == 1)
// Snapshot 2
for i := 1; i <= maxEntryID; i += 3 {
if err := collection.Insert(bson.M{"_id": i, "answer": i}); err != nil {
t.Fatal(err)
}
}
_, err = manager.CreateSnapshot(replicaID)
ensure.Nil(t, err)
err = manager.SaveMetadataForReplica(replicaID)
ensure.Nil(t, err)
err = manager.RefreshMetadata()
ensure.Nil(t, err)
// There should be new files but no garbage
gcStats2, err := manager.CollectGarbage(replicaID)
ensure.Nil(t, err)
ensure.True(t, gcStats2.NumNeeded > gcStats0.NumNeeded)
ensure.True(t, gcStats2.NumNeeded == gcStats2.NumFMNeeded)
ensure.True(t, gcStats2.NumStatsNeeded == 2)
ensure.True(t, gcStats2.NumGarbage == 0)
ensure.True(t, gcStats2.NumFMGarbage == 0)
ensure.True(t, gcStats2.NumStatsGarbage == 0)
// Delete snapshot 2. This should cause a known number of files to become garbage.
metadata, err = manager.GetSnapshotMetadata(replicaID, "1")
ensure.Nil(t, err)
err = manager.DeleteSnapshot(*metadata)
ensure.Nil(t, err)
err = manager.SaveMetadataForReplica(replicaID)
ensure.Nil(t, err)
err = manager.RefreshMetadata()
ensure.Nil(t, err)
gcStats, err = manager.CollectGarbage(replicaID)
ensure.Nil(t, err)
ensure.True(t, gcStats.NumNeeded == gcStats0.NumNeeded)
ensure.True(t, gcStats.NumFMNeeded == gcStats.NumNeeded)
ensure.True(t, gcStats.NumStatsNeeded == 1)
ensure.True(t, gcStats.NumGarbage == gcStats2.NumNeeded-gcStats0.NumNeeded)
ensure.True(t, gcStats.NumGarbage == gcStats.NumFMGarbage)
ensure.True(t, gcStats.NumStatsGarbage == 1)
ensure.True(t, gcStats.NumErrsDeleting == 0)
// Delete snapshot 0
metadata, err = manager.GetSnapshotMetadata(replicaID, "0")
ensure.Nil(t, err)
err = manager.DeleteSnapshot(*metadata)
ensure.Nil(t, err)
err = manager.SaveMetadataForReplica(replicaID)
ensure.Nil(t, err)
err = manager.RefreshMetadata()
ensure.Nil(t, err)
gcStats, err = manager.CollectGarbage(replicaID)
ensure.Nil(t, err)
ensure.True(t, gcStats.NumNeeded == 0)
ensure.True(t, gcStats.NumFMNeeded == 0)
ensure.True(t, gcStats.NumStatsNeeded == 0)
ensure.True(t, gcStats.NumGarbage == gcStats0.NumNeeded)
ensure.True(t, gcStats.NumFMGarbage == gcStats.NumGarbage)
ensure.True(t, gcStats.NumStatsGarbage == 1)
ensure.True(t, gcStats.NumErrsDeleting == 0)
// Garbage collect again. There shouldn't be any files.
gcStats, err = manager.CollectGarbage(replicaID)
ensure.Nil(t, err)
ensure.True(t, gcStats.NumNeeded == 0)
ensure.True(t, gcStats.NumGarbage == 0)
ensure.True(t, gcStats.NumFMNeeded == 0)
ensure.True(t, gcStats.NumStatsNeeded == 0)
// Stop mongo before we delete its dbpath
mongo.Stop()
err = os.RemoveAll(dbpath)
ensure.Nil(t, err)
}