func testStorageLockNodeHealingAfterTimeout(storage cluster.Storage, t *testing.T) {
defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(100))
node := cluster.Node{Address: "addr-xyz"}
defer storage.RemoveNode("addr-xyz")
err := storage.StoreNode(node)
assertIsNil(err, t)
locked, err := storage.LockNodeForHealing("addr-xyz", true, 200*time.Millisecond)
assertIsNil(err, t)
locked, err = storage.LockNodeForHealing("addr-xyz", true, 200*time.Millisecond)
assertIsNil(err, t)
if locked {
t.Fatal("Expected LockNodeForHealing to return false before timeout")
}
time.Sleep(300 * time.Millisecond)
successCount := int32(0)
wg := sync.WaitGroup{}
wg.Add(50)
for i := 0; i < 50; i++ {
go func() {
defer wg.Done()
locked, err := storage.LockNodeForHealing("addr-xyz", true, 5*time.Second)
assertIsNil(err, t)
if locked {
atomic.AddInt32(&successCount, 1)
}
}()
}
wg.Wait()
if successCount != 1 {
t.Fatalf("Expected LockNodeForHealing after timeout to lock only once, got: %d", successCount)
}
}
func testStorageStoreRetrieveNodes(storage cluster.Storage, t *testing.T) {
node1 := cluster.Node{Address: "my-addr-1"}
defer storage.RemoveNode("my-addr-1")
err := storage.StoreNode(node1)
assertIsNil(err, t)
node2 := cluster.Node{Address: "my-addr-2", Metadata: map[string]string{"foo": "bar"}}
defer storage.RemoveNode("my-addr-2")
err = storage.StoreNode(node2)
assertIsNil(err, t)
nodes, err := storage.RetrieveNodes()
assertIsNil(err, t)
if len(nodes) != 2 {
t.Fatalf("unexpected number of nodes, expected 2, got: %d", len(nodes))
}
sort.Sort(cluster.NodeList(nodes))
if nodes[0].Address != node1.Address || nodes[1].Address != node2.Address {
t.Errorf("unexpected nodes: %#v", nodes)
}
if !reflect.DeepEqual(node2.Metadata, nodes[1].Metadata) {
t.Errorf("unexpected node metadata. expected: %#v got: %#v", node2.Metadata, nodes[1].Metadata)
}
if !reflect.DeepEqual(nodes[0].Metadata, map[string]string{}) {
t.Errorf("unexpected node metadata. expected empty map got: %#v", nodes[0].Metadata)
}
}
func testStorageStoreRepeatedNodes(storage cluster.Storage, t *testing.T) {
defer storage.RemoveNode("my-addr-1")
err := storage.StoreNode(cluster.Node{Address: "my-addr-1"})
assertIsNil(err, t)
err = storage.StoreNode(cluster.Node{Address: "my-addr-1"})
if err != cstorage.ErrDuplicatedNodeAddress {
t.Fatalf("Expected error cstorage.ErrDuplicatedNodeAddress, got: %#v", err)
}
}
func testStorageStoreEmptyMetadata(storage cluster.Storage, t *testing.T) {
defer storage.RemoveNode("my-addr-1")
node1 := cluster.Node{Address: "my-addr-1", Metadata: map[string]string{}}
err := storage.StoreNode(node1)
assertIsNil(err, t)
nodes, err := storage.RetrieveNodes()
assertIsNil(err, t)
if len(nodes) != 1 || nodes[0].Address != node1.Address {
t.Errorf("unexpected nodes: %#v", nodes)
}
}
func testStorageUnlockNode(storage cluster.Storage, t *testing.T) {
node := cluster.Node{Address: "addr-xyz"}
defer storage.RemoveNode("addr-xyz")
err := storage.StoreNode(node)
assertIsNil(err, t)
locked, err := storage.LockNodeForHealing("addr-xyz", true, 200*time.Millisecond)
assertIsNil(err, t)
err = storage.UnlockNode("addr-xyz")
assertIsNil(err, t)
locked, err = storage.LockNodeForHealing("addr-xyz", true, 200*time.Millisecond)
assertIsNil(err, t)
if !locked {
t.Fatal("Expected LockNodeForHealing to return true after unlocking")
}
}
func testStorageStoreRetrieveNode(storage cluster.Storage, t *testing.T) {
node1 := cluster.Node{Address: "my-addr-1", Metadata: map[string]string{"abc": "def"}}
defer storage.RemoveNode("my-addr-1")
err := storage.StoreNode(node1)
assertIsNil(err, t)
nd, err := storage.RetrieveNode("my-addr-1")
assertIsNil(err, t)
if !reflect.DeepEqual(nd, node1) {
t.Errorf("unexpected node, expected: %#v, got: %#v", node1, nd)
}
_, err = storage.RetrieveNode("my-addr-xxxx")
if err != cstorage.ErrNoSuchNode {
t.Errorf("Expected ErrNoSuchNode got: %#v", err)
}
}
func testStorageExtendNodeLock(storage cluster.Storage, t *testing.T) {
node := cluster.Node{Address: "addr-xyz"}
defer storage.RemoveNode("addr-xyz")
err := storage.StoreNode(node)
assertIsNil(err, t)
locked, err := storage.LockNodeForHealing("addr-xyz", true, 200*time.Millisecond)
assertIsNil(err, t)
time.Sleep(300 * time.Millisecond)
err = storage.ExtendNodeLock("addr-xyz", 200*time.Millisecond)
assertIsNil(err, t)
locked, err = storage.LockNodeForHealing("addr-xyz", true, 200*time.Millisecond)
assertIsNil(err, t)
if locked {
t.Fatal("Expected LockNodeForHealing to return false after extending timeout")
}
}
func testStorageStoreRemoveNode(storage cluster.Storage, t *testing.T) {
node1 := cluster.Node{Address: "my-addr-1"}
err := storage.StoreNode(node1)
assertIsNil(err, t)
err = storage.RemoveNode("my-addr-1")
assertIsNil(err, t)
err = storage.RemoveNode("my-addr-1")
if err != cstorage.ErrNoSuchNode {
t.Errorf("cstorage.ErrNoSuchNode was expected, got: %s", err)
}
nodes, err := storage.RetrieveNodes()
assertIsNil(err, t)
if len(nodes) > 0 {
t.Errorf("nodes should be empty, found: %#v", nodes)
}
}
func testStorageStoreClearMetadata(storage cluster.Storage, t *testing.T) {
defer storage.RemoveNode("my-addr-1")
err := storage.StoreNode(cluster.Node{Address: "my-addr-1", Metadata: map[string]string{"pool": "p1"}})
assertIsNil(err, t)
err = storage.RemoveNode("my-addr-1")
assertIsNil(err, t)
err = storage.StoreNode(cluster.Node{Address: "my-addr-1"})
assertIsNil(err, t)
nodes, err := storage.RetrieveNodes()
assertIsNil(err, t)
if len(nodes) != 1 || nodes[0].Address != "my-addr-1" {
t.Errorf("unexpected nodes: %#v", nodes)
}
if !reflect.DeepEqual(nodes[0].Metadata, map[string]string{}) {
t.Errorf("unexpected node metadata. expected empty map, got: %#v", nodes[0].Metadata)
}
}
func testStorageStoreAlreadyLocked(storage cluster.Storage, t *testing.T) {
node1 := cluster.Node{
Address: "my-addr-locked",
Metadata: map[string]string{},
Healing: cluster.HealingData{LockedUntil: time.Now().UTC().Add(5 * time.Second), IsFailure: true},
}
defer storage.RemoveNode("my-addr-locked")
err := storage.StoreNode(node1)
assertIsNil(err, t)
nd, err := storage.RetrieveNode("my-addr-locked")
assertIsNil(err, t)
duration := nd.Healing.LockedUntil.Sub(node1.Healing.LockedUntil)
if duration < 0 {
duration = -duration
}
if duration > 1*time.Second {
t.Errorf("unexpected node, expected: %#v, got: %#v", node1, nd)
}
}
func testStorageLockNodeHealing(storage cluster.Storage, t *testing.T) {
defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(100))
node := cluster.Node{Address: "addr-xyz"}
defer storage.RemoveNode("addr-xyz")
err := storage.StoreNode(node)
assertIsNil(err, t)
successCount := 0
wg := sync.WaitGroup{}
wg.Add(50)
for i := 0; i < 50; i++ {
go func() {
defer wg.Done()
locked, err := storage.LockNodeForHealing("addr-xyz", true, 5*time.Second)
assertIsNil(err, t)
if locked {
successCount++
}
}()
}
wg.Wait()
if successCount != 1 {
t.Fatalf("Expected success in only one goroutine, got: %d", successCount)
}
dbNode, err := storage.RetrieveNode("addr-xyz")
assertIsNil(err, t)
if dbNode.Healing.LockedUntil.IsZero() {
t.Fatal("Expected node Healing.LockedUntil not to be zero")
}
if !dbNode.Healing.IsFailure {
t.Fatal("Expected node healing.isFailure to be true")
}
dbNode.Healing = cluster.HealingData{}
err = storage.UpdateNode(dbNode)
assertIsNil(err, t)
dbNode, err = storage.RetrieveNode("addr-xyz")
assertIsNil(err, t)
if !dbNode.Healing.LockedUntil.IsZero() {
t.Fatal("Expected node Healing.LockedUntil to be zero")
}
if dbNode.Healing.IsFailure {
t.Fatal("Expected node Healing.IsFailure to be false")
}
}
func testStorageStoreRetrieveNodesForMetadata(storage cluster.Storage, t *testing.T) {
node1 := cluster.Node{Address: "my-addr-1", Metadata: map[string]string{
"region": "reg1",
"foo": "bar",
}}
err := storage.StoreNode(node1)
assertIsNil(err, t)
defer storage.RemoveNode("my-addr-1")
node2 := cluster.Node{Address: "my-addr-2", Metadata: map[string]string{
"region": "reg2",
"foo": "bar",
}}
err = storage.StoreNode(node2)
assertIsNil(err, t)
defer storage.RemoveNode("my-addr-2")
nodes, err := storage.RetrieveNodesByMetadata(map[string]string{"region": "reg2"})
assertIsNil(err, t)
if len(nodes) != 1 {
t.Fatalf("unexpected nodes len: %d", len(nodes))
}
if nodes[0].Address != node2.Address {
t.Errorf("unexpected node: %s", nodes[0].Address)
}
if !reflect.DeepEqual(node2.Metadata, nodes[0].Metadata) {
t.Errorf("unexpected node metadata. expected: %#v got: %#v", node2.Metadata, nodes[0].Metadata)
}
nodes, err = storage.RetrieveNodesByMetadata(map[string]string{"foo": "bar"})
assertIsNil(err, t)
if len(nodes) != 2 {
t.Fatalf("unexpected nodes len: %d", len(nodes))
}
sort.Sort(cluster.NodeList(nodes))
if nodes[0].Address != node1.Address || nodes[1].Address != node2.Address {
t.Errorf("unexpected nodes: %#v", nodes)
}
if !reflect.DeepEqual(node1.Metadata, nodes[0].Metadata) {
t.Errorf("unexpected node metadata. expected: %#v got: %#v", node1.Metadata, nodes[0].Metadata)
}
if !reflect.DeepEqual(node2.Metadata, nodes[1].Metadata) {
t.Errorf("unexpected node metadata. expected: %#v got: %#v", node2.Metadata, nodes[1].Metadata)
}
}