func testStorageUnlockNode(storage cluster.Storage, t *testing.T) {
node := cluster.Node{Address: "addr-xyz"}
defer storage.RemoveNode("addr-xyz")
err := storage.StoreNode(node)
assertIsNil(err, t)
locked, err := storage.LockNodeForHealing("addr-xyz", true, 200*time.Millisecond)
assertIsNil(err, t)
err = storage.UnlockNode("addr-xyz")
assertIsNil(err, t)
locked, err = storage.LockNodeForHealing("addr-xyz", true, 200*time.Millisecond)
assertIsNil(err, t)
if !locked {
t.Fatal("Expected LockNodeForHealing to return true after unlocking")
}
}
func testStorageExtendNodeLock(storage cluster.Storage, t *testing.T) {
node := cluster.Node{Address: "addr-xyz"}
defer storage.RemoveNode("addr-xyz")
err := storage.StoreNode(node)
assertIsNil(err, t)
locked, err := storage.LockNodeForHealing("addr-xyz", true, 200*time.Millisecond)
assertIsNil(err, t)
time.Sleep(300 * time.Millisecond)
err = storage.ExtendNodeLock("addr-xyz", 200*time.Millisecond)
assertIsNil(err, t)
locked, err = storage.LockNodeForHealing("addr-xyz", true, 200*time.Millisecond)
assertIsNil(err, t)
if locked {
t.Fatal("Expected LockNodeForHealing to return false after extending timeout")
}
}
func testStorageLockNodeHealing(storage cluster.Storage, t *testing.T) {
defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(100))
node := cluster.Node{Address: "addr-xyz"}
defer storage.RemoveNode("addr-xyz")
err := storage.StoreNode(node)
assertIsNil(err, t)
successCount := 0
wg := sync.WaitGroup{}
wg.Add(50)
for i := 0; i < 50; i++ {
go func() {
defer wg.Done()
locked, err := storage.LockNodeForHealing("addr-xyz", true, 5*time.Second)
assertIsNil(err, t)
if locked {
successCount++
}
}()
}
wg.Wait()
if successCount != 1 {
t.Fatalf("Expected success in only one goroutine, got: %d", successCount)
}
dbNode, err := storage.RetrieveNode("addr-xyz")
assertIsNil(err, t)
if dbNode.Healing.LockedUntil.IsZero() {
t.Fatal("Expected node Healing.LockedUntil not to be zero")
}
if !dbNode.Healing.IsFailure {
t.Fatal("Expected node healing.isFailure to be true")
}
dbNode.Healing = cluster.HealingData{}
err = storage.UpdateNode(dbNode)
assertIsNil(err, t)
dbNode, err = storage.RetrieveNode("addr-xyz")
assertIsNil(err, t)
if !dbNode.Healing.LockedUntil.IsZero() {
t.Fatal("Expected node Healing.LockedUntil to be zero")
}
if dbNode.Healing.IsFailure {
t.Fatal("Expected node Healing.IsFailure to be false")
}
}
func testStorageLockNodeHealingAfterTimeout(storage cluster.Storage, t *testing.T) {
defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(100))
node := cluster.Node{Address: "addr-xyz"}
defer storage.RemoveNode("addr-xyz")
err := storage.StoreNode(node)
assertIsNil(err, t)
locked, err := storage.LockNodeForHealing("addr-xyz", true, 200*time.Millisecond)
assertIsNil(err, t)
locked, err = storage.LockNodeForHealing("addr-xyz", true, 200*time.Millisecond)
assertIsNil(err, t)
if locked {
t.Fatal("Expected LockNodeForHealing to return false before timeout")
}
time.Sleep(300 * time.Millisecond)
successCount := int32(0)
wg := sync.WaitGroup{}
wg.Add(50)
for i := 0; i < 50; i++ {
go func() {
defer wg.Done()
locked, err := storage.LockNodeForHealing("addr-xyz", true, 5*time.Second)
assertIsNil(err, t)
if locked {
atomic.AddInt32(&successCount, 1)
}
}()
}
wg.Wait()
if successCount != 1 {
t.Fatalf("Expected LockNodeForHealing after timeout to lock only once, got: %d", successCount)
}
}