func testStorageUnlockNode(storage cluster.Storage, t *testing.T) { node := cluster.Node{Address: "addr-xyz"} defer storage.RemoveNode("addr-xyz") err := storage.StoreNode(node) assertIsNil(err, t) locked, err := storage.LockNodeForHealing("addr-xyz", true, 200*time.Millisecond) assertIsNil(err, t) err = storage.UnlockNode("addr-xyz") assertIsNil(err, t) locked, err = storage.LockNodeForHealing("addr-xyz", true, 200*time.Millisecond) assertIsNil(err, t) if !locked { t.Fatal("Expected LockNodeForHealing to return true after unlocking") } }
func testStorageExtendNodeLock(storage cluster.Storage, t *testing.T) { node := cluster.Node{Address: "addr-xyz"} defer storage.RemoveNode("addr-xyz") err := storage.StoreNode(node) assertIsNil(err, t) locked, err := storage.LockNodeForHealing("addr-xyz", true, 200*time.Millisecond) assertIsNil(err, t) time.Sleep(300 * time.Millisecond) err = storage.ExtendNodeLock("addr-xyz", 200*time.Millisecond) assertIsNil(err, t) locked, err = storage.LockNodeForHealing("addr-xyz", true, 200*time.Millisecond) assertIsNil(err, t) if locked { t.Fatal("Expected LockNodeForHealing to return false after extending timeout") } }
func testStorageLockNodeHealing(storage cluster.Storage, t *testing.T) { defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(100)) node := cluster.Node{Address: "addr-xyz"} defer storage.RemoveNode("addr-xyz") err := storage.StoreNode(node) assertIsNil(err, t) successCount := 0 wg := sync.WaitGroup{} wg.Add(50) for i := 0; i < 50; i++ { go func() { defer wg.Done() locked, err := storage.LockNodeForHealing("addr-xyz", true, 5*time.Second) assertIsNil(err, t) if locked { successCount++ } }() } wg.Wait() if successCount != 1 { t.Fatalf("Expected success in only one goroutine, got: %d", successCount) } dbNode, err := storage.RetrieveNode("addr-xyz") assertIsNil(err, t) if dbNode.Healing.LockedUntil.IsZero() { t.Fatal("Expected node Healing.LockedUntil not to be zero") } if !dbNode.Healing.IsFailure { t.Fatal("Expected node healing.isFailure to be true") } dbNode.Healing = cluster.HealingData{} err = storage.UpdateNode(dbNode) assertIsNil(err, t) dbNode, err = storage.RetrieveNode("addr-xyz") assertIsNil(err, t) if !dbNode.Healing.LockedUntil.IsZero() { t.Fatal("Expected node Healing.LockedUntil to be zero") } if dbNode.Healing.IsFailure { t.Fatal("Expected node Healing.IsFailure to be false") } }
func testStorageLockNodeHealingAfterTimeout(storage cluster.Storage, t *testing.T) { defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(100)) node := cluster.Node{Address: "addr-xyz"} defer storage.RemoveNode("addr-xyz") err := storage.StoreNode(node) assertIsNil(err, t) locked, err := storage.LockNodeForHealing("addr-xyz", true, 200*time.Millisecond) assertIsNil(err, t) locked, err = storage.LockNodeForHealing("addr-xyz", true, 200*time.Millisecond) assertIsNil(err, t) if locked { t.Fatal("Expected LockNodeForHealing to return false before timeout") } time.Sleep(300 * time.Millisecond) successCount := int32(0) wg := sync.WaitGroup{} wg.Add(50) for i := 0; i < 50; i++ { go func() { defer wg.Done() locked, err := storage.LockNodeForHealing("addr-xyz", true, 5*time.Second) assertIsNil(err, t) if locked { atomic.AddInt32(&successCount, 1) } }() } wg.Wait() if successCount != 1 { t.Fatalf("Expected LockNodeForHealing after timeout to lock only once, got: %d", successCount) } }