// UnlockKeyspace unlocks a previously locked keyspace.
func (n *ActionNode) UnlockKeyspace(ctx context.Context, ts topo.Server, keyspace string, lockPath string, actionError error) error {
// Detach from the parent timeout, but copy the trace span.
// We need to still release the lock even if the parent context timed out.
ctx = trace.CopySpan(context.TODO(), ctx)
ctx, cancel := context.WithTimeout(ctx, DefaultLockTimeout)
defer cancel()
span := trace.NewSpanFromContext(ctx)
span.StartClient("TopoServer.UnlockKeyspaceForAction")
span.Annotate("action", n.Action)
span.Annotate("keyspace", keyspace)
defer span.Finish()
// first update the actionNode
if actionError != nil {
log.Infof("Unlocking keyspace %v for action %v with error %v", keyspace, n.Action, actionError)
n.Error = actionError.Error()
n.State = ActionStateFailed
} else {
log.Infof("Unlocking keyspace %v for successful action %v", keyspace, n.Action)
n.Error = ""
n.State = ActionStateDone
}
err := ts.UnlockKeyspaceForAction(ctx, keyspace, lockPath, n.ToJSON())
if actionError != nil {
if err != nil {
// this will be masked
log.Warningf("UnlockKeyspaceForAction failed: %v", err)
}
return actionError
}
return err
}
func checkKeyspaceLockTimeout(ctx context.Context, t *testing.T, ts topo.Server) {
lockPath, err := ts.LockKeyspaceForAction(ctx, "test_keyspace", "fake-content")
if err != nil {
t.Fatalf("LockKeyspaceForAction: %v", err)
}
// test we can't take the lock again
fastCtx, cancel := context.WithTimeout(ctx, timeUntilLockIsTaken)
if _, err := ts.LockKeyspaceForAction(fastCtx, "test_keyspace", "unused-fake-content"); err != topo.ErrTimeout {
t.Fatalf("LockKeyspaceForAction(again): %v", err)
}
cancel()
// test we can interrupt taking the lock
interruptCtx, cancel := context.WithCancel(ctx)
go func() {
time.Sleep(timeUntilLockIsTaken)
cancel()
}()
if _, err := ts.LockKeyspaceForAction(interruptCtx, "test_keyspace", "unused-fake-content"); err != topo.ErrInterrupted {
t.Fatalf("LockKeyspaceForAction(interrupted): %v", err)
}
if err := ts.UnlockKeyspaceForAction(ctx, "test_keyspace", lockPath, "fake-results"); err != nil {
t.Fatalf("UnlockKeyspaceForAction(): %v", err)
}
// test we can't unlock again
if err := ts.UnlockKeyspaceForAction(ctx, "test_keyspace", lockPath, "fake-results"); err == nil {
t.Fatalf("UnlockKeyspaceForAction(again) worked")
}
}
// UnlockKeyspace unlocks a previously locked keyspace.
func (n *ActionNode) UnlockKeyspace(ctx context.Context, ts topo.Server, keyspace string, lockPath string, actionError error) error {
span := trace.NewSpanFromContext(ctx)
span.StartClient("TopoServer.UnlockKeyspaceForAction")
span.Annotate("action", n.Action)
span.Annotate("keyspace", keyspace)
defer span.Finish()
// first update the actionNode
if actionError != nil {
log.Infof("Unlocking keyspace %v for action %v with error %v", keyspace, n.Action, actionError)
n.Error = actionError.Error()
n.State = ActionStateFailed
} else {
log.Infof("Unlocking keyspace %v for successful action %v", keyspace, n.Action)
n.Error = ""
n.State = ActionStateDone
}
err := ts.UnlockKeyspaceForAction(ctx, keyspace, lockPath, n.ToJSON())
if actionError != nil {
if err != nil {
// this will be masked
log.Warningf("UnlockKeyspaceForAction failed: %v", err)
}
return actionError
}
return err
}
// UnlockKeyspace unlocks a previously locked keyspace.
func (n *ActionNode) UnlockKeyspace(ts topo.Server, keyspace string, lockPath string, actionError error) error {
// first update the actionNode
if actionError != nil {
log.Infof("Unlocking keyspace %v for action %v with error %v", keyspace, n.Action, actionError)
n.Error = actionError.Error()
n.State = ACTION_STATE_FAILED
} else {
log.Infof("Unlocking keyspace %v for successful action %v", keyspace, n.Action)
n.Error = ""
n.State = ACTION_STATE_DONE
}
err := ts.UnlockKeyspaceForAction(keyspace, lockPath, n.ToJson())
if actionError != nil {
if err != nil {
// this will be masked
log.Warningf("UnlockKeyspaceForAction failed: %v", err)
}
return actionError
}
return err
}
func CheckKeyspaceLock(t *testing.T, ts topo.Server) {
if err := ts.CreateKeyspace("test_keyspace", &topo.Keyspace{}); err != nil {
t.Fatalf("CreateKeyspace: %v", err)
}
interrupted := make(chan struct{}, 1)
lockPath, err := ts.LockKeyspaceForAction("test_keyspace", "fake-content", 5*time.Second, interrupted)
if err != nil {
t.Fatalf("LockKeyspaceForAction: %v", err)
}
// test we can't take the lock again
if _, err := ts.LockKeyspaceForAction("test_keyspace", "unused-fake-content", time.Second/10, interrupted); err != topo.ErrTimeout {
t.Errorf("LockKeyspaceForAction(again): %v", err)
}
// test we can interrupt taking the lock
go func() {
time.Sleep(time.Second / 10)
close(interrupted)
}()
if _, err := ts.LockKeyspaceForAction("test_keyspace", "unused-fake-content", 5*time.Second, interrupted); err != topo.ErrInterrupted {
t.Errorf("LockKeyspaceForAction(interrupted): %v", err)
}
if err := ts.UnlockKeyspaceForAction("test_keyspace", lockPath, "fake-results"); err != nil {
t.Errorf("UnlockKeyspaceForAction(): %v", err)
}
// test we can't unlock again
if err := ts.UnlockKeyspaceForAction("test_keyspace", lockPath, "fake-results"); err == nil {
t.Error("UnlockKeyspaceForAction(again) worked")
}
// test we can't lock a non-existing keyspace
interrupted = make(chan struct{}, 1)
if _, err := ts.LockKeyspaceForAction("test_keyspace_666", "fake-content", 5*time.Second, interrupted); err == nil {
t.Fatalf("LockKeyspaceForAction(test_keyspace_666) worked for non-existing keyspace")
}
}
// checkKeyspaceLockUnblocks makes sure that a routine waiting on a lock
// is unblocked when another routine frees the lock
func checkKeyspaceLockUnblocks(ctx context.Context, t *testing.T, ts topo.Server) {
unblock := make(chan struct{})
finished := make(chan struct{})
// as soon as we're unblocked, we try to lock the keyspace
go func() {
<-unblock
lockPath, err := ts.LockKeyspaceForAction(ctx, "test_keyspace", "fake-content")
if err != nil {
t.Fatalf("LockKeyspaceForAction(test_keyspace) failed: %v", err)
}
if err = ts.UnlockKeyspaceForAction(ctx, "test_keyspace", lockPath, "fake-results"); err != nil {
t.Fatalf("UnlockKeyspaceForAction(test_keyspace): %v", err)
}
close(finished)
}()
// lock the keyspace
lockPath2, err := ts.LockKeyspaceForAction(ctx, "test_keyspace", "fake-content")
if err != nil {
t.Fatalf("LockKeyspaceForAction(test_keyspace) failed: %v", err)
}
// unblock the go routine so it starts waiting
close(unblock)
// sleep for a while so we're sure the go routine is blocking
time.Sleep(timeUntilLockIsTaken)
if err = ts.UnlockKeyspaceForAction(ctx, "test_keyspace", lockPath2, "fake-results"); err != nil {
t.Fatalf("UnlockKeyspaceForAction(test_keyspace): %v", err)
}
timeout := time.After(10 * time.Second)
select {
case <-finished:
case <-timeout:
t.Fatalf("unlocking timed out")
}
}