// TestManagerSimpleRun starts and stops a job within a Manager.
func TestManagerSimpleRun(t *testing.T) {
ts := topo.Server{Impl: memorytopo.NewMemoryTopo([]string{"cell1"})}
m := NewManager(ts)
// Run the manager in the background.
wg, cancel := startManager(t, m)
// Create a Sleep job.
uuid, err := m.Create(context.Background(), sleepFactoryName, []string{"-duration", "60"})
if err != nil {
t.Fatalf("cannot create sleep workflow: %v", err)
}
// Start the job
if err := m.Start(context.Background(), uuid); err != nil {
t.Fatalf("cannot start sleep workflow: %v", err)
}
// Stop the job
if err := m.Stop(context.Background(), uuid); err != nil {
t.Fatalf("cannot start sleep workflow: %v", err)
}
cancel()
wg.Wait()
}
func TestWatchSrvKeyspaceCancel(t *testing.T) {
cell := "cell1"
keyspace := "ks1"
ctx := context.Background()
mt := memorytopo.NewMemoryTopo([]string{"global", cell})
ts := topo.Server{Impl: mt}
// No SrvKeyspace -> ErrNoNode
current, changes, cancel := ts.WatchSrvKeyspace(ctx, cell, keyspace)
if current.Err != topo.ErrNoNode {
t.Errorf("Got invalid result from WatchSrvKeyspace(not there): %v", current.Err)
}
// Create initial value
wanted := &topodatapb.SrvKeyspace{
ShardingColumnName: "scn2",
}
contents, err := proto.Marshal(wanted)
if err != nil {
t.Fatalf("proto.Marshal(wanted) failed: %v", err)
}
if _, err := mt.Create(ctx, cell, "/keyspaces/"+keyspace+"/SrvKeyspace", contents); err != nil {
t.Fatalf("Update(/keyspaces/ks1/SrvKeyspace) failed: %v", err)
}
// Starting the watch should now work.
current, changes, cancel = waitForInitialSrvKeyspace(t, ts, cell, keyspace)
if !proto.Equal(current.Value, wanted) {
t.Fatalf("got bad data: %v expected: %v", current.Value, wanted)
}
// Cancel watch, wait for error.
cancel()
for {
wd, ok := <-changes
if !ok {
t.Fatalf("watch channel unexpectedly closed")
}
if wd.Err == topo.ErrInterrupted {
break
}
if wd.Err != nil {
t.Fatalf("watch channel unexpectedly got unknown error: %v", wd.Err)
}
if !proto.Equal(wd.Value, wanted) {
t.Fatalf("got bad data: %v expected: %v", wd.Value, wanted)
}
t.Log("got duplicate right value, skipping.")
}
// Cancel should still work here, although it does nothing.
cancel()
}
func TestWatchSrvKeyspaceNoNode(t *testing.T) {
cell := "cell1"
keyspace := "ks1"
ctx := context.Background()
mt := memorytopo.NewMemoryTopo([]string{"global", cell})
ts := topo.Server{Impl: mt}
// No SrvKeyspace -> ErrNoNode
current, _, _ := ts.WatchSrvKeyspace(ctx, cell, keyspace)
if current.Err != topo.ErrNoNode {
t.Errorf("Got invalid result from WatchSrvKeyspace(not there): %v", current.Err)
}
}
func TestMemoryTopo(t *testing.T) {
factory := func() topo.Impl {
return memorytopo.NewMemoryTopo([]string{"global", "test"})
}
t.Log("=== checkDirectory")
ts := factory()
checkDirectory(t, ts)
ts.Close()
t.Log("=== checkFile")
ts = factory()
checkFile(t, ts)
ts.Close()
}
func TestWebSocket(t *testing.T) {
ts := memorytopo.NewMemoryTopo([]string{"cell1"})
m := NewManager(topo.Server{Impl: ts})
// Register the manager to a web handler, start a web server.
m.HandleHTTPWebSocket("/workflow")
listener, err := net.Listen("tcp", ":0")
if err != nil {
t.Fatalf("Cannot listen: %v", err)
}
go http.Serve(listener, nil)
// Run the manager in the background.
wg, cancel := startManager(t, m)
// Start a client websocket.
u := url.URL{Scheme: "ws", Host: listener.Addr().String(), Path: "/workflow"}
c, _, err := websocket.DefaultDialer.Dial(u.String(), nil)
if err != nil {
t.Fatalf("WebSocket dial failed: %v", err)
}
// Read the original full dump.
_, tree, err := c.ReadMessage()
if err != nil {
t.Fatalf("WebSocket first read failed: %v", err)
}
if string(tree) != `{"nodes":null,"deletes":null,"fullUpdate":true}` {
t.Errorf("unexpected first result: %v", string(tree))
}
// Add a node, make sure we get the update.
tw := &testWorkflow{}
n := &Node{
workflow: tw,
Name: "name",
Path: "/uuid1",
Children: []*Node{},
LastChanged: 143,
}
if err := m.NodeManager().AddRootNode(n); err != nil {
t.Fatalf("adding root node failed: %v", err)
}
_, tree, err = c.ReadMessage()
if err != nil {
t.Fatalf("WebSocket first read failed: %v", err)
}
if string(tree) != `{"nodes":[{"name":"name","path":"/uuid1","children":[],"lastChanged":143,"progress":0,"progressMsg":"","state":0,"display":0,"message":"","log":"","disabled":false,"actions":null}],"deletes":null,"fullUpdate":false}` {
t.Errorf("unexpected first result: %v", string(tree))
}
// Trigger an action, make sure it goes through.
message := `{"path":"/uuid1","name":"button1"}`
if err := c.WriteMessage(websocket.TextMessage, []byte(message)); err != nil {
t.Errorf("unexpected WebSocket.WriteMessage error: %v", err)
}
for timeout := 0; ; timeout++ {
// This is an asynchronous action, need to take the lock.
tw.mu.Lock()
if len(tw.actions) == 1 && tw.actions[0].Path == n.Path && tw.actions[0].Name == "button1" {
tw.mu.Unlock()
break
}
tw.mu.Unlock()
timeout++
if timeout == 1000 {
t.Fatalf("failed to wait for action")
}
time.Sleep(time.Millisecond)
}
// Send an update, make sure we see it.
n.Modify(func() {
n.Name = "name2"
})
_, tree, err = c.ReadMessage()
if err != nil {
t.Fatalf("WebSocket update read failed: %v", err)
}
if !strings.Contains(string(tree), `"name":"name2"`) {
t.Errorf("unexpected update result: %v", string(tree))
}
// Close websocket, stop the manager.
c.Close()
cancel()
wg.Wait()
}
func TestWatchSrvKeyspace(t *testing.T) {
cell := "cell1"
keyspace := "ks1"
ctx := context.Background()
mt := memorytopo.NewMemoryTopo([]string{"global", cell})
ts := topo.Server{Impl: mt}
// Create initial value
if _, err := mt.Create(ctx, cell, "/keyspaces/"+keyspace+"/SrvKeyspace", []byte{}); err != nil {
t.Fatalf("Update(/keyspaces/ks1/SrvKeyspace) failed: %v", err)
}
// Starting the watch should now work, and return an empty
// SrvKeyspace.
wanted := &topodatapb.SrvKeyspace{}
current, changes, cancel := waitForInitialSrvKeyspace(t, ts, cell, keyspace)
if !proto.Equal(current.Value, wanted) {
t.Fatalf("got bad data: %v expected: %v", current.Value, wanted)
}
// Update the value with good data, wait until we see it
wanted.ShardingColumnName = "scn1"
contents, err := proto.Marshal(wanted)
if err != nil {
t.Fatalf("proto.Marshal(wanted) failed: %v", err)
}
if _, err := mt.Update(ctx, cell, "/keyspaces/"+keyspace+"/SrvKeyspace", contents, nil); err != nil {
t.Fatalf("Update(/keyspaces/ks1/SrvKeyspace) failed: %v", err)
}
for {
wd, ok := <-changes
if !ok {
t.Fatalf("watch channel unexpectedly closed")
}
if wd.Err != nil {
t.Fatalf("watch channel unexpectedly got error: %v", wd.Err)
}
if proto.Equal(wd.Value, wanted) {
break
}
if proto.Equal(wd.Value, &topodatapb.SrvKeyspace{}) {
t.Log("got duplicate empty value, skipping.")
}
t.Fatalf("got bad data: %v expected: %v", wd.Value, wanted)
}
// Update the value with bad data, wait until error.
if _, err := mt.Update(ctx, cell, "/keyspaces/"+keyspace+"/SrvKeyspace", []byte("BAD PROTO DATA"), nil); err != nil {
t.Fatalf("Update(/keyspaces/ks1/SrvKeyspace) failed: %v", err)
}
for {
wd, ok := <-changes
if !ok {
t.Fatalf("watch channel unexpectedly closed")
}
if wd.Err != nil {
if strings.Contains(wd.Err.Error(), "error unpacking SrvKeyspace object") {
break
}
t.Fatalf("watch channel unexpectedly got unknown error: %v", wd.Err)
}
if !proto.Equal(wd.Value, wanted) {
t.Fatalf("got bad data: %v expected: %v", wd.Value, wanted)
}
t.Log("got duplicate right value, skipping.")
}
// Cancel should still work here, although it does nothing.
cancel()
// Bad data in topo, setting the watch should now fail.
current, changes, cancel = ts.WatchSrvKeyspace(ctx, cell, keyspace)
if current.Err == nil || !strings.Contains(current.Err.Error(), "error unpacking initial SrvKeyspace object") {
t.Fatalf("expected an initial error setting watch on bad content, but got: %v", current.Err)
}
// Update content, wait until Watch works again
if _, err := mt.Update(ctx, cell, "/keyspaces/"+keyspace+"/SrvKeyspace", contents, nil); err != nil {
t.Fatalf("Update(/keyspaces/ks1/SrvKeyspace) failed: %v", err)
}
start := time.Now()
for {
current, changes, cancel = ts.WatchSrvKeyspace(ctx, cell, keyspace)
if current.Err != nil {
if strings.Contains(current.Err.Error(), "error unpacking initial SrvKeyspace object") {
// hasn't changed yet
if time.Now().Sub(start) > 10*time.Second {
t.Fatalf("time out waiting for file to appear")
}
time.Sleep(10 * time.Millisecond)
continue
}
t.Fatalf("got unexpected error while setting watch: %v", err)
}
if !proto.Equal(current.Value, wanted) {
t.Fatalf("got bad data: %v expected: %v", current.Value, wanted)
}
break
}
// Delete node, wait for error (skip any duplicate).
if err := mt.Delete(ctx, cell, "/keyspaces/"+keyspace+"/SrvKeyspace", nil); err != nil {
t.Fatalf("Delete(/keyspaces/ks1/SrvKeyspace) failed: %v", err)
}
for {
wd, ok := <-changes
if !ok {
t.Fatalf("watch channel unexpectedly closed")
}
if wd.Err == topo.ErrNoNode {
break
}
if wd.Err != nil {
t.Fatalf("watch channel unexpectedly got unknown error: %v", wd.Err)
}
if !proto.Equal(wd.Value, wanted) {
t.Fatalf("got bad data: %v expected: %v", wd.Value, wanted)
}
t.Log("got duplicate right value, skipping.")
}
}
// TestManagerRestart starts a job within a manager, stops the
// manager, restarts a manager, and stops the job.
func TestManagerRestart(t *testing.T) {
ts := topo.Server{Impl: memorytopo.NewMemoryTopo([]string{"cell1"})}
m := NewManager(ts)
// Run the manager in the background.
wg, cancel := startManager(t, m)
// Create a Sleep job.
uuid, err := m.Create(context.Background(), sleepFactoryName, []string{"-duration", "60"})
if err != nil {
t.Fatalf("cannot create sleep workflow: %v", err)
}
// Start the job.
if err := m.Start(context.Background(), uuid); err != nil {
t.Fatalf("cannot start sleep workflow: %v", err)
}
// Stop the manager.
cancel()
wg.Wait()
// Make sure the workflow is still in the topo server. This
// validates that interrupting the Manager leaves the jobs in
// the right state in the topo server.
wi, err := ts.GetWorkflow(context.Background(), uuid)
if err != nil {
t.Fatalf("cannot read workflow %v: %v", uuid, err)
}
if wi.State != workflowpb.WorkflowState_Running {
t.Fatalf("unexpected workflow state %v was expecting %v", wi.State, workflowpb.WorkflowState_Running)
}
// Restart the manager.
wg, cancel = startManager(t, m)
// Make sure the job is in there shortly.
timeout := 0
for {
tree, err := m.NodeManager().GetFullTree()
if err != nil {
t.Fatalf("cannot get full node tree: %v", err)
}
if strings.Contains(string(tree), uuid) {
break
}
timeout++
if timeout == 1000 {
t.Fatalf("failed to wait for full node tree to appear: %v", string(tree))
}
time.Sleep(time.Millisecond)
}
// Stop the job. Note Stop() waits until the background go
// routine that saves the job is done, so when we return from
// this call, the job is saved with the right updated State
// inside the topo server.
if err := m.Stop(context.Background(), uuid); err != nil {
t.Fatalf("cannot stop sleep workflow: %v", err)
}
// And stop the manager.
cancel()
wg.Wait()
// Make sure the workflow is stopped in the topo server.
wi, err = ts.GetWorkflow(context.Background(), uuid)
if err != nil {
t.Fatalf("cannot read workflow %v: %v", uuid, err)
}
if wi.State != workflowpb.WorkflowState_Done {
t.Fatalf("unexpected workflow state %v was expecting %v", wi.State, workflowpb.WorkflowState_Running)
}
if !strings.Contains(wi.Error, "canceled") {
t.Errorf("invalid workflow error: %v", wi.Error)
}
}