func (t *ZZKTest) TestActionListener_Listen(c *C) {
conn, err := zzk.GetLocalConnection("/")
c.Assert(err, IsNil)
handler := &TestActionHandler{
ResultMap: map[string]ActionResult{
"success": ActionResult{2 * time.Second, []byte("success"), nil},
"failure": ActionResult{time.Second, []byte("message failure"), fmt.Errorf("failure")},
},
}
c.Log("Start actions and shutdown")
shutdown := make(chan interface{})
done := make(chan interface{})
listener := NewActionListener(handler, "test-host-1")
go func() {
zzk.Listen(shutdown, make(chan error, 1), conn, listener)
close(done)
}()
// send actions
var wg sync.WaitGroup
sendAction := func(dockerID string, command []string) {
id, err := SendAction(conn, &Action{
HostID: listener.hostID,
DockerID: dockerID,
Command: command,
})
c.Assert(err, IsNil)
// There *might* be a race condition here if the node is processed before
// we acquire the event data (see duration timeouts above)
event, err := conn.GetW(actionPath(listener.hostID, id), &Action{})
c.Assert(err, IsNil)
wg.Add(1)
go func() {
defer wg.Done()
<-event
}()
return
}
sendAction("success", []string{"do", "some", "command"})
sendAction("failure", []string{"do", "some", "bad", "command"})
c.Log("Waiting for actions to complete")
wg.Wait()
c.Log("Actions completed")
close(shutdown)
<-done
}
func (t *ZZKTest) TestHostStateListener_Listen_BadState(c *C) {
conn, err := zzk.GetLocalConnection("/base_badstate")
c.Assert(err, IsNil)
shutdown := make(chan interface{})
defer close(shutdown)
errC := make(chan error, 1)
handler := new(TestHostStateHandler).init()
listener := NewHostStateListener(handler, "test-host-1")
// Add a service
svc := service.Service{ID: "test-service-1", Instances: 3}
err = UpdateService(conn, &svc)
c.Assert(err, IsNil)
// Add the host
err = AddHost(conn, &host.Host{ID: "test-host-1"})
c.Assert(err, IsNil)
// Create a host state without a service instance (this should not spin!)
badstate := HostState{
HostID: listener.hostID,
ServiceID: svc.ID,
ServiceStateID: "fail123",
DesiredState: int(service.SVCRun),
}
err = conn.Create(hostpath(badstate.HostID, badstate.ServiceStateID), &badstate)
c.Assert(err, IsNil)
err = conn.Set(hostpath(badstate.HostID, badstate.ServiceStateID), &badstate)
c.Assert(err, IsNil)
// Set up a watch
event, err := conn.GetW(hostpath(badstate.HostID, badstate.ServiceStateID), &HostState{})
c.Assert(err, IsNil)
// Start the listener
go zzk.Listen(shutdown, errC, conn, listener)
select {
case e := <-event:
c.Assert(e.Type, Equals, client.EventNodeDeleted)
case <-time.After(zzk.ZKTestTimeout):
c.Fatalf("timeout waiting for event")
}
}
func (t *ZZKTest) TestHostStateListener_Listen(c *C) {
conn, err := zzk.GetLocalConnection("/base")
c.Assert(err, IsNil)
shutdown := make(chan interface{})
defer close(shutdown)
errC := make(chan error, 1)
handler := new(TestHostStateHandler).init()
listener := NewHostStateListener(handler, "test-host-1")
go zzk.Listen(shutdown, errC, conn, listener)
// Add a service
svc := service.Service{ID: "test-service-1", Instances: 3}
err = UpdateService(conn, &svc)
c.Assert(err, IsNil)
// Add host
err = AddHost(conn, &host.Host{ID: "test-host-1"})
c.Assert(err, IsNil)
// Verify that the host is registered
c.Logf("Waiting for 'test-host-1' to be registered")
select {
case err := <-errC:
c.Assert(err, IsNil)
c.Assert(listener.registry, Not(Equals), "")
exists, err := conn.Exists(listener.registry)
c.Assert(err, IsNil)
c.Assert(exists, Equals, true)
case <-time.After(zzk.ZKTestTimeout):
// NOTE: this timeout may be adjusted to satisfy race conditions
c.Fatalf("timeout waiting for host to be ready")
}
// Add states
addstates := func(hostID string, svc *service.Service, count int) []string {
c.Logf("Adding %d service states for service %s on host %s", count, svc.ID, hostID)
stateIDs := make([]string, count)
for i := 0; i < count; i++ {
state, err := servicestate.BuildFromService(svc, hostID)
c.Assert(err, IsNil)
c.Assert(state.IsRunning(), Equals, false)
err = addInstance(conn, *state)
c.Assert(err, IsNil)
_, err = LoadRunningService(conn, state.ServiceID, state.ID)
c.Assert(err, IsNil)
stateIDs[i] = state.ID
}
return stateIDs
}
stateIDs := addstates("test-host-1", &svc, 3)
wait := func(serviceID string, dState service.DesiredState) {
errC := make(chan error)
c.Logf("Waiting for service instances on 'test-host-1' to %s", dState)
go func() {
errC <- WaitService(shutdown, conn, serviceID, dState)
}()
// Wait on services or fail trying
select {
case err := <-errC:
c.Assert(err, IsNil)
case <-time.After(zzk.ZKTestTimeout):
c.Fatalf("timeout waiting for instances to %s", dState)
}
}
wait(svc.ID, service.SVCRun)
// Pause states
for _, stateID := range stateIDs {
err = pauseInstance(conn, "test-host-1", stateID)
c.Assert(err, IsNil)
}
wait(svc.ID, service.SVCPause)
// Resume states
for _, stateID := range stateIDs {
err = resumeInstance(conn, "test-host-1", stateID)
c.Assert(err, IsNil)
}
wait(svc.ID, service.SVCRun)
// Stop states
for _, stateID := range stateIDs {
err = StopServiceInstance(conn, "test-host-1", stateID)
c.Assert(err, IsNil)
}
wait(svc.ID, service.SVCStop)
}
func (t *ZZKTest) TestHostRegistryListener_Listen(c *C) {
conn, err := zzk.GetLocalConnection("/base")
c.Assert(err, IsNil)
// Initialize the host registry
err = InitHostRegistry(conn)
c.Assert(err, IsNil)
shutdown := make(chan interface{})
defer close(shutdown)
listener := NewHostRegistryListener()
go zzk.Listen(shutdown, make(chan error, 1), conn, listener)
// Add a service
svc := service.Service{ID: "test-service-1"}
err = UpdateService(conn, &svc)
c.Assert(err, IsNil)
// Add hosts
hosts := make(map[string]string)
register := func(hostID string) string {
c.Logf("Registering host %s", hostID)
host := host.Host{ID: hostID}
err := AddHost(conn, &host)
c.Assert(err, IsNil)
p, err := conn.CreateEphemeral(hostregpath(hostID), &HostNode{Host: &host})
c.Assert(err, IsNil)
return path.Base(p)
}
hosts["test-host-1"] = register("test-host-1")
hosts["test-host-2"] = register("test-host-2")
// Add states
addstates := func(hostID string, svc *service.Service, count int) []string {
c.Logf("Adding %d service states for service %s on host %s", count, svc.ID, hostID)
stateIDs := make([]string, count)
for i := 0; i < count; i++ {
state, err := servicestate.BuildFromService(svc, hostID)
c.Assert(err, IsNil)
err = addInstance(conn, *state)
c.Assert(err, IsNil)
_, err = LoadRunningService(conn, state.ServiceID, state.ID)
c.Assert(err, IsNil)
stateIDs[i] = state.ID
}
return stateIDs
}
addstates("test-host-1", &svc, 2)
addstates("test-host-2", &svc, 2)
// unregister a host and verify the states have been removed
unregister := func(hostID, ehostID string) {
var wg sync.WaitGroup
hsids, err := conn.Children(hostpath(hostID))
c.Assert(err, IsNil)
// Monitor the states per service
for _, hsid := range hsids {
var hs HostState
err = conn.Get(hostpath(hostID, hsid), &hs)
c.Assert(err, IsNil)
wg.Add(1)
go func(hsid, serviceID string) {
defer wg.Done()
for {
event, err := conn.GetW(servicepath(serviceID, hsid), &HostNode{})
c.Assert(err, IsNil)
if e := <-event; e.Type == client.EventNodeDeleted {
return
}
}
}(hs.ServiceStateID, hs.ServiceID)
}
// Monitor the host state
wg.Add(1)
go func() {
defer wg.Done()
for {
hsids, event, err := conn.ChildrenW(hostpath(hostID))
c.Assert(err, IsNil)
if len(hsids) == 0 {
return
}
<-event
}
}()
c.Logf("Unregistering host %s", hostID)
err = conn.Delete(hostregpath(ehostID))
c.Assert(err, IsNil)
wg.Wait()
}
done := make(chan struct{})
go func() {
done <- struct{}{}
unregister("test-host-1", hosts["test-host-1"])
}()
select {
case <-done:
case <-time.After(1 * time.Minute):
c.Errorf("timeout")
}
}