func (zk *zkf) RemoveService(service *service.Service) error {
// acquire the service lock to prevent that service from being scheduled
// as it is being deleted
conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(service.PoolID))
if err != nil {
return err
}
// remove the global list of all vhosts deployed
if rootconn, err := zzk.GetLocalConnection("/"); err != nil {
return err
} else if err := zkservice.RemoveServiceVhosts(rootconn, service); err != nil {
return err
}
// Ensure that the service's pool is locked for the duration
finish := make(chan interface{})
defer close(finish)
if err := zkservice.EnsureServiceLock(nil, finish, conn); err != nil {
return err
}
// FIXME: this may be a long-running operation, should we institute a timeout?
return zkservice.RemoveService(conn, service.ID)
}
func getTestConn(c *C, path string) client.Connection {
root, err := zzk.GetLocalConnection("/")
c.Assert(err, IsNil)
err = root.CreateDir(path)
c.Assert(err, IsNil)
conn, err := zzk.GetLocalConnection(path)
c.Assert(err, IsNil)
return conn
}
func (this *ControlPlaneDao) GetRunningServicesForHost(hostID string, services *[]dao.RunningService) error {
// we initialize the data container to something here in case it has not been initialized yet
*services = make([]dao.RunningService, 0)
myHost, err := this.facade.GetHost(datastore.Get(), hostID)
if err != nil {
glog.Errorf("Unable to get host %v: %v", hostID, err)
return err
} else if myHost == nil {
return nil
}
poolBasedConn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(myHost.PoolID))
if err != nil {
glog.Errorf("Error in getting a connection based on pool %v: %v", myHost.PoolID, err)
return err
}
*services, err = zkservice.LoadRunningServicesByHost(poolBasedConn, hostID)
if err != nil {
glog.Errorf("zkservice.LoadRunningServicesByHost (conn: %+v host: %v) failed: %v", poolBasedConn, hostID, err)
return err
}
return nil
}
func (this *ControlPlaneDao) GetRunningServices(request dao.EntityRequest, allRunningServices *[]dao.RunningService) error {
// we initialize the data container to something here in case it has not been initialized yet
*allRunningServices = make([]dao.RunningService, 0)
allPools, err := this.facade.GetResourcePools(datastore.Get())
if err != nil {
glog.Error("runningservice.go failed to get resource pool")
return err
} else if allPools == nil || len(allPools) == 0 {
return fmt.Errorf("no resource pools found")
}
for _, aPool := range allPools {
poolBasedConn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(aPool.ID))
if err != nil {
glog.Error("runningservice.go Failed to get connection based on pool: %v", aPool.ID)
return err
}
singlePoolRunningServices := []dao.RunningService{}
singlePoolRunningServices, err = zkservice.LoadRunningServices(poolBasedConn)
if err != nil {
glog.Errorf("Failed GetAllRunningServices: %v", err)
return err
}
for _, rs := range singlePoolRunningServices {
*allRunningServices = append(*allRunningServices, rs)
}
}
return nil
}
func (z *zkf) CheckRunningVHost(vhostName, serviceID string) error {
rootBasedConnection, err := zzk.GetLocalConnection("/")
if err != nil {
return err
}
vr, err := zkregistry.VHostRegistry(rootBasedConnection)
if err != nil {
glog.Errorf("Error getting vhost registry: %v", err)
return err
}
vhostEphemeralNodes, err := vr.GetVHostKeyChildren(rootBasedConnection, vhostName)
if err != nil {
glog.Errorf("GetVHostKeyChildren failed %v: %v", vhostName, err)
return err
}
if len(vhostEphemeralNodes) > 0 {
if vhost := vhostEphemeralNodes[0]; vhost.ServiceID != serviceID {
err := fmt.Errorf("virtual host %s is already running under service %s", vhostName, vhost.ServiceID)
return err
}
}
return nil
}
func (this *ControlPlaneDao) GetRunningService(request dao.ServiceStateRequest, running *dao.RunningService) error {
glog.V(3).Infof("ControlPlaneDao.GetRunningService: request=%v", request)
*running = dao.RunningService{}
serviceID := request.ServiceID
poolID, err := this.facade.GetPoolForService(datastore.Get(), serviceID)
if err != nil {
glog.Errorf("Unable to get service %v: %v", serviceID, err)
return err
}
poolBasedConn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(poolID))
if err != nil {
glog.Errorf("Error in getting a connection based on pool %v: %v", poolID, err)
return err
}
if thisRunning, err := zkservice.LoadRunningService(poolBasedConn, request.ServiceID, request.ServiceStateID); err != nil {
glog.Errorf("zkservice.LoadRunningService failed (conn: %+v serviceID: %v): %v", poolBasedConn, request.ServiceID, err)
return err
} else {
if thisRunning != nil {
*running = *thisRunning
}
}
return nil
}
func (z *zkf) UpdateResourcePool(pool *pool.ResourcePool) error {
conn, err := zzk.GetLocalConnection("/")
if err != nil {
return err
}
return zkservice.UpdateResourcePool(conn, pool)
}
func (z *zkf) UpdateHost(host *host.Host) error {
conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(host.PoolID))
if err != nil {
return err
}
return zkhost.UpdateHost(conn, host)
}
func (dfs *DistributedFilesystem) IsLocked() (bool, error) {
conn, err := zzk.GetLocalConnection("/")
if err != nil {
return false, err
}
return zkservice.IsServiceLocked(conn)
}
func (z *zkf) RemoveResourcePool(poolID string) error {
conn, err := zzk.GetLocalConnection("/")
if err != nil {
return err
}
return zkservice.RemoveResourcePool(conn, poolID)
}
func (z *zkf) RemoveVirtualIP(virtualIP *pool.VirtualIP) error {
conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(virtualIP.PoolID))
if err != nil {
return err
}
return zkvirtualip.RemoveVirtualIP(conn, virtualIP.IP)
}
func (this *ControlPlaneDao) GetRunningServicesForService(serviceID string, services *[]dao.RunningService) error {
// we initialize the data container to something here in case it has not been initialized yet
*services = make([]dao.RunningService, 0)
poolID, err := this.facade.GetPoolForService(datastore.Get(), serviceID)
if err != nil {
glog.Errorf("Unable to get service %v: %v", serviceID, err)
return err
}
poolBasedConn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(poolID))
if err != nil {
glog.Errorf("Error in getting a connection based on pool %v: %v", poolID, err)
return err
}
svcs, err := zkservice.LoadRunningServicesByService(poolBasedConn, serviceID)
if err != nil {
glog.Errorf("LoadRunningServicesByService failed (conn: %+v serviceID: %v): %v", poolBasedConn, serviceID, err)
return err
}
for _, svc := range svcs {
*services = append(*services, svc)
}
return nil
}
func (zk *zkf) StopServiceInstance(poolID, hostID, stateID string) error {
conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(poolID))
if err != nil {
return err
}
return zkservice.StopServiceInstance(conn, hostID, stateID)
}
func (z *zkf) GetActiveHosts(poolID string, hosts *[]string) error {
conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(poolID))
if err != nil {
return err
}
*hosts, err = zkhost.GetActiveHosts(conn)
return err
}
func (zk *zkf) UpdateService(service *service.Service) error {
conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(service.PoolID))
if err != nil {
return err
}
if err := zkservice.UpdateService(conn, service); err != nil {
return err
}
rootconn, err := zzk.GetLocalConnection("/")
if err != nil {
return err
}
return zkservice.UpdateServiceVhosts(rootconn, service)
}
func (zk *zkf) WaitService(service *service.Service, state service.DesiredState, cancel <-chan interface{}) error {
conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(service.PoolID))
if err != nil {
return err
}
return zkservice.WaitService(cancel, conn, service.ID, state)
}
func TestClient(t *testing.T) {
t.Skipf("Test cluster is not set up properly")
zookeeper.EnsureZkFatjar()
basePath := ""
tc, err := zklib.StartTestCluster(1, nil, nil)
if err != nil {
t.Fatalf("could not start test zk cluster: %s", err)
}
defer os.RemoveAll(tc.Path)
defer tc.Stop()
time.Sleep(time.Second)
servers := []string{fmt.Sprintf("127.0.0.1:%d", tc.Servers[0].Port)}
dsnBytes, err := json.Marshal(zookeeper.DSN{Servers: servers, Timeout: time.Second * 15})
if err != nil {
t.Fatal("unexpected error creating zk DSN: %s", err)
}
dsn := string(dsnBytes)
zClient, err := client.New("zookeeper", dsn, basePath, nil)
zzk.InitializeLocalClient(zClient)
conn, err := zzk.GetLocalConnection("/")
if err != nil {
t.Fatal("unexpected error getting connection")
}
h := host.New()
h.ID = "nodeID"
h.IPAddr = "192.168.1.5"
h.PoolID = "default1"
defer func(old func(string, os.FileMode) error) {
mkdirAll = old
}(mkdirAll)
dir, err := ioutil.TempDir("", "serviced_var_")
if err != nil {
t.Fatalf("could not create tempdir: %s", err)
}
defer os.RemoveAll(dir)
c, err := NewClient(h, dir)
if err != nil {
t.Fatalf("unexpected error creating client: %s", err)
}
defer c.Close()
time.Sleep(time.Second * 5)
// therefore, we need to check that the client was added under the pool from root
nodePath := fmt.Sprintf("/storage/clients/%s", h.IPAddr)
glog.Infof("about to check for %s", nodePath)
if exists, err := conn.Exists(nodePath); err != nil {
t.Fatalf("did not expect error checking for existence of %s: %s", nodePath, err)
} else {
if !exists {
t.Fatalf("could not find %s", nodePath)
}
}
}
func (zk *zkf) GetServiceStates(poolID string, states *[]servicestate.ServiceState, serviceIDs ...string) error {
conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(poolID))
if err != nil {
return err
}
*states, err = zkservice.GetServiceStates(conn, serviceIDs...)
return err
}
func (c *Controller) unregisterVhosts() {
conn, err := zzk.GetLocalConnection("/")
if err != nil {
return
}
for _, path := range c.vhostZKPaths {
glog.V(1).Infof("controller shutdown deleting vhost %v", path)
conn.Delete(path)
}
}
func (t *ZZKTest) TestActionListener_Listen(c *C) {
conn, err := zzk.GetLocalConnection("/")
c.Assert(err, IsNil)
handler := &TestActionHandler{
ResultMap: map[string]ActionResult{
"success": ActionResult{2 * time.Second, []byte("success"), nil},
"failure": ActionResult{time.Second, []byte("message failure"), fmt.Errorf("failure")},
},
}
c.Log("Start actions and shutdown")
shutdown := make(chan interface{})
done := make(chan interface{})
listener := NewActionListener(handler, "test-host-1")
go func() {
zzk.Listen(shutdown, make(chan error, 1), conn, listener)
close(done)
}()
// send actions
var wg sync.WaitGroup
sendAction := func(dockerID string, command []string) {
id, err := SendAction(conn, &Action{
HostID: listener.hostID,
DockerID: dockerID,
Command: command,
})
c.Assert(err, IsNil)
// There *might* be a race condition here if the node is processed before
// we acquire the event data (see duration timeouts above)
event, err := conn.GetW(actionPath(listener.hostID, id), &Action{})
c.Assert(err, IsNil)
wg.Add(1)
go func() {
defer wg.Done()
<-event
}()
return
}
sendAction("success", []string{"do", "some", "command"})
sendAction("failure", []string{"do", "some", "bad", "command"})
c.Log("Waiting for actions to complete")
wg.Wait()
c.Log("Actions completed")
close(shutdown)
<-done
}
func (dfs *DistributedFilesystem) desynchronize(image *docker.Image) error {
// inspect the image
dImg, err := image.Inspect()
if err != nil {
glog.Errorf("Could not inspect image %s (%s): %s", image.ID, image.UUID, err)
return err
}
// look up services for that tenant
svcs, err := dfs.facade.GetServices(datastore.Get(), dao.ServiceRequest{TenantID: image.ID.User})
if err != nil {
glog.Errorf("Could not get services for tenant %s from %s (%s): %s", image.ID.User, image.ID, image.UUID, err)
return err
}
for _, svc := range svcs {
// figure out which services are using the provided image
svcImageID, err := commons.ParseImageID(svc.ImageID)
if err != nil {
glog.Warningf("Could not parse image %s for %s (%s): %s", svc.ImageID, svc.Name, svc.ID)
continue
} else if !svcImageID.Equals(image.ID) {
continue
}
// TODO: we need to switch to using dao.ControlPlane
conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(svc.PoolID))
if err != nil {
glog.Warningf("Could not acquire connection to the coordinator (%s): %s", svc.PoolID, err)
continue
}
states, err := zkservice.GetServiceStates(conn, svc.ID)
if err != nil {
glog.Warningf("Could not get running services for %s (%s): %s", svc.Name, svc.ID)
continue
}
for _, state := range states {
// check if the instance has been running since before the commit
if state.IsRunning() && state.Started.Before(dImg.Created) {
state.InSync = false
if err := zkservice.UpdateServiceState(conn, &state); err != nil {
glog.Warningf("Could not update service state %s for %s (%s) as out of sync: %s", state.ID, svc.Name, svc.ID, err)
continue
}
}
}
}
return nil
}
func (this *ControlPlaneDao) getPoolBasedConnection(serviceID string) (client.Connection, error) {
poolID, err := this.facade.GetPoolForService(datastore.Get(), serviceID)
if err != nil {
glog.V(2).Infof("ControlPlaneDao.GetPoolForService service=%+v err=%s", serviceID, err)
return nil, err
}
poolBasedConn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(poolID))
if err != nil {
glog.Errorf("Error in getting a connection based on pool %v: %v", poolID, err)
return nil, err
}
return poolBasedConn, nil
}
//SetUpSuite is run before the tests to ensure elastic, zookeeper etc. are running.
func (dt *DaoTest) SetUpSuite(c *C) {
docker.SetUseRegistry(true)
dt.Port = 9202
isvcs.Init()
isvcs.Mgr.SetVolumesDir("/tmp/serviced-test")
esServicedClusterName, _ := utils.NewUUID36()
if err := isvcs.Mgr.SetConfigurationOption("elasticsearch-serviced", "cluster", esServicedClusterName); err != nil {
c.Fatalf("Could not set elasticsearch-serviced clustername: %s", err)
}
esLogstashClusterName, _ := utils.NewUUID36()
if err := isvcs.Mgr.SetConfigurationOption("elasticsearch-logstash", "cluster", esLogstashClusterName); err != nil {
c.Fatalf("Could not set elasticsearch-logstash clustername: %s", err)
}
isvcs.Mgr.Wipe()
if err := isvcs.Mgr.Start(); err != nil {
c.Fatalf("Could not start es container: %s", err)
}
dt.MappingsFile = "controlplane.json"
dt.FacadeTest.SetUpSuite(c)
dsn := coordzk.NewDSN([]string{"127.0.0.1:2181"}, time.Second*15).String()
glog.Infof("zookeeper dsn: %s", dsn)
zClient, err := coordclient.New("zookeeper", dsn, "", nil)
if err != nil {
glog.Fatalf("Could not start es container: %s", err)
}
zzk.InitializeLocalClient(zClient)
dt.zkConn, err = zzk.GetLocalConnection("/")
if err != nil {
c.Fatalf("could not get zk connection %v", err)
}
dt.Dao, err = NewControlSvc("localhost", int(dt.Port), dt.Facade, "/tmp", "rsync", 4979, time.Minute*5, "localhost:5000", MockStorageDriver{})
if err != nil {
glog.Fatalf("Could not start es container: %s", err)
} else {
for i := 0; i < 10; i += 1 {
id := strconv.Itoa(i)
dt.Dao.RemoveService(id, &unused)
}
for i := 100; i < 110; i += 1 {
id := strconv.Itoa(i)
dt.Dao.RemoveService(id, &unused)
}
}
}
func (t *ZZKTest) TestHostStateListener_Listen_BadState(c *C) {
conn, err := zzk.GetLocalConnection("/base_badstate")
c.Assert(err, IsNil)
shutdown := make(chan interface{})
defer close(shutdown)
errC := make(chan error, 1)
handler := new(TestHostStateHandler).init()
listener := NewHostStateListener(handler, "test-host-1")
// Add a service
svc := service.Service{ID: "test-service-1", Instances: 3}
err = UpdateService(conn, &svc)
c.Assert(err, IsNil)
// Add the host
err = AddHost(conn, &host.Host{ID: "test-host-1"})
c.Assert(err, IsNil)
// Create a host state without a service instance (this should not spin!)
badstate := HostState{
HostID: listener.hostID,
ServiceID: svc.ID,
ServiceStateID: "fail123",
DesiredState: int(service.SVCRun),
}
err = conn.Create(hostpath(badstate.HostID, badstate.ServiceStateID), &badstate)
c.Assert(err, IsNil)
err = conn.Set(hostpath(badstate.HostID, badstate.ServiceStateID), &badstate)
c.Assert(err, IsNil)
// Set up a watch
event, err := conn.GetW(hostpath(badstate.HostID, badstate.ServiceStateID), &HostState{})
c.Assert(err, IsNil)
// Start the listener
go zzk.Listen(shutdown, errC, conn, listener)
select {
case e := <-event:
c.Assert(e.Type, Equals, client.EventNodeDeleted)
case <-time.After(zzk.ZKTestTimeout):
c.Fatalf("timeout waiting for event")
}
}
func (z *zkf) RemoveHost(host *host.Host) error {
// acquire the service lock to prevent services from being scheduled
// to that pool
conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(host.PoolID))
if err != nil {
return err
}
cancel := make(chan interface{})
go func() {
defer close(cancel)
<-time.After(2 * time.Minute)
}()
// Ensure that the service's pool is locked for the duration
finish := make(chan interface{})
defer close(finish)
if err := zkservice.EnsureServiceLock(cancel, finish, conn); err != nil {
return err
}
return zkhost.RemoveHost(cancel, conn, host.ID)
}
func (this *ControlPlaneDao) StopRunningInstance(request dao.HostServiceRequest, unused *int) error {
myHost, err := this.facade.GetHost(datastore.Get(), request.HostID)
if err != nil {
glog.Errorf("Unable to get host %v: %v", request.HostID, err)
return err
}
if myHost == nil {
return fmt.Errorf("Host %s does not exist", request.HostID)
}
poolBasedConn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(myHost.PoolID))
if err != nil {
glog.Errorf("Error in getting a connection based on pool %v: %v", myHost.PoolID, err)
return err
}
if err := zkservice.StopServiceInstance(poolBasedConn, request.HostID, request.ServiceStateID); err != nil {
glog.Errorf("zkservice.StopServiceInstance failed (conn: %+v hostID: %v serviceStateID: %v): %v", poolBasedConn, request.HostID, request.ServiceStateID, err)
return err
}
return nil
}
func (c *Controller) watchregistry() <-chan struct{} {
alert := make(chan struct{}, 1)
go func() {
paths := append(c.vhostZKPaths, c.exportedEndpointZKPaths...)
if len(paths) == 0 {
return
}
conn, err := zzk.GetLocalConnection("/")
if err != nil {
return
}
endpointRegistry, err := registry.CreateEndpointRegistry(conn)
if err != nil {
glog.Errorf("Could not get EndpointRegistry. Endpoints not checked: %v", err)
return
}
interval := time.Tick(60 * time.Second)
defer func() { alert <- struct{}{} }()
for {
select {
case <-interval:
for _, path := range paths {
if _, err := endpointRegistry.GetItem(conn, path); err != nil {
glog.Errorf("Could not get endpoint. %v", err)
return
}
}
}
}
}()
return alert
}
func NewDistributedFilesystem(fsType, varpath, dockerRegistry string, facade *facade.Facade, timeout time.Duration, networkDriver storage.StorageDriver) (*DistributedFilesystem, error) {
host, port, err := parseRegistry(dockerRegistry)
if err != nil {
return nil, err
}
conn, err := zzk.GetLocalConnection("/")
if err != nil {
return nil, err
}
lock := zkservice.ServiceLock(conn)
return &DistributedFilesystem{
fsType: fsType,
varpath: varpath,
dockerHost: host,
dockerPort: port,
facade: facade,
timeout: timeout,
lock: lock,
networkDriver: networkDriver,
}, nil
}
func (t *ZZKTest) TestActionListener_Spawn(c *C) {
conn, err := zzk.GetLocalConnection("/")
c.Assert(err, IsNil)
handler := &TestActionHandler{
ResultMap: map[string]ActionResult{
"success": ActionResult{time.Second, []byte("success"), nil},
"failure": ActionResult{time.Second, []byte("message failure"), fmt.Errorf("failure")},
},
}
listener := NewActionListener(handler, "test-host-1")
listener.SetConnection(conn)
// send actions
c.Logf("Sending successful command")
success, err := SendAction(conn, &Action{
HostID: listener.hostID,
DockerID: "success",
Command: []string{"do", "some", "command"},
})
if err != nil {
c.Fatalf("Could not send success action")
}
listener.Spawn(nil, success)
c.Logf("Sending failure command")
failure, err := SendAction(conn, &Action{
HostID: listener.hostID,
DockerID: "failure",
Command: []string{"do", "some", "bad", "command"},
})
if err != nil {
c.Fatalf("Could not send failure action")
}
listener.Spawn(make(<-chan interface{}), failure)
}
func (this *ControlPlaneDao) Action(request dao.AttachRequest, unused *int) error {
ctx := datastore.Get()
svc, err := this.facade.GetService(ctx, request.Running.ServiceID)
if err != nil {
return err
}
var command []string
if request.Command == "" {
return fmt.Errorf("missing command")
}
if err := svc.EvaluateActionsTemplate(serviceGetter(ctx, this.facade), childFinder(ctx, this.facade), request.Running.InstanceID); err != nil {
return err
}
action, ok := svc.Actions[request.Command]
if !ok {
return fmt.Errorf("action not found for service %s: %s", svc.ID, request.Command)
}
command = append([]string{action}, request.Args...)
req := zkdocker.Action{
HostID: request.Running.HostID,
DockerID: request.Running.DockerID,
Command: command,
}
conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(svc.PoolID))
if err != nil {
return err
}
_, err = zkdocker.SendAction(conn, &req)
return err
}