func (zk *zkf) RemoveService(service *service.Service) error { // acquire the service lock to prevent that service from being scheduled // as it is being deleted conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(service.PoolID)) if err != nil { return err } // remove the global list of all vhosts deployed if rootconn, err := zzk.GetLocalConnection("/"); err != nil { return err } else if err := zkservice.RemoveServiceVhosts(rootconn, service); err != nil { return err } // Ensure that the service's pool is locked for the duration finish := make(chan interface{}) defer close(finish) if err := zkservice.EnsureServiceLock(nil, finish, conn); err != nil { return err } // FIXME: this may be a long-running operation, should we institute a timeout? return zkservice.RemoveService(conn, service.ID) }
func getTestConn(c *C, path string) client.Connection { root, err := zzk.GetLocalConnection("/") c.Assert(err, IsNil) err = root.CreateDir(path) c.Assert(err, IsNil) conn, err := zzk.GetLocalConnection(path) c.Assert(err, IsNil) return conn }
func (this *ControlPlaneDao) GetRunningServicesForHost(hostID string, services *[]dao.RunningService) error { // we initialize the data container to something here in case it has not been initialized yet *services = make([]dao.RunningService, 0) myHost, err := this.facade.GetHost(datastore.Get(), hostID) if err != nil { glog.Errorf("Unable to get host %v: %v", hostID, err) return err } else if myHost == nil { return nil } poolBasedConn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(myHost.PoolID)) if err != nil { glog.Errorf("Error in getting a connection based on pool %v: %v", myHost.PoolID, err) return err } *services, err = zkservice.LoadRunningServicesByHost(poolBasedConn, hostID) if err != nil { glog.Errorf("zkservice.LoadRunningServicesByHost (conn: %+v host: %v) failed: %v", poolBasedConn, hostID, err) return err } return nil }
func (this *ControlPlaneDao) GetRunningServices(request dao.EntityRequest, allRunningServices *[]dao.RunningService) error { // we initialize the data container to something here in case it has not been initialized yet *allRunningServices = make([]dao.RunningService, 0) allPools, err := this.facade.GetResourcePools(datastore.Get()) if err != nil { glog.Error("runningservice.go failed to get resource pool") return err } else if allPools == nil || len(allPools) == 0 { return fmt.Errorf("no resource pools found") } for _, aPool := range allPools { poolBasedConn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(aPool.ID)) if err != nil { glog.Error("runningservice.go Failed to get connection based on pool: %v", aPool.ID) return err } singlePoolRunningServices := []dao.RunningService{} singlePoolRunningServices, err = zkservice.LoadRunningServices(poolBasedConn) if err != nil { glog.Errorf("Failed GetAllRunningServices: %v", err) return err } for _, rs := range singlePoolRunningServices { *allRunningServices = append(*allRunningServices, rs) } } return nil }
func (z *zkf) CheckRunningVHost(vhostName, serviceID string) error { rootBasedConnection, err := zzk.GetLocalConnection("/") if err != nil { return err } vr, err := zkregistry.VHostRegistry(rootBasedConnection) if err != nil { glog.Errorf("Error getting vhost registry: %v", err) return err } vhostEphemeralNodes, err := vr.GetVHostKeyChildren(rootBasedConnection, vhostName) if err != nil { glog.Errorf("GetVHostKeyChildren failed %v: %v", vhostName, err) return err } if len(vhostEphemeralNodes) > 0 { if vhost := vhostEphemeralNodes[0]; vhost.ServiceID != serviceID { err := fmt.Errorf("virtual host %s is already running under service %s", vhostName, vhost.ServiceID) return err } } return nil }
func (this *ControlPlaneDao) GetRunningService(request dao.ServiceStateRequest, running *dao.RunningService) error { glog.V(3).Infof("ControlPlaneDao.GetRunningService: request=%v", request) *running = dao.RunningService{} serviceID := request.ServiceID poolID, err := this.facade.GetPoolForService(datastore.Get(), serviceID) if err != nil { glog.Errorf("Unable to get service %v: %v", serviceID, err) return err } poolBasedConn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(poolID)) if err != nil { glog.Errorf("Error in getting a connection based on pool %v: %v", poolID, err) return err } if thisRunning, err := zkservice.LoadRunningService(poolBasedConn, request.ServiceID, request.ServiceStateID); err != nil { glog.Errorf("zkservice.LoadRunningService failed (conn: %+v serviceID: %v): %v", poolBasedConn, request.ServiceID, err) return err } else { if thisRunning != nil { *running = *thisRunning } } return nil }
func (z *zkf) UpdateResourcePool(pool *pool.ResourcePool) error { conn, err := zzk.GetLocalConnection("/") if err != nil { return err } return zkservice.UpdateResourcePool(conn, pool) }
func (z *zkf) UpdateHost(host *host.Host) error { conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(host.PoolID)) if err != nil { return err } return zkhost.UpdateHost(conn, host) }
func (dfs *DistributedFilesystem) IsLocked() (bool, error) { conn, err := zzk.GetLocalConnection("/") if err != nil { return false, err } return zkservice.IsServiceLocked(conn) }
func (z *zkf) RemoveResourcePool(poolID string) error { conn, err := zzk.GetLocalConnection("/") if err != nil { return err } return zkservice.RemoveResourcePool(conn, poolID) }
func (z *zkf) RemoveVirtualIP(virtualIP *pool.VirtualIP) error { conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(virtualIP.PoolID)) if err != nil { return err } return zkvirtualip.RemoveVirtualIP(conn, virtualIP.IP) }
func (this *ControlPlaneDao) GetRunningServicesForService(serviceID string, services *[]dao.RunningService) error { // we initialize the data container to something here in case it has not been initialized yet *services = make([]dao.RunningService, 0) poolID, err := this.facade.GetPoolForService(datastore.Get(), serviceID) if err != nil { glog.Errorf("Unable to get service %v: %v", serviceID, err) return err } poolBasedConn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(poolID)) if err != nil { glog.Errorf("Error in getting a connection based on pool %v: %v", poolID, err) return err } svcs, err := zkservice.LoadRunningServicesByService(poolBasedConn, serviceID) if err != nil { glog.Errorf("LoadRunningServicesByService failed (conn: %+v serviceID: %v): %v", poolBasedConn, serviceID, err) return err } for _, svc := range svcs { *services = append(*services, svc) } return nil }
func (zk *zkf) StopServiceInstance(poolID, hostID, stateID string) error { conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(poolID)) if err != nil { return err } return zkservice.StopServiceInstance(conn, hostID, stateID) }
func (z *zkf) GetActiveHosts(poolID string, hosts *[]string) error { conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(poolID)) if err != nil { return err } *hosts, err = zkhost.GetActiveHosts(conn) return err }
func (zk *zkf) UpdateService(service *service.Service) error { conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(service.PoolID)) if err != nil { return err } if err := zkservice.UpdateService(conn, service); err != nil { return err } rootconn, err := zzk.GetLocalConnection("/") if err != nil { return err } return zkservice.UpdateServiceVhosts(rootconn, service) }
func (zk *zkf) WaitService(service *service.Service, state service.DesiredState, cancel <-chan interface{}) error { conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(service.PoolID)) if err != nil { return err } return zkservice.WaitService(cancel, conn, service.ID, state) }
func TestClient(t *testing.T) { t.Skipf("Test cluster is not set up properly") zookeeper.EnsureZkFatjar() basePath := "" tc, err := zklib.StartTestCluster(1, nil, nil) if err != nil { t.Fatalf("could not start test zk cluster: %s", err) } defer os.RemoveAll(tc.Path) defer tc.Stop() time.Sleep(time.Second) servers := []string{fmt.Sprintf("127.0.0.1:%d", tc.Servers[0].Port)} dsnBytes, err := json.Marshal(zookeeper.DSN{Servers: servers, Timeout: time.Second * 15}) if err != nil { t.Fatal("unexpected error creating zk DSN: %s", err) } dsn := string(dsnBytes) zClient, err := client.New("zookeeper", dsn, basePath, nil) zzk.InitializeLocalClient(zClient) conn, err := zzk.GetLocalConnection("/") if err != nil { t.Fatal("unexpected error getting connection") } h := host.New() h.ID = "nodeID" h.IPAddr = "192.168.1.5" h.PoolID = "default1" defer func(old func(string, os.FileMode) error) { mkdirAll = old }(mkdirAll) dir, err := ioutil.TempDir("", "serviced_var_") if err != nil { t.Fatalf("could not create tempdir: %s", err) } defer os.RemoveAll(dir) c, err := NewClient(h, dir) if err != nil { t.Fatalf("unexpected error creating client: %s", err) } defer c.Close() time.Sleep(time.Second * 5) // therefore, we need to check that the client was added under the pool from root nodePath := fmt.Sprintf("/storage/clients/%s", h.IPAddr) glog.Infof("about to check for %s", nodePath) if exists, err := conn.Exists(nodePath); err != nil { t.Fatalf("did not expect error checking for existence of %s: %s", nodePath, err) } else { if !exists { t.Fatalf("could not find %s", nodePath) } } }
func (zk *zkf) GetServiceStates(poolID string, states *[]servicestate.ServiceState, serviceIDs ...string) error { conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(poolID)) if err != nil { return err } *states, err = zkservice.GetServiceStates(conn, serviceIDs...) return err }
func (c *Controller) unregisterVhosts() { conn, err := zzk.GetLocalConnection("/") if err != nil { return } for _, path := range c.vhostZKPaths { glog.V(1).Infof("controller shutdown deleting vhost %v", path) conn.Delete(path) } }
func (t *ZZKTest) TestActionListener_Listen(c *C) { conn, err := zzk.GetLocalConnection("/") c.Assert(err, IsNil) handler := &TestActionHandler{ ResultMap: map[string]ActionResult{ "success": ActionResult{2 * time.Second, []byte("success"), nil}, "failure": ActionResult{time.Second, []byte("message failure"), fmt.Errorf("failure")}, }, } c.Log("Start actions and shutdown") shutdown := make(chan interface{}) done := make(chan interface{}) listener := NewActionListener(handler, "test-host-1") go func() { zzk.Listen(shutdown, make(chan error, 1), conn, listener) close(done) }() // send actions var wg sync.WaitGroup sendAction := func(dockerID string, command []string) { id, err := SendAction(conn, &Action{ HostID: listener.hostID, DockerID: dockerID, Command: command, }) c.Assert(err, IsNil) // There *might* be a race condition here if the node is processed before // we acquire the event data (see duration timeouts above) event, err := conn.GetW(actionPath(listener.hostID, id), &Action{}) c.Assert(err, IsNil) wg.Add(1) go func() { defer wg.Done() <-event }() return } sendAction("success", []string{"do", "some", "command"}) sendAction("failure", []string{"do", "some", "bad", "command"}) c.Log("Waiting for actions to complete") wg.Wait() c.Log("Actions completed") close(shutdown) <-done }
func (dfs *DistributedFilesystem) desynchronize(image *docker.Image) error { // inspect the image dImg, err := image.Inspect() if err != nil { glog.Errorf("Could not inspect image %s (%s): %s", image.ID, image.UUID, err) return err } // look up services for that tenant svcs, err := dfs.facade.GetServices(datastore.Get(), dao.ServiceRequest{TenantID: image.ID.User}) if err != nil { glog.Errorf("Could not get services for tenant %s from %s (%s): %s", image.ID.User, image.ID, image.UUID, err) return err } for _, svc := range svcs { // figure out which services are using the provided image svcImageID, err := commons.ParseImageID(svc.ImageID) if err != nil { glog.Warningf("Could not parse image %s for %s (%s): %s", svc.ImageID, svc.Name, svc.ID) continue } else if !svcImageID.Equals(image.ID) { continue } // TODO: we need to switch to using dao.ControlPlane conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(svc.PoolID)) if err != nil { glog.Warningf("Could not acquire connection to the coordinator (%s): %s", svc.PoolID, err) continue } states, err := zkservice.GetServiceStates(conn, svc.ID) if err != nil { glog.Warningf("Could not get running services for %s (%s): %s", svc.Name, svc.ID) continue } for _, state := range states { // check if the instance has been running since before the commit if state.IsRunning() && state.Started.Before(dImg.Created) { state.InSync = false if err := zkservice.UpdateServiceState(conn, &state); err != nil { glog.Warningf("Could not update service state %s for %s (%s) as out of sync: %s", state.ID, svc.Name, svc.ID, err) continue } } } } return nil }
func (this *ControlPlaneDao) getPoolBasedConnection(serviceID string) (client.Connection, error) { poolID, err := this.facade.GetPoolForService(datastore.Get(), serviceID) if err != nil { glog.V(2).Infof("ControlPlaneDao.GetPoolForService service=%+v err=%s", serviceID, err) return nil, err } poolBasedConn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(poolID)) if err != nil { glog.Errorf("Error in getting a connection based on pool %v: %v", poolID, err) return nil, err } return poolBasedConn, nil }
//SetUpSuite is run before the tests to ensure elastic, zookeeper etc. are running. func (dt *DaoTest) SetUpSuite(c *C) { docker.SetUseRegistry(true) dt.Port = 9202 isvcs.Init() isvcs.Mgr.SetVolumesDir("/tmp/serviced-test") esServicedClusterName, _ := utils.NewUUID36() if err := isvcs.Mgr.SetConfigurationOption("elasticsearch-serviced", "cluster", esServicedClusterName); err != nil { c.Fatalf("Could not set elasticsearch-serviced clustername: %s", err) } esLogstashClusterName, _ := utils.NewUUID36() if err := isvcs.Mgr.SetConfigurationOption("elasticsearch-logstash", "cluster", esLogstashClusterName); err != nil { c.Fatalf("Could not set elasticsearch-logstash clustername: %s", err) } isvcs.Mgr.Wipe() if err := isvcs.Mgr.Start(); err != nil { c.Fatalf("Could not start es container: %s", err) } dt.MappingsFile = "controlplane.json" dt.FacadeTest.SetUpSuite(c) dsn := coordzk.NewDSN([]string{"127.0.0.1:2181"}, time.Second*15).String() glog.Infof("zookeeper dsn: %s", dsn) zClient, err := coordclient.New("zookeeper", dsn, "", nil) if err != nil { glog.Fatalf("Could not start es container: %s", err) } zzk.InitializeLocalClient(zClient) dt.zkConn, err = zzk.GetLocalConnection("/") if err != nil { c.Fatalf("could not get zk connection %v", err) } dt.Dao, err = NewControlSvc("localhost", int(dt.Port), dt.Facade, "/tmp", "rsync", 4979, time.Minute*5, "localhost:5000", MockStorageDriver{}) if err != nil { glog.Fatalf("Could not start es container: %s", err) } else { for i := 0; i < 10; i += 1 { id := strconv.Itoa(i) dt.Dao.RemoveService(id, &unused) } for i := 100; i < 110; i += 1 { id := strconv.Itoa(i) dt.Dao.RemoveService(id, &unused) } } }
func (t *ZZKTest) TestHostStateListener_Listen_BadState(c *C) { conn, err := zzk.GetLocalConnection("/base_badstate") c.Assert(err, IsNil) shutdown := make(chan interface{}) defer close(shutdown) errC := make(chan error, 1) handler := new(TestHostStateHandler).init() listener := NewHostStateListener(handler, "test-host-1") // Add a service svc := service.Service{ID: "test-service-1", Instances: 3} err = UpdateService(conn, &svc) c.Assert(err, IsNil) // Add the host err = AddHost(conn, &host.Host{ID: "test-host-1"}) c.Assert(err, IsNil) // Create a host state without a service instance (this should not spin!) badstate := HostState{ HostID: listener.hostID, ServiceID: svc.ID, ServiceStateID: "fail123", DesiredState: int(service.SVCRun), } err = conn.Create(hostpath(badstate.HostID, badstate.ServiceStateID), &badstate) c.Assert(err, IsNil) err = conn.Set(hostpath(badstate.HostID, badstate.ServiceStateID), &badstate) c.Assert(err, IsNil) // Set up a watch event, err := conn.GetW(hostpath(badstate.HostID, badstate.ServiceStateID), &HostState{}) c.Assert(err, IsNil) // Start the listener go zzk.Listen(shutdown, errC, conn, listener) select { case e := <-event: c.Assert(e.Type, Equals, client.EventNodeDeleted) case <-time.After(zzk.ZKTestTimeout): c.Fatalf("timeout waiting for event") } }
func (z *zkf) RemoveHost(host *host.Host) error { // acquire the service lock to prevent services from being scheduled // to that pool conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(host.PoolID)) if err != nil { return err } cancel := make(chan interface{}) go func() { defer close(cancel) <-time.After(2 * time.Minute) }() // Ensure that the service's pool is locked for the duration finish := make(chan interface{}) defer close(finish) if err := zkservice.EnsureServiceLock(cancel, finish, conn); err != nil { return err } return zkhost.RemoveHost(cancel, conn, host.ID) }
func (this *ControlPlaneDao) StopRunningInstance(request dao.HostServiceRequest, unused *int) error { myHost, err := this.facade.GetHost(datastore.Get(), request.HostID) if err != nil { glog.Errorf("Unable to get host %v: %v", request.HostID, err) return err } if myHost == nil { return fmt.Errorf("Host %s does not exist", request.HostID) } poolBasedConn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(myHost.PoolID)) if err != nil { glog.Errorf("Error in getting a connection based on pool %v: %v", myHost.PoolID, err) return err } if err := zkservice.StopServiceInstance(poolBasedConn, request.HostID, request.ServiceStateID); err != nil { glog.Errorf("zkservice.StopServiceInstance failed (conn: %+v hostID: %v serviceStateID: %v): %v", poolBasedConn, request.HostID, request.ServiceStateID, err) return err } return nil }
func (c *Controller) watchregistry() <-chan struct{} { alert := make(chan struct{}, 1) go func() { paths := append(c.vhostZKPaths, c.exportedEndpointZKPaths...) if len(paths) == 0 { return } conn, err := zzk.GetLocalConnection("/") if err != nil { return } endpointRegistry, err := registry.CreateEndpointRegistry(conn) if err != nil { glog.Errorf("Could not get EndpointRegistry. Endpoints not checked: %v", err) return } interval := time.Tick(60 * time.Second) defer func() { alert <- struct{}{} }() for { select { case <-interval: for _, path := range paths { if _, err := endpointRegistry.GetItem(conn, path); err != nil { glog.Errorf("Could not get endpoint. %v", err) return } } } } }() return alert }
func NewDistributedFilesystem(fsType, varpath, dockerRegistry string, facade *facade.Facade, timeout time.Duration, networkDriver storage.StorageDriver) (*DistributedFilesystem, error) { host, port, err := parseRegistry(dockerRegistry) if err != nil { return nil, err } conn, err := zzk.GetLocalConnection("/") if err != nil { return nil, err } lock := zkservice.ServiceLock(conn) return &DistributedFilesystem{ fsType: fsType, varpath: varpath, dockerHost: host, dockerPort: port, facade: facade, timeout: timeout, lock: lock, networkDriver: networkDriver, }, nil }
func (t *ZZKTest) TestActionListener_Spawn(c *C) { conn, err := zzk.GetLocalConnection("/") c.Assert(err, IsNil) handler := &TestActionHandler{ ResultMap: map[string]ActionResult{ "success": ActionResult{time.Second, []byte("success"), nil}, "failure": ActionResult{time.Second, []byte("message failure"), fmt.Errorf("failure")}, }, } listener := NewActionListener(handler, "test-host-1") listener.SetConnection(conn) // send actions c.Logf("Sending successful command") success, err := SendAction(conn, &Action{ HostID: listener.hostID, DockerID: "success", Command: []string{"do", "some", "command"}, }) if err != nil { c.Fatalf("Could not send success action") } listener.Spawn(nil, success) c.Logf("Sending failure command") failure, err := SendAction(conn, &Action{ HostID: listener.hostID, DockerID: "failure", Command: []string{"do", "some", "bad", "command"}, }) if err != nil { c.Fatalf("Could not send failure action") } listener.Spawn(make(<-chan interface{}), failure) }
func (this *ControlPlaneDao) Action(request dao.AttachRequest, unused *int) error { ctx := datastore.Get() svc, err := this.facade.GetService(ctx, request.Running.ServiceID) if err != nil { return err } var command []string if request.Command == "" { return fmt.Errorf("missing command") } if err := svc.EvaluateActionsTemplate(serviceGetter(ctx, this.facade), childFinder(ctx, this.facade), request.Running.InstanceID); err != nil { return err } action, ok := svc.Actions[request.Command] if !ok { return fmt.Errorf("action not found for service %s: %s", svc.ID, request.Command) } command = append([]string{action}, request.Args...) req := zkdocker.Action{ HostID: request.Running.HostID, DockerID: request.Running.DockerID, Command: command, } conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(svc.PoolID)) if err != nil { return err } _, err = zkdocker.SendAction(conn, &req) return err }