func (zk *zkf) GetServiceStates(poolID string, states *[]servicestate.ServiceState, serviceIDs ...string) error {
conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(poolID))
if err != nil {
return err
}
*states, err = zkservice.GetServiceStates(conn, serviceIDs...)
return err
}
/* This method assumes that if a service instance exists, it has not yet been terminated */
func (this *ControlPlaneDao) getNonTerminatedServiceStates(serviceId string, serviceStates *[]servicestate.ServiceState) error {
glog.V(2).Infof("ControlPlaneDao.getNonTerminatedServiceStates: serviceId=%s", serviceId)
poolBasedConn, err := this.getPoolBasedConnection(serviceId)
if err != nil {
return err
}
*serviceStates, err = zkservice.GetServiceStates(poolBasedConn, serviceId)
return err
}
func (dfs *DistributedFilesystem) desynchronize(image *docker.Image) error {
// inspect the image
dImg, err := image.Inspect()
if err != nil {
glog.Errorf("Could not inspect image %s (%s): %s", image.ID, image.UUID, err)
return err
}
// look up services for that tenant
svcs, err := dfs.facade.GetServices(datastore.Get(), dao.ServiceRequest{TenantID: image.ID.User})
if err != nil {
glog.Errorf("Could not get services for tenant %s from %s (%s): %s", image.ID.User, image.ID, image.UUID, err)
return err
}
for _, svc := range svcs {
// figure out which services are using the provided image
svcImageID, err := commons.ParseImageID(svc.ImageID)
if err != nil {
glog.Warningf("Could not parse image %s for %s (%s): %s", svc.ImageID, svc.Name, svc.ID)
continue
} else if !svcImageID.Equals(image.ID) {
continue
}
// TODO: we need to switch to using dao.ControlPlane
conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(svc.PoolID))
if err != nil {
glog.Warningf("Could not acquire connection to the coordinator (%s): %s", svc.PoolID, err)
continue
}
states, err := zkservice.GetServiceStates(conn, svc.ID)
if err != nil {
glog.Warningf("Could not get running services for %s (%s): %s", svc.Name, svc.ID)
continue
}
for _, state := range states {
// check if the instance has been running since before the commit
if state.IsRunning() && state.Started.Before(dImg.Created) {
state.InSync = false
if err := zkservice.UpdateServiceState(conn, &state); err != nil {
glog.Warningf("Could not update service state %s for %s (%s) as out of sync: %s", state.ID, svc.Name, svc.ID, err)
continue
}
}
}
}
return nil
}
func (this *ControlPlaneDao) GetServiceStates(serviceId string, states *[]servicestate.ServiceState) error {
glog.V(2).Infof("ControlPlaneDao.GetServiceStates: serviceId=%s", serviceId)
*states = make([]servicestate.ServiceState, 0)
poolBasedConn, err := this.getPoolBasedConnection(serviceId)
if err != nil {
return err
}
serviceStates, err := zkservice.GetServiceStates(poolBasedConn, serviceId)
if serviceStates != nil {
*states = serviceStates
}
return err
}
func (dfs *DistributedFilesystem) Restore(filename string) error {
// fail if any services are running
dfs.log("Checking running services")
svcs, err := dfs.facade.GetServices(datastore.Get(), dao.ServiceRequest{})
if err != nil {
glog.Errorf("Could not acquire the list of all services: %s", err)
return err
}
for _, svc := range svcs {
conn, err := zzk.GetLocalConnection(zzk.GeneratePoolPath(svc.PoolID))
if err != nil {
glog.Errorf("Could not acquire connection to coordinator (%s): %s", svc.PoolID, err)
return err
}
if states, err := zkservice.GetServiceStates(conn, svc.ID); err != nil {
glog.Errorf("Could not look up running instances for %s (%s): %s", svc.Name, svc.ID, err)
return err
} else if running := len(states); running > 0 {
err := fmt.Errorf("service %s (%s) has %d running instances", svc.Name, svc.ID, running)
glog.Errorf("Cannot restore to %s: %s", filename, err)
return err
}
}
var reloadLogstashContainer bool
defer func() {
if reloadLogstashContainer {
go facade.LogstashContainerReloader(datastore.Get(), dfs.facade) // don't block main thread
}
}()
dirpath := filepath.Join(utils.BackupDir(dfs.varpath), "restore")
if err := os.RemoveAll(dirpath); err != nil {
glog.Errorf("Could not remove %s: %s", dirpath, err)
return err
}
if err := mkdir(dirpath); err != nil {
glog.Errorf("Could neither find nor create %s: %s", dirpath, err)
return err
}
defer func() {
if err := os.RemoveAll(dirpath); err != nil {
glog.Warningf("Could not remove %s: %s", dirpath, err)
}
}()
dfs.log("Extracting backup file %s", filename)
if err := importTGZ(dirpath, filename); err != nil {
glog.Errorf("Could not expand %s to %s: %s", filename, dirpath, err)
return err
}
var metadata metadata
if err := importJSON(filepath.Join(dirpath, meta), &metadata); err != nil {
glog.Errorf("Could not import %s: %s", meta, err)
return err
} else if metadata.FSType != dfs.fsType {
err = fmt.Errorf("this backup can only be run on %s", metadata.FSType)
glog.Errorf("Could not extract backup %s: %s", filename, err)
return err
}
var pools []pool.ResourcePool
if err := importJSON(filepath.Join(dirpath, poolJSON), &pools); err != nil {
glog.Errorf("Could not read resource pools from %s: %s", filename, err)
return err
}
// restore the resource pools (and virtual ips)
dfs.log("Loading resource pools")
for _, pool := range pools {
pool.DatabaseVersion = 0
glog.V(1).Infof("Restoring resource pool %s", pool.ID)
var err error
if err = dfs.facade.AddResourcePool(datastore.Get(), &pool); err == facade.ErrPoolExists {
err = dfs.facade.UpdateResourcePool(datastore.Get(), &pool)
}
if err != nil {
glog.Errorf("Could not restore resource pool %s: %s", pool.ID, err)
return err
}
}
dfs.log("Resource pool load successful")
var hosts []host.Host
if err := importJSON(filepath.Join(dirpath, hostJSON), &hosts); err != nil {
glog.Errorf("Could not read hosts from %s: %s", filename, err)
return err
}
// restore the hosts (add it if it doesn't exist; assume hosts don't need to be updated from backup)
dfs.log("Loading static ips")
for _, host := range hosts {
host.DatabaseVersion = 0
glog.V(1).Infof("Restoring host %s (%s)", host.ID, host.IPAddr)
if exists, err := dfs.facade.HasIP(datastore.Get(), host.PoolID, host.IPAddr); err != nil {
glog.Errorf("Could not check IP %s for pool %s: %s", host.IPAddr, host.PoolID, err)
return err
} else if exists {
glog.Warningf("Could not restore host %s (%s): ip already exists", host.ID, host.IPAddr)
} else if err := dfs.facade.AddHost(datastore.Get(), &host); err != nil {
glog.Errorf("Could not add host %s: %s", host.ID, err)
return err
}
}
dfs.log("Static ip load successful")
var templates map[string]servicetemplate.ServiceTemplate
if err := importJSON(filepath.Join(dirpath, templateJSON), &templates); err != nil {
glog.Errorf("Could not read templates from %s: %s", filename, err)
return err
}
// restore the service templates
dfs.log("Loading service templates")
for templateID, template := range templates {
template.DatabaseVersion = 0
glog.V(1).Infof("Restoring service template %s", templateID)
template.ID = templateID
if err := dfs.facade.UpdateServiceTemplate(datastore.Get(), template); err != nil {
glog.Errorf("Could not restore template %s: %s", templateID, err)
return err
}
reloadLogstashContainer = true
}
dfs.log("Service template load successful")
// Get the tenant of all the services to be restored
snapshotFiles, err := ls(filepath.Join(dirpath, snapshotDir))
tenantIDs := make(map[string]struct{})
for _, f := range snapshotFiles {
tenantIDs[strings.TrimSuffix(f, ".tgz")] = struct{}{}
}
// restore docker images
dfs.log("Loading docker images")
var images []imagemeta
if err := importJSON(filepath.Join(dirpath, imageJSON), &images); err != nil {
glog.Errorf("Could not read images from %s: %s", filename, err)
return err
}
if err := dfs.importImages(filepath.Join(dirpath, imageDir), images, tenantIDs); err != nil {
glog.Errorf("Could not import images from %s: %s", filename, err)
return err
}
dfs.log("Docker image load successful")
// restore snapshots
glog.V(1).Infof("Restoring services and snapshots")
for _, f := range snapshotFiles {
dfs.log("Loading %s", f)
if err := dfs.loadSnapshots(strings.TrimSuffix(f, ".tgz"), filepath.Join(dirpath, snapshotDir, f)); err != nil {
glog.Errorf("Could not import snapshot from %s: %s", f, err)
return err
}
dfs.log("Successfully loaded %s", f)
}
glog.Infof("Restore succeeded with fsType:%s from file:%s", dfs.fsType, filename)
return nil
}
// getServiceState gets the service states for a serviceID
func getServiceStates(conn coordclient.Connection, serviceID string) ([]servicestate.ServiceState, error) {
return zkservice.GetServiceStates(conn, serviceID)
}