// addInstance creates a new service state and host instance
func addInstance(conn client.Connection, state ss.ServiceState) error {
glog.V(2).Infof("Adding instance %+v", state)
// check the object
if err := state.ValidEntity(); err != nil {
glog.Errorf("Could not validate service state %+v: %s", state, err)
return err
}
// CC-1050: we need to trigger the scheduler in case we only have a
// partial create.
svclock := newStateLock(conn, state.ServiceID)
if err := svclock.Lock(); err != nil {
glog.Errorf("Could not set lock on service %s: %s", state.ServiceID, err)
return err
}
defer svclock.Unlock()
lock := newInstanceLock(conn, state.ID)
if err := lock.Lock(); err != nil {
glog.Errorf("Could not set lock for service instance %s for service %s on host %s: %s", state.ID, state.ServiceID, state.HostID, err)
return err
}
glog.V(2).Infof("Acquired lock for instance %s", state.ID)
defer lock.Unlock()
var err error
defer func() {
if err != nil {
conn.Delete(hostpath(state.HostID, state.ID))
conn.Delete(servicepath(state.ServiceID, state.ID))
rmInstanceLock(conn, state.ID)
}
}()
// Create node on the service
spath := servicepath(state.ServiceID, state.ID)
snode := &ServiceStateNode{ServiceState: &state}
if err = conn.Create(spath, snode); err != nil {
glog.Errorf("Could not create service state %s for service %s: %s", state.ID, state.ServiceID, err)
return err
} else if err = conn.Set(spath, snode); err != nil {
glog.Errorf("Could not set service state %s for node %+v: %s", state.ID, snode, err)
return err
}
// Create node on the host
hpath := hostpath(state.HostID, state.ID)
hnode := NewHostState(&state)
glog.V(2).Infof("Host node: %+v", hnode)
if err = conn.Create(hpath, hnode); err != nil {
glog.Errorf("Could not create host state %s for host %s: %s", state.ID, state.HostID, err)
return err
} else if err = conn.Set(hpath, hnode); err != nil {
glog.Errorf("Could not set host state %s for node %+v: %s", state.ID, hnode, err)
return err
}
glog.V(2).Infof("Releasing lock for instance %s", state.ID)
return nil
}
// UpdateServiceState does a full update of a service state
func UpdateServiceState(conn client.Connection, state *ss.ServiceState) error {
if err := state.ValidEntity(); err != nil {
glog.Errorf("Could not validate service state %+v: %s", state, err)
return err
}
return updateInstance(conn, state.HostID, state.ID, func(_ *HostState, ssdata *ss.ServiceState) {
*ssdata = *state
})
}
// getStatus computes the status of a service state
func getStatus(conn client.Connection, state *servicestate.ServiceState) (dao.Status, error) {
var status dao.Status
// Set the state based on the service state object
if !state.IsRunning() {
status = dao.Stopped
} else if state.IsPaused() {
status = dao.Paused
} else {
status = dao.Running
}
// Set the state based on the host state object
var hostState HostState
if err := conn.Get(hostpath(state.HostID, state.ID), &hostState); err != nil && err != client.ErrNoNode {
return dao.Status{}, err
}
if hostState.DesiredState == int(service.SVCStop) {
switch status {
case dao.Running, dao.Paused:
status = dao.Stopping
case dao.Stopped:
// pass
default:
return dao.Status{}, ErrUnknownState
}
} else if hostState.DesiredState == int(service.SVCRun) {
switch status {
case dao.Stopped:
status = dao.Starting
case dao.Paused:
status = dao.Resuming
case dao.Running:
// pass
default:
return dao.Status{}, ErrUnknownState
}
} else if hostState.DesiredState == int(service.SVCPause) {
switch status {
case dao.Running:
status = dao.Pausing
case dao.Paused, dao.Stopped:
// pass
default:
return dao.Status{}, ErrUnknownState
}
} else {
return dao.Status{}, ErrUnknownState
}
return status, nil
}
func updateInstance(state *servicestate.ServiceState, ctr *docker.Container) error {
if _, err := ctr.Inspect(); err != nil {
return err
}
state.DockerID = ctr.ID
state.Started = ctr.Created
state.PrivateIP = ctr.NetworkSettings.IPAddress
state.PortMapping = make(map[string][]domain.HostIPAndPort)
for k, v := range ctr.NetworkSettings.Ports {
pm := []domain.HostIPAndPort{}
for _, pb := range v {
pm = append(pm, domain.HostIPAndPort{HostIP: pb.HostIp, HostPort: pb.HostPort})
state.PortMapping[string(k)] = pm
}
}
return nil
}
// AttachService attempts to attach to a running container
func (a *HostAgent) AttachService(svc *service.Service, state *servicestate.ServiceState, exited func(string)) error {
ctr, err := docker.FindContainer(state.DockerID)
if err != nil {
return err
}
if !ctr.IsRunning() {
defer exited(state.ID)
return nil
}
ctr.OnEvent(docker.Die, func(cid string) {
defer exited(state.ID)
glog.Infof("Instance %s (%s) for %s (%s) has died", state.ID, ctr.ID, svc.Name, svc.ID)
state.DockerID = cid
a.removeInstance(state.ID, ctr)
})
go a.setProxy(svc, ctr)
return nil
}
// StartService starts a new instance of the specified service and updates the control center state accordingly.
func (a *HostAgent) StartService(svc *service.Service, state *servicestate.ServiceState, exited func(string)) error {
glog.V(2).Infof("About to start service %s with name %s", svc.ID, svc.Name)
client, err := NewControlClient(a.master)
if err != nil {
glog.Errorf("Could not start ControlPlane client %v", err)
return err
}
defer client.Close()
// start from a known good state
if state.DockerID != "" {
if ctr, err := docker.FindContainer(state.DockerID); err != nil {
glog.Errorf("Could not find container %s for %s", state.DockerID, state.ID)
} else if err := ctr.Delete(true); err != nil {
glog.Errorf("Could not delete container %s for %s", state.DockerID, state.ID)
}
}
// create the docker client Config and HostConfig structures necessary to create and start the service
config, hostconfig, err := configureContainer(a, client, svc, state, a.virtualAddressSubnet)
if err != nil {
glog.Errorf("can't configure container: %v", err)
return err
}
cjson, _ := json.MarshalIndent(config, "", " ")
glog.V(3).Infof(">>> CreateContainerOptions:\n%s", string(cjson))
hcjson, _ := json.MarshalIndent(hostconfig, "", " ")
glog.V(3).Infof(">>> HostConfigOptions:\n%s", string(hcjson))
cd := &docker.ContainerDefinition{
dockerclient.CreateContainerOptions{Name: state.ID, Config: config},
*hostconfig,
}
ctr, err := docker.NewContainer(cd, false, 10*time.Second, nil, nil)
if err != nil {
glog.Errorf("Error trying to create container %v: %v", config, err)
return err
}
var started sync.WaitGroup
started.Add(1)
ctr.OnEvent(docker.Start, func(cid string) {
glog.Infof("Instance %s (%s) for %s (%s) has started", state.ID, ctr.ID, svc.Name, svc.ID)
started.Done()
})
ctr.OnEvent(docker.Die, func(cid string) {
defer exited(state.ID)
glog.Infof("Instance %s (%s) for %s (%s) has died", state.ID, ctr.ID, svc.Name, svc.ID)
state.DockerID = cid
a.removeInstance(state.ID, ctr)
})
if err := ctr.Start(time.Hour); err != nil {
glog.Errorf("Could not start service state %s (%s) for service %s (%s): %s", state.ID, ctr.ID, svc.Name, svc.ID, err)
a.removeInstance(state.ID, ctr)
return err
}
started.Wait()
if err := updateInstance(state, ctr); err != nil {
glog.Errorf("Could not update instance %s (%s) for service %s (%s): %s", state.ID, ctr.ID, svc.Name, svc.ID, err)
ctr.Stop(45 * time.Second)
return err
}
go a.setProxy(svc, ctr)
return nil
}
// Spawn listens for changes in the host state and manages running instances
func (l *HostStateListener) Spawn(shutdown <-chan interface{}, stateID string) {
var processDone <-chan struct{}
// Let's have exclusive access to this node
lock := newInstanceLock(l.conn, stateID)
if err := lock.Lock(); err != nil {
glog.Errorf("Could not lock service instance %s on host %s: %s", stateID, l.hostID, err)
return
}
// Get the HostState node
var hs HostState
if err := l.conn.Get(hostpath(l.hostID, stateID), &hs); err != nil {
glog.Errorf("Could not load host instance %s on host %s: %s", stateID, l.hostID, err)
l.conn.Delete(hostpath(l.hostID, stateID))
lock.Unlock()
return
}
defer removeInstance(l.conn, hs.ServiceID, hs.HostID, hs.ServiceStateID)
// Get the ServiceState node
var ss servicestate.ServiceState
if err := l.conn.Get(servicepath(hs.ServiceID, hs.ServiceStateID), &ServiceStateNode{ServiceState: &ss}); err != nil {
glog.Errorf("Could not load service instance %s for service %s on host %s: %s", hs.ServiceStateID, hs.ServiceID, hs.HostID, err)
lock.Unlock()
return
}
defer l.stopInstance(processDone, &ss)
lock.Unlock()
for {
// Get the HostState instance
hsEvt, err := l.conn.GetW(hostpath(l.hostID, stateID), &hs)
if err != nil {
glog.Errorf("Could not load host instance %s on host %s: %s", stateID, l.hostID, err)
return
}
// Get the ServiceState instance
ssEvt, err := l.conn.GetW(servicepath(hs.ServiceID, stateID), &ServiceStateNode{ServiceState: &ss})
if err != nil {
glog.Errorf("Could not load service state %s for service %s on host %s: %s", stateID, hs.ServiceID, l.hostID, err)
return
}
// Get the service
var svc service.Service
if err := l.conn.Get(servicepath(hs.ServiceID), &ServiceNode{Service: &svc}); err != nil {
glog.Errorf("Could not load service %s for service instance %s on host %s: %s", hs.ServiceID, stateID, l.hostID, err)
return
}
// Process the desired state
glog.V(2).Infof("Processing %s (%s); Desired State: %d", svc.Name, svc.ID, hs.DesiredState)
switch service.DesiredState(hs.DesiredState) {
case service.SVCRun:
var err error
if !ss.IsRunning() {
// process has stopped
glog.Infof("Starting a new instance for %s (%s): %s", svc.Name, svc.ID, stateID)
if processDone, err = l.startInstance(&svc, &ss); err != nil {
glog.Errorf("Could not start service instance %s for service %s on host %s: %s", hs.ServiceStateID, hs.ServiceID, hs.HostID, err)
return
}
} else if processDone == nil {
glog.Infof("Attaching to instance %s for %s (%s) via %s", stateID, svc.Name, svc.ID, ss.DockerID)
if processDone, err = l.attachInstance(&svc, &ss); err != nil {
glog.Errorf("Could not start service instance %s for service %s on host %s: %s", hs.ServiceStateID, hs.ServiceID, hs.HostID, err)
return
}
}
if ss.IsPaused() {
glog.Infof("Resuming paused instance %s for service %s (%s)", stateID, svc.Name, svc.ID)
if err := l.resumeInstance(&svc, &ss); err != nil {
glog.Errorf("Could not resume paused instance %s for service %s (%s): %s", stateID, svc.Name, svc.ID, err)
return
}
}
case service.SVCPause:
if !ss.IsPaused() {
if err := l.pauseInstance(&svc, &ss); err != nil {
glog.Errorf("Could not pause instance %s for service %s (%s): %s", stateID, svc.Name, svc.ID, err)
return
}
}
case service.SVCStop:
return
default:
glog.V(2).Infof("Unhandled state (%d) of instance %s for service %s (%s)", hs.DesiredState, stateID, svc.Name, svc.ID, err)
}
select {
case <-processDone:
glog.V(2).Infof("Process ended for instance %s for service %s (%s)", stateID, svc.Name, svc.ID)
case e := <-hsEvt:
glog.V(3).Infof("Host instance %s for service %s (%s) received an event: %+v", stateID, svc.Name, svc.ID, e)
if e.Type == client.EventNodeDeleted {
return
}
case e := <-ssEvt:
glog.V(3).Infof("Service instance %s for service %s (%s) received an event: %+v", stateID, svc.Name, svc.ID, e)
if e.Type == client.EventNodeDeleted {
return
}
case <-shutdown:
glog.V(2).Infof("Host instance %s for service %s (%s) received signal to shutdown", stateID, svc.Name, svc.ID)
return
}
}
}