func (r *Orchestrator) handleServiceEvent(ctx context.Context, event events.Event) {
switch v := event.(type) {
case state.EventDeleteService:
if !orchestrator.IsReplicatedService(v.Service) {
return
}
orchestrator.DeleteServiceTasks(ctx, r.store, v.Service)
r.restarts.ClearServiceHistory(v.Service.ID)
case state.EventCreateService:
if !orchestrator.IsReplicatedService(v.Service) {
return
}
r.reconcileServices[v.Service.ID] = v.Service
case state.EventUpdateService:
if !orchestrator.IsReplicatedService(v.Service) {
return
}
r.reconcileServices[v.Service.ID] = v.Service
}
}
// Run contains the global orchestrator event loop
func (g *Orchestrator) Run(ctx context.Context) error {
defer close(g.doneChan)
// Watch changes to services and tasks
queue := g.store.WatchQueue()
watcher, cancel := queue.Watch()
defer cancel()
// lookup the cluster
var err error
g.store.View(func(readTx store.ReadTx) {
var clusters []*api.Cluster
clusters, err = store.FindClusters(readTx, store.ByName("default"))
if len(clusters) != 1 {
return // just pick up the cluster when it is created.
}
g.cluster = clusters[0]
})
if err != nil {
return err
}
// Get list of nodes
var nodes []*api.Node
g.store.View(func(readTx store.ReadTx) {
nodes, err = store.FindNodes(readTx, store.All)
})
if err != nil {
return err
}
for _, n := range nodes {
g.updateNode(n)
}
// Lookup global services
var existingServices []*api.Service
g.store.View(func(readTx store.ReadTx) {
existingServices, err = store.FindServices(readTx, store.All)
})
if err != nil {
return err
}
var reconcileServiceIDs []string
for _, s := range existingServices {
if orchestrator.IsGlobalService(s) {
g.updateService(s)
reconcileServiceIDs = append(reconcileServiceIDs, s.ID)
}
}
g.reconcileServices(ctx, reconcileServiceIDs)
for {
select {
case event := <-watcher:
// TODO(stevvooe): Use ctx to limit running time of operation.
switch v := event.(type) {
case state.EventUpdateCluster:
g.cluster = v.Cluster
case state.EventCreateService:
if !orchestrator.IsGlobalService(v.Service) {
continue
}
g.updateService(v.Service)
g.reconcileServices(ctx, []string{v.Service.ID})
case state.EventUpdateService:
if !orchestrator.IsGlobalService(v.Service) {
continue
}
g.updateService(v.Service)
g.reconcileServices(ctx, []string{v.Service.ID})
case state.EventDeleteService:
if !orchestrator.IsGlobalService(v.Service) {
continue
}
orchestrator.DeleteServiceTasks(ctx, g.store, v.Service)
// delete the service from service map
delete(g.globalServices, v.Service.ID)
g.restarts.ClearServiceHistory(v.Service.ID)
case state.EventCreateNode:
g.updateNode(v.Node)
g.reconcileOneNode(ctx, v.Node)
case state.EventUpdateNode:
g.updateNode(v.Node)
switch v.Node.Status.State {
// NodeStatus_DISCONNECTED is a transient state, no need to make any change
case api.NodeStatus_DOWN:
g.removeTasksFromNode(ctx, v.Node)
case api.NodeStatus_READY:
// node could come back to READY from DOWN or DISCONNECT
g.reconcileOneNode(ctx, v.Node)
}
case state.EventDeleteNode:
g.removeTasksFromNode(ctx, v.Node)
delete(g.nodes, v.Node.ID)
case state.EventUpdateTask:
if _, exists := g.globalServices[v.Task.ServiceID]; !exists {
continue
}
// global orchestrator needs to inspect when a task has terminated
// it should ignore tasks whose DesiredState is past running, which
// means the task has been processed
if isTaskTerminated(v.Task) {
g.restartTask(ctx, v.Task.ID, v.Task.ServiceID)
}
case state.EventDeleteTask:
// CLI allows deleting task
if _, exists := g.globalServices[v.Task.ServiceID]; !exists {
continue
}
g.reconcileServicesOneNode(ctx, []string{v.Task.ServiceID}, v.Task.NodeID)
}
case <-g.stopChan:
return nil
}
}
}