func (r *Orchestrator) handleServiceEvent(ctx context.Context, event events.Event) { switch v := event.(type) { case state.EventDeleteService: if !orchestrator.IsReplicatedService(v.Service) { return } orchestrator.DeleteServiceTasks(ctx, r.store, v.Service) r.restarts.ClearServiceHistory(v.Service.ID) case state.EventCreateService: if !orchestrator.IsReplicatedService(v.Service) { return } r.reconcileServices[v.Service.ID] = v.Service case state.EventUpdateService: if !orchestrator.IsReplicatedService(v.Service) { return } r.reconcileServices[v.Service.ID] = v.Service } }
// Run contains the global orchestrator event loop func (g *Orchestrator) Run(ctx context.Context) error { defer close(g.doneChan) // Watch changes to services and tasks queue := g.store.WatchQueue() watcher, cancel := queue.Watch() defer cancel() // lookup the cluster var err error g.store.View(func(readTx store.ReadTx) { var clusters []*api.Cluster clusters, err = store.FindClusters(readTx, store.ByName("default")) if len(clusters) != 1 { return // just pick up the cluster when it is created. } g.cluster = clusters[0] }) if err != nil { return err } // Get list of nodes var nodes []*api.Node g.store.View(func(readTx store.ReadTx) { nodes, err = store.FindNodes(readTx, store.All) }) if err != nil { return err } for _, n := range nodes { g.updateNode(n) } // Lookup global services var existingServices []*api.Service g.store.View(func(readTx store.ReadTx) { existingServices, err = store.FindServices(readTx, store.All) }) if err != nil { return err } var reconcileServiceIDs []string for _, s := range existingServices { if orchestrator.IsGlobalService(s) { g.updateService(s) reconcileServiceIDs = append(reconcileServiceIDs, s.ID) } } g.reconcileServices(ctx, reconcileServiceIDs) for { select { case event := <-watcher: // TODO(stevvooe): Use ctx to limit running time of operation. switch v := event.(type) { case state.EventUpdateCluster: g.cluster = v.Cluster case state.EventCreateService: if !orchestrator.IsGlobalService(v.Service) { continue } g.updateService(v.Service) g.reconcileServices(ctx, []string{v.Service.ID}) case state.EventUpdateService: if !orchestrator.IsGlobalService(v.Service) { continue } g.updateService(v.Service) g.reconcileServices(ctx, []string{v.Service.ID}) case state.EventDeleteService: if !orchestrator.IsGlobalService(v.Service) { continue } orchestrator.DeleteServiceTasks(ctx, g.store, v.Service) // delete the service from service map delete(g.globalServices, v.Service.ID) g.restarts.ClearServiceHistory(v.Service.ID) case state.EventCreateNode: g.updateNode(v.Node) g.reconcileOneNode(ctx, v.Node) case state.EventUpdateNode: g.updateNode(v.Node) switch v.Node.Status.State { // NodeStatus_DISCONNECTED is a transient state, no need to make any change case api.NodeStatus_DOWN: g.removeTasksFromNode(ctx, v.Node) case api.NodeStatus_READY: // node could come back to READY from DOWN or DISCONNECT g.reconcileOneNode(ctx, v.Node) } case state.EventDeleteNode: g.removeTasksFromNode(ctx, v.Node) delete(g.nodes, v.Node.ID) case state.EventUpdateTask: if _, exists := g.globalServices[v.Task.ServiceID]; !exists { continue } // global orchestrator needs to inspect when a task has terminated // it should ignore tasks whose DesiredState is past running, which // means the task has been processed if isTaskTerminated(v.Task) { g.restartTask(ctx, v.Task.ID, v.Task.ServiceID) } case state.EventDeleteTask: // CLI allows deleting task if _, exists := g.globalServices[v.Task.ServiceID]; !exists { continue } g.reconcileServicesOneNode(ctx, []string{v.Task.ServiceID}, v.Task.NodeID) } case <-g.stopChan: return nil } } }