// Tasks is a stream of tasks state for node. Each message contains full list // of tasks which should be run on node, if task is not present in that list, // it should be terminated. func (d *Dispatcher) Tasks(r *api.TasksRequest, stream api.Dispatcher_TasksServer) error { nodeInfo, err := ca.RemoteNode(stream.Context()) if err != nil { return err } nodeID := nodeInfo.NodeID dctx, err := d.isRunningLocked() if err != nil { return err } fields := logrus.Fields{ "node.id": nodeID, "node.session": r.SessionID, "method": "(*Dispatcher).Tasks", } if nodeInfo.ForwardedBy != nil { fields["forwarder.id"] = nodeInfo.ForwardedBy.NodeID } log.G(stream.Context()).WithFields(fields).Debugf("") if _, err = d.nodes.GetWithSession(nodeID, r.SessionID); err != nil { return err } tasksMap := make(map[string]*api.Task) nodeTasks, cancel, err := store.ViewAndWatch( d.store, func(readTx store.ReadTx) error { tasks, err := store.FindTasks(readTx, store.ByNodeID(nodeID)) if err != nil { return err } for _, t := range tasks { tasksMap[t.ID] = t } return nil }, state.EventCreateTask{Task: &api.Task{NodeID: nodeID}, Checks: []state.TaskCheckFunc{state.TaskCheckNodeID}}, state.EventUpdateTask{Task: &api.Task{NodeID: nodeID}, Checks: []state.TaskCheckFunc{state.TaskCheckNodeID}}, state.EventDeleteTask{Task: &api.Task{NodeID: nodeID}, Checks: []state.TaskCheckFunc{state.TaskCheckNodeID}}, ) if err != nil { return err } defer cancel() for { if _, err := d.nodes.GetWithSession(nodeID, r.SessionID); err != nil { return err } var tasks []*api.Task for _, t := range tasksMap { // dispatcher only sends tasks that have been assigned to a node if t != nil && t.Status.State >= api.TaskStateAssigned { tasks = append(tasks, t) } } if err := stream.Send(&api.TasksMessage{Tasks: tasks}); err != nil { return err } // bursty events should be processed in batches and sent out snapshot var ( modificationCnt int batchingTimer *time.Timer batchingTimeout <-chan time.Time ) batchingLoop: for modificationCnt < modificationBatchLimit { select { case event := <-nodeTasks: switch v := event.(type) { case state.EventCreateTask: tasksMap[v.Task.ID] = v.Task modificationCnt++ case state.EventUpdateTask: if oldTask, exists := tasksMap[v.Task.ID]; exists { // States ASSIGNED and below are set by the orchestrator/scheduler, // not the agent, so tasks in these states need to be sent to the // agent even if nothing else has changed. if equality.TasksEqualStable(oldTask, v.Task) && v.Task.Status.State > api.TaskStateAssigned { // this update should not trigger action at agent tasksMap[v.Task.ID] = v.Task continue } } tasksMap[v.Task.ID] = v.Task modificationCnt++ case state.EventDeleteTask: delete(tasksMap, v.Task.ID) modificationCnt++ } if batchingTimer != nil { batchingTimer.Reset(batchingWaitTime) } else { batchingTimer = time.NewTimer(batchingWaitTime) batchingTimeout = batchingTimer.C } case <-batchingTimeout: break batchingLoop case <-stream.Context().Done(): return stream.Context().Err() case <-dctx.Done(): return dctx.Err() } } if batchingTimer != nil { batchingTimer.Stop() } } }
// Tasks is a stream of tasks state for node. Each message contains full list // of tasks which should be run on node, if task is not present in that list, // it should be terminated. func (d *Dispatcher) Tasks(r *api.TasksRequest, stream api.Dispatcher_TasksServer) error { nodeInfo, err := ca.RemoteNode(stream.Context()) if err != nil { return err } nodeID := nodeInfo.NodeID if err := d.isRunningLocked(); err != nil { return err } fields := logrus.Fields{ "node.id": nodeID, "node.session": r.SessionID, "method": "(*Dispatcher).Tasks", } if nodeInfo.ForwardedBy != nil { fields["forwarder.id"] = nodeInfo.ForwardedBy.NodeID } log.G(stream.Context()).WithFields(fields).Debugf("") if _, err = d.nodes.GetWithSession(nodeID, r.SessionID); err != nil { return err } tasksMap := make(map[string]*api.Task) nodeTasks, cancel, err := store.ViewAndWatch( d.store, func(readTx store.ReadTx) error { tasks, err := store.FindTasks(readTx, store.ByNodeID(nodeID)) if err != nil { return err } for _, t := range tasks { tasksMap[t.ID] = t } return nil }, state.EventCreateTask{Task: &api.Task{NodeID: nodeID}, Checks: []state.TaskCheckFunc{state.TaskCheckNodeID}}, state.EventUpdateTask{Task: &api.Task{NodeID: nodeID}, Checks: []state.TaskCheckFunc{state.TaskCheckNodeID}}, state.EventDeleteTask{Task: &api.Task{NodeID: nodeID}, Checks: []state.TaskCheckFunc{state.TaskCheckNodeID}}, ) if err != nil { return err } defer cancel() for { if _, err := d.nodes.GetWithSession(nodeID, r.SessionID); err != nil { return err } var tasks []*api.Task for _, t := range tasksMap { // dispatcher only sends tasks that have been assigned to a node if t != nil && t.Status.State >= api.TaskStateAssigned { tasks = append(tasks, t) } } if err := stream.Send(&api.TasksMessage{Tasks: tasks}); err != nil { return err } select { case event := <-nodeTasks: switch v := event.(type) { case state.EventCreateTask: tasksMap[v.Task.ID] = v.Task case state.EventUpdateTask: tasksMap[v.Task.ID] = v.Task case state.EventDeleteTask: delete(tasksMap, v.Task.ID) } case <-stream.Context().Done(): return stream.Context().Err() case <-d.ctx.Done(): return d.ctx.Err() } } }
// Tasks is a stream of tasks state for node. Each message contains full list // of tasks which should be run on node, if task is not present in that list, // it should be terminated. func (d *Dispatcher) Tasks(r *api.TasksRequest, stream api.Dispatcher_TasksServer) error { nodeInfo, err := ca.RemoteNode(stream.Context()) if err != nil { return err } nodeID := nodeInfo.NodeID if err := d.isRunningLocked(); err != nil { return err } fields := logrus.Fields{ "node.id": nodeID, "node.session": r.SessionID, "method": "(*Dispatcher).Tasks", } if nodeInfo.ForwardedBy != nil { fields["forwarder.id"] = nodeInfo.ForwardedBy.NodeID } log.G(stream.Context()).WithFields(fields).Debugf("") if _, err = d.nodes.GetWithSession(nodeID, r.SessionID); err != nil { return err } tasksMap := make(map[string]*api.Task) nodeTasks, cancel, err := store.ViewAndWatch( d.store, func(readTx store.ReadTx) error { tasks, err := store.FindTasks(readTx, store.ByNodeID(nodeID)) if err != nil { return err } for _, t := range tasks { tasksMap[t.ID] = t } return nil }, state.EventCreateTask{Task: &api.Task{NodeID: nodeID}, Checks: []state.TaskCheckFunc{state.TaskCheckNodeID}}, state.EventUpdateTask{Task: &api.Task{NodeID: nodeID}, Checks: []state.TaskCheckFunc{state.TaskCheckNodeID}}, state.EventDeleteTask{Task: &api.Task{NodeID: nodeID}, Checks: []state.TaskCheckFunc{state.TaskCheckNodeID}}, ) if err != nil { return err } defer cancel() for { if _, err := d.nodes.GetWithSession(nodeID, r.SessionID); err != nil { return err } var tasks []*api.Task for _, t := range tasksMap { // dispatcher only sends tasks that have been assigned to a node if t != nil && t.Status.State >= api.TaskStateAssigned { tasks = append(tasks, t) } } if err := stream.Send(&api.TasksMessage{Tasks: tasks}); err != nil { return err } // bursty events should be processed in batches and sent out snapshot const modificationBatchLimit = 200 const eventPausedGap = 50 * time.Millisecond var modificationCnt int // eventPaused is true when there have been modifications // but next event has not arrived within eventPausedGap eventPaused := false for modificationCnt < modificationBatchLimit && !eventPaused { select { case event := <-nodeTasks: switch v := event.(type) { case state.EventCreateTask: tasksMap[v.Task.ID] = v.Task modificationCnt++ case state.EventUpdateTask: if oldTask, exists := tasksMap[v.Task.ID]; exists { if equality.TasksEqualStable(oldTask, v.Task) { // this update should not trigger action at agent tasksMap[v.Task.ID] = v.Task continue } } tasksMap[v.Task.ID] = v.Task modificationCnt++ case state.EventDeleteTask: delete(tasksMap, v.Task.ID) modificationCnt++ } case <-time.After(eventPausedGap): if modificationCnt > 0 { eventPaused = true } case <-stream.Context().Done(): return stream.Context().Err() case <-d.ctx.Done(): return d.ctx.Err() } } } }