// Push event to event queue when execute metronome from consul
func Push() (string, error) {
l, err := util.Consul().LockKey(LOCK_KEY)
if err != nil {
return "", err
}
if _, err := l.Lock(nil); err != nil {
return "", err
}
defer l.Unlock()
// Unmarshal STDIN from consul
bytes, err := ioutil.ReadAll(os.Stdin)
if err != nil {
return "Following error has occurred while read STDIN", err
}
var receiveEvents []api.UserEvent
err = json.Unmarshal(bytes, &receiveEvents)
if err != nil {
return "Following error has occurred while unmarshal STDIN", err
}
// Enqueue each event to event queue
eq := &queue.Queue{
Client: util.Consul(),
Key: EVENT_QUEUE_KEY,
}
for _, re := range receiveEvents {
if err := pushSingleEvent(eq, re); err != nil {
return "", err
}
}
return "", nil
}
func (s *Scheduler) dispatchEvent() error {
pq := &queue.Queue{
Client: util.Consul(),
Key: PROGRESS_QUEUE_KEY,
}
eq := &queue.Queue{
Client: util.Consul(),
Key: EVENT_QUEUE_KEY,
}
var consulEvent api.UserEvent
if err, found := eq.DeQueue(&consulEvent); err != nil || !found {
return err
}
result, err := getEventResult(consulEvent.ID)
if err != nil {
return err
}
if result != nil {
log.Debugf("Ignore event(ID: %s, Name: %s) already has been executed", consulEvent.ID, consulEvent.Name)
return nil
}
// Collect events over all task.yml and dispatch tasks to progress task queue
log.Infof("Dispatch event(ID: %s, Name: %s)", consulEvent.ID, consulEvent.Name)
events := s.sortedEvents(consulEvent.Name)
c := 0
for _, v := range events {
switch {
case v.Task != "":
pq.EnQueue(EventTask{
Pattern: v.Pattern,
ID: consulEvent.ID,
No: c,
Task: v.Task,
Skippable: config.Skippable,
})
c += 1
case len(v.OrderedTasks) > 0:
for _, t := range v.OrderedTasks {
t.Pattern = v.Pattern
t.ID = consulEvent.ID
t.No = c
pq.EnQueue(t)
c += 1
}
}
}
// Log starting event as EventResult on KVS
result = &EventResult{
ID: consulEvent.ID,
Name: consulEvent.Name,
Status: "inprogress",
StartedAt: time.Now(),
}
return result.Save()
}
func (et *EventTask) IsFinished(ch chan EventTask) bool {
// Finished task when timeout has occurred
select {
case timeout := <-ch:
if et.ID == timeout.ID && et.No == timeout.No {
log.Errorf("Task has been reached timeout(%s)", et.String())
return true
}
default:
}
nodes, _, err := util.Consul().Catalog().Nodes(&api.QueryOptions{})
if err != nil {
return false
}
filteredNodes := et.filterNodes(nodes)
if len(filteredNodes) == 0 {
return false
}
// Wait for finishing tasks on target node
for _, node := range filteredNodes {
result, err := getNodeTaskResult(et.ID, et.No, node.Node)
if err != nil || result == nil || !result.IsFinished() {
return false
}
}
return true
}
// Trigger channel when current task has been reached timeout
func taskTimeout(ch chan EventTask) {
var prev EventTask
var now EventTask
for {
time.Sleep(1 * time.Second)
pq := &queue.Queue{
Client: util.Consul(),
Key: PROGRESS_QUEUE_KEY,
}
// Wait until task has dispatched
if err := pq.FetchHead(&now); err != nil || now.ID == "" || prev.ID == now.ID && prev.No == now.No {
continue
}
prev = now
// Wait until current task has started on any node or timeout
cancel := make(chan bool)
select {
case <-startTask(now, cancel):
case <-time.After(TASK_TIMEOUT_WITHOUT_START * time.Second):
cancel <- true
ch <- now
continue
}
// Wait until current task has finished or timeout
select {
case <-changeTask(now, cancel):
case <-time.After(time.Duration(TASK_TIMEOUT) * time.Second):
cancel <- true
ch <- now
}
}
}
func (o *ConsulKVSOperation) get(vars map[string]string) error {
// Store value that has been get to variables map
kv, _, err := util.Consul().KV().Get(o.Key, &api.QueryOptions{})
vars[o.Name] = string(kv.Value)
log.Infof("Get %s from %s and store to %s", kv.Value, kv.Key, o.Name)
return err
}
func getAttributes(keys []string, overwriteAttributes map[string]interface{}) (map[string]interface{}, error) {
var attributes map[string]interface{}
var c *api.Client = util.Consul()
attributes = make(map[string]interface{})
// Get attributes from consul KVS
for _, key := range keys {
list, _, err := c.KV().List(key, &api.QueryOptions{})
if err != nil {
return nil, err
}
for _, kv := range list {
var a map[string]interface{}
if err := json.Unmarshal(kv.Value, &a); err != nil {
return nil, err
}
if err := mergo.MergeWithOverwrite(&attributes, a); err != nil {
return nil, errors.New(fmt.Sprintf("Failed to merge attributes(%v)", err))
}
}
}
// Overwrite some attributes by specified parameter in task.yml
if err := mergo.MergeWithOverwrite(&attributes, overwriteAttributes); err != nil {
return nil, err
}
return attributes, nil
}
// Trigger channel when change current task
func changeTask(et EventTask, cancel chan bool) chan bool {
ch := make(chan bool)
go func(chan bool) {
pq := &queue.Queue{
Client: util.Consul(),
Key: PROGRESS_QUEUE_KEY,
}
for {
time.Sleep(1 * time.Second)
// Exit when cancel channel has signalled
select {
case <-cancel:
return
default:
}
// Send signal to change task channel when current task has been changed
var now EventTask
if err := pq.FetchHead(&now); err != nil {
ch <- true
return
}
if et.ID != now.ID || et.No != now.No {
ch <- true
return
}
}
}(ch)
return ch
}
func (s *Scheduler) polling(ch chan EventTask) error {
// Create critical section by consul lock
l, err := util.Consul().LockKey(LOCK_KEY)
if err != nil {
return err
}
if _, err := l.Lock(nil); err != nil {
return err
}
defer l.Unlock()
// Polling tasks from queue
var eventTasks []EventTask
pq := &queue.Queue{
Client: util.Consul(),
Key: PROGRESS_QUEUE_KEY,
}
if err := pq.Items(&eventTasks); err != nil {
return err
}
if config.Debug {
log.Debug("-------- Progress Task Queue --------")
nodes, _, _ := util.Consul().Catalog().Nodes(&api.QueryOptions{})
for _, et := range eventTasks {
log.Debug(et.String())
for _, n := range nodes {
log.Debugf("%s: %t", n.Node, et.Runnable(n.Node))
}
}
}
switch {
case len(eventTasks) == 0:
return s.dispatchEvent()
case eventTasks[0].Runnable(s.node):
// runTask is parallelizable
l.Unlock()
return s.runTask(eventTasks[0])
case eventTasks[0].IsFinished(ch):
return s.finishTask(eventTasks[0])
default:
log.Debugf("Wait a task will have been finished by other instance(%s)", eventTasks[0].String())
}
return nil
}
func (o *ConsulKVSOperation) put(vars map[string]string) error {
kv := &api.KVPair{
Key: o.Key,
Value: []byte(o.Value),
}
_, err := util.Consul().KV().Put(kv, &api.WriteOptions{})
log.Infof("Put %s to %s", o.Value, o.Key)
return err
}
func (r *NodeTaskResult) Save() error {
// Save any result to consul KVS
if err := putResult(r); err != nil {
return err
}
kv := &api.KVPair{
Key: r.Key() + "/log",
Value: []byte(r.Log),
}
_, err := util.Consul().KV().Put(kv, &api.WriteOptions{})
return err
}
func (o *ConsulEventOperation) Run(vars map[string]string) error {
event := &api.UserEvent{
Name: o.Name,
ServiceFilter: o.Filter.Service,
TagFilter: o.Filter.Tag,
Payload: []byte(config.Token),
}
id, _, err := util.Consul().Event().Fire(event, &api.WriteOptions{})
log.Infof("consul-event: Fire %s event(ID: %s)", o.Name, id)
return err
}
// Put any result to consul KVS with JSON format
func putResult(result Result) error {
d, err := json.Marshal(result)
if err != nil {
return err
}
kv := api.KVPair{
Key: result.Key(),
Value: d,
}
_, err = util.Consul().KV().Put(&kv, &api.WriteOptions{})
return err
}
// Get any result from consul KVS
func getResult(key string, result interface{}) (bool, error) {
kv, _, err := util.Consul().KV().Get(key, &api.QueryOptions{})
if err != nil {
return false, err
}
if kv == nil || len(kv.Value) == 0 {
return false, nil
}
if err := json.Unmarshal(kv.Value, &result); err != nil {
return false, err
}
return true, nil
}
func getNodeTaskResult(id string, no int, node string) (*NodeTaskResult, error) {
var result NodeTaskResult
key := EVENT_RESULT_KEY + "/" + id + "/" + strconv.Itoa(no) + "/" + node
found, err := getResult(key, &result)
if !found || err != nil {
return nil, err
}
// Read log from /metronome/result/[EventID]/[No]/[Node]/log
kv, _, err := util.Consul().KV().Get(key+"/log", &api.QueryOptions{})
if err != nil {
return nil, err
}
if kv != nil {
result.Log = string(kv.Value)
}
return &result, err
}
func (r *TaskResult) GetNodeResults() ([]NodeTaskResult, error) {
// Collect all results on node that belongs with this task
var results []NodeTaskResult
prefix := EVENT_RESULT_KEY + "/" + r.EventID + "/" + strconv.Itoa(r.No)
kvs, _, err := util.Consul().KV().List(prefix, &api.QueryOptions{})
if err != nil {
return nil, err
}
for _, kv := range kvs {
// Except log record on each node
node := strings.TrimPrefix(kv.Key, prefix)
if node == "" || strings.HasSuffix(node, "/log") {
continue
}
result, err := getNodeTaskResult(r.EventID, r.No, node)
if err != nil {
return nil, err
}
results = append(results, *result)
}
return results, nil
}
// Finish current task when no node in consul catalog will execute current task
func (s *Scheduler) finishTask(task EventTask) error {
log.Infof("Finish task(%s)", task.String())
pq := &queue.Queue{
Client: util.Consul(),
Key: PROGRESS_QUEUE_KEY,
}
result, err := task.GetResult()
if err != nil {
return err
}
nodeResults, err := result.GetNodeResults()
if err != nil {
return err
}
// Collect task results over all nodes
status := "success"
if len(nodeResults) == 0 {
if task.Skippable {
status = "skip"
} else {
status = "timeout"
}
}
for _, nr := range nodeResults {
if nr.Status == "error" {
status = "error"
break
}
if nr.Status == "inprogress" {
status = "timeout"
}
}
if status == "error" || status == "timeout" {
// remove following tasks in progress task queue when some error occured or task has reached timeout
pq.Clear()
}
// Log finishing task as TaskResult on KVS
result.Status = status
result.FinishedAt = time.Now()
if err := result.Save(); err != nil {
return err
}
// Dequeue task from task queue when finished task over all all nodes
var dummy EventTask
if err, _ := pq.DeQueue(&dummy); err != nil {
return err
}
// Log finishing event as EventResult on KVS when finished all task in a progress task queue
var tasks []EventTask
if err := pq.Items(&tasks); err != nil {
return err
}
if len(tasks) == 0 {
eventResult, err := getEventResult(task.ID)
if err != nil {
return err
}
eventResult.Status = status
eventResult.FinishedAt = time.Now()
if err := eventResult.Save(); err != nil {
return err
}
}
return nil
}
func (o *ConsulKVSOperation) delete(vars map[string]string) error {
_, err := util.Consul().KV().Delete(o.Key, &api.WriteOptions{})
log.Infof("Delete %s", o.Key)
return err
}
