Example #1
0
//	Push event to event queue when execute metronome from consul
func Push() (string, error) {
	l, err := util.Consul().LockKey(LOCK_KEY)
	if err != nil {
		return "", err
	}
	if _, err := l.Lock(nil); err != nil {
		return "", err
	}
	defer l.Unlock()

	//	Unmarshal STDIN from consul
	bytes, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		return "Following error has occurred while read STDIN", err
	}
	var receiveEvents []api.UserEvent
	err = json.Unmarshal(bytes, &receiveEvents)
	if err != nil {
		return "Following error has occurred while unmarshal STDIN", err
	}

	//	Enqueue each event to event queue
	eq := &queue.Queue{
		Client: util.Consul(),
		Key:    EVENT_QUEUE_KEY,
	}
	for _, re := range receiveEvents {
		if err := pushSingleEvent(eq, re); err != nil {
			return "", err
		}
	}
	return "", nil
}
Example #2
0
func (s *Scheduler) dispatchEvent() error {
	pq := &queue.Queue{
		Client: util.Consul(),
		Key:    PROGRESS_QUEUE_KEY,
	}
	eq := &queue.Queue{
		Client: util.Consul(),
		Key:    EVENT_QUEUE_KEY,
	}

	var consulEvent api.UserEvent
	if err, found := eq.DeQueue(&consulEvent); err != nil || !found {
		return err
	}
	result, err := getEventResult(consulEvent.ID)
	if err != nil {
		return err
	}
	if result != nil {
		log.Debugf("Ignore event(ID: %s, Name: %s) already has been executed", consulEvent.ID, consulEvent.Name)
		return nil
	}

	//	Collect events over all task.yml and dispatch tasks to progress task queue
	log.Infof("Dispatch event(ID: %s, Name: %s)", consulEvent.ID, consulEvent.Name)
	events := s.sortedEvents(consulEvent.Name)
	c := 0
	for _, v := range events {
		switch {
		case v.Task != "":
			pq.EnQueue(EventTask{
				Pattern:   v.Pattern,
				ID:        consulEvent.ID,
				No:        c,
				Task:      v.Task,
				Skippable: config.Skippable,
			})
			c += 1
		case len(v.OrderedTasks) > 0:
			for _, t := range v.OrderedTasks {
				t.Pattern = v.Pattern
				t.ID = consulEvent.ID
				t.No = c
				pq.EnQueue(t)
				c += 1
			}
		}
	}

	//	Log starting event as EventResult on KVS
	result = &EventResult{
		ID:        consulEvent.ID,
		Name:      consulEvent.Name,
		Status:    "inprogress",
		StartedAt: time.Now(),
	}
	return result.Save()
}
Example #3
0
func (et *EventTask) IsFinished(ch chan EventTask) bool {
	//	Finished task when timeout has occurred
	select {
	case timeout := <-ch:
		if et.ID == timeout.ID && et.No == timeout.No {
			log.Errorf("Task has been reached timeout(%s)", et.String())
			return true
		}
	default:
	}

	nodes, _, err := util.Consul().Catalog().Nodes(&api.QueryOptions{})
	if err != nil {
		return false
	}

	filteredNodes := et.filterNodes(nodes)
	if len(filteredNodes) == 0 {
		return false
	}

	//	Wait for finishing tasks on target node
	for _, node := range filteredNodes {
		result, err := getNodeTaskResult(et.ID, et.No, node.Node)
		if err != nil || result == nil || !result.IsFinished() {
			return false
		}
	}
	return true
}
Example #4
0
//	Trigger channel when current task has been reached timeout
func taskTimeout(ch chan EventTask) {
	var prev EventTask
	var now EventTask
	for {
		time.Sleep(1 * time.Second)
		pq := &queue.Queue{
			Client: util.Consul(),
			Key:    PROGRESS_QUEUE_KEY,
		}

		//	Wait until task has dispatched
		if err := pq.FetchHead(&now); err != nil || now.ID == "" || prev.ID == now.ID && prev.No == now.No {
			continue
		}
		prev = now

		//	Wait until current task has started on any node or timeout
		cancel := make(chan bool)
		select {
		case <-startTask(now, cancel):
		case <-time.After(TASK_TIMEOUT_WITHOUT_START * time.Second):
			cancel <- true
			ch <- now
			continue
		}

		//	Wait until current task has finished or timeout
		select {
		case <-changeTask(now, cancel):
		case <-time.After(time.Duration(TASK_TIMEOUT) * time.Second):
			cancel <- true
			ch <- now
		}
	}
}
Example #5
0
func (o *ConsulKVSOperation) get(vars map[string]string) error {
	//	Store value that has been get to variables map
	kv, _, err := util.Consul().KV().Get(o.Key, &api.QueryOptions{})
	vars[o.Name] = string(kv.Value)
	log.Infof("Get %s from %s and store to %s", kv.Value, kv.Key, o.Name)
	return err
}
Example #6
0
func getAttributes(keys []string, overwriteAttributes map[string]interface{}) (map[string]interface{}, error) {
	var attributes map[string]interface{}
	var c *api.Client = util.Consul()
	attributes = make(map[string]interface{})

	//	Get attributes from consul KVS
	for _, key := range keys {
		list, _, err := c.KV().List(key, &api.QueryOptions{})
		if err != nil {
			return nil, err
		}

		for _, kv := range list {
			var a map[string]interface{}
			if err := json.Unmarshal(kv.Value, &a); err != nil {
				return nil, err
			}
			if err := mergo.MergeWithOverwrite(&attributes, a); err != nil {
				return nil, errors.New(fmt.Sprintf("Failed to merge attributes(%v)", err))
			}
		}
	}

	//	Overwrite some attributes by specified parameter in task.yml
	if err := mergo.MergeWithOverwrite(&attributes, overwriteAttributes); err != nil {
		return nil, err
	}
	return attributes, nil
}
Example #7
0
//	Trigger channel when change current task
func changeTask(et EventTask, cancel chan bool) chan bool {
	ch := make(chan bool)

	go func(chan bool) {
		pq := &queue.Queue{
			Client: util.Consul(),
			Key:    PROGRESS_QUEUE_KEY,
		}

		for {
			time.Sleep(1 * time.Second)
			//	Exit when cancel channel has signalled
			select {
			case <-cancel:
				return
			default:
			}

			//  Send signal to change task channel when current task has been changed
			var now EventTask
			if err := pq.FetchHead(&now); err != nil {
				ch <- true
				return
			}

			if et.ID != now.ID || et.No != now.No {
				ch <- true
				return
			}
		}
	}(ch)

	return ch
}
Example #8
0
func (s *Scheduler) polling(ch chan EventTask) error {
	//	Create critical section by consul lock
	l, err := util.Consul().LockKey(LOCK_KEY)
	if err != nil {
		return err
	}
	if _, err := l.Lock(nil); err != nil {
		return err
	}
	defer l.Unlock()

	//	Polling tasks from queue
	var eventTasks []EventTask
	pq := &queue.Queue{
		Client: util.Consul(),
		Key:    PROGRESS_QUEUE_KEY,
	}
	if err := pq.Items(&eventTasks); err != nil {
		return err
	}

	if config.Debug {
		log.Debug("-------- Progress Task Queue --------")
		nodes, _, _ := util.Consul().Catalog().Nodes(&api.QueryOptions{})
		for _, et := range eventTasks {
			log.Debug(et.String())
			for _, n := range nodes {
				log.Debugf("%s: %t", n.Node, et.Runnable(n.Node))
			}
		}
	}

	switch {
	case len(eventTasks) == 0:
		return s.dispatchEvent()
	case eventTasks[0].Runnable(s.node):
		//	runTask is parallelizable
		l.Unlock()
		return s.runTask(eventTasks[0])
	case eventTasks[0].IsFinished(ch):
		return s.finishTask(eventTasks[0])
	default:
		log.Debugf("Wait a task will have been finished by other instance(%s)", eventTasks[0].String())
	}
	return nil
}
Example #9
0
func (o *ConsulKVSOperation) put(vars map[string]string) error {
	kv := &api.KVPair{
		Key:   o.Key,
		Value: []byte(o.Value),
	}
	_, err := util.Consul().KV().Put(kv, &api.WriteOptions{})
	log.Infof("Put %s to %s", o.Value, o.Key)
	return err
}
Example #10
0
func (r *NodeTaskResult) Save() error {
	//	Save any result to consul KVS
	if err := putResult(r); err != nil {
		return err
	}
	kv := &api.KVPair{
		Key:   r.Key() + "/log",
		Value: []byte(r.Log),
	}
	_, err := util.Consul().KV().Put(kv, &api.WriteOptions{})
	return err
}
Example #11
0
func (o *ConsulEventOperation) Run(vars map[string]string) error {
	event := &api.UserEvent{
		Name:          o.Name,
		ServiceFilter: o.Filter.Service,
		TagFilter:     o.Filter.Tag,
		Payload:       []byte(config.Token),
	}

	id, _, err := util.Consul().Event().Fire(event, &api.WriteOptions{})
	log.Infof("consul-event: Fire %s event(ID: %s)", o.Name, id)
	return err
}
Example #12
0
//	Put any result to consul KVS with JSON format
func putResult(result Result) error {
	d, err := json.Marshal(result)
	if err != nil {
		return err
	}

	kv := api.KVPair{
		Key:   result.Key(),
		Value: d,
	}
	_, err = util.Consul().KV().Put(&kv, &api.WriteOptions{})
	return err
}
Example #13
0
//	Get any result from consul KVS
func getResult(key string, result interface{}) (bool, error) {
	kv, _, err := util.Consul().KV().Get(key, &api.QueryOptions{})
	if err != nil {
		return false, err
	}

	if kv == nil || len(kv.Value) == 0 {
		return false, nil
	}

	if err := json.Unmarshal(kv.Value, &result); err != nil {
		return false, err
	}
	return true, nil
}
Example #14
0
func getNodeTaskResult(id string, no int, node string) (*NodeTaskResult, error) {
	var result NodeTaskResult
	key := EVENT_RESULT_KEY + "/" + id + "/" + strconv.Itoa(no) + "/" + node
	found, err := getResult(key, &result)
	if !found || err != nil {
		return nil, err
	}

	//	Read log from /metronome/result/[EventID]/[No]/[Node]/log
	kv, _, err := util.Consul().KV().Get(key+"/log", &api.QueryOptions{})
	if err != nil {
		return nil, err
	}
	if kv != nil {
		result.Log = string(kv.Value)
	}
	return &result, err
}
Example #15
0
func (r *TaskResult) GetNodeResults() ([]NodeTaskResult, error) {
	//	Collect all results on node that belongs with this task
	var results []NodeTaskResult

	prefix := EVENT_RESULT_KEY + "/" + r.EventID + "/" + strconv.Itoa(r.No)
	kvs, _, err := util.Consul().KV().List(prefix, &api.QueryOptions{})
	if err != nil {
		return nil, err
	}

	for _, kv := range kvs {
		//	Except log record on each node
		node := strings.TrimPrefix(kv.Key, prefix)
		if node == "" || strings.HasSuffix(node, "/log") {
			continue
		}
		result, err := getNodeTaskResult(r.EventID, r.No, node)
		if err != nil {
			return nil, err
		}
		results = append(results, *result)
	}
	return results, nil
}
Example #16
0
//	Finish current task when no node in consul catalog will execute current task
func (s *Scheduler) finishTask(task EventTask) error {
	log.Infof("Finish task(%s)", task.String())
	pq := &queue.Queue{
		Client: util.Consul(),
		Key:    PROGRESS_QUEUE_KEY,
	}

	result, err := task.GetResult()
	if err != nil {
		return err
	}

	nodeResults, err := result.GetNodeResults()
	if err != nil {
		return err
	}

	//	Collect task results over all nodes
	status := "success"
	if len(nodeResults) == 0 {
		if task.Skippable {
			status = "skip"
		} else {
			status = "timeout"
		}
	}

	for _, nr := range nodeResults {
		if nr.Status == "error" {
			status = "error"
			break
		}
		if nr.Status == "inprogress" {
			status = "timeout"
		}
	}

	if status == "error" || status == "timeout" {
		// remove following tasks in progress task queue when some error occured or task has reached timeout
		pq.Clear()
	}

	//	Log finishing task as TaskResult on KVS
	result.Status = status
	result.FinishedAt = time.Now()
	if err := result.Save(); err != nil {
		return err
	}

	//	Dequeue task from task queue when finished task over all all nodes
	var dummy EventTask
	if err, _ := pq.DeQueue(&dummy); err != nil {
		return err
	}

	//	Log finishing event as EventResult on KVS when finished all task in a progress task queue
	var tasks []EventTask
	if err := pq.Items(&tasks); err != nil {
		return err
	}
	if len(tasks) == 0 {
		eventResult, err := getEventResult(task.ID)
		if err != nil {
			return err
		}
		eventResult.Status = status
		eventResult.FinishedAt = time.Now()
		if err := eventResult.Save(); err != nil {
			return err
		}
	}

	return nil
}
Example #17
0
func (o *ConsulKVSOperation) delete(vars map[string]string) error {
	_, err := util.Consul().KV().Delete(o.Key, &api.WriteOptions{})
	log.Infof("Delete %s", o.Key)
	return err
}