Example #1
0
func taskWorker(ct config.Task, errorBackoff *backoff.Backoff) {
	for task := range workerChan[ct.Type] {
		output.Debug("Executing task type", ct.Type, "with arguments", task.Args)
		stats.IncrTaskCount(ct.Type)

		err := task.Execute(ct.Script)
		if err != nil {
			task.ErrorMessage = fmt.Sprintf("%s", err)
			failedChan <- failedTask{
				configTask: ct,
				queueTask:  task,
			}

			msg := fmt.Sprintf("Failed executing task: %s \"%s\"\n%s", ct.Script, strings.Join(task.Args, "\" \""), err)
			output.NotifyError(msg)
		}

		if errorBackoff != nil {
			if err == nil {
				errorBackoff.Reset()
			} else {
				errorBackoff.Duration()
			}
		}

		output.Debug("Finished task type", ct.Type, "with arguments", task.Args)
	}

	waitGroup.Done()
}
Example #2
0
func dequeue(id int, wg *sync.WaitGroup) {
	defer wg.Done()
	b := backoff.Backoff{Min: time.Second}
	for tx := range queue {
	attempt:
		if err := transfer(tx); err != nil {
			if b.Attempt() == 10 {
				fmt.Printf("\n[ERROR] %s: TRANSFER '%s' (give up)\n", tx.id, err)
				return
			}
			d := b.Duration()
			fmt.Printf("\n[ERROR] %s: TRANSFER '%s' (retrying in %s)\n", tx.id, err, d)
			time.Sleep(d)
			goto attempt
		}
		b.Reset()
	}
}
Example #3
0
func queueWorker() {
	interval := backoff.Backoff{
		Min:    time.Duration(conf.IntervalMin) * time.Millisecond,
		Max:    time.Duration(conf.IntervalMax) * time.Millisecond,
		Factor: conf.IntervalFactor,
	}

	runIntervalLoop := make(chan bool)
	doneIntervalLoop := make(chan bool)

	go func() {
		for {
			<-doneIntervalLoop
			time.Sleep(interval.Duration())

			if isShuttingDown() {
				break
			}

			runIntervalLoop <- true
		}
	}()

	go func() {
		for {
			select {
			case <-shutdownChan:
				runIntervalLoop <- false
			}
		}
	}()

	doneIntervalLoop <- true

	for <-runIntervalLoop {
		for taskType, configTask := range conf.Tasks {
			if isShuttingDown() {
				break
			}

			output.Debug("Checking for new tasks (" + taskType + ")")

			// check if there are available workers
			if !acceptsTasks(taskType) {
				continue
			}

			queueKey := conf.RedisQueueKey + ":" + taskType

			llen, err := redisPool.Cmd("LLEN", queueKey).Int()
			if err != nil {
				// Errors here are likely redis-connection errors, so we'll
				// need to notify about it
				output.NotifyError("redisPool.Cmd() Error:", err)
				break
			}

			// there are no new tasks in redis
			if llen == 0 {
				continue
			}

			// iterate over all entries in redis, until no more are available,
			// or all workers are busy, for a maximum of 2 * workers
			for i := 0; i < (configTask.Workers * 2); i++ {
				if !acceptsTasks(taskType) {
					break
				}

				value, err := redisPool.Cmd("LPOP", queueKey).Str()
				if err != nil {
					// no more tasks found
					break
				}

				output.Debug("Fetched task for type", taskType, "with payload", value)

				task, err := NewQueueTask(value)
				if err != nil {
					output.NotifyError("NewQueueTask():", err)
					continue
				}

				workerChan[taskType] <- task

				// we've actually are handling new tasks so reset the interval
				interval.Reset()
			}
		}

		doneIntervalLoop <- true
	}

	Stop()
	waitGroup.Done()
}
Example #4
0
func main() {
	c := config{
		URL:      "http://localhost:8086",
		Database: "test",
		Interval: 5 * time.Minute,
	}

	opts.New(&c).Name("sysflux").Version(VERSION).Parse()

	//validate config
	u, err := url.Parse(c.URL)
	if err != nil {
		log.Fatal("Invalid URL")
	}
	host, port, err := net.SplitHostPort(u.Host)
	if err != nil {
		log.Fatal("Invalid host and port")
	}
	if u.Path == "" {
		u.Path = "/write"
	}
	v := url.Values{}
	v.Set("db", c.Database)
	u.RawQuery = v.Encode()

	tags := ""
	if c.Tags != "" {
		tags = "," + c.Tags
	}

	//good to go
	log.Printf("Using InfluxDB endpoint: %s", u)

	success := false
	lock := sync.Mutex{}
	entries := []string{}

	send := func() error {
		body := strings.NewReader(strings.Join(entries, "\n"))
		if c.DNS != "" {
			//lookup host every POST
			ips, err := lookup(host, c.DNS)
			if err != nil {
				return fmt.Errorf("Lookup failed: %s", err)
			}
			u.Host = ips[0] + ":" + port
		}
		resp, err := http.Post(u.String(), "application/x-www-form-urlencoded", body)
		if err != nil {
			return fmt.Errorf("HTTP POST failed: %s", err)
		}
		defer resp.Body.Close()
		if resp.StatusCode != http.StatusNoContent {
			msg, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return fmt.Errorf("Response download failed: %s", err)
			}
			return fmt.Errorf("Response: %d => %s", resp.StatusCode, msg)
		}
		//show first success
		if !success {
			log.Printf("Success")
			success = true
		}
		//clear once recieved!
		entries = nil
		return nil
	}

	report := func() {
		t := time.Now().UnixNano()
		if l, err := load.LoadAvg(); err == nil {
			entries = append(entries, fmt.Sprintf("cpu_load_short%s value=%f %d", tags, l.Load1*100, t))
			entries = append(entries, fmt.Sprintf("cpu_load_medium%s value=%f %d", tags, l.Load5*100, t))
			entries = append(entries, fmt.Sprintf("cpu_load_long%s value=%f %d", tags, l.Load15*100, t))
		}
		if v, err := mem.VirtualMemory(); err == nil {
			entries = append(entries, fmt.Sprintf("mem_usage%s value=%f %d", tags, v.UsedPercent, t))
		}
		if len(entries) > MAX_QUEUED {
			entries = entries[len(entries)-MAX_QUEUED:]
		}
	}

	//send loop
	go func() {
		b := backoff.Backoff{}
		for {
			wait := time.Second
			lock.Lock()
			if len(entries) > 0 {
				if err := send(); err == nil {
					b.Reset()
				} else {
					log.Println(err)
					wait = b.Duration()
				}
			}
			lock.Unlock()
			time.Sleep(wait)
		}
	}()

	//report loop
	for {
		lock.Lock()
		report()
		lock.Unlock()
		time.Sleep(c.Interval)
	}
}