Exemple #1
0
func taskWorker(ct config.Task, errorBackoff *backoff.Backoff) {
	for task := range workerChan[ct.Type] {
		output.Debug("Executing task type", ct.Type, "with arguments", task.Args)
		stats.IncrTaskCount(ct.Type)

		err := task.Execute(ct.Script)
		if err != nil {
			task.ErrorMessage = fmt.Sprintf("%s", err)
			failedChan <- failedTask{
				configTask: ct,
				queueTask:  task,
			}

			msg := fmt.Sprintf("Failed executing task: %s \"%s\"\n%s", ct.Script, strings.Join(task.Args, "\" \""), err)
			output.NotifyError(msg)
		}

		if errorBackoff != nil {
			if err == nil {
				errorBackoff.Reset()
			} else {
				errorBackoff.Duration()
			}
		}

		output.Debug("Finished task type", ct.Type, "with arguments", task.Args)
	}

	waitGroup.Done()
}
Exemple #2
0
func dequeue(id int, wg *sync.WaitGroup) {
	defer wg.Done()
	b := backoff.Backoff{Min: time.Second}
	for tx := range queue {
	attempt:
		if err := transfer(tx); err != nil {
			if b.Attempt() == 10 {
				fmt.Printf("\n[ERROR] %s: TRANSFER '%s' (give up)\n", tx.id, err)
				return
			}
			d := b.Duration()
			fmt.Printf("\n[ERROR] %s: TRANSFER '%s' (retrying in %s)\n", tx.id, err, d)
			time.Sleep(d)
			goto attempt
		}
		b.Reset()
	}
}
Exemple #3
0
func queueWorker() {
	interval := backoff.Backoff{
		Min:    time.Duration(conf.IntervalMin) * time.Millisecond,
		Max:    time.Duration(conf.IntervalMax) * time.Millisecond,
		Factor: conf.IntervalFactor,
	}

	runIntervalLoop := make(chan bool)
	doneIntervalLoop := make(chan bool)

	go func() {
		for {
			<-doneIntervalLoop
			time.Sleep(interval.Duration())

			if isShuttingDown() {
				break
			}

			runIntervalLoop <- true
		}
	}()

	go func() {
		for {
			select {
			case <-shutdownChan:
				runIntervalLoop <- false
			}
		}
	}()

	doneIntervalLoop <- true

	for <-runIntervalLoop {
		for taskType, configTask := range conf.Tasks {
			if isShuttingDown() {
				break
			}

			output.Debug("Checking for new tasks (" + taskType + ")")

			// check if there are available workers
			if !acceptsTasks(taskType) {
				continue
			}

			queueKey := conf.RedisQueueKey + ":" + taskType

			llen, err := redisPool.Cmd("LLEN", queueKey).Int()
			if err != nil {
				// Errors here are likely redis-connection errors, so we'll
				// need to notify about it
				output.NotifyError("redisPool.Cmd() Error:", err)
				break
			}

			// there are no new tasks in redis
			if llen == 0 {
				continue
			}

			// iterate over all entries in redis, until no more are available,
			// or all workers are busy, for a maximum of 2 * workers
			for i := 0; i < (configTask.Workers * 2); i++ {
				if !acceptsTasks(taskType) {
					break
				}

				value, err := redisPool.Cmd("LPOP", queueKey).Str()
				if err != nil {
					// no more tasks found
					break
				}

				output.Debug("Fetched task for type", taskType, "with payload", value)

				task, err := NewQueueTask(value)
				if err != nil {
					output.NotifyError("NewQueueTask():", err)
					continue
				}

				workerChan[taskType] <- task

				// we've actually are handling new tasks so reset the interval
				interval.Reset()
			}
		}

		doneIntervalLoop <- true
	}

	Stop()
	waitGroup.Done()
}
Exemple #4
0
func main() {
	c := config{
		URL:      "http://localhost:8086",
		Database: "test",
		Interval: 5 * time.Minute,
	}

	opts.New(&c).Name("sysflux").Version(VERSION).Parse()

	//validate config
	u, err := url.Parse(c.URL)
	if err != nil {
		log.Fatal("Invalid URL")
	}
	host, port, err := net.SplitHostPort(u.Host)
	if err != nil {
		log.Fatal("Invalid host and port")
	}
	if u.Path == "" {
		u.Path = "/write"
	}
	v := url.Values{}
	v.Set("db", c.Database)
	u.RawQuery = v.Encode()

	tags := ""
	if c.Tags != "" {
		tags = "," + c.Tags
	}

	//good to go
	log.Printf("Using InfluxDB endpoint: %s", u)

	success := false
	lock := sync.Mutex{}
	entries := []string{}

	send := func() error {
		body := strings.NewReader(strings.Join(entries, "\n"))
		if c.DNS != "" {
			//lookup host every POST
			ips, err := lookup(host, c.DNS)
			if err != nil {
				return fmt.Errorf("Lookup failed: %s", err)
			}
			u.Host = ips[0] + ":" + port
		}
		resp, err := http.Post(u.String(), "application/x-www-form-urlencoded", body)
		if err != nil {
			return fmt.Errorf("HTTP POST failed: %s", err)
		}
		defer resp.Body.Close()
		if resp.StatusCode != http.StatusNoContent {
			msg, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return fmt.Errorf("Response download failed: %s", err)
			}
			return fmt.Errorf("Response: %d => %s", resp.StatusCode, msg)
		}
		//show first success
		if !success {
			log.Printf("Success")
			success = true
		}
		//clear once recieved!
		entries = nil
		return nil
	}

	report := func() {
		t := time.Now().UnixNano()
		if l, err := load.LoadAvg(); err == nil {
			entries = append(entries, fmt.Sprintf("cpu_load_short%s value=%f %d", tags, l.Load1*100, t))
			entries = append(entries, fmt.Sprintf("cpu_load_medium%s value=%f %d", tags, l.Load5*100, t))
			entries = append(entries, fmt.Sprintf("cpu_load_long%s value=%f %d", tags, l.Load15*100, t))
		}
		if v, err := mem.VirtualMemory(); err == nil {
			entries = append(entries, fmt.Sprintf("mem_usage%s value=%f %d", tags, v.UsedPercent, t))
		}
		if len(entries) > MAX_QUEUED {
			entries = entries[len(entries)-MAX_QUEUED:]
		}
	}

	//send loop
	go func() {
		b := backoff.Backoff{}
		for {
			wait := time.Second
			lock.Lock()
			if len(entries) > 0 {
				if err := send(); err == nil {
					b.Reset()
				} else {
					log.Println(err)
					wait = b.Duration()
				}
			}
			lock.Unlock()
			time.Sleep(wait)
		}
	}()

	//report loop
	for {
		lock.Lock()
		report()
		lock.Unlock()
		time.Sleep(c.Interval)
	}
}
Exemple #5
0
func main() {

	//configuration defaults
	conf := config{
		Server:          "http://localhost:8086",
		Database:        "test",
		Username:        "",
		Password:        "",
		Measurement:     "data",
		BatchSize:       5000,
		ForceFloat:      false,
		ForceString:     false,
		TreatNull:       false,
		TimestampColumn: "timestamp",
		TimestampFormat: "2006-01-02 15:04:05",
		HttpTimeout:     10,
	}

	//parse config
	opts.New(&conf).
		Name("csv-to-influxdb").
		Repo("github.com/jpillora/csv-to-influxdb").
		Version(VERSION).
		Parse()

	//set tag names
	tagNames := map[string]bool{}
	for _, name := range strings.Split(conf.TagColumns, ",") {
		name = strings.TrimSpace(name)
		if name != "" {
			tagNames[name] = true
		}
	}

	//regular expressions
	numbersRe := regexp.MustCompile(`\d`)
	integerRe := regexp.MustCompile(`^\d+$`)
	floatRe := regexp.MustCompile(`^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$`)
	trueRe := regexp.MustCompile(`^(true|T|True|TRUE)$`)
	falseRe := regexp.MustCompile(`^(false|F|False|FALSE)$`)
	nullRe := regexp.MustCompile(`^(null|Null|NULL)$`)
	timestampRe, err := regexp.Compile("^" + numbersRe.ReplaceAllString(conf.TimestampFormat, `\d`) + "$")
	if err != nil {
		log.Fatalf("time stamp regexp creation failed")
	}

	//influxdb client
	//u, err := url.Parse(conf.Server)
	//if err != nil {
	//	log.Fatalf("Invalid server address: %s", err)
	//}
	c, err := client.NewHTTPClient(client.HTTPConfig{Addr: conf.Server, Username: conf.Username, Password: conf.Password, Timeout: time.Duration(conf.HttpTimeout) * time.Second})
	defer c.Close()

	dbsResp, err := c.Query(client.Query{Command: "SHOW DATABASES"})
	if err != nil {
		log.Fatalf("Invalid server address: %s", err)
	}

	dbExists := false
	if len(dbsResp.Results) == 0 {
		log.Fatalf("No databases found, probably an authentication issue, please provide username and password.")
	}
	for _, v := range dbsResp.Results[0].Series[0].Values {
		dbName := v[0].(string)
		if conf.Database == dbName {
			dbExists = true
			break
		}
	}

	if !dbExists {
		if conf.NoAutoCreate {
			log.Fatalf("Database '%s' does not exist", conf.Database)
		}
		_, err := c.Query(client.Query{Command: "CREATE DATABASE \"" + conf.Database + "\""})
		if err != nil {
			log.Fatalf("Failed to create database: %s", err)
		}
	}

	//open csv file
	f, err := os.Open(conf.CSVFile)
	if err != nil {
		log.Fatalf("Failed to open %s", conf.CSVFile)
	}

	//headers and init fn
	var firstField string
	var headers []string
	setHeaders := func(hdrs []string) {
		//check timestamp and tag columns
		hasTs := false
		n := len(tagNames)
		for _, h := range hdrs {
			if h == conf.TimestampColumn {
				hasTs = true
			} else if tagNames[h] {
				log.Println(h)
				n--
			} else if firstField == "" {
				firstField = h
			}
		}
		if firstField == "" {
			log.Fatalf("You must have at least one field (non-tag)")
		}
		if !hasTs {
			log.Fatalf("Timestamp column (%s) does not match any header (%s)", conf.TimestampColumn, strings.Join(headers, ","))
		}
		if n > 0 {
			log.Fatalf("Tag names (%s) to do not all have matching headers (%s)", conf.TagColumns, strings.Join(headers, ","))
		}
		headers = hdrs
	}

	var bpConfig = client.BatchPointsConfig{Database: conf.Database}
	bp, _ := client.NewBatchPoints(bpConfig) //current batch
	bpSize := 0
	totalSize := 0

	// lastCount := ""

	//write the current batch
	write := func() {
		if bpSize == 0 {
			return
		}
		b := backoff.Backoff{}
		for {
			if err := c.Write(bp); err != nil {
				d := b.Duration()
				log.Printf("Write failed: %s (retrying in %s)", err, d)
				if int(b.Attempt()) == conf.Attempts {
					log.Fatalf("Failed to write to db after %d attempts", int(b.Attempt()))
				}
				time.Sleep(d)
				continue
			}
			break
		}
		//TODO(jpillora): wait until the new points become readable
		// count := ""
		// for count == lastCount {
		// 	resp, err := c.Query(client.Query{Command: "SELECT count(" + firstField + ") FROM " + conf.Measurement, Database: conf.Database})
		// 	if err != nil {
		// 		log.Fatal("failed to count rows")
		// 	}
		// 	count = resp.Results[0].Series[0].Values[0][1].(string)
		// }
		//reset
		bp, _ = client.NewBatchPoints(bpConfig)
		bpSize = 0
	}

	//read csv, line by line
	r := csv.NewReader(f)
	for i := 0; ; i++ {
		records, err := r.Read()
		if err != nil {
			if err == io.EOF {
				break
			}
			log.Fatalf("CSV error: %s", err)
		}
		if i == 0 {
			setHeaders(records)
			continue
		}

		// Create a point and add to batch
		tags := map[string]string{}
		fields := map[string]interface{}{}

		var ts time.Time

		//move all into tags and fields
		for hi, h := range headers {
			r := records[hi]
			if len(r) == 0 {
				continue
			}
			//tags are just strings
			if tagNames[h] {
				tags[h] = r
				continue
			}

			//fields require string parsing
			if timestampRe.MatchString(r) {
				t, err := time.Parse(conf.TimestampFormat, r)
				if err != nil {
					fmt.Printf("#%d: %s: Invalid time: %s\n", i, h, err)
					continue
				}
				if conf.TimestampColumn == h {
					ts = t //the timestamp column!
					continue
				}
				fields[h] = t
			} else if !conf.ForceFloat && !conf.ForceString && integerRe.MatchString(r) {
				i, _ := strconv.Atoi(r)
				fields[h] = i
			} else if !conf.ForceString && floatRe.MatchString(r) {
				f, _ := strconv.ParseFloat(r, 64)
				fields[h] = f
			} else if trueRe.MatchString(r) {
				fields[h] = true
			} else if falseRe.MatchString(r) {
				fields[h] = false
			} else if conf.TreatNull && nullRe.MatchString(r) {
				// null values must not be inserted into InfluxDB
				continue
			} else {
				fields[h] = r
			}
		}

		p, err := client.NewPoint(conf.Measurement, tags, fields, ts)

		bp.AddPoint(p)
		bpSize++
		totalSize++
		if bpSize == conf.BatchSize {
			write()
		}
	}
	//send remainder
	write()
	log.Printf("Done (wrote %d points)", totalSize)
}