func taskWorker(ct config.Task, errorBackoff *backoff.Backoff) {
for task := range workerChan[ct.Type] {
output.Debug("Executing task type", ct.Type, "with arguments", task.Args)
stats.IncrTaskCount(ct.Type)
err := task.Execute(ct.Script)
if err != nil {
task.ErrorMessage = fmt.Sprintf("%s", err)
failedChan <- failedTask{
configTask: ct,
queueTask: task,
}
msg := fmt.Sprintf("Failed executing task: %s \"%s\"\n%s", ct.Script, strings.Join(task.Args, "\" \""), err)
output.NotifyError(msg)
}
if errorBackoff != nil {
if err == nil {
errorBackoff.Reset()
} else {
errorBackoff.Duration()
}
}
output.Debug("Finished task type", ct.Type, "with arguments", task.Args)
}
waitGroup.Done()
}
func dequeue(id int, wg *sync.WaitGroup) {
defer wg.Done()
b := backoff.Backoff{Min: time.Second}
for tx := range queue {
attempt:
if err := transfer(tx); err != nil {
if b.Attempt() == 10 {
fmt.Printf("\n[ERROR] %s: TRANSFER '%s' (give up)\n", tx.id, err)
return
}
d := b.Duration()
fmt.Printf("\n[ERROR] %s: TRANSFER '%s' (retrying in %s)\n", tx.id, err, d)
time.Sleep(d)
goto attempt
}
b.Reset()
}
}
func queueWorker() {
interval := backoff.Backoff{
Min: time.Duration(conf.IntervalMin) * time.Millisecond,
Max: time.Duration(conf.IntervalMax) * time.Millisecond,
Factor: conf.IntervalFactor,
}
runIntervalLoop := make(chan bool)
doneIntervalLoop := make(chan bool)
go func() {
for {
<-doneIntervalLoop
time.Sleep(interval.Duration())
if isShuttingDown() {
break
}
runIntervalLoop <- true
}
}()
go func() {
for {
select {
case <-shutdownChan:
runIntervalLoop <- false
}
}
}()
doneIntervalLoop <- true
for <-runIntervalLoop {
for taskType, configTask := range conf.Tasks {
if isShuttingDown() {
break
}
output.Debug("Checking for new tasks (" + taskType + ")")
// check if there are available workers
if !acceptsTasks(taskType) {
continue
}
queueKey := conf.RedisQueueKey + ":" + taskType
llen, err := redisPool.Cmd("LLEN", queueKey).Int()
if err != nil {
// Errors here are likely redis-connection errors, so we'll
// need to notify about it
output.NotifyError("redisPool.Cmd() Error:", err)
break
}
// there are no new tasks in redis
if llen == 0 {
continue
}
// iterate over all entries in redis, until no more are available,
// or all workers are busy, for a maximum of 2 * workers
for i := 0; i < (configTask.Workers * 2); i++ {
if !acceptsTasks(taskType) {
break
}
value, err := redisPool.Cmd("LPOP", queueKey).Str()
if err != nil {
// no more tasks found
break
}
output.Debug("Fetched task for type", taskType, "with payload", value)
task, err := NewQueueTask(value)
if err != nil {
output.NotifyError("NewQueueTask():", err)
continue
}
workerChan[taskType] <- task
// we've actually are handling new tasks so reset the interval
interval.Reset()
}
}
doneIntervalLoop <- true
}
Stop()
waitGroup.Done()
}
func main() {
c := config{
URL: "http://localhost:8086",
Database: "test",
Interval: 5 * time.Minute,
}
opts.New(&c).Name("sysflux").Version(VERSION).Parse()
//validate config
u, err := url.Parse(c.URL)
if err != nil {
log.Fatal("Invalid URL")
}
host, port, err := net.SplitHostPort(u.Host)
if err != nil {
log.Fatal("Invalid host and port")
}
if u.Path == "" {
u.Path = "/write"
}
v := url.Values{}
v.Set("db", c.Database)
u.RawQuery = v.Encode()
tags := ""
if c.Tags != "" {
tags = "," + c.Tags
}
//good to go
log.Printf("Using InfluxDB endpoint: %s", u)
success := false
lock := sync.Mutex{}
entries := []string{}
send := func() error {
body := strings.NewReader(strings.Join(entries, "\n"))
if c.DNS != "" {
//lookup host every POST
ips, err := lookup(host, c.DNS)
if err != nil {
return fmt.Errorf("Lookup failed: %s", err)
}
u.Host = ips[0] + ":" + port
}
resp, err := http.Post(u.String(), "application/x-www-form-urlencoded", body)
if err != nil {
return fmt.Errorf("HTTP POST failed: %s", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusNoContent {
msg, err := ioutil.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("Response download failed: %s", err)
}
return fmt.Errorf("Response: %d => %s", resp.StatusCode, msg)
}
//show first success
if !success {
log.Printf("Success")
success = true
}
//clear once recieved!
entries = nil
return nil
}
report := func() {
t := time.Now().UnixNano()
if l, err := load.LoadAvg(); err == nil {
entries = append(entries, fmt.Sprintf("cpu_load_short%s value=%f %d", tags, l.Load1*100, t))
entries = append(entries, fmt.Sprintf("cpu_load_medium%s value=%f %d", tags, l.Load5*100, t))
entries = append(entries, fmt.Sprintf("cpu_load_long%s value=%f %d", tags, l.Load15*100, t))
}
if v, err := mem.VirtualMemory(); err == nil {
entries = append(entries, fmt.Sprintf("mem_usage%s value=%f %d", tags, v.UsedPercent, t))
}
if len(entries) > MAX_QUEUED {
entries = entries[len(entries)-MAX_QUEUED:]
}
}
//send loop
go func() {
b := backoff.Backoff{}
for {
wait := time.Second
lock.Lock()
if len(entries) > 0 {
if err := send(); err == nil {
b.Reset()
} else {
log.Println(err)
wait = b.Duration()
}
}
lock.Unlock()
time.Sleep(wait)
}
}()
//report loop
for {
lock.Lock()
report()
lock.Unlock()
time.Sleep(c.Interval)
}
}
func main() {
//configuration defaults
conf := config{
Server: "http://localhost:8086",
Database: "test",
Username: "",
Password: "",
Measurement: "data",
BatchSize: 5000,
ForceFloat: false,
ForceString: false,
TreatNull: false,
TimestampColumn: "timestamp",
TimestampFormat: "2006-01-02 15:04:05",
HttpTimeout: 10,
}
//parse config
opts.New(&conf).
Name("csv-to-influxdb").
Repo("github.com/jpillora/csv-to-influxdb").
Version(VERSION).
Parse()
//set tag names
tagNames := map[string]bool{}
for _, name := range strings.Split(conf.TagColumns, ",") {
name = strings.TrimSpace(name)
if name != "" {
tagNames[name] = true
}
}
//regular expressions
numbersRe := regexp.MustCompile(`\d`)
integerRe := regexp.MustCompile(`^\d+$`)
floatRe := regexp.MustCompile(`^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$`)
trueRe := regexp.MustCompile(`^(true|T|True|TRUE)$`)
falseRe := regexp.MustCompile(`^(false|F|False|FALSE)$`)
nullRe := regexp.MustCompile(`^(null|Null|NULL)$`)
timestampRe, err := regexp.Compile("^" + numbersRe.ReplaceAllString(conf.TimestampFormat, `\d`) + "$")
if err != nil {
log.Fatalf("time stamp regexp creation failed")
}
//influxdb client
//u, err := url.Parse(conf.Server)
//if err != nil {
// log.Fatalf("Invalid server address: %s", err)
//}
c, err := client.NewHTTPClient(client.HTTPConfig{Addr: conf.Server, Username: conf.Username, Password: conf.Password, Timeout: time.Duration(conf.HttpTimeout) * time.Second})
defer c.Close()
dbsResp, err := c.Query(client.Query{Command: "SHOW DATABASES"})
if err != nil {
log.Fatalf("Invalid server address: %s", err)
}
dbExists := false
if len(dbsResp.Results) == 0 {
log.Fatalf("No databases found, probably an authentication issue, please provide username and password.")
}
for _, v := range dbsResp.Results[0].Series[0].Values {
dbName := v[0].(string)
if conf.Database == dbName {
dbExists = true
break
}
}
if !dbExists {
if conf.NoAutoCreate {
log.Fatalf("Database '%s' does not exist", conf.Database)
}
_, err := c.Query(client.Query{Command: "CREATE DATABASE \"" + conf.Database + "\""})
if err != nil {
log.Fatalf("Failed to create database: %s", err)
}
}
//open csv file
f, err := os.Open(conf.CSVFile)
if err != nil {
log.Fatalf("Failed to open %s", conf.CSVFile)
}
//headers and init fn
var firstField string
var headers []string
setHeaders := func(hdrs []string) {
//check timestamp and tag columns
hasTs := false
n := len(tagNames)
for _, h := range hdrs {
if h == conf.TimestampColumn {
hasTs = true
} else if tagNames[h] {
log.Println(h)
n--
} else if firstField == "" {
firstField = h
}
}
if firstField == "" {
log.Fatalf("You must have at least one field (non-tag)")
}
if !hasTs {
log.Fatalf("Timestamp column (%s) does not match any header (%s)", conf.TimestampColumn, strings.Join(headers, ","))
}
if n > 0 {
log.Fatalf("Tag names (%s) to do not all have matching headers (%s)", conf.TagColumns, strings.Join(headers, ","))
}
headers = hdrs
}
var bpConfig = client.BatchPointsConfig{Database: conf.Database}
bp, _ := client.NewBatchPoints(bpConfig) //current batch
bpSize := 0
totalSize := 0
// lastCount := ""
//write the current batch
write := func() {
if bpSize == 0 {
return
}
b := backoff.Backoff{}
for {
if err := c.Write(bp); err != nil {
d := b.Duration()
log.Printf("Write failed: %s (retrying in %s)", err, d)
if int(b.Attempt()) == conf.Attempts {
log.Fatalf("Failed to write to db after %d attempts", int(b.Attempt()))
}
time.Sleep(d)
continue
}
break
}
//TODO(jpillora): wait until the new points become readable
// count := ""
// for count == lastCount {
// resp, err := c.Query(client.Query{Command: "SELECT count(" + firstField + ") FROM " + conf.Measurement, Database: conf.Database})
// if err != nil {
// log.Fatal("failed to count rows")
// }
// count = resp.Results[0].Series[0].Values[0][1].(string)
// }
//reset
bp, _ = client.NewBatchPoints(bpConfig)
bpSize = 0
}
//read csv, line by line
r := csv.NewReader(f)
for i := 0; ; i++ {
records, err := r.Read()
if err != nil {
if err == io.EOF {
break
}
log.Fatalf("CSV error: %s", err)
}
if i == 0 {
setHeaders(records)
continue
}
// Create a point and add to batch
tags := map[string]string{}
fields := map[string]interface{}{}
var ts time.Time
//move all into tags and fields
for hi, h := range headers {
r := records[hi]
if len(r) == 0 {
continue
}
//tags are just strings
if tagNames[h] {
tags[h] = r
continue
}
//fields require string parsing
if timestampRe.MatchString(r) {
t, err := time.Parse(conf.TimestampFormat, r)
if err != nil {
fmt.Printf("#%d: %s: Invalid time: %s\n", i, h, err)
continue
}
if conf.TimestampColumn == h {
ts = t //the timestamp column!
continue
}
fields[h] = t
} else if !conf.ForceFloat && !conf.ForceString && integerRe.MatchString(r) {
i, _ := strconv.Atoi(r)
fields[h] = i
} else if !conf.ForceString && floatRe.MatchString(r) {
f, _ := strconv.ParseFloat(r, 64)
fields[h] = f
} else if trueRe.MatchString(r) {
fields[h] = true
} else if falseRe.MatchString(r) {
fields[h] = false
} else if conf.TreatNull && nullRe.MatchString(r) {
// null values must not be inserted into InfluxDB
continue
} else {
fields[h] = r
}
}
p, err := client.NewPoint(conf.Measurement, tags, fields, ts)
bp.AddPoint(p)
bpSize++
totalSize++
if bpSize == conf.BatchSize {
write()
}
}
//send remainder
write()
log.Printf("Done (wrote %d points)", totalSize)
}