func TestMain(m *testing.M) { // start influx // configure influx to connect to (DO NOT TEST ON PRODUCTION) // viper.SetDefault("influx-address", "http://localhost:8086") viper.SetDefault("influx-address", "http://172.28.128.4:8086") viper.SetDefault("aggregate-interval", 1) // initialize influx queries := []string{ // clean influx to test with (DO NOT RUN ON PRODUCTION) "DROP DATABASE statistics", "CREATE DATABASE statistics", `CREATE RETENTION POLICY one_day ON statistics DURATION 2d REPLICATION 1 DEFAULT`, `CREATE RETENTION POLICY one_week ON statistics DURATION 1w REPLICATION 1`, } for _, query := range queries { _, err := influx.Query(query) if err != nil { fmt.Printf("Failed to QUERY/INITIALIZE - '%v' skipping tests\n", err) os.Exit(0) } } rtn := m.Run() _, err := influx.Query("DROP DATABASE statistics") if err != nil { fmt.Println("Failed to CLEANUP - ", err) os.Exit(1) } os.Exit(rtn) }
func startPulse(ccmd *cobra.Command, args []string) error { // re-initialize logger lumber.Level(lumber.LvlInt(viper.GetString("log-level"))) plex := plexer.NewPlexer() if viper.GetString("mist-address") != "" { mist, err := mist.New(viper.GetString("mist-address"), viper.GetString("mist-token")) if err != nil { return fmt.Errorf("Mist failed to start - %s", err.Error()) } plex.AddObserver("mist", mist.Publish) defer mist.Close() } plex.AddBatcher("influx", influx.Insert) err := pulse.Listen(viper.GetString("server-listen-address"), plex.Publish) if err != nil { return fmt.Errorf("Pulse failed to start - %s", err.Error()) } // begin polling the connected servers pollSec := viper.GetInt("poll-interval") if pollSec == 0 { pollSec = 60 } go pulse.StartPolling(nil, nil, time.Duration(pollSec)*time.Second, nil) queries := []string{ "CREATE DATABASE statistics", "CREATE RETENTION POLICY one_day ON statistics DURATION 8h REPLICATION 1 DEFAULT", fmt.Sprintf("CREATE RETENTION POLICY one_week ON statistics DURATION %dw REPLICATION 1", viper.GetInt("retention")), // todo: ALTER as well? } for _, query := range queries { _, err := influx.Query(query) if err != nil { return fmt.Errorf("Failed to query influx - %s", err.Error()) } } go influx.KeepContinuousQueriesUpToDate() if viper.GetString("kapacitor-address") != "" { err := kapacitor.Init() if err != nil { return fmt.Errorf("Kapacitor failed to start - %s", err.Error()) } } err = api.Start() if err != nil { return fmt.Errorf("Api failed to start - %s", err.Error()) } return nil }
func TestQuery(t *testing.T) { // ensure insert worked response, err := influx.Query(`Select * from one_day./.*/`) if err != nil || len(response.Results) < 1 { t.Error("Failed to QUERY influx - ", err) t.FailNow() } cpu_used := response.Results[0].Series[0].Values[0][1] if cpu_used != json.Number("0.34") { t.Error("Failed to QUERY influx - ( BAD INSERT: expected: 0.34, got: ", cpu_used, ")") } }
func TestContinuousQuery(t *testing.T) { // start cq checker go influx.KeepContinuousQueriesUpToDate() // give it a second to update time.Sleep(time.Second) // ensure insert worked response, err := influx.Query(`SHOW CONTINUOUS QUERIES`) if err != nil { t.Error("Failed to QUERY influx - ", err) t.FailNow() } cq := response.Results[0].Series[1].Values[0][1] if cq != "CREATE CONTINUOUS QUERY aggregate ON statistics BEGIN SELECT mean(cpu_used) AS cpu_used, mean(ram_used) AS ram_used INTO statistics.one_week.aggregate FROM statistics.one_day./.*/ GROUP BY time(1m), host END" { t.Error("Failed to UPDATE CONTINUOUS QUERY influx - mismatched create statement") fmt.Printf("%+q\n", cq) } }
// return a list of tags func tagsRequest(res http.ResponseWriter, req *http.Request) { groupBy, err := influx.Query("SHOW TAG KEYS FROM one_day./.*/") if err != nil { panic(err) } columns := make(map[string]struct{}) for _, result := range groupBy.Results { for _, series := range result.Series { for _, val := range series.Values { columns[val[0].(string)] = struct{}{} } } } var tags []string for k, _ := range columns { tags = append(tags, k) } writeBody(tags, res, http.StatusOK, req) }
// return a list of fields func keysRequest(res http.ResponseWriter, req *http.Request) { cols, err := influx.Query("SHOW FIELD KEYS FROM one_day./.*/") // or show measurements? - aggregate if err != nil { panic(err) } columns := make(map[string]struct{}) for _, result := range cols.Results { for _, series := range result.Series { for _, val := range series.Values { columns[val[0].(string)] = struct{}{} } } } var fields []string for k, _ := range columns { fields = append(fields, k) } writeBody(fields, res, http.StatusOK, req) }
// fetches the latest stat for either a single filter (eg. host) or the average of multiple func latestStat(res http.ResponseWriter, req *http.Request) { // todo: prevent sql(like)-injection (start secure, its their own private stat db otherwise) // verb is what to get to the stat (mean, min, max) verb := req.URL.Query().Get("verb") if verb == "" { verb = "mean" // default to average (avg of [x] is x) } // multiFilter allows for information to be given when multiple same-key filters are applied // on a query where verb is 'none'. eg. `host=a&host=b&verb=none` would return 2 results and // that include the host info. multiFilter := []string{} // default how many responses to limit limit := 1 // filters holds the influxDB language for the WHERE clause filters := []string{} for key, val := range req.URL.Query() { if key == ":stat" || key == "verb" || key == "start" || key == "stop" || key == "backfill" || key == "x-auth-token" || key == "X-AUTH-TOKEN" { continue } if len(val) > 1 { multiFilter = append(multiFilter, key) // if filter applied number is greater than the limit, set limit to that if len(val) > limit { limit = len(val) } } // if there are multiple values, "OR" them, and save as one filter element // allows `host=a&host=b` in query if len(val) > 0 { var tfil []string for i := range val { // todo: does req.Url.Query()[intkey] return an int or is it always a string? tfil = append(tfil, fmt.Sprintf("\"%s\" = '%s'", key, val[i])) } filters = append(filters, fmt.Sprintf("(%s)", strings.Join(tfil, " OR "))) continue } filters = append(filters, fmt.Sprintf("\"%s\" = ''", key)) } // stat is the stat we are selecting stat := req.URL.Query().Get(":stat") var query string if verb == "none" { query = fmt.Sprintf(`SELECT "%s"`, stat) // return multiFilter info if len(multiFilter) > 0 { query = fmt.Sprintf(`%s, "%s"`, query, strings.Join(multiFilter, "\", \"")) } } else { query = fmt.Sprintf(`SELECT %s("%s")`, verb, stat) } // add the FROM to the query query = fmt.Sprintf(`%s FROM "%s"`, query, stat) // grab the latest chunk (poll interval used so we don't have too many results) // allow missing 1 stat update (we have to sacrifice accuracy because influx // acts eventually consistent and doesn't return the data in a predictable, // timely manner) filters = append(filters, fmt.Sprintf("time > now() - %ds", viper.GetInt("poll-interval")*2)) // filter by filters if len(filters) > 0 { query = fmt.Sprintf("%s WHERE %s", query, strings.Join(filters, " AND ")) } // if aggregate function used, group by time and limit to 1 if verb != "none" { limit = 1 query = fmt.Sprintf("%s GROUP BY time(%ds) FILL(none)", query, viper.GetInt("poll-interval")) } // grab the latest available query = fmt.Sprintf("%s ORDER BY time DESC LIMIT %d", query, limit) records, err := influx.Query(query) if err != nil { writeBody(apiError{ErrorString: err.Error()}, res, http.StatusInternalServerError, req) return } if len(records.Results[0].Series) != 1 { writeBody(apiError{ErrorString: "Not Found"}, res, http.StatusNotFound, req) return } // if there are multiple values (if a filter was applies more than 1x) if len(records.Results[0].Series[0].Values) > 1 { // create the master result map (will have key, val) resulto := make([]map[string]interface{}, 0) // loop through the values ([][]interface{}) for _, val := range records.Results[0].Series[0].Values { // create a temporary result for holding values by column name tRes := make(map[string]interface{}) // loop through nested slice for i := range val { // time will be at index 0 unless influx changes things // eg. (select "value"..; returns "time, value") if i == 0 { // make time uniform timestamp, _ := val[i].(json.Number).Int64() tRes["time"] = timestamp * 1000 continue } // our value will be at index 1 if i == 1 { // make value generic tRes["value"] = val[i] continue } // assign value to column name tRes[records.Results[0].Series[0].Columns[i]] = val[i] } // append temp result to master result map resulto = append(resulto, tRes) } writeBody(resulto, res, http.StatusOK, req) return } result := point{} val := records.Results[0].Series[0].Values[0] result.Time, _ = val[0].(json.Number).Int64() result.Time = result.Time * 1000 result.Value, _ = val[1].(json.Number).Float64() writeBody(result, res, http.StatusOK, req) }
// fetches the historic values for a single stat and averages each hour (useful for spotting trends - radial) func dailyStat(res http.ResponseWriter, req *http.Request) { // verb is what to get to the stat (mean, min, max) verb := req.URL.Query().Get("verb") if verb == "" { verb = "mean" // default to average (avg of [x] is x) } // start and stop should be an int and one of u,ms,s,m,h,d,w // start is how far back in time to start (will be subtracted from now()) start := req.URL.Query().Get("start") if start == "" { start = "7d" } // stop is how far back in time to stop (will be subtracted from now()) stop := req.URL.Query().Get("stop") if stop == "" { stop = "0s" } // stat is the stat we are selecting stat := req.URL.Query().Get(":stat") query := fmt.Sprintf(`SELECT %s("%v") FROM one_week.aggregate`, verb, stat) filters := []string{} for key, val := range req.URL.Query() { if key == ":stat" || key == "verb" || key == "start" || key == "stop" || key == "backfill" || key == "x-auth-token" || key == "X-AUTH-TOKEN" { continue } // if there are multiple values, "OR" them, and save as one filter element // allows `host=a&host=b` in query if len(val) > 0 { var tfil []string for i := range val { tfil = append(tfil, fmt.Sprintf("\"%s\" = '%s'", key, val[i])) } filters = append(filters, fmt.Sprintf("(%s)", strings.Join(tfil, " OR "))) continue } filters = append(filters, fmt.Sprintf("\"%s\" = ''", key)) } // set time filter filters = append(filters, fmt.Sprintf("time > now() - %v", start)) filters = append(filters, fmt.Sprintf("time < now() - %v", stop)) // filter by filters if len(filters) > 0 { query = fmt.Sprintf("%s WHERE %s", query, strings.Join(filters, " AND ")) } // grab the hourly averages query = fmt.Sprintf("%s GROUP BY time(15m) FILL(none)", query) records, err := influx.Query(query) if err != nil { writeBody(apiError{ErrorString: err.Error()}, res, http.StatusInternalServerError, req) return } if len(records.Results[0].Series) != 1 { writeBody(apiError{ErrorString: "Not Found"}, res, http.StatusNotFound, req) return } result := make(map[string]float64, 24*4) counter := make(map[string]int, 24*4) for _, values := range records.Results[0].Series[0].Values { valTime, _ := values[0].(json.Number).Int64() valUnix := time.Unix(valTime, 0) hour, minute, _ := valUnix.Clock() minute = int(math.Floor(float64(minute/15.0)) * 15) id := fmt.Sprintf("%02d:%02d", hour, minute) valData, _ := values[1].(json.Number).Float64() // keep a running average instead of a total result[id] = ((result[id] * float64(counter[id])) + valData) / float64(counter[id]+1) counter[id] += 1 } bf := req.URL.Query().Get("backfill") // if backfill set to anything, fill in the missing data with 0 value if bf != "" { daytimes := []string{"00:00", "00:15", "00:30", "00:45", "01:00", "01:15", "01:30", "01:45", "02:00", "02:15", "02:30", "02:45", "03:00", "03:15", "03:30", "03:45", "04:00", "04:15", "04:30", "04:45", "05:00", "05:15", "05:30", "05:45", "06:00", "06:15", "06:30", "06:45", "07:00", "07:15", "07:30", "07:45", "08:00", "08:15", "08:30", "08:45", "09:00", "09:15", "09:30", "09:45", "10:00", "10:15", "10:30", "10:45", "11:00", "11:15", "11:30", "11:45", "12:00", "12:15", "12:30", "12:45", "13:00", "13:15", "13:30", "13:45", "14:00", "14:15", "14:30", "14:45", "15:00", "15:15", "15:30", "15:45", "16:00", "16:15", "16:30", "16:45", "17:00", "17:15", "17:30", "17:45", "18:00", "18:15", "18:30", "18:45", "19:00", "19:15", "19:30", "19:45", "20:00", "20:15", "20:30", "20:45", "21:00", "21:15", "21:30", "21:45", "22:00", "22:15", "22:30", "22:45", "23:00", "23:15", "23:30", "23:45"} // set default backfill value bfFloat := 0.0 // if what was passed in is a valid number, reset backfill to that if s, err := strconv.ParseFloat(bf, 64); err == nil { bfFloat = s } for i := range daytimes { if _, ok := result[daytimes[i]]; !ok { result[daytimes[i]] = bfFloat } } } // todo: will it be better&possible(backfill) to just use a struct slice the whole time? // convert result map to struct slice (make more uniform with hourly response) dailies := mapToStructSlice(result) writeBody(dailies, res, http.StatusOK, req) }
// fetches the historic 1h averages for a single stat func hourlyStat(res http.ResponseWriter, req *http.Request) { // verb is what to get to the stat (mean, min, max) verb := req.URL.Query().Get("verb") if verb == "" { verb = "mean" // default to average (avg of [x] is x) } // start and stop should be an int and one of u,ms,s,m,h,d,w // start is how far back in time to start (will be subtracted from now()) start := req.URL.Query().Get("start") if start == "" { start = "1d" } // stop is how far back in time to stop (will be subtracted from now()) stop := req.URL.Query().Get("stop") if stop == "" { stop = "0s" } // stat is the stat we are selecting stat := req.URL.Query().Get(":stat") query := fmt.Sprintf(`SELECT %s("%v") FROM one_week.aggregate`, verb, stat) filters := []string{} for key, val := range req.URL.Query() { if key == ":stat" || key == "verb" || key == "start" || key == "stop" || key == "backfill" || key == "x-auth-token" || key == "X-AUTH-TOKEN" { continue } // if there are multiple values, "OR" them, and save as one filter element // allows `host=a&host=b` in query if len(val) > 0 { var tfil []string for i := range val { tfil = append(tfil, fmt.Sprintf("\"%s\" = '%s'", key, val[i])) } filters = append(filters, fmt.Sprintf("(%s)", strings.Join(tfil, " OR "))) continue } filters = append(filters, fmt.Sprintf("\"%s\" = ''", key)) } // set time filter filters = append(filters, fmt.Sprintf("time > now() - %v", start)) filters = append(filters, fmt.Sprintf("time < now() - %v", stop)) // filter by filters if len(filters) > 0 { query = fmt.Sprintf("%s WHERE %s", query, strings.Join(filters, " AND ")) } bf := req.URL.Query().Get("backfill") if bf == "" { // grab the hourly averages ('group by' should aggregate multiple filters) query = fmt.Sprintf("%s GROUP BY time(1h) FILL(none)", query) } else { if s, err := strconv.ParseFloat(bf, 64); err != nil { // if bf is not a number, use 0 to backfill query = fmt.Sprintf("%s GROUP BY time(1h) FILL(0)", query) } else { // else use the number value of backfill query = fmt.Sprintf("%s GROUP BY time(1h) FILL(%v)", query, s) } } records, err := influx.Query(query) if err != nil { writeBody(apiError{ErrorString: err.Error()}, res, http.StatusInternalServerError, req) return } if len(records.Results[0].Series) != 1 { writeBody(apiError{ErrorString: "Not Found"}, res, http.StatusNotFound, req) return } result := make([]point, len(records.Results[0].Series[0].Values)) for i, values := range records.Results[0].Series[0].Values { // todo: do i need to multiply time by 1000 (to return nanoseconds?)? result[i].Time, _ = values[0].(json.Number).Int64() result[i].Time = result[i].Time * 1000 result[i].Value, _ = values[1].(json.Number).Float64() } writeBody(result, res, http.StatusOK, req) }