// GetPlots retrieves time series data from provider based on a query and a time interval.
func (c *KairosdbConnector) GetPlots(query *plot.Query) ([]*plot.Series, error) {
var (
jsonResponse map[string][]metricQueryResponse
results []*plot.Series
)
if len(query.Series) == 0 {
return nil, fmt.Errorf("kairosdb[%s]: requested series list is empty", c.name)
}
jsonQuery, err := kairosdbBuildJSONQuery(query, c.series)
if err != nil {
return nil, fmt.Errorf("kairosdb[%s]: unable to build or marshal JSON query: %s", c.name, err)
}
client := utils.NewHTTPClient(c.timeout, c.insecureTLS)
logger.Log(logger.LevelDebug, "connector", "kairosdb[%s]: API Call to %s: %s", c.name,
strings.TrimSuffix(c.url, "/")+kairosdbURLQueryMetric,
string(jsonQuery))
r, err := http.NewRequest("POST", strings.TrimSuffix(c.url, "/")+kairosdbURLQueryMetric, bytes.NewBuffer(jsonQuery))
if err != nil {
return nil, fmt.Errorf("kairosdb[%s]: unable to set up HTTP request: %s", c.name, err)
}
r.Header.Add("User-Agent", "Facette")
r.Header.Add("X-Requested-With", "KairosDBConnector")
r.Header.Set("Content-Type", "application/json")
rsp, err := client.Do(r)
if err != nil {
return nil, fmt.Errorf("kairosdb[%s]: unable to perform HTTP request: %s", c.name, err)
}
defer rsp.Body.Close()
if err = kairosdbCheckBackendResponse(rsp); err != nil {
return nil, fmt.Errorf("kairosdb[%s]: invalid HTTP backend response: %s", c.name, err)
}
data, err := ioutil.ReadAll(rsp.Body)
if err != nil {
return nil, fmt.Errorf("kairosdb[%s]: unable to read HTTP response body: %s", c.name, err)
}
if err = json.Unmarshal(data, &jsonResponse); err != nil {
return nil, fmt.Errorf("kairosdb[%s]: unable to unmarshal JSON data: %s", c.name, err)
}
if results, err = kairosdbExtractPlots(query, c.series, jsonResponse["queries"]); err != nil {
return nil, fmt.Errorf("kairosdb[%s]: unable to extract plot values from backend response: %s", c.name, err)
}
return results, nil
}
// GetPlots retrieves time series data from provider based on a query and a time interval.
func (c *GraphiteConnector) GetPlots(query *plot.Query) ([]*plot.Series, error) {
var (
plots []graphitePlot
results []*plot.Series
)
if len(query.Series) == 0 {
return nil, fmt.Errorf("graphite[%s]: requested series list is empty", c.name)
}
// Build query URL
queryURL, err := graphiteBuildQueryURL(query, c.series)
if err != nil {
return nil, fmt.Errorf("graphite[%s]: unable to build query URL: %s", c.name, err)
}
// Request data from backend
client := utils.NewHTTPClient(c.timeout, c.insecureTLS)
r, err := http.NewRequest("GET", strings.TrimSuffix(c.url, "/")+graphiteURLRender+"?"+queryURL, nil)
if err != nil {
return nil, fmt.Errorf("graphite[%s]: unable to set up HTTP request: %s", c.name, err)
}
r.Header.Add("User-Agent", "Facette")
r.Header.Add("X-Requested-With", "GraphiteConnector")
rsp, err := client.Do(r)
if err != nil {
return nil, fmt.Errorf("graphite[%s]: unable to perform HTTP request: %s", c.name, err)
}
defer rsp.Body.Close()
// Parse backend response
if err = graphiteCheckBackendResponse(rsp); err != nil {
return nil, fmt.Errorf("graphite[%s]: invalid HTTP backend response: %s", c.name, err)
}
data, err := ioutil.ReadAll(rsp.Body)
if err != nil {
return nil, fmt.Errorf("graphite[%s]: unable to read HTTP response body: %s", c.name, err)
}
if err = json.Unmarshal(data, &plots); err != nil {
return nil, fmt.Errorf("graphite[%s]: unable to unmarshal JSON data: %s", c.name, err)
}
// Extract results from response
if results, err = graphiteExtractResult(plots); err != nil {
return nil, fmt.Errorf("graphite[%s]: unable to extract plot values from backend response: %s", c.name, err)
}
return results, nil
}
func kairosdbGetVersion(c *KairosdbConnector) (string, [3]int, error) {
var (
array [3]int
jsonVersion map[string]string
)
client := utils.NewHTTPClient(c.timeout, c.insecureTLS)
r, err := http.NewRequest("GET", strings.TrimSuffix(c.url, "/")+kairosdbURLVersion, nil)
if err != nil {
return "", array, fmt.Errorf("unable to set up HTTP request: %s", err)
}
r.Header.Add("User-Agent", "Facette")
r.Header.Add("X-Requested-With", "KairosDBConnector")
rsp, err := client.Do(r)
if err != nil {
return "", array, fmt.Errorf("unable to perform HTTP request: %s", err)
}
defer rsp.Body.Close()
if err = kairosdbCheckBackendResponse(rsp); err != nil {
return "", array, fmt.Errorf("invalid HTTP backend response: %s", err)
}
data, err := ioutil.ReadAll(rsp.Body)
if err != nil {
return "", array, fmt.Errorf("unable to read HTTP response body: %s", err)
}
if err := json.Unmarshal(data, &jsonVersion); err != nil {
return "", array, fmt.Errorf("unable to unmarshal JSON data: %s", err)
}
if v, ok := jsonVersion["version"]; ok {
re, _ := regexp.Compile(`^KairosDB (\d+)\.(\d+)\.(\d+)`) // {"version": "KairosDB 0.9.4-6.20140730155353"}
submatch := re.FindStringSubmatch(v)
if submatch == nil || len(submatch) != 4 {
return "", array, fmt.Errorf("can't match KairosDB version")
}
array[0], _ = strconv.Atoi(submatch[1])
array[1], _ = strconv.Atoi(submatch[2])
array[2], _ = strconv.Atoi(submatch[3])
return v, array, nil
}
return "", array, fmt.Errorf("can't fetch KairosDB version")
}
// Refresh triggers a full connector data update.
func (c *FacetteConnector) Refresh(originName string, outputChan chan<- *catalog.Record) error {
client := utils.NewHTTPClient(c.timeout, c.insecureTLS)
r, err := http.NewRequest("GET", strings.TrimSuffix(c.upstream, "/")+facetteURLCatalog, nil)
if err != nil {
return fmt.Errorf("facette[%s]: unable to set up HTTP request: %s", c.name, err)
}
r.Header.Add("User-Agent", "Facette")
r.Header.Add("X-Requested-With", "FacetteConnector")
rsp, err := client.Do(r)
if err != nil {
return fmt.Errorf("facette[%s]: unable to perform HTTP request: %s", c.name, err)
}
defer rsp.Body.Close()
if err = facetteCheckConnectorResponse(rsp); err != nil {
return fmt.Errorf("facette[%s]: invalid HTTP backend response: %s", c.name, err)
}
data, err := ioutil.ReadAll(rsp.Body)
if err != nil {
return fmt.Errorf("facette[%s]: unable to read HTTP response body: %s", c.name, err)
}
upstreamCatalog := make(map[string]map[string][]string)
if err = json.Unmarshal(data, &upstreamCatalog); err != nil {
return fmt.Errorf("facette[%s]: unable to unmarshal JSON data: %s", c.name, err)
}
// Parse the upstream catalog entries and append them to our local catalog
for upstreamOriginName, upstreamOrigin := range upstreamCatalog {
for sourceName, metrics := range upstreamOrigin {
for _, metric := range metrics {
outputChan <- &catalog.Record{
Origin: upstreamOriginName,
Source: sourceName,
Metric: metric,
Connector: c,
}
}
}
}
return nil
}
// GetPlots retrieves time series data from origin based on a query and a time interval.
func (c *FacetteConnector) GetPlots(query *plot.Query) ([]*plot.Series, error) {
var results []*plot.Series
// Convert plotQuery into plotRequest-like to forward query to upstream Facette API
plotReq := facettePlotRequest{
Time: query.StartTime,
Range: utils.DurationToRange(query.EndTime.Sub(query.StartTime)),
Sample: query.Sample,
Graph: library.Graph{
Item: library.Item{
Name: "facette",
},
Groups: []*library.OperGroup{
&library.OperGroup{
Name: "group0",
Series: func(series []plot.QuerySeries) []*library.Series {
out := make([]*library.Series, len(series))
for i, s := range series {
out[i] = &library.Series{
Name: fmt.Sprintf("series%d", i),
Origin: s.Origin,
Source: s.Source,
Metric: s.Metric,
}
}
return out
}(query.Series),
},
},
},
}
body, err := json.Marshal(plotReq)
if err != nil {
return nil, fmt.Errorf("facette[%s]: unable to marshal plot request: %s", c.name, err)
}
client := utils.NewHTTPClient(c.timeout, c.insecureTLS)
r, err := http.NewRequest("POST", strings.TrimSuffix(c.upstream, "/")+facetteURLPlots, bytes.NewReader(body))
if err != nil {
return nil, fmt.Errorf("facette[%s]: unable to set up HTTP request: %s", c.name, err)
}
r.Header.Add("Content-Type", "application/json")
r.Header.Add("User-Agent", "Facette")
r.Header.Add("X-Requested-With", "FacetteConnector")
if query.Requestor != "" {
r.Header.Add("X-Facette-Requestor", query.Requestor)
} else {
r.Header.Add("X-Facette-Requestor", c.serverID)
}
rsp, err := client.Do(r)
if err != nil {
return nil, fmt.Errorf("facette[%s]: unable to perform HTTP request: %s", c.name, err)
}
if err := facetteCheckConnectorResponse(rsp); err != nil {
return nil, fmt.Errorf("facette[%s]: invalid upstream HTTP response: %s", c.name, err)
}
data, err := ioutil.ReadAll(rsp.Body)
if err != nil {
return nil, fmt.Errorf("facette[%s]: unable to read HTTP response body: %s", c.name, err)
}
defer rsp.Body.Close()
plotRsp := facettePlotResponse{}
if err := json.Unmarshal(data, &plotRsp); err != nil {
return nil, fmt.Errorf("facette[%s]: unable to unmarshal upstream response: %s", c.name, err)
}
for _, s := range plotRsp.Series {
results = append(results, &plot.Series{
Plots: s.Plots,
Summary: s.Summary,
})
}
return results, nil
}
// Refresh triggers a full connector data update.
func (c *KairosdbConnector) Refresh(originName string, outputChan chan<- *catalog.Record) error {
var (
jsonMetrics map[string][]string
jsonQuery map[string][]map[string][]struct {
Name string `json:"name"`
Tags map[string][]string `json:"tags"`
}
)
client := utils.NewHTTPClient(c.timeout, c.insecureTLS)
r, err := http.NewRequest("GET", strings.TrimSuffix(c.url, "/")+kairosdbURLMetricNames, nil)
if err != nil {
return fmt.Errorf("kairosdb[%s]: unable to set up HTTP request: %s", c.name, err)
}
r.Header.Add("User-Agent", "Facette")
r.Header.Add("X-Requested-With", "KairosDBConnector")
rsp, err := client.Do(r)
if err != nil {
return fmt.Errorf("kairosdb[%s]: unable to perform HTTP request: %s", c.name, err)
}
defer rsp.Body.Close()
if err = kairosdbCheckBackendResponse(rsp); err != nil {
return fmt.Errorf("kairosdb[%s]: invalid HTTP backend response: %s", c.name, err)
}
data, err := ioutil.ReadAll(rsp.Body)
if err != nil {
return fmt.Errorf("kairosdb[%s]: unable to read HTTP response body: %s", c.name, err)
}
if err = json.Unmarshal(data, &jsonMetrics); err != nil {
return fmt.Errorf("kairosdb[%s]: unable to unmarshal JSON data: %s", c.name, err)
}
metrics := make([]map[string]string, 0)
for _, m := range jsonMetrics["results"] {
metrics = append(metrics, map[string]string{"name": m})
}
query := map[string]interface{}{"metrics": metrics}
if c.startAbsolute > 0 {
query["start_absolute"] = c.startAbsolute
} else {
query["start_relative"] = c.startRelative
}
if c.endAbsolute > 0 {
query["end_absolute"] = c.endAbsolute
}
if c.endRelative != nil {
query["end_relative"] = c.endRelative
}
jsonData, err := json.Marshal(query)
if err != nil {
return fmt.Errorf("kairosdb[%s]: unable to marshal JSON data: %s", c.name, err)
}
logger.Log(logger.LevelDebug, "connector", "kairosdb[%s]: API Call to %s: %s", c.name,
strings.TrimSuffix(c.url, "/")+kairosdbURLMetricTags, string(jsonData))
r, err = http.NewRequest("POST", strings.TrimSuffix(c.url, "/")+kairosdbURLMetricTags, bytes.NewBuffer(jsonData))
if err != nil {
return fmt.Errorf("kairosdb[%s]: unable to set up HTTP request: %s", c.name, err)
}
r.Header.Add("User-Agent", "Facette")
r.Header.Add("X-Requested-With", "KairosDBConnector")
r.Header.Set("Content-Type", "application/json")
rsp, err = client.Do(r)
if err != nil {
return fmt.Errorf("kairosdb[%s]: unable to perform HTTP request: %s", c.name, err)
}
defer rsp.Body.Close()
if err = kairosdbCheckBackendResponse(rsp); err != nil {
return fmt.Errorf("kairosdb[%s]: invalid HTTP backend response: %s", c.name, err)
}
data, err = ioutil.ReadAll(rsp.Body)
if err != nil {
return fmt.Errorf("kairosdb[%s]: unable to read HTTP response body: %s", c.name, err)
}
logger.Log(logger.LevelDebug, "connector", "kairosdb[%s]: API Response from %s: %s", c.name,
strings.TrimSuffix(c.url, "/")+kairosdbURLMetricTags, string(data))
if err = json.Unmarshal(data, &jsonQuery); err != nil {
return fmt.Errorf("kairosdb[%s]: unable to unmarshal JSON data: %s", c.name, err)
}
for _, q := range jsonQuery["queries"] {
for _, r := range q["results"] {
metricName := r.Name
aggregator := matchAggregatorPattern(c.aggregators, metricName)
if aggregator == nil {
aggregator = c.defaultAggregator
}
for _, t := range c.sourceTags {
if _, ok := r.Tags[t]; !ok {
continue
}
for _, sourceName := range r.Tags[t] {
if _, ok := c.series[sourceName]; !ok {
c.series[sourceName] = make(map[string]kairosdbSeriesEntry)
}
c.series[sourceName][metricName] = kairosdbSeriesEntry{
tag: t,
source: sourceName,
metric: metricName,
aggregator: aggregator,
}
outputChan <- &catalog.Record{
Origin: originName,
Source: sourceName,
Metric: metricName,
Connector: c,
}
}
if aggregator != nil {
a, _ := json.Marshal(aggregator)
logger.Log(logger.LevelInfo, "connector", "kairosdb[%s]: `%s' applied to `%s'", c.name, string(a),
metricName)
}
break
}
}
}
return nil
}
// Refresh triggers a full connector data update.
func (c *GraphiteConnector) Refresh(originName string, outputChan chan<- *catalog.Record) error {
var series []string
// Request metrics from backend
client := utils.NewHTTPClient(c.timeout, c.insecureTLS)
r, err := http.NewRequest("GET", strings.TrimSuffix(c.url, "/")+graphiteURLMetrics, nil)
if err != nil {
return fmt.Errorf("graphite[%s]: unable to set up HTTP request: %s", c.name, err)
}
r.Header.Add("User-Agent", "Facette")
r.Header.Add("X-Requested-With", "GraphiteConnector")
rsp, err := client.Do(r)
if err != nil {
return fmt.Errorf("graphite[%s]: unable to perform HTTP request: %s", c.name, err)
}
defer rsp.Body.Close()
// Parse backend response
if err = graphiteCheckBackendResponse(rsp); err != nil {
return fmt.Errorf("graphite[%s]: invalid HTTP backend response: %s", c.name, err)
}
data, err := ioutil.ReadAll(rsp.Body)
if err != nil {
return fmt.Errorf("graphite[%s]: unable to read HTTP response body: %s", c.name, err)
}
if err = json.Unmarshal(data, &series); err != nil {
return fmt.Errorf("graphite[%s]: unable to unmarshal JSON data: %s", c.name, err)
}
for _, s := range series {
var sourceName, metricName string
// Get pattern matches
m, err := matchSeriesPattern(c.re, s)
if err != nil {
logger.Log(
logger.LevelInfo,
"connector",
"graphite[%s]: file `%s' does not match pattern, ignoring",
c.name,
s,
)
continue
}
sourceName, metricName = m[0], m[1]
if _, ok := c.series[sourceName]; !ok {
c.series[sourceName] = make(map[string]string)
}
c.series[sourceName][metricName] = s
outputChan <- &catalog.Record{
Origin: originName,
Source: sourceName,
Metric: metricName,
Connector: c,
}
}
return nil
}
