Beispiel #1
0
// GetPlots retrieves time series data from provider based on a query and a time interval.
func (c *KairosdbConnector) GetPlots(query *plot.Query) ([]*plot.Series, error) {
	var (
		jsonResponse map[string][]metricQueryResponse
		results      []*plot.Series
	)

	if len(query.Series) == 0 {
		return nil, fmt.Errorf("kairosdb[%s]: requested series list is empty", c.name)
	}

	jsonQuery, err := kairosdbBuildJSONQuery(query, c.series)
	if err != nil {
		return nil, fmt.Errorf("kairosdb[%s]: unable to build or marshal JSON query: %s", c.name, err)
	}

	client := utils.NewHTTPClient(c.timeout, c.insecureTLS)

	logger.Log(logger.LevelDebug, "connector", "kairosdb[%s]: API Call to %s: %s", c.name,
		strings.TrimSuffix(c.url, "/")+kairosdbURLQueryMetric,
		string(jsonQuery))

	r, err := http.NewRequest("POST", strings.TrimSuffix(c.url, "/")+kairosdbURLQueryMetric, bytes.NewBuffer(jsonQuery))
	if err != nil {
		return nil, fmt.Errorf("kairosdb[%s]: unable to set up HTTP request: %s", c.name, err)
	}

	r.Header.Add("User-Agent", "Facette")
	r.Header.Add("X-Requested-With", "KairosDBConnector")
	r.Header.Set("Content-Type", "application/json")

	rsp, err := client.Do(r)
	if err != nil {
		return nil, fmt.Errorf("kairosdb[%s]: unable to perform HTTP request: %s", c.name, err)
	}
	defer rsp.Body.Close()

	if err = kairosdbCheckBackendResponse(rsp); err != nil {
		return nil, fmt.Errorf("kairosdb[%s]: invalid HTTP backend response: %s", c.name, err)
	}

	data, err := ioutil.ReadAll(rsp.Body)
	if err != nil {
		return nil, fmt.Errorf("kairosdb[%s]: unable to read HTTP response body: %s", c.name, err)
	}

	if err = json.Unmarshal(data, &jsonResponse); err != nil {
		return nil, fmt.Errorf("kairosdb[%s]: unable to unmarshal JSON data: %s", c.name, err)
	}

	if results, err = kairosdbExtractPlots(query, c.series, jsonResponse["queries"]); err != nil {
		return nil, fmt.Errorf("kairosdb[%s]: unable to extract plot values from backend response: %s", c.name, err)
	}

	return results, nil
}
Beispiel #2
0
// GetPlots retrieves time series data from provider based on a query and a time interval.
func (c *GraphiteConnector) GetPlots(query *plot.Query) ([]*plot.Series, error) {
	var (
		plots   []graphitePlot
		results []*plot.Series
	)

	if len(query.Series) == 0 {
		return nil, fmt.Errorf("graphite[%s]: requested series list is empty", c.name)
	}

	// Build query URL
	queryURL, err := graphiteBuildQueryURL(query, c.series)
	if err != nil {
		return nil, fmt.Errorf("graphite[%s]: unable to build query URL: %s", c.name, err)
	}

	// Request data from backend
	client := utils.NewHTTPClient(c.timeout, c.insecureTLS)

	r, err := http.NewRequest("GET", strings.TrimSuffix(c.url, "/")+graphiteURLRender+"?"+queryURL, nil)
	if err != nil {
		return nil, fmt.Errorf("graphite[%s]: unable to set up HTTP request: %s", c.name, err)
	}

	r.Header.Add("User-Agent", "Facette")
	r.Header.Add("X-Requested-With", "GraphiteConnector")

	rsp, err := client.Do(r)
	if err != nil {
		return nil, fmt.Errorf("graphite[%s]: unable to perform HTTP request: %s", c.name, err)
	}
	defer rsp.Body.Close()

	// Parse backend response
	if err = graphiteCheckBackendResponse(rsp); err != nil {
		return nil, fmt.Errorf("graphite[%s]: invalid HTTP backend response: %s", c.name, err)
	}

	data, err := ioutil.ReadAll(rsp.Body)
	if err != nil {
		return nil, fmt.Errorf("graphite[%s]: unable to read HTTP response body: %s", c.name, err)
	}

	if err = json.Unmarshal(data, &plots); err != nil {
		return nil, fmt.Errorf("graphite[%s]: unable to unmarshal JSON data: %s", c.name, err)
	}

	// Extract results from response
	if results, err = graphiteExtractResult(plots); err != nil {
		return nil, fmt.Errorf("graphite[%s]: unable to extract plot values from backend response: %s", c.name, err)
	}

	return results, nil
}
Beispiel #3
0
func kairosdbGetVersion(c *KairosdbConnector) (string, [3]int, error) {
	var (
		array       [3]int
		jsonVersion map[string]string
	)

	client := utils.NewHTTPClient(c.timeout, c.insecureTLS)

	r, err := http.NewRequest("GET", strings.TrimSuffix(c.url, "/")+kairosdbURLVersion, nil)
	if err != nil {
		return "", array, fmt.Errorf("unable to set up HTTP request: %s", err)
	}

	r.Header.Add("User-Agent", "Facette")
	r.Header.Add("X-Requested-With", "KairosDBConnector")

	rsp, err := client.Do(r)
	if err != nil {
		return "", array, fmt.Errorf("unable to perform HTTP request: %s", err)
	}
	defer rsp.Body.Close()

	if err = kairosdbCheckBackendResponse(rsp); err != nil {
		return "", array, fmt.Errorf("invalid HTTP backend response: %s", err)
	}

	data, err := ioutil.ReadAll(rsp.Body)
	if err != nil {
		return "", array, fmt.Errorf("unable to read HTTP response body: %s", err)
	}

	if err := json.Unmarshal(data, &jsonVersion); err != nil {
		return "", array, fmt.Errorf("unable to unmarshal JSON data: %s", err)
	}

	if v, ok := jsonVersion["version"]; ok {
		re, _ := regexp.Compile(`^KairosDB (\d+)\.(\d+)\.(\d+)`) // {"version": "KairosDB 0.9.4-6.20140730155353"}

		submatch := re.FindStringSubmatch(v)
		if submatch == nil || len(submatch) != 4 {
			return "", array, fmt.Errorf("can't match KairosDB version")
		}

		array[0], _ = strconv.Atoi(submatch[1])
		array[1], _ = strconv.Atoi(submatch[2])
		array[2], _ = strconv.Atoi(submatch[3])

		return v, array, nil
	}

	return "", array, fmt.Errorf("can't fetch KairosDB version")
}
Beispiel #4
0
// Refresh triggers a full connector data update.
func (c *FacetteConnector) Refresh(originName string, outputChan chan<- *catalog.Record) error {
	client := utils.NewHTTPClient(c.timeout, c.insecureTLS)

	r, err := http.NewRequest("GET", strings.TrimSuffix(c.upstream, "/")+facetteURLCatalog, nil)
	if err != nil {
		return fmt.Errorf("facette[%s]: unable to set up HTTP request: %s", c.name, err)
	}

	r.Header.Add("User-Agent", "Facette")
	r.Header.Add("X-Requested-With", "FacetteConnector")

	rsp, err := client.Do(r)
	if err != nil {
		return fmt.Errorf("facette[%s]: unable to perform HTTP request: %s", c.name, err)
	}
	defer rsp.Body.Close()

	if err = facetteCheckConnectorResponse(rsp); err != nil {
		return fmt.Errorf("facette[%s]: invalid HTTP backend response: %s", c.name, err)
	}

	data, err := ioutil.ReadAll(rsp.Body)
	if err != nil {
		return fmt.Errorf("facette[%s]: unable to read HTTP response body: %s", c.name, err)
	}

	upstreamCatalog := make(map[string]map[string][]string)
	if err = json.Unmarshal(data, &upstreamCatalog); err != nil {
		return fmt.Errorf("facette[%s]: unable to unmarshal JSON data: %s", c.name, err)
	}

	// Parse the upstream catalog entries and append them to our local catalog
	for upstreamOriginName, upstreamOrigin := range upstreamCatalog {
		for sourceName, metrics := range upstreamOrigin {
			for _, metric := range metrics {
				outputChan <- &catalog.Record{
					Origin:    upstreamOriginName,
					Source:    sourceName,
					Metric:    metric,
					Connector: c,
				}
			}
		}
	}

	return nil
}
Beispiel #5
0
// GetPlots retrieves time series data from origin based on a query and a time interval.
func (c *FacetteConnector) GetPlots(query *plot.Query) ([]*plot.Series, error) {
	var results []*plot.Series

	// Convert plotQuery into plotRequest-like to forward query to upstream Facette API
	plotReq := facettePlotRequest{
		Time:   query.StartTime,
		Range:  utils.DurationToRange(query.EndTime.Sub(query.StartTime)),
		Sample: query.Sample,
		Graph: library.Graph{
			Item: library.Item{
				Name: "facette",
			},
			Groups: []*library.OperGroup{
				&library.OperGroup{
					Name: "group0",
					Series: func(series []plot.QuerySeries) []*library.Series {
						out := make([]*library.Series, len(series))

						for i, s := range series {
							out[i] = &library.Series{
								Name:   fmt.Sprintf("series%d", i),
								Origin: s.Origin,
								Source: s.Source,
								Metric: s.Metric,
							}
						}

						return out
					}(query.Series),
				},
			},
		},
	}

	body, err := json.Marshal(plotReq)
	if err != nil {
		return nil, fmt.Errorf("facette[%s]: unable to marshal plot request: %s", c.name, err)
	}

	client := utils.NewHTTPClient(c.timeout, c.insecureTLS)

	r, err := http.NewRequest("POST", strings.TrimSuffix(c.upstream, "/")+facetteURLPlots, bytes.NewReader(body))
	if err != nil {
		return nil, fmt.Errorf("facette[%s]: unable to set up HTTP request: %s", c.name, err)
	}

	r.Header.Add("Content-Type", "application/json")
	r.Header.Add("User-Agent", "Facette")
	r.Header.Add("X-Requested-With", "FacetteConnector")

	if query.Requestor != "" {
		r.Header.Add("X-Facette-Requestor", query.Requestor)
	} else {
		r.Header.Add("X-Facette-Requestor", c.serverID)
	}

	rsp, err := client.Do(r)
	if err != nil {
		return nil, fmt.Errorf("facette[%s]: unable to perform HTTP request: %s", c.name, err)
	}

	if err := facetteCheckConnectorResponse(rsp); err != nil {
		return nil, fmt.Errorf("facette[%s]: invalid upstream HTTP response: %s", c.name, err)
	}

	data, err := ioutil.ReadAll(rsp.Body)
	if err != nil {
		return nil, fmt.Errorf("facette[%s]: unable to read HTTP response body: %s", c.name, err)
	}
	defer rsp.Body.Close()

	plotRsp := facettePlotResponse{}

	if err := json.Unmarshal(data, &plotRsp); err != nil {
		return nil, fmt.Errorf("facette[%s]: unable to unmarshal upstream response: %s", c.name, err)
	}

	for _, s := range plotRsp.Series {
		results = append(results, &plot.Series{
			Plots:   s.Plots,
			Summary: s.Summary,
		})
	}

	return results, nil
}
Beispiel #6
0
// Refresh triggers a full connector data update.
func (c *KairosdbConnector) Refresh(originName string, outputChan chan<- *catalog.Record) error {
	var (
		jsonMetrics map[string][]string
		jsonQuery   map[string][]map[string][]struct {
			Name string              `json:"name"`
			Tags map[string][]string `json:"tags"`
		}
	)

	client := utils.NewHTTPClient(c.timeout, c.insecureTLS)

	r, err := http.NewRequest("GET", strings.TrimSuffix(c.url, "/")+kairosdbURLMetricNames, nil)
	if err != nil {
		return fmt.Errorf("kairosdb[%s]: unable to set up HTTP request: %s", c.name, err)
	}

	r.Header.Add("User-Agent", "Facette")
	r.Header.Add("X-Requested-With", "KairosDBConnector")

	rsp, err := client.Do(r)
	if err != nil {
		return fmt.Errorf("kairosdb[%s]: unable to perform HTTP request: %s", c.name, err)
	}
	defer rsp.Body.Close()

	if err = kairosdbCheckBackendResponse(rsp); err != nil {
		return fmt.Errorf("kairosdb[%s]: invalid HTTP backend response: %s", c.name, err)
	}

	data, err := ioutil.ReadAll(rsp.Body)
	if err != nil {
		return fmt.Errorf("kairosdb[%s]: unable to read HTTP response body: %s", c.name, err)
	}
	if err = json.Unmarshal(data, &jsonMetrics); err != nil {
		return fmt.Errorf("kairosdb[%s]: unable to unmarshal JSON data: %s", c.name, err)
	}

	metrics := make([]map[string]string, 0)
	for _, m := range jsonMetrics["results"] {
		metrics = append(metrics, map[string]string{"name": m})
	}

	query := map[string]interface{}{"metrics": metrics}

	if c.startAbsolute > 0 {
		query["start_absolute"] = c.startAbsolute
	} else {
		query["start_relative"] = c.startRelative
	}

	if c.endAbsolute > 0 {
		query["end_absolute"] = c.endAbsolute
	}

	if c.endRelative != nil {
		query["end_relative"] = c.endRelative
	}

	jsonData, err := json.Marshal(query)
	if err != nil {
		return fmt.Errorf("kairosdb[%s]: unable to marshal JSON data: %s", c.name, err)
	}

	logger.Log(logger.LevelDebug, "connector", "kairosdb[%s]: API Call to %s: %s", c.name,
		strings.TrimSuffix(c.url, "/")+kairosdbURLMetricTags, string(jsonData))

	r, err = http.NewRequest("POST", strings.TrimSuffix(c.url, "/")+kairosdbURLMetricTags, bytes.NewBuffer(jsonData))
	if err != nil {
		return fmt.Errorf("kairosdb[%s]: unable to set up HTTP request: %s", c.name, err)
	}

	r.Header.Add("User-Agent", "Facette")
	r.Header.Add("X-Requested-With", "KairosDBConnector")
	r.Header.Set("Content-Type", "application/json")

	rsp, err = client.Do(r)
	if err != nil {
		return fmt.Errorf("kairosdb[%s]: unable to perform HTTP request: %s", c.name, err)
	}
	defer rsp.Body.Close()

	if err = kairosdbCheckBackendResponse(rsp); err != nil {
		return fmt.Errorf("kairosdb[%s]: invalid HTTP backend response: %s", c.name, err)
	}

	data, err = ioutil.ReadAll(rsp.Body)
	if err != nil {
		return fmt.Errorf("kairosdb[%s]: unable to read HTTP response body: %s", c.name, err)
	}

	logger.Log(logger.LevelDebug, "connector", "kairosdb[%s]: API Response from %s: %s", c.name,
		strings.TrimSuffix(c.url, "/")+kairosdbURLMetricTags, string(data))

	if err = json.Unmarshal(data, &jsonQuery); err != nil {
		return fmt.Errorf("kairosdb[%s]: unable to unmarshal JSON data: %s", c.name, err)
	}

	for _, q := range jsonQuery["queries"] {
		for _, r := range q["results"] {
			metricName := r.Name

			aggregator := matchAggregatorPattern(c.aggregators, metricName)
			if aggregator == nil {
				aggregator = c.defaultAggregator
			}

			for _, t := range c.sourceTags {
				if _, ok := r.Tags[t]; !ok {
					continue
				}

				for _, sourceName := range r.Tags[t] {
					if _, ok := c.series[sourceName]; !ok {
						c.series[sourceName] = make(map[string]kairosdbSeriesEntry)
					}

					c.series[sourceName][metricName] = kairosdbSeriesEntry{
						tag:        t,
						source:     sourceName,
						metric:     metricName,
						aggregator: aggregator,
					}

					outputChan <- &catalog.Record{
						Origin:    originName,
						Source:    sourceName,
						Metric:    metricName,
						Connector: c,
					}
				}

				if aggregator != nil {
					a, _ := json.Marshal(aggregator)
					logger.Log(logger.LevelInfo, "connector", "kairosdb[%s]: `%s' applied to `%s'", c.name, string(a),
						metricName)
				}

				break
			}
		}
	}

	return nil
}
Beispiel #7
0
// Refresh triggers a full connector data update.
func (c *GraphiteConnector) Refresh(originName string, outputChan chan<- *catalog.Record) error {
	var series []string

	// Request metrics from backend
	client := utils.NewHTTPClient(c.timeout, c.insecureTLS)

	r, err := http.NewRequest("GET", strings.TrimSuffix(c.url, "/")+graphiteURLMetrics, nil)
	if err != nil {
		return fmt.Errorf("graphite[%s]: unable to set up HTTP request: %s", c.name, err)
	}

	r.Header.Add("User-Agent", "Facette")
	r.Header.Add("X-Requested-With", "GraphiteConnector")

	rsp, err := client.Do(r)
	if err != nil {
		return fmt.Errorf("graphite[%s]: unable to perform HTTP request: %s", c.name, err)
	}
	defer rsp.Body.Close()

	// Parse backend response
	if err = graphiteCheckBackendResponse(rsp); err != nil {
		return fmt.Errorf("graphite[%s]: invalid HTTP backend response: %s", c.name, err)
	}

	data, err := ioutil.ReadAll(rsp.Body)
	if err != nil {
		return fmt.Errorf("graphite[%s]: unable to read HTTP response body: %s", c.name, err)
	}

	if err = json.Unmarshal(data, &series); err != nil {
		return fmt.Errorf("graphite[%s]: unable to unmarshal JSON data: %s", c.name, err)
	}

	for _, s := range series {
		var sourceName, metricName string

		// Get pattern matches
		m, err := matchSeriesPattern(c.re, s)
		if err != nil {
			logger.Log(
				logger.LevelInfo,
				"connector",
				"graphite[%s]: file `%s' does not match pattern, ignoring",
				c.name,
				s,
			)
			continue
		}

		sourceName, metricName = m[0], m[1]

		if _, ok := c.series[sourceName]; !ok {
			c.series[sourceName] = make(map[string]string)
		}

		c.series[sourceName][metricName] = s

		outputChan <- &catalog.Record{
			Origin:    originName,
			Source:    sourceName,
			Metric:    metricName,
			Connector: c,
		}
	}

	return nil
}