func MarshalPickle(results []*MetricData) []byte {
var p []map[string]interface{}
for _, r := range results {
values := make([]interface{}, len(r.Values))
for i, v := range r.Values {
if r.IsAbsent[i] {
values[i] = pickle.None{}
} else {
values[i] = v
}
}
p = append(p, map[string]interface{}{
"name": r.GetName(),
"start": r.GetStartTime(),
"end": r.GetStopTime(),
"step": r.GetStepTime(),
"values": values,
})
}
var buf bytes.Buffer
penc := pickle.NewEncoder(&buf)
penc.Encode(p)
return buf.Bytes()
}
func findHandler(w http.ResponseWriter, req *http.Request) {
if Debug > 0 {
logger.Logln("request: ", req.URL.RequestURI())
}
Metrics.Requests.Add(1)
requrl := req.URL
if Config.UsePB {
rewrite, _ := url.ParseRequestURI(req.URL.RequestURI())
v := rewrite.Query()
v.Set("format", "protobuf")
rewrite.RawQuery = v.Encode()
requrl = rewrite
}
responses := multiGet(Config.Backends, requrl.RequestURI())
if responses == nil || len(responses) == 0 {
logger.Logln("find: error querying backends for: ", requrl.RequestURI())
http.Error(w, "find: error querying backends", http.StatusInternalServerError)
return
}
var metrics []map[interface{}]interface{}
var paths map[string][]string
var err error
if Config.UsePB {
metrics, paths, err = findHandlerPB(w, req, responses)
} else {
metrics, paths, err = findHandlerPickle(w, req, responses)
}
if err != nil {
// assumed error has already been handled, nothing else to do
return
}
// update our cache of which servers have which metrics
Config.mu.Lock()
for k, v := range paths {
Config.metricPaths[k] = v
}
Config.mu.Unlock()
w.Header().Set("Content-Type", "application/pickle")
pEnc := pickle.NewEncoder(w)
pEnc.Encode(metrics)
}
func returnRender(w http.ResponseWriter, metric cspb.FetchResponse, pvalues []interface{}) {
// create a pickle response
presponse := map[string]interface{}{
"start": metric.StartTime,
"step": metric.StepTime,
"end": metric.StopTime,
"name": metric.Name,
"values": pvalues,
}
w.Header().Set("Content-Type", "application/pickle")
e := pickle.NewEncoder(w)
e.Encode([]interface{}{presponse})
}
func returnRender(w http.ResponseWriter, format string, metrics pb.MultiFetchResponse) {
switch format {
case "protobuf":
w.Header().Set("Content-Type", "application/protobuf")
b, _ := proto.Marshal(&metrics)
w.Write(b)
case "json":
presponse := createRenderResponse(metrics, nil)
w.Header().Set("Content-Type", "application/json")
e := json.NewEncoder(w)
e.Encode(presponse)
case "", "pickle":
presponse := createRenderResponse(metrics, pickle.None{})
w.Header().Set("Content-Type", "application/pickle")
e := pickle.NewEncoder(w)
e.Encode(presponse)
}
}
func findHandler(w http.ResponseWriter, req *http.Request) {
logger.Debugln("request: ", req.URL.RequestURI())
Metrics.FindRequests.Add(1)
rewrite, _ := url.ParseRequestURI(req.URL.RequestURI())
v := rewrite.Query()
format := req.FormValue("format")
v.Set("format", "protobuf")
rewrite.RawQuery = v.Encode()
query := req.FormValue("query")
var tld string
if i := strings.IndexByte(query, '.'); i > 0 {
tld = query[:i]
}
// lookup tld in our map of where they live to reduce the set of
// servers we bug with our find
Config.mu.RLock()
var backends []string
var ok bool
if backends, ok = Config.metricPaths[tld]; !ok || backends == nil || len(backends) == 0 {
backends = Config.Backends
}
Config.mu.RUnlock()
responses := multiGet(backends, rewrite.RequestURI())
if responses == nil || len(responses) == 0 {
logger.Logln("find: error querying backends for: ", rewrite.RequestURI())
http.Error(w, "find: error querying backends", http.StatusInternalServerError)
return
}
metrics, paths := findHandlerPB(w, req, responses)
// update our cache of which servers have which metrics
Config.mu.Lock()
for k, v := range paths {
Config.metricPaths[k] = v
}
Config.mu.Unlock()
switch format {
case "protobuf":
w.Header().Set("Content-Type", "application/protobuf")
var result pb.GlobResponse
query := req.FormValue("query")
result.Name = &query
result.Matches = metrics
b, _ := proto.Marshal(&result)
w.Write(b)
case "json":
w.Header().Set("Content-Type", "application/json")
jEnc := json.NewEncoder(w)
jEnc.Encode(metrics)
case "", "pickle":
w.Header().Set("Content-Type", "application/pickle")
var result []map[string]interface{}
for _, metric := range metrics {
mm := map[string]interface{}{
"metric_path": *metric.Path,
"isLeaf": *metric.IsLeaf,
}
result = append(result, mm)
}
pEnc := pickle.NewEncoder(w)
pEnc.Encode(result)
}
}
func fetchHandler(wr http.ResponseWriter, req *http.Request) {
// URL: /render/?target=the.metric.name&format=pickle&from=1396008021&until=1396022421
Metrics.RenderRequests.Add(1)
req.ParseForm()
metric := req.FormValue("target")
format := req.FormValue("format")
from := req.FormValue("from")
until := req.FormValue("until")
t0 := time.Now()
// Make sure we log which metric caused a panic()
defer func() {
if r := recover(); r != nil {
var buf [1024]byte
runtime.Stack(buf[:], false)
logger.Logf("panic handling request: %s\n%s\n", req.RequestURI, string(buf[:]))
}
}()
if format != "json" && format != "pickle" && format != "protobuf" {
Metrics.RenderErrors.Add(1)
logger.Logf("dropping invalid uri (format=%s): %s",
format, req.URL.RequestURI())
http.Error(wr, "Bad request (unsupported format)",
http.StatusBadRequest)
return
}
files, leafs := expandGlobs(metric)
var badTime bool
i, err := strconv.Atoi(from)
if err != nil {
logger.Debugf("fromTime (%s) invalid: %s (in %s)", from, err, req.URL.RequestURI())
badTime = true
}
fromTime := int(i)
i, err = strconv.Atoi(until)
if err != nil {
logger.Debugf("untilTime (%s) invalid: %s (in %s)", from, err, req.URL.RequestURI())
badTime = true
}
untilTime := int(i)
if badTime {
Metrics.RenderErrors.Add(1)
http.Error(wr, "Bad request (invalid from/until time)", http.StatusBadRequest)
return
}
var multi pb.MultiFetchResponse
for i, metric := range files {
if !leafs[i] {
log.Printf("skipping directory = %q\n", metric)
// can't fetch a directory
continue
}
path := config.WhisperData + "/" + strings.Replace(metric, ".", "/", -1) + ".wsp"
w, err := whisper.Open(path)
if err != nil {
// the FE/carbonzipper often requests metrics we don't have
// We shouldn't really see this any more -- expandGlobs() should filter them out
Metrics.NotFound.Add(1)
log.Printf("error opening %q: %v\n", path, err)
continue
}
points, err := w.Fetch(fromTime, untilTime)
w.Close()
if err != nil {
Metrics.RenderErrors.Add(1)
logger.Logf("failed to fetch points from %s: %s", path, err)
continue
}
if points == nil {
Metrics.NotFound.Add(1)
logger.Debugf("Metric time range not found: metric=%s from=%d to=%d ", metric, fromTime, untilTime)
continue
}
values := points.Values()
fromTime := int32(points.FromTime())
untilTime := int32(points.UntilTime())
step := int32(points.Step())
response := pb.FetchResponse{
Name: proto.String(metric),
StartTime: &fromTime,
StopTime: &untilTime,
StepTime: &step,
Values: make([]float64, len(values)),
IsAbsent: make([]bool, len(values)),
}
for i, p := range values {
if math.IsNaN(p) {
response.Values[i] = 0
response.IsAbsent[i] = true
} else {
response.Values[i] = p
response.IsAbsent[i] = false
}
}
multi.Metrics = append(multi.Metrics, &response)
}
var b []byte
switch format {
case "json":
wr.Header().Set("Content-Type", "application/json")
b, err = json.Marshal(multi)
case "protobuf":
wr.Header().Set("Content-Type", "application/protobuf")
b, err = proto.Marshal(&multi)
case "pickle":
// transform protobuf data into what pickle expects
//[{'start': 1396271100, 'step': 60, 'name': 'metric',
//'values': [9.0, 19.0, None], 'end': 1396273140}
var response []map[string]interface{}
for _, metric := range multi.GetMetrics() {
var m map[string]interface{}
m = make(map[string]interface{})
m["start"] = metric.StartTime
m["step"] = metric.StepTime
m["end"] = metric.StopTime
m["name"] = metric.Name
mv := make([]interface{}, len(metric.Values))
for i, p := range metric.Values {
if metric.IsAbsent[i] {
mv[i] = nil
} else {
mv[i] = p
}
}
m["values"] = mv
response = append(response, m)
}
wr.Header().Set("Content-Type", "application/pickle")
var buf bytes.Buffer
pEnc := pickle.NewEncoder(&buf)
err = pEnc.Encode(response)
b = buf.Bytes()
}
if err != nil {
Metrics.RenderErrors.Add(1)
logger.Logf("failed to create %s data for %s: %s", format, "<metric>", err)
return
}
wr.Write(b)
logger.Debugf("fetch: served %q from %d to %d in %v", metric, fromTime, untilTime, time.Since(t0))
}
func findHandler(wr http.ResponseWriter, req *http.Request) {
// URL: /metrics/find/?local=1&format=pickle&query=the.metric.path.with.glob
t0 := time.Now()
Metrics.FindRequests.Add(1)
req.ParseForm()
format := req.FormValue("format")
query := req.FormValue("query")
if format != "json" && format != "pickle" && format != "protobuf" {
Metrics.FindErrors.Add(1)
logger.Logf("dropping invalid uri (format=%s): %s",
format, req.URL.RequestURI())
http.Error(wr, "Bad request (unsupported format)",
http.StatusBadRequest)
return
}
if query == "" {
Metrics.FindErrors.Add(1)
logger.Logf("dropping invalid request (query=): %s", req.URL.RequestURI())
http.Error(wr, "Bad request (no query)", http.StatusBadRequest)
return
}
files, leafs := expandGlobs(query)
if format == "json" || format == "protobuf" {
name := req.FormValue("query")
response := pb.GlobResponse{
Name: &name,
Matches: make([]*pb.GlobMatch, 0),
}
for i, p := range files {
response.Matches = append(response.Matches, &pb.GlobMatch{Path: proto.String(p), IsLeaf: proto.Bool(leafs[i])})
}
var b []byte
var err error
switch format {
case "json":
b, err = json.Marshal(response)
case "protobuf":
b, err = proto.Marshal(&response)
}
if err != nil {
Metrics.FindErrors.Add(1)
logger.Logf("failed to create %s data for glob %s: %s",
format, *response.Name, err)
return
}
wr.Write(b)
} else if format == "pickle" {
// [{'metric_path': 'metric', 'intervals': [(x,y)], 'isLeaf': True},]
var metrics []map[string]interface{}
var m map[string]interface{}
for i, p := range files {
m = make(map[string]interface{})
m["metric_path"] = p
// m["intervals"] = dunno how to do a tuple here
m["isLeaf"] = leafs[i]
metrics = append(metrics, m)
}
wr.Header().Set("Content-Type", "application/pickle")
pEnc := pickle.NewEncoder(wr)
pEnc.Encode(metrics)
}
if len(files) == 0 {
// to get an idea how often we search for nothing
Metrics.FindZero.Add(1)
}
logger.Debugf("find: %d hits for %s in %v", len(files), req.FormValue("query"), time.Since(t0))
return
}
func findHandler(wr http.ResponseWriter, req *http.Request) {
// URL: /metrics/find/?local=1&format=pickle&query=the.metric.path.with.glob
Metrics.FindRequests.Add(1)
req.ParseForm()
format := req.FormValue("format")
query := req.FormValue("query")
if format != "json" && format != "pickle" && format != "protobuf" {
Metrics.FindErrors.Add(1)
log.Warnf("dropping invalid uri (format=%s): %s",
format, req.URL.RequestURI())
http.Error(wr, "Bad request (unsupported format)",
http.StatusBadRequest)
return
}
if query == "" {
Metrics.FindErrors.Add(1)
log.Warnf("dropping invalid request (query=): %s", req.URL.RequestURI())
http.Error(wr, "Bad request (no query)", http.StatusBadRequest)
return
}
/* things to glob:
* - carbon.relays -> carbon.relays
* - carbon.re -> carbon.relays, carbon.rewhatever
* - carbon.[rz] -> carbon.relays, carbon.zipper
* - carbon.{re,zi} -> carbon.relays, carbon.zipper
* - match is either dir or .wsp file
* unfortunately, filepath.Glob doesn't handle the curly brace
* expansion for us */
query = strings.Replace(query, ".", "/", -1)
var globs []string
if !strings.HasSuffix(query, "*") {
globs = append(globs, query+".wsp")
}
globs = append(globs, query)
for {
bracematch := false
var newglobs []string
for _, glob := range globs {
lbrace := strings.Index(glob, "{")
rbrace := -1
if lbrace > -1 {
rbrace = strings.Index(glob[lbrace:], "}")
if rbrace > -1 {
rbrace += lbrace
}
}
if lbrace > -1 && rbrace > -1 {
bracematch = true
expansion := glob[lbrace+1 : rbrace]
parts := strings.Split(expansion, ",")
for _, sub := range parts {
if len(newglobs) > config.MaxGlobs {
break
}
newglobs = append(newglobs, glob[:lbrace]+sub+glob[rbrace+1:])
}
} else {
if len(newglobs) > config.MaxGlobs {
break
}
newglobs = append(newglobs, glob)
}
}
globs = newglobs
if !bracematch {
break
}
}
var files []string
for _, glob := range globs {
nfiles, err := filepath.Glob(config.WhisperData + "/" + glob)
if err == nil {
files = append(files, nfiles...)
}
}
leafs := make([]bool, len(files))
for i, p := range files {
p = p[len(config.WhisperData+"/"):]
if strings.HasSuffix(p, ".wsp") {
p = p[:len(p)-4]
leafs[i] = true
} else {
leafs[i] = false
}
files[i] = strings.Replace(p, "/", ".", -1)
}
if format == "json" || format == "protobuf" {
name := req.FormValue("query")
response := pb.GlobResponse{
Name: &name,
Matches: make([]*pb.GlobMatch, 0),
}
for i, p := range files {
response.Matches = append(response.Matches, &pb.GlobMatch{Path: proto.String(p), IsLeaf: proto.Bool(leafs[i])})
}
var b []byte
var err error
switch format {
case "json":
b, err = json.Marshal(response)
case "protobuf":
b, err = proto.Marshal(&response)
}
if err != nil {
Metrics.FindErrors.Add(1)
log.Errorf("failed to create %s data for glob %s: %s",
format, *response.Name, err)
return
}
wr.Write(b)
} else if format == "pickle" {
// [{'metric_path': 'metric', 'intervals': [(x,y)], 'isLeaf': True},]
var metrics []map[string]interface{}
var m map[string]interface{}
for i, p := range files {
m = make(map[string]interface{})
m["metric_path"] = p
// m["intervals"] = dunno how to do a tuple here
m["isLeaf"] = leafs[i]
metrics = append(metrics, m)
}
wr.Header().Set("Content-Type", "application/pickle")
pEnc := pickle.NewEncoder(wr)
pEnc.Encode(metrics)
}
log.Infof("find: %d hits for %s", len(files), req.FormValue("query"))
return
}
func fetchHandler(wr http.ResponseWriter, req *http.Request) {
// URL: /render/?target=the.metric.name&format=pickle&from=1396008021&until=1396022421
Metrics.RenderRequests.Add(1)
req.ParseForm()
metric := req.FormValue("target")
format := req.FormValue("format")
from := req.FormValue("from")
until := req.FormValue("until")
if format != "json" && format != "pickle" && format != "protobuf" {
Metrics.RenderErrors.Add(1)
log.Warnf("dropping invalid uri (format=%s): %s",
format, req.URL.RequestURI())
http.Error(wr, "Bad request (unsupported format)",
http.StatusBadRequest)
return
}
path := config.WhisperData + "/" + strings.Replace(metric, ".", "/", -1) + ".wsp"
w, err := whisper.Open(path)
if err != nil {
// the FE/carbonzipper often requests metrics we don't have
Metrics.NotFound.Add(1)
log.Debugf("failed to %s", err)
http.Error(wr, "Metric not found", http.StatusNotFound)
return
}
i, err := strconv.Atoi(from)
if err != nil {
log.Debugf("fromTime (%s) invalid: %s (in %s)",
from, err, req.URL.RequestURI())
if w != nil {
w.Close()
}
w = nil
}
fromTime := int(i)
i, err = strconv.Atoi(until)
if err != nil {
log.Debugf("untilTime (%s) invalid: %s (in %s)",
from, err, req.URL.RequestURI())
if w != nil {
w.Close()
}
w = nil
}
untilTime := int(i)
if w != nil {
defer w.Close()
} else {
Metrics.RenderErrors.Add(1)
http.Error(wr, "Bad request (invalid from/until time)",
http.StatusBadRequest)
return
}
points, err := w.Fetch(fromTime, untilTime)
if err != nil {
Metrics.RenderErrors.Add(1)
log.Errorf("failed to fetch points from %s: %s", path, err)
http.Error(wr, "Fetching data points failed",
http.StatusInternalServerError)
return
}
if points == nil {
Metrics.NotFound.Add(1)
log.Debugf("Metric time range not found: metric=%s from=%d to=%d ", metric, fromTime, untilTime)
http.Error(wr, "Metric time range not found", http.StatusNotFound)
return
}
values := points.Values()
if format == "json" || format == "protobuf" {
fromTime := int32(points.FromTime())
untilTime := int32(points.UntilTime())
step := int32(points.Step())
response := pb.FetchResponse{
Name: &metric,
StartTime: &fromTime,
StopTime: &untilTime,
StepTime: &step,
Values: make([]float64, len(values)),
IsAbsent: make([]bool, len(values)),
}
for i, p := range values {
if math.IsNaN(p) {
response.Values[i] = 0
response.IsAbsent[i] = true
} else {
response.Values[i] = p
response.IsAbsent[i] = false
}
}
var b []byte
var err error
switch format {
case "json":
b, err = json.Marshal(response)
case "protobuf":
b, err = proto.Marshal(&response)
}
if err != nil {
Metrics.RenderErrors.Add(1)
log.Errorf("failed to create %s data for %s: %s", format, path, err)
return
}
wr.Write(b)
} else if format == "pickle" {
//[{'start': 1396271100, 'step': 60, 'name': 'metric',
//'values': [9.0, 19.0, None], 'end': 1396273140}
var metrics []map[string]interface{}
var m map[string]interface{}
m = make(map[string]interface{})
m["start"] = points.FromTime()
m["step"] = points.Step()
m["end"] = points.UntilTime()
m["name"] = metric
mv := make([]interface{}, len(values))
for i, p := range values {
if math.IsNaN(p) {
mv[i] = nil
} else {
mv[i] = p
}
}
m["values"] = mv
metrics = append(metrics, m)
wr.Header().Set("Content-Type", "application/pickle")
pEnc := pickle.NewEncoder(wr)
pEnc.Encode(metrics)
}
log.Infof("served %d points for %s", len(values), metric)
return
}
func handleRenderPickle(w http.ResponseWriter, req *http.Request, responses []serverResponse) {
// nothing to merge
if len(responses) == 1 {
w.Header().Set("Content-Type", "application/pickle")
w.Write(responses[0].response)
return
}
// decode everything
var decoded [][]interface{}
for _, r := range responses {
d := pickle.NewDecoder(bytes.NewReader(r.response))
metric, err := d.Decode()
if err != nil {
logger.Logf("error decoding response from server:%s: req:%s: err=%s", r.server, req.URL.RequestURI(), err)
if Debug > 1 {
logger.Logln("\n" + hex.Dump(r.response))
}
Metrics.Errors.Add(1)
continue
}
marray, ok := metric.([]interface{})
if !ok {
err := fmt.Sprintf("bad type for metric:%d from server:%s req:%s", metric, r.server, req.URL.RequestURI())
logger.Logln(err)
http.Error(w, err, http.StatusInternalServerError)
Metrics.Errors.Add(1)
return
}
if len(marray) == 0 {
continue
}
decoded = append(decoded, marray)
}
if Debug > 2 {
logger.Logf("request: %s: %v", req.URL.RequestURI(), decoded)
}
if len(decoded) == 0 {
logger.Logf("no decoded responses to merge for req:%s", req.URL.RequestURI())
w.Header().Set("Content-Type", "application/pickle")
w.Write(responses[0].response)
return
}
if len(decoded) == 1 {
if Debug > 0 {
logger.Logf("only one decoded responses to merge for req:%s", req.URL.RequestURI())
}
w.Header().Set("Content-Type", "application/pickle")
// send back whatever data we have
e := pickle.NewEncoder(w)
e.Encode(decoded[0])
return
}
if len(decoded[0]) != 1 {
err := fmt.Sprintf("bad length for decoded[]:%d from req:%s", len(decoded[0]), req.URL.RequestURI())
logger.Logln(err)
http.Error(w, err, http.StatusInternalServerError)
Metrics.Errors.Add(1)
return
}
base, ok := decoded[0][0].(map[interface{}]interface{})
if !ok {
err := fmt.Sprintf("bad type for decoded:%t from req:%s", decoded[0][0], req.URL.RequestURI())
logger.Logln(err)
http.Error(w, err, http.StatusInternalServerError)
Metrics.Errors.Add(1)
return
}
values, ok := base["values"].([]interface{})
if !ok {
err := fmt.Sprintf("bad type for values:%t from req:%s", base["values"], req.URL.RequestURI())
logger.Logln(err)
http.Error(w, err, http.StatusInternalServerError)
Metrics.Errors.Add(1)
return
}
fixValues:
for i := 0; i < len(values); i++ {
if _, ok := values[i].(pickle.None); ok {
// find one in the other values arrays
for other := 1; other < len(decoded); other++ {
m, ok := decoded[other][0].(map[interface{}]interface{})
if !ok {
logger.Logln(fmt.Sprintf("bad type for decoded[%d][0]: %t", other, decoded[other][0]))
Metrics.Errors.Add(1)
break fixValues
}
ovalues, ok := m["values"].([]interface{})
if !ok {
logger.Logf("bad type for ovalues:%t from req:%s (skipping)", m["values"], req.URL.RequestURI())
Metrics.Errors.Add(1)
break fixValues
}
if len(ovalues) != len(values) {
logger.Logf("request: %s: unable to merge ovalues: len(values)=%d but len(ovalues)=%d", req.URL.RequestURI(), len(values), len(ovalues))
Metrics.Errors.Add(1)
break fixValues
}
if _, ok := ovalues[i].(pickle.None); !ok {
values[i] = ovalues[i]
break
}
}
}
}
// the first response is where we've been filling in our data, so we're ok just to serialize it as our response
w.Header().Set("Content-Type", "application/pickle")
e := pickle.NewEncoder(w)
e.Encode(decoded[0])
}
