func findHandlerPB(w http.ResponseWriter, req *http.Request, responses []serverResponse) ([]*pb.GlobMatch, map[string][]string) { // metric -> [server1, ... ] paths := make(map[string][]string) seen := make(map[nameleaf]bool) var metrics []*pb.GlobMatch for _, r := range responses { var metric pb.GlobResponse err := metric.Unmarshal(r.response) if err != nil && req != nil { logger.Logf("error decoding protobuf response from server:%s: req:%s: err=%s", r.server, req.URL.RequestURI(), err) logger.Traceln("\n" + hex.Dump(r.response)) Metrics.FindErrors.Add(1) continue } for _, match := range metric.Matches { n := nameleaf{*match.Path, *match.IsLeaf} _, ok := seen[n] if !ok { // we haven't seen this name yet // add the metric to the list of metrics to return metrics = append(metrics, match) seen[n] = true } // add the server to the list of servers that know about this metric p := paths[*match.Path] p = append(p, r.server) paths[*match.Path] = p } } return metrics, paths }
func findCompleter(globs pb.GlobResponse) ([]byte, error) { var b bytes.Buffer var complete = make([]completer, 0) for _, g := range globs.GetMatches() { c := completer{ Path: g.GetPath(), } if g.GetIsLeaf() { c.IsLeaf = "1" } else { c.IsLeaf = "0" } i := strings.LastIndex(c.Path, ".") if i != -1 { c.Name = c.Path[i+1:] } complete = append(complete, c) } err := json.NewEncoder(&b).Encode(struct { Metrics []completer `json:"metrics"` }{ Metrics: complete}, ) return b.Bytes(), err }
func findList(globs pb.GlobResponse) ([]byte, error) { var b bytes.Buffer for _, g := range globs.GetMatches() { var dot string // make sure non-leaves end in one dot if !g.GetIsLeaf() && !strings.HasSuffix(g.GetPath(), ".") { dot = "." } fmt.Fprintln(&b, g.GetPath()+dot) } return b.Bytes(), nil }
func findTreejson(globs pb.GlobResponse) ([]byte, error) { var b bytes.Buffer var tree = make([]treejson, 0) seen := make(map[string]struct{}) basepath := globs.GetName() if i := strings.LastIndex(basepath, "."); i != -1 { basepath = basepath[:i+1] } for _, g := range globs.GetMatches() { name := g.GetPath() if i := strings.LastIndex(name, "."); i != -1 { name = name[i+1:] } if _, ok := seen[name]; ok { continue } seen[name] = struct{}{} t := treejson{ ID: basepath + name, Context: treejsonContext, Text: name, } if g.GetIsLeaf() { t.Leaf = 1 } else { t.AllowChildren = 1 t.Expandable = 1 } tree = append(tree, t) } err := json.NewEncoder(&b).Encode(tree) return b.Bytes(), err }
func findHandler(w http.ResponseWriter, req *http.Request) { logger.Debugln("request: ", req.URL.RequestURI()) Metrics.FindRequests.Add(1) rewrite, _ := url.ParseRequestURI(req.URL.RequestURI()) v := rewrite.Query() format := req.FormValue("format") v.Set("format", "protobuf") rewrite.RawQuery = v.Encode() query := req.FormValue("query") var tld string if i := strings.IndexByte(query, '.'); i > 0 { tld = query[:i] } // lookup tld in our map of where they live to reduce the set of // servers we bug with our find Config.mu.RLock() var backends []string var ok bool if backends, ok = Config.metricPaths[tld]; !ok || backends == nil || len(backends) == 0 { backends = Config.Backends } Config.mu.RUnlock() responses := multiGet(backends, rewrite.RequestURI()) if responses == nil || len(responses) == 0 { logger.Logln("find: error querying backends for: ", rewrite.RequestURI()) http.Error(w, "find: error querying backends", http.StatusInternalServerError) return } metrics, paths := findHandlerPB(w, req, responses) // update our cache of which servers have which metrics Config.mu.Lock() for k, v := range paths { Config.metricPaths[k] = v } Config.mu.Unlock() switch format { case "protobuf": w.Header().Set("Content-Type", "application/protobuf") var result pb.GlobResponse query := req.FormValue("query") result.Name = &query result.Matches = metrics b, _ := proto.Marshal(&result) w.Write(b) case "json": w.Header().Set("Content-Type", "application/json") jEnc := json.NewEncoder(w) jEnc.Encode(metrics) case "", "pickle": w.Header().Set("Content-Type", "application/pickle") var result []map[string]interface{} for _, metric := range metrics { mm := map[string]interface{}{ "metric_path": *metric.Path, "isLeaf": *metric.IsLeaf, } result = append(result, mm) } pEnc := pickle.NewEncoder(w) pEnc.Encode(result) } }
func findHandler(wr http.ResponseWriter, req *http.Request) { // URL: /metrics/find/?local=1&format=pickle&query=the.metric.path.with.glob t0 := time.Now() Metrics.FindRequests.Add(1) req.ParseForm() format := req.FormValue("format") query := req.FormValue("query") if format != "json" && format != "pickle" && format != "protobuf" { Metrics.FindErrors.Add(1) logger.Logf("dropping invalid uri (format=%s): %s", format, req.URL.RequestURI()) http.Error(wr, "Bad request (unsupported format)", http.StatusBadRequest) return } if query == "" { Metrics.FindErrors.Add(1) logger.Logf("dropping invalid request (query=): %s", req.URL.RequestURI()) http.Error(wr, "Bad request (no query)", http.StatusBadRequest) return } files, leafs := expandGlobs(query) if format == "json" || format == "protobuf" { name := req.FormValue("query") response := pb.GlobResponse{ Name: &name, Matches: make([]*pb.GlobMatch, 0), } for i, p := range files { response.Matches = append(response.Matches, &pb.GlobMatch{Path: proto.String(p), IsLeaf: proto.Bool(leafs[i])}) } var b []byte var err error switch format { case "json": b, err = json.Marshal(response) case "protobuf": b, err = proto.Marshal(&response) } if err != nil { Metrics.FindErrors.Add(1) logger.Logf("failed to create %s data for glob %s: %s", format, *response.Name, err) return } wr.Write(b) } else if format == "pickle" { // [{'metric_path': 'metric', 'intervals': [(x,y)], 'isLeaf': True},] var metrics []map[string]interface{} var m map[string]interface{} for i, p := range files { m = make(map[string]interface{}) m["metric_path"] = p // m["intervals"] = dunno how to do a tuple here m["isLeaf"] = leafs[i] metrics = append(metrics, m) } wr.Header().Set("Content-Type", "application/pickle") pEnc := pickle.NewEncoder(wr) pEnc.Encode(metrics) } if len(files) == 0 { // to get an idea how often we search for nothing Metrics.FindZero.Add(1) } logger.Debugf("find: %d hits for %s in %v", len(files), req.FormValue("query"), time.Since(t0)) return }
func renderHandler(w http.ResponseWriter, r *http.Request, stats *renderStats) { Metrics.Requests.Add(1) err := r.ParseForm() if err != nil { http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) return } targets := r.Form["target"] from := r.FormValue("from") until := r.FormValue("until") format := r.FormValue("format") useCache := truthyBool(r.FormValue("noCache")) == false var jsonp string if format == "json" { // TODO(dgryski): check jsonp only has valid characters jsonp = r.FormValue("jsonp") } if format == "" && (truthyBool(r.FormValue("rawData")) || truthyBool(r.FormValue("rawdata"))) { format = "raw" } if format == "" { format = "png" } cacheTimeout := int32(60) if tstr := r.FormValue("cacheTimeout"); tstr != "" { t, err := strconv.Atoi(tstr) if err != nil { log.Printf("failed to parse cacheTimeout: %v: %v", tstr, err) } else { cacheTimeout = int32(t) } } // make sure the cache key doesn't say noCache, because it will never hit r.Form.Del("noCache") // jsonp callback names are frequently autogenerated and hurt our cache r.Form.Del("jsonp") // Strip some cache-busters. If you don't want to cache, use noCache=1 r.Form.Del("_salt") r.Form.Del("_ts") r.Form.Del("_t") // Used by jquery.graphite.js cacheKey := r.Form.Encode() if response, ok := queryCache.get(cacheKey); useCache && ok { Metrics.RequestCacheHits.Add(1) writeResponse(w, response, format, jsonp) return } // normalize from and until values // BUG(dgryski): doesn't handle timezones the same as graphite-web from32 := dateParamToEpoch(from, timeNow().Add(-24*time.Hour).Unix()) until32 := dateParamToEpoch(until, timeNow().Unix()) var results []*metricData metricMap := make(map[metricRequest][]*metricData) for _, target := range targets { exp, e, err := parseExpr(target) if err != nil || e != "" { http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) return } for _, m := range exp.metrics() { mfetch := m mfetch.from += from32 mfetch.until += until32 if _, ok := metricMap[mfetch]; ok { // already fetched this metric for this request continue } var glob pb.GlobResponse var haveCacheData bool if response, ok := findCache.get(m.metric); useCache && ok { Metrics.FindCacheHits.Add(1) err := proto.Unmarshal(response, &glob) haveCacheData = err == nil } if !haveCacheData { var err error Metrics.FindRequests.Add(1) stats.zipperRequests++ glob, err = Zipper.Find(m.metric) if err != nil { log.Printf("Find: %v: %v", m.metric, err) continue } b, err := proto.Marshal(&glob) if err == nil { findCache.set(m.metric, b, 5*60) } } // For each metric returned in the Find response, query Render // This is a conscious decision to *not* cache render data rch := make(chan *metricData, len(glob.GetMatches())) leaves := 0 for _, m := range glob.GetMatches() { if !m.GetIsLeaf() { continue } Metrics.RenderRequests.Add(1) leaves++ Limiter.enter() stats.zipperRequests++ go func(m *pb.GlobMatch, from, until int32) { var rptr *metricData r, err := Zipper.Render(m.GetPath(), from, until) if err == nil { rptr = &r } else { log.Printf("Render: %v: %v", m.GetPath(), err) } rch <- rptr Limiter.leave() }(m, mfetch.from, mfetch.until) } for i := 0; i < leaves; i++ { r := <-rch if r != nil { metricMap[mfetch] = append(metricMap[mfetch], r) } } } func() { defer func() { if r := recover(); r != nil { var buf [1024]byte runtime.Stack(buf[:], false) log.Printf("panic during eval: %s: %s\n%s\n", cacheKey, r, string(buf[:])) } }() exprs := evalExpr(exp, from32, until32, metricMap) results = append(results, exprs...) }() } var body []byte switch format { case "json": body = marshalJSON(results) case "protobuf": body = marshalProtobuf(results) case "raw": body = marshalRaw(results) case "pickle": body = marshalPickle(results) case "png": body = marshalPNG(r, results) } writeResponse(w, body, format, jsonp) if len(results) != 0 { queryCache.set(cacheKey, body, cacheTimeout) } }