func findHandlerPickle(w http.ResponseWriter, req *http.Request, responses []serverResponse) ([]map[interface{}]interface{}, map[string][]string, error) {
// metric -> [server1, ... ]
paths := make(map[string][]string)
var metrics []map[interface{}]interface{}
for _, r := range responses {
d := pickle.NewDecoder(bytes.NewReader(r.response))
metric, err := d.Decode()
if err != nil {
logger.Logf("error decoding response from server:%s: req:%s: err=%s", r.server, req.URL.RequestURI(), err)
if Debug > 1 {
logger.Logln("\n" + hex.Dump(r.response))
}
Metrics.Errors.Add(1)
continue
}
marray, ok := metric.([]interface{})
if !ok {
logger.Logf("bad type for metric:%t from server:%s: req:%s", metric, r.server, req.URL.RequestURI())
http.Error(w, fmt.Sprintf("bad type for metric: %t", metric), http.StatusInternalServerError)
Metrics.Errors.Add(1)
return nil, nil, errors.New("failed")
}
for i, m := range marray {
mm, ok := m.(map[interface{}]interface{})
if !ok {
logger.Logf("bad type for metric[%d]:%t from server:%s: req:%s", i, m, r.server, req.URL.RequestURI())
http.Error(w, fmt.Sprintf("bad type for metric[%d]:%t", i, m), http.StatusInternalServerError)
Metrics.Errors.Add(1)
return nil, nil, errors.New("failed")
}
name, ok := mm["metric_path"].(string)
if !ok {
logger.Logf("bad type for metric_path:%t from server:%s: req:%s", mm["metric_path"], r.server, req.URL.RequestURI())
http.Error(w, fmt.Sprintf("bad type for metric_path: %t", mm["metric_path"]), http.StatusInternalServerError)
Metrics.Errors.Add(1)
return nil, nil, errors.New("failed")
}
p, ok := paths[name]
if !ok {
// we haven't seen this name yet
// add the metric to the list of metrics to return
metrics = append(metrics, mm)
}
// add the server to the list of servers that know about this metric
p = append(p, r.server)
paths[name] = p
}
}
return metrics, paths, nil
}
func handleRenderPickle(w http.ResponseWriter, req *http.Request, responses []serverResponse) {
// nothing to merge
if len(responses) == 1 {
w.Header().Set("Content-Type", "application/pickle")
w.Write(responses[0].response)
return
}
// decode everything
var decoded [][]interface{}
for _, r := range responses {
d := pickle.NewDecoder(bytes.NewReader(r.response))
metric, err := d.Decode()
if err != nil {
logger.Logf("error decoding response from server:%s: req:%s: err=%s", r.server, req.URL.RequestURI(), err)
if Debug > 1 {
logger.Logln("\n" + hex.Dump(r.response))
}
Metrics.Errors.Add(1)
continue
}
marray, ok := metric.([]interface{})
if !ok {
err := fmt.Sprintf("bad type for metric:%d from server:%s req:%s", metric, r.server, req.URL.RequestURI())
logger.Logln(err)
http.Error(w, err, http.StatusInternalServerError)
Metrics.Errors.Add(1)
return
}
if len(marray) == 0 {
continue
}
decoded = append(decoded, marray)
}
if Debug > 2 {
logger.Logf("request: %s: %v", req.URL.RequestURI(), decoded)
}
if len(decoded) == 0 {
logger.Logf("no decoded responses to merge for req:%s", req.URL.RequestURI())
w.Header().Set("Content-Type", "application/pickle")
w.Write(responses[0].response)
return
}
if len(decoded) == 1 {
if Debug > 0 {
logger.Logf("only one decoded responses to merge for req:%s", req.URL.RequestURI())
}
w.Header().Set("Content-Type", "application/pickle")
// send back whatever data we have
e := pickle.NewEncoder(w)
e.Encode(decoded[0])
return
}
if len(decoded[0]) != 1 {
err := fmt.Sprintf("bad length for decoded[]:%d from req:%s", len(decoded[0]), req.URL.RequestURI())
logger.Logln(err)
http.Error(w, err, http.StatusInternalServerError)
Metrics.Errors.Add(1)
return
}
base, ok := decoded[0][0].(map[interface{}]interface{})
if !ok {
err := fmt.Sprintf("bad type for decoded:%t from req:%s", decoded[0][0], req.URL.RequestURI())
logger.Logln(err)
http.Error(w, err, http.StatusInternalServerError)
Metrics.Errors.Add(1)
return
}
values, ok := base["values"].([]interface{})
if !ok {
err := fmt.Sprintf("bad type for values:%t from req:%s", base["values"], req.URL.RequestURI())
logger.Logln(err)
http.Error(w, err, http.StatusInternalServerError)
Metrics.Errors.Add(1)
return
}
fixValues:
for i := 0; i < len(values); i++ {
if _, ok := values[i].(pickle.None); ok {
// find one in the other values arrays
for other := 1; other < len(decoded); other++ {
m, ok := decoded[other][0].(map[interface{}]interface{})
if !ok {
logger.Logln(fmt.Sprintf("bad type for decoded[%d][0]: %t", other, decoded[other][0]))
Metrics.Errors.Add(1)
break fixValues
}
ovalues, ok := m["values"].([]interface{})
if !ok {
logger.Logf("bad type for ovalues:%t from req:%s (skipping)", m["values"], req.URL.RequestURI())
Metrics.Errors.Add(1)
break fixValues
}
if len(ovalues) != len(values) {
logger.Logf("request: %s: unable to merge ovalues: len(values)=%d but len(ovalues)=%d", req.URL.RequestURI(), len(values), len(ovalues))
Metrics.Errors.Add(1)
break fixValues
}
if _, ok := ovalues[i].(pickle.None); !ok {
values[i] = ovalues[i]
break
}
}
}
}
// the first response is where we've been filling in our data, so we're ok just to serialize it as our response
w.Header().Set("Content-Type", "application/pickle")
e := pickle.NewEncoder(w)
e.Encode(decoded[0])
}
func (p *Pickle) Handle(c net.Conn) {
defer c.Close()
// TODO c.SetTimeout(60e9)
r := bufio.NewReaderSize(c, 4096)
ReadLoop:
for {
// Note that everything in this loop should proceed as fast as it can
// so we're not blocked and can keep processing
// so the validation, the pipeline initiated via table.Dispatch(), etc
// must never block.
var length uint32
err := binary.Read(r, binary.BigEndian, &length)
if err != nil {
if io.EOF != err {
log.Error("couldn't read payload length: " + err.Error())
}
break
}
lengthTotal := int(length)
lengthRead := 0
payload := make([]byte, lengthTotal, lengthTotal)
for {
tmpLengthRead, err := r.Read(payload[lengthRead:])
if err != nil {
log.Error("couldn't read payload: " + err.Error())
break ReadLoop
}
lengthRead += tmpLengthRead
if lengthRead == lengthTotal {
break
}
if lengthRead > lengthTotal {
log.Error(fmt.Sprintf("expected to read %d bytes, but read %d", length, lengthRead))
break ReadLoop
}
}
decoder := ogorek.NewDecoder(bytes.NewBuffer(payload))
rawDecoded, err := decoder.Decode()
if err != nil {
if io.EOF != err {
log.Error("error reading pickled data " + err.Error())
}
break
}
decoded, ok := rawDecoded.([]interface{})
if !ok {
log.Error(fmt.Sprintf("Unrecognized type %T for pickled data", rawDecoded))
break
}
ItemLoop:
for _, rawItem := range decoded {
numIn.Inc(1)
item, ok := rawItem.([]interface{})
if !ok {
log.Error(fmt.Sprintf("Unrecognized type %T for item", rawItem))
numInvalid.Inc(1)
continue
}
if len(item) != 2 {
log.Error(fmt.Sprintf("item length must be 2, got %d", len(item)))
numInvalid.Inc(1)
continue
}
metric, ok := item[0].(string)
if !ok {
log.Error(fmt.Sprintf("item metric must be a string, got %T", item[0]))
numInvalid.Inc(1)
continue
}
data, ok := item[1].([]interface{})
if !ok {
log.Error(fmt.Sprintf("item data must be an array, got %T", item[1]))
numInvalid.Inc(1)
continue
}
if len(data) != 2 {
log.Error(fmt.Sprintf("item data length must be 2, got %d", len(data)))
numInvalid.Inc(1)
continue
}
var value string
switch data[1].(type) {
case string:
value = data[1].(string)
case uint8, uint16, uint32, uint64, int8, int16, int32, int64:
value = fmt.Sprintf("%d", data[1])
case float32, float64:
value = fmt.Sprintf("%f", data[1])
default:
log.Error(fmt.Sprintf("Unrecognized type %T for value", data[1]))
numInvalid.Inc(1)
continue ItemLoop
}
var timestamp string
switch data[0].(type) {
case string:
timestamp = data[0].(string)
case uint8, uint16, uint32, uint64, int8, int16, int32, int64, (*big.Int):
timestamp = fmt.Sprintf("%d", data[0])
case float32, float64:
timestamp = fmt.Sprintf("%.0f", data[0])
default:
log.Error(fmt.Sprintf("Unrecognized type %T for timestamp", data[0]))
numInvalid.Inc(1)
continue ItemLoop
}
buf := []byte(metric + " " + value + " " + timestamp)
key, _, ts, err := m20.ValidatePacket(buf, p.config.Validation_level_legacy.Level, p.config.Validation_level_m20.Level)
if err != nil {
p.bad.Add(key, buf, err)
numInvalid.Inc(1)
continue
}
if p.config.Validate_order {
err = validate.Ordered(key, ts)
if err != nil {
p.bad.Add(key, buf, err)
numOutOfOrder.Inc(1)
continue
}
}
p.table.Dispatch(buf)
}
}
}