func handler(w http.ResponseWriter, r *http.Request) { log.Println("new request from", r.RemoteAddr) defer r.Body.Close() var q query var err error dec := json.NewDecoder(r.Body) if err = dec.Decode(&q); err != nil { log.Println("failed to decode request", err) http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) return } t := time.Unix(int64(q.From), 0).UTC() idx, err := mcache.getIndex(t) if err != nil { log.Println("failed to get index", err) http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) return } idx.borrow() defer idx.release() var v indexPartitions if len(q.And) > 0 { v = parseQuery(idx, mergeAnd, q.And) } else { v = parseQuery(idx, mergeOr, q.Or) } var substreams []string for k, v := range r.URL.Query() { if k == "sub" { substreams = v break } } log.Println("going to send N offsets", v.size(), substreams) h := r.Header h.Set("Content-Type", "text/event-stream") h.Set("Cache-Control", "no-cache") var buf []byte buf4 := make([]byte, 4, 4) buf64K := make([]byte, 65536, 65536) send := func(b []byte) error { if _, err = w.Write(intToByteArray(len(b), buf4)); err != nil { return err } if _, err = w.Write(b); err != nil { return err } return nil } data := pb.Message{} for p, offsets := range v { db, err := mcache.getDatabase(int(p), t) if err != nil { log.Println(err) continue } db.borrow() defer db.release() for _, off := range offsets { offset := int(off) if offset+uint32Size > len(db.data) { log.Println("invalid offset", offset, len(db.data)) break } length := byteArrayToInt(db.data[offset : offset+uint32Size]) if offset+length+uint32Size > len(db.data) { log.Println("invalid length", offset+length+uint32Size, len(db.data)) break } buf = db.data[offset+uint32Size : offset+uint32Size+length] if len(substreams) > 0 { data.Reset() if err = data.Unmarshal(buf); err != nil { log.Println("Failed to decode", err) break } var payload []*pb.Message_Payload for _, sub := range substreams { for _, frame := range data.GetFrames() { if frame.GetId() == sub { payload = append(payload, frame) } } } data.Frames = payload if length, err = data.MarshalTo(buf64K); err != nil { log.Println("Failed to encode", err) break } if err = send(buf64K[:length]); err != nil { log.Println(err) break } } else { if err = send(buf); err != nil { log.Println(err) break } } } } }