func resetMessage(msg *msgpb.Message) {
msg.Frames = msg.Frames[:0]
if msg.Header != nil {
msg.Header.Tags = msg.Header.Tags[:0]
}
}
func handler(w http.ResponseWriter, r *http.Request) {
log.Println("new request from", r.RemoteAddr)
defer r.Body.Close()
var q query
var err error
dec := json.NewDecoder(r.Body)
if err = dec.Decode(&q); err != nil {
log.Println("failed to decode request", err)
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
return
}
t := time.Unix(int64(q.From), 0).UTC()
idx, err := mcache.getIndex(t)
if err != nil {
log.Println("failed to get index", err)
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
return
}
idx.borrow()
defer idx.release()
var v indexPartitions
if len(q.And) > 0 {
v = parseQuery(idx, mergeAnd, q.And)
} else {
v = parseQuery(idx, mergeOr, q.Or)
}
var substreams []string
for k, v := range r.URL.Query() {
if k == "sub" {
substreams = v
break
}
}
log.Println("going to send N offsets", v.size(), substreams)
h := r.Header
h.Set("Content-Type", "text/event-stream")
h.Set("Cache-Control", "no-cache")
var buf []byte
buf4 := make([]byte, 4, 4)
buf64K := make([]byte, 65536, 65536)
send := func(b []byte) error {
if _, err = w.Write(intToByteArray(len(b), buf4)); err != nil {
return err
}
if _, err = w.Write(b); err != nil {
return err
}
return nil
}
data := pb.Message{}
for p, offsets := range v {
db, err := mcache.getDatabase(int(p), t)
if err != nil {
log.Println(err)
continue
}
db.borrow()
defer db.release()
for _, off := range offsets {
offset := int(off)
if offset+uint32Size > len(db.data) {
log.Println("invalid offset", offset, len(db.data))
break
}
length := byteArrayToInt(db.data[offset : offset+uint32Size])
if offset+length+uint32Size > len(db.data) {
log.Println("invalid length", offset+length+uint32Size, len(db.data))
break
}
buf = db.data[offset+uint32Size : offset+uint32Size+length]
if len(substreams) > 0 {
data.Reset()
if err = data.Unmarshal(buf); err != nil {
log.Println("Failed to decode", err)
break
}
var payload []*pb.Message_Payload
for _, sub := range substreams {
for _, frame := range data.GetFrames() {
if frame.GetId() == sub {
payload = append(payload, frame)
}
}
}
data.Frames = payload
if length, err = data.MarshalTo(buf64K); err != nil {
log.Println("Failed to encode", err)
break
}
if err = send(buf64K[:length]); err != nil {
log.Println(err)
break
}
} else {
if err = send(buf); err != nil {
log.Println(err)
break
}
}
}
}
}
func startWorkers(proc int, start []chan epoch, quit <-chan struct{}, wg *sync.WaitGroup) (msgchs []chan<- msg, ticks []chan<- tick) {
worker := func(partition int, ch <-chan msg, tick <-chan tick) {
log.Printf("start worker %d", partition)
wg.Add(1)
defer wg.Done()
var err error
omsg := msgpb.Message{}
jobs := make([]*job, indexingWindow)
getJob := func(e epoch) (j *job, bucket int) {
bucket = int(e) % len(jobs)
if jobs[bucket] == nil {
jobs[bucket] = &job{
partition: partition,
index: make(map[string]*deque),
dir: newEpochDir(rootDir, int64(e), partition),
}
}
return jobs[bucket], bucket
}
start[0] <- 0
min := <-start[1]
max := min + indexingWindow
mainLoop:
for {
select {
case t := <-tick:
log.Printf("worker [%d]: got new base %d, will send %d", partition, t.base, t.send)
if t.send != min {
log.Printf("worker [%d]: send epoch %d doesn't match min epoch %d, this results in data loss!!!", partition, t.send, min)
t.ch <- job{partition: partition}
jobs[int(t.send)%len(jobs)] = nil
break
}
j, bucket := getJob(t.send)
jobs[bucket] = nil
t.ch <- *j
close(t.ch)
select {
case <-quit:
log.Printf("worker [%d]: should quit", partition)
// if t.base > max {
log.Printf("worker [%d]: quiting", partition)
return
// }
// log.Printf("worker [%d]: wait %d seconds and quit", partition, max-t.base)
// min = t.base
default:
min, max = t.base, t.base+indexingWindow
}
case msg := <-ch:
resetMessage(&omsg)
needEncode := false
if err = omsg.Unmarshal(msg.buf); err != nil {
msg.err <- err
continue mainLoop
}
header := omsg.Header
if header.Time == 0 {
needEncode = true
header.Time = time.Now().UTC().Unix()
}
htime := epoch(header.Time)
if htime < min || htime > max {
msg.err <- fmt.Errorf("Message time is outside acceptable window time=%d min=%d max=%d", htime, min, max)
continue mainLoop
}
tags := omsg.Header.Tags
if len(tags) > maxTagsInMessage {
msg.err <- fmt.Errorf("Too many tags in message %d", len(tags))
continue mainLoop
}
for _, tag := range tags {
if len(tag) > maxTagLength {
msg.err <- fmt.Errorf("Too long tag name %d", len(tag))
continue mainLoop
}
}
j, _ := getJob(htime)
var f *dfile
if f, err = j.dir.getDataFile(); err != nil {
msg.err <- err
continue mainLoop
}
if needEncode {
msg.buf = msg.buf[:cap(msg.buf)]
if length, err := omsg.MarshalTo(msg.buf); err != nil {
msg.err <- err
continue mainLoop
} else {
msg.buf = msg.buf[:length]
}
}
var offset int64
if offset, err = f.writeBuffer(msg.buf); err != nil {
msg.err <- err
continue mainLoop
}
index := j.index
tags = append(tags, anyTag)
for _, tag := range tags {
deque, ok := index[string(tag)]
if !ok {
deque = newDeque(partition)
index[string(tag)] = deque
}
deque.Append(uint32(offset))
}
j.totalMsg++
msg.err <- nil
}
}
}
for i := 0; i < proc; i++ {
t := make(chan tick, 1)
ticks = append(ticks, t)
m := make(chan msg, 1)
msgchs = append(msgchs, m)
go worker(i, m, t)
}
return msgchs, ticks
}