コード例 #1
0
ファイル: indexer.go プロジェクト: jackdoe/no
func resetMessage(msg *msgpb.Message) {
	msg.Frames = msg.Frames[:0]
	if msg.Header != nil {
		msg.Header.Tags = msg.Header.Tags[:0]
	}
}
コード例 #2
0
ファイル: searcher.go プロジェクト: jackdoe/no
func handler(w http.ResponseWriter, r *http.Request) {
	log.Println("new request from", r.RemoteAddr)
	defer r.Body.Close()

	var q query
	var err error
	dec := json.NewDecoder(r.Body)
	if err = dec.Decode(&q); err != nil {
		log.Println("failed to decode request", err)
		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
		return
	}

	t := time.Unix(int64(q.From), 0).UTC()
	idx, err := mcache.getIndex(t)
	if err != nil {
		log.Println("failed to get index", err)
		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
		return
	}

	idx.borrow()
	defer idx.release()

	var v indexPartitions
	if len(q.And) > 0 {
		v = parseQuery(idx, mergeAnd, q.And)
	} else {
		v = parseQuery(idx, mergeOr, q.Or)
	}

	var substreams []string
	for k, v := range r.URL.Query() {
		if k == "sub" {
			substreams = v
			break
		}
	}

	log.Println("going to send N offsets", v.size(), substreams)

	h := r.Header
	h.Set("Content-Type", "text/event-stream")
	h.Set("Cache-Control", "no-cache")

	var buf []byte
	buf4 := make([]byte, 4, 4)
	buf64K := make([]byte, 65536, 65536)

	send := func(b []byte) error {
		if _, err = w.Write(intToByteArray(len(b), buf4)); err != nil {
			return err
		}

		if _, err = w.Write(b); err != nil {
			return err
		}

		return nil
	}

	data := pb.Message{}
	for p, offsets := range v {
		db, err := mcache.getDatabase(int(p), t)
		if err != nil {
			log.Println(err)
			continue
		}

		db.borrow()
		defer db.release()

		for _, off := range offsets {
			offset := int(off)
			if offset+uint32Size > len(db.data) {
				log.Println("invalid offset", offset, len(db.data))
				break
			}

			length := byteArrayToInt(db.data[offset : offset+uint32Size])
			if offset+length+uint32Size > len(db.data) {
				log.Println("invalid length", offset+length+uint32Size, len(db.data))
				break
			}

			buf = db.data[offset+uint32Size : offset+uint32Size+length]

			if len(substreams) > 0 {
				data.Reset()
				if err = data.Unmarshal(buf); err != nil {
					log.Println("Failed to decode", err)
					break
				}

				var payload []*pb.Message_Payload
				for _, sub := range substreams {
					for _, frame := range data.GetFrames() {
						if frame.GetId() == sub {
							payload = append(payload, frame)
						}
					}
				}

				data.Frames = payload
				if length, err = data.MarshalTo(buf64K); err != nil {
					log.Println("Failed to encode", err)
					break
				}

				if err = send(buf64K[:length]); err != nil {
					log.Println(err)
					break
				}
			} else {
				if err = send(buf); err != nil {
					log.Println(err)
					break
				}
			}
		}
	}
}
コード例 #3
0
ファイル: indexer.go プロジェクト: jackdoe/no
func startWorkers(proc int, start []chan epoch, quit <-chan struct{}, wg *sync.WaitGroup) (msgchs []chan<- msg, ticks []chan<- tick) {
	worker := func(partition int, ch <-chan msg, tick <-chan tick) {
		log.Printf("start worker %d", partition)

		wg.Add(1)
		defer wg.Done()

		var err error
		omsg := msgpb.Message{}

		jobs := make([]*job, indexingWindow)
		getJob := func(e epoch) (j *job, bucket int) {
			bucket = int(e) % len(jobs)
			if jobs[bucket] == nil {
				jobs[bucket] = &job{
					partition: partition,
					index:     make(map[string]*deque),
					dir:       newEpochDir(rootDir, int64(e), partition),
				}
			}
			return jobs[bucket], bucket
		}

		start[0] <- 0
		min := <-start[1]
		max := min + indexingWindow

	mainLoop:
		for {
			select {
			case t := <-tick:
				log.Printf("worker [%d]: got new base %d, will send %d", partition, t.base, t.send)

				if t.send != min {
					log.Printf("worker [%d]: send epoch %d doesn't match min epoch %d, this results in data loss!!!", partition, t.send, min)
					t.ch <- job{partition: partition}
					jobs[int(t.send)%len(jobs)] = nil
					break
				}

				j, bucket := getJob(t.send)
				jobs[bucket] = nil
				t.ch <- *j
				close(t.ch)

				select {
				case <-quit:
					log.Printf("worker [%d]: should quit", partition)

					// if t.base > max {
					log.Printf("worker [%d]: quiting", partition)
					return
					// }

					// log.Printf("worker [%d]: wait %d seconds and quit", partition, max-t.base)
					// min = t.base

				default:
					min, max = t.base, t.base+indexingWindow
				}

			case msg := <-ch:
				resetMessage(&omsg)
				needEncode := false

				if err = omsg.Unmarshal(msg.buf); err != nil {
					msg.err <- err
					continue mainLoop
				}

				header := omsg.Header
				if header.Time == 0 {
					needEncode = true
					header.Time = time.Now().UTC().Unix()
				}

				htime := epoch(header.Time)
				if htime < min || htime > max {
					msg.err <- fmt.Errorf("Message time is outside acceptable window time=%d min=%d max=%d", htime, min, max)
					continue mainLoop
				}

				tags := omsg.Header.Tags
				if len(tags) > maxTagsInMessage {
					msg.err <- fmt.Errorf("Too many tags in message %d", len(tags))
					continue mainLoop
				}

				for _, tag := range tags {
					if len(tag) > maxTagLength {
						msg.err <- fmt.Errorf("Too long tag name %d", len(tag))
						continue mainLoop
					}
				}

				j, _ := getJob(htime)

				var f *dfile
				if f, err = j.dir.getDataFile(); err != nil {
					msg.err <- err
					continue mainLoop
				}

				if needEncode {
					msg.buf = msg.buf[:cap(msg.buf)]
					if length, err := omsg.MarshalTo(msg.buf); err != nil {
						msg.err <- err
						continue mainLoop
					} else {
						msg.buf = msg.buf[:length]
					}
				}

				var offset int64
				if offset, err = f.writeBuffer(msg.buf); err != nil {
					msg.err <- err
					continue mainLoop
				}

				index := j.index
				tags = append(tags, anyTag)
				for _, tag := range tags {
					deque, ok := index[string(tag)]
					if !ok {
						deque = newDeque(partition)
						index[string(tag)] = deque
					}

					deque.Append(uint32(offset))
				}

				j.totalMsg++
				msg.err <- nil
			}
		}
	}

	for i := 0; i < proc; i++ {
		t := make(chan tick, 1)
		ticks = append(ticks, t)

		m := make(chan msg, 1)
		msgchs = append(msgchs, m)
		go worker(i, m, t)
	}

	return msgchs, ticks
}