Example #1
0
File: indexer.go Project: mrares/no
func processor(partition int, conn *net.UDPConn, tick <-chan compactionTick, quit chan struct{}, wg *sync.WaitGroup) {
	wg.Add(1)
	defer wg.Done()

	shouldQuit := false
	buf := make([]byte, 65536+4, 65536+4)
	index := make(map[string]*deque)
	data := pb.Data{}
	totalMsg := 0

	log.Printf("new processor %d", partition)

	var file *os.File
	defer file.Close()
	getFile := func(t time.Time) (*os.File, int64) {
		if file == nil {
			_, fileName, err := makePath(t)
			if err != nil {
				log.Println("Failed to create path", fileName, err)
				return nil, 0
			}

			fileName += fmt.Sprintf("%d.%d.data", t.Unix(), partition)
			log.Printf("[%d] open new file %s for epoch %d", partition, fileName, t.Unix())

			if file, err = createFile(fileName); err != nil {
				log.Println("Failed to open file", fileName, err)
				return nil, 0
			}

			if _, err = file.WriteString(dataFileHeader); err != nil {
				log.Println("Failed to write to file", err)
				return nil, 0
			}
		}

		offset, _ := file.Seek(0, 1)
		return file, offset
	}

	ctick := <-tick

loop:
	for {
		select {
		case <-quit:
			shouldQuit = true
			log.Printf("[%d] will quit", partition)

		case ct := <-tick:
			log.Printf("[%d] send job to compactor %d\n", partition, ctick.t.Unix())
			ctick.ch <- compactionJob{partition, totalMsg, index}
			index = make(map[string]*deque)
			file.Close()
			file = nil
			totalMsg = 0
			ctick = ct

			if shouldQuit {
				log.Printf("[%d] quiting", partition)
				break loop
			}

		default:
			conn.SetReadDeadline(time.Now().Add(time.Millisecond))
			length, err := conn.Read(buf[4:])
			if err != nil {
				if nerr, ok := err.(net.Error); ok && nerr.Timeout() == false {
					log.Println("UDP read error", err)
				}
				continue
			}

			data.Reset()
			if err := data.Unmarshal(buf[4 : length+4]); err != nil {
				log.Println("Failed to decode", err)
				continue
			}

			file, offset := getFile(ctick.t)
			if file == nil {
				log.Println("Failed to get file")
				continue
			}

			intToByteArray(uint32(length), buf[0:4])
			if _, err := file.Write(buf[:length+4]); err != nil {
				log.Println("Failed to write to file", err)
				continue
			}

			tags := data.GetHeader().GetTags()
			if len(tags) > maxTagsInMessage {
				log.Println("Too many tags in message ", len(tags))
				continue
			}

			for _, tag := range tags {
				if len(tag) > maxTagLength {
					log.Println("Too long tag")
					continue
				}

				deque, ok := index[tag]
				if !ok {
					deque = newDeque(partition)
					index[tag] = deque
				}

				deque.Append(uint32(offset))
			}

			totalMsg++
		}
	}
}