func processor(partition int, conn *net.UDPConn, tick <-chan compactionTick, quit chan struct{}, wg *sync.WaitGroup) { wg.Add(1) defer wg.Done() shouldQuit := false buf := make([]byte, 65536+4, 65536+4) index := make(map[string]*deque) data := pb.Data{} totalMsg := 0 log.Printf("new processor %d", partition) var file *os.File defer file.Close() getFile := func(t time.Time) (*os.File, int64) { if file == nil { _, fileName, err := makePath(t) if err != nil { log.Println("Failed to create path", fileName, err) return nil, 0 } fileName += fmt.Sprintf("%d.%d.data", t.Unix(), partition) log.Printf("[%d] open new file %s for epoch %d", partition, fileName, t.Unix()) if file, err = createFile(fileName); err != nil { log.Println("Failed to open file", fileName, err) return nil, 0 } if _, err = file.WriteString(dataFileHeader); err != nil { log.Println("Failed to write to file", err) return nil, 0 } } offset, _ := file.Seek(0, 1) return file, offset } ctick := <-tick loop: for { select { case <-quit: shouldQuit = true log.Printf("[%d] will quit", partition) case ct := <-tick: log.Printf("[%d] send job to compactor %d\n", partition, ctick.t.Unix()) ctick.ch <- compactionJob{partition, totalMsg, index} index = make(map[string]*deque) file.Close() file = nil totalMsg = 0 ctick = ct if shouldQuit { log.Printf("[%d] quiting", partition) break loop } default: conn.SetReadDeadline(time.Now().Add(time.Millisecond)) length, err := conn.Read(buf[4:]) if err != nil { if nerr, ok := err.(net.Error); ok && nerr.Timeout() == false { log.Println("UDP read error", err) } continue } data.Reset() if err := data.Unmarshal(buf[4 : length+4]); err != nil { log.Println("Failed to decode", err) continue } file, offset := getFile(ctick.t) if file == nil { log.Println("Failed to get file") continue } intToByteArray(uint32(length), buf[0:4]) if _, err := file.Write(buf[:length+4]); err != nil { log.Println("Failed to write to file", err) continue } tags := data.GetHeader().GetTags() if len(tags) > maxTagsInMessage { log.Println("Too many tags in message ", len(tags)) continue } for _, tag := range tags { if len(tag) > maxTagLength { log.Println("Too long tag") continue } deque, ok := index[tag] if !ok { deque = newDeque(partition) index[tag] = deque } deque.Append(uint32(offset)) } totalMsg++ } } }