Esempio n. 1
0
File: command.go Progetto: h12w/kpax
func (cmd *TailCommand) Exec(cl model.Cluster) error {
	if cmd.Format == "" {
		var err error
		cmd.Format, err = formatDetector{cl, cmd.Topic}.detect()
		if err != nil {
			return err
		}
	}
	var partitions []int32
	if cmd.Partition == -1 {
		var err error
		partitions, err = cl.Partitions(cmd.Topic)
		if err != nil {
			return err
		}
	} else {
		partitions = []int32{int32(cmd.Partition)}
	}
	cr := consumer.New(cl)
	var wg sync.WaitGroup
	wg.Add(len(partitions))
	for _, partition := range partitions {
		start, err := cr.FetchOffsetByTime(cmd.Topic, partition, proto.Earliest)
		if err != nil {
			return err
		}
		end, err := cr.FetchOffsetByTime(cmd.Topic, partition, proto.Latest)
		if err != nil {
			return err
		}
		if start < end-int64(cmd.Count) {
			start = end - int64(cmd.Count)
		}
		go func(partition int32, start, end int64) {
			defer wg.Done()
			offset := start
			for offset < end {
				messages, err := cr.Consume(cmd.Topic, partition, offset)
				if err != nil {
					log.Println(err)
					return
				}
				if len(messages) == 0 {
					break
				}
				for _, msg := range messages {
					line, err := cmd.Format.Sprint(msg.Value)
					if err != nil {
						log.Println(err)
						break
					}
					fmt.Println(line)
				}
				offset = messages[len(messages)-1].Offset + 1
			}
		}(partition, start, end)
	}
	wg.Wait()
	return nil
}
Esempio n. 2
0
File: command.go Progetto: h12w/kpax
func (cmd *RollbackCommand) Exec(cl model.Cluster) error {
	if cmd.Format == "" {
		var err error
		cmd.Format, err = formatDetector{cl, cmd.Topic}.detect()
		if err != nil {
			return err
		}
	}
	partitions, err := cl.Partitions(cmd.Topic)
	if err != nil {
		return err
	}
	cr := consumer.New(cl)
	var wg sync.WaitGroup
	wg.Add(len(partitions))
	timeFunc := cmd.Format.unmarshalTime(cmd.TimeField)
	for _, partition := range partitions {
		go func(partition int32) {
			defer wg.Done()
			offset, err := cr.SearchOffsetByTime(cmd.Topic, partition, time.Time(cmd.Start), timeFunc)
			if err != nil {
				log.Println(err)
				return
			}
			if err := cr.Commit(cmd.Topic, partition, cmd.Group, offset); err != nil {
				log.Println(err)
			}
			fmt.Printf("\t%d:%d\n", partition, offset)
		}(partition)
	}
	wg.Wait()
	return nil

}
Esempio n. 3
0
File: command.go Progetto: h12w/kpax
func (cmd *ConsumeCommand) Exec(cl model.Cluster) error {
	if cmd.Format == "" {
		var err error
		cmd.Format, err = formatDetector{cl, cmd.Topic}.detect()
		if err != nil {
			return err
		}
	}
	partitions, err := cl.Partitions(cmd.Topic)
	if err != nil {
		return err
	}
	cr := consumer.New(cl)
	var wg sync.WaitGroup
	wg.Add(len(partitions))
	var cnt int64
	timeFunc := cmd.Format.unmarshalTime(cmd.TimeField)
	for _, partition := range partitions {
		go func(partition int32) {
			defer wg.Done()
			partCnt, err := cmd.consumePartition(cr, partition, timeFunc)
			if err != nil {
				log.Println(err)
				return
			}
			atomic.AddInt64(&cnt, partCnt)
		}(partition)
	}
	wg.Wait()
	if cmd.Count {
		fmt.Println(cnt)
	}
	return nil
}
Esempio n. 4
0
func Dump(k *kafka.Cluster, topic string, newObj func() encoding.BinaryUnmarshaler) (string, error) {
	cl := cluster.New(broker.New, k.Brokers())
	partitions, err := cl.Partitions(topic)
	if err != nil {
		return "", err
	}
	cr := consumer.New(cl)
	var lines []string
	for _, partition := range partitions {
		start, err := cr.FetchOffsetByTime(topic, partition, proto.Earliest)
		if err != nil {
			return "", err
		}
		end, err := cr.FetchOffsetByTime(topic, partition, proto.Latest)
		if err != nil {
			return "", err
		}
		for offset := start; offset < end; {
			messages, err := cr.Consume(topic, partition, offset)
			if err != nil {
				return "", err
			}
			if len(messages) == 0 {
				break
			}
			for _, message := range messages {
				obj := newObj()
				if err := obj.UnmarshalBinary(message.Value); err != nil {
					return "", err
				}
				jsonBuf, err := json.MarshalIndent(obj, "", "\t")
				if err != nil {
					return "", err
				}
				lines = append(lines, string(jsonBuf))
			}
			offset += messages[len(messages)-1].Offset + 1
		}
	}
	sort.Strings(lines)
	return strings.Join(lines, "\n"), nil
}
Esempio n. 5
0
File: format.go Progetto: h12w/kpax
func (d formatDetector) detect() (Format, error) {
	cl, topic := d.cl, d.topic
	partitions, err := cl.Partitions(topic)
	if err != nil {
		return UnknownFormat, err
	}
	cr := consumer.New(cl)
	for _, partition := range partitions {
		offset, err := cr.FetchOffsetByTime(topic, partition, proto.Latest)
		if err != nil {
			continue
		}
		messages, err := cr.Consume(topic, partition, offset-1)
		if err != nil {
			continue
		}
		for _, msg := range messages {
			return detectFormat(msg.Value)
		}
	}
	return UnknownFormat, nil
}
Esempio n. 6
0
File: command.go Progetto: h12w/kpax
func (cmd *OffsetCommand) Exec(cl model.Cluster) error {
	partitions, err := cl.Partitions(cmd.Topic)
	if err != nil {
		return err
	}
	cr := consumer.New(cl)
	fmt.Printf("topic: %s, group: %s\n", cmd.Topic, cmd.Group)
	for _, partition := range partitions {
		offset := int64(-1)
		var err error
		if cmd.Latest {
			offset, err = cr.FetchOffsetByTime(cmd.Topic, partition, proto.Latest)
		} else if cmd.Earliest {
			offset, err = cr.FetchOffsetByTime(cmd.Topic, partition, proto.Earliest)
		} else {
			offset, err = cr.Offset(cmd.Topic, partition, cmd.Group)
		}
		if err != nil {
			return err
		}
		fmt.Printf("\t%d:%d\n", partition, offset)
	}
	return nil
}