func (cmd *TailCommand) Exec(cl model.Cluster) error {
if cmd.Format == "" {
var err error
cmd.Format, err = formatDetector{cl, cmd.Topic}.detect()
if err != nil {
return err
}
}
var partitions []int32
if cmd.Partition == -1 {
var err error
partitions, err = cl.Partitions(cmd.Topic)
if err != nil {
return err
}
} else {
partitions = []int32{int32(cmd.Partition)}
}
cr := consumer.New(cl)
var wg sync.WaitGroup
wg.Add(len(partitions))
for _, partition := range partitions {
start, err := cr.FetchOffsetByTime(cmd.Topic, partition, proto.Earliest)
if err != nil {
return err
}
end, err := cr.FetchOffsetByTime(cmd.Topic, partition, proto.Latest)
if err != nil {
return err
}
if start < end-int64(cmd.Count) {
start = end - int64(cmd.Count)
}
go func(partition int32, start, end int64) {
defer wg.Done()
offset := start
for offset < end {
messages, err := cr.Consume(cmd.Topic, partition, offset)
if err != nil {
log.Println(err)
return
}
if len(messages) == 0 {
break
}
for _, msg := range messages {
line, err := cmd.Format.Sprint(msg.Value)
if err != nil {
log.Println(err)
break
}
fmt.Println(line)
}
offset = messages[len(messages)-1].Offset + 1
}
}(partition, start, end)
}
wg.Wait()
return nil
}
func (cmd *RollbackCommand) Exec(cl model.Cluster) error {
if cmd.Format == "" {
var err error
cmd.Format, err = formatDetector{cl, cmd.Topic}.detect()
if err != nil {
return err
}
}
partitions, err := cl.Partitions(cmd.Topic)
if err != nil {
return err
}
cr := consumer.New(cl)
var wg sync.WaitGroup
wg.Add(len(partitions))
timeFunc := cmd.Format.unmarshalTime(cmd.TimeField)
for _, partition := range partitions {
go func(partition int32) {
defer wg.Done()
offset, err := cr.SearchOffsetByTime(cmd.Topic, partition, time.Time(cmd.Start), timeFunc)
if err != nil {
log.Println(err)
return
}
if err := cr.Commit(cmd.Topic, partition, cmd.Group, offset); err != nil {
log.Println(err)
}
fmt.Printf("\t%d:%d\n", partition, offset)
}(partition)
}
wg.Wait()
return nil
}
func (cmd *ConsumeCommand) Exec(cl model.Cluster) error {
if cmd.Format == "" {
var err error
cmd.Format, err = formatDetector{cl, cmd.Topic}.detect()
if err != nil {
return err
}
}
partitions, err := cl.Partitions(cmd.Topic)
if err != nil {
return err
}
cr := consumer.New(cl)
var wg sync.WaitGroup
wg.Add(len(partitions))
var cnt int64
timeFunc := cmd.Format.unmarshalTime(cmd.TimeField)
for _, partition := range partitions {
go func(partition int32) {
defer wg.Done()
partCnt, err := cmd.consumePartition(cr, partition, timeFunc)
if err != nil {
log.Println(err)
return
}
atomic.AddInt64(&cnt, partCnt)
}(partition)
}
wg.Wait()
if cmd.Count {
fmt.Println(cnt)
}
return nil
}
func Dump(k *kafka.Cluster, topic string, newObj func() encoding.BinaryUnmarshaler) (string, error) {
cl := cluster.New(broker.New, k.Brokers())
partitions, err := cl.Partitions(topic)
if err != nil {
return "", err
}
cr := consumer.New(cl)
var lines []string
for _, partition := range partitions {
start, err := cr.FetchOffsetByTime(topic, partition, proto.Earliest)
if err != nil {
return "", err
}
end, err := cr.FetchOffsetByTime(topic, partition, proto.Latest)
if err != nil {
return "", err
}
for offset := start; offset < end; {
messages, err := cr.Consume(topic, partition, offset)
if err != nil {
return "", err
}
if len(messages) == 0 {
break
}
for _, message := range messages {
obj := newObj()
if err := obj.UnmarshalBinary(message.Value); err != nil {
return "", err
}
jsonBuf, err := json.MarshalIndent(obj, "", "\t")
if err != nil {
return "", err
}
lines = append(lines, string(jsonBuf))
}
offset += messages[len(messages)-1].Offset + 1
}
}
sort.Strings(lines)
return strings.Join(lines, "\n"), nil
}
func (d formatDetector) detect() (Format, error) {
cl, topic := d.cl, d.topic
partitions, err := cl.Partitions(topic)
if err != nil {
return UnknownFormat, err
}
cr := consumer.New(cl)
for _, partition := range partitions {
offset, err := cr.FetchOffsetByTime(topic, partition, proto.Latest)
if err != nil {
continue
}
messages, err := cr.Consume(topic, partition, offset-1)
if err != nil {
continue
}
for _, msg := range messages {
return detectFormat(msg.Value)
}
}
return UnknownFormat, nil
}
func (cmd *OffsetCommand) Exec(cl model.Cluster) error {
partitions, err := cl.Partitions(cmd.Topic)
if err != nil {
return err
}
cr := consumer.New(cl)
fmt.Printf("topic: %s, group: %s\n", cmd.Topic, cmd.Group)
for _, partition := range partitions {
offset := int64(-1)
var err error
if cmd.Latest {
offset, err = cr.FetchOffsetByTime(cmd.Topic, partition, proto.Latest)
} else if cmd.Earliest {
offset, err = cr.FetchOffsetByTime(cmd.Topic, partition, proto.Earliest)
} else {
offset, err = cr.Offset(cmd.Topic, partition, cmd.Group)
}
if err != nil {
return err
}
fmt.Printf("\t%d:%d\n", partition, offset)
}
return nil
}