func (z *zipkin) run(ch chan bool, p sarama.SyncProducer) {
t := time.NewTicker(z.opts.BatchInterval)
var buf []*trace.Span
for {
select {
case s := <-z.spans:
buf = append(buf, s)
if len(buf) >= z.opts.BatchSize {
go z.send(buf, p)
buf = nil
}
case <-t.C:
// flush
if len(buf) > 0 {
go z.send(buf, p)
buf = nil
}
case <-ch:
// exit
t.Stop()
p.Close()
return
}
}
}
func main() {
var cmd *exec.Cmd
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
go func() {
for {
select {
case sig := <-sigs:
cmd.Process.Signal(sig)
}
}
}()
var cfg config
if err := gofigure.Gofigure(&cfg); err != nil {
fmt.Fprintln(os.Stderr, "unexpected error configuring ktee")
os.Exit(1)
}
var err error
var producer sarama.SyncProducer
if len(cfg.Brokers) > 0 {
brokers := strings.Split(cfg.Brokers, ",")
producer, err = sarama.NewSyncProducer(brokers, sarama.NewConfig())
if err != nil {
fmt.Fprintf(os.Stderr, "error connecting to Kafka brokers: %s\n", err)
os.Exit(1)
}
defer func() {
producer.Close()
}()
}
args := os.Args[1:]
if len(args) == 0 {
fmt.Fprintln(os.Stderr, "usage: ktee args")
os.Exit(1)
}
kwOut := kafkaWriter{producer, os.Stdout, cfg.OutTopic, new(bytes.Buffer), make(chan sarama.ProducerMessage)}
kwErr := kafkaWriter{producer, os.Stderr, cfg.ErrTopic, new(bytes.Buffer), make(chan sarama.ProducerMessage)}
defer func() {
kwOut.Flush()
kwErr.Flush()
}()
cmd = exec.Command(args[0], args[1:]...)
cmd.Stdin = os.Stdin
cmd.Stdout = kwOut
cmd.Stderr = kwErr
cmd.Env = os.Environ()
err = cmd.Run()
if err != nil {
switch err.(type) {
case *exec.ExitError:
fmt.Fprintf(os.Stderr, "non-zero exit code: %s\n", err)
if status, ok := err.(*exec.ExitError).Sys().(syscall.WaitStatus); ok {
os.Exit(status.ExitStatus())
}
os.Exit(1)
default:
fmt.Fprintf(os.Stderr, "error executing command: %s\n", err)
os.Exit(1)
}
}
}
func PublishSync(input chan *FileEvent, source string, isRetryer bool) {
log.Debug("publishSync loop")
clientConfig := sarama.NewConfig()
clientConfig.Producer.RequiredAcks = sarama.WaitForAll
clientConfig.Producer.Compression = sarama.CompressionSnappy
clientConfig.Producer.Partitioner = sarama.NewRoundRobinPartitioner
clientConfig.Producer.Retry.Max = 10
topic := kafkaTopic
key := hashKey
if isRetryer {
topic = retryTopic
}
//brokerList := []string{"127.0.0.1:9092"}
var producer sarama.SyncProducer
var err error
for {
producer, err = sarama.NewSyncProducer(brokerList, clientConfig)
if err != nil {
log.Error("Sync: Failed to start Sarama producer: ", err)
log.Info("waiting...")
time.Sleep(1 * time.Second)
} else {
break
}
}
defer func() {
if err := producer.Close(); err != nil {
log.Error("Failed to shutdown producer cleanly", err)
}
}()
// if retryer, use retryer backup, others use Registrar
var recorder Recorder
if isRetryer {
// set to global retryer
retryRecorder := &RetryRecorder{file: mainRetryer.vernier}
recorder = retryRecorder
defer retryRecorder.file.Close()
} else {
registrar := &Registrar{source: source, dir: REGISTRAR_DIR}
if _, err := registrar.OpenRecord(registrar.dir); err != nil {
log.Error("PublishSync open record failed, error:", err)
os.Exit(2)
}
recorder = registrar
}
genMessage := func(rawMessage string) string {
return rawMessage
}
// retryer message sample: 0 this is a sample message
// 0 means, haven't retried succeed
// 1 means have been sended
if isRetryer {
genMessage = func(rawMessage string) string {
// 0|1 raw_name_of_log_file log_msg
rawMessage = rawMessage[2:]
idx := strings.Index(rawMessage, " ")
return rawMessage[idx+1:]
}
}
for event := range input {
log.Debugf("%v, %v, %v, %v\n", *event.Source, *event.Text, event.Line, event.Offset)
// if failed, retry send messge until succeed
key = strconv.FormatInt(event.Offset, 10)
rawMessage := *event.Text
if isRetryer {
if retryTopic != kafkaTopic {
topic = retryTopic
} else {
baseName := getSourceName(rawMessage)
if len(topicmap) > 0 {
tmpTopic := genTopic(baseName, topicmap)
if tmpTopic != "" {
topic = tmpTopic
}
}
}
}
message := genMessage(*event.Text)
if rawMessage[0] == '1' {
log.Debug("message[%s] have been seeded", rawMessage)
continue
}
for {
partition, offset, err := producer.SendMessage(&sarama.ProducerMessage{
Topic: topic,
Key: sarama.StringEncoder(key),
Value: sarama.StringEncoder(message),
})
if err != nil {
log.Errorf("Failed: %s, %d, %d\n", *event.Source, event.Line, event.Offset)
time.Sleep(3 * time.Second)
} else {
log.Debugf("OK: %d, %d, %s\n", partition, offset, *event.Source)
recorder.RecordSucceed(event.Offset, event.RawBytes)
break
}
}
}
}
// Close Producer
func closeProducer(producer sarama.SyncProducer) {
err := producer.Close()
if err != nil {
logger.Printf("Error closing Producer: %s\n", err)
}
}