func (isrv *indexerServer) IndexMessage(ctx context.Context, msg *pb.Message) (*pb.Response, error) {
statIncrementCnt(&stat.msgCnt)
tags := msg.Header.GetTags()
if len(tags) > maxTagsInMessage {
return nil, fmt.Errorf("too many tags in message %d", len(tags))
}
for _, tag := range tags {
if len(tag) > maxTagLength {
return nil, fmt.Errorf("too long tag name %d", len(tag))
}
}
frames := msg.GetFrames()
for i := 0; i < len(frames)-1; i++ {
if frames[i].GetName() > frames[i+1].GetName() {
return nil, fmt.Errorf("frames must be sorted")
}
}
start := time.Now()
value, err := msg.Marshal()
statIncrementTook(&stat.msgSerializeTook, start)
statIncrementSize(&stat.msgSendToKafkaSize, len(value))
if err != nil {
return nil, fmt.Errorf("failed to marshal message: %v", err)
}
start = time.Now()
// partition, offset := int32(0), int64(0)
partition, offset, err := isrv.producer.SendMessage(&sarama.ProducerMessage{
Topic: persistedTopic,
Value: sarama.ByteEncoder(value),
})
statIncrementTook(&stat.msgSendToKafkaTook, start)
if err != nil {
return nil, fmt.Errorf("failed to store message: %v", err)
}
start = time.Now()
isrv.appendIndexCh <- &indexBuilderMessage{
tags: tags,
partition: partition,
offset: offset,
}
statIncrementTook(&stat.msgSendToChTook, start)
return &pb.Response{}, nil
}
func main() {
var proc = flag.Int("proc", 8, "parallelizm")
var srv = flag.String("srv", "localhost:8004", "destination host:port")
var path = flag.String("path", ".", "path_to_srl_files")
flag.Parse()
if !flag.Parsed() {
flag.PrintDefaults()
os.Exit(1)
}
hosts := strings.Split(*srv, ",")
files, _ := filepath.Glob(*path + "/*.srl")
log.Printf("reading %d files", len(files))
var content [][]byte
for _, file := range files {
c, err := ioutil.ReadFile(file)
if err != nil {
log.Fatal("failed to read file", file, err)
}
content = append(content, c)
}
var total, bytes int64
go func(ch <-chan time.Time) {
for {
<-ch
t := atomic.SwapInt64(&total, 0)
b := atomic.SwapInt64(&bytes, 0)
log.Printf("sent %d %d bytes", t, b)
}
}(time.Tick(time.Second))
var wg sync.WaitGroup
send := func() {
defer wg.Done()
src := rand.NewSource(time.Now().UnixNano())
rnd := rand.New(src)
var clients []pb.IndexerClient
for _, host := range hosts {
log.Println("connection to", host)
conn, err := grpc.Dial(host, grpc.WithInsecure())
defer conn.Close()
if err != nil {
log.Fatal("failed to connect", err)
}
clients = append(clients, pb.NewIndexerClient(conn))
}
var msg pb.Message
var frame pb.Frame
msg.Header = &pb.Header{Tags: []string{"a", "b", "c", "d", "e", "f"}}
msg.Frames = append(msg.Frames, &frame)
for {
for _ = range content {
c := content[rnd.Intn(len(content))]
client := clients[rnd.Intn(len(hosts))]
frame.Data = c
if _, err := client.IndexMessage(context.Background(), &msg); err != nil {
log.Println("failed to send message", err)
}
atomic.AddInt64(&total, 1)
atomic.AddInt64(&bytes, int64(len(c)))
}
}
}
log.Printf("spawning %d goroutines", *proc)
for i := 0; i < *proc; i++ {
wg.Add(1)
go send()
}
wg.Wait()
}