Пример #1
0
func (isrv *indexerServer) IndexMessage(ctx context.Context, msg *pb.Message) (*pb.Response, error) {
	statIncrementCnt(&stat.msgCnt)

	tags := msg.Header.GetTags()
	if len(tags) > maxTagsInMessage {
		return nil, fmt.Errorf("too many tags in message %d", len(tags))
	}

	for _, tag := range tags {
		if len(tag) > maxTagLength {
			return nil, fmt.Errorf("too long tag name %d", len(tag))
		}
	}

	frames := msg.GetFrames()
	for i := 0; i < len(frames)-1; i++ {
		if frames[i].GetName() > frames[i+1].GetName() {
			return nil, fmt.Errorf("frames must be sorted")
		}
	}

	start := time.Now()
	value, err := msg.Marshal()
	statIncrementTook(&stat.msgSerializeTook, start)
	statIncrementSize(&stat.msgSendToKafkaSize, len(value))

	if err != nil {
		return nil, fmt.Errorf("failed to marshal message: %v", err)
	}

	start = time.Now()
	// partition, offset := int32(0), int64(0)
	partition, offset, err := isrv.producer.SendMessage(&sarama.ProducerMessage{
		Topic: persistedTopic,
		Value: sarama.ByteEncoder(value),
	})

	statIncrementTook(&stat.msgSendToKafkaTook, start)

	if err != nil {
		return nil, fmt.Errorf("failed to store message: %v", err)
	}

	start = time.Now()
	isrv.appendIndexCh <- &indexBuilderMessage{
		tags:      tags,
		partition: partition,
		offset:    offset,
	}

	statIncrementTook(&stat.msgSendToChTook, start)
	return &pb.Response{}, nil
}
Пример #2
0
func main() {
	var proc = flag.Int("proc", 8, "parallelizm")
	var srv = flag.String("srv", "localhost:8004", "destination host:port")
	var path = flag.String("path", ".", "path_to_srl_files")
	flag.Parse()

	if !flag.Parsed() {
		flag.PrintDefaults()
		os.Exit(1)
	}

	hosts := strings.Split(*srv, ",")
	files, _ := filepath.Glob(*path + "/*.srl")
	log.Printf("reading %d files", len(files))

	var content [][]byte
	for _, file := range files {
		c, err := ioutil.ReadFile(file)
		if err != nil {
			log.Fatal("failed to read file", file, err)
		}

		content = append(content, c)
	}

	var total, bytes int64
	go func(ch <-chan time.Time) {
		for {
			<-ch
			t := atomic.SwapInt64(&total, 0)
			b := atomic.SwapInt64(&bytes, 0)
			log.Printf("sent %d %d bytes", t, b)
		}
	}(time.Tick(time.Second))

	var wg sync.WaitGroup
	send := func() {
		defer wg.Done()
		src := rand.NewSource(time.Now().UnixNano())
		rnd := rand.New(src)

		var clients []pb.IndexerClient
		for _, host := range hosts {
			log.Println("connection to", host)
			conn, err := grpc.Dial(host, grpc.WithInsecure())
			defer conn.Close()
			if err != nil {
				log.Fatal("failed to connect", err)
			}

			clients = append(clients, pb.NewIndexerClient(conn))
		}

		var msg pb.Message
		var frame pb.Frame
		msg.Header = &pb.Header{Tags: []string{"a", "b", "c", "d", "e", "f"}}
		msg.Frames = append(msg.Frames, &frame)

		for {
			for _ = range content {
				c := content[rnd.Intn(len(content))]
				client := clients[rnd.Intn(len(hosts))]

				frame.Data = c
				if _, err := client.IndexMessage(context.Background(), &msg); err != nil {
					log.Println("failed to send message", err)
				}

				atomic.AddInt64(&total, 1)
				atomic.AddInt64(&bytes, int64(len(c)))
			}
		}
	}

	log.Printf("spawning %d goroutines", *proc)
	for i := 0; i < *proc; i++ {
		wg.Add(1)
		go send()
	}

	wg.Wait()
}