Ejemplo n.º 1
0
func main() {
	flag.StringVar(&myId, "id", "", "cluster id, typically a consuming service name")
	flag.StringVar(&topic, "topic", "", "topic to consume")
	flag.Parse()

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Printf("Unable to create cpuprofile file: %v, moving on\n", err)
		} else {
			pprof.StartCPUProfile(f)
			defer dumpCPUProfile()
			c := make(chan os.Signal, 1)
			signal.Notify(c, os.Interrupt)
			go func() {
				<-c
				dumpCPUProfile()
				os.Exit(0)
			}()
		}
	}

	log.V(2).Printf("Identified brokers: %v\n", kafkas)

	server, err := comm.Listen(":4987")
	if err != nil {
		log.Printf("Unable to listen on port :4987: %v", err)
		os.Exit(1)
	}

	_, consumers := newConsumer()

	for _, consumer := range consumers {
		go handleConsumer(consumer)
	}

	go func() {
		for message := range blackHole.inbox {
			log.V(2).Printf("Got message in blackhole: %v\n", message)
			go func(message messages.Message) {
				log.V(2).Printf("Gonna re-send message from blackhole: %v\n", message)
				nextRoundRobinClient().inbox <- message
			}(message)
		}
	}()

	log.Println("Listening on port :4987")

	for {
		conn, err := server.Accept()
		if err != nil {
			log.Printf("Unable to accept connection: %v", err)
			continue
		}

		go handleClient(conn)
	}
}
Ejemplo n.º 2
0
func (c *Consumer) listen() error {
	go func() {
		var err error
		c.stream, err = comm.Dial(c.fanouts[0])
		if err != nil {
			log.Printf("Unable to obtain stream: %v\n", err)
			return
		}

		defer c.stream.Close()

		err = c.stream.WriteLine(c.instanceId)
		if err != nil {
			log.Printf("Unable to send own instance id: %v\n", err)
			return
		}

		for {
			select {
			case _, ok := <-c.closeChan:
				if ok {
					return
				}
			default:
			}

			var message messages.Message
			err := c.stream.ReadMessage(&message)
			if err != nil {
				log.Printf("Unable to unmarshal message: %v\n", err)
				continue
			}

			go func(message messages.Message) {
				if c.SendAcks {
					ack := &messages.MessageAck{
						Partition: message.Partition,
						Offset:    message.Offset,
					}

					err := c.stream.WriteMessage(ack)
					if err != nil {
						log.Printf("Unable to send message ack: %v\n", err)
					}

					log.V(2).Printf("Sent ack from %s: %v\n", c.instanceId, *ack)
				}
			}(message)

			if !message.IsAckRequest {
				c.messages <- message
			}
		}
	}()

	return nil
}
Ejemplo n.º 3
0
func handleConsumer(consumer kafka.PartitionConsumer) {
	for message := range consumer.Messages() {
		log.V(2).Printf("Got message: %v\n", message)
		nextRoundRobinClient().inbox <- messages.Message{
			Value:        message.Value,
			Partition:    message.Partition,
			Offset:       message.Offset,
			IsAckRequest: false,
		}
	}
}
Ejemplo n.º 4
0
func handleClient(stream *comm.Stream) {
	defer stream.Close()
	var instanceId string

	instanceId, err := stream.ReadLine()
	if err != nil {
		log.Printf("Unable to identify client: %v\n", err)
		return
	}

	if instanceId == "-PING\n" {
		log.V(1).Printf("got ping: -PING; answering with: +PONG\n")
		err = stream.WriteLine("+PONG")
		if err != nil {
			log.Printf("Unable to answer with +PONG: %v\n", err)
		}
		return
	}

	log.V(1).Printf("got client: %s\n", instanceId)

	inbox := make(chan messages.Message, CHANNEL_BUFFER_SIZE)
	client := clientInbox{
		inbox: inbox,
		count: 1,
	}
	addClient(instanceId, client)

	ackErrors := make(chan error)

	go func() {
		for {
			var ack messages.MessageAck
			err = stream.ReadMessage(&ack)
			if err != nil {
				ackErrors <- err
				continue
			}

			if _, found := acks[ack]; found {
				log.V(2).Printf("Got ack: %v\n", ack)
				acks[ack] <- true
			}
		}
	}()

	dead := make(chan bool)
	for {
		select {
		case message := <-inbox:
			go sendMessageAndExpectAnAck(stream, dead, &message)

		case <-time.Tick(75 * time.Millisecond):
			go sendMessageAndExpectAnAck(stream, dead, ackRequest)

		case err := <-ackErrors:
			log.V(3).Printf("Got ack error: %v\n", err)
		case <-dead:
			log.V(2).Printf("Client %v disconnected\n", instanceId)
			clientDead(instanceId)
		}
	}
}