Пример #1
1
func (k *Kafka) Connect() error {
	config := sarama.NewConfig()

	config.Producer.RequiredAcks = sarama.RequiredAcks(k.RequiredAcks)
	config.Producer.Compression = sarama.CompressionCodec(k.CompressionCodec)
	config.Producer.Retry.Max = k.MaxRetry

	// Legacy support ssl config
	if k.Certificate != "" {
		k.SSLCert = k.Certificate
		k.SSLCA = k.CA
		k.SSLKey = k.Key
	}

	tlsConfig, err := internal.GetTLSConfig(
		k.SSLCert, k.SSLKey, k.SSLCA, k.InsecureSkipVerify)
	if err != nil {
		return err
	}

	if tlsConfig != nil {
		config.Net.TLS.Config = tlsConfig
		config.Net.TLS.Enable = true
	}

	producer, err := sarama.NewSyncProducer(k.Brokers, config)
	if err != nil {
		return err
	}
	k.producer = producer
	return nil
}
Пример #2
0
// a kafka message:
// crc magic attributes keyLen key messageLen message
func (this *Segment) readSegment(filename string) {
	f, err := os.Open(filename) // readonly
	swallow(err)
	defer f.Close()

	const (
		maxKeySize = 10 << 10
		maxValSize = 2 << 20
	)

	var (
		buf = make([]byte, 12)
		key = make([]byte, maxKeySize)
		val = make([]byte, maxValSize)

		msgN        int64
		firstOffset uint64 = math.MaxUint64 // sentry
		endOffset   uint64
	)
	r := bufio.NewReader(f)
	for {
		_, err := r.Read(buf)
		if err != nil {
			if err == io.EOF {
				break
			} else {
				panic(err)
			}
		}

		// offset, size, crc32, magic, attrs, key-len, key-content, msg-len, msg-content
		// crc32 = crc32(magic, attrs, key-len, key-content, msg-len, msg-content)

		// offset+size 8+4
		offset := binary.BigEndian.Uint64(buf[:8])
		size := binary.BigEndian.Uint32(buf[8:12])

		// crc32+magic+attrs+keySize[key] 4+1+1+4
		r.Read(buf[0:10])

		attr := buf[5]
		keySize := binary.BigEndian.Uint32(buf[6:10])
		if keySize > 0 && keySize != math.MaxUint32 {
			_, err = r.Read(key[:keySize])
			swallow(err)
		}

		// valSize[val] 4
		_, err = r.Read(buf[:4])
		swallow(err)
		valSize := binary.BigEndian.Uint32(buf[:4])
		if valSize > 0 {
			_, err = r.Read(val[:valSize])
			swallow(err)
		}

		switch sarama.CompressionCodec(attr) {
		case sarama.CompressionNone:
			fmt.Printf("offset:%d size:%d %s\n", offset, size, string(val[:valSize]))

		case sarama.CompressionGZIP:
			reader, err := gzip.NewReader(bytes.NewReader(val[:valSize]))
			swallow(err)
			v, err := ioutil.ReadAll(reader)
			swallow(err)

			fmt.Printf("offset:%d size:%d gzip %s\n", offset, size, string(v))

		case sarama.CompressionSnappy:
			v, err := this.snappyDecode(val[:valSize])
			swallow(err)

			fmt.Printf("offset:%d size:%d snappy %s\n", offset, size, string(v))
		}

		if firstOffset == math.MaxUint64 {
			firstOffset = offset
		}
		endOffset = offset
		msgN++
	}

	fmt.Printf("Total Messages: %d, %d - %d\n", msgN, firstOffset, endOffset)
}