func (k *Kafka) Connect() error {
config := sarama.NewConfig()
config.Producer.RequiredAcks = sarama.RequiredAcks(k.RequiredAcks)
config.Producer.Compression = sarama.CompressionCodec(k.CompressionCodec)
config.Producer.Retry.Max = k.MaxRetry
// Legacy support ssl config
if k.Certificate != "" {
k.SSLCert = k.Certificate
k.SSLCA = k.CA
k.SSLKey = k.Key
}
tlsConfig, err := internal.GetTLSConfig(
k.SSLCert, k.SSLKey, k.SSLCA, k.InsecureSkipVerify)
if err != nil {
return err
}
if tlsConfig != nil {
config.Net.TLS.Config = tlsConfig
config.Net.TLS.Enable = true
}
producer, err := sarama.NewSyncProducer(k.Brokers, config)
if err != nil {
return err
}
k.producer = producer
return nil
}
// a kafka message:
// crc magic attributes keyLen key messageLen message
func (this *Segment) readSegment(filename string) {
f, err := os.Open(filename) // readonly
swallow(err)
defer f.Close()
const (
maxKeySize = 10 << 10
maxValSize = 2 << 20
)
var (
buf = make([]byte, 12)
key = make([]byte, maxKeySize)
val = make([]byte, maxValSize)
msgN int64
firstOffset uint64 = math.MaxUint64 // sentry
endOffset uint64
)
r := bufio.NewReader(f)
for {
_, err := r.Read(buf)
if err != nil {
if err == io.EOF {
break
} else {
panic(err)
}
}
// offset, size, crc32, magic, attrs, key-len, key-content, msg-len, msg-content
// crc32 = crc32(magic, attrs, key-len, key-content, msg-len, msg-content)
// offset+size 8+4
offset := binary.BigEndian.Uint64(buf[:8])
size := binary.BigEndian.Uint32(buf[8:12])
// crc32+magic+attrs+keySize[key] 4+1+1+4
r.Read(buf[0:10])
attr := buf[5]
keySize := binary.BigEndian.Uint32(buf[6:10])
if keySize > 0 && keySize != math.MaxUint32 {
_, err = r.Read(key[:keySize])
swallow(err)
}
// valSize[val] 4
_, err = r.Read(buf[:4])
swallow(err)
valSize := binary.BigEndian.Uint32(buf[:4])
if valSize > 0 {
_, err = r.Read(val[:valSize])
swallow(err)
}
switch sarama.CompressionCodec(attr) {
case sarama.CompressionNone:
fmt.Printf("offset:%d size:%d %s\n", offset, size, string(val[:valSize]))
case sarama.CompressionGZIP:
reader, err := gzip.NewReader(bytes.NewReader(val[:valSize]))
swallow(err)
v, err := ioutil.ReadAll(reader)
swallow(err)
fmt.Printf("offset:%d size:%d gzip %s\n", offset, size, string(v))
case sarama.CompressionSnappy:
v, err := this.snappyDecode(val[:valSize])
swallow(err)
fmt.Printf("offset:%d size:%d snappy %s\n", offset, size, string(v))
}
if firstOffset == math.MaxUint64 {
firstOffset = offset
}
endOffset = offset
msgN++
}
fmt.Printf("Total Messages: %d, %d - %d\n", msgN, firstOffset, endOffset)
}