func MustNewKafkaBackend(name string, bconf *config.Transport_kafka) *kafkaBackend {
logging.Infof("MustNewKafkaBackend: %+v", bconf)
_kconf := sarama.NewConfig()
_kconf.Net.DialTimeout = newcore.Interval(bconf.Dail_timeout).MustDuration(time.Second * 5)
_kconf.Net.WriteTimeout = newcore.Interval(bconf.Write_timeout).MustDuration(time.Second * 1)
_kconf.Net.ReadTimeout = time.Second * 10
_kconf.Net.KeepAlive = newcore.Interval(bconf.Keepalive).MustDuration(time.Second * 30)
if bconf.Ack_timeout_ms <= 0 {
_kconf.Producer.Timeout = time.Millisecond * 100
} else {
_kconf.Producer.Timeout = time.Millisecond * time.Duration(bconf.Ack_timeout_ms)
}
if bconf.Flush_frequency_ms <= 0 {
_kconf.Producer.Flush.Frequency = time.Millisecond * 100
} else {
_kconf.Producer.Flush.Frequency = time.Millisecond * time.Duration(bconf.Flush_frequency_ms)
}
cc := strings.ToLower(bconf.Compression_codec)
switch {
case cc == "none":
_kconf.Producer.Compression = sarama.CompressionNone
case cc == "gzip":
_kconf.Producer.Compression = sarama.CompressionGZIP // Compress messages
case cc == "snappy":
_kconf.Producer.Compression = sarama.CompressionSnappy // Compress messages
default:
_kconf.Producer.Compression = sarama.CompressionNone
}
ra := strings.ToLower(bconf.Required_acks)
switch {
case ra == "no_response":
_kconf.Producer.RequiredAcks = sarama.NoResponse
case ra == "wait_for_local":
_kconf.Producer.RequiredAcks = sarama.WaitForLocal
case ra == "wait_for_all":
_kconf.Producer.RequiredAcks = sarama.WaitForAll
default:
_kconf.Producer.RequiredAcks = sarama.NoResponse
}
logging.Debugf("kafka conf: %+v", _kconf)
s := &kafkaBackend{
name: name,
closing: make(chan chan error),
updates: make(chan newcore.MultiDataPoint),
conf: bconf, // backend config
kconf: _kconf, // sarama config
}
go s.loop()
return s
}
func NewKafkaSubscription(opts Config_KafkaSubscription) (*kafka_subscription, error) {
var max_batch_size = 100
var state_file_name string
if opts.Max_batch_size > 0 {
max_batch_size = opts.Max_batch_size
}
if opts.Name == "" {
return nil, fmt.Errorf("name should not be empty")
}
if opts.Topic == "" {
return nil, fmt.Errorf("topic should not be empty")
}
state_file_name = filepath.Join(config.SHARED_DIR, fmt.Sprintf("%s_%s.state", opts.Name, opts.Topic))
c := &kafka_subscription{
opts: opts,
updates: make(chan newcore.MultiDataPoint), // for Updates
closing: make(chan chan error), // for Close
flush_interval: newcore.Interval(opts.Flush_interval).MustDuration(time.Millisecond * 100),
max_batch_size: max_batch_size,
state_file_name: state_file_name,
state: newKafkaSubState(state_file_name),
}
go c.loop()
return c, nil
}
func MustNewWinHickwallMemCollector(interval string, tags newcore.TagSet) *win_pdh_collector {
opts := config.Config_win_pdh_collector{
Interval: newcore.Interval(interval),
Tags: tags,
Queries: []config.Config_win_pdh_query{
{
Query: "\\Process(hickwall)\\Working Set - Private",
Metric: "private_working_set.bytes",
},
},
}
return MustNewWinPdhCollector("hickwall_mem", "hickwall.client.mem", opts)
}
func new_core_from_registry(stop chan error) {
logging.Debug("new_core_from_registry started")
if stop == nil {
logging.Panic("stop chan is nil")
}
if len(config.CoreConf.Registry_urls) <= 0 {
logging.Panic("RegistryURLs is empty!!")
}
resp, err := load_reg_response()
if err != nil {
logging.Errorf("we don't have a valid registry info cached.")
next := time.After(0)
// round robin registry machines
r := ring.New(len(config.CoreConf.Registry_urls))
for i := 0; i < r.Len(); i++ {
r.Value = config.CoreConf.Registry_urls[i]
r = r.Next()
}
registry_loop:
for {
select {
case <-next:
r = r.Next()
resp, err = do_registry(r.Value.(string))
if err == nil {
logging.Info("we are registry we got a valid registry response.")
break registry_loop
} else {
logging.Errorf("failed to registry: %v", err)
}
next = time.After(newcore.Interval(config.CoreConf.Registry_delay_on_error).MustDuration(time.Minute))
}
}
}
// TODO: handle error here. like etcd_machines are not working.
// here we got a valid registry info. get config and start to run.
new_core_from_etcd(resp.EtcdMachines, resp.EtcdConfigPath, stop)
}