Esempio n. 1
0
func MustNewKafkaBackend(name string, bconf *config.Transport_kafka) *kafkaBackend {
	logging.Infof("MustNewKafkaBackend: %+v", bconf)
	_kconf := sarama.NewConfig()

	_kconf.Net.DialTimeout = newcore.Interval(bconf.Dail_timeout).MustDuration(time.Second * 5)
	_kconf.Net.WriteTimeout = newcore.Interval(bconf.Write_timeout).MustDuration(time.Second * 1)
	_kconf.Net.ReadTimeout = time.Second * 10
	_kconf.Net.KeepAlive = newcore.Interval(bconf.Keepalive).MustDuration(time.Second * 30)

	if bconf.Ack_timeout_ms <= 0 {
		_kconf.Producer.Timeout = time.Millisecond * 100
	} else {
		_kconf.Producer.Timeout = time.Millisecond * time.Duration(bconf.Ack_timeout_ms)
	}

	if bconf.Flush_frequency_ms <= 0 {
		_kconf.Producer.Flush.Frequency = time.Millisecond * 100
	} else {
		_kconf.Producer.Flush.Frequency = time.Millisecond * time.Duration(bconf.Flush_frequency_ms)
	}

	cc := strings.ToLower(bconf.Compression_codec)
	switch {
	case cc == "none":
		_kconf.Producer.Compression = sarama.CompressionNone
	case cc == "gzip":
		_kconf.Producer.Compression = sarama.CompressionGZIP // Compress messages
	case cc == "snappy":
		_kconf.Producer.Compression = sarama.CompressionSnappy // Compress messages
	default:
		_kconf.Producer.Compression = sarama.CompressionNone
	}

	ra := strings.ToLower(bconf.Required_acks)
	switch {
	case ra == "no_response":
		_kconf.Producer.RequiredAcks = sarama.NoResponse
	case ra == "wait_for_local":
		_kconf.Producer.RequiredAcks = sarama.WaitForLocal
	case ra == "wait_for_all":
		_kconf.Producer.RequiredAcks = sarama.WaitForAll
	default:
		_kconf.Producer.RequiredAcks = sarama.NoResponse
	}

	logging.Debugf("kafka conf: %+v", _kconf)

	s := &kafkaBackend{
		name:    name,
		closing: make(chan chan error),
		updates: make(chan newcore.MultiDataPoint),
		conf:    bconf,  // backend config
		kconf:   _kconf, // sarama config
	}
	go s.loop()
	return s
}
Esempio n. 2
0
func NewKafkaSubscription(opts Config_KafkaSubscription) (*kafka_subscription, error) {
	var max_batch_size = 100
	var state_file_name string

	if opts.Max_batch_size > 0 {
		max_batch_size = opts.Max_batch_size
	}

	if opts.Name == "" {
		return nil, fmt.Errorf("name should not be empty")
	}
	if opts.Topic == "" {
		return nil, fmt.Errorf("topic should not be empty")
	}
	state_file_name = filepath.Join(config.SHARED_DIR, fmt.Sprintf("%s_%s.state", opts.Name, opts.Topic))

	c := &kafka_subscription{
		opts:            opts,
		updates:         make(chan newcore.MultiDataPoint), // for Updates
		closing:         make(chan chan error),             // for Close
		flush_interval:  newcore.Interval(opts.Flush_interval).MustDuration(time.Millisecond * 100),
		max_batch_size:  max_batch_size,
		state_file_name: state_file_name,
		state:           newKafkaSubState(state_file_name),
	}

	go c.loop()
	return c, nil
}
Esempio n. 3
0
func MustNewWinHickwallMemCollector(interval string, tags newcore.TagSet) *win_pdh_collector {
	opts := config.Config_win_pdh_collector{
		Interval: newcore.Interval(interval),
		Tags:     tags,
		Queries: []config.Config_win_pdh_query{
			{
				Query:  "\\Process(hickwall)\\Working Set - Private",
				Metric: "private_working_set.bytes",
			},
		},
	}
	return MustNewWinPdhCollector("hickwall_mem", "hickwall.client.mem", opts)
}
Esempio n. 4
0
func new_core_from_registry(stop chan error) {
	logging.Debug("new_core_from_registry started")
	if stop == nil {
		logging.Panic("stop chan is nil")
	}

	if len(config.CoreConf.Registry_urls) <= 0 {
		logging.Panic("RegistryURLs is empty!!")
	}

	resp, err := load_reg_response()
	if err != nil {
		logging.Errorf("we don't have a valid registry info cached.")
		next := time.After(0)

		// round robin registry machines
		r := ring.New(len(config.CoreConf.Registry_urls))
		for i := 0; i < r.Len(); i++ {
			r.Value = config.CoreConf.Registry_urls[i]
			r = r.Next()
		}

	registry_loop:
		for {
			select {
			case <-next:
				r = r.Next()
				resp, err = do_registry(r.Value.(string))
				if err == nil {
					logging.Info("we are registry we got a valid registry response.")
					break registry_loop
				} else {
					logging.Errorf("failed to registry: %v", err)
				}
				next = time.After(newcore.Interval(config.CoreConf.Registry_delay_on_error).MustDuration(time.Minute))
			}
		}
	}

	// TODO: handle error here. like etcd_machines are not working.
	// here we got a valid registry info. get config and start to run.
	new_core_from_etcd(resp.EtcdMachines, resp.EtcdConfigPath, stop)
}