func protect(h httprouter.Handle, expire time.Duration, trigger string) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
secure := false
if trigger == "read" && config.CoreConf.Secure_api_read {
secure = true
} else if trigger == "write" && config.CoreConf.Secure_api_write {
secure = true
}
logging.Infof("trigger: %s, secure: %v, write: %v, read: %v\n", trigger, secure, config.CoreConf.Secure_api_write, config.CoreConf.Secure_api_read)
if secure {
hostname := r.URL.Query().Get("hostname")
if strings.ToLower(hostname) != newcore.GetHostname() {
logging.Errorf("hostname mismatch: %v", hostname)
http.Error(w, "hostname mismatch", 500)
return
}
time_str := r.URL.Query().Get("time")
tm, err := utils.UTCTimeFromUnixStr(time_str)
if err != nil {
logging.Errorf("invalid time: %v", time_str)
http.Error(w, "Invalid Time", 500)
return
}
if time.Now().Sub(tm) > expire {
// expired reqeust
logging.Errorf("expired request: %v", time.Now().Sub(tm))
http.Error(w, "expired request", 500)
return
}
// we need to verify request.
// request should put signature of this agent hostname into header HICKWALL_ADMIN_SIGN
load_unsigner()
signed_str := r.Header.Get("HICKWALL_ADMIN_SIGN")
signed, err := base64.StdEncoding.DecodeString(signed_str)
if err != nil {
logging.Error("cannot decode sign")
http.Error(w, "cannot decode sign", 500)
return
}
toSign := fmt.Sprintf("%s%s", hostname, time_str)
logging.Trace("unsign started")
err = unsigner.Unsign([]byte(toSign), signed)
logging.Trace("unsign finished")
if err != nil {
logging.Errorf("-> invalid signature: %v <-", string(signed))
http.Error(w, "invalid signature", 500)
return
}
}
h(w, r, ps)
}
}
func serve_api() {
router := httprouter.New()
router.GET("/sys_info", protect_read(serveSysInfo, time.Second))
router.DELETE("/registry/revoke", protect_write(serveRegistryRevoke, time.Second))
// router.GET("/registry", serveRegistryGet)
// router.POST("/registry/accept", serveRegistryAccept)
// router.PUT("/registry/renew", serveRegistryRenew)
// router.PUT("/config/renew", serveConfigRenew)
addr := ":3031"
if config.CoreConf.Listen_port > 0 {
addr = fmt.Sprintf(":%d", config.CoreConf.Listen_port)
}
logging.Infof("api served at: %s", addr)
api_srv_running = true
err := http.ListenAndServe(addr, router)
api_srv_running = false
if err != nil {
logging.Criticalf("api server is not running!: %v", err)
} else {
logging.Info("api server stopped")
}
}
func MustNewKafkaBackend(name string, bconf *config.Transport_kafka) *kafkaBackend {
logging.Infof("MustNewKafkaBackend: %+v", bconf)
_kconf := sarama.NewConfig()
_kconf.Net.DialTimeout = newcore.Interval(bconf.Dail_timeout).MustDuration(time.Second * 5)
_kconf.Net.WriteTimeout = newcore.Interval(bconf.Write_timeout).MustDuration(time.Second * 1)
_kconf.Net.ReadTimeout = time.Second * 10
_kconf.Net.KeepAlive = newcore.Interval(bconf.Keepalive).MustDuration(time.Second * 30)
if bconf.Ack_timeout_ms <= 0 {
_kconf.Producer.Timeout = time.Millisecond * 100
} else {
_kconf.Producer.Timeout = time.Millisecond * time.Duration(bconf.Ack_timeout_ms)
}
if bconf.Flush_frequency_ms <= 0 {
_kconf.Producer.Flush.Frequency = time.Millisecond * 100
} else {
_kconf.Producer.Flush.Frequency = time.Millisecond * time.Duration(bconf.Flush_frequency_ms)
}
cc := strings.ToLower(bconf.Compression_codec)
switch {
case cc == "none":
_kconf.Producer.Compression = sarama.CompressionNone
case cc == "gzip":
_kconf.Producer.Compression = sarama.CompressionGZIP // Compress messages
case cc == "snappy":
_kconf.Producer.Compression = sarama.CompressionSnappy // Compress messages
default:
_kconf.Producer.Compression = sarama.CompressionNone
}
ra := strings.ToLower(bconf.Required_acks)
switch {
case ra == "no_response":
_kconf.Producer.RequiredAcks = sarama.NoResponse
case ra == "wait_for_local":
_kconf.Producer.RequiredAcks = sarama.WaitForLocal
case ra == "wait_for_all":
_kconf.Producer.RequiredAcks = sarama.WaitForAll
default:
_kconf.Producer.RequiredAcks = sarama.NoResponse
}
logging.Debugf("kafka conf: %+v", _kconf)
s := &kafkaBackend{
name: name,
closing: make(chan chan error),
updates: make(chan newcore.MultiDataPoint),
conf: bconf, // backend config
kconf: _kconf, // sarama config
}
go s.loop()
return s
}
func Test_strategy_etcd_LoadConfigStrategyEtcd_Nil(t *testing.T) {
logging.SetLevel("debug")
defer close_core()
stopCh := make(chan error)
for idx, tcase := range nil_core_tests {
request_cnt := 0
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
request_cnt += 1
logging.Infof("case: %d, -- we got request: cnt:%d ------------\n", idx, request_cnt)
// pretty.Println(r)
iswait := r.URL.Query().Get("wait")
if strings.ToLower(iswait) == "true" {
logging.Info("watching")
// watch, long polling
// time.Sleep(time.Second * 1)
} else {
logging.Info("getting")
}
v, _ := _fack_etcd_respose(1, tcase)
fmt.Printf("case: %d, content: %s\n", idx, v)
fmt.Fprintln(w, v)
}))
go new_core_from_etcd([]string{ts.URL}, "/config/host/DST54869.yml", stopCh)
tick := time.After(time.Second * 1)
timeout := time.After(time.Second * 2)
main_loop:
for {
select {
case <-tick:
stopCh <- nil
if the_core != nil {
t.Error("the_core has been created with invalid configuration!")
return
} else {
t.Log("test case successed")
ts.Close()
break main_loop
}
case <-timeout:
t.Errorf("timed out. somethings is blocking")
return
}
}
}
}
func UseConfigCreateCollectors(rconf *config.RuntimeConfig) ([]newcore.Collector, error) {
var clrs []newcore.Collector
var prefixs = make(map[string]bool)
for gid, group := range rconf.Groups {
logging.Infof("gid: %d, prefix: %s\n", gid, group.Prefix)
if len(group.Prefix) <= 0 {
return nil, fmt.Errorf("group (idx:%d) prefix is empty.", gid)
} else {
_, exists := prefixs[group.Prefix]
if exists == false {
prefixs[group.Prefix] = true
} else {
return nil, fmt.Errorf("duplicated group prefix: %s", group.Prefix)
}
}
for cid, conf := range group.Collector_ping {
pings := MustNewPingCollectors(gen_collector_name(gid, cid, "ping"), group.Prefix, conf)
for _, c := range pings {
clrs = append(clrs, c)
}
}
for cid, conf := range group.Collector_win_pdh {
c := windows.MustNewWinPdhCollector(gen_collector_name(gid, cid, "pdh"), group.Prefix, conf)
clrs = append(clrs, c)
}
for cid, conf := range group.Collector_win_wmi {
c := windows.MustNewWinWmiCollector(gen_collector_name(gid, cid, "wmi"), group.Prefix, conf)
clrs = append(clrs, c)
}
if group.Collector_win_sys != nil {
cs := windows.MustNewWinSysCollectors(gen_collector_name(gid, 0, "win_sys"), group.Prefix, group.Collector_win_sys)
for _, c := range cs {
clrs = append(clrs, c)
}
// clrs = append(clrs, cs...)
}
}
logging.Debugf("rconf.Client.Metric_Enabled: %v, rconf.Client.Metric_Interval: %v",
rconf.Client.Metric_Enabled, rconf.Client.Metric_Interval)
if rconf.Client.Metric_Enabled == true {
clrs = append(clrs, MustNewHickwallCollector(rconf.Client.Metric_Interval))
clrs = append(clrs, windows.MustNewWinHickwallMemCollector(rconf.Client.Metric_Interval, rconf.Client.Tags))
}
return clrs[:], nil
}
func (b *kafkaBackend) connect() error {
producer, err := sarama.NewAsyncProducer(b.conf.Broker_list, b.kconf)
if err != nil {
logging.Errorf("failed to start producer: %v, %v", err, b.conf.Broker_list)
return fmt.Errorf("failed to start producer: %v, %v", err, b.conf.Broker_list)
}
go func() {
logging.Debug("consuming from producer.Errors()")
for err := range producer.Errors() {
logging.Errorf("producer error: %v", err)
}
logging.Debug("producer.Errors() closed")
}()
logging.Infof("created new producer: %v", b.conf.Broker_list)
// save producer reference
b.producer = producer
return nil
}
// Update RunningCore with provided RuntimeConfig.
func UpdateRunningCore(rconf *config.RuntimeConfig) error {
logging.Debug("UpdateRunningCore")
if rconf == nil {
return fmt.Errorf("rconf is nil")
}
core, _, err := create_running_core_hooked(rconf, false)
// http pprof
// https://github.com/golang/go/issues/4674
// we can only open http pprof, cannot close it.
if pprof_serving == false && rconf.Client.Pprof_enabled == true {
if rconf.Client.Pprof_listen == "" {
rconf.Client.Pprof_listen = ":6060"
}
go func() {
pprof_serving = true
logging.Infof("http pprof is listen and served on: %v", rconf.Client.Pprof_listen)
err := http.ListenAndServe(rconf.Client.Pprof_listen, nil)
logging.Errorf("pprof ListenAndServe Error: %v", err)
pprof_serving = false
}()
}
// if registry give us an empty config. agent should also reflect this change.
close_core()
if err != nil {
return err
}
// close_core()
the_core = core
the_rconf = rconf
logging.Debug("UpdateRunningCore Finished")
return nil
}
func (c *kafka_subscription) consume() (<-chan newcore.MultiDataPoint, error) {
logging.Info("consume")
var out = make(chan newcore.MultiDataPoint)
var err error
var consumers []sarama.PartitionConsumer
if c.master == nil {
err = c.connect()
if err != nil {
return nil, err
}
}
for _, c := range c.consumers {
c.Close()
}
c.consumers = nil
partitions, err := c.master.Partitions(c.opts.Topic)
if err != nil {
return nil, fmt.Errorf("Cannot get partitions: %v", err)
}
logging.Infof("partitions: %v", partitions)
err = c.state.Load()
if err != nil {
logging.Errorf("failed to load kafka state: %v", err)
} else {
logging.Infof("state: %+v", c.state.State())
}
flush_offset := true
for _, part := range partitions {
offset := int64(0)
if c.state.Length() > 0 {
offset = c.state.Offset(c.opts.Topic, part)
if offset < 0 {
offset = 0
}
}
consumer, err := c.master.ConsumePartition(c.opts.Topic, part, offset)
if err != nil {
logging.Criticalf("Cannot consumer partition: %d, %v", part, err)
return nil, fmt.Errorf("Cannot consumer partition: %d, %v", part, err)
}
logging.Infof("created consumer: %v", consumer)
consumers = append(consumers, consumer)
go func(flush_offset bool, topic string, part int32, out chan newcore.MultiDataPoint, consumer sarama.PartitionConsumer) {
logging.Infof("start goroutine to consume: part: %d, %v", part, &consumer)
var items newcore.MultiDataPoint
var flush_tick = time.Tick(c.flush_interval)
var _out chan newcore.MultiDataPoint
var startConsume <-chan *sarama.ConsumerMessage
var flushing bool
var offset int64
for {
if (flushing == true && len(items) > 0) || len(items) >= c.max_batch_size {
_out = out // enable output branch
startConsume = nil // disable consuming branch
} else if len(items) < c.max_batch_size {
startConsume = consumer.Messages() // enable consuming branch
_out = nil // disable output branch
}
select {
case message := <-startConsume:
offset = message.Offset
dp, err := newcore.NewDPFromJson(message.Value)
if err != nil {
logging.Tracef("[ERROR]failed to parse datapoint: %v", err)
}
logging.Tracef("kafka dp --> %v", dp)
items = append(items, dp)
case <-flush_tick:
flushing = true
// every part consumer will record offset with interval
c.state.Update(topic, part, offset)
// only 1 goroutine will save state to disk
if flush_offset == true && c.state.Changed() == true {
logging.Tracef("flusing to disk: part: %d, offset: %d", part, offset)
c.state.Save()
}
case _out <- items:
items = nil // clear items
_out = nil // disable output branch
startConsume = consumer.Messages() // enable consuming branch
flushing = false // disable flusing
case err := <-consumer.Errors():
logging.Infof("consumer.Errors: part:%d, %v", part, err)
}
}
}(flush_offset, c.opts.Topic, part, out, consumer)
flush_offset = false // only 1st goroutine is responsible for flushing state back into disk
}
c.consumers = consumers
return out, nil
}
func WatchRuntimeConfFromEtcd(etcd_machines []string, etcd_path string, stop chan error) <-chan RespConfig {
logging.Info("WatchRuntimeConfFromEtcd Started")
var (
out = make(chan RespConfig, 1)
sleep_duration = time.Second
// sleep_duration = time.Second * 5
)
if stop == nil {
panic("stop chan is nil")
}
go func() {
var (
the_first_time = true
watching = false
chGetConf <-chan time.Time
chWaching <-chan time.Time
)
client := etcd.NewClient(etcd_machines)
cached_conf, _ := LoadRuntimeConfFromPath(CONF_CACHE_PATH)
watch_stop := make(chan bool, 0)
loop:
for {
if watching == false && chGetConf == nil {
if the_first_time == true {
chGetConf = time.After(0)
} else {
chGetConf = time.After(sleep_duration)
}
}
if watching == true && chWaching == nil {
chWaching = time.After(sleep_duration)
}
select {
case <-stop:
logging.Info("stop watching etcd.")
watch_stop <- true
logging.Info("watching etcd stopped.")
break loop
case <-chGetConf:
the_first_time = false
chGetConf = nil
tmp_conf, err := getRuntimeConfFromEtcd(client, etcd_path)
if err != nil {
if cached_conf != nil {
// if failed to get config from etcd but we have a cached copy. then use
// this cached version first.
out <- RespConfig{cached_conf, nil}
cached_conf = nil // cached copy only need to emit once.
} else {
out <- RespConfig{nil, logging.SErrorf("failed to getRuntimeConfFromEtcd: %v", err)}
}
} else {
out <- RespConfig{tmp_conf, nil}
watching = true
}
case <-chWaching:
chWaching = nil
logging.Infof("watching etcd remote config: %s, %s", etcd_machines, etcd_path)
resp, err := client.Watch(etcd_path, 0, false, nil, watch_stop)
if err != nil {
logging.Errorf("watching etcd error: %v", err)
break
}
r := bytes.NewReader([]byte(resp.Node.Value))
tmp_conf, err := ReadRuntimeConfig(r)
if err != nil {
logging.Errorf("watching etcd. changes detected but faild to parse config: %v", err)
break
}
logging.Infof("a new config is comming")
out <- RespConfig{tmp_conf, nil}
}
}
}()
return out
}