Ejemplo n.º 1
0
func protect(h httprouter.Handle, expire time.Duration, trigger string) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {

		secure := false
		if trigger == "read" && config.CoreConf.Secure_api_read {
			secure = true
		} else if trigger == "write" && config.CoreConf.Secure_api_write {
			secure = true
		}
		logging.Infof("trigger: %s, secure: %v, write: %v, read: %v\n", trigger, secure, config.CoreConf.Secure_api_write, config.CoreConf.Secure_api_read)

		if secure {
			hostname := r.URL.Query().Get("hostname")
			if strings.ToLower(hostname) != newcore.GetHostname() {
				logging.Errorf("hostname mismatch: %v", hostname)
				http.Error(w, "hostname mismatch", 500)
				return
			}

			time_str := r.URL.Query().Get("time")
			tm, err := utils.UTCTimeFromUnixStr(time_str)
			if err != nil {
				logging.Errorf("invalid time: %v", time_str)
				http.Error(w, "Invalid Time", 500)
				return
			}

			if time.Now().Sub(tm) > expire {
				// expired reqeust
				logging.Errorf("expired request: %v", time.Now().Sub(tm))
				http.Error(w, "expired request", 500)
				return
			}

			// we need to verify request.
			// request should put signature of this agent hostname into header HICKWALL_ADMIN_SIGN
			load_unsigner()

			signed_str := r.Header.Get("HICKWALL_ADMIN_SIGN")
			signed, err := base64.StdEncoding.DecodeString(signed_str)
			if err != nil {
				logging.Error("cannot decode sign")
				http.Error(w, "cannot decode sign", 500)
				return
			}

			toSign := fmt.Sprintf("%s%s", hostname, time_str)
			logging.Trace("unsign started")
			err = unsigner.Unsign([]byte(toSign), signed)
			logging.Trace("unsign finished")
			if err != nil {
				logging.Errorf("-> invalid signature: %v <-", string(signed))
				http.Error(w, "invalid signature", 500)
				return
			}
		}

		h(w, r, ps)
	}
}
Ejemplo n.º 2
0
func serve_api() {
	router := httprouter.New()
	router.GET("/sys_info", protect_read(serveSysInfo, time.Second))

	router.DELETE("/registry/revoke", protect_write(serveRegistryRevoke, time.Second))

	//	router.GET("/registry", serveRegistryGet)
	//	router.POST("/registry/accept", serveRegistryAccept)

	//	router.PUT("/registry/renew", serveRegistryRenew)
	//	router.PUT("/config/renew", serveConfigRenew)

	addr := ":3031"
	if config.CoreConf.Listen_port > 0 {
		addr = fmt.Sprintf(":%d", config.CoreConf.Listen_port)
	}
	logging.Infof("api served at: %s", addr)

	api_srv_running = true
	err := http.ListenAndServe(addr, router)
	api_srv_running = false
	if err != nil {
		logging.Criticalf("api server is not running!: %v", err)
	} else {
		logging.Info("api server stopped")
	}
}
Ejemplo n.º 3
0
func MustNewKafkaBackend(name string, bconf *config.Transport_kafka) *kafkaBackend {
	logging.Infof("MustNewKafkaBackend: %+v", bconf)
	_kconf := sarama.NewConfig()

	_kconf.Net.DialTimeout = newcore.Interval(bconf.Dail_timeout).MustDuration(time.Second * 5)
	_kconf.Net.WriteTimeout = newcore.Interval(bconf.Write_timeout).MustDuration(time.Second * 1)
	_kconf.Net.ReadTimeout = time.Second * 10
	_kconf.Net.KeepAlive = newcore.Interval(bconf.Keepalive).MustDuration(time.Second * 30)

	if bconf.Ack_timeout_ms <= 0 {
		_kconf.Producer.Timeout = time.Millisecond * 100
	} else {
		_kconf.Producer.Timeout = time.Millisecond * time.Duration(bconf.Ack_timeout_ms)
	}

	if bconf.Flush_frequency_ms <= 0 {
		_kconf.Producer.Flush.Frequency = time.Millisecond * 100
	} else {
		_kconf.Producer.Flush.Frequency = time.Millisecond * time.Duration(bconf.Flush_frequency_ms)
	}

	cc := strings.ToLower(bconf.Compression_codec)
	switch {
	case cc == "none":
		_kconf.Producer.Compression = sarama.CompressionNone
	case cc == "gzip":
		_kconf.Producer.Compression = sarama.CompressionGZIP // Compress messages
	case cc == "snappy":
		_kconf.Producer.Compression = sarama.CompressionSnappy // Compress messages
	default:
		_kconf.Producer.Compression = sarama.CompressionNone
	}

	ra := strings.ToLower(bconf.Required_acks)
	switch {
	case ra == "no_response":
		_kconf.Producer.RequiredAcks = sarama.NoResponse
	case ra == "wait_for_local":
		_kconf.Producer.RequiredAcks = sarama.WaitForLocal
	case ra == "wait_for_all":
		_kconf.Producer.RequiredAcks = sarama.WaitForAll
	default:
		_kconf.Producer.RequiredAcks = sarama.NoResponse
	}

	logging.Debugf("kafka conf: %+v", _kconf)

	s := &kafkaBackend{
		name:    name,
		closing: make(chan chan error),
		updates: make(chan newcore.MultiDataPoint),
		conf:    bconf,  // backend config
		kconf:   _kconf, // sarama config
	}
	go s.loop()
	return s
}
Ejemplo n.º 4
0
func Test_strategy_etcd_LoadConfigStrategyEtcd_Nil(t *testing.T) {
	logging.SetLevel("debug")
	defer close_core()

	stopCh := make(chan error)

	for idx, tcase := range nil_core_tests {
		request_cnt := 0
		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			request_cnt += 1
			logging.Infof("case: %d, -- we got request: cnt:%d ------------\n", idx, request_cnt)
			// pretty.Println(r)
			iswait := r.URL.Query().Get("wait")
			if strings.ToLower(iswait) == "true" {
				logging.Info("watching")
				// watch, long polling
				// time.Sleep(time.Second * 1)
			} else {
				logging.Info("getting")
			}

			v, _ := _fack_etcd_respose(1, tcase)
			fmt.Printf("case: %d, content: %s\n", idx, v)

			fmt.Fprintln(w, v)

		}))

		go new_core_from_etcd([]string{ts.URL}, "/config/host/DST54869.yml", stopCh)
		tick := time.After(time.Second * 1)
		timeout := time.After(time.Second * 2)

	main_loop:
		for {
			select {
			case <-tick:
				stopCh <- nil
				if the_core != nil {
					t.Error("the_core has been created with invalid configuration!")
					return
				} else {
					t.Log("test case successed")
					ts.Close()
					break main_loop
				}
			case <-timeout:
				t.Errorf("timed out. somethings is blocking")
				return
			}
		}
	}

}
Ejemplo n.º 5
0
func UseConfigCreateCollectors(rconf *config.RuntimeConfig) ([]newcore.Collector, error) {
	var clrs []newcore.Collector
	var prefixs = make(map[string]bool)

	for gid, group := range rconf.Groups {
		logging.Infof("gid: %d, prefix: %s\n", gid, group.Prefix)
		if len(group.Prefix) <= 0 {
			return nil, fmt.Errorf("group (idx:%d) prefix is empty.", gid)
		} else {

			_, exists := prefixs[group.Prefix]
			if exists == false {
				prefixs[group.Prefix] = true
			} else {
				return nil, fmt.Errorf("duplicated group prefix: %s", group.Prefix)
			}
		}

		for cid, conf := range group.Collector_ping {
			pings := MustNewPingCollectors(gen_collector_name(gid, cid, "ping"), group.Prefix, conf)
			for _, c := range pings {
				clrs = append(clrs, c)
			}
		}

		for cid, conf := range group.Collector_win_pdh {
			c := windows.MustNewWinPdhCollector(gen_collector_name(gid, cid, "pdh"), group.Prefix, conf)
			clrs = append(clrs, c)
		}

		for cid, conf := range group.Collector_win_wmi {
			c := windows.MustNewWinWmiCollector(gen_collector_name(gid, cid, "wmi"), group.Prefix, conf)
			clrs = append(clrs, c)
		}

		if group.Collector_win_sys != nil {
			cs := windows.MustNewWinSysCollectors(gen_collector_name(gid, 0, "win_sys"), group.Prefix, group.Collector_win_sys)
			for _, c := range cs {
				clrs = append(clrs, c)
			}
			//			clrs = append(clrs, cs...)
		}

	}

	logging.Debugf("rconf.Client.Metric_Enabled: %v, rconf.Client.Metric_Interval: %v",
		rconf.Client.Metric_Enabled, rconf.Client.Metric_Interval)
	if rconf.Client.Metric_Enabled == true {
		clrs = append(clrs, MustNewHickwallCollector(rconf.Client.Metric_Interval))
		clrs = append(clrs, windows.MustNewWinHickwallMemCollector(rconf.Client.Metric_Interval, rconf.Client.Tags))
	}
	return clrs[:], nil
}
Ejemplo n.º 6
0
func (b *kafkaBackend) connect() error {
	producer, err := sarama.NewAsyncProducer(b.conf.Broker_list, b.kconf)
	if err != nil {
		logging.Errorf("failed to start producer: %v, %v", err, b.conf.Broker_list)
		return fmt.Errorf("failed to start producer: %v, %v", err, b.conf.Broker_list)
	}

	go func() {
		logging.Debug("consuming from producer.Errors()")
		for err := range producer.Errors() {
			logging.Errorf("producer error: %v", err)
		}
		logging.Debug("producer.Errors() closed")
	}()

	logging.Infof("created new producer: %v", b.conf.Broker_list)

	// save producer reference
	b.producer = producer
	return nil
}
Ejemplo n.º 7
0
// Update RunningCore with provided RuntimeConfig.
func UpdateRunningCore(rconf *config.RuntimeConfig) error {
	logging.Debug("UpdateRunningCore")
	if rconf == nil {
		return fmt.Errorf("rconf is nil")
	}
	core, _, err := create_running_core_hooked(rconf, false)

	// http pprof
	// https://github.com/golang/go/issues/4674
	// we can only open http pprof, cannot close it.
	if pprof_serving == false && rconf.Client.Pprof_enabled == true {
		if rconf.Client.Pprof_listen == "" {
			rconf.Client.Pprof_listen = ":6060"
		}
		go func() {
			pprof_serving = true
			logging.Infof("http pprof is listen and served on: %v", rconf.Client.Pprof_listen)
			err := http.ListenAndServe(rconf.Client.Pprof_listen, nil)
			logging.Errorf("pprof ListenAndServe Error: %v", err)
			pprof_serving = false
		}()
	}

	// if registry give us an empty config. agent should also reflect this change.
	close_core()

	if err != nil {
		return err
	}

	//	close_core()
	the_core = core
	the_rconf = rconf
	logging.Debug("UpdateRunningCore Finished")
	return nil
}
Ejemplo n.º 8
0
func (c *kafka_subscription) consume() (<-chan newcore.MultiDataPoint, error) {
	logging.Info("consume")

	var out = make(chan newcore.MultiDataPoint)
	var err error
	var consumers []sarama.PartitionConsumer
	if c.master == nil {
		err = c.connect()
		if err != nil {
			return nil, err
		}
	}

	for _, c := range c.consumers {
		c.Close()
	}
	c.consumers = nil

	partitions, err := c.master.Partitions(c.opts.Topic)
	if err != nil {
		return nil, fmt.Errorf("Cannot get partitions: %v", err)
	}
	logging.Infof("partitions: %v", partitions)

	err = c.state.Load()
	if err != nil {
		logging.Errorf("failed to load kafka state: %v", err)
	} else {
		logging.Infof("state: %+v", c.state.State())
	}

	flush_offset := true

	for _, part := range partitions {
		offset := int64(0)
		if c.state.Length() > 0 {
			offset = c.state.Offset(c.opts.Topic, part)
			if offset < 0 {
				offset = 0
			}
		}
		consumer, err := c.master.ConsumePartition(c.opts.Topic, part, offset)
		if err != nil {
			logging.Criticalf("Cannot consumer partition: %d, %v", part, err)
			return nil, fmt.Errorf("Cannot consumer partition: %d, %v", part, err)
		}
		logging.Infof("created consumer: %v", consumer)

		consumers = append(consumers, consumer)

		go func(flush_offset bool, topic string, part int32, out chan newcore.MultiDataPoint, consumer sarama.PartitionConsumer) {
			logging.Infof("start goroutine to consume: part: %d,  %v", part, &consumer)

			var items newcore.MultiDataPoint
			var flush_tick = time.Tick(c.flush_interval)
			var _out chan newcore.MultiDataPoint
			var startConsume <-chan *sarama.ConsumerMessage
			var flushing bool
			var offset int64

			for {
				if (flushing == true && len(items) > 0) || len(items) >= c.max_batch_size {
					_out = out         // enable output branch
					startConsume = nil // disable consuming branch
				} else if len(items) < c.max_batch_size {
					startConsume = consumer.Messages() // enable consuming branch
					_out = nil                         // disable output branch
				}

				select {
				case message := <-startConsume:
					offset = message.Offset
					dp, err := newcore.NewDPFromJson(message.Value)
					if err != nil {
						logging.Tracef("[ERROR]failed to parse datapoint: %v", err)
					}
					logging.Tracef("kafka dp --> %v", dp)
					items = append(items, dp)
				case <-flush_tick:
					flushing = true
					// every part consumer will record offset with interval
					c.state.Update(topic, part, offset)

					// only 1 goroutine will save state to disk
					if flush_offset == true && c.state.Changed() == true {
						logging.Tracef("flusing to disk: part: %d, offset: %d", part, offset)
						c.state.Save()
					}
				case _out <- items:
					items = nil                        // clear items
					_out = nil                         // disable output branch
					startConsume = consumer.Messages() // enable consuming branch
					flushing = false                   // disable flusing
				case err := <-consumer.Errors():
					logging.Infof("consumer.Errors: part:%d,  %v", part, err)
				}
			}
		}(flush_offset, c.opts.Topic, part, out, consumer)

		flush_offset = false // only 1st goroutine is responsible for flushing state back into disk
	}
	c.consumers = consumers
	return out, nil
}
Ejemplo n.º 9
0
func WatchRuntimeConfFromEtcd(etcd_machines []string, etcd_path string, stop chan error) <-chan RespConfig {
	logging.Info("WatchRuntimeConfFromEtcd Started")
	var (
		out            = make(chan RespConfig, 1)
		sleep_duration = time.Second
		// sleep_duration = time.Second * 5
	)

	if stop == nil {
		panic("stop chan is nil")
	}

	go func() {
		var (
			the_first_time = true
			watching       = false
			chGetConf      <-chan time.Time
			chWaching      <-chan time.Time
		)

		client := etcd.NewClient(etcd_machines)

		cached_conf, _ := LoadRuntimeConfFromPath(CONF_CACHE_PATH)

		watch_stop := make(chan bool, 0)

	loop:
		for {
			if watching == false && chGetConf == nil {
				if the_first_time == true {
					chGetConf = time.After(0)
				} else {
					chGetConf = time.After(sleep_duration)
				}
			}

			if watching == true && chWaching == nil {
				chWaching = time.After(sleep_duration)
			}

			select {
			case <-stop:
				logging.Info("stop watching etcd.")
				watch_stop <- true
				logging.Info("watching etcd stopped.")
				break loop
			case <-chGetConf:
				the_first_time = false
				chGetConf = nil

				tmp_conf, err := getRuntimeConfFromEtcd(client, etcd_path)
				if err != nil {
					if cached_conf != nil {
						// if failed to get config from etcd but we have a cached copy. then use
						// this cached version first.
						out <- RespConfig{cached_conf, nil}
						cached_conf = nil // cached copy only need to emit once.
					} else {
						out <- RespConfig{nil, logging.SErrorf("failed to getRuntimeConfFromEtcd: %v", err)}
					}
				} else {
					out <- RespConfig{tmp_conf, nil}
					watching = true
				}
			case <-chWaching:
				chWaching = nil
				logging.Infof("watching etcd remote config: %s, %s", etcd_machines, etcd_path)
				resp, err := client.Watch(etcd_path, 0, false, nil, watch_stop)
				if err != nil {
					logging.Errorf("watching etcd error: %v", err)
					break
				}

				r := bytes.NewReader([]byte(resp.Node.Value))
				tmp_conf, err := ReadRuntimeConfig(r)
				if err != nil {
					logging.Errorf("watching etcd. changes detected but faild to parse config: %v", err)
					break
				}

				logging.Infof("a new config is comming")
				out <- RespConfig{tmp_conf, nil}
			}
		}
	}()
	return out
}