Exemplo n.º 1
0
func TestVariance(t *testing.T) {
	// Setup sender
	tierConfig := make(map[string]coco.TierConfig)
	tierConfig["a"] = coco.TierConfig{Targets: []string{"127.0.0.1:25881", "127.0.0.1:25882", "127.0.0.1:25883", "127.0.0.1:25884", "127.0.0.1:25885", "127.0.0.1:25886", "127.0.0.1:25887", "127.0.0.1:25888"}}

	var tiers []coco.Tier
	for k, v := range tierConfig {
		tier := coco.Tier{Name: k, Targets: v.Targets}
		tiers = append(tiers, tier)
	}

	filtered := make(chan collectd.Packet)
	go coco.Send(&tiers, filtered)

	// Test dispatch
	for i := 0; i < 100000; i++ {
		send := collectd.Packet{
			Hostname: "foo" + string(i),
			Plugin:   "load",
			Type:     "load",
		}
		filtered <- send
	}

	// Breathe a moment so packet works its way through
	for {
		time.Sleep(10 * time.Millisecond)
		if len(filtered) == 0 {
			break
		}
	}

	for _, tier := range tiers {
		var min float64
		var max float64
		for _, v := range tier.Mappings {
			size := float64(len(v))
			if min == 0 || size < min {
				min = size
			}
			if size > max {
				max = size
			}
		}
		variance := max / min
		maxVariance := 1.2
		t.Logf("Min: %.2f\n", min)
		t.Logf("Max: %.2f\n", max)
		t.Logf("Variance: %.4f\n", variance)
		if variance > maxVariance {
			t.Fatalf("Variance was %.4f, expected < %.4f", variance, maxVariance)
		}
	}
}
Exemplo n.º 2
0
/*
Send
 - Hash lookup
 - Encode a packet
*/
func TestSend(t *testing.T) {
	// Setup listener
	listenConfig := coco.ListenConfig{
		Bind:    "127.0.0.1:25887",
		Typesdb: "../types.db",
	}
	samples := make(chan collectd.Packet)
	go coco.Listen(listenConfig, samples)

	var receive collectd.Packet
	done := make(chan bool)
	go func() {
		receive = <-samples
		// https://ariejan.net/2014/08/29/synchronize-goroutines-in-your-tests/
		done <- true
	}()

	// Setup sender
	tierConfig := make(map[string]coco.TierConfig)
	tierConfig["a"] = coco.TierConfig{Targets: []string{listenConfig.Bind}}

	var tiers []coco.Tier
	for k, v := range tierConfig {
		tier := coco.Tier{Name: k, Targets: v.Targets}
		tiers = append(tiers, tier)
	}
	t.Logf("tiers: %+v\n", tiers)

	filtered := make(chan collectd.Packet)
	go coco.Send(&tiers, filtered)

	// Test dispatch
	send := collectd.Packet{
		Hostname: "foo",
		Plugin:   "load",
		Type:     "load",
	}

	filtered <- send
	<-done

	if send.Hostname != receive.Hostname {
		t.Errorf("Expected %s got %s", send.Hostname, receive.Hostname)
	}
	if send.Plugin != receive.Plugin {
		t.Errorf("Expected %s got %s", send.Plugin, receive.Plugin)
	}
	if send.Type != receive.Type {
		t.Errorf("Expected %s got %s", send.Type, receive.Type)
	}
}
Exemplo n.º 3
0
func TestSendWhenMissingConnection(t *testing.T) {
	// Setup tiers
	tierConfig := make(map[string]coco.TierConfig)
	tierConfig["a"] = coco.TierConfig{Targets: []string{"127.0.0.1:29000", "a.example:29000", "b.example:29000", "c.example:29000"}}

	var tiers []coco.Tier
	for k, v := range tierConfig {
		tier := coco.Tier{Name: k, Targets: v.Targets}
		tiers = append(tiers, tier)
	}

	// Launch Api so we can query expvars
	apiConfig := coco.ApiConfig{
		Bind: "127.0.0.1:26880",
	}
	blacklisted := map[string]map[string]int64{}
	go coco.Api(apiConfig, &tiers, &blacklisted)

	poll(t, apiConfig.Bind)

	// Launch Send so we can test dispatch behaviour
	filtered := make(chan collectd.Packet)
	go coco.Send(&tiers, filtered)

	// Query the expvars
	var actual float64
	var expected float64
	var vars map[string]interface{}
	vars = fetchExpvar(t, apiConfig.Bind)
	actual = vars["coco"].(map[string]interface{})["errors"].(map[string]interface{})["send.disconnected"].(float64)
	expected = 0
	if actual != expected {
		t.Fatalf("Expected coco.errors.send.disconnected to be %d, was %d", expected, actual)
	}

	// Dispatch a single packet
	send := collectd.Packet{
		Hostname: "foo",
		Plugin:   "load",
		Type:     "load",
	}
	filtered <- send

	// Check the failure count has increased
	vars = fetchExpvar(t, apiConfig.Bind)
	actual = vars["coco"].(map[string]interface{})["errors"].(map[string]interface{})["send.disconnected"].(float64)
	expected = 1
	if actual != expected {
		t.Fatalf("Expected coco.errors.send.disconnected to be %d, was %d", expected, actual)
	}
}
Exemplo n.º 4
0
func TestVirtualReplicasMagic(t *testing.T) {
	// Setup tiers
	tierConfig := make(map[string]coco.TierConfig)
	tierConfig["a"] = coco.TierConfig{Targets: []string{"127.0.0.1:25811", "127.0.0.1:25812", "127.0.0.1:25813"}}
	tierConfig["b"] = coco.TierConfig{Targets: []string{"127.0.0.1:25821", "127.0.0.1:25822", "127.0.0.1:25823"}}
	tierConfig["c"] = coco.TierConfig{Targets: []string{"127.0.0.1:25831", "127.0.0.1:25832", "127.0.0.1:25833"}}

	var tiers []coco.Tier
	for k, v := range tierConfig {
		tier := coco.Tier{Name: k, Targets: v.Targets}
		tiers = append(tiers, tier)
	}

	// Setup Send
	filtered := make(chan collectd.Packet)
	go coco.Send(&tiers, filtered)

	// Setup Api
	apiConfig := coco.ApiConfig{
		Bind: "127.0.0.1:26840",
	}
	blacklisted := map[string]map[string]int64{}
	go coco.Api(apiConfig, &tiers, &blacklisted)
	poll(t, apiConfig.Bind)

	// Fetch exposed tiers
	resp, err := http.Get("http://127.0.0.1:26840/tiers")
	if err != nil {
		t.Fatalf("HTTP GET failed: %s", err)
	}
	body, err := ioutil.ReadAll(resp.Body)
	//var result []coco.Tier
	var result []map[string]interface{}
	err = json.Unmarshal(body, &result)
	if err != nil {
		t.Errorf("Error when decoding JSON %+v.", err)
		t.Errorf("Response body: %s", string(body))
		t.FailNow()
	}

	for _, tier := range result {
		if tier["virtual_replicas"] == nil {
			t.Errorf("tier '%s' doesn't expose virtual replicas", tier["name"])
		}
		t.Logf("tier '%s' has %.0f virtual replicas", tier["name"], tier["virtual_replicas"])
	}
}
Exemplo n.º 5
0
func TestSendTiers(t *testing.T) {
	// Setup listen
	listenConfig := coco.ListenConfig{
		Bind:    "127.0.0.1:25888",
		Typesdb: "../types.db",
	}
	raw := make(chan collectd.Packet)
	go coco.Listen(listenConfig, raw)

	count := 0
	go func() {
		for {
			<-raw
			count += 1
		}
	}()

	// Setup sender
	tierConfig := make(map[string]coco.TierConfig)
	tierConfig["a"] = coco.TierConfig{Targets: []string{"127.0.0.1:25888"}}
	tierConfig["b"] = coco.TierConfig{Targets: []string{"127.0.0.1:25888"}}
	tierConfig["c"] = coco.TierConfig{Targets: []string{"127.0.0.1:25888"}}

	var tiers []coco.Tier
	for k, v := range tierConfig {
		tier := coco.Tier{Name: k, Targets: v.Targets}
		tiers = append(tiers, tier)
	}

	filtered := make(chan collectd.Packet)
	go coco.Send(&tiers, filtered)

	// Test dispatch
	send := collectd.Packet{
		Hostname: "foo",
		Plugin:   "load",
		Type:     "load",
	}

	filtered <- send

	// Breathe a moment so packet works its way through
	time.Sleep(100 * time.Millisecond)
	if count != len(tierConfig) {
		t.Errorf("Expected %d packets, got %d", len(tierConfig), count)
	}
}
Exemplo n.º 6
0
func TestTierLookup(t *testing.T) {
	// Setup sender
	tierConfig := make(map[string]coco.TierConfig)
	tierConfig["a"] = coco.TierConfig{Targets: []string{"127.0.0.1:25887"}}
	tierConfig["b"] = coco.TierConfig{Targets: []string{"127.0.0.1:25888"}}
	tierConfig["c"] = coco.TierConfig{Targets: []string{"127.0.0.1:25889"}}

	var tiers []coco.Tier
	for k, v := range tierConfig {
		tier := coco.Tier{Name: k, Targets: v.Targets}
		tiers = append(tiers, tier)
	}

	filtered := make(chan collectd.Packet)
	go coco.Send(&tiers, filtered)

	// Setup API
	apiConfig := coco.ApiConfig{
		Bind: "0.0.0.0:25999",
	}
	blacklisted := map[string]map[string]int64{}
	go coco.Api(apiConfig, &tiers, &blacklisted)

	poll(t, apiConfig.Bind)

	// Test
	resp, err := http.Get("http://127.0.0.1:25999/lookup?name=abc")
	if err != nil {
		t.Fatalf("HTTP GET failed: %s", err)
	}

	body, err := ioutil.ReadAll(resp.Body)
	var result map[string]string
	err = json.Unmarshal(body, &result)
	if err != nil {
		t.Fatalf("Error when decoding JSON %+v. Response body: %s", err, string(body))
	}

	for k, v := range tierConfig {
		if result[k] != v.Targets[0] {
			t.Errorf("Couldn't find tier %s in response: %s", k, string(body))
		}
	}
}
Exemplo n.º 7
0
func main() {
	kingpin.Version("1.0.0")
	kingpin.Parse()

	var config coco.Config
	if _, err := toml.DecodeFile(*configPath, &config); err != nil {
		log.Fatalln("fatal:", err)
		return
	}

	// Setup data structures to be shared across components
	blacklisted := map[string]map[string]int64{}
	raw := make(chan collectd.Packet, 1000000)
	filtered := make(chan collectd.Packet, 1000000)
	items := make(chan coco.BlacklistItem, 1000000)

	var tiers []coco.Tier
	for k, v := range config.Tiers {
		tier := coco.Tier{Name: k, Targets: v.Targets}
		tiers = append(tiers, tier)
	}

	if len(tiers) == 0 {
		log.Fatal("No tiers configured. Exiting.")
	}

	chans := map[string]chan collectd.Packet{
		"raw":      raw,
		"filtered": filtered,
		//"blacklist_items": items,
	}
	go coco.Measure(config.Measure, chans, &tiers)

	// Launch components to do the work
	go coco.Listen(config.Listen, raw)
	for i := 0; i < 4; i++ {
		go coco.Filter(config.Filter, raw, filtered, items)
	}
	go coco.Blacklist(items, &blacklisted)
	go coco.Send(&tiers, filtered)
	coco.Api(config.Api, &tiers, &blacklisted)
}
Exemplo n.º 8
0
func TestMeasureDistributionSummaryStats(t *testing.T) {
	// Setup tiers
	tierConfig := make(map[string]coco.TierConfig)
	tierConfig["a"] = coco.TierConfig{Targets: []string{"127.0.0.1:25811", "127.0.0.1:25812", "127.0.0.1:25813"}}
	tierConfig["b"] = coco.TierConfig{Targets: []string{"127.0.0.1:25821", "127.0.0.1:25822", "127.0.0.1:25823"}}
	tierConfig["c"] = coco.TierConfig{Targets: []string{"127.0.0.1:25831", "127.0.0.1:25832", "127.0.0.1:25833"}}

	for _, v := range tierConfig {
		for _, target := range v.Targets {
			go MockListener(t, target)
		}
	}

	var tiers []coco.Tier
	for k, v := range tierConfig {
		tier := coco.Tier{Name: k, Targets: v.Targets}
		tiers = append(tiers, tier)
	}

	// Setup Api
	apiConfig := coco.ApiConfig{
		Bind: "127.0.0.1:26810",
	}
	blacklisted := map[string]map[string]int64{}
	go coco.Api(apiConfig, &tiers, &blacklisted)
	poll(t, apiConfig.Bind)

	// Setup Measure
	chans := map[string]chan collectd.Packet{}
	measureConfig := coco.MeasureConfig{
		TickInterval: *new(coco.Duration),
	}
	measureConfig.TickInterval.UnmarshalText([]byte("100ms"))
	go coco.Measure(measureConfig, chans, &tiers)

	// Setup Send
	filtered := make(chan collectd.Packet)
	go coco.Send(&tiers, filtered)

	// Push packets to Send
	// 1000 hosts
	for i := 0; i < 1000; i++ {
		// up to 24 cpus per host
		iter := rand.Intn(24)
		for n := 0; n < iter; n++ {
			types := []string{"user", "system", "steal", "wait"}
			for _, typ := range types {
				filtered <- collectd.Packet{
					Hostname: "foo" + string(i),
					Plugin:   "cpu-" + strconv.Itoa(n),
					Type:     "cpu-" + typ,
				}
			}
		}
	}

	time.Sleep(10 * time.Millisecond)

	// Fetch exposed expvars
	resp, err := http.Get("http://127.0.0.1:26810/debug/vars")
	if err != nil {
		t.Fatalf("HTTP GET failed: %s", err)
	}
	body, err := ioutil.ReadAll(resp.Body)
	var result map[string]interface{}
	err = json.Unmarshal(body, &result)
	if err != nil {
		t.Errorf("Error when decoding JSON %+v.", err)
		t.Errorf("Response body: %s", string(body))
		t.FailNow()
	}

	// Test the exposed expvars data looks sane
	tierProps := result["coco"].(map[string]interface{})["hash.metrics_per_host"].(map[string]interface{})
	t.Logf("Expvar tier props: %+v\n", tierProps)
	if len(tiers) != len(tierProps) {
		t.Errorf("Expected %d tiers to be exposed, got %d\n", len(tiers), len(tierProps))
		//t.Errorf("Tiers: %+v\n", tiers)
		t.Errorf("Exposed tiers: %+v\n", tierProps)
	}
	for _, tier := range tiers {
		targetProps := tierProps[tier.Name].(map[string]interface{})
		if len(targetProps) != len(tier.Targets)+1 {
			t.Errorf("Expected %d targets to be exposed, got %d\n", len(tier.Targets)+1, len(targetProps))
			t.Errorf("Tier targets: %+v\n", tier.Targets)
			t.Errorf("Exposed target properties: %+v\n", targetProps)
		}
		for _, target := range tier.Targets {
			props := targetProps[target].(map[string]interface{})
			for _, k := range []string{"95e", "avg", "max", "min", "sum"} {
				if props[k] == nil {
					t.Errorf("Expected %s metric was not exposed on %s\n", k, target)
				}
			}
		}
	}
}
Exemplo n.º 9
0
func TestEncodeGauge(t *testing.T) {
	// Setup listen
	listenConfig := coco.ListenConfig{
		Bind:    "127.0.0.1:25961",
		Typesdb: "../types.db",
	}
	raw := make(chan collectd.Packet, 500)
	go coco.Listen(listenConfig, raw)

	// Setup sender
	tierConfig := make(map[string]coco.TierConfig)
	tierConfig["a"] = coco.TierConfig{Targets: []string{listenConfig.Bind}}

	var tiers []coco.Tier
	for k, v := range tierConfig {
		tier := coco.Tier{Name: k, Targets: v.Targets}
		tiers = append(tiers, tier)
	}

	filtered := make(chan collectd.Packet)
	go coco.Send(&tiers, filtered)

	// Dispatch a sample
	value := collectd.Value{
		Name:     "shortterm",
		Type:     uint8(1),
		TypeName: "gauge",
		Value:    0.5,
	}
	sample := collectd.Packet{
		Hostname: "foo",
		Plugin:   "load",
		Type:     "load",
		Interval: 10,
		Time:     uint64(time.Now().Unix()),
		Values:   []collectd.Value{value},
	}
	filtered <- sample

	// Breathe a moment so packet works its way through
	time.Sleep(100 * time.Millisecond)
	if len(raw) != 1 {
		t.Errorf("Expected %d packets, got %d\n", 1, len(raw))
		t.FailNow()
	}

	// Grab a sample
	s := <-raw
	if len(s.Values) != 1 {
		t.Errorf("Expected %d values in sample, got %d\n", 1, len(s.Values))
		t.FailNow()
	}

	v := s.Values[0]
	t.Logf("before: %+v\n", value)
	t.Logf("after:  %+v\n", v)

	if value.Value != v.Value {
		t.Errorf("Expected value to be %d, got %d\n", value.Value, v.Value)
		t.FailNow()
	}
}