func TestVariance(t *testing.T) { // Setup sender tierConfig := make(map[string]coco.TierConfig) tierConfig["a"] = coco.TierConfig{Targets: []string{"127.0.0.1:25881", "127.0.0.1:25882", "127.0.0.1:25883", "127.0.0.1:25884", "127.0.0.1:25885", "127.0.0.1:25886", "127.0.0.1:25887", "127.0.0.1:25888"}} var tiers []coco.Tier for k, v := range tierConfig { tier := coco.Tier{Name: k, Targets: v.Targets} tiers = append(tiers, tier) } filtered := make(chan collectd.Packet) go coco.Send(&tiers, filtered) // Test dispatch for i := 0; i < 100000; i++ { send := collectd.Packet{ Hostname: "foo" + string(i), Plugin: "load", Type: "load", } filtered <- send } // Breathe a moment so packet works its way through for { time.Sleep(10 * time.Millisecond) if len(filtered) == 0 { break } } for _, tier := range tiers { var min float64 var max float64 for _, v := range tier.Mappings { size := float64(len(v)) if min == 0 || size < min { min = size } if size > max { max = size } } variance := max / min maxVariance := 1.2 t.Logf("Min: %.2f\n", min) t.Logf("Max: %.2f\n", max) t.Logf("Variance: %.4f\n", variance) if variance > maxVariance { t.Fatalf("Variance was %.4f, expected < %.4f", variance, maxVariance) } } }
/* Send - Hash lookup - Encode a packet */ func TestSend(t *testing.T) { // Setup listener listenConfig := coco.ListenConfig{ Bind: "127.0.0.1:25887", Typesdb: "../types.db", } samples := make(chan collectd.Packet) go coco.Listen(listenConfig, samples) var receive collectd.Packet done := make(chan bool) go func() { receive = <-samples // https://ariejan.net/2014/08/29/synchronize-goroutines-in-your-tests/ done <- true }() // Setup sender tierConfig := make(map[string]coco.TierConfig) tierConfig["a"] = coco.TierConfig{Targets: []string{listenConfig.Bind}} var tiers []coco.Tier for k, v := range tierConfig { tier := coco.Tier{Name: k, Targets: v.Targets} tiers = append(tiers, tier) } t.Logf("tiers: %+v\n", tiers) filtered := make(chan collectd.Packet) go coco.Send(&tiers, filtered) // Test dispatch send := collectd.Packet{ Hostname: "foo", Plugin: "load", Type: "load", } filtered <- send <-done if send.Hostname != receive.Hostname { t.Errorf("Expected %s got %s", send.Hostname, receive.Hostname) } if send.Plugin != receive.Plugin { t.Errorf("Expected %s got %s", send.Plugin, receive.Plugin) } if send.Type != receive.Type { t.Errorf("Expected %s got %s", send.Type, receive.Type) } }
func TestSendWhenMissingConnection(t *testing.T) { // Setup tiers tierConfig := make(map[string]coco.TierConfig) tierConfig["a"] = coco.TierConfig{Targets: []string{"127.0.0.1:29000", "a.example:29000", "b.example:29000", "c.example:29000"}} var tiers []coco.Tier for k, v := range tierConfig { tier := coco.Tier{Name: k, Targets: v.Targets} tiers = append(tiers, tier) } // Launch Api so we can query expvars apiConfig := coco.ApiConfig{ Bind: "127.0.0.1:26880", } blacklisted := map[string]map[string]int64{} go coco.Api(apiConfig, &tiers, &blacklisted) poll(t, apiConfig.Bind) // Launch Send so we can test dispatch behaviour filtered := make(chan collectd.Packet) go coco.Send(&tiers, filtered) // Query the expvars var actual float64 var expected float64 var vars map[string]interface{} vars = fetchExpvar(t, apiConfig.Bind) actual = vars["coco"].(map[string]interface{})["errors"].(map[string]interface{})["send.disconnected"].(float64) expected = 0 if actual != expected { t.Fatalf("Expected coco.errors.send.disconnected to be %d, was %d", expected, actual) } // Dispatch a single packet send := collectd.Packet{ Hostname: "foo", Plugin: "load", Type: "load", } filtered <- send // Check the failure count has increased vars = fetchExpvar(t, apiConfig.Bind) actual = vars["coco"].(map[string]interface{})["errors"].(map[string]interface{})["send.disconnected"].(float64) expected = 1 if actual != expected { t.Fatalf("Expected coco.errors.send.disconnected to be %d, was %d", expected, actual) } }
func TestVirtualReplicasMagic(t *testing.T) { // Setup tiers tierConfig := make(map[string]coco.TierConfig) tierConfig["a"] = coco.TierConfig{Targets: []string{"127.0.0.1:25811", "127.0.0.1:25812", "127.0.0.1:25813"}} tierConfig["b"] = coco.TierConfig{Targets: []string{"127.0.0.1:25821", "127.0.0.1:25822", "127.0.0.1:25823"}} tierConfig["c"] = coco.TierConfig{Targets: []string{"127.0.0.1:25831", "127.0.0.1:25832", "127.0.0.1:25833"}} var tiers []coco.Tier for k, v := range tierConfig { tier := coco.Tier{Name: k, Targets: v.Targets} tiers = append(tiers, tier) } // Setup Send filtered := make(chan collectd.Packet) go coco.Send(&tiers, filtered) // Setup Api apiConfig := coco.ApiConfig{ Bind: "127.0.0.1:26840", } blacklisted := map[string]map[string]int64{} go coco.Api(apiConfig, &tiers, &blacklisted) poll(t, apiConfig.Bind) // Fetch exposed tiers resp, err := http.Get("http://127.0.0.1:26840/tiers") if err != nil { t.Fatalf("HTTP GET failed: %s", err) } body, err := ioutil.ReadAll(resp.Body) //var result []coco.Tier var result []map[string]interface{} err = json.Unmarshal(body, &result) if err != nil { t.Errorf("Error when decoding JSON %+v.", err) t.Errorf("Response body: %s", string(body)) t.FailNow() } for _, tier := range result { if tier["virtual_replicas"] == nil { t.Errorf("tier '%s' doesn't expose virtual replicas", tier["name"]) } t.Logf("tier '%s' has %.0f virtual replicas", tier["name"], tier["virtual_replicas"]) } }
func TestSendTiers(t *testing.T) { // Setup listen listenConfig := coco.ListenConfig{ Bind: "127.0.0.1:25888", Typesdb: "../types.db", } raw := make(chan collectd.Packet) go coco.Listen(listenConfig, raw) count := 0 go func() { for { <-raw count += 1 } }() // Setup sender tierConfig := make(map[string]coco.TierConfig) tierConfig["a"] = coco.TierConfig{Targets: []string{"127.0.0.1:25888"}} tierConfig["b"] = coco.TierConfig{Targets: []string{"127.0.0.1:25888"}} tierConfig["c"] = coco.TierConfig{Targets: []string{"127.0.0.1:25888"}} var tiers []coco.Tier for k, v := range tierConfig { tier := coco.Tier{Name: k, Targets: v.Targets} tiers = append(tiers, tier) } filtered := make(chan collectd.Packet) go coco.Send(&tiers, filtered) // Test dispatch send := collectd.Packet{ Hostname: "foo", Plugin: "load", Type: "load", } filtered <- send // Breathe a moment so packet works its way through time.Sleep(100 * time.Millisecond) if count != len(tierConfig) { t.Errorf("Expected %d packets, got %d", len(tierConfig), count) } }
func TestTierLookup(t *testing.T) { // Setup sender tierConfig := make(map[string]coco.TierConfig) tierConfig["a"] = coco.TierConfig{Targets: []string{"127.0.0.1:25887"}} tierConfig["b"] = coco.TierConfig{Targets: []string{"127.0.0.1:25888"}} tierConfig["c"] = coco.TierConfig{Targets: []string{"127.0.0.1:25889"}} var tiers []coco.Tier for k, v := range tierConfig { tier := coco.Tier{Name: k, Targets: v.Targets} tiers = append(tiers, tier) } filtered := make(chan collectd.Packet) go coco.Send(&tiers, filtered) // Setup API apiConfig := coco.ApiConfig{ Bind: "0.0.0.0:25999", } blacklisted := map[string]map[string]int64{} go coco.Api(apiConfig, &tiers, &blacklisted) poll(t, apiConfig.Bind) // Test resp, err := http.Get("http://127.0.0.1:25999/lookup?name=abc") if err != nil { t.Fatalf("HTTP GET failed: %s", err) } body, err := ioutil.ReadAll(resp.Body) var result map[string]string err = json.Unmarshal(body, &result) if err != nil { t.Fatalf("Error when decoding JSON %+v. Response body: %s", err, string(body)) } for k, v := range tierConfig { if result[k] != v.Targets[0] { t.Errorf("Couldn't find tier %s in response: %s", k, string(body)) } } }
func main() { kingpin.Version("1.0.0") kingpin.Parse() var config coco.Config if _, err := toml.DecodeFile(*configPath, &config); err != nil { log.Fatalln("fatal:", err) return } // Setup data structures to be shared across components blacklisted := map[string]map[string]int64{} raw := make(chan collectd.Packet, 1000000) filtered := make(chan collectd.Packet, 1000000) items := make(chan coco.BlacklistItem, 1000000) var tiers []coco.Tier for k, v := range config.Tiers { tier := coco.Tier{Name: k, Targets: v.Targets} tiers = append(tiers, tier) } if len(tiers) == 0 { log.Fatal("No tiers configured. Exiting.") } chans := map[string]chan collectd.Packet{ "raw": raw, "filtered": filtered, //"blacklist_items": items, } go coco.Measure(config.Measure, chans, &tiers) // Launch components to do the work go coco.Listen(config.Listen, raw) for i := 0; i < 4; i++ { go coco.Filter(config.Filter, raw, filtered, items) } go coco.Blacklist(items, &blacklisted) go coco.Send(&tiers, filtered) coco.Api(config.Api, &tiers, &blacklisted) }
func TestMeasureDistributionSummaryStats(t *testing.T) { // Setup tiers tierConfig := make(map[string]coco.TierConfig) tierConfig["a"] = coco.TierConfig{Targets: []string{"127.0.0.1:25811", "127.0.0.1:25812", "127.0.0.1:25813"}} tierConfig["b"] = coco.TierConfig{Targets: []string{"127.0.0.1:25821", "127.0.0.1:25822", "127.0.0.1:25823"}} tierConfig["c"] = coco.TierConfig{Targets: []string{"127.0.0.1:25831", "127.0.0.1:25832", "127.0.0.1:25833"}} for _, v := range tierConfig { for _, target := range v.Targets { go MockListener(t, target) } } var tiers []coco.Tier for k, v := range tierConfig { tier := coco.Tier{Name: k, Targets: v.Targets} tiers = append(tiers, tier) } // Setup Api apiConfig := coco.ApiConfig{ Bind: "127.0.0.1:26810", } blacklisted := map[string]map[string]int64{} go coco.Api(apiConfig, &tiers, &blacklisted) poll(t, apiConfig.Bind) // Setup Measure chans := map[string]chan collectd.Packet{} measureConfig := coco.MeasureConfig{ TickInterval: *new(coco.Duration), } measureConfig.TickInterval.UnmarshalText([]byte("100ms")) go coco.Measure(measureConfig, chans, &tiers) // Setup Send filtered := make(chan collectd.Packet) go coco.Send(&tiers, filtered) // Push packets to Send // 1000 hosts for i := 0; i < 1000; i++ { // up to 24 cpus per host iter := rand.Intn(24) for n := 0; n < iter; n++ { types := []string{"user", "system", "steal", "wait"} for _, typ := range types { filtered <- collectd.Packet{ Hostname: "foo" + string(i), Plugin: "cpu-" + strconv.Itoa(n), Type: "cpu-" + typ, } } } } time.Sleep(10 * time.Millisecond) // Fetch exposed expvars resp, err := http.Get("http://127.0.0.1:26810/debug/vars") if err != nil { t.Fatalf("HTTP GET failed: %s", err) } body, err := ioutil.ReadAll(resp.Body) var result map[string]interface{} err = json.Unmarshal(body, &result) if err != nil { t.Errorf("Error when decoding JSON %+v.", err) t.Errorf("Response body: %s", string(body)) t.FailNow() } // Test the exposed expvars data looks sane tierProps := result["coco"].(map[string]interface{})["hash.metrics_per_host"].(map[string]interface{}) t.Logf("Expvar tier props: %+v\n", tierProps) if len(tiers) != len(tierProps) { t.Errorf("Expected %d tiers to be exposed, got %d\n", len(tiers), len(tierProps)) //t.Errorf("Tiers: %+v\n", tiers) t.Errorf("Exposed tiers: %+v\n", tierProps) } for _, tier := range tiers { targetProps := tierProps[tier.Name].(map[string]interface{}) if len(targetProps) != len(tier.Targets)+1 { t.Errorf("Expected %d targets to be exposed, got %d\n", len(tier.Targets)+1, len(targetProps)) t.Errorf("Tier targets: %+v\n", tier.Targets) t.Errorf("Exposed target properties: %+v\n", targetProps) } for _, target := range tier.Targets { props := targetProps[target].(map[string]interface{}) for _, k := range []string{"95e", "avg", "max", "min", "sum"} { if props[k] == nil { t.Errorf("Expected %s metric was not exposed on %s\n", k, target) } } } } }
func TestEncodeGauge(t *testing.T) { // Setup listen listenConfig := coco.ListenConfig{ Bind: "127.0.0.1:25961", Typesdb: "../types.db", } raw := make(chan collectd.Packet, 500) go coco.Listen(listenConfig, raw) // Setup sender tierConfig := make(map[string]coco.TierConfig) tierConfig["a"] = coco.TierConfig{Targets: []string{listenConfig.Bind}} var tiers []coco.Tier for k, v := range tierConfig { tier := coco.Tier{Name: k, Targets: v.Targets} tiers = append(tiers, tier) } filtered := make(chan collectd.Packet) go coco.Send(&tiers, filtered) // Dispatch a sample value := collectd.Value{ Name: "shortterm", Type: uint8(1), TypeName: "gauge", Value: 0.5, } sample := collectd.Packet{ Hostname: "foo", Plugin: "load", Type: "load", Interval: 10, Time: uint64(time.Now().Unix()), Values: []collectd.Value{value}, } filtered <- sample // Breathe a moment so packet works its way through time.Sleep(100 * time.Millisecond) if len(raw) != 1 { t.Errorf("Expected %d packets, got %d\n", 1, len(raw)) t.FailNow() } // Grab a sample s := <-raw if len(s.Values) != 1 { t.Errorf("Expected %d values in sample, got %d\n", 1, len(s.Values)) t.FailNow() } v := s.Values[0] t.Logf("before: %+v\n", value) t.Logf("after: %+v\n", v) if value.Value != v.Value { t.Errorf("Expected value to be %d, got %d\n", value.Value, v.Value) t.FailNow() } }