func TestCreateDeleteDomain(t *testing.T) { config.Reset() utils.SetupTests() defer utils.TearDownTests() m := NewManager() info := datamodel.NewEmptyInfo() info.Properties.MaxUniqueItems = utils.Int64p(10000) info.Properties.Size = utils.Int64p(10000) info.Name = utils.Stringp(fmt.Sprintf("marvel")) if err := m.CreateDomain(info); err != nil { t.Error("Expected no errors, got", err) } if sketches := m.GetSketches(); len(sketches) != 4 { t.Error("Expected 1 sketches, got", len(sketches)) } else if sketches[0][0] != "marvel" || sketches[0][1] != "card" { t.Error("Expected [[marvel card]], got", sketches) } // Create a second Sketch info2 := datamodel.NewEmptyInfo() info2.Properties.MaxUniqueItems = utils.Int64p(10000) info2.Name = utils.Stringp("dc") if err := m.CreateDomain(info2); err != nil { t.Error("Expected no errors, got", err) } if sketches := m.GetSketches(); len(sketches) != 8 { t.Error("Expected 8 sketches, got", len(sketches)) } else if sketches[0][0] != "dc" || sketches[0][1] != "card" { t.Error("Expected [[dc card]], got", sketches[0][0], sketches[0][1]) } else if sketches[1][0] != "dc" || sketches[1][1] != "freq" { t.Error("Expected [[dc freq]], got", sketches[1][0], sketches[1][1]) } }
func TestDeleteSketch(t *testing.T) { config.Reset() utils.SetupTests() defer utils.TearDownTests() m := NewManager() info := datamodel.NewEmptyInfo() typ := pb.SketchType_CARD info.Properties.MaxUniqueItems = utils.Int64p(10000) info.Name = utils.Stringp(fmt.Sprintf("marvel")) info.Type = &typ if err := m.CreateSketch(info); err != nil { t.Error("Expected no errors, got", err) } if sketches := m.GetSketches(); len(sketches) != 1 { t.Error("Expected 1 sketches, got", len(sketches)) } else if sketches[0][0] != "marvel" || sketches[0][1] != "card" { t.Error("Expected [[marvel card]], got", sketches) } if err := m.DeleteSketch(info.ID()); err != nil { t.Error("Expected no errors, got", err) } if sketches := m.GetSketches(); len(sketches) != 0 { t.Error("Expected 0 sketches, got", len(sketches)) } }
func TestAdd(t *testing.T) { utils.SetupTests() defer utils.TearDownTests() info := datamodel.NewEmptyInfo() info.Properties.MaxUniqueItems = utils.Int64p(1000000) info.Name = utils.Stringp("marvel") sketch, err := NewCMLSketch(info) if err != nil { t.Error("expected avengers to have no error, got", err) } values := [][]byte{ []byte("sabertooth"), []byte("thunderbolt"), []byte("havoc"), []byte("cyclops"), []byte("cyclops"), []byte("cyclops"), []byte("havoc")} if _, err := sketch.Add(values); err != nil { t.Error("expected no errors, got", err) } if res, err := sketch.Get([][]byte{[]byte("cyclops")}); err != nil { t.Error("expected no errors, got", err) } else if res.(*pb.FrequencyResult).Frequencies[0].GetCount() != 3 { t.Error("expected 'cyclops' count == 3, got", res.(*pb.FrequencyResult).Frequencies[0].GetCount()) } }
func TestStressHLLPP(t *testing.T) { utils.SetupTests() defer utils.TearDownTests() values := make([][]byte, 10) for i := 0; i < 1024; i++ { avenger := "avenger" + strconv.Itoa(i) values = append(values, []byte(avenger)) } for i := 0; i < 1024; i++ { info := datamodel.NewEmptyInfo() info.Properties.MaxUniqueItems = utils.Int64p(1024) info.Name = utils.Stringp("marvel" + strconv.Itoa(i)) sketch, err := NewHLLPPSketch(info) if err != nil { t.Error("expected avengers to have no error, got", err) } if _, err := sketch.Add(values); err != nil { t.Error("expected no errors, got", err) } } }
func TestAddBloom(t *testing.T) { testutils.SetupTests() defer testutils.TearDownTests() info := datamodel.NewEmptyInfo() info.Properties.MaxUniqueItems = utils.Int64p(1024) info.Name = utils.Stringp("marvel") sketch, err := NewBloomSketch(info) if err != nil { t.Error("expected avengers to have no error, got", err) } values := [][]byte{ []byte("sabertooth"), []byte("thunderbolt"), []byte("havoc"), []byte("cyclops"), []byte("cyclops"), []byte("cyclops"), []byte("havoc")} if _, err := sketch.Add(values); err != nil { t.Error("expected no errors, got", err) } check := map[string]bool{ "sabertooth": true, "thunderbolt": true, "havoc": true, "cyclops": true, "wolverine": false, "iceman": false, "rogue": false, "storm": false} if res, err := sketch.Get(values); err != nil { t.Error("expected no errors, got", err) } else { tmp := res.(*pb.MembershipResult) mres := tmp.GetMemberships() // FIXME: Flatten to avoid O(n^2) complexity for key := range check { for i := 0; i < len(mres); i++ { if mres[i].GetValue() == key && mres[i].GetIsMember() != check[key] { // FIXME: Use t.Errorf (using + is ugly) t.Error("expected member == "+strconv.FormatBool(check[key])+", got", mres[i].GetIsMember()) } } } } }
func TestCreateAndSaveSketch(t *testing.T) { config.Reset() testutils.SetupTests() defer testutils.TearDownTests() m := NewManager() info := datamodel.NewEmptyInfo() typ := pb.SketchType_CARD info.Properties.MaxUniqueItems = utils.Int64p(10000) info.Name = utils.Stringp(fmt.Sprintf("marvel")) info.Type = &typ if err := m.CreateSketch(info); err != nil { t.Error("Expected no errors, got", err) } if sketches := m.GetSketches(); len(sketches) != 1 { t.Error("Expected 1 sketches, got", len(sketches)) } else if sketches[0][0] != "marvel" || sketches[0][1] != "card" { t.Error("Expected [[marvel card]], got", sketches) } // Create a second Sketch info2 := datamodel.NewEmptyInfo() typ2 := pb.SketchType_RANK info2.Properties.MaxUniqueItems = utils.Int64p(10000) info2.Name = utils.Stringp(fmt.Sprintf("marvel")) info2.Type = &typ2 if err := m.CreateSketch(info2); err != nil { t.Error("Expected no errors, got", err) } if sketches := m.GetSketches(); len(sketches) != 2 { t.Error("Expected 2 sketches, got", len(sketches)) } else if sketches[0][0] != "marvel" || sketches[0][1] != "card" { t.Error("Expected [[marvel card]], got", sketches[0][0], sketches[0][1]) } else if sketches[1][0] != "marvel" || sketches[1][1] != "rank" { t.Error("Expected [[marvel rank]], got", sketches[1][0], sketches[1][1]) } }
func TestAddCMLThreshold(t *testing.T) { testutils.SetupTests() defer testutils.TearDownTests() info := datamodel.NewEmptyInfo() info.Properties.MaxUniqueItems = utils.Int64p(1024) info.Name = utils.Stringp("marvel") typ := pb.SketchType_FREQ info.Type = &typ sketch, err := NewCMLSketch(info) if err != nil { t.Error("expected avengers to have no error, got", err) } rValues := make(map[string]uint64) var sValues [][]byte thresholdSize := int64(sketch.threshold.size) for i := int64(0); i < info.GetProperties().GetMaxUniqueItems()/10; i++ { value := fmt.Sprintf("value-%d", i) freq := uint64(rand.Int63()) % 100 values := make([][]byte, freq, freq) for i := range values { values[i] = []byte(value) } if _, err := sketch.Add(values); err != nil { t.Error("expected no errors, got", err) } rValues[value] = freq sValues = append(sValues, []byte(value)) // Threshold should be nil once more than 10% is filled if sketch.threshold != nil && i >= thresholdSize { t.Error("expected threshold == nil for i ==", i) } if res, err := sketch.Get(sValues); err != nil { t.Error("expected no errors, got", err) } else { tmp := res.(*pb.FrequencyResult) mres := tmp.GetFrequencies() for i := 0; i < len(mres); i++ { if key := mres[i].GetValue(); rValues[key] != uint64(mres[i].GetCount()) { t.Fatalf("expected %s: %d, got %d", mres[i].GetValue(), rValues[key], uint64(mres[i].GetCount())) } } } } }
func TestCreateInvalidSketch(t *testing.T) { config.Reset() utils.SetupTests() defer utils.TearDownTests() m := NewManager() info := datamodel.NewEmptyInfo() info.Properties.MaxUniqueItems = utils.Int64p(10000) info.Name = utils.Stringp("avengers") info.Type = nil if err := m.CreateSketch(info); err == nil { t.Error("Expected error invalid sketch, got", err) } if sketches := m.GetSketches(); len(sketches) != 0 { t.Error("Expected 0 sketches, got", len(sketches)) } }
func TestAddHLLPPThreshold(t *testing.T) { testutils.SetupTests() defer testutils.TearDownTests() info := datamodel.NewEmptyInfo() info.Properties.MaxUniqueItems = utils.Int64p(1024) info.Name = utils.Stringp("marvel") typ := pb.SketchType_CARD info.Type = &typ sketch, err := NewHLLPPSketch(info) if err != nil { t.Error("expected avengers to have no error, got", err) } rValues := make(map[string]uint64) thresholdSize := int64(sketch.threshold.size) for i := int64(0); i < info.GetProperties().GetMaxUniqueItems()/10; i++ { freq := uint64(rand.Int63()) % 100 value := fmt.Sprintf("value-%d", i) values := make([][]byte, freq+1, freq+1) for i := range values { values[i] = []byte(value) } if _, err := sketch.Add(values); err != nil { t.Error("expected no errors, got", err) } rValues[value] = freq // Threshold should be nil once more than 10% is filled if sketch.threshold != nil && i >= thresholdSize { t.Error("expected threshold == nil for i ==", i) } if res, err := sketch.Get(nil); err != nil { t.Error("expected no errors, got", err) } else { tmp := res.(*pb.CardinalityResult) mres := tmp.GetCardinality() if int64(len(rValues)) != mres { t.Fatalf("expected cardinality %d, got %d", len(rValues), mres) } } } }
func TestAddBloomThreshold(t *testing.T) { testutils.SetupTests() defer testutils.TearDownTests() info := datamodel.NewEmptyInfo() info.Properties.MaxUniqueItems = utils.Int64p(1024) info.Name = utils.Stringp("marvel") sketch, err := NewBloomSketch(info) if err != nil { t.Error("expected avengers to have no error, got", err) } var rValues [][]byte thresholdSize := int64(sketch.threshold.size) for i := int64(0); i < info.GetProperties().GetMaxUniqueItems()/3; i++ { value := fmt.Sprintf("value-%d", i) values := [][]byte{ []byte(value), } if _, err := sketch.Add(values); err != nil { t.Error("expected no errors, got", err) } rValues = append(rValues, []byte(value)) // Threshold should be nil once more than 10% is filled if sketch.threshold != nil && i >= thresholdSize { t.Error("expected threshold == nil for i ==", i) } if res, err := sketch.Get(rValues); err != nil { t.Error("expected no errors, got", err) } else { tmp := res.(*pb.MembershipResult) mres := tmp.GetMemberships() for i := 0; i < len(mres); i++ { if !mres[i].GetIsMember() { t.Fatalf("expected %s ==> member == true, got false", mres[i].GetValue()) break } } } } }
func (s *serverStruct) add(ctx context.Context, in *pb.AddRequest) (*pb.AddReply, error) { info := datamodel.NewEmptyInfo() // FIXME: use domain or sketch directly and stop casting to Info if dom := in.GetDomain(); dom != nil { info.Name = dom.Name err := s.manager.AddToDomain(info.GetName(), in.GetValues()) if err != nil { return nil, err } } else if sketch := in.GetSketch(); sketch != nil { info := &datamodel.Info{Sketch: sketch} err := s.manager.AddToSketch(info.ID(), in.GetValues()) if err != nil { return nil, err } } return &pb.AddReply{}, nil }
func TestDeleteNonExistingSketch(t *testing.T) { config.Reset() utils.SetupTests() defer utils.TearDownTests() m := NewManager() info := datamodel.NewEmptyInfo() typ := pb.SketchType_CARD info.Properties.MaxUniqueItems = utils.Int64p(10000) info.Name = utils.Stringp(fmt.Sprintf("marvel")) info.Type = &typ if err := m.DeleteSketch(info.ID()); err == nil { t.Error("Expected errors deleting non-existing sketch, got", err) } if sketches := m.GetSketches(); len(sketches) != 0 { t.Error("Expected 0 sketches, got", len(sketches)) } }
func (s *serverStruct) createDomain(ctx context.Context, in *pb.Domain) (*pb.Domain, error) { info := datamodel.NewEmptyInfo() info.Name = in.Name // FIXME: A Domain's info should have an array of properties for each Sketch (or just an array // of Sketches, like what the proto has). This is just a hack to choose the first Sketch and // use it's info for now info.Properties.MaxUniqueItems = in.GetSketches()[0].GetProperties().MaxUniqueItems info.Properties.Size = in.GetSketches()[0].GetProperties().Size if info.Properties.Size == nil || *info.Properties.Size == 0 { var defaultSize int64 = 100 info.Properties.Size = &defaultSize } // FIXME: We should be passing a pb.Domain and not a datamodel.Info to manager.CreateDomain err := s.manager.CreateDomain(info) if err != nil { return nil, err } return in, nil }
func TestMembershipSaveLoad(t *testing.T) { config.Reset() utils.SetupTests() defer utils.TearDownTests() m := NewManager() info := datamodel.NewEmptyInfo() typ := pb.SketchType_MEMB info.Properties.MaxUniqueItems = utils.Int64p(1000) info.Name = utils.Stringp(fmt.Sprintf("marvel")) info.Type = &typ if err := m.CreateSketch(info); err != nil { t.Error("Expected no errors, got", err) } if sketches := m.GetSketches(); len(sketches) != 1 { t.Error("Expected 1 sketch, got", len(sketches)) } else if sketches[0][0] != "marvel" || sketches[0][1] != "memb" { t.Error("Expected [[marvel memb]], got", sketches) } if err := m.AddToSketch(info.ID(), []string{"hulk", "hulk", "thor", "iron man", "hawk-eye"}); err != nil { t.Error("Expected no errors, got", err) } if err := m.AddToSketch(info.ID(), []string{"hulk", "black widow", "black widow", "black widow", "black widow"}); err != nil { t.Error("Expected no errors, got", err) } if res, err := m.GetFromSketch(info.ID(), []string{"hulk", "captian america", "black widow"}); err != nil { t.Error("Expected no errors, got", err) } else if len(res.(*pb.MembershipResult).GetMemberships()) != 3 { t.Error("Expected len(res) = 3, got", len(res.(*pb.MembershipResult).GetMemberships())) } else if v := res.(*pb.MembershipResult).GetMemberships()[0].GetIsMember(); !v { t.Error("Expected 'hulk' == true , got", v) } else if v := res.(*pb.MembershipResult).GetMemberships()[1].GetIsMember(); v { t.Error("Expected 'captian america' == false , got", v) } else if v := res.(*pb.MembershipResult).GetMemberships()[2].GetIsMember(); !v { t.Error("Expected 'captian america' == true , got", v) } }
func TestAddHLLPP(t *testing.T) { testutils.SetupTests() defer testutils.TearDownTests() info := datamodel.NewEmptyInfo() info.Properties.MaxUniqueItems = utils.Int64p(1024) info.Name = utils.Stringp("marvel") typ := pb.SketchType_CARD info.Type = &typ sketch, err := NewHLLPPSketch(info) if err != nil { t.Error("expected avengers to have no error, got", err) } values := [][]byte{ []byte("sabertooth"), []byte("thunderbolt"), []byte("havoc"), []byte("cyclops"), []byte("cyclops"), []byte("cyclops"), []byte("havoc")} if _, err := sketch.Add(values); err != nil { t.Error("expected no errors, got", err) } const expectedCardinality int64 = 4 if res, err := sketch.Get(values); err != nil { t.Error("expected no errors, got", err) } else { tmp := res.(*pb.CardinalityResult) mres := tmp.GetCardinality() if mres != int64(expectedCardinality) { t.Error("expected cardinality == "+strconv.FormatInt(expectedCardinality, 10)+", got", mres) } } }
func BenchmarkHLLPP(b *testing.B) { values := make([][]byte, 10) for i := 0; i < 1024; i++ { avenger := "avenger" + strconv.Itoa(i) values = append(values, []byte(avenger)) } for n := 0; n < b.N; n++ { info := datamodel.NewEmptyInfo() info.Properties.Size = utils.Int64p(1000) info.Name = utils.Stringp("marvel3") sketch, err := NewHLLPPSketch(info) if err != nil { b.Error("expected no errors, got", err) } for i := 0; i < 1000; i++ { if _, err := sketch.Add(values); err != nil { b.Error("expected no errors, got", err) } } } }
func TestRankSaveLoad(t *testing.T) { config.Reset() utils.SetupTests() defer utils.TearDownTests() m := NewManager() info := datamodel.NewEmptyInfo() typ := pb.SketchType_RANK info.Properties.Size = utils.Int64p(10) info.Name = utils.Stringp(fmt.Sprintf("marvel")) info.Type = &typ if err := m.CreateSketch(info); err != nil { t.Error("Expected no errors, got", err) } if sketches := m.GetSketches(); len(sketches) != 1 { t.Error("Expected 1 sketch, got", len(sketches)) } else if sketches[0][0] != "marvel" || sketches[0][1] != "rank" { t.Error("Expected [[marvel rank]], got", sketches) } if err := m.AddToSketch(info.ID(), []string{"hulk", "hulk", "thor", "iron man", "hawk-eye"}); err != nil { t.Error("Expected no errors, got", err) } if err := m.AddToSketch(info.ID(), []string{"hulk", "black widow", "black widow", "black widow", "black widow"}); err != nil { t.Error("Expected no errors, got", err) } if res, err := m.GetFromSketch(info.ID(), nil); err != nil { t.Error("Expected no errors, got", err) } else if len(res.(*pb.RankingsResult).GetRankings()) != 5 { t.Error("Expected len(res) = 5, got", len(res.(*pb.RankingsResult).GetRankings())) } else if res.(*pb.RankingsResult).GetRankings()[0].GetValue() != "black widow" { t.Error("Expected 'black widow', got", res.(*pb.RankingsResult).GetRankings()[0].GetValue()) } }
func BenchmarkBloom(b *testing.B) { utils.SetupTests() defer utils.TearDownTests() values := make([][]byte, 10) for i := 0; i < 1024; i++ { avenger := "avenger" + strconv.Itoa(i) values = append(values, []byte(avenger)) } for n := 0; n < b.N; n++ { info := datamodel.NewEmptyInfo() info.Properties.MaxUniqueItems = utils.Int64p(1000) info.Name = utils.Stringp("marvel") sketch, err := NewBloomSketch(info) if err != nil { b.Error("expected no errors, got", err) } for i := 0; i < 1000; i++ { if _, err := sketch.Add(values); err != nil { b.Error("expected no errors, got", err) } } } }
func TestFreqSaveLoad(t *testing.T) { config.Reset() utils.SetupTests() defer utils.TearDownTests() m := NewManager() info := datamodel.NewEmptyInfo() typ := pb.SketchType_FREQ info.Properties.MaxUniqueItems = utils.Int64p(10000) info.Name = utils.Stringp(fmt.Sprintf("marvel")) info.Type = &typ if err := m.CreateSketch(info); err != nil { t.Error("Expected no errors, got", err) } if sketches := m.GetSketches(); len(sketches) != 1 { t.Error("Expected 1 sketch, got", len(sketches)) } else if sketches[0][0] != "marvel" || sketches[0][1] != "freq" { t.Error("Expected [[marvel freq]], got", sketches) } if err := m.AddToSketch(info.ID(), []string{"hulk", "thor", "iron man", "hawk-eye"}); err != nil { t.Error("Expected no errors, got", err) } if err := m.AddToSketch(info.ID(), []string{"hulk", "black widow"}); err != nil { t.Error("Expected no errors, got", err) } if res, err := m.GetFromSketch(info.ID(), []string{"hulk", "thor", "iron man", "hawk-eye"}); err != nil { t.Error("Expected no errors, got", err) } else if res.(*pb.FrequencyResult).GetFrequencies()[0].GetCount() != 2 { t.Error("Expected res = 2, got", res.(*pb.FrequencyResult).GetFrequencies()[0].GetCount()) } }
func TestAddTopK(t *testing.T) { utils.SetupTests() defer utils.TearDownTests() info := datamodel.NewEmptyInfo() info.Properties.Size = utils.Int64p(3) info.Name = utils.Stringp("marvel") sketch, err := NewTopKSketch(info) if err != nil { t.Error("expected avengers to have no error, got", err) } values := [][]byte{ []byte("sabertooth"), []byte("thunderbolt"), []byte("thunderbolt"), []byte("havoc"), []byte("cyclops"), []byte("cyclops"), []byte("cyclops"), []byte("havoc")} if _, err := sketch.Add(values); err != nil { t.Error("expected no errors, got", err) } type RankingsStruct struct { Value string Position int64 Count int64 } expectedRankings := make([]*RankingsStruct, 4, 4) expectedRankings[0] = &RankingsStruct{ Value: "cyclops", Count: 3, Position: 1, } expectedRankings[1] = &RankingsStruct{ Value: "thunderbolt", Count: 2, Position: 2, } expectedRankings[2] = &RankingsStruct{ Value: "havoc", Count: 2, Position: 3, } expectedRankings[3] = &RankingsStruct{ Value: "sabertooth", Count: 1, Position: 4, } if res, err := sketch.Get(values); err != nil { t.Error("expected no errors, got", err) } else { tmp := res.(*pb.RankingsResult) rres := tmp.GetRankings() for i := 0; i < len(rres); i++ { count := rres[i].GetCount() value := rres[i].GetValue() for j := 0; j < len(expectedRankings); j++ { if expectedRankings[j].Value == value && expectedRankings[j].Count != count && expectedRankings[j].Position != int64(i) { t.Error("expected ranking == "+strconv.FormatInt(expectedRankings[j].Position, 10)+", got", count) } } } } }