Exemplo n.º 1
0
func TestSaveSnapshot(t *testing.T) {
	const (
		site     = "TestUpdateReadDeletePackageHistory.com"
		path     = "gcse"
		foundWay = "testing"
	)
	assert.NoError(t, UpdatePackageHistory(site, path, func(info *sppb.HistoryInfo) error {
		assert.Equal(t, "info", info, &sppb.HistoryInfo{})
		info.FoundWay = foundWay
		return nil
	}))
	h, err := ReadPackageHistory(site, path)
	assert.NoError(t, err)
	assert.Equal(t, "h", h, &sppb.HistoryInfo{FoundWay: foundWay})

	outPath := villa.Path(os.TempDir()).Join("TestSaveSnapshot").S()
	assert.NoError(t, SaveSnapshot(outPath))
	box := &bh.RefCountBox{
		DataPath: func() string {
			return outPath
		},
	}
	h, err = ReadPackageHistoryOf(box, site, path)
	assert.NoError(t, err)
	assert.Equal(t, "h", h, &sppb.HistoryInfo{FoundWay: foundWay})
}
Exemplo n.º 2
0
func TestIndex_DescNotIndexedBug(t *testing.T) {
	const (
		description = "description"
		readme      = "readme"
	)
	hits := []HitInfo{{
		DocInfo: DocInfo{
			Package:     "github.com/daviddengcn/gcse",
			Name:        "gcse",
			Description: description,
			ReadmeData:  readme,
		},
	}}
	idxs := []int{0}
	fullHitSaved := 0
	ts := &index.TokenSetSearcher{}
	assert.NoError(t, indexAndSaveHits(ts, hits, idxs, func(hit *HitInfo) error {
		fullHitSaved++
		assert.Equal(t, "Description", hit.Description, description)
		assert.Equal(t, "Readme", hit.ReadmeData, readme)
		return nil
	}))
	assert.Equal(t, "fullHitSaved", fullHitSaved, 1)
	results, err := search(ts, IndexTextField, description)
	assert.NoError(t, err)
	assert.Equal(t, "results", results, hits)

	results, err = search(ts, IndexTextField, readme)
	assert.NoError(t, err)
	assert.Equal(t, "results", results, hits)
}
Exemplo n.º 3
0
func TestStrSet_nil(t *testing.T) {
	var s, ss StrSet
	assert.Equal(t, "nil.In(david)", s.In("david"), false)
	assert.Equal(t, "nil.Equals(nil)", s.Equals(ss), true)
	assert.StringEqual(t, "nil.Elements()", s.Elements(), StringSlice{})
	s.Delete("david")
}
Exemplo n.º 4
0
func TestRefCountBox_UpdateView(t *testing.T) {
	fn := path.Join(os.TempDir(), "TestRefCountBox_UpdateView.bolt")
	assert.NoError(t, os.RemoveAll(fn))

	b := RefCountBox{
		DataPath: func() string { return fn },
	}
	k := bytes.Split([]byte("a.b"), []byte("."))
	v := "hello"
	assert.NoErrorOrDie(t, b.Update(func(tx Tx) error {
		return tx.Put(k, []byte(v))
	}))
	assert.Equal(t, "b.count", b.count, 0)
	assert.ValueShould(t, "b.db.DB", b.db.DB, b.db.DB == nil, "is not nil")

	found := false
	assert.NoErrorOrDie(t, b.View(func(tx Tx) error {
		return tx.Value(k, func(bs bytesp.Slice) error {
			found = true
			assert.Equal(t, "bs", string(v), string(bs))
			return nil
		})
	}))
	assert.Equal(t, "b.count", b.count, 0)
	assert.ValueShould(t, "b.db.DB", b.db.DB, b.db.DB == nil, "is not nil")
}
Exemplo n.º 5
0
func TestCheckPackageStatus(t *testing.T) {
	// No crawling info, new package
	assert.Equal(t, "CheckPackageStatus", CheckPackageStatus(&stpb.PackageInfo{}, nil), OutOfDate)
	pkgCrawlTime, _ := ptypes.TimestampProto(time.Now().Add(-5 * timep.Day))

	newRepoInfoCrawlTime, _ := ptypes.TimestampProto(time.Now().Add(-3 * timep.Day))
	newPkgUpdateTime, _ := ptypes.TimestampProto(time.Now().Add(-4 * timep.Day))
	assert.Equal(t, "CheckPackageStatus", CheckPackageStatus(&stpb.PackageInfo{
		CrawlingInfo: &sppb.CrawlingInfo{
			CrawlingTime: pkgCrawlTime,
		},
	}, &sppb.RepoInfo{
		CrawlingTime: newRepoInfoCrawlTime,
		LastUpdated:  newPkgUpdateTime,
	}), OutOfDate)

	newPkgUpdateTime, _ = ptypes.TimestampProto(time.Now().Add(-6 * timep.Day))
	assert.Equal(t, "CheckPackageStatus", CheckPackageStatus(&stpb.PackageInfo{
		CrawlingInfo: &sppb.CrawlingInfo{
			CrawlingTime: pkgCrawlTime,
		},
	}, &sppb.RepoInfo{
		CrawlingTime: newRepoInfoCrawlTime,
		LastUpdated:  newPkgUpdateTime,
	}), UpToDate)
}
Exemplo n.º 6
0
func TestAggregateMethod_String(t *testing.T) {
	assert.Equal(t, "String()", Sum.String(), "Sum")
	assert.Equal(t, "String()", Max.String(), "Max")
	assert.Equal(t, "String()", Min.String(), "Min")
	assert.Equal(t, "String()", Average.String(), "Average")
	assert.Equal(t, "String()", AggregateMethod(-1).String(), "-1(invalid)")
}
Exemplo n.º 7
0
func TestByteSlice_Bug_Read(t *testing.T) {
	var s ByteSlice
	n, err := s.Read(make([]byte, 1))
	t.Logf("n: %d, err: %v", n, err)
	assert.Equal(t, "n", 0, 0)
	assert.Equal(t, "err", err, io.EOF)
}
Exemplo n.º 8
0
func TestCounter_Append(t *testing.T) {
	tests := []struct {
		sum       int
		div       int
		aggr      AggregateMethod
		value     int
		final_sum int
		final_div int
	}{
		{sum: 11, div: 4, aggr: Sum, value: 2, final_sum: 5, final_div: 1},
		{sum: 11, div: 4, aggr: Min, value: 2, final_sum: 2, final_div: 1},
		{sum: 11, div: 4, aggr: Max, value: 5, final_sum: 5, final_div: 1},
		{sum: 11, div: 4, aggr: Average, value: 2, final_sum: 13, final_div: 5},
	}
	for _, test := range tests {
		t.Logf("test: %v, %v", test, test.aggr)
		c := Counter{
			Sum: test.sum,
			Div: test.div,
		}
		c.Append(test.aggr, test.value)
		assert.Equal(t, "sum", c.Sum, test.final_sum)
		assert.Equal(t, "div", c.Div, test.final_div)
	}
}
Exemplo n.º 9
0
func TestForEachPackageOfSite(t *testing.T) {
	// Clear the database
	configs.StoreBoltPath().RemoveAll()

	const (
		site  = "TestForEachPackageOfSite.com"
		path1 = "gcse"
		name1 = "pkgname"
		path2 = "gcse2"
		name2 = "TestForEachPackageOfSite"
	)
	assert.NoError(t, UpdatePackage(site, path1, func(info *stpb.PackageInfo) error {
		info.Name = name1
		return nil
	}))
	assert.NoError(t, UpdatePackage(site, path2, func(info *stpb.PackageInfo) error {
		info.Name = name2
		return nil
	}))
	var paths, names []string
	assert.NoError(t, ForEachPackageOfSite(site, func(path string, info *stpb.PackageInfo) error {
		paths = append(paths, path)
		names = append(names, info.Name)
		return nil
	}))
	assert.Equal(t, "paths", paths, []string{path1, path2})
	assert.Equal(t, "names", names, []string{name1, name2})
}
Exemplo n.º 10
0
func TestInc(t *testing.T) {
	DataPath = "./tmp.bolt"
	FlushPeriod = time.Second
	err := os.RemoveAll(DataPath)
	if err != nil && err != os.ErrNotExist {
		assert.NoError(t, err)
		return
	}

	now := time.Now()
	Inc("abc")
	Flush()
	Process()

	names, err := ReadNames()
	assert.NoError(t, err)
	assert.Equal(t, "names", names, []string{"abc"})

	data, err := ReadDataOfName("daily", "abc")
	assert.NoError(t, err)
	assert.Equal(t, "data", data, []LabeledCounter{{
		Counter{
			Sum: 1,
			Div: 1,
		},
		now.Format("2006-01-02"),
	}})
}
Exemplo n.º 11
0
func TestMaxStackDepth(t *testing.T) {
	MaxStackDepth = 1
	err := WithStacks(fmt.Errorf("myerror")).(*ErrorWithStacks)
	assert.Equal(t, "len(err.Stacks)", len(err.Stacks), 2)
	assert.Equal(t, "err.Stacks[1]", err.Stacks[1], "...")
	assert.Equal(t, "lines of err.Error()", len(strings.Split(err.Error(), "\n")), 3)
}
Exemplo n.º 12
0
func TestI(t *testing.T) {
	assert.Equal(t, "I(123)", I(123), 123)
	assert.Equal(t, "I(int64(123))", I(int64(123)), 123)
	assert.Equal(t, "I(abc)", I("abc"), 0)
	assert.Equal(t, "I(abc, 50)", I("abc", 50), 50)
	assert.Equal(t, "I(abc, def)", I("abc", "def"), 0)
}
Exemplo n.º 13
0
func TestWithStacks(t *testing.T) {
	origin := fmt.Errorf("myerror")
	wrapped, isErrWithStacks := WithStacks(origin).(*ErrorWithStacks)
	assert.True(t, "isErrWithStacks", isErrWithStacks)
	assert.Equal(t, "wrapped.Err", wrapped.Err, origin)

	wrapped2, isErrWithStacks := WithStacks(wrapped).(*ErrorWithStacks)
	assert.True(t, "isErrWithStacks", isErrWithStacks)
	assert.Equal(t, "wrapped2", wrapped2, wrapped)
}
Exemplo n.º 14
0
func TestRefCountBox_UnsetDataPath(t *testing.T) {
	fn := path.Join(os.TempDir(), "TestRefCountBox.bolt")
	assert.NoError(t, os.RemoveAll(fn))

	b := RefCountBox{}
	db, err := b.Alloc()
	assert.Equal(t, "err", errorsp.Cause(err), ErrBoxDataPathNotSpecified)
	assert.ValueShould(t, "db.DB", db.DB, db.DB == nil, "is not nil")
	assert.ValueShould(t, "b.db.DB", b.db.DB, b.db.DB == nil, "is not nil")
	assert.Equal(t, "b.count", b.count, 0)
}
Exemplo n.º 15
0
func TestWithStacksAndMessage(t *testing.T) {
	origin := fmt.Errorf("origin")
	wrapped, isErrWithStacks := WithStacksAndMessage(origin, "%s:%d", "a", 123).(*ErrorWithStacks)
	assert.True(t, "isErrWithStacks", isErrWithStacks)
	assert.Equal(t, "wrapped.Err", wrapped.Err, origin)
	assert.ValueShould(t, "wrapped.Stacks[0]", wrapped.Stacks[0], strings.HasSuffix(wrapped.Stacks[0], ": a:123"), "does not contain message correctly")

	wrapped2, isErrWithStacks := WithStacksAndMessage(wrapped, "%s:%d", "b", 456).(*ErrorWithStacks)
	assert.True(t, "isErrWithStacks", isErrWithStacks)
	assert.Equal(t, "wrapped.Err", wrapped2.Err, origin)
	assert.ValueShould(t, "wrapped2.Stacks[0]", wrapped2.Stacks[0], strings.HasSuffix(wrapped2.Stacks[0], ": a:123: b:456"), "does not contain message correctly")
}
Exemplo n.º 16
0
func TestSlice_ReadFrom(t *testing.T) {
	src := make([]byte, 63*1024)
	for i := range src {
		src[i] = byte(i)
	}
	b := bytes.NewBuffer(src)
	var s Slice
	n, err := s.ReadFrom(b)
	assert.Equal(t, "n", n, int64(len(src)))
	assert.NoError(t, err)
	assert.Equal(t, "s", s, Slice(src))
}
Exemplo n.º 17
0
func TestMoveData(t *testing.T) {
	DataPath = "./tmp.bolt"
	FlushPeriod = time.Second
	err := os.RemoveAll(DataPath)
	if err != nil && err != os.ErrNotExist {
		assert.NoError(t, err)
		return
	}

	now := time.Now()
	checkValue := func(tp, name string, sum, div int) {
		data, err := ReadDataOfName(tp, name)
		assert.NoError(t, err)
		if sum == 0 && div == 0 {
			assert.Equal(t, fmt.Sprintf("%v:%v", name, tp), len(data), 0)
			return
		}
		assert.Equal(t, fmt.Sprintf("%v:%v", name, tp), data, []LabeledCounter{{
			Counter{Sum: sum, Div: div},
			now.Format("2006-01-02"),
		}})
	}
	checkNames := func(exp []string) {
		names, err := ReadNames()
		assert.NoError(t, err)
		assert.Equal(t, "names", names, exp)
	}

	AddValue(Average, "TestMoveData-abc", 123)
	AddValue(Average, "TestMoveData-def", 456)
	Flush()
	Process()

	t.Log("before move")
	checkNames([]string{"TestMoveData-abc", "TestMoveData-def"})
	checkValue(Daily, "TestMoveData-abc", 123, 1)
	checkValue(Daily, "TestMoveData-def", 456, 1)

	assert.NoError(t, MoveData("TestMoveData-abc", "TestMoveData-def", Average))
	t.Log("after move from abc to def")
	checkNames([]string{"TestMoveData-def"})
	checkValue(Daily, "TestMoveData-abc", 0, 0)
	checkValue(Daily, "TestMoveData-def", 579, 2)

	assert.NoError(t, MoveData("TestMoveData-def", "TestMoveData-none", Average))
	t.Log("after move from def to none")
	checkNames([]string{"TestMoveData-none"})
	checkValue(Daily, "TestMoveData-abc", 0, 0)
	checkValue(Daily, "TestMoveData-def", 0, 0)
	checkValue(Daily, "TestMoveData-none", 579, 2)
}
Exemplo n.º 18
0
func TestAppendPackageEvent(t *testing.T) {
	const (
		site     = "TestAppendPackageEvent.com"
		path     = "gcse"
		foundWay = "test"
	)
	// Insert a found only event, no action.
	foundTm := time.Now()
	foundTs, _ := ptypes.TimestampProto(foundTm)
	assert.NoError(t, AppendPackageEvent(site, path, "test", foundTm, sppb.HistoryEvent_Action_None))
	h, err := ReadPackageHistory(site, path)
	assert.NoError(t, err)
	assert.Equal(t, "h", h, &sppb.HistoryInfo{FoundWay: foundWay, FoundTime: foundTs})

	// Inser a Success action
	succTm := foundTm.Add(time.Hour)
	succTs, _ := ptypes.TimestampProto(succTm)
	assert.NoError(t, AppendPackageEvent(site, path, "non-test", succTm, sppb.HistoryEvent_Action_Success))
	h, err = ReadPackageHistory(site, path)
	assert.NoError(t, err)
	assert.Equal(t, "h", h, &sppb.HistoryInfo{
		FoundWay:  foundWay,
		FoundTime: foundTs,
		Events: []*sppb.HistoryEvent{{
			Timestamp: succTs,
			Action:    sppb.HistoryEvent_Action_Success,
		}},
		LatestSuccess: succTs,
	})
	// Inser a Failed action
	failedTm := succTm.Add(time.Hour)
	failedTs, _ := ptypes.TimestampProto(failedTm)
	assert.NoError(t, AppendPackageEvent(site, path, "", failedTm, sppb.HistoryEvent_Action_Failed))
	h, err = ReadPackageHistory(site, path)
	assert.NoError(t, err)
	assert.Equal(t, "h", h, &sppb.HistoryInfo{
		FoundWay:  foundWay,
		FoundTime: foundTs,
		Events: []*sppb.HistoryEvent{{
			Timestamp: failedTs,
			Action:    sppb.HistoryEvent_Action_Failed,
		}, {
			Timestamp: succTs,
			Action:    sppb.HistoryEvent_Action_Success,
		}},
		LatestSuccess: succTs,
		LatestFailed:  failedTs,
	})
}
Exemplo n.º 19
0
func TestSlice_ReadFromEOF(t *testing.T) {
	r1 := readerReturningEOFLastRead(1)
	var s Slice
	n, err := s.ReadFrom(&r1)
	assert.Equal(t, "n", n, int64(1))
	assert.NoError(t, err)
	assert.Equal(t, "s", s, Slice([]byte{1}))

	r2 := readerReturningEOFAfterLastRead(1)
	s = nil
	n, err = s.ReadFrom(&r2)
	assert.Equal(t, "n", n, int64(1))
	assert.NoError(t, err)
	assert.Equal(t, "s", s, Slice([]byte{2}))
}
Exemplo n.º 20
0
func TestStringFull(t *testing.T) {
	test := func(a, b string, d int, lcs string) {
		actD, actLcs := StringFull(a, b)
		assert.Equal(t, fmt.Sprintf("Edit-distance between %s and %s", a, b), actD, d)
		assert.Equal(t, fmt.Sprintf("Longest-common-string between %s and %s", a, b), actLcs, lcs)
	}

	test("abcd", "bcde", 2, "bcd")
	test("abcde", "", 5, "")
	test("", "abcde", 5, "")
	test("", "", 0, "")
	test("abcde", "abcde", 0, "abcde")
	test("abcde", "dabce", 2, "abce")
	test("abcde", "abfde", 1, "abde")
}
Exemplo n.º 21
0
func TestBoltFileCache(t *testing.T) {
	fn := filepath.Join(os.TempDir(), "TestBoltFileCache.bolt")
	assert.NoErrorOrDie(t, os.RemoveAll(fn))

	db, err := bh.Open(fn, 0755, nil)
	assert.NoErrorOrDie(t, err)

	counter := make(map[string]int)
	c := BoltFileCache{
		DB: db,
		IncCounter: func(name string) {
			counter[name] = counter[name] + 1
		},
	}
	const (
		sign1      = "abc"
		sign2      = "def"
		sign3      = "ghi"
		gofile     = "file.go"
		rootfolder = "root"
		sub        = "sub"
		subfolder  = "root/sub"
	)
	fi := &sppb.GoFileInfo{}

	//////////////////////////////////////////////////////////////
	// New file found.
	//////////////////////////////////////////////////////////////
	// Get before set, should return false
	assert.False(t, "c.Get", c.Get(sign1, fi))
	assert.Equal(t, "counter", counter, map[string]int{
		"crawler.filecache.missed": 1,
	})
	// Set the info.
	c.Set(sign1, &sppb.GoFileInfo{Status: sppb.GoFileInfo_ShouldIgnore})
	assert.Equal(t, "counter", counter, map[string]int{
		"crawler.filecache.missed":     1,
		"crawler.filecache.sign_saved": 1,
	})
	// Now, should fetch the cache
	assert.True(t, "c.Get", c.Get(sign1, fi))
	assert.Equal(t, "fi", fi, &sppb.GoFileInfo{Status: sppb.GoFileInfo_ShouldIgnore})
	assert.Equal(t, "counter", counter, map[string]int{
		"crawler.filecache.missed":     1,
		"crawler.filecache.sign_saved": 1,
		"crawler.filecache.hit":        1,
	})
}
Exemplo n.º 22
0
func TestSliceRemove(t *testing.T) {
	defer __(o_(t))

	var s Slice
	s.Add(1, 2, 3, 4, 5, 6, 7)
	assert.Equal(t, "len(s)", len(s), 7)
	assert.StringEqual(t, "s", s, "[1 2 3 4 5 6 7]")

	s.RemoveRange(2, 5)
	assert.Equal(t, "len(s)", len(s), 4)
	assert.StringEqual(t, "s", s, "[1 2 6 7]")

	s.Remove(2)
	assert.Equal(t, "len(s)", len(s), 3)
	assert.StringEqual(t, "s", s, "[1 2 7]")
}
Exemplo n.º 23
0
func TestStringSliceRemove(t *testing.T) {
	defer __(o_(t))

	var s StringSlice
	s.Add("A", "B", "C", "D", "E", "F", "G")
	assert.Equal(t, "len(s)", len(s), 7)
	assert.StringEqual(t, "s", s, "[A B C D E F G]")

	s.RemoveRange(2, 5)
	assert.Equal(t, "len(s)", len(s), 4)
	assert.StringEqual(t, "s", s, "[A B F G]")

	s.Remove(2)
	assert.Equal(t, "len(s)", len(s), 3)
	assert.StringEqual(t, "s", s, "[A B G]")
}
Exemplo n.º 24
0
func TestTokenize(t *testing.T) {
	text := []byte("abc 3d 中文输入")
	tokens := AppendTokens(nil, text)
	assert.Equal(t, "tokens", tokens,
		stringsp.NewSet("abc", "3", "d", "3-d", "中", "文", "输", "入", "中文",
			"文输", "输入"))
}
Exemplo n.º 25
0
func TestTokenize2(t *testing.T) {
	text := []byte("PubSubHub")
	tokens := AppendTokens(nil, text)
	assert.Equal(t, "tokens", tokens,
		stringsp.NewSet("pub", "sub", "hub", "pubsub", "subhub",
			"pubsubhub"))
}
Exemplo n.º 26
0
func TestMemDB_Recover(t *testing.T) {
	path := villa.Path(".").Join("testmemdb.gob")
	if path.Exists() {
		path.Remove()
	}

	db := NewMemDB(".", "testmemdb")
	db.Put("s", 1)
	if err := db.Sync(); err != nil {
		t.Error(err)
		return
	}

	if err := path.Rename(path + ".new"); err != nil {
		t.Error(err)
		return
	}
	// Now in the status of fn.new exists, fn not exist

	if err := db.Load(); err != nil {
		t.Error(err)
		return
	}
	var vl int
	if ok := db.Get("s", &vl); !ok {
		t.Error("Recover failed!")
		return
	}
	assert.Equal(t, "vl", vl, 1)
}
Exemplo n.º 27
0
func TestByteSlice_WriteItoa(t *testing.T) {
	var s ByteSlice
	s.WriteItoa(1234, 10)
	s.WriteItoa(255, 16)

	assert.Equal(t, "s", string(s), "1234ff")
}
Exemplo n.º 28
0
func TestIntMatrix(t *testing.T) {
	defer __(o_(t))

	mat := NewIntMatrix(5, 4)
	assert.Equal(t, "mat.Rows()", mat.Rows(), 5)
	assert.Equal(t, "mat.Cols()", mat.Cols(), 4)
	assert.StringEqual(t, "mat", mat, "[[0 0 0 0] [0 0 0 0] [0 0 0 0] [0 0 0 0] [0 0 0 0]]")

	mat.Fill(10)
	assert.StringEqual(t, "mat", mat, "[[10 10 10 10] [10 10 10 10] [10 10 10 10] [10 10 10 10] [10 10 10 10]]")

	mat[1][1] = 0
	mat[3][2] = 12345
	mat[2][0] = -998
	t.Logf("%s", mat.PrettyString())
}
Exemplo n.º 29
0
func TestUnmarshalFile(t *testing.T) {
	fn := path.Join(os.TempDir(), "TestMarshalToFile.json")
	assert.NoError(t, ioutil.WriteFile(fn, []byte(`{"S":"abc"}`), 0644))
	var s T
	assert.NoError(t, UnmarshalFile(fn, &s))
	assert.Equal(t, "s.S", s.S, "abc")
}
Exemplo n.º 30
0
func TestDocDB_Export(t *testing.T) {
	var db DocDB = PackedDocDB{NewMemDB("", "")}

	info := DocInfo{
		Name: "github.com/daviddengcn/gcse",
	}

	db.Put("go", info)

	if err := db.Export(villa.Path("."), "testexport_db"); err != nil {
		t.Errorf("db.Export failed: %v", err)
		return
	}

	var newDB DocDB = PackedDocDB{NewMemDB(villa.Path("."), "testexport_db")}
	count := 0
	if err := newDB.Iterate(func(key string, val interface{}) error {
		info, ok := val.(DocInfo)
		if !ok {
			return errors.New("Not a DocInfo object")
		}
		assert.StringEqual(t, "info.Name", info.Name,
			"github.com/daviddengcn/gcse")
		count++
		return nil
	}); err != nil {
		t.Errorf("newDB.Iterate failed: %v", err)
	}

	assert.Equal(t, "count", count, 1)
}