func TestSaveSnapshot(t *testing.T) {
const (
site = "TestUpdateReadDeletePackageHistory.com"
path = "gcse"
foundWay = "testing"
)
assert.NoError(t, UpdatePackageHistory(site, path, func(info *sppb.HistoryInfo) error {
assert.Equal(t, "info", info, &sppb.HistoryInfo{})
info.FoundWay = foundWay
return nil
}))
h, err := ReadPackageHistory(site, path)
assert.NoError(t, err)
assert.Equal(t, "h", h, &sppb.HistoryInfo{FoundWay: foundWay})
outPath := villa.Path(os.TempDir()).Join("TestSaveSnapshot").S()
assert.NoError(t, SaveSnapshot(outPath))
box := &bh.RefCountBox{
DataPath: func() string {
return outPath
},
}
h, err = ReadPackageHistoryOf(box, site, path)
assert.NoError(t, err)
assert.Equal(t, "h", h, &sppb.HistoryInfo{FoundWay: foundWay})
}
func TestIndex_DescNotIndexedBug(t *testing.T) {
const (
description = "description"
readme = "readme"
)
hits := []HitInfo{{
DocInfo: DocInfo{
Package: "github.com/daviddengcn/gcse",
Name: "gcse",
Description: description,
ReadmeData: readme,
},
}}
idxs := []int{0}
fullHitSaved := 0
ts := &index.TokenSetSearcher{}
assert.NoError(t, indexAndSaveHits(ts, hits, idxs, func(hit *HitInfo) error {
fullHitSaved++
assert.Equal(t, "Description", hit.Description, description)
assert.Equal(t, "Readme", hit.ReadmeData, readme)
return nil
}))
assert.Equal(t, "fullHitSaved", fullHitSaved, 1)
results, err := search(ts, IndexTextField, description)
assert.NoError(t, err)
assert.Equal(t, "results", results, hits)
results, err = search(ts, IndexTextField, readme)
assert.NoError(t, err)
assert.Equal(t, "results", results, hits)
}
func TestStrSet_nil(t *testing.T) {
var s, ss StrSet
assert.Equal(t, "nil.In(david)", s.In("david"), false)
assert.Equal(t, "nil.Equals(nil)", s.Equals(ss), true)
assert.StringEqual(t, "nil.Elements()", s.Elements(), StringSlice{})
s.Delete("david")
}
func TestRefCountBox_UpdateView(t *testing.T) {
fn := path.Join(os.TempDir(), "TestRefCountBox_UpdateView.bolt")
assert.NoError(t, os.RemoveAll(fn))
b := RefCountBox{
DataPath: func() string { return fn },
}
k := bytes.Split([]byte("a.b"), []byte("."))
v := "hello"
assert.NoErrorOrDie(t, b.Update(func(tx Tx) error {
return tx.Put(k, []byte(v))
}))
assert.Equal(t, "b.count", b.count, 0)
assert.ValueShould(t, "b.db.DB", b.db.DB, b.db.DB == nil, "is not nil")
found := false
assert.NoErrorOrDie(t, b.View(func(tx Tx) error {
return tx.Value(k, func(bs bytesp.Slice) error {
found = true
assert.Equal(t, "bs", string(v), string(bs))
return nil
})
}))
assert.Equal(t, "b.count", b.count, 0)
assert.ValueShould(t, "b.db.DB", b.db.DB, b.db.DB == nil, "is not nil")
}
func TestCheckPackageStatus(t *testing.T) {
// No crawling info, new package
assert.Equal(t, "CheckPackageStatus", CheckPackageStatus(&stpb.PackageInfo{}, nil), OutOfDate)
pkgCrawlTime, _ := ptypes.TimestampProto(time.Now().Add(-5 * timep.Day))
newRepoInfoCrawlTime, _ := ptypes.TimestampProto(time.Now().Add(-3 * timep.Day))
newPkgUpdateTime, _ := ptypes.TimestampProto(time.Now().Add(-4 * timep.Day))
assert.Equal(t, "CheckPackageStatus", CheckPackageStatus(&stpb.PackageInfo{
CrawlingInfo: &sppb.CrawlingInfo{
CrawlingTime: pkgCrawlTime,
},
}, &sppb.RepoInfo{
CrawlingTime: newRepoInfoCrawlTime,
LastUpdated: newPkgUpdateTime,
}), OutOfDate)
newPkgUpdateTime, _ = ptypes.TimestampProto(time.Now().Add(-6 * timep.Day))
assert.Equal(t, "CheckPackageStatus", CheckPackageStatus(&stpb.PackageInfo{
CrawlingInfo: &sppb.CrawlingInfo{
CrawlingTime: pkgCrawlTime,
},
}, &sppb.RepoInfo{
CrawlingTime: newRepoInfoCrawlTime,
LastUpdated: newPkgUpdateTime,
}), UpToDate)
}
func TestAggregateMethod_String(t *testing.T) {
assert.Equal(t, "String()", Sum.String(), "Sum")
assert.Equal(t, "String()", Max.String(), "Max")
assert.Equal(t, "String()", Min.String(), "Min")
assert.Equal(t, "String()", Average.String(), "Average")
assert.Equal(t, "String()", AggregateMethod(-1).String(), "-1(invalid)")
}
func TestByteSlice_Bug_Read(t *testing.T) {
var s ByteSlice
n, err := s.Read(make([]byte, 1))
t.Logf("n: %d, err: %v", n, err)
assert.Equal(t, "n", 0, 0)
assert.Equal(t, "err", err, io.EOF)
}
func TestCounter_Append(t *testing.T) {
tests := []struct {
sum int
div int
aggr AggregateMethod
value int
final_sum int
final_div int
}{
{sum: 11, div: 4, aggr: Sum, value: 2, final_sum: 5, final_div: 1},
{sum: 11, div: 4, aggr: Min, value: 2, final_sum: 2, final_div: 1},
{sum: 11, div: 4, aggr: Max, value: 5, final_sum: 5, final_div: 1},
{sum: 11, div: 4, aggr: Average, value: 2, final_sum: 13, final_div: 5},
}
for _, test := range tests {
t.Logf("test: %v, %v", test, test.aggr)
c := Counter{
Sum: test.sum,
Div: test.div,
}
c.Append(test.aggr, test.value)
assert.Equal(t, "sum", c.Sum, test.final_sum)
assert.Equal(t, "div", c.Div, test.final_div)
}
}
func TestForEachPackageOfSite(t *testing.T) {
// Clear the database
configs.StoreBoltPath().RemoveAll()
const (
site = "TestForEachPackageOfSite.com"
path1 = "gcse"
name1 = "pkgname"
path2 = "gcse2"
name2 = "TestForEachPackageOfSite"
)
assert.NoError(t, UpdatePackage(site, path1, func(info *stpb.PackageInfo) error {
info.Name = name1
return nil
}))
assert.NoError(t, UpdatePackage(site, path2, func(info *stpb.PackageInfo) error {
info.Name = name2
return nil
}))
var paths, names []string
assert.NoError(t, ForEachPackageOfSite(site, func(path string, info *stpb.PackageInfo) error {
paths = append(paths, path)
names = append(names, info.Name)
return nil
}))
assert.Equal(t, "paths", paths, []string{path1, path2})
assert.Equal(t, "names", names, []string{name1, name2})
}
func TestInc(t *testing.T) {
DataPath = "./tmp.bolt"
FlushPeriod = time.Second
err := os.RemoveAll(DataPath)
if err != nil && err != os.ErrNotExist {
assert.NoError(t, err)
return
}
now := time.Now()
Inc("abc")
Flush()
Process()
names, err := ReadNames()
assert.NoError(t, err)
assert.Equal(t, "names", names, []string{"abc"})
data, err := ReadDataOfName("daily", "abc")
assert.NoError(t, err)
assert.Equal(t, "data", data, []LabeledCounter{{
Counter{
Sum: 1,
Div: 1,
},
now.Format("2006-01-02"),
}})
}
func TestMaxStackDepth(t *testing.T) {
MaxStackDepth = 1
err := WithStacks(fmt.Errorf("myerror")).(*ErrorWithStacks)
assert.Equal(t, "len(err.Stacks)", len(err.Stacks), 2)
assert.Equal(t, "err.Stacks[1]", err.Stacks[1], "...")
assert.Equal(t, "lines of err.Error()", len(strings.Split(err.Error(), "\n")), 3)
}
func TestI(t *testing.T) {
assert.Equal(t, "I(123)", I(123), 123)
assert.Equal(t, "I(int64(123))", I(int64(123)), 123)
assert.Equal(t, "I(abc)", I("abc"), 0)
assert.Equal(t, "I(abc, 50)", I("abc", 50), 50)
assert.Equal(t, "I(abc, def)", I("abc", "def"), 0)
}
func TestWithStacks(t *testing.T) {
origin := fmt.Errorf("myerror")
wrapped, isErrWithStacks := WithStacks(origin).(*ErrorWithStacks)
assert.True(t, "isErrWithStacks", isErrWithStacks)
assert.Equal(t, "wrapped.Err", wrapped.Err, origin)
wrapped2, isErrWithStacks := WithStacks(wrapped).(*ErrorWithStacks)
assert.True(t, "isErrWithStacks", isErrWithStacks)
assert.Equal(t, "wrapped2", wrapped2, wrapped)
}
func TestRefCountBox_UnsetDataPath(t *testing.T) {
fn := path.Join(os.TempDir(), "TestRefCountBox.bolt")
assert.NoError(t, os.RemoveAll(fn))
b := RefCountBox{}
db, err := b.Alloc()
assert.Equal(t, "err", errorsp.Cause(err), ErrBoxDataPathNotSpecified)
assert.ValueShould(t, "db.DB", db.DB, db.DB == nil, "is not nil")
assert.ValueShould(t, "b.db.DB", b.db.DB, b.db.DB == nil, "is not nil")
assert.Equal(t, "b.count", b.count, 0)
}
func TestWithStacksAndMessage(t *testing.T) {
origin := fmt.Errorf("origin")
wrapped, isErrWithStacks := WithStacksAndMessage(origin, "%s:%d", "a", 123).(*ErrorWithStacks)
assert.True(t, "isErrWithStacks", isErrWithStacks)
assert.Equal(t, "wrapped.Err", wrapped.Err, origin)
assert.ValueShould(t, "wrapped.Stacks[0]", wrapped.Stacks[0], strings.HasSuffix(wrapped.Stacks[0], ": a:123"), "does not contain message correctly")
wrapped2, isErrWithStacks := WithStacksAndMessage(wrapped, "%s:%d", "b", 456).(*ErrorWithStacks)
assert.True(t, "isErrWithStacks", isErrWithStacks)
assert.Equal(t, "wrapped.Err", wrapped2.Err, origin)
assert.ValueShould(t, "wrapped2.Stacks[0]", wrapped2.Stacks[0], strings.HasSuffix(wrapped2.Stacks[0], ": a:123: b:456"), "does not contain message correctly")
}
func TestSlice_ReadFrom(t *testing.T) {
src := make([]byte, 63*1024)
for i := range src {
src[i] = byte(i)
}
b := bytes.NewBuffer(src)
var s Slice
n, err := s.ReadFrom(b)
assert.Equal(t, "n", n, int64(len(src)))
assert.NoError(t, err)
assert.Equal(t, "s", s, Slice(src))
}
func TestMoveData(t *testing.T) {
DataPath = "./tmp.bolt"
FlushPeriod = time.Second
err := os.RemoveAll(DataPath)
if err != nil && err != os.ErrNotExist {
assert.NoError(t, err)
return
}
now := time.Now()
checkValue := func(tp, name string, sum, div int) {
data, err := ReadDataOfName(tp, name)
assert.NoError(t, err)
if sum == 0 && div == 0 {
assert.Equal(t, fmt.Sprintf("%v:%v", name, tp), len(data), 0)
return
}
assert.Equal(t, fmt.Sprintf("%v:%v", name, tp), data, []LabeledCounter{{
Counter{Sum: sum, Div: div},
now.Format("2006-01-02"),
}})
}
checkNames := func(exp []string) {
names, err := ReadNames()
assert.NoError(t, err)
assert.Equal(t, "names", names, exp)
}
AddValue(Average, "TestMoveData-abc", 123)
AddValue(Average, "TestMoveData-def", 456)
Flush()
Process()
t.Log("before move")
checkNames([]string{"TestMoveData-abc", "TestMoveData-def"})
checkValue(Daily, "TestMoveData-abc", 123, 1)
checkValue(Daily, "TestMoveData-def", 456, 1)
assert.NoError(t, MoveData("TestMoveData-abc", "TestMoveData-def", Average))
t.Log("after move from abc to def")
checkNames([]string{"TestMoveData-def"})
checkValue(Daily, "TestMoveData-abc", 0, 0)
checkValue(Daily, "TestMoveData-def", 579, 2)
assert.NoError(t, MoveData("TestMoveData-def", "TestMoveData-none", Average))
t.Log("after move from def to none")
checkNames([]string{"TestMoveData-none"})
checkValue(Daily, "TestMoveData-abc", 0, 0)
checkValue(Daily, "TestMoveData-def", 0, 0)
checkValue(Daily, "TestMoveData-none", 579, 2)
}
func TestAppendPackageEvent(t *testing.T) {
const (
site = "TestAppendPackageEvent.com"
path = "gcse"
foundWay = "test"
)
// Insert a found only event, no action.
foundTm := time.Now()
foundTs, _ := ptypes.TimestampProto(foundTm)
assert.NoError(t, AppendPackageEvent(site, path, "test", foundTm, sppb.HistoryEvent_Action_None))
h, err := ReadPackageHistory(site, path)
assert.NoError(t, err)
assert.Equal(t, "h", h, &sppb.HistoryInfo{FoundWay: foundWay, FoundTime: foundTs})
// Inser a Success action
succTm := foundTm.Add(time.Hour)
succTs, _ := ptypes.TimestampProto(succTm)
assert.NoError(t, AppendPackageEvent(site, path, "non-test", succTm, sppb.HistoryEvent_Action_Success))
h, err = ReadPackageHistory(site, path)
assert.NoError(t, err)
assert.Equal(t, "h", h, &sppb.HistoryInfo{
FoundWay: foundWay,
FoundTime: foundTs,
Events: []*sppb.HistoryEvent{{
Timestamp: succTs,
Action: sppb.HistoryEvent_Action_Success,
}},
LatestSuccess: succTs,
})
// Inser a Failed action
failedTm := succTm.Add(time.Hour)
failedTs, _ := ptypes.TimestampProto(failedTm)
assert.NoError(t, AppendPackageEvent(site, path, "", failedTm, sppb.HistoryEvent_Action_Failed))
h, err = ReadPackageHistory(site, path)
assert.NoError(t, err)
assert.Equal(t, "h", h, &sppb.HistoryInfo{
FoundWay: foundWay,
FoundTime: foundTs,
Events: []*sppb.HistoryEvent{{
Timestamp: failedTs,
Action: sppb.HistoryEvent_Action_Failed,
}, {
Timestamp: succTs,
Action: sppb.HistoryEvent_Action_Success,
}},
LatestSuccess: succTs,
LatestFailed: failedTs,
})
}
func TestSlice_ReadFromEOF(t *testing.T) {
r1 := readerReturningEOFLastRead(1)
var s Slice
n, err := s.ReadFrom(&r1)
assert.Equal(t, "n", n, int64(1))
assert.NoError(t, err)
assert.Equal(t, "s", s, Slice([]byte{1}))
r2 := readerReturningEOFAfterLastRead(1)
s = nil
n, err = s.ReadFrom(&r2)
assert.Equal(t, "n", n, int64(1))
assert.NoError(t, err)
assert.Equal(t, "s", s, Slice([]byte{2}))
}
func TestStringFull(t *testing.T) {
test := func(a, b string, d int, lcs string) {
actD, actLcs := StringFull(a, b)
assert.Equal(t, fmt.Sprintf("Edit-distance between %s and %s", a, b), actD, d)
assert.Equal(t, fmt.Sprintf("Longest-common-string between %s and %s", a, b), actLcs, lcs)
}
test("abcd", "bcde", 2, "bcd")
test("abcde", "", 5, "")
test("", "abcde", 5, "")
test("", "", 0, "")
test("abcde", "abcde", 0, "abcde")
test("abcde", "dabce", 2, "abce")
test("abcde", "abfde", 1, "abde")
}
func TestBoltFileCache(t *testing.T) {
fn := filepath.Join(os.TempDir(), "TestBoltFileCache.bolt")
assert.NoErrorOrDie(t, os.RemoveAll(fn))
db, err := bh.Open(fn, 0755, nil)
assert.NoErrorOrDie(t, err)
counter := make(map[string]int)
c := BoltFileCache{
DB: db,
IncCounter: func(name string) {
counter[name] = counter[name] + 1
},
}
const (
sign1 = "abc"
sign2 = "def"
sign3 = "ghi"
gofile = "file.go"
rootfolder = "root"
sub = "sub"
subfolder = "root/sub"
)
fi := &sppb.GoFileInfo{}
//////////////////////////////////////////////////////////////
// New file found.
//////////////////////////////////////////////////////////////
// Get before set, should return false
assert.False(t, "c.Get", c.Get(sign1, fi))
assert.Equal(t, "counter", counter, map[string]int{
"crawler.filecache.missed": 1,
})
// Set the info.
c.Set(sign1, &sppb.GoFileInfo{Status: sppb.GoFileInfo_ShouldIgnore})
assert.Equal(t, "counter", counter, map[string]int{
"crawler.filecache.missed": 1,
"crawler.filecache.sign_saved": 1,
})
// Now, should fetch the cache
assert.True(t, "c.Get", c.Get(sign1, fi))
assert.Equal(t, "fi", fi, &sppb.GoFileInfo{Status: sppb.GoFileInfo_ShouldIgnore})
assert.Equal(t, "counter", counter, map[string]int{
"crawler.filecache.missed": 1,
"crawler.filecache.sign_saved": 1,
"crawler.filecache.hit": 1,
})
}
func TestSliceRemove(t *testing.T) {
defer __(o_(t))
var s Slice
s.Add(1, 2, 3, 4, 5, 6, 7)
assert.Equal(t, "len(s)", len(s), 7)
assert.StringEqual(t, "s", s, "[1 2 3 4 5 6 7]")
s.RemoveRange(2, 5)
assert.Equal(t, "len(s)", len(s), 4)
assert.StringEqual(t, "s", s, "[1 2 6 7]")
s.Remove(2)
assert.Equal(t, "len(s)", len(s), 3)
assert.StringEqual(t, "s", s, "[1 2 7]")
}
func TestStringSliceRemove(t *testing.T) {
defer __(o_(t))
var s StringSlice
s.Add("A", "B", "C", "D", "E", "F", "G")
assert.Equal(t, "len(s)", len(s), 7)
assert.StringEqual(t, "s", s, "[A B C D E F G]")
s.RemoveRange(2, 5)
assert.Equal(t, "len(s)", len(s), 4)
assert.StringEqual(t, "s", s, "[A B F G]")
s.Remove(2)
assert.Equal(t, "len(s)", len(s), 3)
assert.StringEqual(t, "s", s, "[A B G]")
}
func TestTokenize(t *testing.T) {
text := []byte("abc 3d 中文输入")
tokens := AppendTokens(nil, text)
assert.Equal(t, "tokens", tokens,
stringsp.NewSet("abc", "3", "d", "3-d", "中", "文", "输", "入", "中文",
"文输", "输入"))
}
func TestTokenize2(t *testing.T) {
text := []byte("PubSubHub")
tokens := AppendTokens(nil, text)
assert.Equal(t, "tokens", tokens,
stringsp.NewSet("pub", "sub", "hub", "pubsub", "subhub",
"pubsubhub"))
}
func TestMemDB_Recover(t *testing.T) {
path := villa.Path(".").Join("testmemdb.gob")
if path.Exists() {
path.Remove()
}
db := NewMemDB(".", "testmemdb")
db.Put("s", 1)
if err := db.Sync(); err != nil {
t.Error(err)
return
}
if err := path.Rename(path + ".new"); err != nil {
t.Error(err)
return
}
// Now in the status of fn.new exists, fn not exist
if err := db.Load(); err != nil {
t.Error(err)
return
}
var vl int
if ok := db.Get("s", &vl); !ok {
t.Error("Recover failed!")
return
}
assert.Equal(t, "vl", vl, 1)
}
func TestByteSlice_WriteItoa(t *testing.T) {
var s ByteSlice
s.WriteItoa(1234, 10)
s.WriteItoa(255, 16)
assert.Equal(t, "s", string(s), "1234ff")
}
func TestIntMatrix(t *testing.T) {
defer __(o_(t))
mat := NewIntMatrix(5, 4)
assert.Equal(t, "mat.Rows()", mat.Rows(), 5)
assert.Equal(t, "mat.Cols()", mat.Cols(), 4)
assert.StringEqual(t, "mat", mat, "[[0 0 0 0] [0 0 0 0] [0 0 0 0] [0 0 0 0] [0 0 0 0]]")
mat.Fill(10)
assert.StringEqual(t, "mat", mat, "[[10 10 10 10] [10 10 10 10] [10 10 10 10] [10 10 10 10] [10 10 10 10]]")
mat[1][1] = 0
mat[3][2] = 12345
mat[2][0] = -998
t.Logf("%s", mat.PrettyString())
}
func TestUnmarshalFile(t *testing.T) {
fn := path.Join(os.TempDir(), "TestMarshalToFile.json")
assert.NoError(t, ioutil.WriteFile(fn, []byte(`{"S":"abc"}`), 0644))
var s T
assert.NoError(t, UnmarshalFile(fn, &s))
assert.Equal(t, "s.S", s.S, "abc")
}
func TestDocDB_Export(t *testing.T) {
var db DocDB = PackedDocDB{NewMemDB("", "")}
info := DocInfo{
Name: "github.com/daviddengcn/gcse",
}
db.Put("go", info)
if err := db.Export(villa.Path("."), "testexport_db"); err != nil {
t.Errorf("db.Export failed: %v", err)
return
}
var newDB DocDB = PackedDocDB{NewMemDB(villa.Path("."), "testexport_db")}
count := 0
if err := newDB.Iterate(func(key string, val interface{}) error {
info, ok := val.(DocInfo)
if !ok {
return errors.New("Not a DocInfo object")
}
assert.StringEqual(t, "info.Name", info.Name,
"github.com/daviddengcn/gcse")
count++
return nil
}); err != nil {
t.Errorf("newDB.Iterate failed: %v", err)
}
assert.Equal(t, "count", count, 1)
}
