func TestRefCountBox_UpdateView(t *testing.T) {
fn := path.Join(os.TempDir(), "TestRefCountBox_UpdateView.bolt")
assert.NoError(t, os.RemoveAll(fn))
b := RefCountBox{
DataPath: func() string { return fn },
}
k := bytes.Split([]byte("a.b"), []byte("."))
v := "hello"
assert.NoErrorOrDie(t, b.Update(func(tx Tx) error {
return tx.Put(k, []byte(v))
}))
assert.Equal(t, "b.count", b.count, 0)
assert.ValueShould(t, "b.db.DB", b.db.DB, b.db.DB == nil, "is not nil")
found := false
assert.NoErrorOrDie(t, b.View(func(tx Tx) error {
return tx.Value(k, func(bs bytesp.Slice) error {
found = true
assert.Equal(t, "bs", string(v), string(bs))
return nil
})
}))
assert.Equal(t, "b.count", b.count, 0)
assert.ValueShould(t, "b.db.DB", b.db.DB, b.db.DB == nil, "is not nil")
}
func TestBoltFileCache(t *testing.T) {
fn := filepath.Join(os.TempDir(), "TestBoltFileCache.bolt")
assert.NoErrorOrDie(t, os.RemoveAll(fn))
db, err := bh.Open(fn, 0755, nil)
assert.NoErrorOrDie(t, err)
counter := make(map[string]int)
c := BoltFileCache{
DB: db,
IncCounter: func(name string) {
counter[name] = counter[name] + 1
},
}
const (
sign1 = "abc"
sign2 = "def"
sign3 = "ghi"
gofile = "file.go"
rootfolder = "root"
sub = "sub"
subfolder = "root/sub"
)
fi := &sppb.GoFileInfo{}
//////////////////////////////////////////////////////////////
// New file found.
//////////////////////////////////////////////////////////////
// Get before set, should return false
assert.False(t, "c.Get", c.Get(sign1, fi))
assert.Equal(t, "counter", counter, map[string]int{
"crawler.filecache.missed": 1,
})
// Set the info.
c.Set(sign1, &sppb.GoFileInfo{Status: sppb.GoFileInfo_ShouldIgnore})
assert.Equal(t, "counter", counter, map[string]int{
"crawler.filecache.missed": 1,
"crawler.filecache.sign_saved": 1,
})
// Now, should fetch the cache
assert.True(t, "c.Get", c.Get(sign1, fi))
assert.Equal(t, "fi", fi, &sppb.GoFileInfo{Status: sppb.GoFileInfo_ShouldIgnore})
assert.Equal(t, "counter", counter, map[string]int{
"crawler.filecache.missed": 1,
"crawler.filecache.sign_saved": 1,
"crawler.filecache.hit": 1,
})
}
func open(t *testing.T, path string) DB {
fn := filepath.Join(os.TempDir(), path)
os.RemoveAll(fn)
db, err := Open(fn, 0644, nil)
assert.NoErrorOrDie(t, err)
return db
}
func TestSearchRepositories(t *testing.T) {
s := NewSpiderWithToken("")
assert.Should(t, s != nil, "s == nil")
rs, err := s.SearchRepositories("")
assert.NoErrorOrDie(t, err)
assert.ValueShould(t, "len(rs)", len(rs), len(rs) > 0, "> 0")
}
func TestReadRepository(t *testing.T) {
s := NewSpiderWithToken("")
assert.Should(t, s != nil, "s == nil")
repo, err := s.ReadRepository("daviddengcn", "gosl")
assert.NoErrorOrDie(t, err)
assert.ValueShould(t, "repo.Stars", repo.Stars, repo.Stars > 0, "> 0")
}
func TestReadUser(t *testing.T) {
s := NewSpiderWithToken("")
assert.Should(t, s != nil, "s == nil")
da, err := s.ReadUser("daviddengcn")
assert.NoErrorOrDie(t, err)
assert.ValueShould(t, "len(da.Repos)", len(da.Repos), len(da.Repos) > 0, "> 0")
}
func TestReadPackage(t *testing.T) {
s := NewSpiderWithToken("")
assert.Should(t, s != nil, "s == nil")
pkg, folders, err := s.ReadPackage("daviddengcn", "gcse", "spider/github/testdata")
assert.NoErrorOrDie(t, err)
assert.Equal(t, "pkg.Name", pkg.Name, "pkg")
sort.Strings(pkg.Imports)
assert.Equal(t, "pkg.Imports", pkg.Imports, []string{
"github.com/daviddengcn/gcse/spider/github",
"github.com/golangplus/strings",
})
assert.Equal(t, "pkg.TestImports", pkg.TestImports, []string{"github.com/golangplus/testing/assert"})
assert.Equal(t, "len(folders)", len(folders), 1)
assert.Equal(t, "folders[0].Name", folders[0].Name, "sub")
assert.Equal(t, "folders[0].Path", folders[0].Path, "spider/github/testdata/sub")
}
func TestIndex(t *testing.T) {
const (
package0 = "github.com/daviddengcn/gcse"
package1 = "github.com/daviddengcn/gcse/indexer"
package2 = "github.com/daviddengcn/go-villa"
)
docs := []DocInfo{
{
Package: package0,
Name: "gcse",
TestImports: []string{
package2, package0,
},
}, {
Package: package1,
Name: "main",
Imports: []string{
package0,
package2,
package1,
},
}, {
Package: package2,
Name: "villa",
},
}
ts, err := Index(&mr.InputStruct{
PartCountF: func() (int, error) {
return 1, nil
},
IteratorF: func(int) (sophie.IterateCloser, error) {
index := 0
return &sophie.IterateCloserStruct{
NextF: func(key, val sophie.SophieReader) error {
if index >= len(docs) {
return io.EOF
}
*key.(*sophie.RawString) = sophie.RawString(
docs[index].Package)
*val.(*DocInfo) = docs[index]
val.(*DocInfo).Imports = append([]string{}, docs[index].Imports...)
val.(*DocInfo).TestImports = append([]string{}, docs[index].TestImports...)
index++
return nil
},
}, nil
},
}, "./tmp")
assert.NoErrorOrDie(t, err)
hitsArr, err := index.OpenConstArray(path.Join("./tmp", HitsArrFn))
for _, doc := range docs {
idx := -1
ts.Search(index.SingleFieldQuery(IndexPkgField, doc.Package), func(docID int32, data interface{}) error {
idx = int(docID)
return nil
})
d, err := hitsArr.GetGob(idx)
assert.NoError(t, err)
assert.Equal(t, "d.Package", d.(HitInfo).Package, doc.Package)
}
numDocs := ts.DocCount()
assert.Equal(t, "DocCount", numDocs, 3)
var pkgs []string
if err := ts.Search(map[string]stringsp.Set{IndexTextField: nil},
func(docID int32, data interface{}) error {
hit := data.(HitInfo)
pkgs = append(pkgs, hit.Package)
return nil
},
); err != nil {
t.Error(err)
return
}
assert.StringEqual(t, "all", pkgs,
[]string{
"github.com/daviddengcn/gcse",
"github.com/daviddengcn/go-villa",
"github.com/daviddengcn/gcse/indexer",
})
var gcseInfo HitInfo
if err := ts.Search(map[string]stringsp.Set{
IndexPkgField: stringsp.NewSet("github.com/daviddengcn/gcse"),
}, func(docID int32, data interface{}) error {
gcseInfo = data.(HitInfo)
return nil
}); err != nil {
t.Errorf("ts.Search: %v", err)
return
}
assert.Equal(t, "gcseInfo.Imported", gcseInfo.Imported, []string(nil))
assert.Equal(t, "gcseInfo.ImportedLen", gcseInfo.ImportedLen, 1)
assert.Equal(t, "gcseInfo.TestImports", gcseInfo.TestImports, []string{"github.com/daviddengcn/go-villa"})
var indexerInfo HitInfo
if err := ts.Search(map[string]stringsp.Set{
IndexPkgField: stringsp.NewSet("github.com/daviddengcn/gcse/indexer"),
}, func(docID int32, data interface{}) error {
gcseInfo = data.(HitInfo)
return nil
}); err != nil {
t.Errorf("ts.Search: %v", err)
return
}
assert.StringEqual(t, "indexerInfo.Imported",
indexerInfo.Imported, []string{})
assert.StringEqual(t, "indexerInfo.Imports",
indexerInfo.Imports, []string{})
if err := ts.Search(map[string]stringsp.Set{
IndexPkgField: stringsp.NewSet("github.com/daviddengcn/go-villa"),
}, func(docID int32, data interface{}) error {
gcseInfo = data.(HitInfo)
return nil
}); err != nil {
t.Errorf("ts.Search: %v", err)
return
}
assert.Equal(t, "indexerInfo.Imported", indexerInfo.Imported, []string(nil))
assert.Equal(t, "gcseInfo.TestImportedLen", gcseInfo.TestImportedLen, 1)
assert.Equal(t, "gcseInfo.TestImported", gcseInfo.TestImported, []string(nil))
}
func TestGobReadWrite(t *testing.T) {
db := open(t, "TestGobReadWrite")
defer db.Close()
k := [][]byte{[]byte("a"), []byte("b")}
// Testing Tx.PutGob and Tx.GobValue
assert.NoErrorOrDie(t, db.Update(func(tx Tx) error {
return tx.PutGob(k, &Da{S: "hello"})
}))
found1 := false
assert.NoError(t, db.View(func(tx Tx) error {
return tx.GobValue(k, func(v interface{}) error {
found1 = true
da, ok := v.(Da)
assert.True(t, "ok", ok)
assert.Equal(t, "da", da, Da{S: "hello"})
return nil
})
}))
assert.True(t, "found1", found1)
found2 := false
assert.NoError(t, db.View(func(tx Tx) error {
return tx.ForEachGob(k[:len(k)-1], func(_ Bucket, k bytesp.Slice, v interface{}) error {
found2 = true
da, ok := v.(Da)
assert.True(t, "ok", ok)
assert.Equal(t, "da", da, Da{S: "hello"})
return nil
})
}))
assert.True(t, "found2", found2)
// Testing Bucket.PutGob, Bucket.GobValue, Bucket.ForEachGob
bk := [][]byte{[]byte("bbb")}
assert.NoErrorOrDie(t, db.Update(func(tx Tx) error {
b, err := tx.CreateBucketIfNotExists(bk)
assert.NoErrorOrDie(t, err)
return b.PutGob(k, &Da{S: "world"})
}))
found3 := false
assert.NoError(t, db.View(func(tx Tx) error {
return tx.Bucket(bk, func(b Bucket) error {
return b.GobValue(k, func(v interface{}) error {
found3 = true
da, ok := v.(Da)
assert.True(t, "ok", ok)
assert.Equal(t, "da", da, Da{S: "world"})
return nil
})
})
}))
assert.True(t, "found3", found3)
found4 := false
assert.NoError(t, db.View(func(tx Tx) error {
return tx.Bucket(bk, func(b Bucket) error {
return b.ForEachGob(k[:len(k)-1], func(k bytesp.Slice, v interface{}) error {
found4 = true
da, ok := v.(Da)
assert.True(t, "ok", ok)
assert.Equal(t, "da", da, Da{S: "world"})
return nil
})
})
}))
assert.True(t, "found4", found4)
}