Example #1
0
func TestRefCountBox_UpdateView(t *testing.T) {
	fn := path.Join(os.TempDir(), "TestRefCountBox_UpdateView.bolt")
	assert.NoError(t, os.RemoveAll(fn))

	b := RefCountBox{
		DataPath: func() string { return fn },
	}
	k := bytes.Split([]byte("a.b"), []byte("."))
	v := "hello"
	assert.NoErrorOrDie(t, b.Update(func(tx Tx) error {
		return tx.Put(k, []byte(v))
	}))
	assert.Equal(t, "b.count", b.count, 0)
	assert.ValueShould(t, "b.db.DB", b.db.DB, b.db.DB == nil, "is not nil")

	found := false
	assert.NoErrorOrDie(t, b.View(func(tx Tx) error {
		return tx.Value(k, func(bs bytesp.Slice) error {
			found = true
			assert.Equal(t, "bs", string(v), string(bs))
			return nil
		})
	}))
	assert.Equal(t, "b.count", b.count, 0)
	assert.ValueShould(t, "b.db.DB", b.db.DB, b.db.DB == nil, "is not nil")
}
Example #2
0
func TestBoltFileCache(t *testing.T) {
	fn := filepath.Join(os.TempDir(), "TestBoltFileCache.bolt")
	assert.NoErrorOrDie(t, os.RemoveAll(fn))

	db, err := bh.Open(fn, 0755, nil)
	assert.NoErrorOrDie(t, err)

	counter := make(map[string]int)
	c := BoltFileCache{
		DB: db,
		IncCounter: func(name string) {
			counter[name] = counter[name] + 1
		},
	}
	const (
		sign1      = "abc"
		sign2      = "def"
		sign3      = "ghi"
		gofile     = "file.go"
		rootfolder = "root"
		sub        = "sub"
		subfolder  = "root/sub"
	)
	fi := &sppb.GoFileInfo{}

	//////////////////////////////////////////////////////////////
	// New file found.
	//////////////////////////////////////////////////////////////
	// Get before set, should return false
	assert.False(t, "c.Get", c.Get(sign1, fi))
	assert.Equal(t, "counter", counter, map[string]int{
		"crawler.filecache.missed": 1,
	})
	// Set the info.
	c.Set(sign1, &sppb.GoFileInfo{Status: sppb.GoFileInfo_ShouldIgnore})
	assert.Equal(t, "counter", counter, map[string]int{
		"crawler.filecache.missed":     1,
		"crawler.filecache.sign_saved": 1,
	})
	// Now, should fetch the cache
	assert.True(t, "c.Get", c.Get(sign1, fi))
	assert.Equal(t, "fi", fi, &sppb.GoFileInfo{Status: sppb.GoFileInfo_ShouldIgnore})
	assert.Equal(t, "counter", counter, map[string]int{
		"crawler.filecache.missed":     1,
		"crawler.filecache.sign_saved": 1,
		"crawler.filecache.hit":        1,
	})
}
func open(t *testing.T, path string) DB {
	fn := filepath.Join(os.TempDir(), path)
	os.RemoveAll(fn)
	db, err := Open(fn, 0644, nil)
	assert.NoErrorOrDie(t, err)
	return db
}
Example #4
0
func TestSearchRepositories(t *testing.T) {
	s := NewSpiderWithToken("")
	assert.Should(t, s != nil, "s == nil")

	rs, err := s.SearchRepositories("")
	assert.NoErrorOrDie(t, err)
	assert.ValueShould(t, "len(rs)", len(rs), len(rs) > 0, "> 0")
}
Example #5
0
func TestReadRepository(t *testing.T) {
	s := NewSpiderWithToken("")
	assert.Should(t, s != nil, "s == nil")

	repo, err := s.ReadRepository("daviddengcn", "gosl")
	assert.NoErrorOrDie(t, err)
	assert.ValueShould(t, "repo.Stars", repo.Stars, repo.Stars > 0, "> 0")
}
Example #6
0
func TestReadUser(t *testing.T) {
	s := NewSpiderWithToken("")
	assert.Should(t, s != nil, "s == nil")

	da, err := s.ReadUser("daviddengcn")
	assert.NoErrorOrDie(t, err)
	assert.ValueShould(t, "len(da.Repos)", len(da.Repos), len(da.Repos) > 0, "> 0")
}
Example #7
0
func TestReadPackage(t *testing.T) {
	s := NewSpiderWithToken("")
	assert.Should(t, s != nil, "s == nil")

	pkg, folders, err := s.ReadPackage("daviddengcn", "gcse", "spider/github/testdata")
	assert.NoErrorOrDie(t, err)
	assert.Equal(t, "pkg.Name", pkg.Name, "pkg")
	sort.Strings(pkg.Imports)
	assert.Equal(t, "pkg.Imports", pkg.Imports, []string{
		"github.com/daviddengcn/gcse/spider/github",
		"github.com/golangplus/strings",
	})
	assert.Equal(t, "pkg.TestImports", pkg.TestImports, []string{"github.com/golangplus/testing/assert"})
	assert.Equal(t, "len(folders)", len(folders), 1)
	assert.Equal(t, "folders[0].Name", folders[0].Name, "sub")
	assert.Equal(t, "folders[0].Path", folders[0].Path, "spider/github/testdata/sub")
}
Example #8
0
func TestIndex(t *testing.T) {
	const (
		package0 = "github.com/daviddengcn/gcse"
		package1 = "github.com/daviddengcn/gcse/indexer"
		package2 = "github.com/daviddengcn/go-villa"
	)

	docs := []DocInfo{
		{
			Package: package0,
			Name:    "gcse",
			TestImports: []string{
				package2, package0,
			},
		}, {
			Package: package1,
			Name:    "main",
			Imports: []string{
				package0,
				package2,
				package1,
			},
		}, {
			Package: package2,
			Name:    "villa",
		},
	}
	ts, err := Index(&mr.InputStruct{
		PartCountF: func() (int, error) {
			return 1, nil
		},
		IteratorF: func(int) (sophie.IterateCloser, error) {
			index := 0
			return &sophie.IterateCloserStruct{
				NextF: func(key, val sophie.SophieReader) error {
					if index >= len(docs) {
						return io.EOF
					}
					*key.(*sophie.RawString) = sophie.RawString(
						docs[index].Package)
					*val.(*DocInfo) = docs[index]
					val.(*DocInfo).Imports = append([]string{}, docs[index].Imports...)
					val.(*DocInfo).TestImports = append([]string{}, docs[index].TestImports...)

					index++
					return nil
				},
			}, nil
		},
	}, "./tmp")
	assert.NoErrorOrDie(t, err)

	hitsArr, err := index.OpenConstArray(path.Join("./tmp", HitsArrFn))
	for _, doc := range docs {
		idx := -1
		ts.Search(index.SingleFieldQuery(IndexPkgField, doc.Package), func(docID int32, data interface{}) error {
			idx = int(docID)
			return nil
		})
		d, err := hitsArr.GetGob(idx)
		assert.NoError(t, err)
		assert.Equal(t, "d.Package", d.(HitInfo).Package, doc.Package)
	}
	numDocs := ts.DocCount()
	assert.Equal(t, "DocCount", numDocs, 3)

	var pkgs []string
	if err := ts.Search(map[string]stringsp.Set{IndexTextField: nil},
		func(docID int32, data interface{}) error {
			hit := data.(HitInfo)
			pkgs = append(pkgs, hit.Package)
			return nil
		},
	); err != nil {
		t.Error(err)
		return
	}
	assert.StringEqual(t, "all", pkgs,
		[]string{
			"github.com/daviddengcn/gcse",
			"github.com/daviddengcn/go-villa",
			"github.com/daviddengcn/gcse/indexer",
		})

	var gcseInfo HitInfo
	if err := ts.Search(map[string]stringsp.Set{
		IndexPkgField: stringsp.NewSet("github.com/daviddengcn/gcse"),
	}, func(docID int32, data interface{}) error {
		gcseInfo = data.(HitInfo)
		return nil
	}); err != nil {
		t.Errorf("ts.Search: %v", err)
		return
	}
	assert.Equal(t, "gcseInfo.Imported", gcseInfo.Imported, []string(nil))
	assert.Equal(t, "gcseInfo.ImportedLen", gcseInfo.ImportedLen, 1)
	assert.Equal(t, "gcseInfo.TestImports", gcseInfo.TestImports, []string{"github.com/daviddengcn/go-villa"})

	var indexerInfo HitInfo
	if err := ts.Search(map[string]stringsp.Set{
		IndexPkgField: stringsp.NewSet("github.com/daviddengcn/gcse/indexer"),
	}, func(docID int32, data interface{}) error {
		gcseInfo = data.(HitInfo)
		return nil
	}); err != nil {
		t.Errorf("ts.Search: %v", err)
		return
	}
	assert.StringEqual(t, "indexerInfo.Imported",
		indexerInfo.Imported, []string{})
	assert.StringEqual(t, "indexerInfo.Imports",
		indexerInfo.Imports, []string{})

	if err := ts.Search(map[string]stringsp.Set{
		IndexPkgField: stringsp.NewSet("github.com/daviddengcn/go-villa"),
	}, func(docID int32, data interface{}) error {
		gcseInfo = data.(HitInfo)
		return nil
	}); err != nil {
		t.Errorf("ts.Search: %v", err)
		return
	}
	assert.Equal(t, "indexerInfo.Imported", indexerInfo.Imported, []string(nil))
	assert.Equal(t, "gcseInfo.TestImportedLen", gcseInfo.TestImportedLen, 1)
	assert.Equal(t, "gcseInfo.TestImported", gcseInfo.TestImported, []string(nil))
}
func TestGobReadWrite(t *testing.T) {
	db := open(t, "TestGobReadWrite")
	defer db.Close()

	k := [][]byte{[]byte("a"), []byte("b")}

	// Testing Tx.PutGob and Tx.GobValue
	assert.NoErrorOrDie(t, db.Update(func(tx Tx) error {
		return tx.PutGob(k, &Da{S: "hello"})
	}))
	found1 := false
	assert.NoError(t, db.View(func(tx Tx) error {
		return tx.GobValue(k, func(v interface{}) error {
			found1 = true
			da, ok := v.(Da)
			assert.True(t, "ok", ok)
			assert.Equal(t, "da", da, Da{S: "hello"})
			return nil
		})
	}))
	assert.True(t, "found1", found1)
	found2 := false
	assert.NoError(t, db.View(func(tx Tx) error {
		return tx.ForEachGob(k[:len(k)-1], func(_ Bucket, k bytesp.Slice, v interface{}) error {
			found2 = true
			da, ok := v.(Da)
			assert.True(t, "ok", ok)
			assert.Equal(t, "da", da, Da{S: "hello"})
			return nil
		})
	}))
	assert.True(t, "found2", found2)

	// Testing Bucket.PutGob, Bucket.GobValue, Bucket.ForEachGob
	bk := [][]byte{[]byte("bbb")}
	assert.NoErrorOrDie(t, db.Update(func(tx Tx) error {
		b, err := tx.CreateBucketIfNotExists(bk)
		assert.NoErrorOrDie(t, err)
		return b.PutGob(k, &Da{S: "world"})
	}))
	found3 := false
	assert.NoError(t, db.View(func(tx Tx) error {
		return tx.Bucket(bk, func(b Bucket) error {
			return b.GobValue(k, func(v interface{}) error {
				found3 = true
				da, ok := v.(Da)
				assert.True(t, "ok", ok)
				assert.Equal(t, "da", da, Da{S: "world"})
				return nil
			})
		})
	}))
	assert.True(t, "found3", found3)
	found4 := false
	assert.NoError(t, db.View(func(tx Tx) error {
		return tx.Bucket(bk, func(b Bucket) error {
			return b.ForEachGob(k[:len(k)-1], func(k bytesp.Slice, v interface{}) error {
				found4 = true
				da, ok := v.(Da)
				assert.True(t, "ok", ok)
				assert.Equal(t, "da", da, Da{S: "world"})
				return nil
			})
		})
	}))
	assert.True(t, "found4", found4)
}