func benchGet(b *testing.B, store crawler.Store, name string) {
// start := time.Now()
b.ResetTimer()
for i := 0; i < b.N; i++ {
n := rand.Intn(2 * bench_get_size)
store.Get(mustParse(fmt.Sprintf("http://example.com/foo/bar/%d", n)))
}
// d := time.Now().Sub(start) / 2
// d /= time.Duration(int64(b.N))
// b.Log(name, "Get:", d)
}
func benchPut(b *testing.B, store crawler.Store, name string) {
// start := time.Now()
for i := 0; i < b.N; i++ {
now := time.Now().UTC()
store.PutNX(&crawler.URL{
URL: *mustParse(fmt.Sprintf("http://example.com/foo/bar/%d", i)),
Last: now,
})
}
// d := time.Now().Sub(start)
// d /= time.Duration(int64(b.N))
// b.Log(name, "Put:", d)
}
func StoreTest(t *testing.T, s crawler.Store) {
equalTime := func(t1, t2 time.Time) bool {
return t1.Round(time.Microsecond).Equal(
t2.Round(time.Microsecond))
}
cmp := func(u, uu *crawler.URL) bool {
return u.URL.String() == uu.URL.String() &&
u.Depth == uu.Depth &&
u.Status == uu.Status &&
equalTime(u.Last, uu.Last) &&
u.NumVisit == uu.NumVisit &&
u.NumRetry == uu.NumRetry
}
tm := time.Now().UTC()
assert := assert.New(t)
u, _ := url.Parse("http://localhost:6060")
uu := &crawler.URL{
URL: *u,
Last: tm,
}
ok, err := s.PutNX(uu)
assert.NoError(err)
assert.True(ok)
ok, err = s.PutNX(uu)
assert.NoError(err)
assert.False(ok)
ok, err = s.Exist(u)
assert.NoError(err)
assert.True(ok)
uuu, err := s.Get(u)
assert.NoError(err)
// assert.Equal(*uu, *uuu)
assert.True(cmp(uu, uuu))
uuu.NumVisit++
uuu.Last = time.Now().UTC()
assert.NoError(s.Update(uuu))
uu, err = s.Get(u)
assert.NoError(err)
// assert.Equal(*uuu, *uu)
assert.True(cmp(uu, uuu))
uu.Status = crawler.URLStatusError
assert.NoError(s.Update(uuu))
uuu, err = s.Get(u)
assert.NoError(err)
// assert.NotEqual(*uu, *uuu)
assert.False(cmp(uu, uuu))
ok, err = s.IsFinished()
assert.NoError(err)
assert.False(ok)
assert.NoError(s.Complete(u))
ok, err = s.IsFinished()
assert.NoError(err)
assert.True(ok)
u.Path = "/hello"
uu = &crawler.URL{
URL: *u,
}
ok, err = s.PutNX(uu)
assert.NoError(err)
assert.True(ok)
ok, err = s.IsFinished()
assert.NoError(err)
assert.False(ok)
assert.NoError(s.Complete(u))
ok, err = s.IsFinished()
assert.NoError(err)
assert.True(ok)
}