Beispiel #1
0
func benchGet(b *testing.B, store crawler.Store, name string) {
	// start := time.Now()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		n := rand.Intn(2 * bench_get_size)
		store.Get(mustParse(fmt.Sprintf("http://example.com/foo/bar/%d", n)))
	}
	// d := time.Now().Sub(start) / 2
	// d /= time.Duration(int64(b.N))
	// b.Log(name, "Get:", d)
}
Beispiel #2
0
func benchPut(b *testing.B, store crawler.Store, name string) {
	// start := time.Now()
	for i := 0; i < b.N; i++ {
		now := time.Now().UTC()
		store.PutNX(&crawler.URL{
			URL:  *mustParse(fmt.Sprintf("http://example.com/foo/bar/%d", i)),
			Last: now,
		})
	}
	// d := time.Now().Sub(start)
	// d /= time.Duration(int64(b.N))
	// b.Log(name, "Put:", d)
}
Beispiel #3
0
func StoreTest(t *testing.T, s crawler.Store) {
	equalTime := func(t1, t2 time.Time) bool {
		return t1.Round(time.Microsecond).Equal(
			t2.Round(time.Microsecond))
	}
	cmp := func(u, uu *crawler.URL) bool {
		return u.URL.String() == uu.URL.String() &&
			u.Depth == uu.Depth &&
			u.Status == uu.Status &&
			equalTime(u.Last, uu.Last) &&
			u.NumVisit == uu.NumVisit &&
			u.NumRetry == uu.NumRetry
	}
	tm := time.Now().UTC()
	assert := assert.New(t)
	u, _ := url.Parse("http://localhost:6060")
	uu := &crawler.URL{
		URL:  *u,
		Last: tm,
	}
	ok, err := s.PutNX(uu)
	assert.NoError(err)
	assert.True(ok)

	ok, err = s.PutNX(uu)
	assert.NoError(err)
	assert.False(ok)

	ok, err = s.Exist(u)
	assert.NoError(err)
	assert.True(ok)

	uuu, err := s.Get(u)
	assert.NoError(err)
	// assert.Equal(*uu, *uuu)
	assert.True(cmp(uu, uuu))

	uuu.NumVisit++
	uuu.Last = time.Now().UTC()
	assert.NoError(s.Update(uuu))
	uu, err = s.Get(u)
	assert.NoError(err)
	// assert.Equal(*uuu, *uu)
	assert.True(cmp(uu, uuu))

	uu.Status = crawler.URLStatusError
	assert.NoError(s.Update(uuu))
	uuu, err = s.Get(u)
	assert.NoError(err)
	// assert.NotEqual(*uu, *uuu)
	assert.False(cmp(uu, uuu))

	ok, err = s.IsFinished()
	assert.NoError(err)
	assert.False(ok)

	assert.NoError(s.Complete(u))
	ok, err = s.IsFinished()
	assert.NoError(err)
	assert.True(ok)

	u.Path = "/hello"
	uu = &crawler.URL{
		URL: *u,
	}
	ok, err = s.PutNX(uu)
	assert.NoError(err)
	assert.True(ok)
	ok, err = s.IsFinished()
	assert.NoError(err)
	assert.False(ok)

	assert.NoError(s.Complete(u))
	ok, err = s.IsFinished()
	assert.NoError(err)
	assert.True(ok)
}