Example #1
0
func Test_Agent_Recorder_Cancel(t *testing.T) {
	var agent *Agent
	var r Recorder
	assert := wcg.NewAssert(t)
	now, _ := wcg.ParseDateTime("2016-05-15T16:05:00Z")

	httptest.StartMockServer(func(mock *httptest.MockServer) {
		mock.Routes().GET("/api/intern/pt/iepg/records/", middleware.ServeFile("./fixtures/mocks/agent_tests.json"))
		agent = NewAgent(mock.BaseURL(), "test-token", nil)
		agent.recorderFactory = func(iepg *pt.IEPG) Recorder {
			return NewMockRecorder(iepg)
		}
		lib.TemporarySetNow(now, func() {
			agent.RunOnce()
			time.Sleep(100 * time.Millisecond)
			r = agent.recorders["200171201605160105"]
			assert.EqInt(1, len(agent.recorders), "agent should have a recorder thread.")
			assert.EqInt(int(pt.IEPGProcessing), int(r.GetStats().Status), "the recorder status should be pt.IEPGProcessing.")
		})
	})

	httptest.StartMockServer(func(mock *httptest.MockServer) {
		// preparation for dummy entry
		mock.Routes().GET("/api/intern/pt/iepg/records/", middleware.ServeFile("./fixtures/mocks/agent_tests_no_data.json"))
		agent.subscriber = NewSubscriber(mock.BaseURL(), "test-token")
		lib.TemporarySetNow(now, func() {
			agent.RunOnce()
			time.Sleep(100 * time.Millisecond)
			assert.EqInt(0, len(agent.recorders), "agent should have no recorder thread.")
			assert.EqInt(int(r.GetStats().Status), int(pt.IEPGCanceled), "the recorder status should be pt.IEPGPRocessing.")
		})
	})
}
Example #2
0
func Test_API_Tasks_Crawlers_Ameblo_Posts(t *testing.T) {
	assert := gaetest.NewAssert(t)
	assert.Nil(gaetest.ResetFixtureFromFile(ts.Context, "./fixtures/Test_API_Tasks_Crawlers_Ameblo_Posts.json", nil, "hplink"), "fixture")

	p := entities.AmebloPost.Query().MustExecute(ts.GET("/").Request)
	postList := p.Data.([]hplink.AmebloPost)
	assert.EqInt(1, len(postList))

	httptest.StartMockServer(func(mock *httptest.MockServer) {
		mock.Routes().GET("/morningmusume-9ki/post.html", middleware.ServeFile("./fixtures/mocks/ameblo.jp//morningmusume-9ki/post.html"))
		crawler.MockExternalResource(map[string]string{
			"http://ameblo.jp/morningmusume-9ki/entry-12201946654.html": fmt.Sprintf("%s//morningmusume-9ki/post.html", mock.BaseURL()),
		}, func() {
			runner := testhelper.NewAsyncTaskTestRunner(t, instance.Routes(), ts)
			runner.OnMonitor(func(testreq *httptest.TestRequest, f func()) {
				testhelper.TemporaryAllow(testreq, "hplink.admin", f)
			})
			runner.OnTrigger(func(testreq *httptest.TestRequest, f func()) {
				testhelper.TemporaryAllow(testreq, "hplink.admin", f)
			})
			runner.Run("/api/hplink/tasks/crawlers/ameblo/posts/", url.Values{})

			p := entities.AmebloPost.Query().MustExecute(ts.GET("/").Request)
			postList := p.Data.([]hplink.AmebloPost)
			assert.EqInt(1, len(postList))
			assert.EqStr("ノーメイク☆譜久村聖", postList[0].Title)
			assert.EqStr("譜久村聖", postList[0].Theme)
			assert.EqStr("morningmusume.mizuki_fukumura", postList[0].MemberKey)

			// It also creates a URL cache
			assert.EqInt(1, entities.URLCache.Query().MustCount(ts.GET("/").Request))
		})
	})
}
Example #3
0
func Test_API_Tasks_Crawlers_Ameblo_EntryLists(t *testing.T) {
	assert := gaetest.NewAssert(t)
	assert.Nil(gaetest.ResetFixtureFromFile(ts.Context, "./fixtures/Test_API_Tasks_Crawlers_Ameblo_EntryLists.json", nil, "hplink"), "fixture")

	httptest.StartMockServer(func(mock *httptest.MockServer) {
		mock.Routes().GET("/morningmusume-9ki/entrylist.html", middleware.ServeFile("./fixtures/mocks/ameblo.jp//morningmusume-9ki/entrylist.html"))
		crawler.MockExternalResource(map[string]string{
			"http://ameblo.jp/morningmusume-9ki/entrylist.html": fmt.Sprintf("%s//morningmusume-9ki/entrylist.html", mock.BaseURL()),
		}, func() {
			runner := testhelper.NewAsyncTaskTestRunner(t, instance.Routes(), ts)
			runner.OnMonitor(func(testreq *httptest.TestRequest, f func()) {
				testhelper.TemporaryAllow(testreq, "hplink.admin", f)
			})
			runner.OnTrigger(func(testreq *httptest.TestRequest, f func()) {
				testhelper.TemporaryAllow(testreq, "hplink.admin", f)
			})
			runner.Run("/api/hplink/tasks/crawlers/ameblo/entrylists/", url.Values{})

			p := entities.AmebloPost.Query().Order("-PostAt").MustExecute(ts.GET("/").Request)
			postList := p.Data.([]hplink.AmebloPost)
			assert.EqInt(20, len(postList))
			assert.EqStr("ノーメイク☆譜久村聖", postList[0].Title)
			assert.EqStr("", postList[0].Theme)
			assert.EqStr("morningmusume.mizuki_fukumura", postList[0].MemberKey)
			assert.EqStr("http://ameblo.jp/morningmusume-9ki/", postList[0].SettingsURL)

			p = entities.CrawlerSettings.Query().Filter("URL=", "http://ameblo.jp/morningmusume-9ki/").MustExecute(ts.GET("/").Request)
			assert.EqInt(1, p.Length())
			settings := p.Head().(*hplink.CrawlerSettings)
			assert.EqInt(int(hplink.CrawlerStatusSuccess), int(settings.Status))
		})
	})
}
Example #4
0
func Test_API_Task_Sync(t *testing.T) {
	assert := gaetest.NewAssert(t)
	gaetest.CleanupDatastore(ts.Context, "hplink")

	httptest.StartMockServer(func(mock *httptest.MockServer) {
		mock.Routes().GET("/", middleware.ServeFile("./fixtures/mocks/helloproject.html"))
		mock.Routes().GET("/c-ute/", middleware.ServeFile("./fixtures/mocks/c-ute/index.html"))
		mock.Routes().GET("/c-ute/*", middleware.StaticFile("", "fixtures/mocks/c-ute/"))

		crawler.MockExternalResource(map[string]string{
			"http://www.helloproject.com/artist/":                             fmt.Sprintf("%s", mock.BaseURL()),
			"http://www.helloproject.com/artist/c-ute/":                       fmt.Sprintf("%sc-ute/", mock.BaseURL()),
			"http://www.helloproject.com/artist/c-ute/profile/maimi_yajima/":  fmt.Sprintf("%sc-ute/maimi_yajima.html", mock.BaseURL()),
			"http://www.helloproject.com/artist/c-ute/profile/saki_nakajima/": fmt.Sprintf("%sc-ute/saki_nkajima.html", mock.BaseURL()),
			"http://www.helloproject.com/artist/c-ute/profile/airi_suzuki/":   fmt.Sprintf("%sc-ute/airi_suzuki.html", mock.BaseURL()),
			"http://www.helloproject.com/artist/c-ute/profile/chisato_okai/":  fmt.Sprintf("%sc-ute/chisato_okai.html", mock.BaseURL()),
			"http://www.helloproject.com/artist/c-ute/profile/mai_hagiwara/":  fmt.Sprintf("%sc-ute/mai_hagiwara.html", mock.BaseURL()),
		}, func() {
			runner := testhelper.NewAsyncTaskTestRunner(t, instance.Routes(), ts)
			runner.OnMonitor(func(testreq *httptest.TestRequest, f func()) {
				testhelper.TemporaryAllow(testreq, "hplink.admin", f)
			})
			runner.OnTrigger(func(testreq *httptest.TestRequest, f func()) {
				testhelper.TemporaryAllow(testreq, "hplink.admin", f)
			})
			runner.Run("/api/hplink/tasks/profile/", url.Values{})

			p := entities.Artist.Query().Order("Index").MustExecute(ts.GET("/").Request)
			artistList := p.Data.([]hplink.Artist)
			assert.EqInt(1, len(artistList))
			assert.EqInt(0, artistList[0].Index)
			assert.EqStr("c-ute", artistList[0].Key)
			assert.EqStr("℃-ute", artistList[0].Name)

			p = entities.Member.Query().Order("Index").Filter("ArtistKey=", "c-ute").MustExecute(ts.GET("/").Request)
			memberList := p.Data.([]hplink.Member)
			assert.EqInt(5, len(memberList))
			assert.EqInt(0, memberList[0].Index)
			assert.EqStr("c-ute", memberList[0].ArtistKey)
			assert.EqStr("c-ute.maimi_yajima", memberList[0].Key)
			assert.EqStr("矢島舞美", memberList[0].Name)

			entities.FillMemberPublicProfiles(ts.GET("/").Request, memberList)
			assert.NotNil(memberList[0].PublicProfile)
			assert.EqStr("c-ute.maimi_yajima", memberList[0].PublicProfile.Key)
			assert.EqStr("埼玉県", memberList[0].PublicProfile.Hometown)

			_, settings := entities.CrawlerSettings.Get().Key("http://ameblo.jp/c-ute-official").MustOne(ts.GET("/").Request)
			assert.NotNil(settings)
			assert.EqStr("c-ute", settings.(*hplink.CrawlerSettings).ArtistKey)
			assert.EqInt(int(hplink.CrawlerSettingsTypeAmeblo), int(settings.(*hplink.CrawlerSettings).Type))
		})
	})
}
Example #5
0
func Test_Run_With_404(t *testing.T) {
	assert := wcg.NewAssert(t)
	httptest.StartMockServer(func(s *httptest.MockServer) {
		url := s.BaseURL()

		runner := &Runner{
			Crawler: &testCrawlerImpl{url},
		}
		_, err := runner.Run(url)
		assert.NotNil(err)
	})
}
Example #6
0
func Test_CollectReadyNASSMART(t *testing.T) {
	assert := wcg.NewAssert(t)
	httptest.StartMockServer(func(mock *httptest.MockServer) {
		mock.Routes().GET("/*", wcg.AnonymousHandler(func(res *wcg.Response, req *wcg.Request) {
			http.ServeFile(res, req.HTTPRequest(), "./fixtures/SMART.xml")
		}))
		stats, err := CollectReadyNASSMART(mock.Addr(), "", "", 6)
		assert.Nil(err, "getReadyNASSMart should not return an error")
		assert.EqInt(6, len(stats), "len(stats)")
		for _, s := range stats {
			assert.EqInt(5, s.StartStopCount, "StartStopCount")
		}
	})
}
Example #7
0
func Test_getReadyNASSMART(t *testing.T) {
	assert := wcg.NewAssert(t)
	httptest.StartMockServer(func(mock *httptest.MockServer) {
		mock.Routes().GET("/*", wcg.AnonymousHandler(func(res *wcg.Response, req *wcg.Request) {
			http.ServeFile(res, req.HTTPRequest(), "./fixtures/SMART.xml")
		}))
		s, err := getSMARTStatsFromReadyNAS(mock.BaseURL(), "", "")
		assert.Nil(err, "getReadyNASSMart should not return an error")
		assert.EqInt(1, s.SpinUpTime, "SpinUpTime")
		assert.EqInt(5, s.StartStopCount, "StartStopCount")
		assert.EqInt64(252668631068583, s.HeadFlyingHours, "HeadFlyingHours")
		assert.EqInt64(76931166535, s.TotalLBAsWritten, "TotalLBAsWritten")
		assert.EqInt64(186750896472, s.TotalLBAsRead, "TotalLBAsRead")
	})
}
Example #8
0
func Test_Run(t *testing.T) {
	assert := wcg.NewAssert(t)
	httptest.StartMockServer(func(s *httptest.MockServer) {
		s.Routes().GET("/*", wcg.AnonymousHandler(func(res *wcg.Response, req *wcg.Request) {
			res.WriteString("OK")
		}))
		url := s.BaseURL()

		runner := &Runner{
			Crawler: &testCrawlerImpl{url},
		}
		result, err := runner.Run(url)
		assert.Nil(err)
		assert.EqStr("OK", result.Data.(string))
	})
}
Example #9
0
func Test_getReadyNASSMART_With_BasicAuth(t *testing.T) {
	assert := wcg.NewAssert(t)
	httptest.StartMockServer(func(mock *httptest.MockServer) {
		mock.Routes().GET("/*", wcg.AnonymousHandler(func(res *wcg.Response, req *wcg.Request) {
			username, password, _ := req.HTTPRequest().BasicAuth()
			if username == "testuser" && password == "passw0rd" {
				http.ServeFile(res, req.HTTPRequest(), "./fixtures/SMART.xml")
			} else {
				res.WriteHeader(401)
			}
		}))
		_, err := getSMARTStatsFromReadyNAS(mock.BaseURL(), "testuser", "passw0rd")
		assert.Nil(err, "getReadyNASSMart should not return an error")
		_, err = getSMARTStatsFromReadyNAS(mock.BaseURL(), "", "")
		assert.OK(err != nil, "getReadyNASSMart should return an error")
	})
}
Example #10
0
func Test_HelloProject_Execute(t *testing.T) {
	assert := wcg.NewAssert(t)

	httptest.StartMockServer(func(mock *httptest.MockServer) {
		// preparation for dummy entry
		mock.Routes().GET("/*", middleware.ServeFile("./fixtures/mocks/helloproject.com/top.html"))
		crawler.MockExternalResource(map[string]string{
			"http://www.helloproject.com/artist/": mock.BaseURL(),
		}, func() {
			data, err := NewHelloProject(http.DefaultClient).Run()
			assert.Nil(err)
			assert.EqInt(10, len(data))
			assert.EqStr("c-ute", data[0].Key)
			assert.EqStr("℃-ute", data[0].Name)
			assert.EqStr("http://www.helloproject.com/c-ute/", data[0].URL)
			assert.EqStr("http://cdn.helloproject.com/img/artist/m/80bc0be0d26cfc7720b747e965bc5edbadf3e726.jpg", data[0].ImageURL)
		})
	})
}
Example #11
0
func Test_Artist_Run_NonGroup(t *testing.T) {
	assert := wcg.NewAssert(t)

	httptest.StartMockServer(func(mock *httptest.MockServer) {
		// preparation for dummy entry
		mock.Routes().GET("/profile/", middleware.ServeFile("./fixtures/mocks/helloproject.com/rihosayashi.html"))
		mock.Routes().GET("/", middleware.ServeFile("./fixtures/mocks/helloproject.com/rihosayashi_top.html"))
		crawler.MockExternalResource(map[string]string{
			"http://www.helloproject.com/rihosayashi/profile/": mock.BaseURL() + "/profile/",
			"http://www.helloproject.com/rihosayashi/":         mock.BaseURL(),
		}, func() {
			data, err := NewArtist(http.DefaultClient, "rihosayashi").Run()
			assert.Nil(err)
			assert.EqInt(1, len(data.ArtistImageURLs))
			assert.EqInt(1, len(data.Members))
			assert.EqStr("http://cdn.helloproject.com/img/rotation/4ae1ab86fe60194c0a6c1e6a188a166eafa8686d.jpg", data.ArtistImageURLs[0])
			assert.EqStr("rihosayashi", data.Members[0].Key)
			assert.EqStr("鞘師里保", data.Members[0].Name)
		})
	})
}
Example #12
0
func Test_Subscribe(t *testing.T) {
	assert := wcg.NewAssert(t)

	httptest.StartMockServer(func(mock *httptest.MockServer) {
		// preparation for dummy entry
		mock.Routes().GET("/api/intern/pt/iepg/records/", middleware.ServeFile("./fixtures/mocks/subscriber_tests.json"))
		s := NewSubscriber(mock.BaseURL(), "test-token")

		t, _ := wcg.ParseDateTime("2016-05-12T16:00:00Z")
		lib.TemporarySetNow(t, func() {
			list, err := s.Subscribe()
			assert.Nil(err, "error should be nil")
			assert.EqInt(1, len(list), "Subscribe should return the list of *pt.IEPG")
		})

		t, _ = wcg.ParseDateTime("2016-05-13T18:00:00Z")
		lib.TemporarySetNow(t, func() {
			list, err := s.Subscribe()
			assert.Nil(err, "error should be nil")
			assert.EqInt(0, len(list), "Subscribe should not return any IEPG")
		})
	})
}
Example #13
0
func Test_Artist_Run(t *testing.T) {
	assert := wcg.NewAssert(t)

	httptest.StartMockServer(func(mock *httptest.MockServer) {
		// preparation for dummy entry
		mock.Routes().GET("/*", middleware.ServeFile("./fixtures/mocks/helloproject.com/c-ute.html"))
		crawler.MockExternalResource(map[string]string{
			"http://www.helloproject.com/c-ute/profile/": mock.BaseURL(),
		}, func() {
			data, err := NewArtist(http.DefaultClient, "c-ute").Run()
			assert.Nil(err)
			assert.EqInt(3, len(data.ArtistImageURLs))
			assert.EqInt(5, len(data.Members))
			assert.EqStr("http://cdn.helloproject.com/img/rotation/a974df522ac1ec6fa7668a43e0edac1b28c2aecd.jpg", data.ArtistImageURLs[0])
			assert.EqStr("http://cdn.helloproject.com/img/rotation/836f19e6688f3e46b9c7373d729331eb67b29f7f.jpg", data.ArtistImageURLs[1])
			assert.EqStr("http://cdn.helloproject.com/img/rotation/783e45e9d41c1bcc083fb62390ace0da94e63910.jpg", data.ArtistImageURLs[2])
			assert.EqStr("maimi_yajima", data.Members[0].Key)
			assert.EqStr("矢島舞美", data.Members[0].Name)
			assert.EqInt(1, len(data.CrawlableURLs))
			assert.EqStr("http://ameblo.jp/c-ute-official", data.CrawlableURLs[0].Link)
		})
	})
}
Example #14
0
func Test_URLCacher_(t *testing.T) {
	assert := wcg.NewAssert(t)
	assert.Nil(gaetest.CleanupDatastore(ts.Context, "hplink"))

	httptest.StartMockServer(func(mock *httptest.MockServer) {
		// preparation for dummy entry
		var count = 0
		mock.Routes().GET("/*", wcg.AnonymousHandler(func(res *wcg.Response, req *wcg.Request) {
			if req.Header("if-none-match") == "etagvalue" {
				res.WriteHeader(304)
				res.Close()
				return
			}
			count++
			res.Header().Set("Etag", "etagvalue")
			res.WriteString("FOO")
		}))

		textURL := fmt.Sprintf("%s%s", mock.BaseURL(), "/")
		client := NewURLCacheAwareClient(ts.GET("/").Request, urlCacheKind)
		resp, _ := client.Get(textURL)
		body, _ := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		assert.EqInt(200, resp.StatusCode)
		assert.EqStr("FOO", string(body))

		_, v := urlCacheKind.Get().Key(textURL).MustOne(ts.GET("/").Request)
		assert.NotNil(v)

		resp, _ = client.Get(textURL)
		body, _ = ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		assert.EqInt(304, resp.StatusCode)
		assert.EqStr("FOO", string(body))
	})
}
Example #15
0
func Test_Agent_Recorder_Completed(t *testing.T) {
	assert := wcg.NewAssert(t)

	httptest.StartMockServer(func(mock *httptest.MockServer) {
		// preparation for dummy entry
		mock.Routes().GET("/api/intern/pt/iepg/records/", middleware.ServeFile("./fixtures/mocks/agent_tests.json"))
		agent := NewAgent(mock.BaseURL(), "test-token", nil)
		agent.recorderFactory = func(iepg *pt.IEPG) Recorder {
			return NewMockRecorder(iepg)
		}
		now, _ := wcg.ParseDateTime("2016-05-15T16:05:00Z")
		lib.TemporarySetNow(now, func() {
			agent.RunOnce() // started
			r := agent.recorders["200171201605160105"]

			agent.RunOnce() // notify update
			agent.RunOnce() // notify update
			agent.RunOnce() // notify update
			agent.RunOnce() // notify update and confirmed to be completed.
			assert.EqInt(0, len(agent.recorders), "agent should have no recorder thread.")
			assert.EqInt(int(r.GetStats().Status), int(pt.IEPGCompleted), "the recorder status should be pt.IEPGCompleted.")
		})
	})
}
Example #16
0
func Test_Run_With_UserAgent(t *testing.T) {
	assert := wcg.NewAssert(t)
	httptest.StartMockServer(func(s *httptest.MockServer) {
		s.Routes().GET("/*", wcg.AnonymousHandler(func(res *wcg.Response, req *wcg.Request) {
			if req.UserAgent() == "Special UserAgent" {
				res.WriteHeader(403)
				res.WriteString("Forbidden")
			}
			res.WriteString("OK")
		}))
		url := s.BaseURL()

		runner := &Runner{
			Crawler: &testCrawlerImpl{url},
		}
		result, err := runner.Run(url)
		assert.Nil(err)
		assert.EqStr("OK", result.Data.(string))

		runner.UserAgent = "Special UserAgent"
		_, err = runner.Run(url)
		assert.NotNil(err)
	})
}
func Test_API_IEPGKeyword_Crawl_Preview(t *testing.T) {
	assert := gaetest.NewAssert(t)
	assert.Nil(gaetest.ResetMemcache(ts.Context))
	assert.Nil(gaetest.ResetFixtureFromFile(ts.Context, "./fixtures/Test_API_IEPGKeyword_Crawl.json", nil, "intern.pt"), "fixture")

	httptest.StartMockServer(func(mock *httptest.MockServer) {
		// preparation for dummy entry
		mock.Routes().GET("/*", middleware.StaticFile("", "./fixtures/mocks/"))

		// Preview Ttest
		req := ts.POSTForm("/api/intern/pt/iepg/keywords/preview/", url.Values{
			"keyword": []string{"テスト"},
			"scope":   []string{"0"},
		})

		var feedIDs []string
		req.Request.SetLocal("__test_iepg_url_resolver", &iepgURLResolver{
			getFeedURL: func(keyword string, scope pt.IEPGCrawlerScope) string {
				assert.EqStr("テスト", keyword, "keyword")
				assert.EqInt(int(pt.IEPGCrawlerScopeAll), int(scope), "scope")
				return fmt.Sprintf(
					"%s/feed/feed.rss",
					mock.BaseURL(),
				)
			},
			getDataURL: func(id string) string {
				feedIDs = append(feedIDs, id)
				return fmt.Sprintf(
					"%s/data/%s.txt",
					mock.BaseURL(),
					id,
				)
			},
		})

		helper.TemporaryAllow(req, "family", func() {
			res := req.RouteTo(instance.Routes())
			assert.HTTPStatus(200, res)
			assert.EqInt(4, len(feedIDs), "collected Feed IDs")
			assert.EqStr("succ", feedIDs[0], "feedIDs[0]")
			assert.EqStr("opted-out", feedIDs[1], "feedIDs[1]")
			assert.EqStr("excluded", feedIDs[2], "feedIDs[2]")
			assert.EqStr("multiple-categories", feedIDs[3], "feedIDs[3]")
			var got map[string][]pt.IEPG
			assert.JSONResponse(&got, res)
			assert.EqInt(4, len(got["hits"]), "len(got[\"hits\"])")

			var hits = got["hits"]
			var succ = hits[0]
			var optout = hits[1]
			var excluded = hits[2]
			var multiplecategories = hits[3]

			// assertion for succ
			assert.EqStr("DFS00430", succ.StationID, "succ.StationID")
			assert.EqStr("テレビ東京", succ.StationName, "succ.StationName")
			assert.EqStr(
				"The\u3000Girls\u3000Live\u3000▽道重さゆみ卒業ライブに密着▽LoVendoЯスタジオライブ",
				succ.ProgramTitle,
				"succ.ProgramTitle",
			)
			assert.EqInt(2014, succ.StartAt.Year(), "succ.StartAt.Year()")
			assert.EqInt(12, int(succ.StartAt.Month()), "succ.StartAt.Month()")
			assert.EqInt(5, succ.StartAt.Day(), "succ.StartAt.Day()")
			assert.EqInt(23, succ.StartAt.Hour(), "succ.StartAt.Hour()")
			assert.EqInt(0, succ.StartAt.Minute(), "succ.StartAt.Minute()")
			assert.EqInt(2014, succ.EndAt.Year(), "succ.EndAt.Year()")
			assert.EqInt(12, int(succ.EndAt.Month()), "succ.EndAt.Month()")
			assert.EqInt(6, succ.EndAt.Day(), "succ.EndAt.Day()")
			assert.EqInt(1, succ.EndAt.Hour(), "succ.EndAt.Hour()")
			assert.EqInt(30, succ.EndAt.Minute(), "succ.EndAt.Minute()")
			assert.EqStr("23", succ.CID, "succ.CID")
			assert.EqStr("hd", succ.SID, "succ.SID")
			assert.Not(succ.OptIn, "succ.OptIn")
			assert.Not(succ.OptOut, "succ.OptOut")
			assert.EqInt(0, len(succ.ExcludedBy), "len(succ.ExcludedBy)")
			assert.EqInt(0, len(succ.CrawledBy), "len(succ.CrawledBy)")

			// assertion for optout
			assert.Not(optout.OptIn, "OptIn")
			assert.OK(optout.OptOut, "OptOut")
			assert.EqInt(0, len(optout.ExcludedBy), "len(ExcludedBy)")

			// assertion for excluded
			assert.Not(excluded.OptIn, "excluded.OptIn")
			assert.Not(excluded.OptOut, "excluded.OptOut")
			assert.EqInt(1, len(excluded.ExcludedBy), "len(excluded.ExcludedBy)")
			assert.EqStr("crawl-test-exclusion", excluded.ExcludedBy[0], "excluded.ExcludedBy[0]")

			// assertion for multiple categories.
			assert.EqInt(1, len(multiplecategories.CrawledBy), "len(multiplecategories.CrawledBy)")
			assert.EqStr("another-crawl-keyword", multiplecategories.CrawledBy[0], "multiplecategories.CrawledBy[0]")

		})
	})
}