func Test_Agent_Recorder_Cancel(t *testing.T) {
var agent *Agent
var r Recorder
assert := wcg.NewAssert(t)
now, _ := wcg.ParseDateTime("2016-05-15T16:05:00Z")
httptest.StartMockServer(func(mock *httptest.MockServer) {
mock.Routes().GET("/api/intern/pt/iepg/records/", middleware.ServeFile("./fixtures/mocks/agent_tests.json"))
agent = NewAgent(mock.BaseURL(), "test-token", nil)
agent.recorderFactory = func(iepg *pt.IEPG) Recorder {
return NewMockRecorder(iepg)
}
lib.TemporarySetNow(now, func() {
agent.RunOnce()
time.Sleep(100 * time.Millisecond)
r = agent.recorders["200171201605160105"]
assert.EqInt(1, len(agent.recorders), "agent should have a recorder thread.")
assert.EqInt(int(pt.IEPGProcessing), int(r.GetStats().Status), "the recorder status should be pt.IEPGProcessing.")
})
})
httptest.StartMockServer(func(mock *httptest.MockServer) {
// preparation for dummy entry
mock.Routes().GET("/api/intern/pt/iepg/records/", middleware.ServeFile("./fixtures/mocks/agent_tests_no_data.json"))
agent.subscriber = NewSubscriber(mock.BaseURL(), "test-token")
lib.TemporarySetNow(now, func() {
agent.RunOnce()
time.Sleep(100 * time.Millisecond)
assert.EqInt(0, len(agent.recorders), "agent should have no recorder thread.")
assert.EqInt(int(r.GetStats().Status), int(pt.IEPGCanceled), "the recorder status should be pt.IEPGPRocessing.")
})
})
}
func Test_API_Task_Sync(t *testing.T) {
assert := gaetest.NewAssert(t)
gaetest.CleanupDatastore(ts.Context, "hplink")
httptest.StartMockServer(func(mock *httptest.MockServer) {
mock.Routes().GET("/", middleware.ServeFile("./fixtures/mocks/helloproject.html"))
mock.Routes().GET("/c-ute/", middleware.ServeFile("./fixtures/mocks/c-ute/index.html"))
mock.Routes().GET("/c-ute/*", middleware.StaticFile("", "fixtures/mocks/c-ute/"))
crawler.MockExternalResource(map[string]string{
"http://www.helloproject.com/artist/": fmt.Sprintf("%s", mock.BaseURL()),
"http://www.helloproject.com/artist/c-ute/": fmt.Sprintf("%sc-ute/", mock.BaseURL()),
"http://www.helloproject.com/artist/c-ute/profile/maimi_yajima/": fmt.Sprintf("%sc-ute/maimi_yajima.html", mock.BaseURL()),
"http://www.helloproject.com/artist/c-ute/profile/saki_nakajima/": fmt.Sprintf("%sc-ute/saki_nkajima.html", mock.BaseURL()),
"http://www.helloproject.com/artist/c-ute/profile/airi_suzuki/": fmt.Sprintf("%sc-ute/airi_suzuki.html", mock.BaseURL()),
"http://www.helloproject.com/artist/c-ute/profile/chisato_okai/": fmt.Sprintf("%sc-ute/chisato_okai.html", mock.BaseURL()),
"http://www.helloproject.com/artist/c-ute/profile/mai_hagiwara/": fmt.Sprintf("%sc-ute/mai_hagiwara.html", mock.BaseURL()),
}, func() {
runner := testhelper.NewAsyncTaskTestRunner(t, instance.Routes(), ts)
runner.OnMonitor(func(testreq *httptest.TestRequest, f func()) {
testhelper.TemporaryAllow(testreq, "hplink.admin", f)
})
runner.OnTrigger(func(testreq *httptest.TestRequest, f func()) {
testhelper.TemporaryAllow(testreq, "hplink.admin", f)
})
runner.Run("/api/hplink/tasks/profile/", url.Values{})
p := entities.Artist.Query().Order("Index").MustExecute(ts.GET("/").Request)
artistList := p.Data.([]hplink.Artist)
assert.EqInt(1, len(artistList))
assert.EqInt(0, artistList[0].Index)
assert.EqStr("c-ute", artistList[0].Key)
assert.EqStr("℃-ute", artistList[0].Name)
p = entities.Member.Query().Order("Index").Filter("ArtistKey=", "c-ute").MustExecute(ts.GET("/").Request)
memberList := p.Data.([]hplink.Member)
assert.EqInt(5, len(memberList))
assert.EqInt(0, memberList[0].Index)
assert.EqStr("c-ute", memberList[0].ArtistKey)
assert.EqStr("c-ute.maimi_yajima", memberList[0].Key)
assert.EqStr("矢島舞美", memberList[0].Name)
entities.FillMemberPublicProfiles(ts.GET("/").Request, memberList)
assert.NotNil(memberList[0].PublicProfile)
assert.EqStr("c-ute.maimi_yajima", memberList[0].PublicProfile.Key)
assert.EqStr("埼玉県", memberList[0].PublicProfile.Hometown)
_, settings := entities.CrawlerSettings.Get().Key("http://ameblo.jp/c-ute-official").MustOne(ts.GET("/").Request)
assert.NotNil(settings)
assert.EqStr("c-ute", settings.(*hplink.CrawlerSettings).ArtistKey)
assert.EqInt(int(hplink.CrawlerSettingsTypeAmeblo), int(settings.(*hplink.CrawlerSettings).Type))
})
})
}
func Test_API_Tasks_Crawlers_Ameblo_EntryLists(t *testing.T) {
assert := gaetest.NewAssert(t)
assert.Nil(gaetest.ResetFixtureFromFile(ts.Context, "./fixtures/Test_API_Tasks_Crawlers_Ameblo_EntryLists.json", nil, "hplink"), "fixture")
httptest.StartMockServer(func(mock *httptest.MockServer) {
mock.Routes().GET("/morningmusume-9ki/entrylist.html", middleware.ServeFile("./fixtures/mocks/ameblo.jp//morningmusume-9ki/entrylist.html"))
crawler.MockExternalResource(map[string]string{
"http://ameblo.jp/morningmusume-9ki/entrylist.html": fmt.Sprintf("%s//morningmusume-9ki/entrylist.html", mock.BaseURL()),
}, func() {
runner := testhelper.NewAsyncTaskTestRunner(t, instance.Routes(), ts)
runner.OnMonitor(func(testreq *httptest.TestRequest, f func()) {
testhelper.TemporaryAllow(testreq, "hplink.admin", f)
})
runner.OnTrigger(func(testreq *httptest.TestRequest, f func()) {
testhelper.TemporaryAllow(testreq, "hplink.admin", f)
})
runner.Run("/api/hplink/tasks/crawlers/ameblo/entrylists/", url.Values{})
p := entities.AmebloPost.Query().Order("-PostAt").MustExecute(ts.GET("/").Request)
postList := p.Data.([]hplink.AmebloPost)
assert.EqInt(20, len(postList))
assert.EqStr("ノーメイク☆譜久村聖", postList[0].Title)
assert.EqStr("", postList[0].Theme)
assert.EqStr("morningmusume.mizuki_fukumura", postList[0].MemberKey)
assert.EqStr("http://ameblo.jp/morningmusume-9ki/", postList[0].SettingsURL)
p = entities.CrawlerSettings.Query().Filter("URL=", "http://ameblo.jp/morningmusume-9ki/").MustExecute(ts.GET("/").Request)
assert.EqInt(1, p.Length())
settings := p.Head().(*hplink.CrawlerSettings)
assert.EqInt(int(hplink.CrawlerStatusSuccess), int(settings.Status))
})
})
}
func Test_API_Tasks_Crawlers_Ameblo_Posts(t *testing.T) {
assert := gaetest.NewAssert(t)
assert.Nil(gaetest.ResetFixtureFromFile(ts.Context, "./fixtures/Test_API_Tasks_Crawlers_Ameblo_Posts.json", nil, "hplink"), "fixture")
p := entities.AmebloPost.Query().MustExecute(ts.GET("/").Request)
postList := p.Data.([]hplink.AmebloPost)
assert.EqInt(1, len(postList))
httptest.StartMockServer(func(mock *httptest.MockServer) {
mock.Routes().GET("/morningmusume-9ki/post.html", middleware.ServeFile("./fixtures/mocks/ameblo.jp//morningmusume-9ki/post.html"))
crawler.MockExternalResource(map[string]string{
"http://ameblo.jp/morningmusume-9ki/entry-12201946654.html": fmt.Sprintf("%s//morningmusume-9ki/post.html", mock.BaseURL()),
}, func() {
runner := testhelper.NewAsyncTaskTestRunner(t, instance.Routes(), ts)
runner.OnMonitor(func(testreq *httptest.TestRequest, f func()) {
testhelper.TemporaryAllow(testreq, "hplink.admin", f)
})
runner.OnTrigger(func(testreq *httptest.TestRequest, f func()) {
testhelper.TemporaryAllow(testreq, "hplink.admin", f)
})
runner.Run("/api/hplink/tasks/crawlers/ameblo/posts/", url.Values{})
p := entities.AmebloPost.Query().MustExecute(ts.GET("/").Request)
postList := p.Data.([]hplink.AmebloPost)
assert.EqInt(1, len(postList))
assert.EqStr("ノーメイク☆譜久村聖", postList[0].Title)
assert.EqStr("譜久村聖", postList[0].Theme)
assert.EqStr("morningmusume.mizuki_fukumura", postList[0].MemberKey)
// It also creates a URL cache
assert.EqInt(1, entities.URLCache.Query().MustCount(ts.GET("/").Request))
})
})
}
func Test_Artist_Run_NonGroup(t *testing.T) {
assert := wcg.NewAssert(t)
httptest.StartMockServer(func(mock *httptest.MockServer) {
// preparation for dummy entry
mock.Routes().GET("/profile/", middleware.ServeFile("./fixtures/mocks/helloproject.com/rihosayashi.html"))
mock.Routes().GET("/", middleware.ServeFile("./fixtures/mocks/helloproject.com/rihosayashi_top.html"))
crawler.MockExternalResource(map[string]string{
"http://www.helloproject.com/rihosayashi/profile/": mock.BaseURL() + "/profile/",
"http://www.helloproject.com/rihosayashi/": mock.BaseURL(),
}, func() {
data, err := NewArtist(http.DefaultClient, "rihosayashi").Run()
assert.Nil(err)
assert.EqInt(1, len(data.ArtistImageURLs))
assert.EqInt(1, len(data.Members))
assert.EqStr("http://cdn.helloproject.com/img/rotation/4ae1ab86fe60194c0a6c1e6a188a166eafa8686d.jpg", data.ArtistImageURLs[0])
assert.EqStr("rihosayashi", data.Members[0].Key)
assert.EqStr("鞘師里保", data.Members[0].Name)
})
})
}
func Test_HelloProject_Execute(t *testing.T) {
assert := wcg.NewAssert(t)
httptest.StartMockServer(func(mock *httptest.MockServer) {
// preparation for dummy entry
mock.Routes().GET("/*", middleware.ServeFile("./fixtures/mocks/helloproject.com/top.html"))
crawler.MockExternalResource(map[string]string{
"http://www.helloproject.com/artist/": mock.BaseURL(),
}, func() {
data, err := NewHelloProject(http.DefaultClient).Run()
assert.Nil(err)
assert.EqInt(10, len(data))
assert.EqStr("c-ute", data[0].Key)
assert.EqStr("℃-ute", data[0].Name)
assert.EqStr("http://www.helloproject.com/c-ute/", data[0].URL)
assert.EqStr("http://cdn.helloproject.com/img/artist/m/80bc0be0d26cfc7720b747e965bc5edbadf3e726.jpg", data[0].ImageURL)
})
})
}
func Test_Artist_Run(t *testing.T) {
assert := wcg.NewAssert(t)
httptest.StartMockServer(func(mock *httptest.MockServer) {
// preparation for dummy entry
mock.Routes().GET("/*", middleware.ServeFile("./fixtures/mocks/helloproject.com/c-ute.html"))
crawler.MockExternalResource(map[string]string{
"http://www.helloproject.com/c-ute/profile/": mock.BaseURL(),
}, func() {
data, err := NewArtist(http.DefaultClient, "c-ute").Run()
assert.Nil(err)
assert.EqInt(3, len(data.ArtistImageURLs))
assert.EqInt(5, len(data.Members))
assert.EqStr("http://cdn.helloproject.com/img/rotation/a974df522ac1ec6fa7668a43e0edac1b28c2aecd.jpg", data.ArtistImageURLs[0])
assert.EqStr("http://cdn.helloproject.com/img/rotation/836f19e6688f3e46b9c7373d729331eb67b29f7f.jpg", data.ArtistImageURLs[1])
assert.EqStr("http://cdn.helloproject.com/img/rotation/783e45e9d41c1bcc083fb62390ace0da94e63910.jpg", data.ArtistImageURLs[2])
assert.EqStr("maimi_yajima", data.Members[0].Key)
assert.EqStr("矢島舞美", data.Members[0].Name)
assert.EqInt(1, len(data.CrawlableURLs))
assert.EqStr("http://ameblo.jp/c-ute-official", data.CrawlableURLs[0].Link)
})
})
}
func Test_Subscribe(t *testing.T) {
assert := wcg.NewAssert(t)
httptest.StartMockServer(func(mock *httptest.MockServer) {
// preparation for dummy entry
mock.Routes().GET("/api/intern/pt/iepg/records/", middleware.ServeFile("./fixtures/mocks/subscriber_tests.json"))
s := NewSubscriber(mock.BaseURL(), "test-token")
t, _ := wcg.ParseDateTime("2016-05-12T16:00:00Z")
lib.TemporarySetNow(t, func() {
list, err := s.Subscribe()
assert.Nil(err, "error should be nil")
assert.EqInt(1, len(list), "Subscribe should return the list of *pt.IEPG")
})
t, _ = wcg.ParseDateTime("2016-05-13T18:00:00Z")
lib.TemporarySetNow(t, func() {
list, err := s.Subscribe()
assert.Nil(err, "error should be nil")
assert.EqInt(0, len(list), "Subscribe should not return any IEPG")
})
})
}
func Test_Agent_Recorder_Completed(t *testing.T) {
assert := wcg.NewAssert(t)
httptest.StartMockServer(func(mock *httptest.MockServer) {
// preparation for dummy entry
mock.Routes().GET("/api/intern/pt/iepg/records/", middleware.ServeFile("./fixtures/mocks/agent_tests.json"))
agent := NewAgent(mock.BaseURL(), "test-token", nil)
agent.recorderFactory = func(iepg *pt.IEPG) Recorder {
return NewMockRecorder(iepg)
}
now, _ := wcg.ParseDateTime("2016-05-15T16:05:00Z")
lib.TemporarySetNow(now, func() {
agent.RunOnce() // started
r := agent.recorders["200171201605160105"]
agent.RunOnce() // notify update
agent.RunOnce() // notify update
agent.RunOnce() // notify update
agent.RunOnce() // notify update and confirmed to be completed.
assert.EqInt(0, len(agent.recorders), "agent should have no recorder thread.")
assert.EqInt(int(r.GetStats().Status), int(pt.IEPGCompleted), "the recorder status should be pt.IEPGCompleted.")
})
})
}
