func Test_AsyncAPI_Define(t *testing.T) {
assert := gaetest.NewAssert(t)
app := NewApp("testapp")
async := app.AsyncAPI()
var processed = false
var q1 = ""
config := async.Define("/async/")
config.OnProcess(AsyncTaskHandler(func(req *wcg.Request, _ *models.AsyncAPITask) (*models.AsyncAPITaskProgress, error) {
processed = true
q1 = req.Query("q1")
return nil, nil
}))
assert.EqStr("api-testapp-async", config.Queue.Name)
var path = "/api/testapp/async/"
var params = url.Values{
"q1": []string{"a"},
}
runner := testhelper.NewAsyncTaskTestRunner(t, app.Routes(), ts)
runner.Run(fmt.Sprintf("%s", path), params)
assert.OK(processed)
assert.EqStr("a", q1)
}
func Test_API_Tasks_Crawlers_Ameblo_EntryLists(t *testing.T) {
assert := gaetest.NewAssert(t)
assert.Nil(gaetest.ResetFixtureFromFile(ts.Context, "./fixtures/Test_API_Tasks_Crawlers_Ameblo_EntryLists.json", nil, "hplink"), "fixture")
httptest.StartMockServer(func(mock *httptest.MockServer) {
mock.Routes().GET("/morningmusume-9ki/entrylist.html", middleware.ServeFile("./fixtures/mocks/ameblo.jp//morningmusume-9ki/entrylist.html"))
crawler.MockExternalResource(map[string]string{
"http://ameblo.jp/morningmusume-9ki/entrylist.html": fmt.Sprintf("%s//morningmusume-9ki/entrylist.html", mock.BaseURL()),
}, func() {
runner := testhelper.NewAsyncTaskTestRunner(t, instance.Routes(), ts)
runner.OnMonitor(func(testreq *httptest.TestRequest, f func()) {
testhelper.TemporaryAllow(testreq, "hplink.admin", f)
})
runner.OnTrigger(func(testreq *httptest.TestRequest, f func()) {
testhelper.TemporaryAllow(testreq, "hplink.admin", f)
})
runner.Run("/api/hplink/tasks/crawlers/ameblo/entrylists/", url.Values{})
p := entities.AmebloPost.Query().Order("-PostAt").MustExecute(ts.GET("/").Request)
postList := p.Data.([]hplink.AmebloPost)
assert.EqInt(20, len(postList))
assert.EqStr("ノーメイク☆譜久村聖", postList[0].Title)
assert.EqStr("", postList[0].Theme)
assert.EqStr("morningmusume.mizuki_fukumura", postList[0].MemberKey)
assert.EqStr("http://ameblo.jp/morningmusume-9ki/", postList[0].SettingsURL)
p = entities.CrawlerSettings.Query().Filter("URL=", "http://ameblo.jp/morningmusume-9ki/").MustExecute(ts.GET("/").Request)
assert.EqInt(1, p.Length())
settings := p.Head().(*hplink.CrawlerSettings)
assert.EqInt(int(hplink.CrawlerStatusSuccess), int(settings.Status))
})
})
}
func Test_API_Tasks_Crawlers_Ameblo_Posts(t *testing.T) {
assert := gaetest.NewAssert(t)
assert.Nil(gaetest.ResetFixtureFromFile(ts.Context, "./fixtures/Test_API_Tasks_Crawlers_Ameblo_Posts.json", nil, "hplink"), "fixture")
p := entities.AmebloPost.Query().MustExecute(ts.GET("/").Request)
postList := p.Data.([]hplink.AmebloPost)
assert.EqInt(1, len(postList))
httptest.StartMockServer(func(mock *httptest.MockServer) {
mock.Routes().GET("/morningmusume-9ki/post.html", middleware.ServeFile("./fixtures/mocks/ameblo.jp//morningmusume-9ki/post.html"))
crawler.MockExternalResource(map[string]string{
"http://ameblo.jp/morningmusume-9ki/entry-12201946654.html": fmt.Sprintf("%s//morningmusume-9ki/post.html", mock.BaseURL()),
}, func() {
runner := testhelper.NewAsyncTaskTestRunner(t, instance.Routes(), ts)
runner.OnMonitor(func(testreq *httptest.TestRequest, f func()) {
testhelper.TemporaryAllow(testreq, "hplink.admin", f)
})
runner.OnTrigger(func(testreq *httptest.TestRequest, f func()) {
testhelper.TemporaryAllow(testreq, "hplink.admin", f)
})
runner.Run("/api/hplink/tasks/crawlers/ameblo/posts/", url.Values{})
p := entities.AmebloPost.Query().MustExecute(ts.GET("/").Request)
postList := p.Data.([]hplink.AmebloPost)
assert.EqInt(1, len(postList))
assert.EqStr("ノーメイク☆譜久村聖", postList[0].Title)
assert.EqStr("譜久村聖", postList[0].Theme)
assert.EqStr("morningmusume.mizuki_fukumura", postList[0].MemberKey)
// It also creates a URL cache
assert.EqInt(1, entities.URLCache.Query().MustCount(ts.GET("/").Request))
})
})
}
func Test_API_Task_Sync(t *testing.T) {
assert := gaetest.NewAssert(t)
gaetest.CleanupDatastore(ts.Context, "hplink")
httptest.StartMockServer(func(mock *httptest.MockServer) {
mock.Routes().GET("/", middleware.ServeFile("./fixtures/mocks/helloproject.html"))
mock.Routes().GET("/c-ute/", middleware.ServeFile("./fixtures/mocks/c-ute/index.html"))
mock.Routes().GET("/c-ute/*", middleware.StaticFile("", "fixtures/mocks/c-ute/"))
crawler.MockExternalResource(map[string]string{
"http://www.helloproject.com/artist/": fmt.Sprintf("%s", mock.BaseURL()),
"http://www.helloproject.com/artist/c-ute/": fmt.Sprintf("%sc-ute/", mock.BaseURL()),
"http://www.helloproject.com/artist/c-ute/profile/maimi_yajima/": fmt.Sprintf("%sc-ute/maimi_yajima.html", mock.BaseURL()),
"http://www.helloproject.com/artist/c-ute/profile/saki_nakajima/": fmt.Sprintf("%sc-ute/saki_nkajima.html", mock.BaseURL()),
"http://www.helloproject.com/artist/c-ute/profile/airi_suzuki/": fmt.Sprintf("%sc-ute/airi_suzuki.html", mock.BaseURL()),
"http://www.helloproject.com/artist/c-ute/profile/chisato_okai/": fmt.Sprintf("%sc-ute/chisato_okai.html", mock.BaseURL()),
"http://www.helloproject.com/artist/c-ute/profile/mai_hagiwara/": fmt.Sprintf("%sc-ute/mai_hagiwara.html", mock.BaseURL()),
}, func() {
runner := testhelper.NewAsyncTaskTestRunner(t, instance.Routes(), ts)
runner.OnMonitor(func(testreq *httptest.TestRequest, f func()) {
testhelper.TemporaryAllow(testreq, "hplink.admin", f)
})
runner.OnTrigger(func(testreq *httptest.TestRequest, f func()) {
testhelper.TemporaryAllow(testreq, "hplink.admin", f)
})
runner.Run("/api/hplink/tasks/profile/", url.Values{})
p := entities.Artist.Query().Order("Index").MustExecute(ts.GET("/").Request)
artistList := p.Data.([]hplink.Artist)
assert.EqInt(1, len(artistList))
assert.EqInt(0, artistList[0].Index)
assert.EqStr("c-ute", artistList[0].Key)
assert.EqStr("℃-ute", artistList[0].Name)
p = entities.Member.Query().Order("Index").Filter("ArtistKey=", "c-ute").MustExecute(ts.GET("/").Request)
memberList := p.Data.([]hplink.Member)
assert.EqInt(5, len(memberList))
assert.EqInt(0, memberList[0].Index)
assert.EqStr("c-ute", memberList[0].ArtistKey)
assert.EqStr("c-ute.maimi_yajima", memberList[0].Key)
assert.EqStr("矢島舞美", memberList[0].Name)
entities.FillMemberPublicProfiles(ts.GET("/").Request, memberList)
assert.NotNil(memberList[0].PublicProfile)
assert.EqStr("c-ute.maimi_yajima", memberList[0].PublicProfile.Key)
assert.EqStr("埼玉県", memberList[0].PublicProfile.Hometown)
_, settings := entities.CrawlerSettings.Get().Key("http://ameblo.jp/c-ute-official").MustOne(ts.GET("/").Request)
assert.NotNil(settings)
assert.EqStr("c-ute", settings.(*hplink.CrawlerSettings).ArtistKey)
assert.EqInt(int(hplink.CrawlerSettingsTypeAmeblo), int(settings.(*hplink.CrawlerSettings).Type))
})
})
}
func Test_AsyncAPI_Define_RecursiveCall(t *testing.T) {
assert := gaetest.NewAssert(t)
app := NewApp("testapp")
async := app.AsyncAPI()
var processed = 0
var q1 string
config := async.Define("/async/")
config.OnProcess(AsyncTaskHandler(func(req *wcg.Request, t *models.AsyncAPITask) (*models.AsyncAPITaskProgress, error) {
var progress models.AsyncAPITaskProgress
lastp := t.LastProgress()
if lastp != nil {
progress.Current = lastp.Current + 1
} else {
progress.Current++
}
progress.Total = 4
// confirm progress text is updated.
processed++
if processed == 4 {
q1 = req.Query("q1")
// no progress returns to stop the task
return &progress, nil
}
if processed == 3 {
progress.Next = url.Values{
"q1": []string{"b"},
}
} else {
progress.Next = url.Values{}
}
return &progress, nil
}))
assert.EqStr("api-testapp-async", config.Queue.Name)
var path = "/api/testapp/async/"
var params = url.Values{}
runner := testhelper.NewAsyncTaskTestRunner(t, app.Routes(), ts)
task := runner.Run(fmt.Sprintf("%s", path), params)
assert.OK(models.AsyncAPIStatusSuccess == task.Status)
assert.EqInt(4, len(task.Progress))
assert.EqInt(4, task.LastProgress().Current)
assert.EqInt(4, processed)
assert.EqStr("b", q1)
}
func Test_AsyncAPI_Define_Fail(t *testing.T) {
assert := gaetest.NewAssert(t)
app := NewApp("testapp")
async := app.AsyncAPI()
config := async.Define("/async/")
config.OnProcess(AsyncTaskHandler(func(req *wcg.Request, _ *models.AsyncAPITask) (*models.AsyncAPITaskProgress, error) {
return nil, fmt.Errorf("taskfail")
}))
assert.EqStr("api-testapp-async", config.Queue.Name)
var path = "/api/testapp/async/"
var params = url.Values{}
runner := testhelper.NewAsyncTaskTestRunner(t, app.Routes(), ts)
task := runner.Run(fmt.Sprintf("%s", path), params)
assert.OK(models.AsyncAPIStatusFailure == task.Status)
assert.EqStr("taskfail", task.Error)
}
func Test_API_IEPGExclusion_RunTask(t *testing.T) {
assert := gaetest.NewAssert(t)
assert.Nil(gaetest.ResetFixtureFromFile(ts.Context, "./fixtures/Test_API_IEPGExclusion_RunTask.json", nil, "intern.pt"))
now, _ := wcg.ParseDateTime("2016-01-01T00:00:00Z")
lib.TemporarySetNow(now, func() {
runner := helper.NewAsyncTaskTestRunner(t, instance.Routes(), ts)
runner.OnTrigger(func(req *httptest.TestRequest, trigger func()) {
helper.TemporaryAllow(req, "family", trigger)
})
runner.OnMonitor(func(req *httptest.TestRequest, monitor func()) {
helper.TemporaryAllow(req, "family", monitor)
})
runner.Run("/api/intern/pt/iepg/exclusions/task/", url.Values{})
req := ts.GET("/dummy")
_, flagged := IEPG.Get().Key("test1").MustOne(req.Request)
assert.OK(len(flagged.(*pt.IEPG).ExcludedBy) > 0)
})
}