func Test_AsyncAPI_Define(t *testing.T) { assert := gaetest.NewAssert(t) app := NewApp("testapp") async := app.AsyncAPI() var processed = false var q1 = "" config := async.Define("/async/") config.OnProcess(AsyncTaskHandler(func(req *wcg.Request, _ *models.AsyncAPITask) (*models.AsyncAPITaskProgress, error) { processed = true q1 = req.Query("q1") return nil, nil })) assert.EqStr("api-testapp-async", config.Queue.Name) var path = "/api/testapp/async/" var params = url.Values{ "q1": []string{"a"}, } runner := testhelper.NewAsyncTaskTestRunner(t, app.Routes(), ts) runner.Run(fmt.Sprintf("%s", path), params) assert.OK(processed) assert.EqStr("a", q1) }
func Test_API_Tasks_Crawlers_Ameblo_EntryLists(t *testing.T) { assert := gaetest.NewAssert(t) assert.Nil(gaetest.ResetFixtureFromFile(ts.Context, "./fixtures/Test_API_Tasks_Crawlers_Ameblo_EntryLists.json", nil, "hplink"), "fixture") httptest.StartMockServer(func(mock *httptest.MockServer) { mock.Routes().GET("/morningmusume-9ki/entrylist.html", middleware.ServeFile("./fixtures/mocks/ameblo.jp//morningmusume-9ki/entrylist.html")) crawler.MockExternalResource(map[string]string{ "http://ameblo.jp/morningmusume-9ki/entrylist.html": fmt.Sprintf("%s//morningmusume-9ki/entrylist.html", mock.BaseURL()), }, func() { runner := testhelper.NewAsyncTaskTestRunner(t, instance.Routes(), ts) runner.OnMonitor(func(testreq *httptest.TestRequest, f func()) { testhelper.TemporaryAllow(testreq, "hplink.admin", f) }) runner.OnTrigger(func(testreq *httptest.TestRequest, f func()) { testhelper.TemporaryAllow(testreq, "hplink.admin", f) }) runner.Run("/api/hplink/tasks/crawlers/ameblo/entrylists/", url.Values{}) p := entities.AmebloPost.Query().Order("-PostAt").MustExecute(ts.GET("/").Request) postList := p.Data.([]hplink.AmebloPost) assert.EqInt(20, len(postList)) assert.EqStr("ノーメイク☆譜久村聖", postList[0].Title) assert.EqStr("", postList[0].Theme) assert.EqStr("morningmusume.mizuki_fukumura", postList[0].MemberKey) assert.EqStr("http://ameblo.jp/morningmusume-9ki/", postList[0].SettingsURL) p = entities.CrawlerSettings.Query().Filter("URL=", "http://ameblo.jp/morningmusume-9ki/").MustExecute(ts.GET("/").Request) assert.EqInt(1, p.Length()) settings := p.Head().(*hplink.CrawlerSettings) assert.EqInt(int(hplink.CrawlerStatusSuccess), int(settings.Status)) }) }) }
func Test_API_Tasks_Crawlers_Ameblo_Posts(t *testing.T) { assert := gaetest.NewAssert(t) assert.Nil(gaetest.ResetFixtureFromFile(ts.Context, "./fixtures/Test_API_Tasks_Crawlers_Ameblo_Posts.json", nil, "hplink"), "fixture") p := entities.AmebloPost.Query().MustExecute(ts.GET("/").Request) postList := p.Data.([]hplink.AmebloPost) assert.EqInt(1, len(postList)) httptest.StartMockServer(func(mock *httptest.MockServer) { mock.Routes().GET("/morningmusume-9ki/post.html", middleware.ServeFile("./fixtures/mocks/ameblo.jp//morningmusume-9ki/post.html")) crawler.MockExternalResource(map[string]string{ "http://ameblo.jp/morningmusume-9ki/entry-12201946654.html": fmt.Sprintf("%s//morningmusume-9ki/post.html", mock.BaseURL()), }, func() { runner := testhelper.NewAsyncTaskTestRunner(t, instance.Routes(), ts) runner.OnMonitor(func(testreq *httptest.TestRequest, f func()) { testhelper.TemporaryAllow(testreq, "hplink.admin", f) }) runner.OnTrigger(func(testreq *httptest.TestRequest, f func()) { testhelper.TemporaryAllow(testreq, "hplink.admin", f) }) runner.Run("/api/hplink/tasks/crawlers/ameblo/posts/", url.Values{}) p := entities.AmebloPost.Query().MustExecute(ts.GET("/").Request) postList := p.Data.([]hplink.AmebloPost) assert.EqInt(1, len(postList)) assert.EqStr("ノーメイク☆譜久村聖", postList[0].Title) assert.EqStr("譜久村聖", postList[0].Theme) assert.EqStr("morningmusume.mizuki_fukumura", postList[0].MemberKey) // It also creates a URL cache assert.EqInt(1, entities.URLCache.Query().MustCount(ts.GET("/").Request)) }) }) }
func Test_API_Task_Sync(t *testing.T) { assert := gaetest.NewAssert(t) gaetest.CleanupDatastore(ts.Context, "hplink") httptest.StartMockServer(func(mock *httptest.MockServer) { mock.Routes().GET("/", middleware.ServeFile("./fixtures/mocks/helloproject.html")) mock.Routes().GET("/c-ute/", middleware.ServeFile("./fixtures/mocks/c-ute/index.html")) mock.Routes().GET("/c-ute/*", middleware.StaticFile("", "fixtures/mocks/c-ute/")) crawler.MockExternalResource(map[string]string{ "http://www.helloproject.com/artist/": fmt.Sprintf("%s", mock.BaseURL()), "http://www.helloproject.com/artist/c-ute/": fmt.Sprintf("%sc-ute/", mock.BaseURL()), "http://www.helloproject.com/artist/c-ute/profile/maimi_yajima/": fmt.Sprintf("%sc-ute/maimi_yajima.html", mock.BaseURL()), "http://www.helloproject.com/artist/c-ute/profile/saki_nakajima/": fmt.Sprintf("%sc-ute/saki_nkajima.html", mock.BaseURL()), "http://www.helloproject.com/artist/c-ute/profile/airi_suzuki/": fmt.Sprintf("%sc-ute/airi_suzuki.html", mock.BaseURL()), "http://www.helloproject.com/artist/c-ute/profile/chisato_okai/": fmt.Sprintf("%sc-ute/chisato_okai.html", mock.BaseURL()), "http://www.helloproject.com/artist/c-ute/profile/mai_hagiwara/": fmt.Sprintf("%sc-ute/mai_hagiwara.html", mock.BaseURL()), }, func() { runner := testhelper.NewAsyncTaskTestRunner(t, instance.Routes(), ts) runner.OnMonitor(func(testreq *httptest.TestRequest, f func()) { testhelper.TemporaryAllow(testreq, "hplink.admin", f) }) runner.OnTrigger(func(testreq *httptest.TestRequest, f func()) { testhelper.TemporaryAllow(testreq, "hplink.admin", f) }) runner.Run("/api/hplink/tasks/profile/", url.Values{}) p := entities.Artist.Query().Order("Index").MustExecute(ts.GET("/").Request) artistList := p.Data.([]hplink.Artist) assert.EqInt(1, len(artistList)) assert.EqInt(0, artistList[0].Index) assert.EqStr("c-ute", artistList[0].Key) assert.EqStr("℃-ute", artistList[0].Name) p = entities.Member.Query().Order("Index").Filter("ArtistKey=", "c-ute").MustExecute(ts.GET("/").Request) memberList := p.Data.([]hplink.Member) assert.EqInt(5, len(memberList)) assert.EqInt(0, memberList[0].Index) assert.EqStr("c-ute", memberList[0].ArtistKey) assert.EqStr("c-ute.maimi_yajima", memberList[0].Key) assert.EqStr("矢島舞美", memberList[0].Name) entities.FillMemberPublicProfiles(ts.GET("/").Request, memberList) assert.NotNil(memberList[0].PublicProfile) assert.EqStr("c-ute.maimi_yajima", memberList[0].PublicProfile.Key) assert.EqStr("埼玉県", memberList[0].PublicProfile.Hometown) _, settings := entities.CrawlerSettings.Get().Key("http://ameblo.jp/c-ute-official").MustOne(ts.GET("/").Request) assert.NotNil(settings) assert.EqStr("c-ute", settings.(*hplink.CrawlerSettings).ArtistKey) assert.EqInt(int(hplink.CrawlerSettingsTypeAmeblo), int(settings.(*hplink.CrawlerSettings).Type)) }) }) }
func Test_AsyncAPI_Define_RecursiveCall(t *testing.T) { assert := gaetest.NewAssert(t) app := NewApp("testapp") async := app.AsyncAPI() var processed = 0 var q1 string config := async.Define("/async/") config.OnProcess(AsyncTaskHandler(func(req *wcg.Request, t *models.AsyncAPITask) (*models.AsyncAPITaskProgress, error) { var progress models.AsyncAPITaskProgress lastp := t.LastProgress() if lastp != nil { progress.Current = lastp.Current + 1 } else { progress.Current++ } progress.Total = 4 // confirm progress text is updated. processed++ if processed == 4 { q1 = req.Query("q1") // no progress returns to stop the task return &progress, nil } if processed == 3 { progress.Next = url.Values{ "q1": []string{"b"}, } } else { progress.Next = url.Values{} } return &progress, nil })) assert.EqStr("api-testapp-async", config.Queue.Name) var path = "/api/testapp/async/" var params = url.Values{} runner := testhelper.NewAsyncTaskTestRunner(t, app.Routes(), ts) task := runner.Run(fmt.Sprintf("%s", path), params) assert.OK(models.AsyncAPIStatusSuccess == task.Status) assert.EqInt(4, len(task.Progress)) assert.EqInt(4, task.LastProgress().Current) assert.EqInt(4, processed) assert.EqStr("b", q1) }
func Test_AsyncAPI_Define_Fail(t *testing.T) { assert := gaetest.NewAssert(t) app := NewApp("testapp") async := app.AsyncAPI() config := async.Define("/async/") config.OnProcess(AsyncTaskHandler(func(req *wcg.Request, _ *models.AsyncAPITask) (*models.AsyncAPITaskProgress, error) { return nil, fmt.Errorf("taskfail") })) assert.EqStr("api-testapp-async", config.Queue.Name) var path = "/api/testapp/async/" var params = url.Values{} runner := testhelper.NewAsyncTaskTestRunner(t, app.Routes(), ts) task := runner.Run(fmt.Sprintf("%s", path), params) assert.OK(models.AsyncAPIStatusFailure == task.Status) assert.EqStr("taskfail", task.Error) }
func Test_API_IEPGExclusion_RunTask(t *testing.T) { assert := gaetest.NewAssert(t) assert.Nil(gaetest.ResetFixtureFromFile(ts.Context, "./fixtures/Test_API_IEPGExclusion_RunTask.json", nil, "intern.pt")) now, _ := wcg.ParseDateTime("2016-01-01T00:00:00Z") lib.TemporarySetNow(now, func() { runner := helper.NewAsyncTaskTestRunner(t, instance.Routes(), ts) runner.OnTrigger(func(req *httptest.TestRequest, trigger func()) { helper.TemporaryAllow(req, "family", trigger) }) runner.OnMonitor(func(req *httptest.TestRequest, monitor func()) { helper.TemporaryAllow(req, "family", monitor) }) runner.Run("/api/intern/pt/iepg/exclusions/task/", url.Values{}) req := ts.GET("/dummy") _, flagged := IEPG.Get().Key("test1").MustOne(req.Request) assert.OK(len(flagged.(*pt.IEPG).ExcludedBy) > 0) }) }