Beispiel #1
0
func Test_AsyncAPI_Define(t *testing.T) {
	assert := gaetest.NewAssert(t)
	app := NewApp("testapp")
	async := app.AsyncAPI()

	var processed = false
	var q1 = ""
	config := async.Define("/async/")
	config.OnProcess(AsyncTaskHandler(func(req *wcg.Request, _ *models.AsyncAPITask) (*models.AsyncAPITaskProgress, error) {
		processed = true
		q1 = req.Query("q1")
		return nil, nil
	}))
	assert.EqStr("api-testapp-async", config.Queue.Name)

	var path = "/api/testapp/async/"
	var params = url.Values{
		"q1": []string{"a"},
	}

	runner := testhelper.NewAsyncTaskTestRunner(t, app.Routes(), ts)
	runner.Run(fmt.Sprintf("%s", path), params)

	assert.OK(processed)
	assert.EqStr("a", q1)
}
Beispiel #2
0
func Test_API_Tasks_Crawlers_Ameblo_EntryLists(t *testing.T) {
	assert := gaetest.NewAssert(t)
	assert.Nil(gaetest.ResetFixtureFromFile(ts.Context, "./fixtures/Test_API_Tasks_Crawlers_Ameblo_EntryLists.json", nil, "hplink"), "fixture")

	httptest.StartMockServer(func(mock *httptest.MockServer) {
		mock.Routes().GET("/morningmusume-9ki/entrylist.html", middleware.ServeFile("./fixtures/mocks/ameblo.jp//morningmusume-9ki/entrylist.html"))
		crawler.MockExternalResource(map[string]string{
			"http://ameblo.jp/morningmusume-9ki/entrylist.html": fmt.Sprintf("%s//morningmusume-9ki/entrylist.html", mock.BaseURL()),
		}, func() {
			runner := testhelper.NewAsyncTaskTestRunner(t, instance.Routes(), ts)
			runner.OnMonitor(func(testreq *httptest.TestRequest, f func()) {
				testhelper.TemporaryAllow(testreq, "hplink.admin", f)
			})
			runner.OnTrigger(func(testreq *httptest.TestRequest, f func()) {
				testhelper.TemporaryAllow(testreq, "hplink.admin", f)
			})
			runner.Run("/api/hplink/tasks/crawlers/ameblo/entrylists/", url.Values{})

			p := entities.AmebloPost.Query().Order("-PostAt").MustExecute(ts.GET("/").Request)
			postList := p.Data.([]hplink.AmebloPost)
			assert.EqInt(20, len(postList))
			assert.EqStr("ノーメイク☆譜久村聖", postList[0].Title)
			assert.EqStr("", postList[0].Theme)
			assert.EqStr("morningmusume.mizuki_fukumura", postList[0].MemberKey)
			assert.EqStr("http://ameblo.jp/morningmusume-9ki/", postList[0].SettingsURL)

			p = entities.CrawlerSettings.Query().Filter("URL=", "http://ameblo.jp/morningmusume-9ki/").MustExecute(ts.GET("/").Request)
			assert.EqInt(1, p.Length())
			settings := p.Head().(*hplink.CrawlerSettings)
			assert.EqInt(int(hplink.CrawlerStatusSuccess), int(settings.Status))
		})
	})
}
Beispiel #3
0
func Test_API_Tasks_Crawlers_Ameblo_Posts(t *testing.T) {
	assert := gaetest.NewAssert(t)
	assert.Nil(gaetest.ResetFixtureFromFile(ts.Context, "./fixtures/Test_API_Tasks_Crawlers_Ameblo_Posts.json", nil, "hplink"), "fixture")

	p := entities.AmebloPost.Query().MustExecute(ts.GET("/").Request)
	postList := p.Data.([]hplink.AmebloPost)
	assert.EqInt(1, len(postList))

	httptest.StartMockServer(func(mock *httptest.MockServer) {
		mock.Routes().GET("/morningmusume-9ki/post.html", middleware.ServeFile("./fixtures/mocks/ameblo.jp//morningmusume-9ki/post.html"))
		crawler.MockExternalResource(map[string]string{
			"http://ameblo.jp/morningmusume-9ki/entry-12201946654.html": fmt.Sprintf("%s//morningmusume-9ki/post.html", mock.BaseURL()),
		}, func() {
			runner := testhelper.NewAsyncTaskTestRunner(t, instance.Routes(), ts)
			runner.OnMonitor(func(testreq *httptest.TestRequest, f func()) {
				testhelper.TemporaryAllow(testreq, "hplink.admin", f)
			})
			runner.OnTrigger(func(testreq *httptest.TestRequest, f func()) {
				testhelper.TemporaryAllow(testreq, "hplink.admin", f)
			})
			runner.Run("/api/hplink/tasks/crawlers/ameblo/posts/", url.Values{})

			p := entities.AmebloPost.Query().MustExecute(ts.GET("/").Request)
			postList := p.Data.([]hplink.AmebloPost)
			assert.EqInt(1, len(postList))
			assert.EqStr("ノーメイク☆譜久村聖", postList[0].Title)
			assert.EqStr("譜久村聖", postList[0].Theme)
			assert.EqStr("morningmusume.mizuki_fukumura", postList[0].MemberKey)

			// It also creates a URL cache
			assert.EqInt(1, entities.URLCache.Query().MustCount(ts.GET("/").Request))
		})
	})
}
Beispiel #4
0
func Test_API_Task_Sync(t *testing.T) {
	assert := gaetest.NewAssert(t)
	gaetest.CleanupDatastore(ts.Context, "hplink")

	httptest.StartMockServer(func(mock *httptest.MockServer) {
		mock.Routes().GET("/", middleware.ServeFile("./fixtures/mocks/helloproject.html"))
		mock.Routes().GET("/c-ute/", middleware.ServeFile("./fixtures/mocks/c-ute/index.html"))
		mock.Routes().GET("/c-ute/*", middleware.StaticFile("", "fixtures/mocks/c-ute/"))

		crawler.MockExternalResource(map[string]string{
			"http://www.helloproject.com/artist/":                             fmt.Sprintf("%s", mock.BaseURL()),
			"http://www.helloproject.com/artist/c-ute/":                       fmt.Sprintf("%sc-ute/", mock.BaseURL()),
			"http://www.helloproject.com/artist/c-ute/profile/maimi_yajima/":  fmt.Sprintf("%sc-ute/maimi_yajima.html", mock.BaseURL()),
			"http://www.helloproject.com/artist/c-ute/profile/saki_nakajima/": fmt.Sprintf("%sc-ute/saki_nkajima.html", mock.BaseURL()),
			"http://www.helloproject.com/artist/c-ute/profile/airi_suzuki/":   fmt.Sprintf("%sc-ute/airi_suzuki.html", mock.BaseURL()),
			"http://www.helloproject.com/artist/c-ute/profile/chisato_okai/":  fmt.Sprintf("%sc-ute/chisato_okai.html", mock.BaseURL()),
			"http://www.helloproject.com/artist/c-ute/profile/mai_hagiwara/":  fmt.Sprintf("%sc-ute/mai_hagiwara.html", mock.BaseURL()),
		}, func() {
			runner := testhelper.NewAsyncTaskTestRunner(t, instance.Routes(), ts)
			runner.OnMonitor(func(testreq *httptest.TestRequest, f func()) {
				testhelper.TemporaryAllow(testreq, "hplink.admin", f)
			})
			runner.OnTrigger(func(testreq *httptest.TestRequest, f func()) {
				testhelper.TemporaryAllow(testreq, "hplink.admin", f)
			})
			runner.Run("/api/hplink/tasks/profile/", url.Values{})

			p := entities.Artist.Query().Order("Index").MustExecute(ts.GET("/").Request)
			artistList := p.Data.([]hplink.Artist)
			assert.EqInt(1, len(artistList))
			assert.EqInt(0, artistList[0].Index)
			assert.EqStr("c-ute", artistList[0].Key)
			assert.EqStr("℃-ute", artistList[0].Name)

			p = entities.Member.Query().Order("Index").Filter("ArtistKey=", "c-ute").MustExecute(ts.GET("/").Request)
			memberList := p.Data.([]hplink.Member)
			assert.EqInt(5, len(memberList))
			assert.EqInt(0, memberList[0].Index)
			assert.EqStr("c-ute", memberList[0].ArtistKey)
			assert.EqStr("c-ute.maimi_yajima", memberList[0].Key)
			assert.EqStr("矢島舞美", memberList[0].Name)

			entities.FillMemberPublicProfiles(ts.GET("/").Request, memberList)
			assert.NotNil(memberList[0].PublicProfile)
			assert.EqStr("c-ute.maimi_yajima", memberList[0].PublicProfile.Key)
			assert.EqStr("埼玉県", memberList[0].PublicProfile.Hometown)

			_, settings := entities.CrawlerSettings.Get().Key("http://ameblo.jp/c-ute-official").MustOne(ts.GET("/").Request)
			assert.NotNil(settings)
			assert.EqStr("c-ute", settings.(*hplink.CrawlerSettings).ArtistKey)
			assert.EqInt(int(hplink.CrawlerSettingsTypeAmeblo), int(settings.(*hplink.CrawlerSettings).Type))
		})
	})
}
Beispiel #5
0
func Test_AsyncAPI_Define_RecursiveCall(t *testing.T) {
	assert := gaetest.NewAssert(t)
	app := NewApp("testapp")
	async := app.AsyncAPI()

	var processed = 0
	var q1 string
	config := async.Define("/async/")
	config.OnProcess(AsyncTaskHandler(func(req *wcg.Request, t *models.AsyncAPITask) (*models.AsyncAPITaskProgress, error) {
		var progress models.AsyncAPITaskProgress
		lastp := t.LastProgress()
		if lastp != nil {
			progress.Current = lastp.Current + 1
		} else {
			progress.Current++
		}
		progress.Total = 4
		// confirm progress text is updated.
		processed++
		if processed == 4 {
			q1 = req.Query("q1")
			// no progress returns to stop the task
			return &progress, nil
		}
		if processed == 3 {
			progress.Next = url.Values{
				"q1": []string{"b"},
			}
		} else {
			progress.Next = url.Values{}
		}
		return &progress, nil
	}))
	assert.EqStr("api-testapp-async", config.Queue.Name)

	var path = "/api/testapp/async/"
	var params = url.Values{}

	runner := testhelper.NewAsyncTaskTestRunner(t, app.Routes(), ts)
	task := runner.Run(fmt.Sprintf("%s", path), params)

	assert.OK(models.AsyncAPIStatusSuccess == task.Status)
	assert.EqInt(4, len(task.Progress))
	assert.EqInt(4, task.LastProgress().Current)
	assert.EqInt(4, processed)
	assert.EqStr("b", q1)
}
Beispiel #6
0
func Test_AsyncAPI_Define_Fail(t *testing.T) {
	assert := gaetest.NewAssert(t)
	app := NewApp("testapp")
	async := app.AsyncAPI()

	config := async.Define("/async/")
	config.OnProcess(AsyncTaskHandler(func(req *wcg.Request, _ *models.AsyncAPITask) (*models.AsyncAPITaskProgress, error) {
		return nil, fmt.Errorf("taskfail")
	}))
	assert.EqStr("api-testapp-async", config.Queue.Name)

	var path = "/api/testapp/async/"
	var params = url.Values{}

	runner := testhelper.NewAsyncTaskTestRunner(t, app.Routes(), ts)
	task := runner.Run(fmt.Sprintf("%s", path), params)
	assert.OK(models.AsyncAPIStatusFailure == task.Status)
	assert.EqStr("taskfail", task.Error)
}
func Test_API_IEPGExclusion_RunTask(t *testing.T) {
	assert := gaetest.NewAssert(t)
	assert.Nil(gaetest.ResetFixtureFromFile(ts.Context, "./fixtures/Test_API_IEPGExclusion_RunTask.json", nil, "intern.pt"))

	now, _ := wcg.ParseDateTime("2016-01-01T00:00:00Z")
	lib.TemporarySetNow(now, func() {
		runner := helper.NewAsyncTaskTestRunner(t, instance.Routes(), ts)
		runner.OnTrigger(func(req *httptest.TestRequest, trigger func()) {
			helper.TemporaryAllow(req, "family", trigger)
		})
		runner.OnMonitor(func(req *httptest.TestRequest, monitor func()) {
			helper.TemporaryAllow(req, "family", monitor)
		})
		runner.Run("/api/intern/pt/iepg/exclusions/task/", url.Values{})

		req := ts.GET("/dummy")
		_, flagged := IEPG.Get().Key("test1").MustOne(req.Request)
		assert.OK(len(flagged.(*pt.IEPG).ExcludedBy) > 0)

	})
}