Ejemplo n.º 1
0
func TestDelChannelApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)
		// prepare
		d := NewTvChannelDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
		ent1 := &tv.TvChannel{"c1", "s1", "foo", "bar"}
		ent2 := &tv.TvChannel{"c2", "s2", "hoge", "piyo"}
		d.Put(d.NewKey(ent1.Key(), 0, nil), ent1)
		d.Put(d.NewKey(ent2.Key(), 0, nil), ent2)

		err := util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 2
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm TvChannel entities has been stored within a timeout window.")
		p := app.Api.Path("/channels/c1/s1.json")
		req := ts.Delete(p)
		lib.SetApiTokenForTest(req, lib.Admin)
		res := req.RouteTo(app.Routes())
		assert.HttpStatus(200, res)

		err = util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 1
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "DELETE %s Confirm TvChannel entities has been deleted via API within a timeout window.", p)

		// Confirm cache invalidation
		mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
		assert.Ok(!mc.Exists(MC_KEY_CHANNELS), "DELETE %s should invalidate the cache", p)

	})
}
Ejemplo n.º 2
0
func TestAddKeywordApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)
		p := app.Api.Path("/keywords/")
		req := ts.PostForm(p, url.Values{
			"keyword":  []string{"モーニング娘。'15"},
			"category": []string{"モーニング娘。"},
			"scope":    []string{"1"},
		})
		lib.SetApiTokenForTest(req, lib.Admin)
		var got map[string]interface{}
		res := req.RouteTo(app.Routes())
		res.Json(&got)
		assert.HttpStatus(201, res)
		assert.EqStr(
			"http://localhost:8080/api/pt/keywords/モーニング娘。'15.json",
			got["location"].(string),
			"POST %s location",
			p,
		)

		// Confirm cache invalidation
		mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
		assert.Ok(!mc.Exists(MC_KEY_KEYWORDS), "POST %s should invalidate the cache", p)

	})
}
Ejemplo n.º 3
0
func TestAddChannelApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)
		p := app.Api.Path("/channels/")
		req := ts.PostForm(p, url.Values{
			"cid":             []string{"c1"},
			"sid":             []string{"s1"},
			"name":            []string{"foo"},
			"iepg_station_id": []string{"bar"},
		})
		lib.SetApiTokenForTest(req, lib.Admin)
		var got map[string]interface{}
		res := req.RouteTo(app.Routes())
		res.Json(&got)
		assert.HttpStatus(201, res)
		assert.EqStr(
			"http://localhost:8080/api/pt/channels/c1/s1.json",
			got["location"].(string),
			"POST %s location",
			p,
		)

		// Confirm cache invalidation
		mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
		assert.Ok(!mc.Exists(MC_KEY_CHANNELS), "POST %s should invalidate the cache", p)

	})
}
Ejemplo n.º 4
0
func NewRecordCacheDriver(ns string, ctx appengine.Context, logger wcg.Logger) *RecordCacheDriver {
	return &RecordCacheDriver{
		memcache.NewDriver(ctx, logger),
		NewTvRecordDriver(ns, ctx, logger),
		NewIEpgDriver(ns, ctx, logger),
		util.Today().UTC().Add((9 + 3) * time.Hour), // Cache key cycle = 03:00am in JST for the less impact on recording.
	}
}
Ejemplo n.º 5
0
func init() {
	app.Api.Get("/channels/",
		func(res *wcg.Response, req *wcg.Request) {
			if list, err := listTvChannels(res, req); err != nil {
				app.Api.InternalError(res, req, err)
			} else {
				res.WriteJson(list)
			}
		},
	)
	app.Api.Post("/channels/", lib.Admin.Required(
		func(res *wcg.Response, req *wcg.Request) {
			ctx := gae.NewContext(req)
			d := NewTvChannelDriver(app.Key, ctx, req.Logger)
			mc := memcache.NewDriver(ctx, req.Logger)
			if err := d.AddChannel(req.Form("cid"), req.Form("sid"), req.Form("name"), req.Form("iepg_station_id")); err != nil {
				lib.InternalError(res, req, err)
			} else {
				mc.Delete(MC_KEY_CHANNELS)
				id := fmt.Sprintf("%s/%s", req.Form("cid"), req.Form("sid"))
				app.Api.Created(res, req, id)
			}
		},
	))
	app.Api.Delete("/channels/:cid/:sid.json", lib.Admin.Required(
		func(res *wcg.Response, req *wcg.Request) {
			ctx := gae.NewContext(req)
			d := NewTvChannelDriver(app.Key, ctx, req.Logger)
			mc := memcache.NewDriver(ctx, req.Logger)
			if err := d.DelChannel(req.Param("cid"), req.Param("sid")); err != nil {
				lib.InternalError(res, req, err)
			} else {
				mc.Delete(MC_KEY_CHANNELS)
				app.Api.Ok(res, req)
			}
		},
	))
}
Ejemplo n.º 6
0
func TestListChannelApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)
		p := app.Api.Path("/channels/")
		err := util.WaitFor(func() bool {
			var got []tv.TvChannel
			req := ts.Get(p)
			res := req.RouteTo(app.Routes())
			assert.HttpStatus(200, res)
			res.Json(&got)
			return len(got) > 0
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "GET %s should return the list of TvChannels within a timeout window.", p)

		// Confirm cache invalidation
		mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
		assert.Ok(mc.Exists(MC_KEY_CHANNELS), "GET %s should create the cache", p)
	})
}
Ejemplo n.º 7
0
func TestDelKeywordApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)
		// prepare
		d := NewCrawlerConfigDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
		d.Add(&tv.CrawlerConfig{
			Keyword:  "モーニング娘。'15",
			Category: "モーニング娘",
			Scope:    tv.FEED_SCOPE_ALL,
		})
		d.Add(&tv.CrawlerConfig{
			Keyword:  "SPEED",
			Category: "SPEED",
			Scope:    tv.FEED_SCOPE_ALL,
		})

		err := util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 2
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm CrawlerConfig entities has been stored within a timeout window.")

		p := app.Api.Path("/keywords/モーニング娘。'15.json")
		req := ts.Delete(p)
		lib.SetApiTokenForTest(req, lib.Admin)
		res := req.RouteTo(app.Routes())
		assert.HttpStatus(200, res)

		err = util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 1
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "DELETE %s Confirm CrawlerConfig entities has been deleted via API within a timeout window.", p)

		// Confirm cache invalidation
		mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
		assert.Ok(!mc.Exists(MC_KEY_KEYWORDS), "DELETE %s should invalidate the cache", p)
	})
}
Ejemplo n.º 8
0
func TestListKeywordApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)

		// prepare
		d := NewCrawlerConfigDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
		d.Add(&tv.CrawlerConfig{
			Keyword:  "キーワード1",
			Category: "カテゴリー1",
			Scope:    1,
		})
		d.Add(&tv.CrawlerConfig{
			Keyword:  "キーワード2",
			Category: "カテゴリー2",
			Scope:    1,
		})

		err := util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 2
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm CrawlerConfig entities has been stored within a timeout window.")

		var got []*tv.CrawlerConfig
		p := app.Api.Path("/keywords/") + "?force=true"
		req := ts.Get(p)
		res := req.RouteTo(app.Routes())
		assert.HttpStatus(200, res)
		res.Json(&got)
		assert.EqStr("キーワード2", got[0].Keyword, "GET %s should return the list ordered by creation time.", p)
		assert.EqStr("キーワード1", got[1].Keyword, "GET %s should return the list ordered by creation time.", p)

		// Confirm cache invalidation
		mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
		assert.Ok(mc.Exists(MC_KEY_KEYWORDS), "GET %s should create the cache", p)
	})
}
Ejemplo n.º 9
0
func listTvChannels(res *wcg.Response, req *wcg.Request) ([]*tv.TvChannel, error) {
	var list []*tv.TvChannel
	app := lib.GetCurrentApp(req)
	ctx := gae.NewContext(req)
	d := NewTvChannelDriver(app.Key, ctx, req.Logger)
	mc := memcache.NewDriver(ctx, req.Logger)
	err := mc.CachedObject(MC_KEY_CHANNELS, &list, func() (interface{}, error) {
		return d.AllAsList()
	}, req.Query("force") == "1")
	if err != nil {
		return nil, err
	} else {
		if len(list) == 0 {
			req.Logger.Warn("No channel is defined. Reset the configuraiton.")
			d.AddChannelList(defaultChannels)
			mc.Delete(MC_KEY_CHANNELS)
			mc.Set(MC_KEY_CHANNELS, defaultChannels)
			res.WriteJson(defaultChannels)
			return defaultChannels, nil
		} else {
			return list, nil
		}
	}
}
Ejemplo n.º 10
0
func init() {
	app.Cron.Get(
		"Crawl keyword IEPGs",
		"every 3 hours",
		"/keywords/crawl/",
		lib.Admin.Required(
			func(res *wcg.Response, req *wcg.Request) {
				var channels []*tv.TvChannel
				var cfglist []*tv.CrawlerConfig
				ctx := gae.NewContext(req)
				d := NewCrawlerConfigDriver(app.Key, ctx, req.Logger)
				mc := memcache.NewDriver(ctx, req.Logger)
				err := mc.CachedObject(MC_KEY_CHANNELS, &channels, func() (interface{}, error) {
					return NewTvChannelDriver(app.Key, ctx, req.Logger).AllAsList()
				}, false)
				if err != nil {
					app.Api.InternalError(res, req, err)
					return
				}

				req.Logger.Info("Retrieving all cralwer configurations...")
				q := d.NewQuery()
				q.Order("-CreatedAt")
				if _, err := q.GetAll(&cfglist); err != nil {
					app.Api.InternalError(res, req, err)
					return
				} else if len(cfglist) == 0 {
					req.Logger.Info("No cralwer configuration is found.")
					res.WriteJson(map[string]int{
						"updates": 0,
					})
					return
				} else {
					req.Logger.Info("Found %d configs, start crawling", len(cfglist))
					iepglist := make([]*tv.IEpg, 0)
					for i := range cfglist {
						if list, err := getIEpgListFromCrawlerConfig(res, req, cfglist[i], channels); err != nil {
							req.Logger.Warn("Failed to crawl %v: %v", cfglist[i], err)
						} else {
							iepglist = append(iepglist, list...)
						}
					}

					// TODO: Sync bulk update
					d := NewRecordCacheDriver(app.Key, ctx, req.Logger)
					if keys, err := d.IEpg.BulkUpdate(iepglist); err != nil {
						app.Api.InternalError(res, req, err)
						return
					} else {
						req.Logger.Debug("Updated %d iepg entries...", len(keys))
						// invalidate cache after 10 seconds
						time.Sleep(10 * time.Second)
						d.GetRecords(true)
						res.WriteJson(map[string]int{
							"updates": len(iepglist),
						})
						return
					}
				}
			},
		))
}
Ejemplo n.º 11
0
func init() {
	app.Api.Get("/keywords/",
		func(res *wcg.Response, req *wcg.Request) {
			var list []*tv.CrawlerConfig
			ctx := gae.NewContext(req)
			mc := memcache.NewDriver(ctx, req.Logger)
			err := mc.CachedObject(MC_KEY_KEYWORDS, &list, func() (interface{}, error) {
				var list []*tv.CrawlerConfig
				d := NewCrawlerConfigDriver(app.Key, ctx, req.Logger)
				q := d.NewQuery().Order("-CreatedAt")
				_, err := q.GetAll(&list)
				if list == nil {
					return make([]*tv.CrawlerConfig, 0), nil
				} else {
					return list, err
				}
			}, req.Query("force") == "1")

			if err != nil {
				lib.InternalError(res, req, err)
			} else {
				res.WriteJson(list)
			}
		},
	)

	app.Api.Get("/keywords/preview/:keyword.json", lib.Family.Required(
		func(res *wcg.Response, req *wcg.Request) {
			var channels []*tv.TvChannel
			ctx := gae.NewContext(req)
			mc := memcache.NewDriver(ctx, req.Logger)
			err := mc.CachedObject(MC_KEY_CHANNELS, &channels, func() (interface{}, error) {
				return NewTvChannelDriver(app.Key, ctx, req.Logger).AllAsList()
			}, false)

			keyword := req.Param("keyword")
			scope, _ := strconv.Atoi(req.Query("scope"))
			list, err := getIEpgListFromCrawlerConfig(res, req, &tv.CrawlerConfig{
				Keyword:  keyword,
				Scope:    scope,
				Category: "dummy",
			}, channels)
			if err != nil {
				lib.InternalError(res, req, err)
			} else {
				res.WriteJson(map[string]interface{}{
					"samples": list,
					"total":   len(list),
				})
			}
		},
	))

	app.Api.Post("/keywords/", lib.Family.Required(
		func(res *wcg.Response, req *wcg.Request) {
			ctx := gae.NewContext(req)
			mc := memcache.NewDriver(ctx, req.Logger)
			d := NewCrawlerConfigDriver(app.Key, ctx, req.Logger)
			scope, _ := strconv.Atoi(req.Form("scope"))
			cfg := &tv.CrawlerConfig{
				Keyword:  req.Form("keyword"),
				Category: req.Form("category"),
				Scope:    scope,
			}
			if err := d.Add(cfg); err != nil {
				app.Api.InternalError(res, req, err)
				return
			} else {
				mc.Delete(MC_KEY_KEYWORDS)
				app.Api.Created(res, req, req.Form("keyword"))
				return
			}
		},
	))

	app.Api.Delete("/keywords/:keyword.json", lib.Family.Required(
		func(res *wcg.Response, req *wcg.Request) {
			ctx := gae.NewContext(req)
			d := NewCrawlerConfigDriver(app.Key, ctx, req.Logger)
			mc := memcache.NewDriver(ctx, req.Logger)
			keyword := req.Param("keyword")
			if err := d.Delete(keyword); err != nil {
				lib.InternalError(res, req, err)
				return
			} else {
				mc.Delete(MC_KEY_KEYWORDS)
				app.Api.Ok(res, req)
				return
			}
		},
	))

}
Ejemplo n.º 12
0
func (appCtx *AppContext) NewMemcacheDriver() *memcache.Driver {
	return memcache.NewDriver(appCtx.Context, appCtx.Logger)
}