func TestDelChannelApi(t *testing.T) { test.RunTestServer(func(ts *test.TestServer) { assert := test.NewAssert(t) // prepare d := NewTvChannelDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil)) ent1 := &tv.TvChannel{"c1", "s1", "foo", "bar"} ent2 := &tv.TvChannel{"c2", "s2", "hoge", "piyo"} d.Put(d.NewKey(ent1.Key(), 0, nil), ent1) d.Put(d.NewKey(ent2.Key(), 0, nil), ent2) err := util.WaitFor(func() bool { c, _ := d.NewQuery().Count() return c == 2 }, util.DefaultWaitForTimeout) assert.Nil(err, "Confirm TvChannel entities has been stored within a timeout window.") p := app.Api.Path("/channels/c1/s1.json") req := ts.Delete(p) lib.SetApiTokenForTest(req, lib.Admin) res := req.RouteTo(app.Routes()) assert.HttpStatus(200, res) err = util.WaitFor(func() bool { c, _ := d.NewQuery().Count() return c == 1 }, util.DefaultWaitForTimeout) assert.Nil(err, "DELETE %s Confirm TvChannel entities has been deleted via API within a timeout window.", p) // Confirm cache invalidation mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil)) assert.Ok(!mc.Exists(MC_KEY_CHANNELS), "DELETE %s should invalidate the cache", p) }) }
func TestAddKeywordApi(t *testing.T) { test.RunTestServer(func(ts *test.TestServer) { assert := test.NewAssert(t) p := app.Api.Path("/keywords/") req := ts.PostForm(p, url.Values{ "keyword": []string{"モーニング娘。'15"}, "category": []string{"モーニング娘。"}, "scope": []string{"1"}, }) lib.SetApiTokenForTest(req, lib.Admin) var got map[string]interface{} res := req.RouteTo(app.Routes()) res.Json(&got) assert.HttpStatus(201, res) assert.EqStr( "http://localhost:8080/api/pt/keywords/モーニング娘。'15.json", got["location"].(string), "POST %s location", p, ) // Confirm cache invalidation mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil)) assert.Ok(!mc.Exists(MC_KEY_KEYWORDS), "POST %s should invalidate the cache", p) }) }
func TestAddChannelApi(t *testing.T) { test.RunTestServer(func(ts *test.TestServer) { assert := test.NewAssert(t) p := app.Api.Path("/channels/") req := ts.PostForm(p, url.Values{ "cid": []string{"c1"}, "sid": []string{"s1"}, "name": []string{"foo"}, "iepg_station_id": []string{"bar"}, }) lib.SetApiTokenForTest(req, lib.Admin) var got map[string]interface{} res := req.RouteTo(app.Routes()) res.Json(&got) assert.HttpStatus(201, res) assert.EqStr( "http://localhost:8080/api/pt/channels/c1/s1.json", got["location"].(string), "POST %s location", p, ) // Confirm cache invalidation mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil)) assert.Ok(!mc.Exists(MC_KEY_CHANNELS), "POST %s should invalidate the cache", p) }) }
func NewRecordCacheDriver(ns string, ctx appengine.Context, logger wcg.Logger) *RecordCacheDriver { return &RecordCacheDriver{ memcache.NewDriver(ctx, logger), NewTvRecordDriver(ns, ctx, logger), NewIEpgDriver(ns, ctx, logger), util.Today().UTC().Add((9 + 3) * time.Hour), // Cache key cycle = 03:00am in JST for the less impact on recording. } }
func init() { app.Api.Get("/channels/", func(res *wcg.Response, req *wcg.Request) { if list, err := listTvChannels(res, req); err != nil { app.Api.InternalError(res, req, err) } else { res.WriteJson(list) } }, ) app.Api.Post("/channels/", lib.Admin.Required( func(res *wcg.Response, req *wcg.Request) { ctx := gae.NewContext(req) d := NewTvChannelDriver(app.Key, ctx, req.Logger) mc := memcache.NewDriver(ctx, req.Logger) if err := d.AddChannel(req.Form("cid"), req.Form("sid"), req.Form("name"), req.Form("iepg_station_id")); err != nil { lib.InternalError(res, req, err) } else { mc.Delete(MC_KEY_CHANNELS) id := fmt.Sprintf("%s/%s", req.Form("cid"), req.Form("sid")) app.Api.Created(res, req, id) } }, )) app.Api.Delete("/channels/:cid/:sid.json", lib.Admin.Required( func(res *wcg.Response, req *wcg.Request) { ctx := gae.NewContext(req) d := NewTvChannelDriver(app.Key, ctx, req.Logger) mc := memcache.NewDriver(ctx, req.Logger) if err := d.DelChannel(req.Param("cid"), req.Param("sid")); err != nil { lib.InternalError(res, req, err) } else { mc.Delete(MC_KEY_CHANNELS) app.Api.Ok(res, req) } }, )) }
func TestListChannelApi(t *testing.T) { test.RunTestServer(func(ts *test.TestServer) { assert := test.NewAssert(t) p := app.Api.Path("/channels/") err := util.WaitFor(func() bool { var got []tv.TvChannel req := ts.Get(p) res := req.RouteTo(app.Routes()) assert.HttpStatus(200, res) res.Json(&got) return len(got) > 0 }, util.DefaultWaitForTimeout) assert.Nil(err, "GET %s should return the list of TvChannels within a timeout window.", p) // Confirm cache invalidation mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil)) assert.Ok(mc.Exists(MC_KEY_CHANNELS), "GET %s should create the cache", p) }) }
func TestDelKeywordApi(t *testing.T) { test.RunTestServer(func(ts *test.TestServer) { assert := test.NewAssert(t) // prepare d := NewCrawlerConfigDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil)) d.Add(&tv.CrawlerConfig{ Keyword: "モーニング娘。'15", Category: "モーニング娘", Scope: tv.FEED_SCOPE_ALL, }) d.Add(&tv.CrawlerConfig{ Keyword: "SPEED", Category: "SPEED", Scope: tv.FEED_SCOPE_ALL, }) err := util.WaitFor(func() bool { c, _ := d.NewQuery().Count() return c == 2 }, util.DefaultWaitForTimeout) assert.Nil(err, "Confirm CrawlerConfig entities has been stored within a timeout window.") p := app.Api.Path("/keywords/モーニング娘。'15.json") req := ts.Delete(p) lib.SetApiTokenForTest(req, lib.Admin) res := req.RouteTo(app.Routes()) assert.HttpStatus(200, res) err = util.WaitFor(func() bool { c, _ := d.NewQuery().Count() return c == 1 }, util.DefaultWaitForTimeout) assert.Nil(err, "DELETE %s Confirm CrawlerConfig entities has been deleted via API within a timeout window.", p) // Confirm cache invalidation mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil)) assert.Ok(!mc.Exists(MC_KEY_KEYWORDS), "DELETE %s should invalidate the cache", p) }) }
func TestListKeywordApi(t *testing.T) { test.RunTestServer(func(ts *test.TestServer) { assert := test.NewAssert(t) // prepare d := NewCrawlerConfigDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil)) d.Add(&tv.CrawlerConfig{ Keyword: "キーワード1", Category: "カテゴリー1", Scope: 1, }) d.Add(&tv.CrawlerConfig{ Keyword: "キーワード2", Category: "カテゴリー2", Scope: 1, }) err := util.WaitFor(func() bool { c, _ := d.NewQuery().Count() return c == 2 }, util.DefaultWaitForTimeout) assert.Nil(err, "Confirm CrawlerConfig entities has been stored within a timeout window.") var got []*tv.CrawlerConfig p := app.Api.Path("/keywords/") + "?force=true" req := ts.Get(p) res := req.RouteTo(app.Routes()) assert.HttpStatus(200, res) res.Json(&got) assert.EqStr("キーワード2", got[0].Keyword, "GET %s should return the list ordered by creation time.", p) assert.EqStr("キーワード1", got[1].Keyword, "GET %s should return the list ordered by creation time.", p) // Confirm cache invalidation mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil)) assert.Ok(mc.Exists(MC_KEY_KEYWORDS), "GET %s should create the cache", p) }) }
func listTvChannels(res *wcg.Response, req *wcg.Request) ([]*tv.TvChannel, error) { var list []*tv.TvChannel app := lib.GetCurrentApp(req) ctx := gae.NewContext(req) d := NewTvChannelDriver(app.Key, ctx, req.Logger) mc := memcache.NewDriver(ctx, req.Logger) err := mc.CachedObject(MC_KEY_CHANNELS, &list, func() (interface{}, error) { return d.AllAsList() }, req.Query("force") == "1") if err != nil { return nil, err } else { if len(list) == 0 { req.Logger.Warn("No channel is defined. Reset the configuraiton.") d.AddChannelList(defaultChannels) mc.Delete(MC_KEY_CHANNELS) mc.Set(MC_KEY_CHANNELS, defaultChannels) res.WriteJson(defaultChannels) return defaultChannels, nil } else { return list, nil } } }
func init() { app.Cron.Get( "Crawl keyword IEPGs", "every 3 hours", "/keywords/crawl/", lib.Admin.Required( func(res *wcg.Response, req *wcg.Request) { var channels []*tv.TvChannel var cfglist []*tv.CrawlerConfig ctx := gae.NewContext(req) d := NewCrawlerConfigDriver(app.Key, ctx, req.Logger) mc := memcache.NewDriver(ctx, req.Logger) err := mc.CachedObject(MC_KEY_CHANNELS, &channels, func() (interface{}, error) { return NewTvChannelDriver(app.Key, ctx, req.Logger).AllAsList() }, false) if err != nil { app.Api.InternalError(res, req, err) return } req.Logger.Info("Retrieving all cralwer configurations...") q := d.NewQuery() q.Order("-CreatedAt") if _, err := q.GetAll(&cfglist); err != nil { app.Api.InternalError(res, req, err) return } else if len(cfglist) == 0 { req.Logger.Info("No cralwer configuration is found.") res.WriteJson(map[string]int{ "updates": 0, }) return } else { req.Logger.Info("Found %d configs, start crawling", len(cfglist)) iepglist := make([]*tv.IEpg, 0) for i := range cfglist { if list, err := getIEpgListFromCrawlerConfig(res, req, cfglist[i], channels); err != nil { req.Logger.Warn("Failed to crawl %v: %v", cfglist[i], err) } else { iepglist = append(iepglist, list...) } } // TODO: Sync bulk update d := NewRecordCacheDriver(app.Key, ctx, req.Logger) if keys, err := d.IEpg.BulkUpdate(iepglist); err != nil { app.Api.InternalError(res, req, err) return } else { req.Logger.Debug("Updated %d iepg entries...", len(keys)) // invalidate cache after 10 seconds time.Sleep(10 * time.Second) d.GetRecords(true) res.WriteJson(map[string]int{ "updates": len(iepglist), }) return } } }, )) }
func init() { app.Api.Get("/keywords/", func(res *wcg.Response, req *wcg.Request) { var list []*tv.CrawlerConfig ctx := gae.NewContext(req) mc := memcache.NewDriver(ctx, req.Logger) err := mc.CachedObject(MC_KEY_KEYWORDS, &list, func() (interface{}, error) { var list []*tv.CrawlerConfig d := NewCrawlerConfigDriver(app.Key, ctx, req.Logger) q := d.NewQuery().Order("-CreatedAt") _, err := q.GetAll(&list) if list == nil { return make([]*tv.CrawlerConfig, 0), nil } else { return list, err } }, req.Query("force") == "1") if err != nil { lib.InternalError(res, req, err) } else { res.WriteJson(list) } }, ) app.Api.Get("/keywords/preview/:keyword.json", lib.Family.Required( func(res *wcg.Response, req *wcg.Request) { var channels []*tv.TvChannel ctx := gae.NewContext(req) mc := memcache.NewDriver(ctx, req.Logger) err := mc.CachedObject(MC_KEY_CHANNELS, &channels, func() (interface{}, error) { return NewTvChannelDriver(app.Key, ctx, req.Logger).AllAsList() }, false) keyword := req.Param("keyword") scope, _ := strconv.Atoi(req.Query("scope")) list, err := getIEpgListFromCrawlerConfig(res, req, &tv.CrawlerConfig{ Keyword: keyword, Scope: scope, Category: "dummy", }, channels) if err != nil { lib.InternalError(res, req, err) } else { res.WriteJson(map[string]interface{}{ "samples": list, "total": len(list), }) } }, )) app.Api.Post("/keywords/", lib.Family.Required( func(res *wcg.Response, req *wcg.Request) { ctx := gae.NewContext(req) mc := memcache.NewDriver(ctx, req.Logger) d := NewCrawlerConfigDriver(app.Key, ctx, req.Logger) scope, _ := strconv.Atoi(req.Form("scope")) cfg := &tv.CrawlerConfig{ Keyword: req.Form("keyword"), Category: req.Form("category"), Scope: scope, } if err := d.Add(cfg); err != nil { app.Api.InternalError(res, req, err) return } else { mc.Delete(MC_KEY_KEYWORDS) app.Api.Created(res, req, req.Form("keyword")) return } }, )) app.Api.Delete("/keywords/:keyword.json", lib.Family.Required( func(res *wcg.Response, req *wcg.Request) { ctx := gae.NewContext(req) d := NewCrawlerConfigDriver(app.Key, ctx, req.Logger) mc := memcache.NewDriver(ctx, req.Logger) keyword := req.Param("keyword") if err := d.Delete(keyword); err != nil { lib.InternalError(res, req, err) return } else { mc.Delete(MC_KEY_KEYWORDS) app.Api.Ok(res, req) return } }, )) }
func (appCtx *AppContext) NewMemcacheDriver() *memcache.Driver { return memcache.NewDriver(appCtx.Context, appCtx.Logger) }