func TestDelChannelApi(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := test.NewAssert(t)
// prepare
d := NewTvChannelDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
ent1 := &tv.TvChannel{"c1", "s1", "foo", "bar"}
ent2 := &tv.TvChannel{"c2", "s2", "hoge", "piyo"}
d.Put(d.NewKey(ent1.Key(), 0, nil), ent1)
d.Put(d.NewKey(ent2.Key(), 0, nil), ent2)
err := util.WaitFor(func() bool {
c, _ := d.NewQuery().Count()
return c == 2
}, util.DefaultWaitForTimeout)
assert.Nil(err, "Confirm TvChannel entities has been stored within a timeout window.")
p := app.Api.Path("/channels/c1/s1.json")
req := ts.Delete(p)
lib.SetApiTokenForTest(req, lib.Admin)
res := req.RouteTo(app.Routes())
assert.HttpStatus(200, res)
err = util.WaitFor(func() bool {
c, _ := d.NewQuery().Count()
return c == 1
}, util.DefaultWaitForTimeout)
assert.Nil(err, "DELETE %s Confirm TvChannel entities has been deleted via API within a timeout window.", p)
// Confirm cache invalidation
mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
assert.Ok(!mc.Exists(MC_KEY_CHANNELS), "DELETE %s should invalidate the cache", p)
})
}
func TestAddKeywordApi(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := test.NewAssert(t)
p := app.Api.Path("/keywords/")
req := ts.PostForm(p, url.Values{
"keyword": []string{"モーニング娘。'15"},
"category": []string{"モーニング娘。"},
"scope": []string{"1"},
})
lib.SetApiTokenForTest(req, lib.Admin)
var got map[string]interface{}
res := req.RouteTo(app.Routes())
res.Json(&got)
assert.HttpStatus(201, res)
assert.EqStr(
"http://localhost:8080/api/pt/keywords/モーニング娘。'15.json",
got["location"].(string),
"POST %s location",
p,
)
// Confirm cache invalidation
mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
assert.Ok(!mc.Exists(MC_KEY_KEYWORDS), "POST %s should invalidate the cache", p)
})
}
func TestAddChannelApi(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := test.NewAssert(t)
p := app.Api.Path("/channels/")
req := ts.PostForm(p, url.Values{
"cid": []string{"c1"},
"sid": []string{"s1"},
"name": []string{"foo"},
"iepg_station_id": []string{"bar"},
})
lib.SetApiTokenForTest(req, lib.Admin)
var got map[string]interface{}
res := req.RouteTo(app.Routes())
res.Json(&got)
assert.HttpStatus(201, res)
assert.EqStr(
"http://localhost:8080/api/pt/channels/c1/s1.json",
got["location"].(string),
"POST %s location",
p,
)
// Confirm cache invalidation
mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
assert.Ok(!mc.Exists(MC_KEY_CHANNELS), "POST %s should invalidate the cache", p)
})
}
func NewRecordCacheDriver(ns string, ctx appengine.Context, logger wcg.Logger) *RecordCacheDriver {
return &RecordCacheDriver{
memcache.NewDriver(ctx, logger),
NewTvRecordDriver(ns, ctx, logger),
NewIEpgDriver(ns, ctx, logger),
util.Today().UTC().Add((9 + 3) * time.Hour), // Cache key cycle = 03:00am in JST for the less impact on recording.
}
}
func init() {
app.Api.Get("/channels/",
func(res *wcg.Response, req *wcg.Request) {
if list, err := listTvChannels(res, req); err != nil {
app.Api.InternalError(res, req, err)
} else {
res.WriteJson(list)
}
},
)
app.Api.Post("/channels/", lib.Admin.Required(
func(res *wcg.Response, req *wcg.Request) {
ctx := gae.NewContext(req)
d := NewTvChannelDriver(app.Key, ctx, req.Logger)
mc := memcache.NewDriver(ctx, req.Logger)
if err := d.AddChannel(req.Form("cid"), req.Form("sid"), req.Form("name"), req.Form("iepg_station_id")); err != nil {
lib.InternalError(res, req, err)
} else {
mc.Delete(MC_KEY_CHANNELS)
id := fmt.Sprintf("%s/%s", req.Form("cid"), req.Form("sid"))
app.Api.Created(res, req, id)
}
},
))
app.Api.Delete("/channels/:cid/:sid.json", lib.Admin.Required(
func(res *wcg.Response, req *wcg.Request) {
ctx := gae.NewContext(req)
d := NewTvChannelDriver(app.Key, ctx, req.Logger)
mc := memcache.NewDriver(ctx, req.Logger)
if err := d.DelChannel(req.Param("cid"), req.Param("sid")); err != nil {
lib.InternalError(res, req, err)
} else {
mc.Delete(MC_KEY_CHANNELS)
app.Api.Ok(res, req)
}
},
))
}
func TestListChannelApi(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := test.NewAssert(t)
p := app.Api.Path("/channels/")
err := util.WaitFor(func() bool {
var got []tv.TvChannel
req := ts.Get(p)
res := req.RouteTo(app.Routes())
assert.HttpStatus(200, res)
res.Json(&got)
return len(got) > 0
}, util.DefaultWaitForTimeout)
assert.Nil(err, "GET %s should return the list of TvChannels within a timeout window.", p)
// Confirm cache invalidation
mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
assert.Ok(mc.Exists(MC_KEY_CHANNELS), "GET %s should create the cache", p)
})
}
func TestDelKeywordApi(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := test.NewAssert(t)
// prepare
d := NewCrawlerConfigDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
d.Add(&tv.CrawlerConfig{
Keyword: "モーニング娘。'15",
Category: "モーニング娘",
Scope: tv.FEED_SCOPE_ALL,
})
d.Add(&tv.CrawlerConfig{
Keyword: "SPEED",
Category: "SPEED",
Scope: tv.FEED_SCOPE_ALL,
})
err := util.WaitFor(func() bool {
c, _ := d.NewQuery().Count()
return c == 2
}, util.DefaultWaitForTimeout)
assert.Nil(err, "Confirm CrawlerConfig entities has been stored within a timeout window.")
p := app.Api.Path("/keywords/モーニング娘。'15.json")
req := ts.Delete(p)
lib.SetApiTokenForTest(req, lib.Admin)
res := req.RouteTo(app.Routes())
assert.HttpStatus(200, res)
err = util.WaitFor(func() bool {
c, _ := d.NewQuery().Count()
return c == 1
}, util.DefaultWaitForTimeout)
assert.Nil(err, "DELETE %s Confirm CrawlerConfig entities has been deleted via API within a timeout window.", p)
// Confirm cache invalidation
mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
assert.Ok(!mc.Exists(MC_KEY_KEYWORDS), "DELETE %s should invalidate the cache", p)
})
}
func TestListKeywordApi(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := test.NewAssert(t)
// prepare
d := NewCrawlerConfigDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
d.Add(&tv.CrawlerConfig{
Keyword: "キーワード1",
Category: "カテゴリー1",
Scope: 1,
})
d.Add(&tv.CrawlerConfig{
Keyword: "キーワード2",
Category: "カテゴリー2",
Scope: 1,
})
err := util.WaitFor(func() bool {
c, _ := d.NewQuery().Count()
return c == 2
}, util.DefaultWaitForTimeout)
assert.Nil(err, "Confirm CrawlerConfig entities has been stored within a timeout window.")
var got []*tv.CrawlerConfig
p := app.Api.Path("/keywords/") + "?force=true"
req := ts.Get(p)
res := req.RouteTo(app.Routes())
assert.HttpStatus(200, res)
res.Json(&got)
assert.EqStr("キーワード2", got[0].Keyword, "GET %s should return the list ordered by creation time.", p)
assert.EqStr("キーワード1", got[1].Keyword, "GET %s should return the list ordered by creation time.", p)
// Confirm cache invalidation
mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
assert.Ok(mc.Exists(MC_KEY_KEYWORDS), "GET %s should create the cache", p)
})
}
func listTvChannels(res *wcg.Response, req *wcg.Request) ([]*tv.TvChannel, error) {
var list []*tv.TvChannel
app := lib.GetCurrentApp(req)
ctx := gae.NewContext(req)
d := NewTvChannelDriver(app.Key, ctx, req.Logger)
mc := memcache.NewDriver(ctx, req.Logger)
err := mc.CachedObject(MC_KEY_CHANNELS, &list, func() (interface{}, error) {
return d.AllAsList()
}, req.Query("force") == "1")
if err != nil {
return nil, err
} else {
if len(list) == 0 {
req.Logger.Warn("No channel is defined. Reset the configuraiton.")
d.AddChannelList(defaultChannels)
mc.Delete(MC_KEY_CHANNELS)
mc.Set(MC_KEY_CHANNELS, defaultChannels)
res.WriteJson(defaultChannels)
return defaultChannels, nil
} else {
return list, nil
}
}
}
func init() {
app.Cron.Get(
"Crawl keyword IEPGs",
"every 3 hours",
"/keywords/crawl/",
lib.Admin.Required(
func(res *wcg.Response, req *wcg.Request) {
var channels []*tv.TvChannel
var cfglist []*tv.CrawlerConfig
ctx := gae.NewContext(req)
d := NewCrawlerConfigDriver(app.Key, ctx, req.Logger)
mc := memcache.NewDriver(ctx, req.Logger)
err := mc.CachedObject(MC_KEY_CHANNELS, &channels, func() (interface{}, error) {
return NewTvChannelDriver(app.Key, ctx, req.Logger).AllAsList()
}, false)
if err != nil {
app.Api.InternalError(res, req, err)
return
}
req.Logger.Info("Retrieving all cralwer configurations...")
q := d.NewQuery()
q.Order("-CreatedAt")
if _, err := q.GetAll(&cfglist); err != nil {
app.Api.InternalError(res, req, err)
return
} else if len(cfglist) == 0 {
req.Logger.Info("No cralwer configuration is found.")
res.WriteJson(map[string]int{
"updates": 0,
})
return
} else {
req.Logger.Info("Found %d configs, start crawling", len(cfglist))
iepglist := make([]*tv.IEpg, 0)
for i := range cfglist {
if list, err := getIEpgListFromCrawlerConfig(res, req, cfglist[i], channels); err != nil {
req.Logger.Warn("Failed to crawl %v: %v", cfglist[i], err)
} else {
iepglist = append(iepglist, list...)
}
}
// TODO: Sync bulk update
d := NewRecordCacheDriver(app.Key, ctx, req.Logger)
if keys, err := d.IEpg.BulkUpdate(iepglist); err != nil {
app.Api.InternalError(res, req, err)
return
} else {
req.Logger.Debug("Updated %d iepg entries...", len(keys))
// invalidate cache after 10 seconds
time.Sleep(10 * time.Second)
d.GetRecords(true)
res.WriteJson(map[string]int{
"updates": len(iepglist),
})
return
}
}
},
))
}
func init() {
app.Api.Get("/keywords/",
func(res *wcg.Response, req *wcg.Request) {
var list []*tv.CrawlerConfig
ctx := gae.NewContext(req)
mc := memcache.NewDriver(ctx, req.Logger)
err := mc.CachedObject(MC_KEY_KEYWORDS, &list, func() (interface{}, error) {
var list []*tv.CrawlerConfig
d := NewCrawlerConfigDriver(app.Key, ctx, req.Logger)
q := d.NewQuery().Order("-CreatedAt")
_, err := q.GetAll(&list)
if list == nil {
return make([]*tv.CrawlerConfig, 0), nil
} else {
return list, err
}
}, req.Query("force") == "1")
if err != nil {
lib.InternalError(res, req, err)
} else {
res.WriteJson(list)
}
},
)
app.Api.Get("/keywords/preview/:keyword.json", lib.Family.Required(
func(res *wcg.Response, req *wcg.Request) {
var channels []*tv.TvChannel
ctx := gae.NewContext(req)
mc := memcache.NewDriver(ctx, req.Logger)
err := mc.CachedObject(MC_KEY_CHANNELS, &channels, func() (interface{}, error) {
return NewTvChannelDriver(app.Key, ctx, req.Logger).AllAsList()
}, false)
keyword := req.Param("keyword")
scope, _ := strconv.Atoi(req.Query("scope"))
list, err := getIEpgListFromCrawlerConfig(res, req, &tv.CrawlerConfig{
Keyword: keyword,
Scope: scope,
Category: "dummy",
}, channels)
if err != nil {
lib.InternalError(res, req, err)
} else {
res.WriteJson(map[string]interface{}{
"samples": list,
"total": len(list),
})
}
},
))
app.Api.Post("/keywords/", lib.Family.Required(
func(res *wcg.Response, req *wcg.Request) {
ctx := gae.NewContext(req)
mc := memcache.NewDriver(ctx, req.Logger)
d := NewCrawlerConfigDriver(app.Key, ctx, req.Logger)
scope, _ := strconv.Atoi(req.Form("scope"))
cfg := &tv.CrawlerConfig{
Keyword: req.Form("keyword"),
Category: req.Form("category"),
Scope: scope,
}
if err := d.Add(cfg); err != nil {
app.Api.InternalError(res, req, err)
return
} else {
mc.Delete(MC_KEY_KEYWORDS)
app.Api.Created(res, req, req.Form("keyword"))
return
}
},
))
app.Api.Delete("/keywords/:keyword.json", lib.Family.Required(
func(res *wcg.Response, req *wcg.Request) {
ctx := gae.NewContext(req)
d := NewCrawlerConfigDriver(app.Key, ctx, req.Logger)
mc := memcache.NewDriver(ctx, req.Logger)
keyword := req.Param("keyword")
if err := d.Delete(keyword); err != nil {
lib.InternalError(res, req, err)
return
} else {
mc.Delete(MC_KEY_KEYWORDS)
app.Api.Ok(res, req)
return
}
},
))
}
func (appCtx *AppContext) NewMemcacheDriver() *memcache.Driver {
return memcache.NewDriver(appCtx.Context, appCtx.Logger)
}