func main() {
app := cli.NewApp()
app.Name = "speedland-agent"
app.Usage = "A agent command for speedland.net"
app.Version = "1.0.0"
app.Flags = []cli.Flag{
cli.StringFlag{
"config, c",
"wcg.ini",
"configuration file",
"WCG_INI_FILE",
},
}
app.Before = func(c *cli.Context) error {
wcg.ConfigureProcess(c.String("config"))
// normalize path
lib.Config.Endpoint.Path = ""
wcg.NewLogger(nil).Info("Used configurn file: %q", c.String("config"))
wcg.NewLogger(nil).Info("Target Endpoint: %q", lib.Config.Endpoint)
wcg.NewLogger(nil).Debug("Token: %q", lib.Config.Token)
return nil
}
app.Commands = commands.AllCommands()
app.Run(os.Args)
wcg.WaitLogs()
}
func TestDelChannelApi(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := test.NewAssert(t)
// prepare
d := NewTvChannelDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
ent1 := &tv.TvChannel{"c1", "s1", "foo", "bar"}
ent2 := &tv.TvChannel{"c2", "s2", "hoge", "piyo"}
d.Put(d.NewKey(ent1.Key(), 0, nil), ent1)
d.Put(d.NewKey(ent2.Key(), 0, nil), ent2)
err := util.WaitFor(func() bool {
c, _ := d.NewQuery().Count()
return c == 2
}, util.DefaultWaitForTimeout)
assert.Nil(err, "Confirm TvChannel entities has been stored within a timeout window.")
p := app.Api.Path("/channels/c1/s1.json")
req := ts.Delete(p)
lib.SetApiTokenForTest(req, lib.Admin)
res := req.RouteTo(app.Routes())
assert.HttpStatus(200, res)
err = util.WaitFor(func() bool {
c, _ := d.NewQuery().Count()
return c == 1
}, util.DefaultWaitForTimeout)
assert.Nil(err, "DELETE %s Confirm TvChannel entities has been deleted via API within a timeout window.", p)
// Confirm cache invalidation
mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
assert.Ok(!mc.Exists(MC_KEY_CHANNELS), "DELETE %s should invalidate the cache", p)
})
}
func TestAddKeywordApi(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := test.NewAssert(t)
p := app.Api.Path("/keywords/")
req := ts.PostForm(p, url.Values{
"keyword": []string{"モーニング娘。'15"},
"category": []string{"モーニング娘。"},
"scope": []string{"1"},
})
lib.SetApiTokenForTest(req, lib.Admin)
var got map[string]interface{}
res := req.RouteTo(app.Routes())
res.Json(&got)
assert.HttpStatus(201, res)
assert.EqStr(
"http://localhost:8080/api/pt/keywords/モーニング娘。'15.json",
got["location"].(string),
"POST %s location",
p,
)
// Confirm cache invalidation
mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
assert.Ok(!mc.Exists(MC_KEY_KEYWORDS), "POST %s should invalidate the cache", p)
})
}
func TestChannel_AddAndDelCrawlerConfig(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := wcg.NewAssert(t)
// prepare
d := NewCrawlerConfigDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
d.Add(&tv.CrawlerConfig{
Keyword: "キーワード1",
Category: "カテゴリー1",
Scope: 1,
})
d.Add(&tv.CrawlerConfig{
Keyword: "キーワード2",
Category: "カテゴリー2",
Scope: 1,
})
err := util.WaitFor(func() bool {
c, _ := d.NewQuery().Count()
return c == 2
}, util.DefaultWaitForTimeout)
assert.Nil(err, "Confirm CrawlerConfig entities has been stored within a timeout window.")
d.Delete("キーワード1")
err = util.WaitFor(func() bool {
c, _ := d.NewQuery().Count()
return c == 1
}, util.DefaultWaitForTimeout)
assert.Nil(err, "Confirm CrawlerConfig entities has been deleted within a timeout window.")
err = d.Delete("Not Exists")
assert.Nil(err, "Delete should not return even trying to delete the unexising keyword.")
})
}
func TestAddChannelApi(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := test.NewAssert(t)
p := app.Api.Path("/channels/")
req := ts.PostForm(p, url.Values{
"cid": []string{"c1"},
"sid": []string{"s1"},
"name": []string{"foo"},
"iepg_station_id": []string{"bar"},
})
lib.SetApiTokenForTest(req, lib.Admin)
var got map[string]interface{}
res := req.RouteTo(app.Routes())
res.Json(&got)
assert.HttpStatus(201, res)
assert.EqStr(
"http://localhost:8080/api/pt/channels/c1/s1.json",
got["location"].(string),
"POST %s location",
p,
)
// Confirm cache invalidation
mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
assert.Ok(!mc.Exists(MC_KEY_CHANNELS), "POST %s should invalidate the cache", p)
})
}
func NewAppContextFromTestServer(app *lib.App, ts *test.TestServer) *lib.AppContext {
return &lib.AppContext{
app,
ts.Context,
wcg.NewLogger(nil),
}
}
func TestGetOverlaps(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
var found []*tv.TvRecord
const timeForm = "2006/01/02 15:04:05"
assert := wcg.NewAssert(t)
d := NewTvRecordDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
base := genTestRecord()
base.StartAt, _ = time.Parse(timeForm, "2014/01/01 12:00:00")
base.EndAt, _ = time.Parse(timeForm, "2014/01/01 13:00:00")
d.Save(base)
err := util.WaitFor(func() bool {
c, _ := d.NewQuery().Count()
return c == 1
}, util.DefaultWaitForTimeout)
assert.Nil(err, "Confirm TvRecord entities has been stored within a timeout window.")
// case1: no overlaps
r := genTestRecord()
r.StartAt, _ = time.Parse(timeForm, "2014/01/01 10:50:00")
r.EndAt, _ = time.Parse(timeForm, "2014/01/01 11:10:00")
found, err = d.GetOverlaps(r)
assert.Nil(err, "TvRecordDriver#GetOverlaps should not return an error (case1)")
assert.EqInt(0, len(found), "TvRecordDriver#GetOverlaps should return empty array (case1)")
// case 2: base.end < r.start
r = genTestRecord()
r.StartAt, _ = time.Parse(timeForm, "2014/01/01 13:50:00")
r.EndAt, _ = time.Parse(timeForm, "2014/01/01 14:10:00")
found, err = d.GetOverlaps(r)
assert.Nil(err, "TvRecordDriver#GetOverlaps should not return an error (case2)")
assert.EqInt(0, len(found), "TvRecordDriver#GetOverlaps should return empty array (case2)")
// case 3: base.start < r.start < base.end
r = genTestRecord()
r.StartAt, _ = time.Parse(timeForm, "2014/01/01 11:50:00")
r.EndAt, _ = time.Parse(timeForm, "2014/01/01 12:10:00")
found, err = d.GetOverlaps(r)
assert.Nil(err, "TvRecordDriver#GetOverlaps should not return an error (case3)")
assert.EqInt(1, len(found), "TvRecordDriver#GetOverlaps should return an array with an entity (case3)")
// case 4: r.start < base.end < r.end
r = genTestRecord()
r.StartAt, _ = time.Parse(timeForm, "2014/01/01 12:50:00")
r.EndAt, _ = time.Parse(timeForm, "2014/01/01 13:10:00")
found, err = d.GetOverlaps(r)
assert.Nil(err, "TvRecordDriver#GetOverlaps should not return an error (case4)")
assert.EqInt(1, len(found), "TvRecordDriver#GetOverlaps should return an array with an entity (case4)")
// case 5: r.start < base.start < base.end < r.end
r = genTestRecord()
r.StartAt, _ = time.Parse(timeForm, "2014/01/01 12:10:00")
r.EndAt, _ = time.Parse(timeForm, "2014/01/01 12:50:00")
found, err = d.GetOverlaps(r)
assert.Nil(err, "TvRecordDriver#GetOverlaps should not return an error (case4)")
assert.EqInt(1, len(found), "TvRecordDriver#GetOverlaps should return an array with an entity (case4)")
})
}
func FixtureFromMap(ctx appengine.Context, arr []map[string]interface{}) error {
logger := wcg.NewLogger(nil)
for _, v := range arr {
if err := loadJsonToDatastore(ctx, nil, v, logger); err != nil {
return err
}
}
return nil
}
func Run() {
go ping.Run()
go recorder.Run()
go statsd.Run()
// block the main until receiving the signal
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGTERM, syscall.SIGINT)
s := <-c
wcg.NewLogger(nil).Info("Exitting the signal with %s", s)
// TODO: delegate the notification to each job
}
func TestDeleteRecordApi(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := test.NewAssert(t)
d := NewRecordCacheDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
// create an empty cache
var got []tv.TvRecord
p := app.Api.Path("/records/")
req := ts.Get(p)
res := req.RouteTo(app.Routes())
assert.HttpStatus(200, res)
res.Json(&got)
assert.EqInt(0, len(got), "GET %s should return 2 TvRecord entities.")
assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "GET %s should create RecordCacheTypeTvRecord cache", p)
assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "GET %s should create RecordCacheTypeIEpg cache", p)
// prepare
in_window := genTestRecord()
in_window.StartAt = d.today.Add(1 * time.Hour)
in_window.EndAt = d.today.Add(2 * time.Hour)
d.TvRecord.Save(in_window)
future := genTestRecord()
future.StartAt = d.today.Add(RECORD_TIME_WINDOW + 1*time.Hour)
future.EndAt = future.StartAt.Add(1 * time.Hour)
d.TvRecord.Save(future)
err := util.WaitFor(func() bool {
records, _ := d.TvRecord.NewQuery().Count()
return records == 2
}, util.DefaultWaitForTimeout)
assert.Nil(err, "Confirm TvRecord entities has been stored within a timeout window.")
// case 1: future
p = fmt.Sprintf("%s%s.json", app.Api.Path("/records/"), future.Key())
req = ts.Delete(p)
lib.SetApiTokenForTest(req, lib.Admin)
res = req.RouteTo(app.Routes())
assert.HttpStatus(200, res)
assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "POST %s should not invalidate RecordCacheTypeTvRecord cache (future record)", p)
assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "GET %s should not invalidate RecordCacheTypeIEpg cache (future record)", p)
// case 2: in recording window
p = fmt.Sprintf("%s%s.json", app.Api.Path("/records/"), in_window.Key())
req = ts.Delete(p)
lib.SetApiTokenForTest(req, lib.Admin)
res = req.RouteTo(app.Routes())
assert.HttpStatus(200, res)
assert.Ok(!d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "POST %s should invalidate RecordCacheTypeTvRecord cache (in_window record)", p)
assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "GET %s should not invalidate RecordCacheTypeIEpg cache (in_window record)", p)
})
}
func TestDelKeywordApi(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := test.NewAssert(t)
// prepare
d := NewCrawlerConfigDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
d.Add(&tv.CrawlerConfig{
Keyword: "モーニング娘。'15",
Category: "モーニング娘",
Scope: tv.FEED_SCOPE_ALL,
})
d.Add(&tv.CrawlerConfig{
Keyword: "SPEED",
Category: "SPEED",
Scope: tv.FEED_SCOPE_ALL,
})
err := util.WaitFor(func() bool {
c, _ := d.NewQuery().Count()
return c == 2
}, util.DefaultWaitForTimeout)
assert.Nil(err, "Confirm CrawlerConfig entities has been stored within a timeout window.")
p := app.Api.Path("/keywords/モーニング娘。'15.json")
req := ts.Delete(p)
lib.SetApiTokenForTest(req, lib.Admin)
res := req.RouteTo(app.Routes())
assert.HttpStatus(200, res)
err = util.WaitFor(func() bool {
c, _ := d.NewQuery().Count()
return c == 1
}, util.DefaultWaitForTimeout)
assert.Nil(err, "DELETE %s Confirm CrawlerConfig entities has been deleted via API within a timeout window.", p)
// Confirm cache invalidation
mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
assert.Ok(!mc.Exists(MC_KEY_KEYWORDS), "DELETE %s should invalidate the cache", p)
})
}
func TestRecordCache_GetRecords_and_Invalidate(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := wcg.NewAssert(t)
d := NewRecordCacheDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
// prepare
in_window := genTestRecord()
in_window.StartAt = d.today.Add(1 * time.Hour)
in_window.EndAt = d.today.Add(2 * time.Hour)
d.TvRecord.Save(in_window)
future := genTestRecord()
future.StartAt = d.today.Add(RECORD_TIME_WINDOW + 1*time.Hour)
future.EndAt = future.StartAt.Add(1 * time.Hour)
d.TvRecord.Save(future)
past := genTestRecord()
past.StartAt = d.today.Add(-RECORD_TIME_WINDOW - 1*time.Hour)
past.EndAt = past.StartAt.Add(30 * time.Minute)
d.TvRecord.Save(past)
in_window_iepg := genTestIEpg()
in_window.StartAt = d.today.Add(4 * time.Hour)
in_window.EndAt = d.today.Add(5 * time.Hour)
d.IEpg.Save(in_window_iepg)
err := util.WaitFor(func() bool {
records, _ := d.TvRecord.NewQuery().Count()
iepgs, _ := d.IEpg.NewQuery().Count()
return records+iepgs == 4
}, util.DefaultWaitForTimeout)
assert.Nil(err, "Confirm TvRecord/IEpg entities has been stored within a timeout window.")
list, err := d.GetRecords(false)
assert.Nil(err, "RecordCache#GetRecords should not return an error")
assert.EqInt(2, len(list), "RecordCache#GetRecords should return 2 records in a time window")
// cache check
assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "RecordCache#GetRecords should create RecordCacheTypeTvRecord on memcache.")
assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "RecordCache#GetRecords should create RecordCacheTypeIEpg on memcache.")
// invalidate the keys
assert.Ok(!d.Invalidate(future), "RecordCache#Invalidate should not return true if the passed record is out of window (future)")
assert.Ok(!d.Invalidate(past), "RecordCache#Invalidate should not return true if the passed record is out of window (future)")
assert.Ok(d.Invalidate(in_window), "RecordCache#Invalidate should return true if the passed record is in window")
assert.Ok(!d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "RecordCache#GetRecords should invalidate RecordCacheTypeTvRecord on memcache.")
assert.Ok(d.Invalidate(in_window_iepg), "RecordCache#Invalidate should return true if the passed record is in window (iepg)")
assert.Ok(!d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "RecordCache#GetRecords should invalidate RecordCacheTypeIEpg on memcache.")
})
}
func TestListKeywordApi(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := test.NewAssert(t)
// prepare
d := NewCrawlerConfigDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
d.Add(&tv.CrawlerConfig{
Keyword: "キーワード1",
Category: "カテゴリー1",
Scope: 1,
})
d.Add(&tv.CrawlerConfig{
Keyword: "キーワード2",
Category: "カテゴリー2",
Scope: 1,
})
err := util.WaitFor(func() bool {
c, _ := d.NewQuery().Count()
return c == 2
}, util.DefaultWaitForTimeout)
assert.Nil(err, "Confirm CrawlerConfig entities has been stored within a timeout window.")
var got []*tv.CrawlerConfig
p := app.Api.Path("/keywords/") + "?force=true"
req := ts.Get(p)
res := req.RouteTo(app.Routes())
assert.HttpStatus(200, res)
res.Json(&got)
assert.EqStr("キーワード2", got[0].Keyword, "GET %s should return the list ordered by creation time.", p)
assert.EqStr("キーワード1", got[1].Keyword, "GET %s should return the list ordered by creation time.", p)
// Confirm cache invalidation
mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
assert.Ok(mc.Exists(MC_KEY_KEYWORDS), "GET %s should create the cache", p)
})
}
func TestCreateRecordApi(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := test.NewAssert(t)
d := NewRecordCacheDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
// create an empty cache
var got []tv.TvRecord
p := app.Api.Path("/records/")
req := ts.Get(p)
res := req.RouteTo(app.Routes())
assert.HttpStatus(200, res)
res.Json(&got)
assert.EqInt(0, len(got), "GET %s should return 2 TvRecord entities.")
assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "GET %s should create RecordCacheTypeTvRecord cache", p)
assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "GET %s should create RecordCacheTypeIEpg cache", p)
now := time.Now()
// case 1: future
req = ts.PostForm(p, url.Values{
"title": []string{"Title"},
"category": []string{"Category"},
"cid": []string{"27"},
"sid": []string{"hd"},
"start_at": []string{util.FormatDateTime(now.Add(RECORD_TIME_WINDOW + 24*time.Hour))},
"end_at": []string{util.FormatDateTime(now.Add(RECORD_TIME_WINDOW + 25*time.Hour))},
})
lib.SetApiTokenForTest(req, lib.Admin)
res = req.RouteTo(app.Routes())
assert.HttpStatus(201, res)
assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "POST %s should not invalidate RecordCacheTypeTvRecord cache (future record)", p)
assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "GET %s should not invalidate RecordCacheTypeIEpg cache (future record)", p)
// case 2: in recording window
req = ts.PostForm(p, url.Values{
"title": []string{"Title"},
"category": []string{"Category"},
"cid": []string{"27"},
"sid": []string{"hd"},
"start_at": []string{util.FormatDateTime(now)},
"end_at": []string{util.FormatDateTime(now.Add(1 * time.Hour))},
})
lib.SetApiTokenForTest(req, lib.Admin)
res = req.RouteTo(app.Routes())
assert.HttpStatus(201, res)
assert.Ok(!d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "POST %s should invalidate RecordCacheTypeTvRecord cache (in window)", p)
assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "GET %s should not invalidate RecordCacheTypeIEpg cache (in window)", p)
})
}
func DatastoreFixture(ctx appengine.Context, path string, bindings interface{}) error {
logger := wcg.NewLogger(nil)
logger.Debug("[Fixture] load from %s", path)
data, err := loadFile(path, bindings)
if err != nil {
return err
}
var arr []map[string]interface{}
if err = json.Unmarshal(data, &arr); err != nil {
return fmt.Errorf("Could not load the json file from %q - JSON Parse error: %v", path, err)
}
for _, v := range arr {
if err := loadJsonToDatastore(ctx, nil, v, logger); err != nil {
return err
}
}
return nil
}
func (*TweetStream) Run() {
logger := wcg.NewLogger(nil)
if tweetStreamConfig.ConsumerKey == "" {
logger.Error("[TweetStream] consumer_key is missing or empty.")
return
}
if tweetStreamConfig.ConsumerSecret == "" {
logger.Error("[TweetStream] consumer_secret is missing or empty.")
return
}
if tweetStreamConfig.Token == "" {
logger.Error("[TweetStream] oauth_token is missing or empty.")
return
}
if tweetStreamConfig.TokenSecret == "" {
logger.Error("[TweetStream] oauth_token_secret is missing or empty.")
return
}
for {
consumerKey := tweetStreamConfig.ConsumerKey
consumerSecret := tweetStreamConfig.ConsumerSecret
client := twitter.NewTwitterClient(consumerKey, consumerSecret, nil)
client.Token = &twitter.OAuthToken{
ConsumerKey: consumerKey,
ConsumerSecret: consumerSecret,
Token: tweetStreamConfig.Token,
TokenSecret: tweetStreamConfig.TokenSecret,
}
ts, err := client.Stream(&twitter.StreamingParams{
Track: "#morningmusume15",
})
if err != nil {
logger.Error("[TweetStream] Error building streaming connection: %v", err)
} else {
logger.Info("[TweetStream] Connected to the streaming server.")
ch := ts.GetStreamChannel()
for tweet := range ch {
logger.Debug("[TweetStream] %v", tweet)
}
logger.Info("[TweetStream] Disconnected from server.")
}
}
}
func TestListRecordApi(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := test.NewAssert(t)
d := NewRecordCacheDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
// prepare
in_window := genTestRecord()
in_window.StartAt = d.today.Add(1 * time.Hour)
in_window.EndAt = d.today.Add(2 * time.Hour)
d.TvRecord.Save(in_window)
future := genTestRecord()
future.StartAt = d.today.Add(RECORD_TIME_WINDOW + 1*time.Hour)
future.EndAt = future.StartAt.Add(1 * time.Hour)
d.TvRecord.Save(future)
past := genTestRecord()
past.StartAt = d.today.Add(-RECORD_TIME_WINDOW - 1*time.Hour)
past.EndAt = past.StartAt.Add(30 * time.Minute)
d.TvRecord.Save(past)
in_window_iepg := genTestIEpg()
in_window.StartAt = d.today.Add(4 * time.Hour)
in_window.EndAt = d.today.Add(5 * time.Hour)
d.IEpg.Save(in_window_iepg)
err := util.WaitFor(func() bool {
records, _ := d.TvRecord.NewQuery().Count()
iepgs, _ := d.IEpg.NewQuery().Count()
return records+iepgs == 4
}, util.DefaultWaitForTimeout)
assert.Nil(err, "Confirm TvRecord/IEpg entities has been stored within a timeout window.")
var got []tv.TvRecord
p := app.Api.Path("/records/")
req := ts.Get(p)
res := req.RouteTo(app.Routes())
assert.HttpStatus(200, res)
res.Json(&got)
assert.EqInt(2, len(got), "GET %s should return 2 TvRecord entities.", p)
assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "GET %s should create RecordCacheTypeTvRecord cache", p)
assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "GET %s should create RecordCacheTypeIEpg cache", p)
})
}
func (*Ping) Run() {
logger := wcg.NewLogger(nil)
down := 0
for {
_, err := api.Ping()
if err != nil {
logger.Error("Ping error: %v", err)
down = down + 1
} else {
if down > 0 {
down = 0
logger.Info("Ping recovered.")
} else {
logger.Debug("Successfully pinged to the endpoint")
}
}
time.Sleep(PingConfig.Interval)
}
}
func TestTvRecord_SyncSave_and_SyncDelete(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := wcg.NewAssert(t)
d := NewTvRecordDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
r := genTestRecord()
err := d.SyncSave(r)
assert.Nil(err, "TvRecordDriver#SyncSave should not return an error")
c, _ := d.NewQuery().Count()
assert.EqInt(1, c, "Confirm TvRecord entities has been stored.")
err = d.SyncDelete(r.Key())
assert.Nil(err, "TvRecordDriver#SyncDelete should not return an error")
c, _ = d.NewQuery().Count()
assert.EqInt(0, c, "Confirm TvRecord entities has been deleted.")
})
}
func TestListChannelApi(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := test.NewAssert(t)
p := app.Api.Path("/channels/")
err := util.WaitFor(func() bool {
var got []tv.TvChannel
req := ts.Get(p)
res := req.RouteTo(app.Routes())
assert.HttpStatus(200, res)
res.Json(&got)
return len(got) > 0
}, util.DefaultWaitForTimeout)
assert.Nil(err, "GET %s should return the list of TvChannels within a timeout window.", p)
// Confirm cache invalidation
mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
assert.Ok(mc.Exists(MC_KEY_CHANNELS), "GET %s should create the cache", p)
})
}
func TestTvRecord_GetRecords(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
const timeForm = "2006/01/02 15:04:05"
assert := wcg.NewAssert(t)
d := NewTvRecordDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
// prepare
r0 := genTestRecord()
r0.StartAt, _ = time.Parse(timeForm, "2014/01/01 12:00:00")
r0.EndAt, _ = time.Parse(timeForm, "2014/01/01 13:00:00")
d.Save(r0)
r1 := genTestRecord()
r1.StartAt, _ = time.Parse(timeForm, "2014/01/02 15:00:00")
r1.EndAt, _ = time.Parse(timeForm, "2014/01/02 16:00:00")
d.Save(r1)
err := util.WaitFor(func() bool {
c, _ := d.NewQuery().Count()
return c == 2
}, util.DefaultWaitForTimeout)
assert.Nil(err, "Confirm TvRecord entities has been stored within a timeout window")
since, _ := time.Parse(timeForm, "2014/01/01 01:00:00")
until, _ := time.Parse(timeForm, "2014/01/02 23:00:00")
list, err := d.GetRecords(since, until)
assert.Nil(err, "TvRecordDriver#GetRecords should not return an error")
assert.EqInt(2, len(list), "TvRecordDriver#GetRecords should return 2 records")
since, _ = time.Parse(timeForm, "2014/01/02 01:00:00")
until, _ = time.Parse(timeForm, "2014/01/02 23:00:00")
list, err = d.GetRecords(since, until)
assert.Nil(err, "TvRecordDriver#GetRecords should not return an error")
assert.EqInt(1, len(list), "TvRecordDriver#GetRecords should return 1 record")
since, _ = time.Parse(timeForm, "2014/01/03 01:00:00")
until, _ = time.Parse(timeForm, "2014/01/03 23:00:00")
list, err = d.GetRecords(since, until)
assert.Nil(err, "TvRecordDriver#GetRecords should not return an error")
assert.EqInt(0, len(list), "TvRecordDriver#GetRecords should return 0 records")
})
}
func TestSyncPutAndDelete(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := wcg.NewAssert(t)
d := NewDriver(ts.Context, "testSyncStruct", wcg.NewLogger(nil))
key := d.NewKey("foo", 0, nil)
tss := &testSyncStruct{}
var tss1 testSyncStruct
key, err := d.SyncPut(key, tss)
assert.Nil(err, "Driver#PutSync should not return an error")
d.Get(key, &tss1)
assert.Ok(!wcg.IsZero(tss1.Timestamp), "Driver#SyncPut should update timestamp")
err = d.SyncDelete(key)
assert.Nil(err, "Driver#SyncDelete should not return an error")
err = d.Get(key, &tss1)
assert.Ok(err == ErrNoSuchEntity, "Driver#SyncDelete should delete the entity")
})
}
func CleanupDatastore(ctx appengine.Context) error {
var dummy []interface{}
logger := wcg.NewLogger(nil)
logger.Debug("[Fixture] --------- CleanupDatastore ---------")
return wcg.RetryUntil(func() error {
if keys, err := datastore.NewQuery("").KeysOnly().GetAll(ctx, dummy); err != nil {
return err
} else {
if err := datastore.DeleteMulti(ctx, keys); err != nil {
return err
}
count, _ := datastore.NewQuery("").KeysOnly().Count(ctx)
if count == 0 {
return nil
} else {
return fmt.Errorf("Still have %d keys.", count)
}
}
}, 10*time.Second, 100*time.Millisecond)
}
func TestChannel_AllAsList(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := wcg.NewAssert(t)
// prepare
d := NewTvChannelDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
ent1 := &tv.TvChannel{"c1", "s1", "foo", "bar"}
ent2 := &tv.TvChannel{"c2", "s2", "hoge", "piyo"}
d.Put(d.NewKey(ent1.Key(), 0, nil), ent1)
d.Put(d.NewKey(ent2.Key(), 0, nil), ent2)
err := util.WaitFor(func() bool {
c, _ := d.NewQuery().Count()
return c == 2
}, util.DefaultWaitForTimeout)
assert.Nil(err, "Confirm TvChannel entities has been stored within a timeout window.")
list, err := d.AllAsList()
assert.EqInt(2, len(list), "AllAsList length")
})
}
func TestTvRecord_SaveAndLoad(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := wcg.NewAssert(t)
d := NewTvRecordDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
r := genTestRecord()
err := d.Save(r)
assert.Nil(err, "TvRecordDriver#Save should not return an error")
err = util.WaitFor(func() bool {
c, _ := d.NewQuery().Count()
return c == 1
}, util.DefaultWaitForTimeout)
assert.Nil(err, "Confirm TvRecord entities has been stored within a timeout window.")
r1, err := d.Load(r.Key())
assert.Nil(err, "TvRecordDriver#Load should not return an error")
assert.NotNil(r1, "TvRecordDriver#Load should return an valid TvRecord")
assert.EqStr("title", r1.Title, "TvRecordDriver#Load should return an valid TvRecord")
})
}
func TestChannel_AddAndDelChannel(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := wcg.NewAssert(t)
// prepare
d := NewTvChannelDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
d.AddChannel("c1", "s1", "foo", "bar")
d.AddChannel("c2", "s2", "foo", "bar")
err := util.WaitFor(func() bool {
c, _ := d.NewQuery().Count()
return c == 2
}, util.DefaultWaitForTimeout)
assert.Nil(err, "Confirm TvChannel entities has been stored within a timeout window.")
d.DelChannel("c1", "s1")
err = util.WaitFor(func() bool {
c, _ := d.NewQuery().Count()
return c == 1
}, util.DefaultWaitForTimeout)
assert.Nil(err, "Confirm TvChannel entities has been deleted within a timeout window.")
var list []*tv.TvChannel
d.NewQuery().GetAll(&list)
assert.EqStr("c2", list[0].Cid, "Confirm TvChannel.Cid")
d.AddChannelList([]*tv.TvChannel{
&tv.TvChannel{"c3", "s3", "aaa", "aaa"},
&tv.TvChannel{"c4", "s4", "bbb", "bbb"},
&tv.TvChannel{"c5", "s5", "ccc", "ccc"},
})
err = util.WaitFor(func() bool {
c, _ := d.NewQuery().Count()
return c == 4
}, util.DefaultWaitForTimeout)
assert.Nil(err, "Confirm TvChannel entities has been stored by AddChannelList within a timeout window.")
})
}
func TestIepgBulkUpdate(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := wcg.NewAssert(t)
d := NewIEpgDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
iepg1 := genTestIEpg()
iepg1.ProgramTitle = "new"
iepg1.StartAt, _ = wcg.ParseDateTime("2014/11/10 12:00")
iepg1.EndAt, _ = wcg.ParseDateTime("2014/11/10 12:30")
_, err := d.BulkUpdate([]*tv.IEpg{iepg1})
assert.Nil(err, "IEpgDriver#BulkUpdate should not return an error (insert).")
// insert
err = util.WaitFor(func() bool {
c, _ := d.NewQuery().Count()
return c == 1
}, util.DefaultWaitForTimeout)
assert.Nil(err, "Confirm Iepg entities has been stored within a timeout window.")
// update
iepg1.ProgramTitle = "update"
_, err = d.BulkUpdate([]*tv.IEpg{iepg1})
assert.Nil(err, "IEpgDriver#BulkUpdate should not return an error (update).")
err = util.WaitFor(func() bool {
found, err := d.Load(iepg1.Id)
return err == nil && found.ProgramTitle == "update"
}, util.DefaultWaitForTimeout)
assert.Nil(err, "Confirm Iepg entities has been updated within a timeout window.")
// Optout
iepg1.Optout = true
d.SyncSave(iepg1)
iepg1.Optout = false
iepg1.ProgramTitle = "optout"
keys, _ := d.BulkUpdate([]*tv.IEpg{iepg1})
assert.EqInt(0, len(keys), "IEpgDriver#BulkUpdate should not update optout entity.")
})
}
func TestCronKeywordCrawl(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := test.NewAssert(t)
// prepare
d := NewCrawlerConfigDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
d.Add(&tv.CrawlerConfig{
Keyword: "モーニング娘。",
Category: "モーニング娘",
Scope: tv.FEED_SCOPE_ALL,
})
err := util.WaitFor(func() bool {
c, _ := d.NewQuery().Count()
return c == 1
}, util.DefaultWaitForTimeout)
assert.Nil(err, "Confirm CrawlerConfig entities has been stored within a timeout window.")
p := app.Cron.Path("/keywords/crawl/")
req := ts.Get(p)
lib.SetApiTokenForTest(req, lib.Admin)
res := req.RouteTo(app.Routes())
assert.HttpStatus(200, res)
})
}
func TestPreviewKeywordApi(t *testing.T) {
test.RunTestServer(func(ts *test.TestServer) {
assert := test.NewAssert(t)
// prepare
dchan := NewTvChannelDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
dchan.AddChannelList(defaultChannels)
err := util.WaitFor(func() bool {
c, _ := dchan.NewQuery().Count()
return c > 0
}, util.DefaultWaitForTimeout)
assert.Nil(err, "Prepare channels within a timeout window")
var got map[string]interface{}
p := app.Api.Path("/keywords/preview/モーニング娘。'14.json")
req := ts.Get(p)
lib.SetApiTokenForTest(req, lib.Admin)
res := req.RouteTo(app.Routes())
assert.HttpStatus(200, res)
res.Json(&got)
assert.NotNil(got["samples"], "GET %s should return the sample list", p)
assert.NotNil(got["total"], "GET %s should return the total entries", p)
})
}
func (rt *apiRoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) {
logger := wcg.NewLogger(nil)
req.Header.Set("X-SPEEDLAND-API-TOKEN", lib.Config.Token)
logger.Debug("[Api] %s %s", req.Method, req.URL.Path)
return rt.RoundTripper.RoundTrip(req)
}