Example #1
0
func main() {
	app := cli.NewApp()
	app.Name = "speedland-agent"
	app.Usage = "A agent command for speedland.net"
	app.Version = "1.0.0"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			"config, c",
			"wcg.ini",
			"configuration file",
			"WCG_INI_FILE",
		},
	}
	app.Before = func(c *cli.Context) error {
		wcg.ConfigureProcess(c.String("config"))
		// normalize path
		lib.Config.Endpoint.Path = ""
		wcg.NewLogger(nil).Info("Used configurn file: %q", c.String("config"))
		wcg.NewLogger(nil).Info("Target Endpoint: %q", lib.Config.Endpoint)
		wcg.NewLogger(nil).Debug("Token: %q", lib.Config.Token)
		return nil
	}
	app.Commands = commands.AllCommands()
	app.Run(os.Args)
	wcg.WaitLogs()
}
Example #2
0
func TestDelChannelApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)
		// prepare
		d := NewTvChannelDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
		ent1 := &tv.TvChannel{"c1", "s1", "foo", "bar"}
		ent2 := &tv.TvChannel{"c2", "s2", "hoge", "piyo"}
		d.Put(d.NewKey(ent1.Key(), 0, nil), ent1)
		d.Put(d.NewKey(ent2.Key(), 0, nil), ent2)

		err := util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 2
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm TvChannel entities has been stored within a timeout window.")
		p := app.Api.Path("/channels/c1/s1.json")
		req := ts.Delete(p)
		lib.SetApiTokenForTest(req, lib.Admin)
		res := req.RouteTo(app.Routes())
		assert.HttpStatus(200, res)

		err = util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 1
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "DELETE %s Confirm TvChannel entities has been deleted via API within a timeout window.", p)

		// Confirm cache invalidation
		mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
		assert.Ok(!mc.Exists(MC_KEY_CHANNELS), "DELETE %s should invalidate the cache", p)

	})
}
Example #3
0
func TestAddKeywordApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)
		p := app.Api.Path("/keywords/")
		req := ts.PostForm(p, url.Values{
			"keyword":  []string{"モーニング娘。'15"},
			"category": []string{"モーニング娘。"},
			"scope":    []string{"1"},
		})
		lib.SetApiTokenForTest(req, lib.Admin)
		var got map[string]interface{}
		res := req.RouteTo(app.Routes())
		res.Json(&got)
		assert.HttpStatus(201, res)
		assert.EqStr(
			"http://localhost:8080/api/pt/keywords/モーニング娘。'15.json",
			got["location"].(string),
			"POST %s location",
			p,
		)

		// Confirm cache invalidation
		mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
		assert.Ok(!mc.Exists(MC_KEY_KEYWORDS), "POST %s should invalidate the cache", p)

	})
}
func TestChannel_AddAndDelCrawlerConfig(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := wcg.NewAssert(t)

		// prepare
		d := NewCrawlerConfigDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
		d.Add(&tv.CrawlerConfig{
			Keyword:  "キーワード1",
			Category: "カテゴリー1",
			Scope:    1,
		})
		d.Add(&tv.CrawlerConfig{
			Keyword:  "キーワード2",
			Category: "カテゴリー2",
			Scope:    1,
		})

		err := util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 2
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm CrawlerConfig entities has been stored within a timeout window.")

		d.Delete("キーワード1")
		err = util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 1
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm CrawlerConfig entities has been deleted within a timeout window.")

		err = d.Delete("Not Exists")
		assert.Nil(err, "Delete should not return even trying to delete the unexising keyword.")
	})
}
Example #5
0
func TestAddChannelApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)
		p := app.Api.Path("/channels/")
		req := ts.PostForm(p, url.Values{
			"cid":             []string{"c1"},
			"sid":             []string{"s1"},
			"name":            []string{"foo"},
			"iepg_station_id": []string{"bar"},
		})
		lib.SetApiTokenForTest(req, lib.Admin)
		var got map[string]interface{}
		res := req.RouteTo(app.Routes())
		res.Json(&got)
		assert.HttpStatus(201, res)
		assert.EqStr(
			"http://localhost:8080/api/pt/channels/c1/s1.json",
			got["location"].(string),
			"POST %s location",
			p,
		)

		// Confirm cache invalidation
		mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
		assert.Ok(!mc.Exists(MC_KEY_CHANNELS), "POST %s should invalidate the cache", p)

	})
}
Example #6
0
func NewAppContextFromTestServer(app *lib.App, ts *test.TestServer) *lib.AppContext {
	return &lib.AppContext{
		app,
		ts.Context,
		wcg.NewLogger(nil),
	}
}
Example #7
0
func TestGetOverlaps(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		var found []*tv.TvRecord
		const timeForm = "2006/01/02 15:04:05"
		assert := wcg.NewAssert(t)
		d := NewTvRecordDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))

		base := genTestRecord()
		base.StartAt, _ = time.Parse(timeForm, "2014/01/01 12:00:00")
		base.EndAt, _ = time.Parse(timeForm, "2014/01/01 13:00:00")
		d.Save(base)
		err := util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 1
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm TvRecord entities has been stored within a timeout window.")

		// case1: no overlaps
		r := genTestRecord()
		r.StartAt, _ = time.Parse(timeForm, "2014/01/01 10:50:00")
		r.EndAt, _ = time.Parse(timeForm, "2014/01/01 11:10:00")
		found, err = d.GetOverlaps(r)
		assert.Nil(err, "TvRecordDriver#GetOverlaps should not return an error (case1)")
		assert.EqInt(0, len(found), "TvRecordDriver#GetOverlaps should return empty array (case1)")

		// case 2: base.end < r.start
		r = genTestRecord()
		r.StartAt, _ = time.Parse(timeForm, "2014/01/01 13:50:00")
		r.EndAt, _ = time.Parse(timeForm, "2014/01/01 14:10:00")
		found, err = d.GetOverlaps(r)
		assert.Nil(err, "TvRecordDriver#GetOverlaps should not return an error (case2)")
		assert.EqInt(0, len(found), "TvRecordDriver#GetOverlaps should return empty array (case2)")

		// case 3: base.start < r.start < base.end
		r = genTestRecord()
		r.StartAt, _ = time.Parse(timeForm, "2014/01/01 11:50:00")
		r.EndAt, _ = time.Parse(timeForm, "2014/01/01 12:10:00")
		found, err = d.GetOverlaps(r)
		assert.Nil(err, "TvRecordDriver#GetOverlaps should not return an error (case3)")
		assert.EqInt(1, len(found), "TvRecordDriver#GetOverlaps should return an array with an entity (case3)")

		// case 4: r.start < base.end < r.end
		r = genTestRecord()
		r.StartAt, _ = time.Parse(timeForm, "2014/01/01 12:50:00")
		r.EndAt, _ = time.Parse(timeForm, "2014/01/01 13:10:00")
		found, err = d.GetOverlaps(r)
		assert.Nil(err, "TvRecordDriver#GetOverlaps should not return an error (case4)")
		assert.EqInt(1, len(found), "TvRecordDriver#GetOverlaps should return an array with an entity (case4)")

		// case 5: r.start < base.start < base.end < r.end
		r = genTestRecord()
		r.StartAt, _ = time.Parse(timeForm, "2014/01/01 12:10:00")
		r.EndAt, _ = time.Parse(timeForm, "2014/01/01 12:50:00")
		found, err = d.GetOverlaps(r)
		assert.Nil(err, "TvRecordDriver#GetOverlaps should not return an error (case4)")
		assert.EqInt(1, len(found), "TvRecordDriver#GetOverlaps should return an array with an entity (case4)")
	})
}
Example #8
0
func FixtureFromMap(ctx appengine.Context, arr []map[string]interface{}) error {
	logger := wcg.NewLogger(nil)
	for _, v := range arr {
		if err := loadJsonToDatastore(ctx, nil, v, logger); err != nil {
			return err
		}
	}
	return nil
}
Example #9
0
func Run() {
	go ping.Run()
	go recorder.Run()
	go statsd.Run()

	// block the main until receiving the signal
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGTERM, syscall.SIGINT)
	s := <-c
	wcg.NewLogger(nil).Info("Exitting the signal with %s", s)
	// TODO: delegate the notification to each job
}
Example #10
0
func TestDeleteRecordApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)
		d := NewRecordCacheDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))

		// create an empty cache
		var got []tv.TvRecord
		p := app.Api.Path("/records/")
		req := ts.Get(p)
		res := req.RouteTo(app.Routes())
		assert.HttpStatus(200, res)
		res.Json(&got)
		assert.EqInt(0, len(got), "GET %s should return 2 TvRecord entities.")
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "GET %s should create RecordCacheTypeTvRecord cache", p)
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "GET %s should create RecordCacheTypeIEpg cache", p)

		// prepare
		in_window := genTestRecord()
		in_window.StartAt = d.today.Add(1 * time.Hour)
		in_window.EndAt = d.today.Add(2 * time.Hour)
		d.TvRecord.Save(in_window)

		future := genTestRecord()
		future.StartAt = d.today.Add(RECORD_TIME_WINDOW + 1*time.Hour)
		future.EndAt = future.StartAt.Add(1 * time.Hour)
		d.TvRecord.Save(future)

		err := util.WaitFor(func() bool {
			records, _ := d.TvRecord.NewQuery().Count()
			return records == 2
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm TvRecord entities has been stored within a timeout window.")

		// case 1: future
		p = fmt.Sprintf("%s%s.json", app.Api.Path("/records/"), future.Key())
		req = ts.Delete(p)
		lib.SetApiTokenForTest(req, lib.Admin)
		res = req.RouteTo(app.Routes())
		assert.HttpStatus(200, res)
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "POST %s should not invalidate RecordCacheTypeTvRecord cache (future record)", p)
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "GET %s should not invalidate RecordCacheTypeIEpg cache (future record)", p)

		// case 2: in recording window
		p = fmt.Sprintf("%s%s.json", app.Api.Path("/records/"), in_window.Key())
		req = ts.Delete(p)
		lib.SetApiTokenForTest(req, lib.Admin)
		res = req.RouteTo(app.Routes())
		assert.HttpStatus(200, res)
		assert.Ok(!d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "POST %s should invalidate RecordCacheTypeTvRecord cache (in_window record)", p)
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "GET %s should not invalidate RecordCacheTypeIEpg cache (in_window record)", p)
	})
}
Example #11
0
func TestDelKeywordApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)
		// prepare
		d := NewCrawlerConfigDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
		d.Add(&tv.CrawlerConfig{
			Keyword:  "モーニング娘。'15",
			Category: "モーニング娘",
			Scope:    tv.FEED_SCOPE_ALL,
		})
		d.Add(&tv.CrawlerConfig{
			Keyword:  "SPEED",
			Category: "SPEED",
			Scope:    tv.FEED_SCOPE_ALL,
		})

		err := util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 2
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm CrawlerConfig entities has been stored within a timeout window.")

		p := app.Api.Path("/keywords/モーニング娘。'15.json")
		req := ts.Delete(p)
		lib.SetApiTokenForTest(req, lib.Admin)
		res := req.RouteTo(app.Routes())
		assert.HttpStatus(200, res)

		err = util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 1
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "DELETE %s Confirm CrawlerConfig entities has been deleted via API within a timeout window.", p)

		// Confirm cache invalidation
		mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
		assert.Ok(!mc.Exists(MC_KEY_KEYWORDS), "DELETE %s should invalidate the cache", p)
	})
}
func TestRecordCache_GetRecords_and_Invalidate(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := wcg.NewAssert(t)
		d := NewRecordCacheDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))

		// prepare
		in_window := genTestRecord()
		in_window.StartAt = d.today.Add(1 * time.Hour)
		in_window.EndAt = d.today.Add(2 * time.Hour)
		d.TvRecord.Save(in_window)

		future := genTestRecord()
		future.StartAt = d.today.Add(RECORD_TIME_WINDOW + 1*time.Hour)
		future.EndAt = future.StartAt.Add(1 * time.Hour)
		d.TvRecord.Save(future)

		past := genTestRecord()
		past.StartAt = d.today.Add(-RECORD_TIME_WINDOW - 1*time.Hour)
		past.EndAt = past.StartAt.Add(30 * time.Minute)
		d.TvRecord.Save(past)

		in_window_iepg := genTestIEpg()
		in_window.StartAt = d.today.Add(4 * time.Hour)
		in_window.EndAt = d.today.Add(5 * time.Hour)
		d.IEpg.Save(in_window_iepg)

		err := util.WaitFor(func() bool {
			records, _ := d.TvRecord.NewQuery().Count()
			iepgs, _ := d.IEpg.NewQuery().Count()
			return records+iepgs == 4
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm TvRecord/IEpg entities has been stored within a timeout window.")

		list, err := d.GetRecords(false)
		assert.Nil(err, "RecordCache#GetRecords should not return an error")
		assert.EqInt(2, len(list), "RecordCache#GetRecords should return 2 records in a time window")

		// cache check
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "RecordCache#GetRecords should create RecordCacheTypeTvRecord on memcache.")
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "RecordCache#GetRecords should create RecordCacheTypeIEpg on memcache.")

		// invalidate the keys
		assert.Ok(!d.Invalidate(future), "RecordCache#Invalidate should not return true if the passed record is out of window (future)")
		assert.Ok(!d.Invalidate(past), "RecordCache#Invalidate should not return true if the passed record is out of window (future)")
		assert.Ok(d.Invalidate(in_window), "RecordCache#Invalidate should return true if the passed record is in window")
		assert.Ok(!d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "RecordCache#GetRecords should invalidate RecordCacheTypeTvRecord on memcache.")

		assert.Ok(d.Invalidate(in_window_iepg), "RecordCache#Invalidate should return true if the passed record is in window (iepg)")
		assert.Ok(!d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "RecordCache#GetRecords should invalidate RecordCacheTypeIEpg on memcache.")
	})
}
Example #13
0
func TestListKeywordApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)

		// prepare
		d := NewCrawlerConfigDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
		d.Add(&tv.CrawlerConfig{
			Keyword:  "キーワード1",
			Category: "カテゴリー1",
			Scope:    1,
		})
		d.Add(&tv.CrawlerConfig{
			Keyword:  "キーワード2",
			Category: "カテゴリー2",
			Scope:    1,
		})

		err := util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 2
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm CrawlerConfig entities has been stored within a timeout window.")

		var got []*tv.CrawlerConfig
		p := app.Api.Path("/keywords/") + "?force=true"
		req := ts.Get(p)
		res := req.RouteTo(app.Routes())
		assert.HttpStatus(200, res)
		res.Json(&got)
		assert.EqStr("キーワード2", got[0].Keyword, "GET %s should return the list ordered by creation time.", p)
		assert.EqStr("キーワード1", got[1].Keyword, "GET %s should return the list ordered by creation time.", p)

		// Confirm cache invalidation
		mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
		assert.Ok(mc.Exists(MC_KEY_KEYWORDS), "GET %s should create the cache", p)
	})
}
Example #14
0
func TestCreateRecordApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)
		d := NewRecordCacheDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))

		// create an empty cache
		var got []tv.TvRecord
		p := app.Api.Path("/records/")
		req := ts.Get(p)
		res := req.RouteTo(app.Routes())
		assert.HttpStatus(200, res)
		res.Json(&got)
		assert.EqInt(0, len(got), "GET %s should return 2 TvRecord entities.")
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "GET %s should create RecordCacheTypeTvRecord cache", p)
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "GET %s should create RecordCacheTypeIEpg cache", p)

		now := time.Now()

		// case 1: future
		req = ts.PostForm(p, url.Values{
			"title":    []string{"Title"},
			"category": []string{"Category"},
			"cid":      []string{"27"},
			"sid":      []string{"hd"},
			"start_at": []string{util.FormatDateTime(now.Add(RECORD_TIME_WINDOW + 24*time.Hour))},
			"end_at":   []string{util.FormatDateTime(now.Add(RECORD_TIME_WINDOW + 25*time.Hour))},
		})
		lib.SetApiTokenForTest(req, lib.Admin)
		res = req.RouteTo(app.Routes())
		assert.HttpStatus(201, res)
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "POST %s should not invalidate RecordCacheTypeTvRecord cache (future record)", p)
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "GET %s should not invalidate RecordCacheTypeIEpg cache (future record)", p)

		// case 2: in recording window
		req = ts.PostForm(p, url.Values{
			"title":    []string{"Title"},
			"category": []string{"Category"},
			"cid":      []string{"27"},
			"sid":      []string{"hd"},
			"start_at": []string{util.FormatDateTime(now)},
			"end_at":   []string{util.FormatDateTime(now.Add(1 * time.Hour))},
		})
		lib.SetApiTokenForTest(req, lib.Admin)
		res = req.RouteTo(app.Routes())
		assert.HttpStatus(201, res)
		assert.Ok(!d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "POST %s should invalidate RecordCacheTypeTvRecord cache (in window)", p)
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "GET %s should not invalidate RecordCacheTypeIEpg cache (in window)", p)
	})
}
Example #15
0
func DatastoreFixture(ctx appengine.Context, path string, bindings interface{}) error {
	logger := wcg.NewLogger(nil)
	logger.Debug("[Fixture] load from %s", path)
	data, err := loadFile(path, bindings)
	if err != nil {
		return err
	}
	var arr []map[string]interface{}
	if err = json.Unmarshal(data, &arr); err != nil {
		return fmt.Errorf("Could not load the json file from %q - JSON Parse error: %v", path, err)
	}
	for _, v := range arr {
		if err := loadJsonToDatastore(ctx, nil, v, logger); err != nil {
			return err
		}
	}
	return nil
}
Example #16
0
func (*TweetStream) Run() {
	logger := wcg.NewLogger(nil)
	if tweetStreamConfig.ConsumerKey == "" {
		logger.Error("[TweetStream] consumer_key is missing or empty.")
		return
	}
	if tweetStreamConfig.ConsumerSecret == "" {
		logger.Error("[TweetStream] consumer_secret is missing or empty.")
		return
	}
	if tweetStreamConfig.Token == "" {
		logger.Error("[TweetStream] oauth_token is missing or empty.")
		return
	}
	if tweetStreamConfig.TokenSecret == "" {
		logger.Error("[TweetStream] oauth_token_secret is missing or empty.")
		return
	}

	for {
		consumerKey := tweetStreamConfig.ConsumerKey
		consumerSecret := tweetStreamConfig.ConsumerSecret
		client := twitter.NewTwitterClient(consumerKey, consumerSecret, nil)
		client.Token = &twitter.OAuthToken{
			ConsumerKey:    consumerKey,
			ConsumerSecret: consumerSecret,
			Token:          tweetStreamConfig.Token,
			TokenSecret:    tweetStreamConfig.TokenSecret,
		}
		ts, err := client.Stream(&twitter.StreamingParams{
			Track: "#morningmusume15",
		})
		if err != nil {
			logger.Error("[TweetStream] Error building streaming connection: %v", err)
		} else {
			logger.Info("[TweetStream] Connected to the streaming server.")
			ch := ts.GetStreamChannel()
			for tweet := range ch {
				logger.Debug("[TweetStream] %v", tweet)
			}
			logger.Info("[TweetStream] Disconnected from server.")
		}
	}
}
Example #17
0
func TestListRecordApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)
		d := NewRecordCacheDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))

		// prepare
		in_window := genTestRecord()
		in_window.StartAt = d.today.Add(1 * time.Hour)
		in_window.EndAt = d.today.Add(2 * time.Hour)
		d.TvRecord.Save(in_window)

		future := genTestRecord()
		future.StartAt = d.today.Add(RECORD_TIME_WINDOW + 1*time.Hour)
		future.EndAt = future.StartAt.Add(1 * time.Hour)
		d.TvRecord.Save(future)

		past := genTestRecord()
		past.StartAt = d.today.Add(-RECORD_TIME_WINDOW - 1*time.Hour)
		past.EndAt = past.StartAt.Add(30 * time.Minute)
		d.TvRecord.Save(past)

		in_window_iepg := genTestIEpg()
		in_window.StartAt = d.today.Add(4 * time.Hour)
		in_window.EndAt = d.today.Add(5 * time.Hour)
		d.IEpg.Save(in_window_iepg)

		err := util.WaitFor(func() bool {
			records, _ := d.TvRecord.NewQuery().Count()
			iepgs, _ := d.IEpg.NewQuery().Count()
			return records+iepgs == 4
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm TvRecord/IEpg entities has been stored within a timeout window.")

		var got []tv.TvRecord
		p := app.Api.Path("/records/")
		req := ts.Get(p)
		res := req.RouteTo(app.Routes())
		assert.HttpStatus(200, res)
		res.Json(&got)
		assert.EqInt(2, len(got), "GET %s should return 2 TvRecord entities.", p)
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "GET %s should create RecordCacheTypeTvRecord cache", p)
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "GET %s should create RecordCacheTypeIEpg cache", p)
	})
}
Example #18
0
func (*Ping) Run() {
	logger := wcg.NewLogger(nil)
	down := 0
	for {
		_, err := api.Ping()
		if err != nil {
			logger.Error("Ping error: %v", err)
			down = down + 1
		} else {
			if down > 0 {
				down = 0
				logger.Info("Ping recovered.")
			} else {
				logger.Debug("Successfully pinged to the endpoint")
			}
		}
		time.Sleep(PingConfig.Interval)
	}
}
Example #19
0
func TestTvRecord_SyncSave_and_SyncDelete(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := wcg.NewAssert(t)
		d := NewTvRecordDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))

		r := genTestRecord()
		err := d.SyncSave(r)
		assert.Nil(err, "TvRecordDriver#SyncSave should not return an error")

		c, _ := d.NewQuery().Count()
		assert.EqInt(1, c, "Confirm TvRecord entities has been stored.")

		err = d.SyncDelete(r.Key())
		assert.Nil(err, "TvRecordDriver#SyncDelete should not return an error")

		c, _ = d.NewQuery().Count()
		assert.EqInt(0, c, "Confirm TvRecord entities has been deleted.")
	})
}
Example #20
0
func TestListChannelApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)
		p := app.Api.Path("/channels/")
		err := util.WaitFor(func() bool {
			var got []tv.TvChannel
			req := ts.Get(p)
			res := req.RouteTo(app.Routes())
			assert.HttpStatus(200, res)
			res.Json(&got)
			return len(got) > 0
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "GET %s should return the list of TvChannels within a timeout window.", p)

		// Confirm cache invalidation
		mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
		assert.Ok(mc.Exists(MC_KEY_CHANNELS), "GET %s should create the cache", p)
	})
}
Example #21
0
func TestTvRecord_GetRecords(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		const timeForm = "2006/01/02 15:04:05"
		assert := wcg.NewAssert(t)
		d := NewTvRecordDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))

		// prepare
		r0 := genTestRecord()
		r0.StartAt, _ = time.Parse(timeForm, "2014/01/01 12:00:00")
		r0.EndAt, _ = time.Parse(timeForm, "2014/01/01 13:00:00")
		d.Save(r0)

		r1 := genTestRecord()
		r1.StartAt, _ = time.Parse(timeForm, "2014/01/02 15:00:00")
		r1.EndAt, _ = time.Parse(timeForm, "2014/01/02 16:00:00")
		d.Save(r1)

		err := util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 2
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm TvRecord entities has been stored within a timeout window")

		since, _ := time.Parse(timeForm, "2014/01/01 01:00:00")
		until, _ := time.Parse(timeForm, "2014/01/02 23:00:00")
		list, err := d.GetRecords(since, until)
		assert.Nil(err, "TvRecordDriver#GetRecords should not return an error")
		assert.EqInt(2, len(list), "TvRecordDriver#GetRecords should return 2 records")

		since, _ = time.Parse(timeForm, "2014/01/02 01:00:00")
		until, _ = time.Parse(timeForm, "2014/01/02 23:00:00")
		list, err = d.GetRecords(since, until)
		assert.Nil(err, "TvRecordDriver#GetRecords should not return an error")
		assert.EqInt(1, len(list), "TvRecordDriver#GetRecords should return 1 record")

		since, _ = time.Parse(timeForm, "2014/01/03 01:00:00")
		until, _ = time.Parse(timeForm, "2014/01/03 23:00:00")
		list, err = d.GetRecords(since, until)
		assert.Nil(err, "TvRecordDriver#GetRecords should not return an error")
		assert.EqInt(0, len(list), "TvRecordDriver#GetRecords should return 0 records")
	})
}
Example #22
0
func TestSyncPutAndDelete(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := wcg.NewAssert(t)
		d := NewDriver(ts.Context, "testSyncStruct", wcg.NewLogger(nil))

		key := d.NewKey("foo", 0, nil)
		tss := &testSyncStruct{}
		var tss1 testSyncStruct
		key, err := d.SyncPut(key, tss)
		assert.Nil(err, "Driver#PutSync should not return an error")
		d.Get(key, &tss1)
		assert.Ok(!wcg.IsZero(tss1.Timestamp), "Driver#SyncPut should update timestamp")

		err = d.SyncDelete(key)
		assert.Nil(err, "Driver#SyncDelete should not return an error")
		err = d.Get(key, &tss1)
		assert.Ok(err == ErrNoSuchEntity, "Driver#SyncDelete should delete the entity")

	})
}
Example #23
0
func CleanupDatastore(ctx appengine.Context) error {
	var dummy []interface{}
	logger := wcg.NewLogger(nil)
	logger.Debug("[Fixture] --------- CleanupDatastore ---------")
	return wcg.RetryUntil(func() error {
		if keys, err := datastore.NewQuery("").KeysOnly().GetAll(ctx, dummy); err != nil {
			return err
		} else {
			if err := datastore.DeleteMulti(ctx, keys); err != nil {
				return err
			}
			count, _ := datastore.NewQuery("").KeysOnly().Count(ctx)
			if count == 0 {
				return nil
			} else {
				return fmt.Errorf("Still have %d keys.", count)
			}
		}
	}, 10*time.Second, 100*time.Millisecond)
}
Example #24
0
func TestChannel_AllAsList(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := wcg.NewAssert(t)

		// prepare
		d := NewTvChannelDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
		ent1 := &tv.TvChannel{"c1", "s1", "foo", "bar"}
		ent2 := &tv.TvChannel{"c2", "s2", "hoge", "piyo"}
		d.Put(d.NewKey(ent1.Key(), 0, nil), ent1)
		d.Put(d.NewKey(ent2.Key(), 0, nil), ent2)

		err := util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 2
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm TvChannel entities has been stored within a timeout window.")

		list, err := d.AllAsList()
		assert.EqInt(2, len(list), "AllAsList length")
	})
}
Example #25
0
func TestTvRecord_SaveAndLoad(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := wcg.NewAssert(t)
		d := NewTvRecordDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))

		r := genTestRecord()
		err := d.Save(r)
		assert.Nil(err, "TvRecordDriver#Save should not return an error")

		err = util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 1
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm TvRecord entities has been stored within a timeout window.")

		r1, err := d.Load(r.Key())
		assert.Nil(err, "TvRecordDriver#Load should not return an error")
		assert.NotNil(r1, "TvRecordDriver#Load should return an valid TvRecord")
		assert.EqStr("title", r1.Title, "TvRecordDriver#Load should return an valid TvRecord")
	})
}
Example #26
0
func TestChannel_AddAndDelChannel(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := wcg.NewAssert(t)

		// prepare
		d := NewTvChannelDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
		d.AddChannel("c1", "s1", "foo", "bar")
		d.AddChannel("c2", "s2", "foo", "bar")

		err := util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 2
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm TvChannel entities has been stored within a timeout window.")

		d.DelChannel("c1", "s1")
		err = util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 1
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm TvChannel entities has been deleted within a timeout window.")

		var list []*tv.TvChannel
		d.NewQuery().GetAll(&list)
		assert.EqStr("c2", list[0].Cid, "Confirm TvChannel.Cid")

		d.AddChannelList([]*tv.TvChannel{
			&tv.TvChannel{"c3", "s3", "aaa", "aaa"},
			&tv.TvChannel{"c4", "s4", "bbb", "bbb"},
			&tv.TvChannel{"c5", "s5", "ccc", "ccc"},
		})

		err = util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 4
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm TvChannel entities has been stored by AddChannelList within a timeout window.")
	})
}
Example #27
0
func TestIepgBulkUpdate(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := wcg.NewAssert(t)
		d := NewIEpgDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
		iepg1 := genTestIEpg()
		iepg1.ProgramTitle = "new"
		iepg1.StartAt, _ = wcg.ParseDateTime("2014/11/10 12:00")
		iepg1.EndAt, _ = wcg.ParseDateTime("2014/11/10 12:30")
		_, err := d.BulkUpdate([]*tv.IEpg{iepg1})
		assert.Nil(err, "IEpgDriver#BulkUpdate should not return an error (insert).")

		// insert
		err = util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 1
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm Iepg entities has been stored within a timeout window.")

		// update
		iepg1.ProgramTitle = "update"
		_, err = d.BulkUpdate([]*tv.IEpg{iepg1})
		assert.Nil(err, "IEpgDriver#BulkUpdate should not return an error (update).")
		err = util.WaitFor(func() bool {
			found, err := d.Load(iepg1.Id)
			return err == nil && found.ProgramTitle == "update"
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm Iepg entities has been updated within a timeout window.")

		// Optout
		iepg1.Optout = true
		d.SyncSave(iepg1)

		iepg1.Optout = false
		iepg1.ProgramTitle = "optout"
		keys, _ := d.BulkUpdate([]*tv.IEpg{iepg1})
		assert.EqInt(0, len(keys), "IEpgDriver#BulkUpdate should not update optout entity.")
	})
}
Example #28
0
func TestCronKeywordCrawl(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)

		// prepare
		d := NewCrawlerConfigDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
		d.Add(&tv.CrawlerConfig{
			Keyword:  "モーニング娘。",
			Category: "モーニング娘",
			Scope:    tv.FEED_SCOPE_ALL,
		})
		err := util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 1
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm CrawlerConfig entities has been stored within a timeout window.")

		p := app.Cron.Path("/keywords/crawl/")
		req := ts.Get(p)
		lib.SetApiTokenForTest(req, lib.Admin)
		res := req.RouteTo(app.Routes())
		assert.HttpStatus(200, res)
	})
}
Example #29
0
func TestPreviewKeywordApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)

		// prepare
		dchan := NewTvChannelDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
		dchan.AddChannelList(defaultChannels)
		err := util.WaitFor(func() bool {
			c, _ := dchan.NewQuery().Count()
			return c > 0
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Prepare channels within a timeout window")

		var got map[string]interface{}
		p := app.Api.Path("/keywords/preview/モーニング娘。'14.json")
		req := ts.Get(p)
		lib.SetApiTokenForTest(req, lib.Admin)
		res := req.RouteTo(app.Routes())
		assert.HttpStatus(200, res)
		res.Json(&got)
		assert.NotNil(got["samples"], "GET %s should return the sample list", p)
		assert.NotNil(got["total"], "GET %s should return the total entries", p)
	})
}
Example #30
0
func (rt *apiRoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	logger := wcg.NewLogger(nil)
	req.Header.Set("X-SPEEDLAND-API-TOKEN", lib.Config.Token)
	logger.Debug("[Api] %s %s", req.Method, req.URL.Path)
	return rt.RoundTripper.RoundTrip(req)
}