// CleanupDatastore is to remove all data in datastore
func CleanupDatastore(ctx context.Context, namespaces ...string) error {
var dummy []interface{}
logger := wcg.NewLogger(nil)
logger.Debugf("[Fixture] --------- CleanupDatastore ---------")
namespaceList := append(namespaces, "")
for _, ns := range namespaceList {
logger.Debugf("[Fixture] Cleanup: ns=%q", ns)
var _ctx = ctx
if ns != "" {
_ctx, _ = appengine.Namespace(_ctx, ns)
}
err := wcg.RetryUntil(func() error {
var keys []*datastore.Key
var err error
if keys, err = datastore.NewQuery("").KeysOnly().GetAll(_ctx, dummy); err != nil {
return err
}
if err := datastore.DeleteMulti(_ctx, keys); err != nil {
return err
}
count, _ := datastore.NewQuery("").KeysOnly().Count(_ctx)
if count == 0 {
return nil
}
return fmt.Errorf("Still have %d keys.", count)
}, 10*time.Second, 100*time.Millisecond)
if err != nil {
return err
}
}
return nil
}
func Test_Query_WithCache(t *testing.T) {
assert := gaetest.NewAssert(t)
gaetest.CleanupDatastore(ts.Context)
gaetest.FixtureFromMap(ts.Context, []map[string]interface{}{
map[string]interface{}{"_kind": testEntity.String(), "_key": "1", "ID": "1"},
map[string]interface{}{"_kind": testEntity.String(), "_key": "2", "ID": "2"},
map[string]interface{}{"_kind": testEntity.String(), "_key": "3", "ID": "3"},
map[string]interface{}{"_kind": testEntity.String(), "_key": "4", "ID": "4"},
map[string]interface{}{"_kind": testEntity.String(), "_key": "5", "ID": "5"},
map[string]interface{}{"_kind": testEntity.String(), "_key": "6", "ID": "6"},
map[string]interface{}{"_kind": testEntity.String(), "_key": "7", "ID": "7"},
})
page, err := testEntity.Query().Cache("123").Execute(ts.GET("/").Request)
got := page.Data.([]TEntity)
assert.Nil(err)
assert.EqInt(7, len(got), "len(Pagination.Data)")
driver := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
assert.OK(driver.Exists("query.TestEntity.123"))
// cache still exists even after removing data.
gaetest.CleanupDatastore(ts.Context)
page, err = testEntity.Query().Cache("123").Execute(ts.GET("/").Request)
got = page.Data.([]TEntity)
assert.Nil(err)
assert.EqInt(7, len(got))
assert.EqStr("1", got[0].ID)
}
func Test_Delete_CacheInvalidation(t *testing.T) {
assert := gaetest.NewAssert(t)
driver := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
gaetest.CleanupDatastore(ts.Context)
gaetest.FixtureFromMap(ts.Context, []map[string]interface{}{
map[string]interface{}{"_kind": testEntity.String(), "_key": "1", "ID": "1"},
map[string]interface{}{"_kind": testEntity.String(), "_key": "2", "ID": "2"},
map[string]interface{}{"_kind": testEntity.String(), "_key": "3", "ID": "3"},
map[string]interface{}{"_kind": testEntity.String(), "_key": "4", "ID": "4"},
map[string]interface{}{"_kind": testEntity.String(), "_key": "5", "ID": "5"},
map[string]interface{}{"_kind": testEntity.String(), "_key": "6", "ID": "6"},
})
// create caches
testEntity.Query().Cache("123").MustExecute(ts.GET("/").Request)
testEntity.Get().Key("1").Cache(true).MustOne(ts.GET("/").Request)
assert.OK(driver.Exists("ent.TestEntity.1"), "An entyty cache should exist")
assert.OK(driver.Exists("query.TestEntity.123"), "A query cache should exist")
testEntity.Delete().Key("1").Cache("123").MustCommit(ts.GET("/").Request)
assert.Not(driver.Exists("ent.TestEntity.1"), "Entity cache should be invalidated")
assert.Not(driver.Exists("query.TestEntity.123"), "Query cache should be invalidated")
}
func Test_GetMulti_Cache(t *testing.T) {
assert := gaetest.NewAssert(t)
gaetest.CleanupDatastore(ts.Context)
gaetest.FixtureFromMap(ts.Context, []map[string]interface{}{
map[string]interface{}{"_kind": testEntity.String(), "_key": "1", "ID": "1"},
})
driver := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
_, list, err := testEntity.GetMulti().Keys("1", "3").Cache(true).List(ts.GET("/").Request)
got := list.([]*TEntity)
assert.Nil(err)
assert.EqInt(2, len(got))
assert.EqStr("1", got[0].ID)
assert.Nil(got[1])
assert.OK(driver.Exists("ent.TestEntity.1"))
assert.OK(driver.Exists("ent.TestEntity.3"))
var t1, t3 TEntity
driver.Get("ent.TestEntity.1", &t1)
driver.Get("ent.TestEntity.3", &t3)
assert.EqStr("1", t1.ID)
assert.EqStr("", t3.ID)
got = make([]*TEntity, 2)
driver.GetMulti([]string{"ent.TestEntity.1", "ent.TestEntity.3"}, got)
assert.EqStr("1", got[0].ID)
assert.Nil(got[1])
}
// FixtureFromMap is to load fixtures from []]map[string]interface{}
func FixtureFromMap(ctx context.Context, arr []map[string]interface{}) error {
logger := wcg.NewLogger(nil)
for _, v := range arr {
var vv map[string]interface{}
buff, _ := json.Marshal(v)
json.Unmarshal(buff, vv)
if err := loadJsonToDatastore(ctx, nil, v, logger); err != nil {
return err
}
}
return nil
}
func TestDatastoreFixture(t *testing.T) {
filepath := mkTempfile(`[{
"_kind": "FixtureKind",
"_key": "key1",
"IntValue": 10,
"FloatValue": 2.4,
"BoolValue": true,
"StringValue": "foobar",
"BytesValue": "[]bytesfoobar",
"DateTimeValue": "2014-01-02T14:02:50Z",
"DateValue": "2014-01-02",
"Slice": ["a", "b", "c"],
"Struct": {
"Foo": "bar"
}
},{
"_kind": "FixtureKind",
"_key": "key1",
"_ns": "ns1",
"StringValue": "withns1"
}
]`)
assert := wcg.NewAssert(t)
var fk FixtureKind
assert.Nil(DatastoreFixture(ts.Context, filepath, nil), "DatastoreFixture")
key := datastore.NewKey(ts.Context, "FixtureKind", "key1", 0, nil)
wcg.NewLogger(nil).Infof("GET: %s", key)
assert.Nil(datastore.Get(ts.Context, key, &fk), "datastore.Get('key1') ")
assert.EqInt(10, fk.IntValue, "IntValue should be 10")
assert.EqFloat32(2.4, fk.FloatValue, "FloatValue should be 2.4")
assert.EqStr("foobar", fk.StringValue, "StringValue should be 'foobar'")
assert.EqStr("bytesfoobar", string(fk.BytesValue), "BytesValue should be 'foobar'")
assert.EqInt(3, len(fk.Slice), "len(Slice) should be 3")
assert.EqStr("a", string(fk.Slice[0]), "Slice[0] should be 'a'")
assert.EqStr("b", string(fk.Slice[1]), "Slice[0] should be 'a'")
assert.EqStr("c", string(fk.Slice[2]), "Slice[0] should be 'a'")
assert.EqTime(time.Date(2014, 01, 02, 14, 02, 50, 0, time.UTC), fk.DateTimeValue, "DateTimeValue should be 2014-01-02T14:02:50Z")
assert.EqTime(time.Date(2014, 01, 02, 0, 0, 0, 0, time.UTC), fk.DateValue, "DateTimeValue should be 2014-01-02T00:00:00Z")
assert.EqStr("bar", string(fk.Struct.Foo), "Struct.Foo should be 'bar'")
// namespace
ns1, _ := appengine.Namespace(ts.Context, "ns1")
key = datastore.NewKey(ns1, "FixtureKind", "key1", 0, nil)
assert.Nil(datastore.Get(ns1, key, &fk), "datastore.Get('key1') /w ns1")
assert.EqStr("withns1", fk.StringValue, "StringValue should be 'withns1'")
}
// DatastoreFixture loads the fixtures from path to datastore.
func DatastoreFixture(ctx context.Context, path string, bindings interface{}) error {
logger := wcg.NewLogger(nil)
logger.Debugf("[Fixture] load from %s", path)
data, err := loadFile(path, bindings)
if err != nil {
return fmt.Errorf("Could not load fixture file from %s: %v", err)
}
var arr []map[string]interface{}
if err = json.Unmarshal(data, &arr); err != nil {
return fmt.Errorf("Could not load the json file from %q - JSON Parse error: %v", path, err)
}
for _, v := range arr {
if err := loadJsonToDatastore(ctx, nil, v, logger); err != nil {
return err
}
}
return nil
}
func InitLog(configPath string) {
logsink.UseFileConfig()
wcg.ConfigureProcess(configPath)
var logger = wcg.NewLogger(nil)
if wcg.DefaultLogConfig.NumSinks() == 0 {
wcg.DefaultLogConfig.AddSink(
logsink.NewIOLogSink(
"",
os.Stderr,
true,
),
wcg.DefaultLogConfig.Level,
)
logger.Warnf("No LogSink is specified, fail back to stderr")
}
// normalize path
logger.Infof("Used configuration file: %q", configPath)
logger.Infof("Target Endpoint: %q", DefaultConfig.Endpoint)
logger.Debugf("Token: %q", DefaultConfig.Token)
}
func RunAgent(agent async.Agent) {
var logger = wcg.NewLogger(nil)
var s os.Signal
var err error
go func() {
err = agent.Start()
if err != nil {
logger.Errorf("agent.Start() returns an error: %#v", err)
}
}()
defer func() {
logger.Infof("Got a signal %d, going to exit.", s)
agent.Stop()
logger.Infof("Instance stopped. Exiting.")
}()
// block the main until receiving the signal
logger.Infof("An agent process started")
logger.Infof("Press ctrl-c or `kill %d` to stop the agent", os.Getpid())
ch := make(chan os.Signal, 1)
signal.Notify(ch, syscall.SIGTERM, syscall.SIGINT)
s = <-ch
}
func Test_Put_CacheInvalidation(t *testing.T) {
assert := gaetest.NewAssert(t)
driver := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
gaetest.CleanupDatastore(ts.Context)
gaetest.FixtureFromMap(ts.Context, []map[string]interface{}{
map[string]interface{}{"_kind": testEntity.String(), "_key": "1", "ID": "1"},
})
// create caches
testEntity.Query().Cache("123").MustExecute(ts.GET("/").Request)
testEntity.Get().Key("1").Cache(true).MustOne(ts.GET("/").Request)
assert.OK(driver.Exists("ent.TestEntity.1"), "An entyty cache should exist")
assert.OK(driver.Exists("query.TestEntity.123"), "A query cache should exist")
item := &TEntity{ID: "1", Digit: 30}
testEntity.Put().Cache("123").MustUpdate(ts.PUTForm("/", nil).Request, item)
assert.Not(driver.Exists("ent.TestEntity.1"), "Entity cache should be invalidated")
assert.Not(driver.Exists("query.TestEntity.123"), "Query cache should be invalidated")
}