// Increment increments the named counter.
func Increment(ctx context.Context, name string) error {
// Get counter config.
var cfg counterConfig
ckey := datastore.NewKey(ctx, configKind, name, 0, nil)
err := datastore.RunInTransaction(ctx, func(ctx context.Context) error {
err := datastore.Get(ctx, ckey, &cfg)
if err == datastore.ErrNoSuchEntity {
cfg.Shards = defaultShards
_, err = datastore.Put(ctx, ckey, &cfg)
}
return err
}, nil)
if err != nil {
return err
}
var s shard
err = datastore.RunInTransaction(ctx, func(ctx context.Context) error {
shardName := fmt.Sprintf("%s-shard%d", name, rand.Intn(cfg.Shards))
key := datastore.NewKey(ctx, shardKind, shardName, 0, nil)
err := datastore.Get(ctx, key, &s)
// A missing entity and a present entity will both work.
if err != nil && err != datastore.ErrNoSuchEntity {
return err
}
s.Name = name
s.Count++
_, err = datastore.Put(ctx, key, &s)
return err
}, nil)
if err != nil {
return err
}
memcache.IncrementExisting(ctx, memcacheKey(name), 1)
return nil
}
// Increment increments the named counter.
func Increment(c context.Context, valName string) error {
// Get counter config.
shardsTotal := dsu.WrapInt{}
dsu.McacheGet(c, mcKeyShardsTotal(valName), &shardsTotal)
if shardsTotal.I < 1 {
ckey := datastore.NewKey(c, dsKindNumShards, mcKeyShardsTotal(valName), 0, nil)
errTx := datastore.RunInTransaction(c,
func(c context.Context) error {
err := datastore.Get(c, ckey, &shardsTotal)
if err == datastore.ErrNoSuchEntity {
shardsTotal.I = defaultNumShards
_, err = datastore.Put(c, ckey, &shardsTotal)
}
return err
}, nil)
if errTx != nil {
return errTx
}
dsu.McacheSet(c, mcKeyShardsTotal(valName), dsu.WrapInt{shardsTotal.I})
}
// pick random counter and increment it
errTx := datastore.RunInTransaction(c,
func(c context.Context) error {
shardId := rand.Intn(shardsTotal.I)
dsKey := datastore.NewKey(c, dsKindShard, keySingleShard(valName, shardId), 0, nil)
var sd WrapShardData
err := datastore.Get(c, dsKey, &sd)
// A missing entity and a present entity will both work.
if err != nil && err != datastore.ErrNoSuchEntity {
return err
}
sd.Name = valName
sd.ShardId = shardId
sd.I++
_, err = datastore.Put(c, dsKey, &sd)
if ll > 2 {
aelog.Infof(c, "ds put %v %v", dsKey, sd)
}
return err
}, nil)
if errTx != nil {
return errTx
}
memcache.Increment(c, mcKey(valName), 1, 0)
// collect number of updates
// per valName per instance in memory
// for every interval of 10 minutes
//
// a batch job checks if the number of shards should be increased or decreased
// and truncates this map
updateSamplingFrequency[valName+util.TimeMarker()[:len("2006-01-02 15:0")]] += 1
return nil
}
func incrementCounter(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
c := appengine.NewContext(r)
key := datastore.NewKey(c, "Counter", "TheCounter", 0, nil)
var result Counter
err := datastore.RunInTransaction(c, func(c context.Context) error {
var counter Counter
err := datastore.Get(c, key, &counter)
if err == datastore.ErrNoSuchEntity {
log.Debugf(c, "No entity, creating a new one.")
} else if err != nil {
log.Errorf(c, "Error getting counter. %v", err)
return err
} else {
}
counter.Value += 1
result = counter
_, err = datastore.Put(c, key, &result)
return err
}, nil)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
w.WriteHeader(http.StatusOK)
w.Write([]byte(strconv.FormatInt(result.Value, 10)))
}
// Remove safely deletes an account and all its associated information in the datastore. This includes
// any objects that are descendants of the Account (i.e., a cascading delete).
func Remove(ctx context.Context, account *Account) error {
return datastore.RunInTransaction(ctx, func(txCtx context.Context) error {
acctKey := account.Key(txCtx)
q := datastore.NewQuery("").
Ancestor(acctKey).
KeysOnly()
if changed, err := HasChanged(txCtx, account); err != nil {
return err
} else if changed {
return ErrConflict
}
keys, err := q.GetAll(txCtx, nil)
if err != nil {
return err
}
keys = append(keys, acctKey)
return nds.DeleteMulti(txCtx, keys)
}, nil)
}
func incrementSoldCounter(c context.Context) error {
_, err := memcache.IncrementExisting(c, "sold", 1)
if err != nil {
if err == datastore.ErrNoSuchEntity {
log.Infof(c, "[counter] Cache miss when incrementing")
} else {
return err
}
}
sold := new(Counter)
err = datastore.RunInTransaction(c, func(c context.Context) error {
key := datastore.NewKey(c, "Counter", "sold", 0, nil)
err := datastore.Get(c, key, sold)
if err != nil && err != datastore.ErrNoSuchEntity {
return err
}
sold.Value++
_, err = datastore.Put(c, key, sold)
if err != nil {
sold.Value--
return err
}
return nil
}, nil)
return err
}
//WriteLinkTweet writes a given Tweet to the datastore
func (mb MapBuilder) writeLinkTweet(tweets <-chan anaconda.Tweet, wg *sync.WaitGroup) {
defer wg.Done()
var keys []*datastore.Key
var values []*int64
for tweet := range tweets {
key := datastore.NewIncompleteKey(mb.c, linkTweetKind, getTweetKey(mb.c))
keys = append(keys, key)
values = append(values, &tweet.Id)
}
err := datastore.RunInTransaction(mb.c, func(c context.Context) error {
_, err := datastore.PutMulti(c, keys, values)
if err != nil {
log.Errorf(c, "Failed to write LinkTweet to datastore. %v", err.Error())
return err
}
return nil
}, nil)
if err != nil {
log.Errorf(mb.c, "Failed to write LinkTweet to datastore. %v", err.Error())
}
}
// initRepoData is called to declare a new active repository in the
// datastore. It should run after the repo has been verified to work.
func initRepoData(c context.Context, user, repo, token string) error {
item := repoStorageData{
User: user,
Repo: repo,
Token: token,
Status: statusValidating,
}
key := makeRepoKey(c, user, repo)
return datastore.RunInTransaction(c, func(c context.Context) error {
var currentItem repoStorageData
err := datastore.Get(c, key, ¤tItem)
if err != datastore.ErrNoSuchEntity {
if err != nil {
return err
}
return &repoExistsError{
User: user,
Repo: repo,
}
}
_, err = datastore.Put(c, key, &item)
return err
}, &datastore.TransactionOptions{})
}
func newPlayer(w http.ResponseWriter, r *http.Request) {
ctx := appengine.NewContext(r)
id := r.FormValue("id")
if id == "" {
http.Error(w, "need a non-blank id from the form data", http.StatusBadRequest)
return
}
k := datastore.NewKey(ctx, "Player", id, 0, nil)
err := datastore.RunInTransaction(ctx, func(ctx context.Context) error {
var p *Player
if err := datastore.Get(ctx, k, p); err != datastore.ErrNoSuchEntity {
return codeErr{fmt.Errorf("player(%s) already exists", id), http.StatusConflict}
}
// TODO(jaguilar): Add another system. Add a system Registry. Leagues
// can have ratings.
p = &Player{Rating: eloSys.InitialRating(), key: k}
if _, err := datastore.Put(ctx, p.key, p); err != nil {
return err
}
return nil
}, nil)
if err != nil {
httpError(w, err)
return
}
fmt.Fprintf(
w,
`<html><head><meta http-equiv="refresh" content="5,/"></head>
<body>Successfully added %s. Redirecting you in five seconds.</body></html>`,
id)
}
func TestGetLockAlreadyLocked(t *testing.T) {
r, _ := instance.NewRequest("GET", "/", nil)
r.Header.Set("X-AppEngine-Request-Log-Id", "locked")
c := appengine.NewContext(r)
getTime = getTimeDefault
f := &Foo{
Value: "test",
Lock: Lock{
Timestamp: getTime(),
RequestID: "previous",
Sequence: 1,
Retries: 0,
},
}
k := datastore.NewKey(c, "foo", "", 1, nil)
datastore.RunInTransaction(c, func(c context.Context) error {
if _, err := datastore.Put(c, k, f); err != nil {
return err
}
return nil
}, nil)
l, _ := NewLocker()
err := l.Aquire(c, k, f, 1)
if err != ErrLockFailed {
t.Errorf("expected failed lock, got %v", err)
}
datastore.Get(c, k, f)
t.Logf("%v", f)
}
// [START uses_for_transactions_1]
func increment(ctx context.Context, key *datastore.Key) error {
return datastore.RunInTransaction(ctx, func(ctx context.Context) error {
count := new(Counter)
if err := datastore.Get(ctx, key, count); err != nil {
return err
}
count.Count++
_, err := datastore.Put(ctx, key, count)
return err
}, nil)
}
// RunInTransaction works just like datastore.RunInTransaction however it
// interacts correctly with memcache. You should always use this method for
// transactions if you are using the NDS package.
func RunInTransaction(c context.Context, f func(tc context.Context) error,
opts *datastore.TransactionOptions) error {
return datastore.RunInTransaction(c, func(tc context.Context) error {
tx := &transaction{}
tc = context.WithValue(tc, &transactionKey, tx)
if err := f(tc); err != nil {
return err
}
return memcacheSetMulti(tc, tx.lockMemcacheItems)
}, opts)
}
// BecomeReplica make the database become the replica.
func BecomeReplica(c context.Context, rs AuthReplicationState) error {
if rs.AuthDBRev != 0 {
return fmt.Errorf("should not have AuthDBRev")
}
if len(rs.PrimaryID) == 0 || len(rs.PrimaryURL) == 0 {
return fmt.Errorf("wrong primary ID (%v) or primary url (%v)", rs.PrimaryID, rs.PrimaryURL)
}
return datastore.RunInTransaction(c, func(c context.Context) error {
_, err := datastore.Put(c, ReplicationStateKey(c), &rs)
return err
}, nil)
}
// updateUserInfo updates information about the currently logged in user.
func (c *context) updateUserInfo(f func(u *userInfo)) error {
key := datastore.NewKey(c.c, "user", c.u.Email, 0, nil)
return datastore.RunInTransaction(c.c, func(ctx appengine.Context) error {
var u userInfo
err := datastore.Get(ctx, key, &u)
if err != nil && err != datastore.ErrNoSuchEntity {
return err
}
f(&u)
_, err = datastore.Put(ctx, key, &u)
return err
}, nil)
}
func example() {
var ctx context.Context
// [START transactional_task_enqueuing]
datastore.RunInTransaction(ctx, func(ctx context.Context) error {
t := &taskqueue.Task{Path: "/path/to/worker"}
if _, err := taskqueue.Add(ctx, t, ""); err != nil {
return err
}
// ...
return nil
}, nil)
// [END transactional_task_enqueuing]
}
func incrementCounter(ctx context.Context, name string) error {
key := datastore.NewKey(ctx, "Counter", name, 0, nil)
return datastore.RunInTransaction(ctx, func(ctx context.Context) error {
var ctr Counter
err := datastore.Get(ctx, key, &ctr)
if err != nil && err != datastore.ErrNoSuchEntity {
return err
}
ctr.Count++
_, err = datastore.Put(ctx, key, &ctr)
return err
}, nil)
}
func GetOrUpdate(ctx context.Context, id, addr, phone string) error {
key := datastore.NewKey(ctx, "Account", id, 0, nil)
return datastore.RunInTransaction(ctx, func(ctx context.Context) error {
acct := new(Account)
err := datastore.Get(ctx, key, acct)
if err != nil && err != datastore.ErrNoSuchEntity {
return err
}
acct.Address = addr
acct.Phone = phone
_, err = datastore.Put(ctx, key, acct)
return err
}, nil)
}
// Save changes the application configuration to
// the values in conf. All HTTP requests subsequent to this one
// are guaranteed to use the new values in their configuration.
//
// Note that subsequent calls to Get with the same request context
// will continue to retrieve the old version of the configuration.
//
// As a special case, calling Save with a *config.Config will replace
// the entire contents of the configuration with the contents of Config.
func Save(ctx context.Context, conf interface{}) error {
if typedConfig, ok := conf.(*Config); ok {
pl := datastore.PropertyList(*typedConfig)
replaceKey := datastore.NewKey(ctx, Entity, Entity, 0, nil)
_, replaceErr := nds.Put(ctx, replaceKey, &pl)
return replaceErr
}
return datastore.RunInTransaction(ctx, func(txCtx context.Context) error {
props := datastore.PropertyList{}
key := datastore.NewKey(txCtx, Entity, Entity, 0, nil)
if err := nds.Get(txCtx, key, &props); err != nil && err != datastore.ErrNoSuchEntity {
return err
}
// merge existing config with the new values
if newProps, err := datastore.SaveStruct(conf); err != nil {
return err
} else {
for _, newProp := range newProps {
newProp.NoIndex = true
replacing := false
for _, prop := range props {
// make sure NoIndex is set
prop.NoIndex = true
if prop.Name == newProp.Name {
replacing = true
prop.Value = newProp.Value
break
}
}
if !replacing {
// append
props = append(props, newProp)
}
}
}
_, err := nds.Put(txCtx, key, &props)
return err
}, nil)
}
// Increment increments the counter.
func Increment(ctx context.Context) error {
return datastore.RunInTransaction(ctx, func(ctx context.Context) error {
shardName := fmt.Sprintf("shard%d", rand.Intn(numShards))
key := datastore.NewKey(ctx, shardKind, shardName, 0, nil)
var s simpleCounterShard
err := datastore.Get(ctx, key, &s)
// A missing entity and a present entity will both work.
if err != nil && err != datastore.ErrNoSuchEntity {
return err
}
s.Count++
_, err = datastore.Put(ctx, key, &s)
return err
}, nil)
}
func (this *MainController) Delete() {
err := datastore.RunInTransaction(this.AppEngineCtx, func(c context.Context) error {
ks, err := datastore.NewQuery("Todo").KeysOnly().Ancestor(models.DefaultTodoList(c)).Filter("Done=", true).GetAll(c, nil)
if err != nil {
return err
}
return datastore.DeleteMulti(c, ks)
}, nil)
if err == nil {
this.Data["json"] = nil
} else {
this.Data["json"] = err
}
}
// SetFavorite changes the favorite status of a post given its UID.
func (PostsAPI) SetFavorite(c context.Context, r *SetFavoriteRequest) error {
if err := checkReferer(c); err != nil {
return err
}
return datastore.RunInTransaction(c, func(c context.Context) error {
var p Post
if err := datastore.Get(c, r.UID, &p); err == datastore.ErrNoSuchEntity {
return endpoints.NewNotFoundError("post not found")
} else if err != nil {
return err
}
p.Favorite = r.Favorite
_, err := datastore.Put(c, r.UID, &p)
return err
}, nil)
}
func result(w http.ResponseWriter, r *http.Request) {
ctx := appengine.NewContext(r)
p1id, p2id, outcome := r.FormValue("p1"), r.FormValue("p2"), rating.ParseWLD(r.FormValue("outcome"))
if p1id == "" || p2id == "" {
http.Error(w, "must specify ids for both players", http.StatusBadRequest)
return
}
if outcome == rating.UnknownOutcome {
http.Error(w, "unrecognized outcome: "+r.FormValue("outcome"), http.StatusBadRequest)
return
}
var p1, p2 Player
p1.key = datastore.NewKey(ctx, "Player", p1id, 0, nil)
p2.key = datastore.NewKey(ctx, "Player", p2id, 0, nil)
err := datastore.RunInTransaction(ctx, func(ctx context.Context) error {
if err := datastore.Get(ctx, p1.key, &p1); err != nil {
return fmt.Errorf("get %v: %v", p1, err)
}
if err := datastore.Get(ctx, p2.key, &p2); err != nil {
return fmt.Errorf("get %v: %v", p2, err)
}
p1.Rating = eloSys.Update(p1.Rating, p2.Rating, outcome)
p2.Rating = eloSys.Update(p2.Rating, p1.Rating, outcome.Opposite())
for _, p := range []*Player{&p1, &p2} {
if _, err := datastore.Put(ctx, p.key, p); err != nil {
return err
}
}
return nil
}, &datastore.TransactionOptions{XG: true})
if err != nil {
httpError(w, err)
return
}
fmt.Fprintf(w, `
<html><body>
<p>Update complete.<p>%s's new elo: %f<p>%s's new elo: %f
<p><a href="/">Go back</a></body></html>`, p1.key.StringID(), p1.Rating, p2.key.StringID(), p2.Rating)
}
// delete world data
func (a *MinecraftApi) Delete(w http.ResponseWriter, r *http.Request) {
ctx := appengine.NewContext(r)
u := user.Current(ctx)
if u == nil {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(http.StatusUnauthorized)
loginURL, err := user.LoginURL(ctx, "")
if err != nil {
log.Errorf(ctx, "get user login URL error, %s", err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
w.Write([]byte(fmt.Sprintf(`{"loginURL":"%s"}`, loginURL)))
return
}
if user.IsAdmin(ctx) == false {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(http.StatusForbidden)
return
}
keyStr := r.FormValue("key")
key, err := datastore.DecodeKey(keyStr)
if err != nil {
log.Infof(ctx, "invalid key, %v", r.Body)
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(`{"message": "invalid key."}`))
return
}
err = datastore.RunInTransaction(ctx, func(c context.Context) error {
return datastore.Delete(ctx, key)
}, nil)
if err != nil {
log.Errorf(ctx, "Minecraft Delete Error. error = %s", err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(`{}`)
}
func f(ctx context.Context) {
err := datastore.RunInTransaction(ctx, func(ctx context.Context) error {
t := taskqueue.NewPOSTTask("/worker", url.Values{
// ...
})
// Use the transaction's context when invoking taskqueue.Add.
_, err := taskqueue.Add(ctx, t, "")
if err != nil {
// Handle error
}
// ...
return nil
}, nil)
if err != nil {
// Handle error
}
// ...
}
func modifyRepoData(c context.Context, user, repo string, f func(*repoStorageData)) error {
return datastore.RunInTransaction(c, func(c context.Context) error {
key := makeRepoKey(c, user, repo)
var item repoStorageData
err := datastore.Get(c, key, &item)
if err != nil {
return err
}
f(&item)
_, err = datastore.Put(c, key, &item)
return err
}, &datastore.TransactionOptions{})
}
// UpdateOverviewerSnapshot is Overviewerを作成したSnapshotのVersionを更新する
func (m *Minecraft) UpdateOverviewerSnapshot(ctx context.Context, key *datastore.Key) error {
return datastore.RunInTransaction(ctx, func(c context.Context) error {
var entity Minecraft
err := datastore.Get(ctx, key, &entity)
if err != datastore.ErrNoSuchEntity && err != nil {
return err
}
entity.OverviewerSnapshot = entity.LatestSnapshot
entity.UpdatedAt = time.Now()
_, err = datastore.Put(ctx, key, &entity)
if err != nil {
return err
}
return nil
}, nil)
}
func (r *record) update(c context.Context) error {
return datastore.RunInTransaction(c,
func(tc context.Context) error {
r1, err := find(c, r.Domain)
if err != nil {
return err
}
if r.Token != r1.Token {
return tokenError
}
return r.insert(c)
},
&datastore.TransactionOptions{XG: false})
}
// RunInTransaction works just like datastore.RunInTransaction however it
// interacts correctly with memcache. You should always use this method for
// transactions if you are using the NDS package.
func RunInTransaction(c context.Context, f func(tc context.Context) error,
opts *datastore.TransactionOptions) error {
return datastore.RunInTransaction(c, func(tc context.Context) error {
tx := &transaction{}
tc = context.WithValue(tc, &transactionKey, tx)
if err := f(tc); err != nil {
return err
}
// tx.Unlock() is not called as the tx context should never be called
//again so we rather block than allow people to misuse the context.
tx.Lock()
memcacheCtx, err := memcacheContext(tc)
if err != nil {
return err
}
return memcacheSetMulti(memcacheCtx, tx.lockMemcacheItems)
}, opts)
}
func initHandler(w http.ResponseWriter, r *http.Request) {
var fileRoot struct {
Root string
}
c := appengine.NewContext(r)
k := rootKey(c)
err := datastore.RunInTransaction(c, func(c context.Context) error {
err := datastore.Get(c, k, &fileRoot)
if err != nil && err != datastore.ErrNoSuchEntity {
return err
}
_, err = datastore.Put(c, k, &fileRoot)
return err
}, nil)
if err != nil {
http.Error(w, err.Error(), 500)
return
}
io.WriteString(w, "OK")
}
func TestGetLockAvailable(t *testing.T) {
r, _ := instance.NewRequest("GET", "/", nil)
r.Header.Set("X-AppEngine-Request-Log-Id", "locked")
c := appengine.NewContext(r)
f := &Foo{
Value: "test",
Lock: Lock{
Timestamp: time.Date(2016, 7, 21, 11, 10, 0, 0, time.UTC),
RequestID: "",
Sequence: 1,
Retries: 0,
},
}
k := datastore.NewKey(c, "foo", "", 1, nil)
datastore.RunInTransaction(c, func(c context.Context) error {
if _, err := datastore.Put(c, k, f); err != nil {
return err
}
return nil
}, nil)
n := time.Date(2016, 7, 21, 11, 15, 0, 0, time.UTC)
getTime = func() time.Time {
return n
}
l, _ := NewLocker()
err := l.Aquire(c, k, f, 1)
if err != nil {
t.Errorf("failed to lock %v", err)
}
datastore.Get(c, k, f)
if f.RequestID != "locked" {
t.Errorf("failed to set request id")
}
if f.Timestamp.UTC() != n {
t.Errorf("failed to set request timestamp %s %s", n, f.Timestamp)
}
}
// IncreaseShards increases the number of shards for the named counter to n.
// It will never decrease the number of shards.
func IncreaseShards(c context.Context, name string, n int) error {
ckey := datastore.NewKey(c, configKind, name, 0, nil)
return datastore.RunInTransaction(c, func(c context.Context) error {
var cfg counterConfig
mod := false
err := datastore.Get(c, ckey, &cfg)
if err == datastore.ErrNoSuchEntity {
cfg.Shards = defaultShards
mod = true
} else if err != nil {
return err
}
if cfg.Shards < n {
cfg.Shards = n
mod = true
}
if mod {
_, err = datastore.Put(c, ckey, &cfg)
}
return err
}, nil)
}
