func ExampleNewKey_withParent() {
ctx := context.Background()
// [START key_with_parent]
parentKey := datastore.NewKey(ctx, "TaskList", "default", 0, nil)
taskKey := datastore.NewKey(ctx, "Task", "sampleTask", 0, parentKey)
// [END key_with_parent]
_ = taskKey // Use the task key for datastore operations.
}
func ExampleNewKey_withMultipleParents() {
ctx := context.Background()
// [START key_with_multilevel_parent]
userKey := datastore.NewKey(ctx, "User", "alice", 0, nil)
parentKey := datastore.NewKey(ctx, "TaskList", "default", 0, userKey)
taskKey := datastore.NewKey(ctx, "Task", "sampleTask", 0, parentKey)
// [END key_with_multilevel_parent]
_ = taskKey // Use the task key for datastore operations.
}
func ExampleGetMulti() {
ctx := Example_auth()
keys := []*datastore.Key{
datastore.NewKey(ctx, "Post", "post1", 0, nil),
datastore.NewKey(ctx, "Post", "post2", 0, nil),
datastore.NewKey(ctx, "Post", "post3", 0, nil),
}
posts := make([]Post, 3)
if err := datastore.GetMulti(ctx, keys, posts); err != nil {
log.Println(err)
}
}
func pollCommits(dir string) {
cmd := execGit(dir, nil, "pull", "origin")
out, err := cmd.CombinedOutput()
if err != nil {
log.Printf("Error running git pull origin master in %s: %v\n%s", dir, err, out)
return
}
log.Printf("Ran git pull.")
// TODO: see if .git/refs/remotes/origin/master
// changed. (quicker than running recentCommits each time)
hashes, err := recentCommits(dir)
if err != nil {
log.Print(err)
return
}
latestHash.Lock()
latestHash.s = hashes[0]
latestHash.Unlock()
for _, commit := range hashes {
if knownCommit[commit] {
continue
}
if dsClient != nil {
ctx := context.Background()
key := datastore.NewKey(ctx, "git_commit", commit, 0, nil)
var gc GitCommit
if err := dsClient.Get(ctx, key, &gc); err == nil && gc.Emailed {
log.Printf("Already emailed about commit %v; skipping", commit)
knownCommit[commit] = true
continue
}
}
if err := emailCommit(dir, commit); err == nil {
log.Printf("Emailed commit %s", commit)
knownCommit[commit] = true
if dsClient != nil {
ctx := context.Background()
key := datastore.NewKey(ctx, "git_commit", commit, 0, nil)
_, err := dsClient.Put(ctx, key, &GitCommit{Emailed: true})
log.Printf("datastore put of git_commit(%v): %v", commit, err)
}
}
}
if githubSSHKey != "" {
if err := syncToGithub(dir, hashes[0]); err != nil {
log.Printf("Failed to push commit %v to github: %v", hashes[0], err)
}
}
}
func (txio *DBTransactionLogIO) Sync() error {
txio.mu.Lock()
batch := txio.nextbatch
txio.nextbatch = make([]inodedb.DBTransaction, 0)
txio.mu.Unlock()
if len(batch) == 0 {
return nil
}
ctx := txio.getContext()
keys := make([]*datastore.Key, 0, len(batch))
stxs := make([]*storedbtx, 0, len(batch))
for _, tx := range batch {
keys = append(keys, datastore.NewKey(ctx, kindTransaction, "", int64(tx.TxID), txio.rootKey))
stx, err := encode(txio.c, tx)
if err != nil {
return err
}
stxs = append(stxs, stx)
}
if _, err := datastore.PutMulti(ctx, keys, stxs); err != nil {
return err
}
log.Printf("Committed %d txs", len(stxs))
return nil
}
func ExampleTransaction() {
ctx := Example_auth()
const retries = 3
// Increment a counter.
// See https://cloud.google.com/appengine/articles/sharding_counters for
// a more scalable solution.
type Counter struct {
Count int
}
key := datastore.NewKey(ctx, "counter", "CounterA", 0, nil)
for i := 0; i < retries; i++ {
tx, err := datastore.NewTransaction(ctx)
if err != nil {
break
}
var c Counter
if err := tx.Get(key, &c); err != nil && err != datastore.ErrNoSuchEntity {
break
}
c.Count++
if _, err := tx.Put(key, &c); err != nil {
break
}
// Attempt to commit the transaction. If there's a conflict, try again.
if _, err := tx.Commit(); err != datastore.ErrConcurrentTransaction {
break
}
}
}
func (fe *FeServiceImpl) getBiosphereTopo(ctx context.Context, biosphereId uint64) (BiosphereTopology, *api.BiosphereEnvConfig, error, *api.TimingTrace) {
trace := InitTrace("getBiosphereTopo")
fe.bsMetaCacheMutex.Lock()
defer fe.bsMetaCacheMutex.Unlock()
bsMeta, ok := fe.bsMetaCache[biosphereId]
if !ok {
authTrace := InitTrace("AuthDatastore")
client, err := fe.AuthDatastore(ctx)
if err != nil {
return nil, nil, err, trace
}
FinishTrace(authTrace, trace)
key := datastore.NewKey(ctx, "BiosphereMeta", "", int64(biosphereId), nil)
var meta BiosphereMeta
err = client.Get(ctx, key, &meta)
if err != nil {
return nil, nil, err, trace
}
fe.bsMetaCache[biosphereId] = &meta
bsMeta = &meta
log.Printf("BiosphereMeta cache entry (bsId: %d) created. Now #(cache entry)=%d", biosphereId, len(fe.bsMetaCache))
}
log.Printf("Found config of %d x %d", bsMeta.Nx, bsMeta.Ny)
envConfig := api.BiosphereEnvConfig{}
err := proto.Unmarshal(bsMeta.Env, &envConfig)
if err != nil {
return nil, nil, err, trace
}
return NewCylinderTopology(biosphereId, int(bsMeta.Nx), int(bsMeta.Ny)), &envConfig, nil, trace
}
func ExampleNewKey() {
ctx := context.Background()
// [START named_key]
taskKey := datastore.NewKey(ctx, "Task", "sampletask", 0, nil)
// [END named_key]
_ = taskKey // Use the task key for datastore operations.
}
func (ck *CkServiceImpl) fetchDatastoreSnapshot(dsKey int64) *api.ChunkState {
strictCtx, _ := context.WithTimeout(context.Background(), 2500*time.Millisecond)
client, err := ck.AuthDatastore(strictCtx)
if err != nil {
log.Printf("ERROR datastore auth failed %#v", err)
return nil
}
key := datastore.NewKey(strictCtx, "PersistentChunkSnapshot", "", dsKey, nil)
snapshot := new(PersistentChunkSnapshot)
err = client.Get(strictCtx, key, snapshot)
if err != nil {
log.Printf("ERROR datastore.Get(%#v) failed %#v", key, err)
return nil
}
snapshotProto := &api.ChunkSnapshot{}
err = proto.Unmarshal(snapshot.Snapshot, snapshotProto)
if err != nil {
log.Printf("ERROR failed to unmarshal; corrupt datastore entry at key %d %#v", dsKey, err)
return nil
}
// Convert to a shard w/o any outgoing grains.
return &api.ChunkState{
Shards: []*api.ChunkShard{
&api.ChunkShard{
Dp: &api.ChunkRel{Dx: 0, Dy: 0},
Grains: snapshotProto.Grains,
},
},
}
}
func (ds *Datastore) GetEntity(t, subject string) (reply []x.Instruction, rerr error) {
skey := datastore.NewKey(ds.ctx, t+"Entity", subject, 0, nil)
log.Infof("skey: %+v", skey)
dkeys, rerr := datastore.NewQuery(t+"Instruction").Ancestor(skey).GetAll(ds.ctx, &reply)
log.Debugf("Got num keys: %+v", len(dkeys))
return
}
func NewDBTransactionLogIO(cfg *Config) *DBTransactionLogIO {
return &DBTransactionLogIO{
cfg: cfg,
rootKey: datastore.NewKey(ctxNoNamespace, kindTransaction, cfg.rootKeyStr, 0, nil),
nextbatch: make([]inodedb.DBTransaction, 0),
}
}
func (loc *INodeDBSSLocator) tryPutOnce(blobpath string, txid int64) error {
start := time.Now()
e := sslocentry{BlobPath: blobpath, TxID: txid, CreatedAt: start}
cli, err := loc.cfg.getClient(context.TODO())
if err != nil {
return err
}
dstx, err := cli.NewTransaction(context.TODO(), datastore.Serializable)
if err != nil {
return err
}
key := datastore.NewKey(ctxNoNamespace, kindINodeDBSS, "", int64(e.TxID), loc.rootKey)
if _, err := dstx.Put(key, &e); err != nil {
dstx.Rollback()
return err
}
if _, err := dstx.Commit(); err != nil {
return err
}
logger.Infof(sslog, "Put(%s, %d) took %s.", blobpath, txid, time.Since(start))
return nil
}
func ExamplePut() {
ctx := context.Background()
client, err := datastore.NewClient(ctx, "project-id")
if err != nil {
log.Fatal(err)
}
type Article struct {
Title string
Description string
Body string `datastore:",noindex"`
Author *datastore.Key
PublishedAt time.Time
}
newKey := datastore.NewIncompleteKey(ctx, "Article", nil)
_, err = client.Put(ctx, newKey, &Article{
Title: "The title of the article",
Description: "The description of the article...",
Body: "...",
Author: datastore.NewKey(ctx, "Author", "jbd", 0, nil),
PublishedAt: time.Now(),
})
if err != nil {
log.Fatal(err)
}
}
func ExampleGetMulti() {
ctx := context.Background()
client, err := datastore.NewClient(ctx, "project-id")
if err != nil {
log.Fatal(err)
}
keys := []*datastore.Key{
datastore.NewKey(ctx, "Post", "post1", 0, nil),
datastore.NewKey(ctx, "Post", "post2", 0, nil),
datastore.NewKey(ctx, "Post", "post3", 0, nil),
}
posts := make([]Post, 3)
if err := client.GetMulti(ctx, keys, posts); err != nil {
log.Println(err)
}
}
func ExampleQuery_keyFilter() {
ctx := context.Background()
// [START key_filter]
key := datastore.NewKey(ctx, "Task", "someTask", 0, nil)
query := datastore.NewQuery("Task").Filter("__key__ >", key)
// [END key_filter]
_ = query // Use client.Run or client.GetAll to execute the query.
}
func ExampleQuery_EventualConsistency() {
ctx := context.Background()
// [START eventual_consistent_query]
ancestor := datastore.NewKey(ctx, "TaskList", "default", 0, nil)
query := datastore.NewQuery("Task").Ancestor(ancestor).EventualConsistency()
// [END eventual_consistent_query]
_ = query // Use client.Run or client.GetAll to execute the query.
}
func ExampleDelete() {
ctx := Example_auth()
key := datastore.NewKey(ctx, "Article", "articled1", 0, nil)
if err := datastore.Delete(ctx, key); err != nil {
log.Fatal(err)
}
}
func ExamplePutMulti_interfaceSlice() {
ctx := Example_auth()
keys := []*datastore.Key{
datastore.NewKey(ctx, "Post", "post1", 0, nil),
datastore.NewKey(ctx, "Post", "post2", 0, nil),
}
// PutMulti with an empty interface slice.
posts := []interface{}{
&Post{Title: "Post 1", PublishedAt: time.Now()},
&Post{Title: "Post 2", PublishedAt: time.Now()},
}
if _, err := datastore.PutMulti(ctx, keys, posts); err != nil {
log.Fatal(err)
}
}
func ExampleClient_Delete() {
ctx := context.Background()
client, _ := datastore.NewClient(ctx, "my-proj")
key := datastore.NewKey(ctx, "Task", "sampletask", 0, nil)
// [START delete]
err := client.Delete(ctx, key)
// [END delete]
_ = err // Make sure you check err.
}
func ExampleClient_Get() {
ctx := context.Background()
client, _ := datastore.NewClient(ctx, "my-proj")
taskKey := datastore.NewKey(ctx, "Task", "sampleTask", 0, nil)
// [START lookup]
var task Task
err := client.Get(ctx, taskKey, &task)
// [END lookup]
_ = err // Make sure you check err.
}
func ExampleDelete() {
ctx := context.Background()
client, err := datastore.NewClient(ctx, "project-id")
if err != nil {
log.Fatal(err)
}
key := datastore.NewKey(ctx, "Article", "articled1", 0, nil)
if err := client.Delete(ctx, key); err != nil {
log.Fatal(err)
}
}
func ExamplePutMulti_interfaceSlice() {
ctx := context.Background()
client, err := datastore.NewClient(ctx, "project-id")
if err != nil {
log.Fatal(err)
}
keys := []*datastore.Key{
datastore.NewKey(ctx, "Post", "post1", 0, nil),
datastore.NewKey(ctx, "Post", "post2", 0, nil),
}
// PutMulti with an empty interface slice.
posts := []interface{}{
&Post{Title: "Post 1", PublishedAt: time.Now()},
&Post{Title: "Post 2", PublishedAt: time.Now()},
}
if _, err := client.PutMulti(ctx, keys, posts); err != nil {
log.Fatal(err)
}
}
func (ds *Datastore) IsNew(t, id string) bool {
dkey := datastore.NewKey(ds.ctx, t+"Entity", id, 0, nil)
keys, err := datastore.NewQuery(t+"Instruction").Ancestor(dkey).
Limit(1).KeysOnly().GetAll(ds.ctx, nil)
if err != nil {
return false
}
if len(keys) > 0 {
return false
}
return true
}
func NewDBTransactionLogIO(projectName, rootKeyStr string, c btncrypt.Cipher, clisrc auth.ClientSource) (*DBTransactionLogIO, error) {
txio := &DBTransactionLogIO{
projectName: projectName,
c: c,
clisrc: clisrc,
nextbatch: make([]inodedb.DBTransaction, 0),
}
ctx := txio.getContext()
txio.rootKey = datastore.NewKey(ctx, kindTransaction, rootKeyStr, 0, nil)
txio.syncer = util.NewSyncScheduler(txio, 300*time.Millisecond)
return txio, nil
}
// [START update_entity]
// MarkDone marks the task done with the given ID.
func MarkDone(ctx context.Context, client *datastore.Client, taskID int64) error {
// Create a key using the given integer ID.
key := datastore.NewKey(ctx, "Task", "", taskID, nil)
// In a transaction load each task, set done to true and store.
_, err := client.RunInTransaction(ctx, func(tx *datastore.Transaction) error {
var task Task
if err := tx.Get(key, &task); err != nil {
return err
}
task.Done = true
_, err := tx.Put(key, &task)
return err
})
return err
}
func (ds *Datastore) GetEntity(subject string) (reply []x.Instruction, rerr error) {
skey := datastore.NewKey(ds.ctx, ds.tablePrefix+"Entity", subject, 0, nil)
log.Infof("skey: %+v", skey)
client, err := datastore.NewClient(ds.ctx, ds.projectId)
if err != nil {
x.LogErr(log, err).Error("While creating client")
return reply, err
}
var dkeys []*datastore.Key
q := datastore.NewQuery(ds.tablePrefix + "Instruction").Ancestor(skey)
dkeys, rerr = client.GetAll(ds.ctx, q, &reply)
log.Debugf("Got num keys: %+v", len(dkeys))
return
}
func ExampleGet() {
ctx := Example_auth()
type Article struct {
Title string
Description string
Body string `datastore:",noindex"`
Author *datastore.Key
PublishedAt time.Time
}
key := datastore.NewKey(ctx, "Article", "articled1", 0, nil)
article := &Article{}
if err := datastore.Get(ctx, key, article); err != nil {
log.Fatal(err)
}
}
func Example_metadataPropertiesForKind() {
ctx := context.Background()
client, _ := datastore.NewClient(ctx, "my-proj")
// [START property_by_kind_run_query]
kindKey := datastore.NewKey(ctx, "__kind__", "Task", 0, nil)
query := datastore.NewQuery("__property__").Ancestor(kindKey)
type Prop struct {
Repr []string `datastore:"property_representation"`
}
var props []Prop
keys, err := client.GetAll(ctx, query, &props)
// [END property_by_kind_run_query]
_ = err // Check error.
_ = keys // Use keys to find property names, and props for their representations.
}
func ExampleTransaction_runQuery() {
ctx := context.Background()
client, _ := datastore.NewClient(ctx, "my-proj")
// [START transactional_single_entity_group_read_only]
tx, err := client.NewTransaction(ctx)
if err != nil {
log.Fatalf("client.NewTransaction: %v", err)
}
defer tx.Rollback() // Transaction only used for read.
ancestor := datastore.NewKey(ctx, "TaskList", "default", 0, nil)
query := datastore.NewQuery("Task").Ancestor(ancestor).Transaction(tx)
var tasks []Task
_, err = client.GetAll(ctx, query, &tasks)
// [END transactional_single_entity_group_read_only]
_ = err // Check error.
}
func (fe *FeServiceImpl) DeleteBiosphere(ctx context.Context, q *api.DeleteBiosphereQ) (*api.DeleteBiosphereS, error) {
ctx = TraceStart(ctx, "/frontend.DeleteBiosphere")
defer TraceEnd(ctx, fe.ServerCred)
if q.Auth == nil {
return nil, errors.New("DeleteBiosphere requires auth")
}
canWrite, err := fe.isWriteAuthorized(ctx, q.Auth)
if err != nil {
return nil, err
}
if !canWrite {
return nil, errors.New("UI must disallow unauthorized actions")
}
client, err := fe.AuthDatastore(ctx)
if err != nil {
return nil, err
}
chunkIdStart := fmt.Sprintf("%d", q.BiosphereId)
chunkIdEnd := fmt.Sprintf("%d", q.BiosphereId+1)
// Delete snapshots.
snapshotQ := datastore.NewQuery("PersistentChunkSnapshot").Filter("ChunkId>=", chunkIdStart).Filter("ChunkId<", chunkIdEnd).KeysOnly()
snapshotKeys, err := client.GetAll(ctx, snapshotQ, nil)
log.Printf("Delete snapshots: [%s, %s): %d PersistentChunkSnapshot keys found", chunkIdStart, chunkIdEnd, len(snapshotKeys))
if err != nil {
return nil, err
}
err = client.DeleteMulti(ctx, snapshotKeys)
if err != nil {
return nil, err
}
// Delete metadata.
key := datastore.NewKey(ctx, "BiosphereMeta", "", int64(q.BiosphereId), nil)
err = client.Delete(ctx, key)
if err != nil {
return nil, err
}
return &api.DeleteBiosphereS{
Deleted: true,
}, nil
}