func ExampleNewKey_withParent() {
	ctx := context.Background()
	// [START key_with_parent]
	parentKey := datastore.NewKey(ctx, "TaskList", "default", 0, nil)
	taskKey := datastore.NewKey(ctx, "Task", "sampleTask", 0, parentKey)
	// [END key_with_parent]
	_ = taskKey // Use the task key for datastore operations.
}
func ExampleNewKey_withMultipleParents() {
	ctx := context.Background()
	// [START key_with_multilevel_parent]
	userKey := datastore.NewKey(ctx, "User", "alice", 0, nil)
	parentKey := datastore.NewKey(ctx, "TaskList", "default", 0, userKey)
	taskKey := datastore.NewKey(ctx, "Task", "sampleTask", 0, parentKey)
	// [END key_with_multilevel_parent]
	_ = taskKey // Use the task key for datastore operations.
}
Beispiel #3
0
func ExampleGetMulti() {
	ctx := Example_auth()

	keys := []*datastore.Key{
		datastore.NewKey(ctx, "Post", "post1", 0, nil),
		datastore.NewKey(ctx, "Post", "post2", 0, nil),
		datastore.NewKey(ctx, "Post", "post3", 0, nil),
	}
	posts := make([]Post, 3)
	if err := datastore.GetMulti(ctx, keys, posts); err != nil {
		log.Println(err)
	}
}
Beispiel #4
0
func pollCommits(dir string) {
	cmd := execGit(dir, nil, "pull", "origin")
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Printf("Error running git pull origin master in %s: %v\n%s", dir, err, out)
		return
	}
	log.Printf("Ran git pull.")
	// TODO: see if .git/refs/remotes/origin/master
	// changed. (quicker than running recentCommits each time)

	hashes, err := recentCommits(dir)
	if err != nil {
		log.Print(err)
		return
	}
	latestHash.Lock()
	latestHash.s = hashes[0]
	latestHash.Unlock()
	for _, commit := range hashes {
		if knownCommit[commit] {
			continue
		}
		if dsClient != nil {
			ctx := context.Background()
			key := datastore.NewKey(ctx, "git_commit", commit, 0, nil)
			var gc GitCommit
			if err := dsClient.Get(ctx, key, &gc); err == nil && gc.Emailed {
				log.Printf("Already emailed about commit %v; skipping", commit)
				knownCommit[commit] = true
				continue
			}
		}
		if err := emailCommit(dir, commit); err == nil {
			log.Printf("Emailed commit %s", commit)
			knownCommit[commit] = true
			if dsClient != nil {
				ctx := context.Background()
				key := datastore.NewKey(ctx, "git_commit", commit, 0, nil)
				_, err := dsClient.Put(ctx, key, &GitCommit{Emailed: true})
				log.Printf("datastore put of git_commit(%v): %v", commit, err)
			}
		}
	}
	if githubSSHKey != "" {
		if err := syncToGithub(dir, hashes[0]); err != nil {
			log.Printf("Failed to push commit %v to github: %v", hashes[0], err)
		}
	}
}
func (txio *DBTransactionLogIO) Sync() error {
	txio.mu.Lock()
	batch := txio.nextbatch
	txio.nextbatch = make([]inodedb.DBTransaction, 0)
	txio.mu.Unlock()

	if len(batch) == 0 {
		return nil
	}

	ctx := txio.getContext()
	keys := make([]*datastore.Key, 0, len(batch))
	stxs := make([]*storedbtx, 0, len(batch))
	for _, tx := range batch {
		keys = append(keys, datastore.NewKey(ctx, kindTransaction, "", int64(tx.TxID), txio.rootKey))
		stx, err := encode(txio.c, tx)
		if err != nil {
			return err
		}
		stxs = append(stxs, stx)
	}

	if _, err := datastore.PutMulti(ctx, keys, stxs); err != nil {
		return err
	}
	log.Printf("Committed %d txs", len(stxs))
	return nil
}
Beispiel #6
0
func ExampleTransaction() {
	ctx := Example_auth()
	const retries = 3

	// Increment a counter.
	// See https://cloud.google.com/appengine/articles/sharding_counters for
	// a more scalable solution.
	type Counter struct {
		Count int
	}

	key := datastore.NewKey(ctx, "counter", "CounterA", 0, nil)

	for i := 0; i < retries; i++ {
		tx, err := datastore.NewTransaction(ctx)
		if err != nil {
			break
		}

		var c Counter
		if err := tx.Get(key, &c); err != nil && err != datastore.ErrNoSuchEntity {
			break
		}
		c.Count++
		if _, err := tx.Put(key, &c); err != nil {
			break
		}

		// Attempt to commit the transaction. If there's a conflict, try again.
		if _, err := tx.Commit(); err != datastore.ErrConcurrentTransaction {
			break
		}
	}

}
Beispiel #7
0
func (fe *FeServiceImpl) getBiosphereTopo(ctx context.Context, biosphereId uint64) (BiosphereTopology, *api.BiosphereEnvConfig, error, *api.TimingTrace) {
	trace := InitTrace("getBiosphereTopo")
	fe.bsMetaCacheMutex.Lock()
	defer fe.bsMetaCacheMutex.Unlock()

	bsMeta, ok := fe.bsMetaCache[biosphereId]
	if !ok {
		authTrace := InitTrace("AuthDatastore")
		client, err := fe.AuthDatastore(ctx)
		if err != nil {
			return nil, nil, err, trace
		}
		FinishTrace(authTrace, trace)

		key := datastore.NewKey(ctx, "BiosphereMeta", "", int64(biosphereId), nil)
		var meta BiosphereMeta
		err = client.Get(ctx, key, &meta)
		if err != nil {
			return nil, nil, err, trace
		}
		fe.bsMetaCache[biosphereId] = &meta
		bsMeta = &meta
		log.Printf("BiosphereMeta cache entry (bsId: %d) created. Now #(cache entry)=%d", biosphereId, len(fe.bsMetaCache))
	}
	log.Printf("Found config of %d x %d", bsMeta.Nx, bsMeta.Ny)
	envConfig := api.BiosphereEnvConfig{}
	err := proto.Unmarshal(bsMeta.Env, &envConfig)
	if err != nil {
		return nil, nil, err, trace
	}
	return NewCylinderTopology(biosphereId, int(bsMeta.Nx), int(bsMeta.Ny)), &envConfig, nil, trace
}
func ExampleNewKey() {
	ctx := context.Background()
	// [START named_key]
	taskKey := datastore.NewKey(ctx, "Task", "sampletask", 0, nil)
	// [END named_key]
	_ = taskKey // Use the task key for datastore operations.
}
Beispiel #9
0
func (ck *CkServiceImpl) fetchDatastoreSnapshot(dsKey int64) *api.ChunkState {
	strictCtx, _ := context.WithTimeout(context.Background(), 2500*time.Millisecond)
	client, err := ck.AuthDatastore(strictCtx)
	if err != nil {
		log.Printf("ERROR datastore auth failed %#v", err)
		return nil
	}

	key := datastore.NewKey(strictCtx, "PersistentChunkSnapshot", "", dsKey, nil)
	snapshot := new(PersistentChunkSnapshot)
	err = client.Get(strictCtx, key, snapshot)
	if err != nil {
		log.Printf("ERROR datastore.Get(%#v) failed %#v", key, err)
		return nil
	}

	snapshotProto := &api.ChunkSnapshot{}
	err = proto.Unmarshal(snapshot.Snapshot, snapshotProto)
	if err != nil {
		log.Printf("ERROR failed to unmarshal; corrupt datastore entry at key %d %#v", dsKey, err)
		return nil
	}

	// Convert to a shard w/o any outgoing grains.
	return &api.ChunkState{
		Shards: []*api.ChunkShard{
			&api.ChunkShard{
				Dp:     &api.ChunkRel{Dx: 0, Dy: 0},
				Grains: snapshotProto.Grains,
			},
		},
	}
}
Beispiel #10
0
func (ds *Datastore) GetEntity(t, subject string) (reply []x.Instruction, rerr error) {
	skey := datastore.NewKey(ds.ctx, t+"Entity", subject, 0, nil)
	log.Infof("skey: %+v", skey)
	dkeys, rerr := datastore.NewQuery(t+"Instruction").Ancestor(skey).GetAll(ds.ctx, &reply)
	log.Debugf("Got num keys: %+v", len(dkeys))
	return
}
Beispiel #11
0
func NewDBTransactionLogIO(cfg *Config) *DBTransactionLogIO {
	return &DBTransactionLogIO{
		cfg:       cfg,
		rootKey:   datastore.NewKey(ctxNoNamespace, kindTransaction, cfg.rootKeyStr, 0, nil),
		nextbatch: make([]inodedb.DBTransaction, 0),
	}
}
Beispiel #12
0
func (loc *INodeDBSSLocator) tryPutOnce(blobpath string, txid int64) error {
	start := time.Now()
	e := sslocentry{BlobPath: blobpath, TxID: txid, CreatedAt: start}

	cli, err := loc.cfg.getClient(context.TODO())
	if err != nil {
		return err
	}
	dstx, err := cli.NewTransaction(context.TODO(), datastore.Serializable)
	if err != nil {
		return err
	}

	key := datastore.NewKey(ctxNoNamespace, kindINodeDBSS, "", int64(e.TxID), loc.rootKey)
	if _, err := dstx.Put(key, &e); err != nil {
		dstx.Rollback()
		return err
	}
	if _, err := dstx.Commit(); err != nil {
		return err
	}

	logger.Infof(sslog, "Put(%s, %d) took %s.", blobpath, txid, time.Since(start))
	return nil
}
Beispiel #13
0
func ExamplePut() {
	ctx := context.Background()
	client, err := datastore.NewClient(ctx, "project-id")
	if err != nil {
		log.Fatal(err)
	}

	type Article struct {
		Title       string
		Description string
		Body        string `datastore:",noindex"`
		Author      *datastore.Key
		PublishedAt time.Time
	}
	newKey := datastore.NewIncompleteKey(ctx, "Article", nil)
	_, err = client.Put(ctx, newKey, &Article{
		Title:       "The title of the article",
		Description: "The description of the article...",
		Body:        "...",
		Author:      datastore.NewKey(ctx, "Author", "jbd", 0, nil),
		PublishedAt: time.Now(),
	})
	if err != nil {
		log.Fatal(err)
	}
}
Beispiel #14
0
func ExampleGetMulti() {
	ctx := context.Background()
	client, err := datastore.NewClient(ctx, "project-id")
	if err != nil {
		log.Fatal(err)
	}

	keys := []*datastore.Key{
		datastore.NewKey(ctx, "Post", "post1", 0, nil),
		datastore.NewKey(ctx, "Post", "post2", 0, nil),
		datastore.NewKey(ctx, "Post", "post3", 0, nil),
	}
	posts := make([]Post, 3)
	if err := client.GetMulti(ctx, keys, posts); err != nil {
		log.Println(err)
	}
}
func ExampleQuery_keyFilter() {
	ctx := context.Background()
	// [START key_filter]
	key := datastore.NewKey(ctx, "Task", "someTask", 0, nil)
	query := datastore.NewQuery("Task").Filter("__key__ >", key)
	// [END key_filter]
	_ = query // Use client.Run or client.GetAll to execute the query.
}
func ExampleQuery_EventualConsistency() {
	ctx := context.Background()
	// [START eventual_consistent_query]
	ancestor := datastore.NewKey(ctx, "TaskList", "default", 0, nil)
	query := datastore.NewQuery("Task").Ancestor(ancestor).EventualConsistency()
	// [END eventual_consistent_query]
	_ = query // Use client.Run or client.GetAll to execute the query.
}
Beispiel #17
0
func ExampleDelete() {
	ctx := Example_auth()

	key := datastore.NewKey(ctx, "Article", "articled1", 0, nil)
	if err := datastore.Delete(ctx, key); err != nil {
		log.Fatal(err)
	}
}
Beispiel #18
0
func ExamplePutMulti_interfaceSlice() {
	ctx := Example_auth()

	keys := []*datastore.Key{
		datastore.NewKey(ctx, "Post", "post1", 0, nil),
		datastore.NewKey(ctx, "Post", "post2", 0, nil),
	}

	// PutMulti with an empty interface slice.
	posts := []interface{}{
		&Post{Title: "Post 1", PublishedAt: time.Now()},
		&Post{Title: "Post 2", PublishedAt: time.Now()},
	}
	if _, err := datastore.PutMulti(ctx, keys, posts); err != nil {
		log.Fatal(err)
	}
}
func ExampleClient_Delete() {
	ctx := context.Background()
	client, _ := datastore.NewClient(ctx, "my-proj")
	key := datastore.NewKey(ctx, "Task", "sampletask", 0, nil)
	// [START delete]
	err := client.Delete(ctx, key)
	// [END delete]
	_ = err // Make sure you check err.
}
func ExampleClient_Get() {
	ctx := context.Background()
	client, _ := datastore.NewClient(ctx, "my-proj")
	taskKey := datastore.NewKey(ctx, "Task", "sampleTask", 0, nil)
	// [START lookup]
	var task Task
	err := client.Get(ctx, taskKey, &task)
	// [END lookup]
	_ = err // Make sure you check err.
}
Beispiel #21
0
func ExampleDelete() {
	ctx := context.Background()
	client, err := datastore.NewClient(ctx, "project-id")
	if err != nil {
		log.Fatal(err)
	}

	key := datastore.NewKey(ctx, "Article", "articled1", 0, nil)
	if err := client.Delete(ctx, key); err != nil {
		log.Fatal(err)
	}
}
Beispiel #22
0
func ExamplePutMulti_interfaceSlice() {
	ctx := context.Background()
	client, err := datastore.NewClient(ctx, "project-id")
	if err != nil {
		log.Fatal(err)
	}

	keys := []*datastore.Key{
		datastore.NewKey(ctx, "Post", "post1", 0, nil),
		datastore.NewKey(ctx, "Post", "post2", 0, nil),
	}

	// PutMulti with an empty interface slice.
	posts := []interface{}{
		&Post{Title: "Post 1", PublishedAt: time.Now()},
		&Post{Title: "Post 2", PublishedAt: time.Now()},
	}
	if _, err := client.PutMulti(ctx, keys, posts); err != nil {
		log.Fatal(err)
	}
}
Beispiel #23
0
func (ds *Datastore) IsNew(t, id string) bool {
	dkey := datastore.NewKey(ds.ctx, t+"Entity", id, 0, nil)
	keys, err := datastore.NewQuery(t+"Instruction").Ancestor(dkey).
		Limit(1).KeysOnly().GetAll(ds.ctx, nil)
	if err != nil {
		return false
	}
	if len(keys) > 0 {
		return false
	}
	return true
}
Beispiel #24
0
func NewDBTransactionLogIO(projectName, rootKeyStr string, c btncrypt.Cipher, clisrc auth.ClientSource) (*DBTransactionLogIO, error) {
	txio := &DBTransactionLogIO{
		projectName: projectName,
		c:           c,
		clisrc:      clisrc,
		nextbatch:   make([]inodedb.DBTransaction, 0),
	}
	ctx := txio.getContext()
	txio.rootKey = datastore.NewKey(ctx, kindTransaction, rootKeyStr, 0, nil)
	txio.syncer = util.NewSyncScheduler(txio, 300*time.Millisecond)

	return txio, nil
}
Beispiel #25
0
// [START update_entity]
// MarkDone marks the task done with the given ID.
func MarkDone(ctx context.Context, client *datastore.Client, taskID int64) error {
	// Create a key using the given integer ID.
	key := datastore.NewKey(ctx, "Task", "", taskID, nil)

	// In a transaction load each task, set done to true and store.
	_, err := client.RunInTransaction(ctx, func(tx *datastore.Transaction) error {
		var task Task
		if err := tx.Get(key, &task); err != nil {
			return err
		}
		task.Done = true
		_, err := tx.Put(key, &task)
		return err
	})
	return err
}
Beispiel #26
0
func (ds *Datastore) GetEntity(subject string) (reply []x.Instruction, rerr error) {
	skey := datastore.NewKey(ds.ctx, ds.tablePrefix+"Entity", subject, 0, nil)
	log.Infof("skey: %+v", skey)

	client, err := datastore.NewClient(ds.ctx, ds.projectId)
	if err != nil {
		x.LogErr(log, err).Error("While creating client")
		return reply, err
	}

	var dkeys []*datastore.Key
	q := datastore.NewQuery(ds.tablePrefix + "Instruction").Ancestor(skey)
	dkeys, rerr = client.GetAll(ds.ctx, q, &reply)
	log.Debugf("Got num keys: %+v", len(dkeys))
	return
}
Beispiel #27
0
func ExampleGet() {
	ctx := Example_auth()

	type Article struct {
		Title       string
		Description string
		Body        string `datastore:",noindex"`
		Author      *datastore.Key
		PublishedAt time.Time
	}
	key := datastore.NewKey(ctx, "Article", "articled1", 0, nil)
	article := &Article{}
	if err := datastore.Get(ctx, key, article); err != nil {
		log.Fatal(err)
	}
}
func Example_metadataPropertiesForKind() {
	ctx := context.Background()
	client, _ := datastore.NewClient(ctx, "my-proj")
	// [START property_by_kind_run_query]
	kindKey := datastore.NewKey(ctx, "__kind__", "Task", 0, nil)
	query := datastore.NewQuery("__property__").Ancestor(kindKey)

	type Prop struct {
		Repr []string `datastore:"property_representation"`
	}

	var props []Prop
	keys, err := client.GetAll(ctx, query, &props)
	// [END property_by_kind_run_query]
	_ = err  // Check error.
	_ = keys // Use keys to find property names, and props for their representations.
}
func ExampleTransaction_runQuery() {
	ctx := context.Background()
	client, _ := datastore.NewClient(ctx, "my-proj")
	// [START transactional_single_entity_group_read_only]
	tx, err := client.NewTransaction(ctx)
	if err != nil {
		log.Fatalf("client.NewTransaction: %v", err)
	}
	defer tx.Rollback() // Transaction only used for read.

	ancestor := datastore.NewKey(ctx, "TaskList", "default", 0, nil)
	query := datastore.NewQuery("Task").Ancestor(ancestor).Transaction(tx)
	var tasks []Task
	_, err = client.GetAll(ctx, query, &tasks)
	// [END transactional_single_entity_group_read_only]
	_ = err // Check error.
}
func (fe *FeServiceImpl) DeleteBiosphere(ctx context.Context, q *api.DeleteBiosphereQ) (*api.DeleteBiosphereS, error) {
	ctx = TraceStart(ctx, "/frontend.DeleteBiosphere")
	defer TraceEnd(ctx, fe.ServerCred)

	if q.Auth == nil {
		return nil, errors.New("DeleteBiosphere requires auth")
	}
	canWrite, err := fe.isWriteAuthorized(ctx, q.Auth)
	if err != nil {
		return nil, err
	}
	if !canWrite {
		return nil, errors.New("UI must disallow unauthorized actions")
	}
	client, err := fe.AuthDatastore(ctx)
	if err != nil {
		return nil, err
	}

	chunkIdStart := fmt.Sprintf("%d", q.BiosphereId)
	chunkIdEnd := fmt.Sprintf("%d", q.BiosphereId+1)

	// Delete snapshots.
	snapshotQ := datastore.NewQuery("PersistentChunkSnapshot").Filter("ChunkId>=", chunkIdStart).Filter("ChunkId<", chunkIdEnd).KeysOnly()
	snapshotKeys, err := client.GetAll(ctx, snapshotQ, nil)
	log.Printf("Delete snapshots: [%s, %s): %d PersistentChunkSnapshot keys found", chunkIdStart, chunkIdEnd, len(snapshotKeys))
	if err != nil {
		return nil, err
	}
	err = client.DeleteMulti(ctx, snapshotKeys)
	if err != nil {
		return nil, err
	}
	// Delete metadata.
	key := datastore.NewKey(ctx, "BiosphereMeta", "", int64(q.BiosphereId), nil)
	err = client.Delete(ctx, key)
	if err != nil {
		return nil, err
	}

	return &api.DeleteBiosphereS{
		Deleted: true,
	}, nil
}