// Next processes the next item
func (x *example6) Next(c context.Context, counters mapper.Counters, key *datastore.Key) error {
// we need to load the entity ourselves
photo := new(Photo)
if err := nds.Get(c, key, photo); err != nil {
return err
}
photo.ID = key.IntID()
suffix := photo.Taken.Format("20060102")
_, err := x.bq.Tabledata.InsertAll(x.appID, "datasetName", "tableName", &bigquery.TableDataInsertAllRequest{
TemplateSuffix: suffix,
Rows: []*bigquery.TableDataInsertAllRequestRows{
{
Json: map[string]bigquery.JsonValue{
"id": photo.ID,
"taken": photo.Taken,
"photographer": map[string]bigquery.JsonValue{
"id": photo.Photographer.ID,
"name": photo.Photographer.Name,
},
},
},
},
}).Context(c).Do()
return err
}
func endTask(c context.Context, ds appwrap.Datastore, taskIntf startTopIntf, jobKey *datastore.Key, taskKey *datastore.Key, resultErr error, result interface{}) error {
if resultErr == nil {
if task, err := updateTask(ds, taskKey, TaskStatusDone, 0, "", result); err != nil {
return fmt.Errorf("Could not update task: %s", err)
} else {
taskIntf.Status(jobKey.IntID(), task)
}
} else {
if _, ok := resultErr.(tryAgainError); ok {
// wasn't fatal, go for it
if retryErr := retryTask(c, ds, taskIntf, jobKey, taskKey); retryErr != nil {
return fmt.Errorf("error retrying: %s (task failed due to: %s)", retryErr, resultErr)
} else {
logInfo(c, "retrying task due to %s", resultErr)
return nil
}
}
// fatal error
if _, err := updateTask(ds, taskKey, TaskStatusFailed, 0, resultErr.Error(), nil); err != nil {
return fmt.Errorf("Could not update task with failure: %s", err)
}
}
return nil
}
func ktoi(key *datastore.Key) id {
return id{
kind: key.Kind(),
stringID: key.StringID(),
intID: key.IntID(),
appID: key.AppID(),
namespace: key.Namespace(),
}
}
func buildDatastoreKey(key *datastore.Key) (map[string]bigquery.JsonValue, error) {
if key == nil {
return map[string]bigquery.JsonValue{
"namespace": "",
"app": "",
"path": "",
"kind": "",
"name": "",
"id": 0,
}, nil
}
var workKey = key
var keys []*datastore.Key
keys = append(keys, key)
for {
if workKey.Parent() == nil {
break
}
keys = append(keys, workKey.Parent())
workKey = workKey.Parent()
}
var buf bytes.Buffer
for i := len(keys) - 1; i >= 0; i-- {
if buf.Len() > 0 {
_, err := buf.WriteString(", ")
if err != nil {
return map[string]bigquery.JsonValue{}, nil
}
}
key := keys[i]
if len(key.StringID()) < 1 {
_, err := buf.WriteString(fmt.Sprintf(`"%s", "%s"`, keys[i].Kind(), keys[i].IntID()))
if err != nil {
return map[string]bigquery.JsonValue{}, nil
}
} else {
_, err := buf.WriteString(fmt.Sprintf(`"%s", "%s"`, keys[i].Kind(), keys[i].StringID()))
if err != nil {
return map[string]bigquery.JsonValue{}, nil
}
}
}
return map[string]bigquery.JsonValue{
"namespace": key.Namespace(),
"app": key.AppID(),
"path": buf.String(),
"kind": key.Kind(),
"name": key.StringID(),
"id": key.IntID(),
}, nil
}
func jobFailed(c context.Context, ds appwrap.Datastore, taskIntf TaskInterface, jobKey *datastore.Key, err error) {
logError(c, "jobFailed: %s", err)
prevJob, _ := markJobFailed(c, ds, jobKey) // this might mark it failed again. whatever.
if prevJob.OnCompleteUrl != "" {
taskIntf.PostStatus(c, fmt.Sprintf("%s?status=error;error=%s;id=%d", prevJob.OnCompleteUrl,
url.QueryEscape(err.Error()), jobKey.IntID()))
}
return
}
// Next processes the next item
func (x *example3) Next(c context.Context, counters mapper.Counters, key *datastore.Key) error {
photo := new(Photo)
if err := nds.Get(c, key, photo); err != nil {
log.Errorf(c, err.Error())
return err
}
photo.ID = key.IntID()
counters.Increment(photo.Photographer.Name, 1)
return nil
}
// Next processes the next item
func (x *example5) Next(c context.Context, counters mapper.Counters, key *datastore.Key) error {
photo := x.photo
photo.ID = key.IntID()
out := &photoOutput{
Photo: photo,
Namespace: key.Namespace(),
}
x.encoder.Encode(out)
return nil
}
func nameObject(key *datastore.Key) string {
name := ""
for key != nil {
id := key.StringID()
if id == "" {
id = strconv.FormatInt(key.IntID(), 10)
}
name = "/" + key.Kind() + "/" + id + name
key = key.Parent()
}
// NOTE: The name of a GCS object must not be prefixed "/",
// this will give you a major headache.
return name[1:]
}
func getJob(ds appwrap.Datastore, jobKey *datastore.Key) (JobInfo, error) {
var job JobInfo
err := backoff.Retry(func() error {
if err := ds.Get(jobKey, &job); err != nil {
return err
}
return nil
}, mrBackOff())
job.Id = jobKey.IntID()
return job, err
}
// EncodeToken translates the key and raw secret of a newly generated token to
// a form suitable for the client.
func encodeToken(key *datastore.Key, raw *[tokenLength]byte) (string, error) {
// Buffer size will be 8 (size of an int64) times the number of keys
// in the hirarchy plus the length of the raw token itself.
var b [len(kinds)*8 + tokenLength]byte
for i := range kinds {
if n := binary.PutVarint(b[i*8:(i+1)*8], key.IntID()); n < 8 {
return "", errors.New("short write when encoding token")
}
if key != nil {
key = key.Parent()
}
}
copy(b[len(kinds)*8:len(kinds)*8+tokenLength], raw[:])
return hex.EncodeToString(b[:]), nil
}
// cmpKey compares k and other, returning -1, 0 or 1 if k is
// less than, equal or grather than other. The algorithm doesn't
// takes into account any ancestors in the two keys. The order
// of comparision is AppID, Kind, IntID and StringID. Keys with
// integer identifiers are smaller than string identifiers.
func cmpKey(k, other *datastore.Key) int {
if k == other {
return 0
}
if r := cmpStr(k.AppID(), other.AppID()); r != 0 {
return r
}
if r := cmpStr(k.Kind(), other.Kind()); r != 0 {
return r
}
if k.IntID() != 0 {
if other.IntID() == 0 {
return -1
}
return cmpInt(k.IntID(), other.IntID())
}
if other.IntID() != 0 {
return 1
}
return cmpStr(k.StringID(), other.StringID())
}
// dsR2F (DS real-to-fake) converts an SDK Key to a ds.Key
func dsR2F(k *datastore.Key) *ds.Key {
if k == nil {
return nil
}
aid := k.AppID()
ns := k.Namespace()
count := 0
for nk := k; nk != nil; nk = nk.Parent() {
count++
}
toks := make([]ds.KeyTok, count)
for ; k != nil; k = k.Parent() {
count--
toks[count].Kind = k.Kind()
toks[count].StringID = k.StringID()
toks[count].IntID = k.IntID()
}
return ds.NewKeyToks(aid, ns, toks)
}
func reduceMonitorTask(c context.Context, ds appwrap.Datastore, pipeline MapReducePipeline, jobKey *datastore.Key, r *http.Request, timeout time.Duration) int {
start := time.Now()
job, err := waitForStageCompletion(c, ds, pipeline, jobKey, StageReducing, StageDone, timeout)
if err != nil {
logCritical(c, "waitForStageCompletion() failed: %S", err)
return 200
} else if job.Stage == StageReducing {
logInfo(c, "wait timed out -- returning an error and letting us automatically restart")
return 500
}
logInfo(c, "reduce complete status: %s", job.Stage)
if job.OnCompleteUrl != "" {
successUrl := fmt.Sprintf("%s?status=%s;id=%d", job.OnCompleteUrl, TaskStatusDone, jobKey.IntID())
logInfo(c, "posting complete status to url %s", successUrl)
pipeline.PostStatus(c, successUrl)
}
logInfo(c, "reduction complete after %s of monitoring ", time.Now().Sub(start))
return 200
}
func (g *Goon) setStructKey(src interface{}, key *datastore.Key) error {
v := reflect.ValueOf(src)
t := v.Type()
k := t.Kind()
if k != reflect.Ptr {
return fmt.Errorf("goon: Expected pointer to struct, got instead: %v", k)
}
v = reflect.Indirect(v)
t = v.Type()
k = t.Kind()
if k != reflect.Struct {
return fmt.Errorf(fmt.Sprintf("goon: Expected struct, got instead: %v", k))
}
idSet := false
kindSet := false
parentSet := false
for i := 0; i < v.NumField(); i++ {
tf := t.Field(i)
vf := v.Field(i)
if !vf.CanSet() {
continue
}
tag := tf.Tag.Get("goon")
tagValues := strings.Split(tag, ",")
if len(tagValues) > 0 {
tagValue := tagValues[0]
if tagValue == "id" {
if idSet {
return fmt.Errorf("goon: Only one field may be marked id")
}
switch vf.Kind() {
case reflect.Int64:
vf.SetInt(key.IntID())
idSet = true
case reflect.String:
vf.SetString(key.StringID())
idSet = true
}
} else if tagValue == "kind" {
if kindSet {
return fmt.Errorf("goon: Only one field may be marked kind")
}
if vf.Kind() == reflect.String {
if (len(tagValues) <= 1 || key.Kind() != tagValues[1]) && g.KindNameResolver(src) != key.Kind() {
vf.Set(reflect.ValueOf(key.Kind()))
}
kindSet = true
}
} else if tagValue == "parent" {
if parentSet {
return fmt.Errorf("goon: Only one field may be marked parent")
}
dskeyType := reflect.TypeOf(&datastore.Key{})
vfType := vf.Type()
if vfType.ConvertibleTo(dskeyType) {
vf.Set(reflect.ValueOf(key.Parent()).Convert(vfType))
parentSet = true
}
}
}
}
if !idSet {
return fmt.Errorf("goon: Could not set id field")
}
return nil
}
func populateKey(k *datastore.Key, b *Battery) {
b.Time = time.Unix(k.IntID(), 0)
}
func (x *logPhotos) Process(c context.Context, key *datastore.Key) {
// just log it
log.Debugf(c, "photo %d taken %s by %d %s", key.IntID(), x.photo.Taken.String(), x.photo.Photographer.ID, x.photo.Photographer.Name)
}
