コード例 #1
0
ファイル: user.go プロジェクト: trythings/trythings
func (s *UserService) Create(ctx context.Context, u *User) error {
	span := trace.FromContext(ctx).NewChild("trythings.user.Create")
	defer span.Finish()

	// TODO Make sure u.GoogleID == user.Current(ctx).ID

	if u.ID != "" {
		return fmt.Errorf("u already has id %q", u.ID)
	}

	if u.CreatedAt.IsZero() {
		u.CreatedAt = time.Now()
	}

	id, _, err := datastore.AllocateIDs(ctx, "User", nil, 1)
	if err != nil {
		return err
	}
	u.ID = fmt.Sprintf("%x", id)

	rootKey := datastore.NewKey(ctx, "Root", "root", 0, nil)
	k := datastore.NewKey(ctx, "User", u.ID, 0, rootKey)
	k, err = datastore.Put(ctx, k, u)
	if err != nil {
		return err
	}

	return nil
}
コード例 #2
0
func NewFileKey(c context.Context) (*datastore.Key, error) {
	parentKey := GetParentKey(c)
	lowID, _, err := datastore.AllocateIDs(c, KIND_FILE, parentKey, 1)
	if err != nil {
		return nil, err
	}
	return datastore.NewKey(c, KIND_FILE, "", lowID, parentKey), nil
}
コード例 #3
0
ファイル: raw_datastore.go プロジェクト: tetrafolium/gae
func (d rdsImpl) AllocateIDs(incomplete *ds.Key, n int) (start int64, err error) {
	par, err := dsF2R(d.aeCtx, incomplete.Parent())
	if err != nil {
		return
	}

	start, _, err = datastore.AllocateIDs(d.aeCtx, incomplete.Kind(), par, n)
	return
}
コード例 #4
0
ファイル: models.go プロジェクト: yosukesuzuki/t2tapp
func (s *SpotGet) key(ctx context.Context) *datastore.Key {
	if s.SpotCode == 0 {
		low, _, err := datastore.AllocateIDs(ctx, "Spot", nil, 1)
		if err != nil {
			return nil
		}
		return datastore.NewKey(ctx, "Spot", "", low, nil)
	}
	return datastore.NewKey(ctx, "Spot", "", s.SpotCode, nil)
}
コード例 #5
0
ファイル: gopherpods.go プロジェクト: rojters/gopherpods
func submissionsAddHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	if err := r.ParseForm(); err != nil {
		serveErr(ctx, err, w)
		return
	}

	ID, _, err := datastore.AllocateIDs(ctx, "Podcast", nil, 1)
	if err != nil {
		serveErr(ctx, err, w)
		return
	}

	date, err := time.Parse(yyyymmdd, r.FormValue("date"))
	if err != nil {
		serveErr(ctx, err, w)
		return
	}

	podcast := Podcast{
		ID:         ID,
		Show:       r.FormValue("show"),
		Title:      r.FormValue("title"),
		Desc:       r.FormValue("desc"),
		URL:        template.URL(r.FormValue("url")),
		MediaURL:   template.URL(r.FormValue("media_url")),
		RuntimeSec: r.FormValue("runtime"),
		Size:       r.FormValue("size"),
		Date:       date,
		Added:      time.Now(),
	}

	if _, err := datastore.Put(ctx, datastore.NewKey(ctx, "Podcast", "", ID, nil), &podcast); err != nil {
		serveErr(ctx, err, w)
		return
	}

	key, err := datastore.DecodeKey(r.FormValue("key"))
	if err != nil {
		serveErr(ctx, err, w)
		return
	}

	if err := datastore.Delete(ctx, key); err != nil {
		serveErr(ctx, err, w)
		return
	}

	if err := memcache.Delete(ctx, cacheKey); err != nil {
		log.Errorf(ctx, "memcache delete error %v", err)
	}

	successTmpl.ExecuteTemplate(w, "base", nil)
}
コード例 #6
0
ファイル: task.go プロジェクト: trythings/trythings
func (s *TaskService) Create(ctx context.Context, t *Task) error {
	span := trace.FromContext(ctx).NewChild("trythings.task.Create")
	defer span.Finish()

	if t.ID != "" {
		return fmt.Errorf("t already has id %q", t.ID)
	}

	if t.CreatedAt.IsZero() {
		t.CreatedAt = time.Now()
	}

	if t.SpaceID == "" {
		return errors.New("SpaceID is required")
	}

	ok, err := s.IsVisible(ctx, t)
	if err != nil {
		return err
	}

	if !ok {
		return errors.New("cannot access space to create task")
	}

	id, _, err := datastore.AllocateIDs(ctx, "Task", nil, 1)
	if err != nil {
		return err
	}
	t.ID = fmt.Sprintf("%x", id)

	rootKey := datastore.NewKey(ctx, "Root", "root", 0, nil)
	k := datastore.NewKey(ctx, "Task", t.ID, 0, rootKey)
	k, err = datastore.Put(ctx, k, t)
	if err != nil {
		return err
	}

	err = s.Index(ctx, t)
	if err != nil {
		return err
	}

	return nil
}
コード例 #7
0
ファイル: task.go プロジェクト: DrenBx/Go-TodoAPI
func createTask(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	ctx := appengine.NewContext(r)
	low, _, err := datastore.AllocateIDs(ctx, "tasks", nil, 1)
	if err != nil {
		sendJSONResponse(w, http.StatusNotModified, nil)
		return
	}
	t, err := maketask(low, r.Body)
	if err != nil {
		sendJSONResponse(w, http.StatusNotModified, nil)
		return
	}
	if _, e := datastore.Put(ctx, datastore.NewKey(ctx, "tasks", "", low, nil), t); e != nil {
		sendJSONResponse(w, http.StatusNotModified, nil)
		return
	}
	sendJSONResponse(w, http.StatusCreated, nil)
}
コード例 #8
0
ファイル: view.go プロジェクト: trythings/trythings
func (s *ViewService) Create(ctx context.Context, v *View) error {
	span := trace.FromContext(ctx).NewChild("trythings.view.Create")
	defer span.Finish()

	if v.ID != "" {
		return fmt.Errorf("v already has id %q", v.ID)
	}

	if v.CreatedAt.IsZero() {
		v.CreatedAt = time.Now()
	}

	if v.Name == "" {
		return errors.New("Name is required")
	}

	if v.SpaceID == "" {
		return errors.New("SpaceID is required")
	}

	ok, err := s.IsVisible(ctx, v)
	if err != nil {
		return err
	}

	if !ok {
		return errors.New("cannot access space to create view")
	}

	id, _, err := datastore.AllocateIDs(ctx, "View", nil, 1)
	if err != nil {
		return err
	}
	v.ID = fmt.Sprintf("%x", id)

	rootKey := datastore.NewKey(ctx, "Root", "root", 0, nil)
	k := datastore.NewKey(ctx, "View", v.ID, 0, rootKey)
	k, err = datastore.Put(ctx, k, v)
	if err != nil {
		return err
	}

	return nil
}
コード例 #9
0
ファイル: space.go プロジェクト: trythings/trythings
func (s *SpaceService) Create(ctx context.Context, sp *Space) error {
	span := trace.FromContext(ctx).NewChild("trythings.space.Create")
	defer span.Finish()

	if sp.ID != "" {
		return fmt.Errorf("sp already has id %q", sp.ID)
	}

	if sp.CreatedAt.IsZero() {
		sp.CreatedAt = time.Now()
	}

	if len(sp.UserIDs) > 0 {
		return errors.New("UserIDs must be empty")
	}

	su, err := IsSuperuser(ctx)
	if err != nil {
		return err
	}
	if !su {
		u, err := s.UserService.FromContext(ctx)
		if err != nil {
			return err
		}
		sp.UserIDs = []string{u.ID}
	}

	id, _, err := datastore.AllocateIDs(ctx, "Space", nil, 1)
	if err != nil {
		return err
	}
	sp.ID = fmt.Sprintf("%x", id)

	rootKey := datastore.NewKey(ctx, "Root", "root", 0, nil)
	k := datastore.NewKey(ctx, "Space", sp.ID, 0, rootKey)
	k, err = datastore.Put(ctx, k, sp)
	if err != nil {
		return err
	}

	return nil
}
コード例 #10
0
ファイル: submission.go プロジェクト: flowlo/coduno-api
// PostSubmission creates a new submission.
func PostSubmission(ctx context.Context, w http.ResponseWriter, r *http.Request) (status int, err error) {
	if r.Method != "POST" {
		return http.StatusMethodNotAllowed, nil
	}

	p, ok := passenger.FromContext(ctx)
	if !ok {
		return http.StatusUnauthorized, nil
	}

	mediaType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
	if err != nil {
		return http.StatusBadRequest, err
	}
	if !strings.HasPrefix(mediaType, "multipart/") {
		return http.StatusUnsupportedMediaType, nil
	}

	resultKey, err := datastore.DecodeKey(mux.Vars(r)["resultKey"])
	if err != nil {
		return http.StatusNotFound, err
	}

	if !util.HasParent(p.User, resultKey) {
		return http.StatusBadRequest, errors.New("cannot submit answer for other users")
	}

	taskKey, err := datastore.DecodeKey(mux.Vars(r)["taskKey"])
	if err != nil {
		return http.StatusNotFound, err
	}

	if err := r.ParseMultipartForm(16 << 20); err != nil {
		return http.StatusBadRequest, err
	}

	files, ok := r.MultipartForm.File["files"]
	if !ok {
		return http.StatusBadRequest, errors.New("missing files")
	}

	var task model.Task
	if err = datastore.Get(ctx, taskKey, &task); err != nil {
		return http.StatusNotFound, err
	}

	// Furthermore, the name of the GCS object is derived from the of the
	// encapsulating Submission. To avoid race conditions, allocate an ID.
	low, _, err := datastore.AllocateIDs(ctx, model.SubmissionKind, resultKey, 1)
	if err != nil {
		return http.StatusInternalServerError, err
	}

	submissionKey := datastore.NewKey(ctx, model.SubmissionKind, "", low, resultKey)
	storedCode := model.StoredObject{
		Bucket: util.SubmissionBucket(),
		Name:   nameObject(submissionKey) + "/Code/",
	}
	submission := model.Submission{
		Task:     taskKey,
		Time:     time.Now(),
		Language: detectLanguage(files),
		Code:     storedCode,
	}

	if _, err = datastore.Put(ctx, submissionKey, &submission); err != nil {
		return http.StatusInternalServerError, err
	}

	var tests model.Tests
	testKeys, err := model.NewQueryForTest().
		Ancestor(taskKey).
		GetAll(ctx, &tests)
	if err != nil {
		return http.StatusInternalServerError, err
	}

	prrs, pwrs := multiPipe(len(tests))

	go maketar(pwrs, files)

	for i, t := range tests {
		go func(i int, t model.Test) {
			if err := test.Tester(t.Tester).Call(ctx, *t.Key(testKeys[i]), *submission.Key(submissionKey), prrs[i]); err != nil {
				log.Warningf(ctx, "%s", err)
			}
		}(i, t)
	}

	if err := upload(util.CloudContext(ctx), storedCode.Bucket, storedCode.Name, files); err != nil {
		return http.StatusInternalServerError, err
	}

	return http.StatusOK, nil
}
コード例 #11
0
ファイル: datastore.go プロジェクト: speedland/service
// AllocateIDs wraps datastore.AllocateIDs
func (d *Driver) AllocateIDs(parent *datastore.Key, n int) (low, high int64, err error) {
	return datastore.AllocateIDs(d.ctx, d.kind, parent, n)
}
コード例 #12
0
ファイル: mapreduce.go プロジェクト: pendo-io/mapreduce
func Run(c context.Context, ds appwrap.Datastore, job MapReduceJob) (int64, error) {
	readerNames, err := job.Inputs.ReaderNames()
	if err != nil {
		return 0, fmt.Errorf("forming reader names: %s", err)
	} else if len(readerNames) == 0 {
		return 0, fmt.Errorf("no input readers")
	}

	writerNames, err := job.Outputs.WriterNames(c)
	if err != nil {
		return 0, fmt.Errorf("forming writer names: %s", err)
	} else if len(writerNames) == 0 {
		return 0, fmt.Errorf("no output writers")
	}

	reducerCount := len(writerNames)

	jobKey, err := createJob(ds, job.UrlPrefix, writerNames, job.OnCompleteUrl, job.SeparateReduceItems, job.JobParameters, job.RetryCount)
	if err != nil {
		return 0, fmt.Errorf("creating job: %s", err)
	}

	firstId, _, err := datastore.AllocateIDs(c, TaskEntity, nil, len(readerNames))
	if err != nil {
		return 0, fmt.Errorf("allocating task ids: %s", err)
	}
	taskKeys := makeTaskKeys(ds, firstId, len(readerNames))
	tasks := make([]JobTask, len(readerNames))

	for i, readerName := range readerNames {
		url := fmt.Sprintf("%s/map?taskKey=%s;reader=%s;shards=%d",
			job.UrlPrefix, taskKeys[i].Encode(), readerName,
			reducerCount)

		tasks[i] = JobTask{
			Status: TaskStatusPending,
			Url:    url,
			Type:   TaskTypeMap,
		}
	}

	if err := createTasks(ds, jobKey, taskKeys, tasks, StageMapping); err != nil {
		if _, innerErr := markJobFailed(c, ds, jobKey); err != nil {
			logError(c, "failed to log job %d as failed: %s", jobKey.IntID(), innerErr)
		}
		return 0, fmt.Errorf("creating tasks: %s", err)
	}

	for i := range tasks {
		if err := job.PostTask(c, tasks[i].Url, job.JobParameters); err != nil {
			if _, innerErr := markJobFailed(c, ds, jobKey); err != nil {
				logError(c, "failed to log job %d as failed: %s", jobKey.IntID(), innerErr)
			}
			return 0, fmt.Errorf("posting task: %s", err)
		}
	}

	if err := job.PostStatus(c, fmt.Sprintf("%s/map-monitor?jobKey=%s", job.UrlPrefix, jobKey.Encode())); err != nil {
		logCritical(c, "failed to start map monitor task: %s", err)
	}

	return jobKey.IntID(), nil
}
コード例 #13
0
ファイル: search.go プロジェクト: trythings/trythings
func (s *SearchService) Create(ctx context.Context, se *Search) error {
	span := trace.FromContext(ctx).NewChild("trythings.search.Create")
	defer span.Finish()

	if se.ID != "" {
		return fmt.Errorf("se already has id %q", se.ID)
	}

	if se.CreatedAt.IsZero() {
		se.CreatedAt = time.Now()
	}

	if se.Name == "" {
		return errors.New("Name is required")
	}

	if se.ViewID == "" {
		return errors.New("ViewID is required")
	}

	v, err := s.ViewService.ByID(ctx, se.ViewID)
	if err != nil {
		return err
	}

	if se.SpaceID == "" {
		se.SpaceID = v.SpaceID
	}

	if se.SpaceID != v.SpaceID {
		return errors.New("Search's SpaceID must match View's")
	}

	if len(se.ViewRank) != 0 {
		return fmt.Errorf("se already has a view rank %x", se.ViewRank)
	}

	// TODO#Performance: Add a shared or per-request cache to support these small, repeated queries.

	if se.Query == "" {
		return errors.New("Query is required")
	}

	rootKey := datastore.NewKey(ctx, "Root", "root", 0, nil)

	// Create a ViewRank for the search.
	// It should come after every other search in the view.
	var ranks []*struct {
		ViewRank datastore.ByteString
	}
	_, err = datastore.NewQuery("Search").
		Ancestor(rootKey).
		Filter("ViewID =", se.ViewID).
		Project("ViewRank").
		Order("-ViewRank").
		Limit(1).
		GetAll(ctx, &ranks)
	if err != nil {
		return err
	}

	maxViewRank := MinRank
	if len(ranks) != 0 {
		maxViewRank = Rank(ranks[0].ViewRank)
	}
	rank, err := NewRank(maxViewRank, MaxRank)
	if err != nil {
		return err
	}
	se.ViewRank = datastore.ByteString(rank)

	ok, err := s.IsVisible(ctx, se)
	if err != nil {
		return err
	}

	if !ok {
		return errors.New("cannot access view to create search")
	}

	id, _, err := datastore.AllocateIDs(ctx, "Search", nil, 1)
	if err != nil {
		return err
	}
	se.ID = fmt.Sprintf("%x", id)

	k := datastore.NewKey(ctx, "Search", se.ID, 0, rootKey)
	k, err = datastore.Put(ctx, k, se)
	if err != nil {
		return err
	}

	return nil
}
コード例 #14
0
ファイル: submission.go プロジェクト: pbochis/api
// PostSubmission creates a new submission.
func PostSubmission(ctx context.Context, w http.ResponseWriter, r *http.Request) (status int, err error) {
	if r.Method != "POST" {
		return http.StatusMethodNotAllowed, nil
	}

	var body = struct {
		Code     string
		Language string
	}{}

	if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
		return http.StatusBadRequest, err
	}

	p, ok := passenger.FromContext(ctx)
	if !ok {
		return http.StatusUnauthorized, nil
	}

	resultKey, err := datastore.DecodeKey(mux.Vars(r)["resultKey"])
	if err != nil {
		return http.StatusNotFound, err
	}

	if !util.HasParent(p.User, resultKey) {
		return http.StatusBadRequest, errors.New("cannot submit answer for other users")
	}

	taskKey, err := datastore.DecodeKey(mux.Vars(r)["taskKey"])
	if err != nil {
		return http.StatusNotFound, err
	}

	var task model.Task
	if err = datastore.Get(ctx, taskKey, &task); err != nil {
		return http.StatusInternalServerError, err
	}

	// Furthermore, the name of the GCS object is derived from the of the
	// encapsulating Submission. To avoid race conditions, allocate an ID.
	low, _, err := datastore.AllocateIDs(ctx, model.SubmissionKind, resultKey, 1)
	if err != nil {
		return http.StatusInternalServerError, err
	}

	submissionKey := datastore.NewKey(ctx, model.SubmissionKind, "", low, resultKey)

	submission := model.Submission{
		Task: taskKey,
		Time: time.Now(),
	}

	if body.Code != "" {
		submission.Code, err = store(ctx, submissionKey, body.Code, body.Language)
		if err != nil {
			return http.StatusInternalServerError, err
		}
		submission.Language = body.Language
	}

	// Set the submission in stone.
	if _, err = datastore.Put(ctx, submissionKey, &submission); err != nil {
		return http.StatusInternalServerError, err
	}

	var tests model.Tests
	_, err = model.NewQueryForTest().
		Ancestor(taskKey).
		GetAll(ctx, &tests)

	if err != nil {
		return http.StatusInternalServerError, err
	}

	for _, t := range tests {
		if err := test.Tester(t.Tester).Call(ctx, t.Params, *submission.Key(submissionKey)); err != nil {
			log.Warningf(ctx, "%s", err)
			continue
		}
	}

	// TODO(flowlo): Return something meaningful.

	return http.StatusOK, nil
}
コード例 #15
0
ファイル: map.go プロジェクト: pendo-io/mapreduce
func mapMonitorTask(c context.Context, ds appwrap.Datastore, pipeline MapReducePipeline, jobKey *datastore.Key, r *http.Request, timeout time.Duration) int {
	start := time.Now()

	job, err := waitForStageCompletion(c, ds, pipeline, jobKey, StageMapping, StageReducing, timeout)
	if err != nil {
		logCritical(c, "waitForStageCompletion() failed: %s", err)
		return 200
	} else if job.Stage == StageMapping {
		logInfo(c, "wait timed out -- returning an error and letting us automatically restart")
		return 500
	}

	logInfo(c, "map stage completed -- stage is now %s", job.Stage)

	// erm... we just did this in jobStageComplete. dumb to do it again
	mapTasks, err := gatherTasks(ds, job)
	if err != nil {
		logError(c, "failed loading tasks: %s", mapTasks)
		jobFailed(c, ds, pipeline, jobKey, fmt.Errorf("error loading tasks after map complete: %s", err.Error()))
		return 200
	}

	// we have one set for each reducer task
	storageNames := make([][]string, len(job.WriterNames))

	for i := range mapTasks {
		var shardNames map[string]int
		if err = json.Unmarshal([]byte(mapTasks[i].Result), &shardNames); err != nil {
			logError(c, `unmarshal error for result from map %d result '%+v'`, job.FirstTaskId+int64(i), mapTasks[i].Result)
			jobFailed(c, ds, pipeline, jobKey, fmt.Errorf("cannot unmarshal map shard names: %s", err.Error()))
			return 200
		} else {
			for name, shard := range shardNames {
				storageNames[shard] = append(storageNames[shard], name)
			}
		}
	}

	firstId, _, err := datastore.AllocateIDs(c, TaskEntity, nil, len(job.WriterNames))
	if err != nil {
		jobFailed(c, ds, pipeline, jobKey, fmt.Errorf("failed to allocate ids for reduce tasks: %s", err.Error()))
		return 200
	}
	taskKeys := makeTaskKeys(ds, firstId, len(job.WriterNames))
	tasks := make([]JobTask, 0, len(job.WriterNames))

	for shard := range job.WriterNames {
		if shards := storageNames[shard]; len(shards) > 0 {
			url := fmt.Sprintf("%s/reduce?taskKey=%s;shard=%d;writer=%s",
				job.UrlPrefix, taskKeys[len(tasks)].Encode(), shard, url.QueryEscape(job.WriterNames[shard]))

			firstId++

			shardJson, _ := json.Marshal(shards)
			shardZ := &bytes.Buffer{}
			w := zlib.NewWriter(shardZ)
			w.Write(shardJson)
			w.Close()

			tasks = append(tasks, JobTask{
				Status:              TaskStatusPending,
				Url:                 url,
				ReadFrom:            shardZ.Bytes(),
				SeparateReduceItems: job.SeparateReduceItems,
				Type:                TaskTypeReduce,
			})
		}
	}

	// this means we got nothing from maps. there is no result. so, we're done! right? that's hard to communicate though
	// so we'll just start a single task with no inputs
	if len(tasks) == 0 {
		logInfo(c, "no results from maps -- starting noop reduce task")
		url := fmt.Sprintf("%s/reduce?taskKey=%s;shard=%d;writer=%s",
			job.UrlPrefix, taskKeys[len(tasks)].Encode(), 0, url.QueryEscape(job.WriterNames[0]))

		tasks = append(tasks, JobTask{
			Status:              TaskStatusPending,
			Url:                 url,
			ReadFrom:            []byte(``),
			SeparateReduceItems: job.SeparateReduceItems,
			Type:                TaskTypeReduce,
		})
	}

	taskKeys = taskKeys[0:len(tasks)]

	if err := createTasks(ds, jobKey, taskKeys, tasks, StageReducing); err != nil {
		jobFailed(c, ds, pipeline, jobKey, fmt.Errorf("failed to create reduce tasks: %s", err.Error()))
		return 200
	}

	for i := range tasks {
		if err := pipeline.PostTask(c, tasks[i].Url, job.JsonParameters); err != nil {
			jobFailed(c, ds, pipeline, jobKey, fmt.Errorf("failed to post reduce task: %s", err.Error()))
			return 200
		}
	}

	if err := pipeline.PostStatus(c, fmt.Sprintf("%s/reduce-monitor?jobKey=%s", job.UrlPrefix, jobKey.Encode())); err != nil {
		jobFailed(c, ds, pipeline, jobKey, fmt.Errorf("failed to start reduce monitor: %s", err.Error()))
		return 200
	}

	logInfo(c, "mapping complete after %s of monitoring ", time.Now().Sub(start))
	return 200
}