Beispiel #1
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	q := datastore.NewQuery("F").KeysOnly().Filter("n <=", time.Now())
	q = q.Limit(10 * 60 * 2) // 10/s queue, 2 min cron
	it := q.Run(appengine.Timeout(c, time.Minute))
	tc := make(chan *taskqueue.Task)
	done := make(chan bool)
	i := 0
	u := routeUrl("update-feed")
	go taskSender(c, "update-feed", tc, done)
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		}
		tc <- taskqueue.NewPOSTTask(u, url.Values{
			"feed": {k.StringID()},
		})
		i++
	}
	close(tc)
	<-done
	c.Infof("updating %d feeds", i)
}
Beispiel #2
0
func UpdateFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(appengine.Timeout(c, time.Minute))
	url := r.FormValue("feed")
	if url == "" {
		c.Errorf("empty update feed")
		return
	}
	c.Debugf("update feed %s", url)
	last := len(r.FormValue("last")) > 0
	f := Feed{Url: url}
	s := ""
	defer func() {
		gn.Put(&Log{
			Parent: gn.Key(&f),
			Id:     time.Now().UnixNano(),
			Text:   "UpdateFeed - " + s,
		})
	}()
	if err := gn.Get(&f); err == datastore.ErrNoSuchEntity {
		c.Errorf("no such entity - " + url)
		s += "NSE"
		return
	} else if err != nil {
		s += "err - " + err.Error()
		return
	} else if last {
		// noop
	}
	if time.Now().Before(f.NextUpdate) {
		c.Errorf("feed %v already updated: %v", url, f.NextUpdate)
		s += "already updated"
		return
	}

	feedError := func(err error) {
		s += "feed err - " + err.Error()
		f.Errors++
		v := f.Errors + 1
		const max = 24 * 7
		if v > max {
			v = max
		} else if f.Errors == 1 {
			v = 0
		}
		f.NextUpdate = time.Now().Add(time.Hour * time.Duration(v))
		gn.Put(&f)
		c.Warningf("error with %v (%v), bump next update to %v, %v", url, f.Errors, f.NextUpdate, err)
	}

	if feed, stories, err := fetchFeed(c, f.Url, f.Url); err == nil {
		if err := updateFeed(c, f.Url, feed, stories, false, false, last); err != nil {
			feedError(err)
		} else {
			s += "success"
		}
	} else {
		feedError(err)
	}
	f.Subscribe(c)
}
Beispiel #3
0
func reportHandler(w http.ResponseWriter, r *http.Request) {
	if r.FormValue("date") == "" {
		var params = map[string]interface{}{
			"Yesterday": date.NowInPdt().AddDate(0, 0, -1),
		}
		if err := templates.ExecuteTemplate(w, "report-form", params); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return
	}

	c := appengine.Timeout(appengine.NewContext(r), 60*time.Second) // Default has a 5s timeout

	s, e, _ := widget.FormValueDateRange(r)
	opt := ReportOptions{
		ClassB_OnePerFlight:       widget.FormValueCheckbox(r, "classb_oneperflight"),
		ClassB_LocalDataOnly:      widget.FormValueCheckbox(r, "classb_localdataonly"),
		Skimmer_AltitudeTolerance: widget.FormValueFloat64(w, r, "skimmer_altitude_tolerance"),
		Skimmer_MinDurationNM:     widget.FormValueFloat64(w, r, "skimmer_min_duration_nm"),
	}

	if fix := strings.ToUpper(r.FormValue("waypoint")); fix != "" {
		if _, exists := sfo.KFixes[fix]; !exists {
			http.Error(w, fmt.Sprintf("Waypoint '%s' not known", fix), http.StatusInternalServerError)
			return
		}
		opt.Waypoint = fix
	}

	reportWriter(c, w, r, s, e, opt, r.FormValue("reportname"), r.FormValue("resultformat"))
}
Beispiel #4
0
// Upgrade the set of complaints for each user.
func upgradeUserHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.Timeout(appengine.NewContext(r), 30*time.Second)
	cdb := ComplaintDB{C: c, Memcache: false}

	p := types.ComplainerProfile{}
	if err := p.Base64Decode(r.FormValue("profile")); err != nil {
		c.Errorf("upgradeUserHandler: decode failed: %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// Get *all* the complaints for this person, unfiltered.
	var data = []types.Complaint{}
	q := datastore.
		NewQuery(kComplaintKind).
		Ancestor(cdb.emailToRootKey(p.EmailAddress)).
		Order("Timestamp")
	keys, err := q.GetAll(c, &data)
	if err != nil {
		c.Errorf("upgradeUserHandler/%s: GetAll failed: %v", p.EmailAddress, err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	nDeleted, nUpdated := 0, 0
	str := ""
	for i, complaint := range data {
		FixupComplaint(&complaint, keys[i]) // Put the key where cdb.* expect to find it

		deleteMe := i < len(data)-1 && ComplaintsAreEquivalent(data[i], data[i+1])
		noProfile := complaint.Profile.EmailAddress == ""
		//str += fmt.Sprintf(" [%03d] [DEL=%5v,PRO=%5v] %s\n", i, deleteMe, noProfile, complaint)

		if true {
			if deleteMe {
				if err := cdb.DeleteComplaints([]string{complaint.DatastoreKey}, p.EmailAddress); err != nil {
					c.Errorf("upgradeUserHandler/%s: deletecomplaints failed: %v", p.EmailAddress, err)
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
				nDeleted++

			} else if noProfile {
				complaint.Profile = p
				if err := cdb.UpdateComplaint(complaint, p.EmailAddress); err != nil {
					c.Errorf("upgradeUserHandler/%s: updatecomplaints failed: %v", p.EmailAddress, err)
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
				nUpdated++
			}
		}
	}

	str += fmt.Sprintf(" *** upgraded {%s} [tot=%d, del=%d, upd=%d]\n",
		p.EmailAddress, len(data), nDeleted, nUpdated)
	c.Infof(" -- Upgrade for %s --\n%s", p.EmailAddress, str)
	w.Write([]byte(fmt.Sprintf("OK, upgraded %s\n", p.EmailAddress)))
}
Beispiel #5
0
func downloadHandler(w http.ResponseWriter, r *http.Request) {
	session := sessions.Get(r)
	if session.Values["email"] == nil {
		http.Error(w, "session was empty; no cookie ? is this browser in privacy mode ?",
			http.StatusInternalServerError)
		return
	}

	c := appengine.Timeout(appengine.NewContext(r), 60*time.Second)

	cdb := complaintdb.ComplaintDB{C: c}
	cap, err := cdb.GetAllByEmailAddress(session.Values["email"].(string), true)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	_ = cap

	filename := date.NowInPdt().Format("complaints-20060102.csv")
	w.Header().Set("Content-Type", "application/csv")
	w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))

	cols := []string{
		"Date", "Time(PDT)", "Notes", "Speedbrakes", "Loudness", "Activity",
		"Flightnumber", "Origin", "Destination", "Speed(Knots)", "Altitude(Feet)",
		"Lat", "Long", "Registration", "Callsign",
		"VerticalSpeed(FeetPerMin)", "Dist2(km)", "Dist3(km)",
	}

	csvWriter := csv.NewWriter(w)
	csvWriter.Write(cols)

	for _, c := range cap.Complaints {
		a := c.AircraftOverhead
		speedbrakes := ""
		if c.HeardSpeedbreaks {
			speedbrakes = "y"
		}
		r := []string{
			c.Timestamp.Format("2006/01/02"),
			c.Timestamp.Format("15:04:05"),
			c.Description, speedbrakes, fmt.Sprintf("%d", c.Loudness), c.Activity,
			a.FlightNumber, a.Origin, a.Destination,
			fmt.Sprintf("%.0f", a.Speed), fmt.Sprintf("%.0f", a.Altitude),
			fmt.Sprintf("%.5f", a.Lat), fmt.Sprintf("%.5f", a.Long),
			a.Registration, a.Callsign, fmt.Sprintf("%.0f", a.VerticalSpeed),
			fmt.Sprintf("%.1f", c.Dist2KM), fmt.Sprintf("%.1f", c.Dist3KM),
		}

		if err := csvWriter.Write(r); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
	}
	csvWriter.Flush()
}
Beispiel #6
0
func NewDriver(ctx appengine.Context, kind string, logger wcg.Logger) *Driver {
	return &Driver{
		ctx:                    appengine.Timeout(ctx, DefaultTimeout),
		kind:                   kind,
		Logger:                 logger,
		EnableOperationLogging: DataStoreConfig.EnableOperationLogging,
		Timeout:                DefaultTimeout,
		namespace:              "",
		PutSplitThreshold:      DefaultPutSplitTreshold,
	}
}
Beispiel #7
0
func cannedSerfr1ComplaintsHandler(w http.ResponseWriter, r *http.Request) {
	s, e := date.WindowForYesterday()
	e = e.Add(-1 * time.Second)

	format := "list"
	if r.FormValue("csv") != "" {
		format = "csv"
	}

	c := appengine.Timeout(appengine.NewContext(r), 60*time.Second) // Default has a 5s timeout
	reportWriter(c, w, r, s, e, ReportOptions{}, "serfr1complaints", format)
}
Beispiel #8
0
func cannedAdsbClassBHandler(w http.ResponseWriter, r *http.Request) {
	s, e := date.WindowForYesterday()
	e = e.Add(-1 * time.Second)

	opt := ReportOptions{}
	format := "list"
	if r.FormValue("csv") != "" {
		format = "csv"
	}

	c := appengine.Timeout(appengine.NewContext(r), 60*time.Second) // Default has a 5s timeout
	reportWriter(c, w, r, s, e, opt, "adsbclassb", format)
}
Beispiel #9
0
func CronFetchHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.Timeout(appengine.NewContext(r), 30*time.Minute)

	q := datastore.NewQuery("Queue").KeysOnly()
	keys, _ := q.GetAll(c, nil)
	db := DB{c}
	start := time.Now()

	for _, v := range keys {
		db.FetchUser(strconv.FormatInt(v.IntID(), 10))
	}

	finish := time.Since(start)

	c.Infof("Finished cron fetch, took %s", finish)

	fmt.Fprint(w, "fetching users")
}
Beispiel #10
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	q := datastore.NewQuery("F").KeysOnly().Filter("n <=", time.Now())
	q = q.Limit(10 * 60 * 20) // 10/s queue, 20 min cron
	var keys []*datastore.Key
	it := q.Run(appengine.Timeout(c, time.Second*60))
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		}
		keys = append(keys, k)
	}

	if len(keys) == 0 {
		c.Errorf("no results")
		return
	}
	c.Infof("updating %d feeds", len(keys))

	var tasks []*taskqueue.Task
	for _, k := range keys {
		tasks = append(tasks, taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		}))
	}
	var ts []*taskqueue.Task
	const taskLimit = 100
	for len(tasks) > 0 {
		if len(tasks) > taskLimit {
			ts = tasks[:taskLimit]
			tasks = tasks[taskLimit:]
		} else {
			ts = tasks
			tasks = tasks[0:0]
		}
		if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
	}
}
func runTestTQ(
	c appengine.Context,
	r *http.Request,
) {
	//u.Debugf(c, "payload: %q", len(payload))
	c = appengine.Timeout(c, deadline)

	_, err := taskqueueAdd(c, &taskqueue.Task{
		Payload: payload,
		Method:  "PULL",
	}, "pull")
	if err != nil {
		u.Errorf(c, "%v", err)
		return
	}

	delayTestTQ.Call(c, "")
	return
}
Beispiel #12
0
func (g *Goon) putMemcache(srcs []interface{}, exists []byte) error {
	items := make([]*memcache.Item, len(srcs))
	payloadSize := 0
	for i, src := range srcs {
		toSerialize := src
		if exists[i] == 0 {
			toSerialize = nil
		}
		data, err := serializeStruct(toSerialize)
		if err != nil {
			g.error(err)
			return err
		}
		key, _, err := g.getStructKey(src)
		if err != nil {
			return err
		}
		// payloadSize will overflow if we push 2+ gigs on a 32bit machine
		payloadSize += len(data)
		items[i] = &memcache.Item{
			Key:   memkey(key),
			Value: data,
		}
	}
	memcacheTimeout := MemcachePutTimeoutSmall
	if payloadSize >= MemcachePutTimeoutThreshold {
		memcacheTimeout = MemcachePutTimeoutLarge
	}
	errc := make(chan error)
	go func() {
		errc <- memcache.SetMulti(appengine.Timeout(g.Context, memcacheTimeout), items)
	}()
	g.putMemoryMulti(srcs, exists)
	err := <-errc
	if appengine.IsTimeoutError(err) {
		g.timeoutError(err)
		err = nil
	} else if err != nil {
		g.error(err)
	}
	return err
}
Beispiel #13
0
func DeleteOldFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	ctx := appengine.Timeout(c, time.Minute)
	gn := goon.FromContext(c)
	q := datastore.NewQuery(gn.Kind(&Feed{})).Filter("n=", timeMax).KeysOnly()
	if cur, err := datastore.DecodeCursor(r.FormValue("c")); err == nil {
		q = q.Start(cur)
	}
	it := q.Run(ctx)
	done := false
	var tasks []*taskqueue.Task
	for i := 0; i < 10000 && len(tasks) < 100; i++ {
		k, err := it.Next(nil)
		if err == datastore.Done {
			c.Criticalf("done")
			done = true
			break
		} else if err != nil {
			c.Errorf("err: %v", err)
			continue
		}
		values := make(url.Values)
		values.Add("f", k.StringID())
		tasks = append(tasks, taskqueue.NewPOSTTask("/tasks/delete-old-feed", values))
	}
	if len(tasks) > 0 {
		c.Errorf("deleting %v feeds", len(tasks))
		if _, err := taskqueue.AddMulti(c, tasks, ""); err != nil {
			c.Errorf("err: %v", err)
		}
	}
	if !done {
		if cur, err := it.Cursor(); err == nil {
			values := make(url.Values)
			values.Add("c", cur.String())
			taskqueue.Add(c, taskqueue.NewPOSTTask("/tasks/delete-old-feeds", values), "")
		} else {
			c.Errorf("err: %v", err)
		}
	}
}
Beispiel #14
0
func DeleteOldFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	ctx := appengine.Timeout(c, time.Minute)
	g := goon.FromContext(ctx)
	oldDate := time.Now().Add(-time.Hour * 24 * 90)
	feed := Feed{Url: r.FormValue("f")}
	if err := g.Get(&feed); err != nil {
		c.Criticalf("err: %v", err)
		return
	}
	if feed.LastViewed.After(oldDate) {
		return
	}
	q := datastore.NewQuery(g.Kind(&Story{})).Ancestor(g.Key(&feed)).KeysOnly()
	keys, err := q.GetAll(ctx, nil)
	if err != nil {
		c.Criticalf("err: %v", err)
		return
	}
	q = datastore.NewQuery(g.Kind(&StoryContent{})).Ancestor(g.Key(&feed)).KeysOnly()
	sckeys, err := q.GetAll(ctx, nil)
	if err != nil {
		c.Criticalf("err: %v", err)
		return
	}
	keys = append(keys, sckeys...)
	c.Infof("delete: %v - %v", feed.Url, len(keys))
	feed.NextUpdate = timeMax.Add(time.Hour)
	if _, err := g.Put(&feed); err != nil {
		c.Criticalf("put err: %v", err)
	}
	if len(keys) == 0 {
		return
	}
	err = g.DeleteMulti(keys)
	if err != nil {
		c.Criticalf("err: %v", err)
	}
}
Beispiel #15
0
func DeleteBlobs(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	ctx := appengine.Timeout(c, time.Minute)
	q := datastore.NewQuery("__BlobInfo__").KeysOnly()
	it := q.Run(ctx)
	wg := sync.WaitGroup{}
	something := false
	for _i := 0; _i < 20; _i++ {
		var bk []appengine.BlobKey
		for i := 0; i < 1000; i++ {
			k, err := it.Next(nil)
			if err == datastore.Done {
				break
			} else if err != nil {
				c.Errorf("err: %v", err)
				continue
			}
			bk = append(bk, appengine.BlobKey(k.StringID()))
		}
		if len(bk) == 0 {
			break
		}
		go func(bk []appengine.BlobKey) {
			something = true
			c.Errorf("deleteing %v blobs", len(bk))
			err := blobstore.DeleteMulti(ctx, bk)
			if err != nil {
				c.Errorf("blobstore delete err: %v", err)
			}
			wg.Done()
		}(bk)
		wg.Add(1)
	}
	wg.Wait()
	if something {
		taskqueue.Add(c, taskqueue.NewPOSTTask("/tasks/delete-blobs", nil), "")
	}
}
func nukeDatastore(w http.ResponseWriter, r *http.Request) {
	c := appengine.Timeout(appengine.NewContext(r), 1*time.Hour)
	kind := r.FormValue("kind")
	c.Debugf("starting query")
	q := datastore.NewQuery(kind).KeysOnly().Limit(-1)
	keys, err := q.GetAll(c, nil)
	if err != nil {
		c.Errorf("query failed: %s", err)
		return
	}
	c.Debugf("query finished %d", len(keys))
	var wg sync.WaitGroup
	count := len(keys)
	batchSize := 500
	for i := 0; i < count; i += batchSize {
		wg.Add(1)
		j := i + batchSize
		if j > count {
			j = count
		}
		c.Infof("will delete keys [%d:%d]", i, j)
		go func(i, j int) {
			defer wg.Done()
			c.Infof("starting delete [%d:%d]", i, j)
			if err := datastore.DeleteMulti(c, keys[i:j]); err != nil {
				c.Errorf("failed deleting keys [%d:%d]: %s", i, j, err)
			} else {
				c.Infof("deleted keys [%d:%d]", i, j)
			}
		}(i, j)
	}

	c.Debugf("waiting...")
	wg.Wait()
	w.Write([]byte(fmt.Sprintf("nuked %s", kind)))
}
Beispiel #17
0
func zipHandler(w http.ResponseWriter, r *http.Request) {
	if r.FormValue("date") == "" {
		var params = map[string]interface{}{
			"Yesterday": date.NowInPdt().AddDate(0, 0, -1),
		}
		if err := templates.ExecuteTemplate(w, "zip-report-form", params); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return
	}

	ctx := appengine.Timeout(appengine.NewContext(r), 60*time.Second) // Default has a 5s timeout
	cdb := complaintdb.ComplaintDB{C: ctx, Memcache: false}

	zip := r.FormValue("zip")
	s, e, _ := widget.FormValueDateRange(r)

	var countsByHour [24]int
	countsByDate := map[string]int{}
	var uniquesByHour [24]map[string]int
	uniquesByDate := map[string]map[string]int{}
	uniquesAll := map[string]int{}

	iter := cdb.NewIter(cdb.QueryInSpanInZip(s, e, zip))
	for {
		c, err := iter.NextWithErr()
		if err != nil {
			http.Error(w, fmt.Sprintf("Zip iterator failed: %v", err), http.StatusInternalServerError)
			return
		} else if c == nil {
			break // We've hit EOF
		}

		h := c.Timestamp.Hour()
		countsByHour[h]++
		if uniquesByHour[h] == nil {
			uniquesByHour[h] = map[string]int{}
		}
		uniquesByHour[h][c.Profile.EmailAddress]++

		d := c.Timestamp.Format("2006.01.02")
		countsByDate[d]++
		if uniquesByDate[d] == nil {
			uniquesByDate[d] = map[string]int{}
		}
		uniquesByDate[d][c.Profile.EmailAddress]++

		uniquesAll[c.Profile.EmailAddress]++
	}

	dateKeys := []string{}
	for k, _ := range countsByDate {
		dateKeys = append(dateKeys, k)
	}
	sort.Strings(dateKeys)

	data := [][]string{}

	data = append(data, []string{"Date", "NumComplaints", "UniqueComplainers"})
	for _, k := range dateKeys {
		data = append(data, []string{
			k,
			fmt.Sprintf("%d", countsByDate[k]),
			fmt.Sprintf("%d", len(uniquesByDate[k])),
		})
	}
	data = append(data, []string{"------"})

	data = append(data, []string{"HourAcrossAllDays", "NumComplaints", "UniqueComplainers"})
	for i, v := range countsByHour {
		data = append(data, []string{
			fmt.Sprintf("%02d:00", i),
			fmt.Sprintf("%d", v),
			fmt.Sprintf("%d", len(uniquesByHour[i])),
		})
	}
	data = append(data, []string{"------"})
	data = append(data, []string{"UniqueComplainersAcrossAllDays", fmt.Sprintf("%d", len(uniquesAll))})

	var params = map[string]interface{}{"Data": data}
	if err := templates.ExecuteTemplate(w, "report", params); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}
Beispiel #18
0
func personalReportHandler(w http.ResponseWriter, r *http.Request) {
	session := sessions.Get(r)
	if session.Values["email"] == nil {
		http.Error(w, "session was empty; no cookie ?", http.StatusInternalServerError)
		return
	}
	email := session.Values["email"].(string)

	if r.FormValue("date") == "" {
		var params = map[string]interface{}{
			"Yesterday": date.NowInPdt().AddDate(0, 0, -1),
		}
		if err := templates.ExecuteTemplate(w, "personal-report-form", params); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return
	}

	start, end, _ := widget.FormValueDateRange(r)

	ctx := appengine.Timeout(appengine.NewContext(r), 60*time.Second)
	cdb := complaintdb.ComplaintDB{C: ctx}

	w.Header().Set("Content-Type", "text/plain")
	// w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", "sc.txt"))
	fmt.Fprintf(w, "Personal disturbances report for <%s>:\n From [%s]\n To   [%s]\n",
		email, start, end)

	complaintStrings := []string{}
	var countsByHour [24]int
	countsByDate := map[string]int{}
	countsByAirline := map[string]int{}

	iter := cdb.NewIter(cdb.QueryInSpanByEmailAddress(start, end, email))
	n := 0
	for {
		c := iter.Next()
		if c == nil {
			break
		}

		str := fmt.Sprintf("Time: %s, Loudness:%d, Speedbrakes:%v, Flight:%6.6s, Notes:%s",
			c.Timestamp.Format("2006.01.02 15:04:05"), c.Loudness, c.HeardSpeedbreaks,
			c.AircraftOverhead.FlightNumber, c.Description)

		n++
		complaintStrings = append(complaintStrings, str)

		countsByHour[c.Timestamp.Hour()]++
		countsByDate[c.Timestamp.Format("2006.01.02")]++
		if airline := c.AircraftOverhead.IATAAirlineCode(); airline != "" {
			countsByAirline[airline]++
		}
	}

	fmt.Fprintf(w, "\nTotal number of disturbance reports, over %d days:  %d\n",
		len(countsByDate), n)

	fmt.Fprintf(w, "\nDisturbance reports, counted by Airline (where known):\n")
	for _, k := range keysByIntValDesc(countsByAirline) {
		fmt.Fprintf(w, " %s: % 4d\n", k, countsByAirline[k])
	}

	fmt.Fprintf(w, "\nDisturbance reports, counted by date:\n")
	for _, k := range keysByKeyAsc(countsByDate) {
		fmt.Fprintf(w, " %s: % 4d\n", k, countsByDate[k])
	}
	fmt.Fprintf(w, "\nDisturbance reports, counted by hour of day (across all dates):\n")
	for i, n := range countsByHour {
		fmt.Fprintf(w, " %02d: % 4d\n", i, n)
	}
	fmt.Fprintf(w, "\nFull dump of all disturbance reports:\n\n")
	for _, s := range complaintStrings {
		fmt.Fprint(w, s+"\n")
	}
}
Beispiel #19
0
func monthTaskHandler(w http.ResponseWriter, r *http.Request) {
	//ctx,_ := context.WithTimeout(appengine.NewContext(r), 599*time.Second)
	ctx := appengine.NewContext(r)

	cdb := complaintdb.ComplaintDB{
		//C: oldappengine.NewContext(r),
		C: oldappengine.Timeout(oldappengine.NewContext(r), 599*time.Second),
	}

	year, err := strconv.ParseInt(r.FormValue("year"), 10, 64)
	if err != nil {
		http.Error(w, "need arg 'year' (2015)", http.StatusInternalServerError)
		return
	}
	month, err := strconv.ParseInt(r.FormValue("month"), 10, 64)
	if err != nil {
		http.Error(w, "need arg 'month' (1-12)", http.StatusInternalServerError)
		return
	}

	now := date.NowInPdt()
	s := time.Date(int(year), time.Month(month), 1, 0, 0, 0, 0, now.Location())
	e := s.AddDate(0, 1, 0).Add(-1 * time.Second)
	log.Infof(ctx, "Starting /be/month: %s", s)

	// One time, at 00:00, for each day of the given month
	days := date.IntermediateMidnights(s.Add(-1*time.Second), e)

	filename := s.Format("complaints-20060102") + e.Format("-20060102.csv")

	gcsHandle, err := gcs.OpenRW(ctx, "serfr0-reports", filename, "text/plain")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	csvWriter := csv.NewWriter(gcsHandle.IOWriter())

	cols := []string{
		"CallerCode", "Name", "Address", "Zip", "Email", "HomeLat", "HomeLong",
		"UnixEpoch", "Date", "Time(PDT)", "Notes", "ActivityDisturbed", "Flightnumber",
		"Notes",
		// Column names above are incorrect, but BKSV are used to them.
		//
		//"CallerCode", "Name", "Address", "Zip", "Email", "HomeLat", "HomeLong",
		//"UnixEpoch", "Date", "Time(PDT)", "Notes", "Flightnumber",
		//"ActivityDisturbed", "CcSFO",
	}
	csvWriter.Write(cols)

	for _, dayStart := range days {
		dayEnd := dayStart.AddDate(0, 0, 1).Add(-1 * time.Second)
		log.Infof(ctx, " /be/month: %s - %s", dayStart, dayEnd)

		iter := cdb.NewIter(cdb.QueryInSpan(dayStart, dayEnd))
		for {
			c, err := iter.NextWithErr()
			if err != nil {
				http.Error(w, fmt.Sprintf("iterator failed: %v", err),
					http.StatusInternalServerError)
				return
			}
			if c == nil {
				break
			}

			r := []string{
				c.Profile.CallerCode,
				c.Profile.FullName,
				c.Profile.Address,
				c.Profile.StructuredAddress.Zip,
				c.Profile.EmailAddress,
				fmt.Sprintf("%.4f", c.Profile.Lat),
				fmt.Sprintf("%.4f", c.Profile.Long),

				fmt.Sprintf("%d", c.Timestamp.UTC().Unix()),
				c.Timestamp.Format("2006/01/02"),
				c.Timestamp.Format("15:04:05"),
				c.Description,
				c.AircraftOverhead.FlightNumber,
				c.Activity,
				fmt.Sprintf("%v", c.Profile.CcSfo),
			}

			if err := csvWriter.Write(r); err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		}
	}
	csvWriter.Flush()

	if err := gcsHandle.Close(); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	log.Infof(ctx, "GCS report '%s' successfully written", filename)

	w.Header().Set("Content-Type", "text/plain")
	w.Write([]byte(fmt.Sprintf("OK!\nGCS file '%s' written to bucket", filename)))
}
Beispiel #20
0
func ListFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	cu := user.Current(c)
	gn := goon.FromContext(c)
	u := &User{Id: cu.ID}
	ud := &UserData{Id: "data", Parent: gn.Key(u)}
	if err := gn.GetMulti([]interface{}{u, ud}); err != nil && !goon.NotFound(err, 1) {
		serveError(w, err)
		return
	}
	l := &Log{
		Parent: ud.Parent,
		Id:     time.Now().UnixNano(),
		Text:   "list feeds",
	}
	l.Text += fmt.Sprintf(", len opml %v", len(ud.Opml))
	putU := false
	putUD := false
	fixRead := false
	if time.Since(u.Read) > oldDuration {
		u.Read = time.Now().Add(-oldDuration)
		putU = true
		fixRead = true
		l.Text += ", u.Read"
	}
	trialRemaining := 0
	if STRIPE_KEY != "" && ud.Opml != nil {
		if u.Created.IsZero() {
			u.Created = time.Now()
			putU = true
		} else if time.Since(u.Created) > accountFreeDuration {
			b, _ := json.Marshal(struct {
				ErrorSubscription bool
			}{
				true,
			})
			w.Write(b)
			return
		}
		trialRemaining = int((accountFreeDuration-time.Since(u.Created))/time.Hour/24) + 1
	}
	read := make(Read)
	var uf Opml
	c.Step("unmarshal user data", func(c mpg.Context) {
		gob.NewDecoder(bytes.NewReader(ud.Read)).Decode(&read)
		json.Unmarshal(ud.Opml, &uf)
	})
	var feeds []*Feed
	opmlMap := make(map[string]*OpmlOutline)
	var merr error
	c.Step("fetch feeds", func(c mpg.Context) {
		gn := goon.FromContext(appengine.Timeout(c, time.Minute))
		for _, outline := range uf.Outline {
			if outline.XmlUrl == "" {
				for _, so := range outline.Outline {
					feeds = append(feeds, &Feed{Url: so.XmlUrl})
					opmlMap[so.XmlUrl] = so
				}
			} else {
				feeds = append(feeds, &Feed{Url: outline.XmlUrl})
				opmlMap[outline.XmlUrl] = outline
			}
		}
		merr = gn.GetMulti(feeds)
	})
	lock := sync.Mutex{}
	fl := make(map[string][]*Story)
	q := datastore.NewQuery(gn.Key(&Story{}).Kind()).
		Filter(IDX_COL+" >=", u.Read).
		KeysOnly().
		Order("-" + IDX_COL).
		Limit(250)
	updatedLinks := false
	now := time.Now()
	numStories := 0

	c.Step(fmt.Sprintf("feed unreads: %v", u.Read), func(c mpg.Context) {
		queue := make(chan *Feed)
		tc := make(chan *taskqueue.Task)
		done := make(chan bool)
		wg := sync.WaitGroup{}
		feedProc := func() {
			for f := range queue {
				c.Step(f.Title, func(c mpg.Context) {
					defer wg.Done()
					var stories []*Story
					gn := goon.FromContext(appengine.Timeout(c, time.Minute))

					if !f.Date.Before(u.Read) {
						fk := gn.Key(f)
						sq := q.Ancestor(fk)
						keys, _ := gn.GetAll(sq, nil)
						stories = make([]*Story, len(keys))
						for j, key := range keys {
							stories[j] = &Story{
								Id:     key.StringID(),
								Parent: fk,
							}
						}
						gn.GetMulti(stories)
					}
					if f.Link != opmlMap[f.Url].HtmlUrl {
						l.Text += fmt.Sprintf(", link: %v -> %v", opmlMap[f.Url].HtmlUrl, f.Link)
						updatedLinks = true
						opmlMap[f.Url].HtmlUrl = f.Link
					}
					manualDone := false
					if time.Since(f.LastViewed) > time.Hour*24*2 {
						if f.NextUpdate.Equal(timeMax) {
							tc <- taskqueue.NewPOSTTask(routeUrl("update-feed-manual"), url.Values{
								"feed": {f.Url},
								"last": {"1"},
							})
							manualDone = true
						} else {
							tc <- taskqueue.NewPOSTTask(routeUrl("update-feed-last"), url.Values{
								"feed": {f.Url},
							})
						}
					}
					if !manualDone && now.Sub(f.NextUpdate) >= 0 {
						tc <- taskqueue.NewPOSTTask(routeUrl("update-feed-manual"), url.Values{
							"feed": {f.Url},
						})
					}
					lock.Lock()
					fl[f.Url] = stories
					numStories += len(stories)
					lock.Unlock()
				})
			}
		}
		go taskSender(c, "update-manual", tc, done)
		for i := 0; i < 20; i++ {
			go feedProc()
		}
		for i, f := range feeds {
			if goon.NotFound(merr, i) {
				continue
			}
			wg.Add(1)
			queue <- f
		}
		close(queue)
		// wait for feeds to complete so there are no more tasks to queue
		wg.Wait()
		// then finish enqueuing tasks
		close(tc)
		<-done
	})
	if numStories > 0 {
		c.Step("numStories", func(c mpg.Context) {
			stories := make([]*Story, 0, numStories)
			for _, v := range fl {
				stories = append(stories, v...)
			}
			sort.Sort(sort.Reverse(Stories(stories)))
			if len(stories) > numStoriesLimit {
				stories = stories[:numStoriesLimit]
				fl = make(map[string][]*Story)
				for _, s := range stories {
					fk := s.Parent.StringID()
					p := fl[fk]
					fl[fk] = append(p, s)
				}
			}
			last := stories[len(stories)-1].Created
			if u.Read.Before(last) {
				u.Read = last
				putU = true
				fixRead = true
			}
		})
	}
	if fixRead {
		c.Step("fix read", func(c mpg.Context) {
			nread := make(Read)
			for k, v := range fl {
				for _, s := range v {
					rs := readStory{Feed: k, Story: s.Id}
					if read[rs] {
						nread[rs] = true
					}
				}
			}
			if len(nread) != len(read) {
				read = nread
				var b bytes.Buffer
				gob.NewEncoder(&b).Encode(&read)
				ud.Read = b.Bytes()
				putUD = true
				l.Text += ", fix read"
			}
		})
	}
	numStories = 0
	for k, v := range fl {
		newStories := make([]*Story, 0, len(v))
		for _, s := range v {
			if !read[readStory{Feed: k, Story: s.Id}] {
				newStories = append(newStories, s)
			}
		}
		numStories += len(newStories)
		fl[k] = newStories
	}
	if numStories == 0 {
		l.Text += ", clear read"
		fixRead = false
		if ud.Read != nil {
			putUD = true
			ud.Read = nil
		}
		last := u.Read
		for _, v := range feeds {
			if last.Before(v.Date) {
				last = v.Date
			}
		}
		c.Infof("nothing here, move up: %v -> %v", u.Read, last)
		if u.Read.Before(last) {
			putU = true
			u.Read = last
		}
	}
	if updatedLinks {
		backupOPML(c)
		if o, err := json.Marshal(&uf); err == nil {
			ud.Opml = o
			putUD = true
			l.Text += ", update links"
		} else {
			c.Errorf("json UL err: %v, %v", err, uf)
		}
	}
	if putU {
		gn.Put(u)
		l.Text += ", putU"
	}
	if putUD {
		gn.Put(ud)
		l.Text += ", putUD"
	}
	l.Text += fmt.Sprintf(", len opml %v", len(ud.Opml))
	gn.Put(l)
	c.Step("json marshal", func(c mpg.Context) {
		gn := goon.FromContext(c)
		o := struct {
			Opml           []*OpmlOutline
			Stories        map[string][]*Story
			Options        string
			TrialRemaining int
			Feeds          []*Feed
		}{
			Opml:           uf.Outline,
			Stories:        fl,
			Options:        u.Options,
			TrialRemaining: trialRemaining,
			Feeds:          feeds,
		}
		b, err := json.Marshal(o)
		if err != nil {
			c.Errorf("cleaning")
			for _, v := range fl {
				for _, s := range v {
					n := sanitizer.CleanNonUTF8(s.Summary)
					if n != s.Summary {
						s.Summary = n
						c.Errorf("cleaned %v", s.Id)
						gn.Put(s)
					}
				}
			}
			b, _ = json.Marshal(o)
		}
		w.Write(b)
	})
}
Beispiel #21
0
func (d *Driver) SetTimeout(to time.Duration) *Driver {
	d.ctx = appengine.Timeout(d.ctx, to)
	return d
}
Beispiel #22
0
func report3Handler(w http.ResponseWriter, r *http.Request) {
	if r.FormValue("rep") == "" {
		var params = map[string]interface{}{
			"Yesterday": date.NowInPdt().AddDate(0, 0, -1),
			"Reports":   report.ListReports(),
			"FormUrl":   "/report3/",
			"Waypoints": sfo.ListWaypoints(),
		}
		if err := templates.ExecuteTemplate(w, "report3-form", params); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return
	}

	rep, err := report.SetupReport(r)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	if r.FormValue("debug") != "" {
		str := fmt.Sprintf("Report Options\n\n%s\n", rep.Options)
		w.Header().Set("Content-Type", "text/plain")
		w.Write([]byte(fmt.Sprintf("OK\n\n%s\n", str)))
		return
	}

	//airframes := ref.NewAirframeCache(c)
	client := newurlfetch.Client(newappengine.NewContext(r))
	metars, err := metar.FetchFromNOAA(client, "KSFO",
		rep.Start.AddDate(0, 0, -1), rep.End.AddDate(0, 0, 1))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	v1idspecs := []string{}
	v2idspecs := []string{}

	v1idspecComplement := []string{}

	reportFunc := func(oldF *oldfdb.Flight) {
		newF, err := oldF.V2()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		} else {
			newF.ComputeIndicatedAltitudes(metars)
			if included, err := rep.Process(newF); err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
			} else if included {
				v1idspecs = append(v1idspecs, oldF.UniqueIdentifier())
				v2idspecs = append(v2idspecs, newF.IdSpec())
			} else {
				v1idspecComplement = append(v1idspecComplement, oldF.UniqueIdentifier())
			}
		}
	}

	tags := tagList(rep.Options)

	db := oldfgae.FlightDB{C: oldappengine.Timeout(oldappengine.NewContext(r), 600*time.Second)}
	s, e := rep.Start, rep.End
	if err := db.LongIterWith(db.QueryTimeRangeByTags(tags, s, e), reportFunc); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	postButtons := ButtonPOST(fmt.Sprintf("%d Matches as a VectorMap", len(v1idspecs)),
		fmt.Sprintf("/fdb/trackset2?%s", rep.ToCGIArgs()), v1idspecs)
	postButtons += ButtonPOST(fmt.Sprintf("%d Non-matches as a VectorMap", len(v1idspecComplement)),
		fmt.Sprintf("/fdb/trackset2?%s", rep.ToCGIArgs()), v1idspecComplement)
	if rep.Name == "sfoclassb" {
		postButtons += ButtonPOST(fmt.Sprintf("%d Matches as ClassBApproaches", len(v1idspecs)),
			fmt.Sprintf("/fdb/approach2?%s", rep.ToCGIArgs()), v1idspecs)
		// This is kinda useless, as it isn't limited to SERFR1 things
		postButtons += ButtonPOST(fmt.Sprintf("%d Non-matches as ClassBApproaches", len(v1idspecComplement)),
			fmt.Sprintf("/fdb/approach2?%s", rep.ToCGIArgs()), v1idspecComplement)

		postButtons += ButtonPOST(fmt.Sprintf("%d Matches as ClassBApproaches (delta)",
			len(v1idspecs)), fmt.Sprintf("/fdb/approach2?%s&colorby=delta", rep.ToCGIArgs()), v1idspecs)
		// This is kinda useless, as it isn't limited to SERFR1 things
		postButtons += ButtonPOST(fmt.Sprintf("%d Non-matches as ClassBApproaches (delta)",
			len(v1idspecComplement)), fmt.Sprintf("/fdb/approach2?%s&colorby=delta",
			rep.ToCGIArgs()), v1idspecComplement)
	}

	var params = map[string]interface{}{
		"R":           rep,
		"Metadata":    rep.MetadataTable(),
		"PostButtons": template.HTML(postButtons),
		"OptStr":      template.HTML(fmt.Sprintf("<pre>%s</pre>\n", rep.Options)),
		"IdSpecs":     template.HTML(strings.Join(v1idspecs, ",")),
	}
	if err := templates.ExecuteTemplate(w, "report3-results", params); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

}
Beispiel #23
0
func CFixer(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	q := datastore.NewQuery("F").KeysOnly()
	q = q.Limit(1000)
	cs := r.FormValue("c")
	if len(cs) > 0 {
		if cur, err := datastore.DecodeCursor(cs); err == nil {
			q = q.Start(cur)
			c.Infof("starting at %v", cur)
		} else {
			c.Errorf("cursor error %v", err.Error())
		}
	}
	var keys []*datastore.Key
	it := q.Run(appengine.Timeout(c, time.Second*15))
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		}
		keys = append(keys, k)
	}

	if len(keys) == 0 {
		c.Errorf("no results")
		return
	} else {
		cur, err := it.Cursor()
		if err != nil {
			c.Errorf("to cur error %v", err.Error())
		} else {
			c.Infof("add with cur %v", cur)
			t := taskqueue.NewPOSTTask("/tasks/cfixer", url.Values{
				"c": {cur.String()},
			})
			taskqueue.Add(c, t, "cfixer")
		}
	}
	c.Infof("fixing %d feeds", len(keys))

	var tasks []*taskqueue.Task
	for _, k := range keys {
		c.Infof("f: %v", k.StringID())
		tasks = append(tasks, taskqueue.NewPOSTTask("/tasks/cfix", url.Values{
			"feed": {k.StringID()},
		}))
	}
	var ts []*taskqueue.Task
	const taskLimit = 100
	for len(tasks) > 0 {
		if len(tasks) > taskLimit {
			ts = tasks[:taskLimit]
			tasks = tasks[taskLimit:]
		} else {
			ts = tasks
			tasks = tasks[0:0]
		}
		if _, err := taskqueue.AddMulti(c, ts, "cfixer"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
	}
}
Beispiel #24
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	q := datastore.NewQuery("F").KeysOnly().Filter("n <=", time.Now())
	q = q.Limit(1000)
	cs := r.FormValue("c")
	hasCursor := false
	if len(cs) > 0 {
		if cur, err := datastore.DecodeCursor(cs); err == nil {
			q = q.Start(cur)
			hasCursor = true
			c.Infof("starting at %v", cur)
		} else {
			c.Errorf("cursor error %v", err.Error())
		}
	}
	if !hasCursor && !isDevServer {
		qs, err := taskqueue.QueueStats(c, []string{"update-feed"}, 0)
		if err != nil || qs[0].Tasks > 0 || qs[0].Executed1Minute > 0 {
			c.Infof("already %v (%v) tasks", qs[0].Tasks, qs[0].Executed1Minute)
			return
		}
	}
	var keys []*datastore.Key
	it := q.Run(appengine.Timeout(c, time.Second*60))
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		}
		keys = append(keys, k)
	}

	if len(keys) == 0 {
		c.Errorf("no results")
		return
	} else {

	}
	c.Infof("updating %d feeds", len(keys))

	var tasks []*taskqueue.Task
	for _, k := range keys {
		tasks = append(tasks, taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		}))
	}
	cur, err := it.Cursor()
	if err != nil {
		c.Errorf("to cur error %v", err.Error())
	} else {
		c.Infof("add with cur %v", cur)
		tasks = append(tasks, taskqueue.NewPOSTTask(routeUrl("update-feeds"), url.Values{
			"c": {cur.String()},
		}))
	}
	var ts []*taskqueue.Task
	const taskLimit = 100
	for len(tasks) > 0 {
		if len(tasks) > taskLimit {
			ts = tasks[:taskLimit]
			tasks = tasks[taskLimit:]
		} else {
			ts = tasks
			tasks = tasks[0:0]
		}
		if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
	}
}
Beispiel #25
0
func Timeout(req *wcg.Request, t time.Duration) appengine.Context {
	ctx := NewContext(req)
	return appengine.Timeout(ctx, t)
}
Beispiel #26
0
// GetMulti is a batch version of Get.
//
// dst must be a *[]S, *[]*S, *[]I, []S, []*S, or []I, for some struct type S,
// or some interface type I. If *[]I or []I, each element must be a struct pointer.
func (g *Goon) GetMulti(dst interface{}) error {
	keys, err := g.extractKeys(dst, false) // don't allow incomplete keys on a Get request
	if err != nil {
		return err
	}

	v := reflect.Indirect(reflect.ValueOf(dst))

	if g.inTransaction {
		// todo: support getMultiLimit in transactions
		return datastore.GetMulti(g.Context, keys, v.Interface())
	}

	var dskeys []*datastore.Key
	var dsdst []interface{}
	var dixs []int

	var memkeys []string
	var mixs []int

	g.cacheLock.RLock()
	for i, key := range keys {
		m := memkey(key)
		vi := v.Index(i)

		if vi.Kind() == reflect.Struct {
			vi = vi.Addr()
		}

		if s, present := g.cache[m]; present {
			if vi.Kind() == reflect.Interface {
				vi = vi.Elem()
			}

			reflect.Indirect(vi).Set(reflect.Indirect(reflect.ValueOf(s)))
		} else {
			memkeys = append(memkeys, m)
			mixs = append(mixs, i)
			dskeys = append(dskeys, key)
			dsdst = append(dsdst, vi.Interface())
			dixs = append(dixs, i)
		}
	}
	g.cacheLock.RUnlock()

	if len(memkeys) == 0 {
		return nil
	}

	multiErr, any := make(appengine.MultiError, len(keys)), false

	memvalues, err := memcache.GetMulti(appengine.Timeout(g.Context, MemcacheGetTimeout), memkeys)
	if appengine.IsTimeoutError(err) {
		g.timeoutError(err)
		err = nil
	} else if err != nil {
		g.error(err) // timing out or another error from memcache isn't something to fail over, but do log it
		// No memvalues found, prepare the datastore fetch list already prepared above
	} else if len(memvalues) > 0 {
		// since memcache fetch was successful, reset the datastore fetch list and repopulate it
		dskeys = dskeys[:0]
		dsdst = dsdst[:0]
		dixs = dixs[:0]
		// we only want to check the returned map if there weren't any errors
		// unlike the datastore, memcache will return a smaller map with no error if some of the keys were missed

		for i, m := range memkeys {
			d := v.Index(mixs[i]).Interface()
			if v.Index(mixs[i]).Kind() == reflect.Struct {
				d = v.Index(mixs[i]).Addr().Interface()
			}
			if s, present := memvalues[m]; present {
				err := deserializeStruct(d, s.Value)
				if err == datastore.ErrNoSuchEntity {
					any = true // this flag tells GetMulti to return multiErr later
					multiErr[mixs[i]] = err
				} else if err != nil {
					g.error(err)
					return err
				} else {
					g.putMemory(d)
				}
			} else {
				dskeys = append(dskeys, keys[mixs[i]])
				dsdst = append(dsdst, d)
				dixs = append(dixs, mixs[i])
			}
		}
		if len(dskeys) == 0 {
			if any {
				return realError(multiErr)
			}
			return nil
		}
	}

	goroutines := (len(dskeys)-1)/getMultiLimit + 1
	var wg sync.WaitGroup
	wg.Add(goroutines)
	for i := 0; i < goroutines; i++ {
		go func(i int) {
			defer wg.Done()
			var toCache []interface{}
			var exists []byte
			lo := i * getMultiLimit
			hi := (i + 1) * getMultiLimit
			if hi > len(dskeys) {
				hi = len(dskeys)
			}
			gmerr := datastore.GetMulti(g.Context, dskeys[lo:hi], dsdst[lo:hi])
			if gmerr != nil {
				any = true // this flag tells GetMulti to return multiErr later
				merr, ok := gmerr.(appengine.MultiError)
				if !ok {
					g.error(gmerr)
					for j := lo; j < hi; j++ {
						multiErr[j] = gmerr
					}
					return
				}
				for i, idx := range dixs[lo:hi] {
					if merr[i] == nil {
						toCache = append(toCache, dsdst[lo+i])
						exists = append(exists, 1)
					} else {
						if merr[i] == datastore.ErrNoSuchEntity {
							toCache = append(toCache, dsdst[lo+i])
							exists = append(exists, 0)
						}
						multiErr[idx] = merr[i]
					}
				}
			} else {
				toCache = append(toCache, dsdst[lo:hi]...)
				exists = append(exists, bytes.Repeat([]byte{1}, hi-lo)...)
			}
			if len(toCache) > 0 {
				if err := g.putMemcache(toCache, exists); err != nil {
					g.error(err)
					// since putMemcache() gives no guarantee it will actually store the data in memcache
					// we log and swallow this error
				}

			}
		}(i)
	}
	wg.Wait()
	if any {
		return realError(multiErr)
	}
	return nil
}