func init() { http.Handle("/admin/commit/load", appstats.NewHandler(startLoad)) http.Handle("/admin/commit/kickoff", appstats.NewHandler(initialLoad)) http.Handle("/admin/commit/status", appstats.NewHandler(status)) http.Handle("/admin/commit/show/", appstats.NewHandler(show)) laterLoad = delay.Func("commit.load", load) laterLoadRev = delay.Func("commit.loadrev", loadRev) }
func uploadHistory(res http.ResponseWriter, req *http.Request) { c := appengine.NewContext(req) password := req.FormValue("password") if password != os.Getenv("SERVER_PASSWORD") { http.Error(res, "uh oh", http.StatusForbidden) return } gobFile, _, err := req.FormFile("gob_file") if err != nil { http.Error(res, "missing file", http.StatusBadRequest) return } defer gobFile.Close() commands := []History{} dec := gob.NewDecoder(gobFile) ctr := 0 for { var h History err := dec.Decode(&h) if err == io.EOF { break } if err != nil { http.Error(res, err.Error(), http.StatusInternalServerError) return } commands = append(commands, h) ctr += 1 } increment := 500 tasks := 0 for i := 0; i < len(commands); i += increment { start := i end := i + 500 if end > len(commands) { end = len(commands) } var laterFunc = delay.Func("key", importHistory) laterFunc.Call(c, commands[start:end]) tasks += 1 } fmt.Fprintf(res, "hi :) queued %d tasks to import %d elements for import\n", tasks, ctr) }
var updateSeries = delay.Func("Fred", func(c appengine.Context, z string) { u := urlForSeries(z) body, err := fetch(u, c) if err != nil { panic(err) } var response FredResponse if err = json.Unmarshal(body, &response); err != nil { panic(err) } now := time.Now() starttime, _ := time.Parse(dateformat, "2010-07-18") datalist := make(map[string]float64) for day := starttime; day.Before(now); day = day.AddDate(0, 0, 1) { datalist[day.Format(dateformat)] = math.NaN() } for _, obs := range response.Observations { if value, err := strconv.ParseFloat(obs.Value, 64); err != nil { // c.Infof("error: %v",err) } else { datalist[obs.Date] = value } } var data DataSet for k, v := range datalist { l := math.Log(v) d := DataPoint{ Date: k, Price: v, Logprice: l, } data = append(data, d) } sort.Sort(DataSet(data)) // ideally, we'd use multiple imputation to fill in weekends and holidays // must find Go package for multiple imputation i := 1 for i < len(data) { data[i].Return = data[i].Logprice - data[i-1].Logprice i += 1 } j := 29 for j < len(data) { subset := data[j-29 : j] var returns []float64 for _, point := range subset { if !math.IsNaN(point.Return) { returns = append(returns, point.Return) } } data[j].Volatility = stats.StatsSampleStandardDeviation(returns) * 100.0 j += 1 } k := 59 for k < len(data) { subset := data[k-59 : k] var returns []float64 for _, point := range subset { if !math.IsNaN(point.Return) { returns = append(returns, point.Return) } } data[k].Volatility60 = stats.StatsSampleStandardDeviation(returns) * 100.0 k += 1 } d := StoredDataSet{ Data: data, } if _, err := datastore.Put(c, datastore.NewKey(c, "StoredDataSet", z, 0, nil), &d); err != nil { panic(err) } })
Freq: kanji.Freq, JLPT: kanji.JLPT, } for _, r := range kanji.Readings { if r.RType == "ja_on" { k.Onyomi = append(k.Onyomi, r.Value) } if r.RType == "ja_kun" { k.Kunyomi = append(k.Kunyomi, r.Value) } } for _, m := range kanji.Meanings { // English meanings do not have an m_lang attribute if m.MLang == "" { k.Meanings = append(k.Meanings, m.Value) } } populateLater.Call(c, k) } } var populateLater = delay.Func("populate", func(c appengine.Context, k Kanji) { key := datastore.NewKey(c, "Kanji", k.Literal, 0, nil) _, err := datastore.Put(c, key, &k) c.Infof("Added kanji %s", k.Literal) if err != nil { c.Errorf(err.Error()) return } })
"math/rand" "resize" ) const BASEURL = "http://pitcam.ccs.neu.edu/%s.jpeg?client=pitcamproxy&rid=%s" // This should be set to the IP of the remote authorized download ping host const REMOTE_PINGBACK_IP = "127.0.0.1" func init() { http.HandleFunc("/random.jpeg", randomredirect) http.HandleFunc("/_private/newimgpingback", pingbackHandler) http.HandleFunc("/", handler) } var downloadLater = delay.Func("imageDownloader", downloadHandler) func pingbackHandler(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) switch r.RemoteAddr { case REMOTE_PINGBACK_IP: c.Infof("pitcam pingback") downloadLater.Call(c) default: c.Errorf("unauthenticated pingback from %s", r.RemoteAddr) } } func downloadHandler(c appengine.Context) { c.Infof("Downloader Called")
func init() { saveLater = delay.Func("save", save) }
import ( "fmt" "http" "appengine" "appengine/delay" ) func init() { http.HandleFunc("/", handle) } func handle(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/" { // ignore /favicon.ico, etc. return } w.Header().Set("Content-Type", "text/plain; charset=utf-8") fmt.Fprint(w, "I will run a function later.\n") c := appengine.NewContext(r) later.Call(c, "Please send my regards to your mother.") } // A function that can be executed later. var later = delay.Func("later", func(c appengine.Context, msg string) { c.Infof("later, %q", msg) })
package job import ( "math/rand" "time" "appengine" "appengine/datastore" "appengine/delay" ) var processJobDelay = delay.Func("processJob", processJob) func processJob(c appengine.Context, jobID string) (err error) { task := TaskNone taskState := &taskState{} state := &State{ID: jobID} for task != TaskHaltProcessing { // Run the next state of transactional processing of the job, // to find out the next task to do and update records. err = datastore.RunInTransaction(c, func(c appengine.Context) error { // Update our copy of the job's state. if err := datastore.Get(c, state.GetKey(c), state); err != nil { return err } // Process it and get the next task to peform. var err error
c := appengine.NewContext(r) q := datastore.NewQuery("Package").Filter("Kind=", "external").KeysOnly() for t := q.Run(c); ; { key, err := t.Next(nil) if err == datastore.Done { break } else if err != nil { c.Errorf("%v", err) return } updateWeeklyLater.Call(c, key) } } var ( installLater = delay.Func("install", install) updateWeeklyLater = delay.Func("updateWeekly", updateWeekly) ) // install validates the provided package path, increments its install count, // and creates the Package record if it doesn't exist. func install(c appengine.Context, path string) { if !validPath(c, path) { return } tx := func(c appengine.Context) error { p := &Package{Path: path, Kind: "external"} err := datastore.Get(c, p.Key(c), p) if err != nil && err != datastore.ErrNoSuchEntity { return err }
buf := new(bytes.Buffer) _, err = io.Copy(buf, resp.Body) if err != nil { c.Errorf("io.Copy", err) return } att := &mail.Attachment{ Name: filename, Data: buf.Bytes(), } //c.Infof(string(att.Data)) msg := "Deliverd successfully. Here is your file." sendEmail(att, email, msg, c) } var later = delay.Func("later", doAnt) func init() { http.HandleFunc("/", indexHandler) http.HandleFunc("/robots.txt", robotsHandler) } func robotsHandler(w http.ResponseWriter, r *http.Request) { io.WriteString(w, robotsPage) } func indexHandler(w http.ResponseWriter, r *http.Request) { if r.Method == "GET" { io.WriteString(w, indexPage) } if r.Method == "POST" {
var laterSaveInvoiceRippleEffects = delay.Func("later-save-invoice-ripple-effects", func(c appengine.Context, invoice *Invoice) error { // 1. Find pending orders // 2. Sort them by date // 3. Start filling the ShippedItemWithInvId in Order // 4. Mark if they are complete // 5. Repeat until all the invoice items have been exhausted. // 6. Report error if you have items left in invoice but no corresponding order for it. pendingOrders, err := getPendingOrdersForPurchaser(c, invoice.PurchaserId) if err != nil { return err } Orders(pendingOrders).Sort() unmappedInvoiceItems := append(Items(nil), invoice.Items...) for oi := range pendingOrders { pendingOrder := &pendingOrders[oi] for ii := range pendingOrder.PendingItems { pendingItem := &pendingOrder.PendingItems[ii] for umi := range unmappedInvoiceItems { unmappedInvoiceItem := &unmappedInvoiceItems[umi] if pendingItem.Qty == 0 { continue } if pendingItem.Qty == 0 { continue } if (*pendingItem).skuEquals(*unmappedInvoiceItem) { //c.Infof("__________________________________") //c.Infof("unmappedInvoiceItem %+v", unmappedInvoiceItem) //c.Infof("pendingItem%+v", pendingItem) //Find minimum of the two quantities minQty := min(pendingItem.Qty, unmappedInvoiceItem.Qty) //Prepare siwi var siwi ShippedItemWithInvId siwi.Item = *unmappedInvoiceItem siwi.Item.Qty = minQty siwi.InvoiceId = invoice.Id //c.Infof("Created siwi %+v", siwi) //Reduce pending pendingItem.Qty -= minQty pendingOrder.ShippedItemsWithInvId = append(pendingOrder.ShippedItemsWithInvId, siwi) //c.Infof("pendingOrder.ShippedItemsWithInvId became %+v", pendingOrder.ShippedItemsWithInvId) //Reduce unmappedInvoiceItem.Qty unmappedInvoiceItem.Qty -= minQty //Store Order id in Invoice for future reference. invoice.OrdersId = append(invoice.OrdersId, pendingOrder.Id) } } } } for i := range pendingOrders { pendingOrder := &pendingOrders[i] c.Infof("___________________________________________") c.Infof("\nBefore Filtering pendingOrder.PendingItems = %#v", pendingOrder.PendingItems) EmptyItems := func(item Item) bool { return item.Qty == 0 } c.Infof("\nafter Filtering pendingOrder.PendingItems = %#v", pendingOrder.PendingItems) pendingOrder.PendingItems = pendingOrder.PendingItems.Filter(EmptyItems) if len(pendingOrder.PendingItems) == 0 { pendingOrder.IsComplete = true } if _, err := pendingOrder.save(c); err != nil { //TODO: If there are too many pending orders for a purchaser, you might want to set a dirty flag or multi save the entities. But this is unlikely. return err } } if _, err := invoice.save(c); err != nil { return err } return nil })
import ( "appengine/delay" "fmt" "github.com/scotch/aego/v1/config" "github.com/scotch/aego/v1/context" "github.com/scotch/aego/v1/mail" "github.com/scotch/aego/v1/user/token" "net/http" ) const ( URL_PREFIX = "/-/email/verify" ) var SendConfirmAddressLater = delay.Func("confirmaddress", SendConfirmAddress) var defaultConfig = map[string]string{ "SiteTitle": "Company Name", "SiteURL": "http://*****:*****@example.com>", "Subject": "Please verify your email '%s'", "BodyTmplPath": "github.com/scotch/aego/v1/user/email/templates/confirm.txt", "BodyHTMLTmplPath": "github.com/scotch/aego/v1/user/email/templates/confirm.html", } func init() { http.HandleFunc(URL_PREFIX, verifyHandler) } func genVerifyURL(baseURL string, token string) string {
var ( ErrUserNotFound = fmt.Errorf("User not found") ErrWrongConfirmationCode = fmt.Errorf("Wrong confirmation code") ErrEmailAlreadyClaimed = fmt.Errorf("The email is already claimed") ) var ( delayedConfirmAccount = delay.Func("confirmAccount", func(c appengine.Context, user Account) error { buf := &bytes.Buffer{} if err := accountConfirmationEmail.Execute(buf, user); err != nil { c.Criticalf("Couldn't execute account confirm email: %s", err) return nil } msg := &mail.Message{ Sender: fmt.Sprintf("no-reply@%s.appspotmail.com", appengine.AppID(c)), To: []string{user.Email}, Subject: "Confirm your account registration with Inner Hearth Yoga", Body: buf.String(), } if err := mail.Send(c, msg); err != nil { c.Criticalf("Couldn't send email to %q: %s", user.Email, err) return fmt.Errorf("failed to send email") } return nil }) ) // Info stores basic user contact & identification data. type Info struct { FirstName string `datastore: ",noindex"` LastName string Email string
} toDelete = append(toDelete, peerKey) } deleted := len(toDelete) c.Infof("Deleting %d expired peers", deleted) if err := ds.DeleteMulti(c, toDelete); err != nil { c.Criticalf("Failed to delete peers: %#v (%s)", err, err) return } if deleted <= 0 { break } } c.Infof("Finished deleting expired peers") } var expireDelay = delay.Func("doExpire", doExpire) func expire(c appengine.Context, w http.ResponseWriter, r *http.Request) { expireDelay.Call(c, time.Now(), "") w.WriteHeader(200) io.WriteString(w, "Expiring peers...") c.Debugf("Successfully began expire") }
u.Infof("Loaded via datastore config[%s] = %q, modified = %s", key, e.Value, e.Modifed) gaeConfigs[key] = e.Value return e.Value } var gaeDelayFunc = delay.Func(GAE_DELAY_KEY, func(c appengine.Context, delayFuncArgs ...interface{}) error { handlerName, ok := delayFuncArgs[0].(string) if !ok { c.Errorf("GAE.Delay: handler name could not be extracted from %v", delayFuncArgs) return nil } handler, ok := DelayHandlers[handlerName] if !ok { c.Errorf("GAE.Delay: handler %s could not be found", handlerName) return nil } u := new(GAE) u.context = c args := delayFuncArgs[1:] Verbosef(u, "GAE.Delay: executing %s(%v)", handlerName, &args) return handler(u, args...) }) func (u GAE) Delay(handlerName string, args ...interface{}) error { delayFuncArgs := append([]interface{}{handlerName}, args...) gaeDelayFunc.Call(u.context, delayFuncArgs...) Verbosef(u, "GAE.Delay: delaying %s(%v)", handlerName, &args)
sum += item.Price } fmt.Fprintln(&b, strings.Repeat("-", 55)) fmt.Fprintf(&b, "\tTotal:\t\t\t\t\t$%.2f\n", sum) w.Write(b.Bytes()) sendReceipt.Call(c, user.Current(c).Email, b.String()) } var sendReceipt = delay.Func("send-receipt", func(c appengine.Context, dst, body string) { msg := &mail.Message{ Sender: appAdmin, To: []string{dst}, Subject: "Your Gopher Mart receipt", Body: body, } if err := mail.Send(c, msg); err != nil { c.Errorf("mail.Send: %v", err) } }) func adminPopulate(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) for i := range [numItems]struct{}{} { // r hates this. tee hee. key := itemKey(c, i) good := goods[rand.Intn(len(goods))] item := &Item{ // TODO: vary names more Name: fmt.Sprintf("%s %dg", good.name, i+1), Price: float64(rand.Intn(1999)+1) / 100,
func init() { laterUpdateKind = delay.Func("app.backgroundUpdateKind", backgroundUpdateKind) }
// license that can be found in the LICENSE file. /* Package mail provides methods for sending e-mails */ package mail import ( "appengine/delay" "appengine/mail" "bytes" "github.com/scotch/aego/v1/context" "html/template" ) var SendLater = delay.Func("send", Send) type TemplateMessage struct { *mail.Message BodyValues map[string]string BodyTmplPath string HTMLBodyTmplPath string } func parseTmpl(tmplPath string, values map[string]string) (s string, err error) { var b bytes.Buffer tmpl := template.Must(template.ParseFiles(tmplPath)) if err := tmpl.Execute(&b, values); err != nil { return s, err } return string(b.Bytes()), nil
return fmt.Errorf("finding result for %q: %+v", builder, com) } return commonNotify(c, broken, builder, r.LogHash) } // firstMatch executes the query q and loads the first entity into v. func firstMatch(c appengine.Context, q *datastore.Query, v interface{}) error { t := q.Limit(1).Run(c) _, err := t.Next(v) if err == datastore.Done { err = datastore.ErrNoSuchEntity } return err } var notifyLater = delay.Func("notify", notify) // notify tries to update the CL for the given Commit with a failure message. // If it doesn't succeed, it sends a failure email to golang-dev. func notify(c appengine.Context, com *Commit, builder, logHash string) { if !updateCL(c, com, builder, logHash) { // Send a mail notification if the CL can't be found. sendFailMail(c, com, builder, logHash) } } // updateCL updates the CL for the given Commit with a failure message // for the given builder. func updateCL(c appengine.Context, com *Commit, builder, logHash string) bool { cl, err := lookupCL(c, com) if err != nil {
func handleDelay(w http.ResponseWriter, r *http.Request) { var later = delay.Func("key", delayedFunction) later.Call(appengine.NewContext(r), "the data") }
} return err } // firstMatch executes the query q and loads the first entity into v. func firstMatch(c appengine.Context, q *datastore.Query, v interface{}) error { t := q.Limit(1).Run(c) _, err := t.Next(v) if err == datastore.Done { err = datastore.ErrNoSuchEntity } return err } var ( sendFailMailLater = delay.Func("sendFailMail", sendFailMail) sendFailMailTmpl = template.Must( template.New("notify.txt"). Funcs(template.FuncMap(tmplFuncs)). ParseFiles("build/notify.txt"), ) ) func init() { gob.Register(&Commit{}) // for delay } // sendFailMail sends a mail notification that the build failed on the // provided commit and builder. func sendFailMail(c appengine.Context, com *Commit, builder string) { // TODO(adg): handle packages
func init() { deleteBlobLater = delay.Func("deleteBlob", deleteBlob) }
} return nil } var propagate = delay.Func("propagate", func(c appengine.Context, ik *datastore.Key) error { var it Item err := datastore.Get(c, ik, &it) if err != nil { return err } si := subscribedItem{it.PubDate} iter := datastore.NewQuery("subscription").Ancestor(ik.Parent()).KeysOnly().Run(c) var sk *datastore.Key for sk, err = iter.Next(nil); err == nil; sk, err = iter.Next(nil) { uk, err := datastore.DecodeKey(sk.StringID()) if err != nil { return err } _, err = datastore.Put(c, datastore.NewKey(c, "subscribedItem", ik.Encode(), 0, uk), &si) if err != nil { return err } } if err != datastore.Done { return err } return nil }) func unsubscriber(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) k, err := datastore.DecodeKey(r.URL.RawQuery)
func init() { nextTurnFunc = delay.Func("models/game.nextTurnFunc", nextTurn) }
} func generateUUID(cid *string) error { b := make([]byte, 16) _, err := rand.Read(b) if err != nil { return err } b[8] = (b[8] | 0x80) & 0xBF // what's the purpose ? b[6] = (b[6] | 0x40) & 0x4F // what's the purpose ? *cid = hex.EncodeToString(b) return nil } var delayHit = delay.Func("collect", logHit) func logHit(c appengine.Context, params []string, ua string, cid string) error { // https://developers.google.com/analytics/devguides/collection/protocol/v1/reference payload := url.Values{ "v": {"1"}, // protocol version = 1 "t": {"pageview"}, // hit type "tid": {params[0]}, // tracking / property ID "cid": {cid}, // unique client ID (server generated UUID) "dp": {params[1]}, // page path } req, _ := http.NewRequest("POST", beaconURL, strings.NewReader(payload.Encode())) req.Header.Add("User-Agent", ua) req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
_, err = datastore.Put(c, broken.Key(c), broken) } return err } // firstMatch executes the query q and loads the first entity into v. func firstMatch(c appengine.Context, q *datastore.Query, v interface{}) error { t := q.Limit(1).Run(c) _, err := t.Next(v) if err == datastore.Done { err = datastore.ErrNoSuchEntity } return err } var notifyLater = delay.Func("notify", notify) // notify tries to update the CL for the given Commit with a failure message. // If it doesn't succeed, it sends a failure email to golang-dev. func notify(c appengine.Context, com *Commit, builder string) { if !updateCL(c, com, builder) { // Send a mail notification if the CL can't be found. sendFailMail(c, com, builder) } } // updateCL updates the CL for the given Commit with a failure message // for the given builder. func updateCL(c appengine.Context, com *Commit, builder string) bool { cl, err := lookupCL(c, com) if err != nil {
package controllers import ( "fmt" "net/http" "appengine" "appengine/delay" ) func DelayFunc(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) testFunc.Call(c, FuncArg{"test", "fuga"}) fmt.Fprintf(w, "ok") } type FuncArg struct { X, Y string } var testFunc = delay.Func("hoge", func(c appengine.Context, arg FuncArg) { c.Debugf("delayFunc %s %s", arg.X, arg.Y) })
package api import ( "github.com/GoogleCloudPlatform/go-endpoints/endpoints" "appengine/delay" ) type JobsApi struct{} var ( pullLeaguesFunc = delay.Func("pull-leagues", PullLeagues) ) // PullLeaguesApi will create a delay task to pull leagues from provider (xmlsoccer) // should handle at // POST jobs/pullleagues func (jobs *JobsApi) PullLeaguesApi(c endpoints.Context) { pullLeaguesFunc.Call(c) }
import ( "appengine" "appengine/datastore" "appengine/delay" "fmt" "net/http" "time" ) func init() { http.HandleFunc("/call", handler) http.HandleFunc("/_ah/stop", handlerStop) } var cellCallFunc = delay.Func("cellCall", cellCall) type Status struct { Value int } func handler(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) name := appengine.ModuleName(c) fmt.Fprintf(w, "Hello from cell %s", appengine.ModuleName(c)) col := int(name[4] - '0') row := int(name[6] - '0') if row < 3 { cellCallFunc.Call(c, col, row+1) }
import ( "appengine" "appengine/datastore" "appengine/delay" ) type BasicDataPoint struct { Date string Volatility float64 `datastore:",noindex"` Volatility60 float64 `datastore:",noindex"` } type BasicDataSet []DataPoint var delayedpreprocess = delay.Func("preprocess", preprocess) func preprocess(c appengine.Context) { var d StoredDataSet if err := datastore.Get(c, datastore.NewKey(c, "StoredDataSet", "data", 0, nil), &d); err != nil { panic(err) } var gold StoredDataSet if err := datastore.Get(c, datastore.NewKey(c, "StoredDataSet", "GOLDAMGBD228NLBM", 0, nil), &gold); err != nil { panic(err) } var euro StoredDataSet if err := datastore.Get(c, datastore.NewKey(c, "StoredDataSet", "DEXUSEU", 0, nil), &euro); err != nil { panic(err) } var brazil StoredDataSet