func init() {
http.Handle("/admin/commit/load", appstats.NewHandler(startLoad))
http.Handle("/admin/commit/kickoff", appstats.NewHandler(initialLoad))
http.Handle("/admin/commit/status", appstats.NewHandler(status))
http.Handle("/admin/commit/show/", appstats.NewHandler(show))
laterLoad = delay.Func("commit.load", load)
laterLoadRev = delay.Func("commit.loadrev", loadRev)
}
func uploadHistory(res http.ResponseWriter, req *http.Request) {
c := appengine.NewContext(req)
password := req.FormValue("password")
if password != os.Getenv("SERVER_PASSWORD") {
http.Error(res, "uh oh", http.StatusForbidden)
return
}
gobFile, _, err := req.FormFile("gob_file")
if err != nil {
http.Error(res, "missing file", http.StatusBadRequest)
return
}
defer gobFile.Close()
commands := []History{}
dec := gob.NewDecoder(gobFile)
ctr := 0
for {
var h History
err := dec.Decode(&h)
if err == io.EOF {
break
}
if err != nil {
http.Error(res, err.Error(), http.StatusInternalServerError)
return
}
commands = append(commands, h)
ctr += 1
}
increment := 500
tasks := 0
for i := 0; i < len(commands); i += increment {
start := i
end := i + 500
if end > len(commands) {
end = len(commands)
}
var laterFunc = delay.Func("key", importHistory)
laterFunc.Call(c, commands[start:end])
tasks += 1
}
fmt.Fprintf(res, "hi :) queued %d tasks to import %d elements for import\n", tasks, ctr)
}
var updateSeries = delay.Func("Fred", func(c appengine.Context, z string) {
u := urlForSeries(z)
body, err := fetch(u, c)
if err != nil {
panic(err)
}
var response FredResponse
if err = json.Unmarshal(body, &response); err != nil {
panic(err)
}
now := time.Now()
starttime, _ := time.Parse(dateformat, "2010-07-18")
datalist := make(map[string]float64)
for day := starttime; day.Before(now); day = day.AddDate(0, 0, 1) {
datalist[day.Format(dateformat)] = math.NaN()
}
for _, obs := range response.Observations {
if value, err := strconv.ParseFloat(obs.Value, 64); err != nil {
// c.Infof("error: %v",err)
} else {
datalist[obs.Date] = value
}
}
var data DataSet
for k, v := range datalist {
l := math.Log(v)
d := DataPoint{
Date: k,
Price: v,
Logprice: l,
}
data = append(data, d)
}
sort.Sort(DataSet(data))
// ideally, we'd use multiple imputation to fill in weekends and holidays
// must find Go package for multiple imputation
i := 1
for i < len(data) {
data[i].Return = data[i].Logprice - data[i-1].Logprice
i += 1
}
j := 29
for j < len(data) {
subset := data[j-29 : j]
var returns []float64
for _, point := range subset {
if !math.IsNaN(point.Return) {
returns = append(returns, point.Return)
}
}
data[j].Volatility = stats.StatsSampleStandardDeviation(returns) * 100.0
j += 1
}
k := 59
for k < len(data) {
subset := data[k-59 : k]
var returns []float64
for _, point := range subset {
if !math.IsNaN(point.Return) {
returns = append(returns, point.Return)
}
}
data[k].Volatility60 = stats.StatsSampleStandardDeviation(returns) * 100.0
k += 1
}
d := StoredDataSet{
Data: data,
}
if _, err := datastore.Put(c, datastore.NewKey(c, "StoredDataSet", z, 0, nil), &d); err != nil {
panic(err)
}
})
Freq: kanji.Freq,
JLPT: kanji.JLPT,
}
for _, r := range kanji.Readings {
if r.RType == "ja_on" {
k.Onyomi = append(k.Onyomi, r.Value)
}
if r.RType == "ja_kun" {
k.Kunyomi = append(k.Kunyomi, r.Value)
}
}
for _, m := range kanji.Meanings {
// English meanings do not have an m_lang attribute
if m.MLang == "" {
k.Meanings = append(k.Meanings, m.Value)
}
}
populateLater.Call(c, k)
}
}
var populateLater = delay.Func("populate", func(c appengine.Context, k Kanji) {
key := datastore.NewKey(c, "Kanji", k.Literal, 0, nil)
_, err := datastore.Put(c, key, &k)
c.Infof("Added kanji %s", k.Literal)
if err != nil {
c.Errorf(err.Error())
return
}
})
"math/rand"
"resize"
)
const BASEURL = "http://pitcam.ccs.neu.edu/%s.jpeg?client=pitcamproxy&rid=%s"
// This should be set to the IP of the remote authorized download ping host
const REMOTE_PINGBACK_IP = "127.0.0.1"
func init() {
http.HandleFunc("/random.jpeg", randomredirect)
http.HandleFunc("/_private/newimgpingback", pingbackHandler)
http.HandleFunc("/", handler)
}
var downloadLater = delay.Func("imageDownloader", downloadHandler)
func pingbackHandler(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
switch r.RemoteAddr {
case REMOTE_PINGBACK_IP:
c.Infof("pitcam pingback")
downloadLater.Call(c)
default:
c.Errorf("unauthenticated pingback from %s", r.RemoteAddr)
}
}
func downloadHandler(c appengine.Context) {
c.Infof("Downloader Called")
func init() {
saveLater = delay.Func("save", save)
}
import (
"fmt"
"http"
"appengine"
"appengine/delay"
)
func init() {
http.HandleFunc("/", handle)
}
func handle(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/" {
// ignore /favicon.ico, etc.
return
}
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
fmt.Fprint(w, "I will run a function later.\n")
c := appengine.NewContext(r)
later.Call(c, "Please send my regards to your mother.")
}
// A function that can be executed later.
var later = delay.Func("later", func(c appengine.Context, msg string) {
c.Infof("later, %q", msg)
})
package job
import (
"math/rand"
"time"
"appengine"
"appengine/datastore"
"appengine/delay"
)
var processJobDelay = delay.Func("processJob", processJob)
func processJob(c appengine.Context, jobID string) (err error) {
task := TaskNone
taskState := &taskState{}
state := &State{ID: jobID}
for task != TaskHaltProcessing {
// Run the next state of transactional processing of the job,
// to find out the next task to do and update records.
err = datastore.RunInTransaction(c, func(c appengine.Context) error {
// Update our copy of the job's state.
if err := datastore.Get(c, state.GetKey(c), state); err != nil {
return err
}
// Process it and get the next task to peform.
var err error
c := appengine.NewContext(r)
q := datastore.NewQuery("Package").Filter("Kind=", "external").KeysOnly()
for t := q.Run(c); ; {
key, err := t.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
c.Errorf("%v", err)
return
}
updateWeeklyLater.Call(c, key)
}
}
var (
installLater = delay.Func("install", install)
updateWeeklyLater = delay.Func("updateWeekly", updateWeekly)
)
// install validates the provided package path, increments its install count,
// and creates the Package record if it doesn't exist.
func install(c appengine.Context, path string) {
if !validPath(c, path) {
return
}
tx := func(c appengine.Context) error {
p := &Package{Path: path, Kind: "external"}
err := datastore.Get(c, p.Key(c), p)
if err != nil && err != datastore.ErrNoSuchEntity {
return err
}
buf := new(bytes.Buffer)
_, err = io.Copy(buf, resp.Body)
if err != nil {
c.Errorf("io.Copy", err)
return
}
att := &mail.Attachment{
Name: filename,
Data: buf.Bytes(),
}
//c.Infof(string(att.Data))
msg := "Deliverd successfully. Here is your file."
sendEmail(att, email, msg, c)
}
var later = delay.Func("later", doAnt)
func init() {
http.HandleFunc("/", indexHandler)
http.HandleFunc("/robots.txt", robotsHandler)
}
func robotsHandler(w http.ResponseWriter, r *http.Request) {
io.WriteString(w, robotsPage)
}
func indexHandler(w http.ResponseWriter, r *http.Request) {
if r.Method == "GET" {
io.WriteString(w, indexPage)
}
if r.Method == "POST" {
var laterSaveInvoiceRippleEffects = delay.Func("later-save-invoice-ripple-effects", func(c appengine.Context, invoice *Invoice) error {
// 1. Find pending orders
// 2. Sort them by date
// 3. Start filling the ShippedItemWithInvId in Order
// 4. Mark if they are complete
// 5. Repeat until all the invoice items have been exhausted.
// 6. Report error if you have items left in invoice but no corresponding order for it.
pendingOrders, err := getPendingOrdersForPurchaser(c, invoice.PurchaserId)
if err != nil {
return err
}
Orders(pendingOrders).Sort()
unmappedInvoiceItems := append(Items(nil), invoice.Items...)
for oi := range pendingOrders {
pendingOrder := &pendingOrders[oi]
for ii := range pendingOrder.PendingItems {
pendingItem := &pendingOrder.PendingItems[ii]
for umi := range unmappedInvoiceItems {
unmappedInvoiceItem := &unmappedInvoiceItems[umi]
if pendingItem.Qty == 0 {
continue
}
if pendingItem.Qty == 0 {
continue
}
if (*pendingItem).skuEquals(*unmappedInvoiceItem) {
//c.Infof("__________________________________")
//c.Infof("unmappedInvoiceItem %+v", unmappedInvoiceItem)
//c.Infof("pendingItem%+v", pendingItem)
//Find minimum of the two quantities
minQty := min(pendingItem.Qty, unmappedInvoiceItem.Qty)
//Prepare siwi
var siwi ShippedItemWithInvId
siwi.Item = *unmappedInvoiceItem
siwi.Item.Qty = minQty
siwi.InvoiceId = invoice.Id
//c.Infof("Created siwi %+v", siwi)
//Reduce pending
pendingItem.Qty -= minQty
pendingOrder.ShippedItemsWithInvId = append(pendingOrder.ShippedItemsWithInvId, siwi)
//c.Infof("pendingOrder.ShippedItemsWithInvId became %+v", pendingOrder.ShippedItemsWithInvId)
//Reduce unmappedInvoiceItem.Qty
unmappedInvoiceItem.Qty -= minQty
//Store Order id in Invoice for future reference.
invoice.OrdersId = append(invoice.OrdersId, pendingOrder.Id)
}
}
}
}
for i := range pendingOrders {
pendingOrder := &pendingOrders[i]
c.Infof("___________________________________________")
c.Infof("\nBefore Filtering pendingOrder.PendingItems = %#v", pendingOrder.PendingItems)
EmptyItems := func(item Item) bool { return item.Qty == 0 }
c.Infof("\nafter Filtering pendingOrder.PendingItems = %#v", pendingOrder.PendingItems)
pendingOrder.PendingItems = pendingOrder.PendingItems.Filter(EmptyItems)
if len(pendingOrder.PendingItems) == 0 {
pendingOrder.IsComplete = true
}
if _, err := pendingOrder.save(c); err != nil {
//TODO: If there are too many pending orders for a purchaser, you might want to set a dirty flag or multi save the entities. But this is unlikely.
return err
}
}
if _, err := invoice.save(c); err != nil {
return err
}
return nil
})
import (
"appengine/delay"
"fmt"
"github.com/scotch/aego/v1/config"
"github.com/scotch/aego/v1/context"
"github.com/scotch/aego/v1/mail"
"github.com/scotch/aego/v1/user/token"
"net/http"
)
const (
URL_PREFIX = "/-/email/verify"
)
var SendConfirmAddressLater = delay.Func("confirmaddress", SendConfirmAddress)
var defaultConfig = map[string]string{
"SiteTitle": "Company Name",
"SiteURL": "http://*****:*****@example.com>",
"Subject": "Please verify your email '%s'",
"BodyTmplPath": "github.com/scotch/aego/v1/user/email/templates/confirm.txt",
"BodyHTMLTmplPath": "github.com/scotch/aego/v1/user/email/templates/confirm.html",
}
func init() {
http.HandleFunc(URL_PREFIX, verifyHandler)
}
func genVerifyURL(baseURL string, token string) string {
var (
ErrUserNotFound = fmt.Errorf("User not found")
ErrWrongConfirmationCode = fmt.Errorf("Wrong confirmation code")
ErrEmailAlreadyClaimed = fmt.Errorf("The email is already claimed")
)
var (
delayedConfirmAccount = delay.Func("confirmAccount", func(c appengine.Context, user Account) error {
buf := &bytes.Buffer{}
if err := accountConfirmationEmail.Execute(buf, user); err != nil {
c.Criticalf("Couldn't execute account confirm email: %s", err)
return nil
}
msg := &mail.Message{
Sender: fmt.Sprintf("no-reply@%s.appspotmail.com", appengine.AppID(c)),
To: []string{user.Email},
Subject: "Confirm your account registration with Inner Hearth Yoga",
Body: buf.String(),
}
if err := mail.Send(c, msg); err != nil {
c.Criticalf("Couldn't send email to %q: %s", user.Email, err)
return fmt.Errorf("failed to send email")
}
return nil
})
)
// Info stores basic user contact & identification data.
type Info struct {
FirstName string `datastore: ",noindex"`
LastName string
Email string
}
toDelete = append(toDelete, peerKey)
}
deleted := len(toDelete)
c.Infof("Deleting %d expired peers", deleted)
if err := ds.DeleteMulti(c, toDelete); err != nil {
c.Criticalf("Failed to delete peers: %#v (%s)", err, err)
return
}
if deleted <= 0 {
break
}
}
c.Infof("Finished deleting expired peers")
}
var expireDelay = delay.Func("doExpire", doExpire)
func expire(c appengine.Context, w http.ResponseWriter, r *http.Request) {
expireDelay.Call(c, time.Now(), "")
w.WriteHeader(200)
io.WriteString(w, "Expiring peers...")
c.Debugf("Successfully began expire")
}
u.Infof("Loaded via datastore config[%s] = %q, modified = %s", key, e.Value, e.Modifed)
gaeConfigs[key] = e.Value
return e.Value
}
var gaeDelayFunc = delay.Func(GAE_DELAY_KEY, func(c appengine.Context, delayFuncArgs ...interface{}) error {
handlerName, ok := delayFuncArgs[0].(string)
if !ok {
c.Errorf("GAE.Delay: handler name could not be extracted from %v", delayFuncArgs)
return nil
}
handler, ok := DelayHandlers[handlerName]
if !ok {
c.Errorf("GAE.Delay: handler %s could not be found", handlerName)
return nil
}
u := new(GAE)
u.context = c
args := delayFuncArgs[1:]
Verbosef(u, "GAE.Delay: executing %s(%v)", handlerName, &args)
return handler(u, args...)
})
func (u GAE) Delay(handlerName string, args ...interface{}) error {
delayFuncArgs := append([]interface{}{handlerName}, args...)
gaeDelayFunc.Call(u.context, delayFuncArgs...)
Verbosef(u, "GAE.Delay: delaying %s(%v)", handlerName, &args)
sum += item.Price
}
fmt.Fprintln(&b, strings.Repeat("-", 55))
fmt.Fprintf(&b, "\tTotal:\t\t\t\t\t$%.2f\n", sum)
w.Write(b.Bytes())
sendReceipt.Call(c, user.Current(c).Email, b.String())
}
var sendReceipt = delay.Func("send-receipt", func(c appengine.Context, dst, body string) {
msg := &mail.Message{
Sender: appAdmin,
To: []string{dst},
Subject: "Your Gopher Mart receipt",
Body: body,
}
if err := mail.Send(c, msg); err != nil {
c.Errorf("mail.Send: %v", err)
}
})
func adminPopulate(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
for i := range [numItems]struct{}{} { // r hates this. tee hee.
key := itemKey(c, i)
good := goods[rand.Intn(len(goods))]
item := &Item{
// TODO: vary names more
Name: fmt.Sprintf("%s %dg", good.name, i+1),
Price: float64(rand.Intn(1999)+1) / 100,
func init() {
laterUpdateKind = delay.Func("app.backgroundUpdateKind", backgroundUpdateKind)
}
// license that can be found in the LICENSE file.
/*
Package mail provides methods for sending e-mails
*/
package mail
import (
"appengine/delay"
"appengine/mail"
"bytes"
"github.com/scotch/aego/v1/context"
"html/template"
)
var SendLater = delay.Func("send", Send)
type TemplateMessage struct {
*mail.Message
BodyValues map[string]string
BodyTmplPath string
HTMLBodyTmplPath string
}
func parseTmpl(tmplPath string, values map[string]string) (s string, err error) {
var b bytes.Buffer
tmpl := template.Must(template.ParseFiles(tmplPath))
if err := tmpl.Execute(&b, values); err != nil {
return s, err
}
return string(b.Bytes()), nil
return fmt.Errorf("finding result for %q: %+v", builder, com)
}
return commonNotify(c, broken, builder, r.LogHash)
}
// firstMatch executes the query q and loads the first entity into v.
func firstMatch(c appengine.Context, q *datastore.Query, v interface{}) error {
t := q.Limit(1).Run(c)
_, err := t.Next(v)
if err == datastore.Done {
err = datastore.ErrNoSuchEntity
}
return err
}
var notifyLater = delay.Func("notify", notify)
// notify tries to update the CL for the given Commit with a failure message.
// If it doesn't succeed, it sends a failure email to golang-dev.
func notify(c appengine.Context, com *Commit, builder, logHash string) {
if !updateCL(c, com, builder, logHash) {
// Send a mail notification if the CL can't be found.
sendFailMail(c, com, builder, logHash)
}
}
// updateCL updates the CL for the given Commit with a failure message
// for the given builder.
func updateCL(c appengine.Context, com *Commit, builder, logHash string) bool {
cl, err := lookupCL(c, com)
if err != nil {
func handleDelay(w http.ResponseWriter, r *http.Request) {
var later = delay.Func("key", delayedFunction)
later.Call(appengine.NewContext(r), "the data")
}
}
return err
}
// firstMatch executes the query q and loads the first entity into v.
func firstMatch(c appengine.Context, q *datastore.Query, v interface{}) error {
t := q.Limit(1).Run(c)
_, err := t.Next(v)
if err == datastore.Done {
err = datastore.ErrNoSuchEntity
}
return err
}
var (
sendFailMailLater = delay.Func("sendFailMail", sendFailMail)
sendFailMailTmpl = template.Must(
template.New("notify.txt").
Funcs(template.FuncMap(tmplFuncs)).
ParseFiles("build/notify.txt"),
)
)
func init() {
gob.Register(&Commit{}) // for delay
}
// sendFailMail sends a mail notification that the build failed on the
// provided commit and builder.
func sendFailMail(c appengine.Context, com *Commit, builder string) {
// TODO(adg): handle packages
func init() {
deleteBlobLater = delay.Func("deleteBlob", deleteBlob)
}
}
return nil
}
var propagate = delay.Func("propagate", func(c appengine.Context, ik *datastore.Key) error {
var it Item
err := datastore.Get(c, ik, &it)
if err != nil {
return err
}
si := subscribedItem{it.PubDate}
iter := datastore.NewQuery("subscription").Ancestor(ik.Parent()).KeysOnly().Run(c)
var sk *datastore.Key
for sk, err = iter.Next(nil); err == nil; sk, err = iter.Next(nil) {
uk, err := datastore.DecodeKey(sk.StringID())
if err != nil {
return err
}
_, err = datastore.Put(c, datastore.NewKey(c, "subscribedItem", ik.Encode(), 0, uk), &si)
if err != nil {
return err
}
}
if err != datastore.Done {
return err
}
return nil
})
func unsubscriber(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
k, err := datastore.DecodeKey(r.URL.RawQuery)
func init() {
nextTurnFunc = delay.Func("models/game.nextTurnFunc", nextTurn)
}
}
func generateUUID(cid *string) error {
b := make([]byte, 16)
_, err := rand.Read(b)
if err != nil {
return err
}
b[8] = (b[8] | 0x80) & 0xBF // what's the purpose ?
b[6] = (b[6] | 0x40) & 0x4F // what's the purpose ?
*cid = hex.EncodeToString(b)
return nil
}
var delayHit = delay.Func("collect", logHit)
func logHit(c appengine.Context, params []string, ua string, cid string) error {
// https://developers.google.com/analytics/devguides/collection/protocol/v1/reference
payload := url.Values{
"v": {"1"}, // protocol version = 1
"t": {"pageview"}, // hit type
"tid": {params[0]}, // tracking / property ID
"cid": {cid}, // unique client ID (server generated UUID)
"dp": {params[1]}, // page path
}
req, _ := http.NewRequest("POST", beaconURL, strings.NewReader(payload.Encode()))
req.Header.Add("User-Agent", ua)
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
_, err = datastore.Put(c, broken.Key(c), broken)
}
return err
}
// firstMatch executes the query q and loads the first entity into v.
func firstMatch(c appengine.Context, q *datastore.Query, v interface{}) error {
t := q.Limit(1).Run(c)
_, err := t.Next(v)
if err == datastore.Done {
err = datastore.ErrNoSuchEntity
}
return err
}
var notifyLater = delay.Func("notify", notify)
// notify tries to update the CL for the given Commit with a failure message.
// If it doesn't succeed, it sends a failure email to golang-dev.
func notify(c appengine.Context, com *Commit, builder string) {
if !updateCL(c, com, builder) {
// Send a mail notification if the CL can't be found.
sendFailMail(c, com, builder)
}
}
// updateCL updates the CL for the given Commit with a failure message
// for the given builder.
func updateCL(c appengine.Context, com *Commit, builder string) bool {
cl, err := lookupCL(c, com)
if err != nil {
package controllers
import (
"fmt"
"net/http"
"appengine"
"appengine/delay"
)
func DelayFunc(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
testFunc.Call(c, FuncArg{"test", "fuga"})
fmt.Fprintf(w, "ok")
}
type FuncArg struct {
X, Y string
}
var testFunc = delay.Func("hoge", func(c appengine.Context, arg FuncArg) {
c.Debugf("delayFunc %s %s", arg.X, arg.Y)
})
package api
import (
"github.com/GoogleCloudPlatform/go-endpoints/endpoints"
"appengine/delay"
)
type JobsApi struct{}
var (
pullLeaguesFunc = delay.Func("pull-leagues", PullLeagues)
)
// PullLeaguesApi will create a delay task to pull leagues from provider (xmlsoccer)
// should handle at
// POST jobs/pullleagues
func (jobs *JobsApi) PullLeaguesApi(c endpoints.Context) {
pullLeaguesFunc.Call(c)
}
import (
"appengine"
"appengine/datastore"
"appengine/delay"
"fmt"
"net/http"
"time"
)
func init() {
http.HandleFunc("/call", handler)
http.HandleFunc("/_ah/stop", handlerStop)
}
var cellCallFunc = delay.Func("cellCall", cellCall)
type Status struct {
Value int
}
func handler(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
name := appengine.ModuleName(c)
fmt.Fprintf(w, "Hello from cell %s", appengine.ModuleName(c))
col := int(name[4] - '0')
row := int(name[6] - '0')
if row < 3 {
cellCallFunc.Call(c, col, row+1)
}
import (
"appengine"
"appengine/datastore"
"appengine/delay"
)
type BasicDataPoint struct {
Date string
Volatility float64 `datastore:",noindex"`
Volatility60 float64 `datastore:",noindex"`
}
type BasicDataSet []DataPoint
var delayedpreprocess = delay.Func("preprocess", preprocess)
func preprocess(c appengine.Context) {
var d StoredDataSet
if err := datastore.Get(c, datastore.NewKey(c, "StoredDataSet", "data", 0, nil), &d); err != nil {
panic(err)
}
var gold StoredDataSet
if err := datastore.Get(c, datastore.NewKey(c, "StoredDataSet", "GOLDAMGBD228NLBM", 0, nil), &gold); err != nil {
panic(err)
}
var euro StoredDataSet
if err := datastore.Get(c, datastore.NewKey(c, "StoredDataSet", "DEXUSEU", 0, nil), &euro); err != nil {
panic(err)
}
var brazil StoredDataSet
