func lookupShelfDatastore(ctx ae.Context, user string) (shelf *data.Bookshelf, err error) { ancestor := ds.NewKey(ctx, data.KindBookshelf, user, 0, nil) query := ds.NewQuery(data.KindBookInfo).Ancestor(ancestor) targetMeta := new(data.BookMetaData) targetShelf := new(data.Bookshelf) switch err = ds.Get(ctx, ancestor, shelf); err { case nil: // all good case ds.ErrNoSuchEntity: targetShelf.Version = Latest default: return } it := query.Run(ctx) for _, err = it.Next(targetMeta); err == nil; _, err = it.Next(targetMeta) { targetShelf.Books = append(targetShelf.Books, *targetMeta) targetMeta = new(data.BookMetaData) } if err == ds.Done { err = nil shelf = targetShelf } ctx.Infof("Found %d items in datastore for key %v (error: %v)", len(targetShelf.Books), ancestor, err) return }
// extracts the URL from an Feed. If the feed has publication date, we only // add the URL since the last feed update func getLinksFromFeed(dsFeed *data.Feed, r *http.Request, c appengine.Context) ([]string, error) { c.Infof("DS Feed @ importer: %v", dsFeed) _, body, err := extract.GetURL(dsFeed.URL, r) if err != nil { return nil, err } rss, err := parseFeedContent(body) if err != nil { return nil, err } var urls []string stopURL := dsFeed.LastURL for index, i := range rss.ItemList { if index == 0 { dsFeed.LastURL = i.Link } if i.Link == stopURL { break } urls = append(urls, i.Link) } return urls, nil }
func dumpMessage(c appengine.Context, msg *mail.Message) { iw := InfoWriter{c} c.Infof("--BEGIN MESSAGE") io.Copy(iw, msg.Body) c.Infof("END MESSAGE--") }
func read(c appengine.Context, name string) (fi *FileInfo, data []byte, err error) { name, _, _ = mangle(c, name) fi1, err := stat(c, name) if err != nil { return nil, nil, err } if fi1.IsDir { dt, err := readdir(c, name) if err != nil { return nil, nil, err } fi = fi1 data = dt return fi, data, nil } root := datastore.NewKey(c, "RootKey", "v2:", 0, nil) dkey := datastore.NewKey(c, "FileData", "", fi1.Qid, root) var fd FileData c.Infof("DATASTORE Read %q", name) if err := datastore.Get(c, dkey, &fd); err != nil { return nil, nil, err } fi = fi1 data = fd.Data return fi, data, nil }
func cachePathTime(c appengine.Context, path string) (t int64, err error) { t, err = cacheTime(c) if err != nil { return 0, err } key := fmt.Sprintf("%d,mtime,%s", t, path) item, err := memcache.Get(c, key) if err == nil { v, err := strconv.ParseInt(string(item.Value), 10, 64) if err == nil { if chatty { c.Infof("cachePathTime %q = %v", key, v) } return v, nil } c.Criticalf("memcache.Get %q = %q (%v) - deleting", key, item.Value, err) memcache.Delete(c, key) } var seq int64 if fi, err := stat(c, path); err == nil { seq = fi.Seq } c.Infof("cachePathTime save %q = %v", key, seq) item = &memcache.Item{Key: key, Value: []byte(strconv.FormatInt(seq, 10))} if err := memcache.Set(c, item); err != nil { c.Criticalf("memcache.Set %q %q: %v", key, item.Value, err) } return seq, nil }
// Object usually too big (1MB limit), so shard. // http://stackoverflow.com/questions/9127982/ func BytesToShardedMemcache(c appengine.Context, key string, b []byte) { /* var buf bytes.Buffer if err := gob.NewEncoder(&buf).Encode(f); err != nil { db.C.Errorf("fdb error encoding item: %v", err) return } b := buf.Bytes() */ items := []*memcache.Item{} for i := 0; i < len(b); i += chunksize { k := fmt.Sprintf("=%d=%s", i, key) s, e := i, i+chunksize-1 if e >= len(b) { e = len(b) - 1 } c.Infof(" #=== [%7d, %7d] (%d) %s", s, e, len(b), k) items = append(items, &memcache.Item{Key: k, Value: b[s : e+1]}) // slice sytax is [s,e) } if err := memcache.SetMulti(c, items); err != nil { c.Errorf(" #=== cdb sharded store fail: %v", err) } c.Infof(" #=== Stored '%s' (len=%d)!", key, len(b)) }
// This will reimburse the bid's price and fee to the buyer. func RetireBid(c appengine.Context, key *datastore.Key) error { f := func(c appengine.Context) error { now := time.Now() dao := NewGaeAccountingDao(c, true) var bid Bid if err := datastore.Get(c, key, bidCodec{&bid}); err != nil { return err } if bid.State == Matched { c.Infof("Not retiring matched bid %v", key) return nil } if err := bid.Retire(dao, key.Encode(), now); err != nil { return err } if _, err := datastore.Put(c, key, bidCodec{&bid}); err != nil { return err } return dao.Flush() } if err := datastore.RunInTransaction(c, f, &datastore.TransactionOptions{XG: true}); err != nil { return err } return nil }
func UpdateUnreadCounts(c appengine.Context, ch chan<- Subscription, subscriptionKey *datastore.Key, subscription Subscription) { originalSubscriptionCount := subscription.UnreadCount q := datastore.NewQuery("Article").Ancestor(subscriptionKey).Filter("Properties =", "unread") if count, err := q.Count(c); err != nil { c.Errorf("Error getting unread count: %s", err) goto done } else if count != originalSubscriptionCount { subscription.UnreadCount = count if _, err := datastore.Put(c, subscriptionKey, &subscription); err != nil { c.Errorf("Error writing unread count: %s", err) goto done } } if originalSubscriptionCount != subscription.UnreadCount { c.Infof("Subscription count corrected to %d (was: %d)", subscription.UnreadCount, originalSubscriptionCount) } done: if ch != nil { ch <- subscription } }
func sendSubscription(addr string, c appengine.Context) { code := getSubCode() msg := &gaeMail.Message{ Sender: "*****@*****.**", To: []string{addr}, Subject: "confirm " + code, Body: "Reply without changing subject", } if err := gaeMail.Send(c, msg); err != nil { c.Errorf("Couldn't send email to %s for %s: %v", addr, code, err) } // XXXX if successful, register the code as (email, code, 0 (retry)) tuple. confirmation := Confirmation{ Email: addr, Code: code, Retry: 0, } _, err := datastore.Put(c, datastore.NewIncompleteKey(c, CONFIRM_MODEL, nil), &confirmation) if err != nil { c.Errorf("Couldn't write confirmation code for %s, %s: %v", addr, code, err) return } c.Infof("Wrote confirmation successfully for %s, %s", addr, code) }
// sendAlert sends an alert email about the potential car hijacking. func sendAlert(c appengine.Context, accKeyID int64, alertMsg, bodyTempl string) { // load account acc := new(ds.Account) key := datastore.NewKey(c, ds.ENameAccount, "", accKeyID, nil) if err := datastore.Get(c, key, acc); err != nil { c.Errorf("Failed to load account: %v", err) return } const adminEmail = "Andras Belicza <*****@*****.**>" msg := &mail.Message{ Sender: adminEmail, To: []string{acc.Email}, ReplyTo: adminEmail, Subject: "[IczaGPS] ALERT: " + alertMsg, Body: fmt.Sprintf(bodyTempl, acc.Email), } if len(acc.ContactEmail) > 0 { msg.Cc = []string{acc.ContactEmail} } if err := mail.Send(c, msg); err == nil { c.Infof("Sent successful alert email: %s", alertMsg) } else { c.Errorf("Couldn't send alert email: %s, %v", alertMsg, err) } }
func UpdateAllSubscriptions(c appengine.Context, userID UserID) error { userKey, err := userID.key(c) if err != nil { return err } var subscriptions []Subscription q := datastore.NewQuery("Subscription").Ancestor(userKey).Limit(defaultBatchSize) subscriptionKeys, err := q.GetAll(c, &subscriptions) if err != nil { return err } started := time.Now() doneChannel := make(chan Subscription) subscriptionCount := len(subscriptions) for i := 0; i < subscriptionCount; i++ { go updateSubscriptionAsync(c, subscriptionKeys[i], subscriptions[i], doneChannel) } for i := 0; i < subscriptionCount; i++ { <-doneChannel } c.Infof("%d subscriptions completed in %s", subscriptionCount, time.Since(started)) return nil }
func parse_chunk_map(c appengine.Context) error { var err error if chunkMapParsed == true { return nil } bytes, err := readGzFile("chunks.json.gz") if err != nil { return err } //log.Fatal("read bytes:", string(bytes)) err = json.Unmarshal(bytes, &chunkMap) c.Infof("Parsed %d bytes of JSON into %d map entries", len(bytes), len(chunkMap)) if err != nil { return err } chunkMapParsed = true return nil }
func SearchIndex(c appengine.Context, q string) (pairs [][]string, err error) { client := urlfetch.Client(c) u := searchFindUrl u.RawQuery = url.Values{"q": []string{q}}.Encode() res, err := client.Get(u.String()) if err == nil { defer res.Body.Close() if res.StatusCode != 200 { err = fmt.Errorf("%s -> %d", u.String(), res.StatusCode) return } } b, err := ioutil.ReadAll(res.Body) if err != nil { return } c.Infof("Response from search: %s", b) m := make(map[string]interface{}) err = json.Unmarshal(b, &m) if err != nil { return } nuts := m["Nuts"].([]interface{}) pairs = make([][]string, len(nuts)) for i, n := range nuts { nm := n.(map[string]interface{}) pairs[i] = []string{nm["Vendor"].(string), nm["Name"].(string)} } return }
// Sends post notification mail func SendMail(c appengine.Context, entry Entry) error { config, err := ParseConfig("./config/mailConfig.json") if err != nil { return err } // Prepares email message msg := new(mail.Message) msg.Sender = config.Sender msg.To = make([]string, 1) msg.To[0] = config.To msg.Subject = "New post made from Legacy-BBS-Go" var body bytes.Buffer var mailTemplate = template.Must(template.ParseFiles("template/notificationMailTemplate.txt")) if err := mailTemplate.Execute(&body, entry); err != nil { return err } msg.Body = body.String() if err := mail.Send(c, msg); err != nil { return err } c.Infof("Notification mail sent to \"" + config.To + "\"") return nil }
func orderHandler(c appengine.Context, w http.ResponseWriter, r *http.Request) (interface{}, error) { oid := r.URL.Path[len(ORDERS_API):] c.Infof("Received oid %v", oid) if len(oid) > 0 { switch r.Method { case "GET": oid64, err := strconv.ParseInt(oid, 10, 64) if err != nil { return nil, err } order := new(Order) order.Id = OrderId(oid64) return order.get(c) default: return nil, fmt.Errorf(r.Method + " on " + r.URL.Path + " not implemented") } } else { switch r.Method { case "POST": return orderSaveEntryPoint(c, r) case "GET": return getAllOrders(c) default: return nil, fmt.Errorf(r.Method + " on " + r.URL.Path + " not implemented") } } return nil, nil }
func unregisterUser(addr string, c appengine.Context) { q := datastore.NewQuery(USER_MODEL). Filter("Email =", addr). KeysOnly() keys, err := q.GetAll(c, nil) if err != nil { c.Errorf("Cound not query the model for %s: %v", addr, err) return } if len(keys) == 0 { c.Infof("No such user to unregister: %s", addr) return } for i := range keys { datastore.Delete(c, keys[i]) } c.Infof("Removed user %s", addr) msg := &gaeMail.Message{ Sender: "*****@*****.**", To: []string{addr}, Subject: "Email unregistered", Body: "user " + addr + " has been unregistered", } gaeMail.Send(c, msg) }
// notifyOnFailure checks whether the supplied Commit or the subsequent // Commit (if present) breaks the build for this builder. // If either of those commits break the build an email notification is sent // from a delayed task. (We use a task because this way the mail won't be // sent if the enclosing datastore transaction fails.) // // This must be run in a datastore transaction, and the provided *Commit must // have been retrieved from the datastore within that transaction. func notifyOnFailure(c appengine.Context, com *Commit, builder string) error { if failIgnore[builder] { return nil } // TODO(adg): implement notifications for packages if com.PackagePath != "" { return nil } p := &Package{Path: com.PackagePath} var broken *Commit cr := com.Result(builder, "") if cr == nil { return fmt.Errorf("no result for %s/%s", com.Hash, builder) } q := datastore.NewQuery("Commit").Ancestor(p.Key(c)) if cr.OK { // This commit is OK. Notify if next Commit is broken. next := new(Commit) q = q.Filter("ParentHash=", com.Hash) if err := firstMatch(c, q, next); err != nil { if err == datastore.ErrNoSuchEntity { // OK at tip, no notification necessary. return nil } return err } if nr := next.Result(builder, ""); nr != nil && !nr.OK { c.Debugf("commit ok: %#v\nresult: %#v", com, cr) c.Debugf("next commit broken: %#v\nnext result:%#v", next, nr) broken = next } } else { // This commit is broken. Notify if the previous Commit is OK. prev := new(Commit) q = q.Filter("Hash=", com.ParentHash) if err := firstMatch(c, q, prev); err != nil { if err == datastore.ErrNoSuchEntity { // No previous result, let the backfill of // this result trigger the notification. return nil } return err } if pr := prev.Result(builder, ""); pr != nil && pr.OK { c.Debugf("commit broken: %#v\nresult: %#v", com, cr) c.Debugf("previous commit ok: %#v\nprevious result:%#v", prev, pr) broken = com } } var err error if broken != nil && !broken.FailNotificationSent { c.Infof("%s is broken commit; notifying", broken.Hash) sendFailMailLater.Call(c, broken, builder) // add task to queue broken.FailNotificationSent = true _, err = datastore.Put(c, broken.Key(c), broken) } return err }
func loadRev(c appengine.Context, uuid string, ch chan revRes) { rv := revRes{} if item, err := memcache.Get(c, "rev."+uuid); err == nil { rv.rev = string(item.Value) } if rv.rev == "" { req, err := http.NewRequest("HEAD", baseDBURL+uniqueDB+"/unique_"+uuid, nil) if err != nil { rv.err = err ch <- rv return } req.Header.Set("Authorization", authHeader) t := &urlfetch.Transport{Context: c} resp, err := t.RoundTrip(req) if err != nil { rv.err = err ch <- rv return } if resp.StatusCode == 200 { rv.rev = resp.Header.Get("etag") if len(rv.rev) > 1 { rv.rev = rv.rev[1 : len(rv.rev)-1] } } else { c.Infof("Got unfavorable response for key: %v", resp.Status) } } ch <- rv }
/* Does the actual google books API call. */ func lookupISBN(ctx appengine.Context, country string, isbn isbn13.ISBN13) (resp *data.BookMetaData, err error) { var r *http.Response url := fmt.Sprintf(lookupURLTemplate, uint64(isbn), country, apiKey) ctx.Debugf("Calling %s", fmt.Sprintf(lookupURLTemplate, uint64(isbn), country, "<hidden>")) client := urlfetch.Client(ctx) if r, err = client.Get(url); err == nil { if r.StatusCode != http.StatusOK { err = fmt.Errorf("Google API returned %s", r.Status) } else { reply := new(data.LookupReply) decode := json.NewDecoder(r.Body) defer r.Body.Close() ctx.Infof("Completed API call, result %s\n", r.Status) if err = decode.Decode(reply); err == nil { if reply.Count == 1 { resp = &reply.BookInfos[0] } else { ctx.Infof("Google books reported %d matching items: %v", reply.Count, reply) resp = new(data.BookMetaData) resp.Volume.Title = placeHolderText + isbn.String() } resp.Parent = country resp.ISBN = isbn.String() } } } return }
func mailissue(ctxt appengine.Context, kind, key string) error { ctxt.Infof("mailissue %s", key) var cl CL err := app.ReadData(ctxt, "CL", key, &cl) if err != nil { return nil // error already logged } if len(cl.NeedMailIssue) == 0 { return nil } var mailed []string for _, issue := range cl.NeedMailIssue { err := postIssueComment(ctxt, issue, "CL https://codereview.appspot.com/"+cl.CL+" mentions this issue.") if err != nil { ctxt.Criticalf("posting to issue %v: %v", issue, err) continue } mailed = append(mailed, issue) } err = app.Transaction(ctxt, func(ctxt appengine.Context) error { var old CL if err := app.ReadData(ctxt, "CL", key, &old); err != nil { return err } old.MailedIssue = append(old.MailedIssue, mailed...) return app.WriteData(ctxt, "CL", key, &old) }) return err }
// Marks a bid as placed. This is purely informational for the user. func PlaceBid(c appengine.Context, bidId string) error { var key *datastore.Key if k, err := datastore.DecodeKey(bidId); err != nil { return err } else { key = k } f := func(c appengine.Context) error { var bid Bid if err := datastore.Get(c, key, bidCodec{&bid}); err != nil { return err } if bid.State != InQueue { c.Infof("Not placing bid %v : State=%v", key, bid.State) return nil } bid.State = Placed if _, err := datastore.Put(c, key, bidCodec{&bid}); err != nil { return err } return nil } return datastore.RunInTransaction(c, f, nil) }
func fetchNomRoll(c appengine.Context, skey string, actId int) os.Error { /// req,err := http.NewRequest("GET", fmt.Sprintf("https://cadetone.aafc.org.au/activities/nominalroll.php?ActID=%d",actId), nil) /// if err != nil { /// c.Errorf("Fetch Roll - actId:%d - %s", actId, err.String()) /// return err /// } /// session := c1SessionCookie(skey) /// injectSession(req,session) /// resp, err := GetClient(c).Do(req) /// if err != nil { /// c.Errorf("Fetch Roll - actId:%d - %s", actId, err.String()) /// return err /// } /// defer resp.Body.Close() /// c.Infof("Fetch Roll OK - actId:%d", actId) /// res,err := parseNomRoll(c, &resp.Body) res, err := parseNomRoll(c) if err != nil { c.Errorf("Fetch Roll - actId:%d - %s", actId, err.String()) return err } res.ActId = actId c.Infof(fmt.Sprint(res.ActId)) for _, r := range res.Roll { c.Infof(fmt.Sprint(*r)) } return err }
func updateCacheTime(c appengine.Context, seq int64) { const key = rootMemcacheKey bseq := []byte(strconv.FormatInt(seq, 10)) for tries := 0; tries < 10; tries++ { item, err := memcache.Get(c, key) if err != nil { c.Infof("memcache.Get %q: %v", key, err) err = memcache.Add(c, &memcache.Item{Key: key, Value: bseq}) if err == nil { c.Infof("memcache.Add %q %q ok", key, bseq) return } c.Infof("memcache.Add %q %q: %v", key, bseq, err) } v, err := strconv.ParseInt(string(item.Value), 10, 64) if err != nil { c.Criticalf("memcache.Get %q = %q (%v)", key, item.Value, err) return } if v >= seq { return } item.Value = bseq err = memcache.CompareAndSwap(c, item) if err == nil { c.Infof("memcache.CAS %q %d->%d ok", key, v, seq) return } c.Infof("memcache.CAS %q %d->%d: %v", key, v, seq, err) } c.Criticalf("repeatedly failed to update root key") }
func loadFullFeed(domain string, ctx appengine.Context, r *http.Request) (buf *bytes.Buffer) { newctx := newappengine.NewContext(r) tok, err := config.Exchange(newctx, r.FormValue("code")) if err != nil { ctx.Errorf("exchange error: %v", err) return } ctx.Infof("tok: %v", tok) client := config.Client(newctx, tok) expFeedUrl := fmt.Sprintf(feedUrl, domain) expFeedUrl = expFeedUrl + "&max-results=50000" res, err := client.Get(expFeedUrl) if err != nil { ctx.Errorf("get: %v", err) return } defer res.Body.Close() buf = new(bytes.Buffer) io.Copy(buf, res.Body) return }
func GetAllAccounts(c appengine.Context) ([]string, error) { accounts := []string{} _, err := memcache.Gob.Get(c, "AccountsArray", &accounts) if err == nil { return accounts, nil } var accountsDS []AccountDS query := datastore.NewQuery("Account") _, err = query.GetAll(c, &accountsDS) if err != nil { return []string{}, err } for _, account := range accountsDS { accounts = append(accounts, account.UserID) } cacheItem := &memcache.Item{ Key: "AccountsArray", Object: accounts, } err = memcache.Gob.Set(c, cacheItem) if err != nil { c.Infof("Accounts array not cached (%v)", err) } return accounts, err }
func verifyMassageWithPaypal(ctx appengine.Context, content string, testIpnField string) error { paypalIpnUrl := PaypalIpn if testIpnField != "" { if testIpnField == "1" { paypalIpnUrl = PaypalIpnSandBox } else { paypalIpnUrl = localIpn } } ctx.Infof("Sending msg to: " + paypalIpnUrl) extraData := []byte("cmd=_notify-validate&") client := urlfetch.Client(ctx) resp, err := client.Post(paypalIpnUrl, FromEncodedContentType, bytes.NewBuffer(append(extraData, content...))) if err != nil { return err } respBody, err := ioutil.ReadAll(resp.Body) resp.Body.Close() ctx.Debugf("Ipn Validation response " + string(respBody)) if err == nil && string(respBody) != "VERIFIED" { return IpnMessageCouldNotBeValidated } return err }
// AppEngineLogHandler sends logs to AppEngine. // The record must contain the appengine request context. func AppEngineLogHandler() log15.Handler { logFormat := log15.JsonFormat() return log15.FuncHandler(func(r *log15.Record) error { var c appengine.Context index := 0 for i, e := range r.Ctx { if ct, ok := e.(appengine.Context); ok { c = ct index = i break } } if c == nil { // not in the context of a request return nil } r.Ctx = append(r.Ctx[:index-1], r.Ctx[index+1:]...) log := string(logFormat.Format(r)) switch r.Lvl { case log15.LvlCrit: c.Criticalf(log) case log15.LvlError: c.Errorf(log) case log15.LvlWarn: c.Warningf(log) case log15.LvlInfo: c.Infof(log) case log15.LvlDebug: c.Debugf(log) } return nil }) }
// Log is a helper function that logs the given message to appenging // with the given priority. Accepted priorities are "debug", "info", // "warn", "error", and "crit". Other values default to "error". func Log(c appengine.Context, r *http.Request, priority string, message string, params ...interface{}) { message = fmt.Sprintf("[%s] [%s] [%s]: %s", r.RemoteAddr, r.Method, r.URL, message) switch priority { case "debug": c.Debugf(message, params...) case "info": c.Infof(message, params...) case "warn": c.Warningf(message, params...) case "error": c.Errorf(message, params...) case "crit": c.Criticalf(message, params...) default: c.Errorf(message, params...) } }
func (e Event) GetHTMLView(c appengine.Context) string { buffer := new(bytes.Buffer) var tmpltxt = `<label>Event Title: </label><a href="https://orgreminders.appspot.com/editevent?id={{.Key}}">{{.Title}}</a> <br> <label>When Due: </label>{{.DueFormatted}} <br> <label>Organization(s): </label>{{range .Orgs}}{{.}},{{end}} <br> <label>Email enabled: </label>{{.Email}} <br> <label>Text Enabled: </label>{{.Text}} <br> <label>Email Message: </label><br><div class="msgbody">{{.EmailMessage}}</div> <br> <label>Text Message: </label><br><div class="msgbody"><pre>{{.TextMessage}}</pre></div> <br>` template, terr := template.New("foo").Parse(tmpltxt) if terr != nil { c.Infof("error parsing event html template: %v", terr) return "" } template.Execute(buffer, e) return buffer.String() }
func SaveQuestion(c appengine.Context, question Question) { key := datastore.NewIncompleteKey(c, "Question", QuestionKey(c)) _, err := datastore.Put(c, key, &question) if err != nil { c.Infof("Error in saving question: ", err) } }