// AppEngineLogHandler sends logs to AppEngine.
// The record must contain the appengine request context.
func AppEngineLogHandler() log15.Handler {
logFormat := log15.JsonFormat()
return log15.FuncHandler(func(r *log15.Record) error {
var c appengine.Context
index := 0
for i, e := range r.Ctx {
if ct, ok := e.(appengine.Context); ok {
c = ct
index = i
break
}
}
if c == nil {
// not in the context of a request
return nil
}
r.Ctx = append(r.Ctx[:index-1], r.Ctx[index+1:]...)
log := string(logFormat.Format(r))
switch r.Lvl {
case log15.LvlCrit:
c.Criticalf(log)
case log15.LvlError:
c.Errorf(log)
case log15.LvlWarn:
c.Warningf(log)
case log15.LvlInfo:
c.Infof(log)
case log15.LvlDebug:
c.Debugf(log)
}
return nil
})
}
func webuserOK(c appengine.Context, w http.ResponseWriter, r *http.Request) bool {
if !userauthenticated(c) {
url, err := user.LoginURL(c, r.URL.String())
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return false
}
w.Header().Set("Location", url)
w.WriteHeader(http.StatusFound)
return false
}
u := user.Current(c)
authzed, err := userauthorized(c, u.Email)
if err != nil {
c.Errorf("authorization error: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return false
}
if !authzed {
c.Warningf("authorization failure: %v", u.Email)
w.WriteHeader(http.StatusForbidden)
err = templates.ExecuteTemplate(w, "unauthorized.html", nil)
if err != nil {
c.Errorf("unauthorized user and got err on template: %v", err)
}
return false
}
return true
}
func getCached(c appengine.Context, key string, missing func(appengine.Context, string) (*bytes.Buffer, error)) (*memcache.Item, error) {
item, err := memcache.Get(c, key)
if err == memcache.ErrCacheMiss {
c.Infof("item not in the cache: %v", key)
result, err := missing(c, key)
if err != nil {
return nil, err
}
item = &memcache.Item{
Key: key,
Value: result.Bytes(),
}
if err := memcache.Add(c, item); err == memcache.ErrNotStored {
c.Warningf("item with key %q already exists", item.Key)
} else if err != nil {
return item, err
}
} else if err != nil {
return item, err
} else {
c.Infof("Cache hit: %v", key)
}
return item, nil
}
// CacheAccount puts the specified Account into the cache (memcache).
func CacheAccount(c appengine.Context, acc *ds.Account) {
mk := prefixAccForUID + acc.UserID
if err := memcache.Set(c, &memcache.Item{Key: mk, Value: acc.Encode(), Expiration: cachedAccExpiration}); err != nil {
c.Warningf("Failed to set %s in memcache: %v", mk, err)
}
}
func runFunc(c appengine.Context, w http.ResponseWriter, req *http.Request) {
defer req.Body.Close()
var inv invocation
if err := gob.NewDecoder(req.Body).Decode(&inv); err != nil {
c.Errorf("delay: failed decoding task payload: %v", err)
c.Warningf("delay: dropping task")
return
}
f := funcs[inv.Key]
if f == nil {
c.Errorf("delay: no func with key %q found", inv.Key)
c.Warningf("delay: dropping task")
return
}
// TODO: This is broken for variadic functions.
ft := f.fv.Type()
in := make([]reflect.Value, ft.NumIn())
in[0] = reflect.ValueOf(c)
for i := 1; i < len(in); i++ {
in[i] = reflect.ValueOf(inv.Args[i-1])
}
out := f.fv.Call(in)
if n := ft.NumOut(); n > 0 && ft.Out(n-1) == osErrorType {
if errv := out[n-1]; !errv.IsNil() {
c.Errorf("delay: func failed (will retry): %v", errv.Interface())
w.WriteHeader(http.StatusInternalServerError)
return
}
}
}
// CacheDevice puts the specified Device into the cache (memcache).
func CacheDevice(c appengine.Context, dev *ds.Device) {
mk := prefixDevForRandID + dev.RandID
if err := memcache.Set(c, &memcache.Item{Key: mk, Value: dev.Encode()}); err != nil {
c.Warningf("Failed to set %s in memcache: %v", mk, err)
}
}
// ClearDeviceForRandID clears the cached Device for the specified RandID.
func ClearDeviceForRandID(c appengine.Context, RandID string) {
mk := prefixDevForRandID + RandID
if err := memcache.Delete(c, mk); err != nil {
c.Warningf("Failed to delete %s from memcache: %v", mk, err)
}
}
// locateFavIconURL attempts to determine the "favicon" URL for a particular
// site URL. It does this by checking the source document for explicit icon
// directives (in the LINK tags), as well as by attempting to fetch favicon.ico
func locateFavIconURL(context appengine.Context, feedHomeURL string) (string, error) {
if feedHomeURL != "" {
// Attempt to extract the favicon from the source document
if favIconURL, err := extractFavIconURL(context, feedHomeURL); err != nil {
context.Warningf("FavIcon extraction failed for %s: %s", feedHomeURL, err)
} else if favIconURL != "" {
if contains, err := containsFavIcon(context, favIconURL); err != nil {
context.Warningf("FavIcon lookup failed for %s: %s", feedHomeURL, err)
} else if contains {
return favIconURL, nil
}
}
// If that fails, try the usual location (/favicon.ico)
if url, err := url.Parse(feedHomeURL); err != nil {
return "", err
} else {
attemptURL := fmt.Sprintf("%s://%s/favicon.ico", url.Scheme, url.Host)
if contains, err := containsFavIcon(context, attemptURL); err != nil {
return "", err
} else if contains {
return attemptURL, nil
}
}
}
return "", nil
}
func EditBlogCube(model *M.BlogCube, c appengine.Context) (err error) {
key, entity := dsHelper.Model2Entity(model)
blogCubeEntity := entity.(M.BlogCubeEntity)
c.Warningf("The entity is %v", blogCubeEntity)
_, err = datastore.Put(c, key, &blogCubeEntity)
return err
}
// Log is a helper function that logs the given message to appenging
// with the given priority. Accepted priorities are "debug", "info",
// "warn", "error", and "crit". Other values default to "error".
func Log(c appengine.Context, r *http.Request, priority string,
message string, params ...interface{}) {
message = fmt.Sprintf("[%s] [%s] [%s]: %s", r.RemoteAddr, r.Method,
r.URL, message)
switch priority {
case "debug":
c.Debugf(message, params...)
case "info":
c.Infof(message, params...)
case "warn":
c.Warningf(message, params...)
case "error":
c.Errorf(message, params...)
case "crit":
c.Criticalf(message, params...)
default:
c.Errorf(message, params...)
}
}
func TriggerBatchProcessing(c appengine.Context, article ArticleId) error {
// Instead of submitting a task to match incoming bids, resulting in one task per bid,
// we collect bids for up to two seconds and batch-process them afterwards.
semaphoreKey := "semaphore-" + string(article)
if semaphore, err := memcache.Increment(c, semaphoreKey, 1, 0); err != nil {
return err
} else if semaphore >= 2 {
c.Infof("Batch processing already triggered for article %v", article)
memcache.IncrementExisting(c, semaphoreKey, -1)
return nil
} else {
time.Sleep(1 * time.Second)
c.Infof("Starting batch processing...")
memcache.IncrementExisting(c, semaphoreKey, -1)
time_before := time.Now()
matchingErr := MatchIncomingBids(c, article)
time_after := time.Now()
duration := time_after.Sub(time_before)
if duration > 1000*time.Millisecond {
c.Errorf("Batch processing finished after %v. Limit exceeded!", duration)
} else if duration > 500*time.Millisecond {
c.Warningf("Batch processing finished after %v. Limit in danger.", duration)
} else {
c.Infof("Batch processing finished after %v.", duration)
}
return matchingErr
}
}
func storeStatsObject(c appengine.Context, st Stats, uuid string) error {
cherr := make(chan error)
oldSize := len(st.Data)
if err := st.compress(); err != nil {
return err
}
c.Infof("Compressed stat data from %v -> %v", oldSize, len(st.Data))
k, err := datastore.Put(c,
datastore.NewIncompleteKey(c, "Stats", nil), &st)
if err != nil {
c.Warningf("Error storing stats item: %v", err)
return err
}
go func() {
err := couchit(c, k, nil)
if err != nil {
c.Errorf("Error queuing store of stats item: %v", err)
}
cherr <- err
}()
go func() {
err := updateUnique(c, k, uuid)
if err != nil {
c.Errorf("Error queueing update unique: %v", err)
}
cherr <- err
}()
return anyErr(<-cherr, <-cherr)
}
func ParseFeed(c appengine.Context, contentType, origUrl, fetchUrl string, body []byte) (*Feed, []*Story, error) {
cr := defaultCharsetReader
if !bytes.EqualFold(body[:len(xml.Header)], []byte(xml.Header)) {
enc, err := encodingReader(body, contentType)
if err != nil {
return nil, nil, err
}
if enc != encoding.Nop {
cr = nilCharsetReader
body, err = ioutil.ReadAll(transform.NewReader(bytes.NewReader(body), enc.NewDecoder()))
if err != nil {
return nil, nil, err
}
}
}
var feed *Feed
var stories []*Story
var atomerr, rsserr, rdferr error
feed, stories, atomerr = parseAtom(c, body, cr)
if feed == nil {
feed, stories, rsserr = parseRSS(c, body, cr)
}
if feed == nil {
feed, stories, rdferr = parseRDF(c, body, cr)
}
if feed == nil {
c.Warningf("atom parse error: %s", atomerr.Error())
c.Warningf("xml parse error: %s", rsserr.Error())
c.Warningf("rdf parse error: %s", rdferr.Error())
return nil, nil, fmt.Errorf("Could not parse feed data")
}
feed.Url = origUrl
return parseFix(c, feed, stories, fetchUrl)
}
func AddBlogCube(model *M.BlogCube, c appengine.Context) (err error) {
c.Warningf("addBlogCube function")
entity := &(M.BlogCubeEntity{})
entity.BlogName = model.BlogName
entityName := "BlogCubeEntity"
_, err = datastore.Put(c, datastore.NewIncompleteKey(c, entityName, nil), entity)
return err
}
func (appUser *AppUser) DSKeyID(c appengine.Context) (string, error) {
if appUser.ID == "" {
c.Warningf("Attempted to create an AppUser entity key with no ID!")
return "", errors.New("AppUser has no ID!")
}
return "user:" + appUser.ID, nil
}
func SetProperty(c appengine.Context, ref ArticleRef, propertyName string, propertyValue bool) ([]string, error) {
articleKey, err := ref.key(c)
if err != nil {
return nil, err
}
article := new(Article)
if err := datastore.Get(c, articleKey, article); err != nil && !IsFieldMismatch(err) {
return nil, err
}
if propertyValue != article.HasProperty(propertyName) {
wasUnread := article.IsUnread()
wasLiked := article.IsLiked()
unreadDelta := 0
article.SetProperty(propertyName, propertyValue)
// Update unread counts if necessary
if wasUnread != article.IsUnread() {
if wasUnread {
unreadDelta = -1
} else {
unreadDelta = 1
}
}
if _, err := datastore.Put(c, articleKey, article); err != nil {
return nil, err
}
if wasLiked != article.IsLiked() {
if wasLiked {
article.updateLikeCount(c, -1)
} else {
article.updateLikeCount(c, 1)
}
}
if unreadDelta != 0 {
// Update unread counts - not critical
subscriptionKey := articleKey.Parent()
subscription := new(Subscription)
if err := datastore.Get(c, subscriptionKey, subscription); err != nil {
c.Warningf("Unread count update failed: subscription read error (%s)", err)
} else if subscription.UnreadCount+unreadDelta >= 0 {
subscription.UnreadCount += unreadDelta
if _, err := datastore.Put(c, subscriptionKey, subscription); err != nil {
c.Warningf("Unread count update failed: subscription write error (%s)", err)
}
}
}
}
return article.Properties, nil
}
func parseRSS(c appengine.Context, body []byte, charsetReader func(string, io.Reader) (io.Reader, error)) (*Feed, []*Story, error) {
var f Feed
var s []*Story
r := rss.Rss{}
d := xml.NewDecoder(bytes.NewReader(body))
d.CharsetReader = charsetReader
d.DefaultSpace = "DefaultSpace"
if err := d.Decode(&r); err != nil {
return nil, nil, err
}
f.Title = r.Title
if t, err := parseDate(c, &f, r.LastBuildDate, r.PubDate); err == nil {
f.Updated = t
} else {
c.Warningf("no rss feed date: %v", f.Link)
}
f.Link = r.BaseLink()
f.Hub = r.Hub()
for _, i := range r.Items {
st := Story{
Link: i.Link,
Author: i.Author,
}
if i.Content != "" {
st.content = i.Content
} else if i.Description != "" {
st.content = i.Description
}
if i.Title != "" {
st.Title = i.Title
} else if i.Description != "" {
st.Title = i.Description
}
if st.content == st.Title {
st.Title = ""
}
st.Title = textTitle(st.Title)
if i.Guid != nil {
st.Id = i.Guid.Guid
}
if i.Enclosure != nil && strings.HasPrefix(i.Enclosure.Type, "audio/") {
st.MediaContent = i.Enclosure.Url
} else if i.Media != nil && strings.HasPrefix(i.Media.Type, "audio/") {
st.MediaContent = i.Media.URL
}
if t, err := parseDate(c, &f, i.PubDate, i.Date, i.Published); err == nil {
st.Published = t
st.Updated = t
}
s = append(s, &st)
}
return &f, s, nil
}
func storeInCouch(c appengine.Context, ob map[string]interface{}, k string) (rv string, err error) {
// log.Printf("Storing in couchdb: %+v", ob)
if baseDBURL == "" {
c.Warningf("CouchDB is not configured. Deferring")
return "", errors.New("not configured")
}
dbname := miscDB
switch {
case strings.HasPrefix(k, "stats_"):
dbname = statsDB
case strings.HasPrefix(k, "unique_"):
dbname = uniqueDB
}
js, err := json.Marshal(ob)
if err != nil {
return "", err
}
req, err := http.NewRequest("PUT", baseDBURL+dbname+"/"+k, bytes.NewReader(js))
if err != nil {
return "", err
}
req.Header.Set("Authorization", authHeader)
req.Header.Set("Content-Type", "application/json")
t := &urlfetch.Transport{Context: c}
resp, err := t.RoundTrip(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
switch {
case resp.StatusCode == 409:
c.Infof("Conflict updating %s", k)
case resp.StatusCode >= 200 && resp.StatusCode < 300:
c.Debugf("Updated couch with %v", resp.StatusCode)
withRev := struct {
Rev string `json:"rev"`
}{}
d := json.NewDecoder(resp.Body)
err = d.Decode(&withRev)
rv = withRev.Rev
default:
return "", fmt.Errorf("Failed to update couch: %v", resp.Status)
}
return
}
func cacheJson(c appengine.Context, id int64, data []byte) error {
memId := jsonMemId(id)
item := &memcache.Item{
Key: memId,
Value: data,
}
err := memcache.Set(c, item)
if err != nil {
c.Warningf("can't write %v to memcache: %v", memId, err)
}
return err
}
func retrieveAuth(context appengine.Context) *Auth {
// First, check memcache
auth := Auth{}
_, err := memcache.Gob.Get(context, "rackspace-auth", &auth)
if err == memcache.ErrCacheMiss {
return newAuth(context)
} else if err != nil {
context.Warningf("error retrieving item %v", err)
return newAuth(context)
}
return &auth
}
func (f Feed) Subscribe(c appengine.Context) {
if !f.IsSubscribed() {
t := taskqueue.NewPOSTTask(routeUrl("subscribe-feed"), url.Values{
"feed": {f.Url},
})
if _, err := taskqueue.Add(c, t, "update-manual"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
} else {
c.Warningf("subscribe feed: %v", f.Url)
}
}
}
// cacheDevListForAccKey puts the specified Device list into the cache (memcache).
func cacheDevListForAccKey(c appengine.Context, accKey *datastore.Key, devices []*ds.Device) {
mk := prefixDevListForAccKey + strconv.FormatInt(accKey.IntID(), 10)
data, err := json.Marshal(devices) // This can't really fail
if err != nil {
c.Errorf("Failed to encode device list to JSON: %v", err)
}
if err = memcache.Set(c, &memcache.Item{Key: mk, Value: data}); err != nil {
c.Warningf("Failed to set %s in memcache: %v", mk, err)
}
}
func Read(c appengine.Context, w http.ResponseWriter, r *http.Request) error {
w.Header().Set("Content-Type", "application/xhtml+xml; charset=UTF-8")
var id string
if strings.HasPrefix(r.URL.Path, "/read/") {
id = r.URL.Path[len("/read/"):]
} else {
return NotFound(r.URL.Path)
}
type metadata struct {
Label string
Value string
}
type renderdata struct {
Title string
Meta []metadata
HTML string
}
var data renderdata
s, err := GetStory(c, id)
if err != nil {
return err
}
data.Title = html.EscapeString(s.Title)
for name, prop := range s.Meta {
if len(name) == 0 || len(prop.Name) == 0 {
c.Warningf("Zero-length property name?")
continue
}
data.Meta = append(data.Meta, metadata{
Label: html.EscapeString(strings.ToUpper(prop.Name[:1]) + prop.Name[1:]),
Value: html.EscapeString(prop.Value),
})
}
if node, err := fictex.ParseBytes(s.Source); err == nil {
b := new(bytes.Buffer)
if err := fictex.HTMLRenderer.Render(b, node); err == nil {
data.HTML = b.String()
}
} else {
data.HTML = html.EscapeString(fmt.Sprintf("Error: %s", err))
}
return templates.ExecuteTemplate(w, "render.html", data)
}
// Add attempts to write a new Student entity; it will not overwrite
// any existing Students. Returns ErrClassFull if the class is full as
// of the given date. The number of students "currently registered"
// for a class is the number of session-registered students plus any
// future drop ins. This may be smaller than the number of students
// registered on a particular day, and so may prevent drop-ins which
// may otherwise have succeeded. In other words, a student can only
// drop in if we can prove that there is room for them to register for
// the rest of the session.
func (s *Student) Add(c appengine.Context, asOf time.Time) error {
key := s.key(c)
var txnErr error
for i := 0; i < 25; i++ {
txnErr = datastore.RunInTransaction(c, func(c appengine.Context) error {
old := &Student{}
switch err := datastore.Get(c, key, old); err {
case datastore.ErrNoSuchEntity:
break
case nil:
if old.DropIn && old.Date.Before(asOf) {
// Old registration is an expired drop-in. Allow re-registering.
break
}
// Old registration is still active; do nothing.
c.Warningf("Attempted duplicate registration of %q in %d", s.ID, s.ClassID)
return nil
default:
return fmt.Errorf("students: failed to look up existing student: %s", err)
}
class, err := classes.ClassWithID(c, s.ClassID)
if err != nil {
return err
}
in := In(c, class, asOf)
if int32(len(in)) >= class.Capacity {
return ErrClassIsFull
}
if err := class.Update(c); err != nil {
return fmt.Errorf("students: failed to update class: %s", err)
}
if _, err := datastore.Put(c, key, s); err != nil {
return fmt.Errorf("students: failed to write student: %s", err)
}
return nil
}, nil)
if txnErr != datastore.ErrConcurrentTransaction {
break
}
}
switch txnErr {
case nil:
return nil
case datastore.ErrConcurrentTransaction:
return fmt.Errorf("students: too many concurrent updates to class %d", s.ClassID)
default:
return txnErr
}
}
func writeLogMessage(c appengine.Context, level logLevel, msg string) {
const fmt = "%s"
switch level {
case levelDebug:
c.Debugf(fmt, msg)
case levelWarning:
c.Warningf(fmt, msg)
case levelError:
c.Errorf(fmt, msg)
case levelCritical:
c.Criticalf(fmt, msg)
default:
c.Infof(fmt, msg)
}
}
func postToCouch(c appengine.Context, w http.ResponseWriter, r *http.Request) {
err := postToCouchCommon(c, r.FormValue("key"), r.FormValue("isnew") == "true")
if err != nil {
c.Errorf("Error posting to couchdb %q: %v", r.FormValue("key"), err)
http.Error(w, err.Error(), 500)
return
}
if err != nil {
c.Warningf("Could not couch %v: %v", r.FormValue("key"), err)
http.Error(w, err.Error(), 503)
} else {
w.WriteHeader(201)
}
}
func (self *AppCubeFilter) Filte(w http.ResponseWriter, r *http.Request, c appengine.Context) bool {
c.Warningf("app cube filte")
// (self.App.GetAppCube())["BlogName"] = "Core"
if _, ok := (self.App.GetAppCube())["BlogCube"]; !ok {
cube, _ := ds.GetBlogCube(c)
if cube == nil {
cube = &M.BlogCube{}
cube.BlogName = "Blog"
cube.NoRight = false
ds.AddBlogCube(cube, c)
}
self.App.GetAppCube()["BlogCube"] = cube
}
return true
}
func Unsubscribe(c appengine.Context, ref SubscriptionRef) error {
subscriptionKey, err := ref.key(c)
if err != nil {
return err
}
if err := datastore.Delete(c, subscriptionKey); err != nil {
return err
}
if err := updateSubscriberCount(c, ref.SubscriptionID, -1); err != nil {
c.Warningf("Error decrementing subscriber count: %s", err)
}
return nil
}
// DELETE ./groups/xxx", xxx: Group name
// Header {"Instance-Id":"..."}
// Success returns 204 No Content
// Failure returns 400 Bad Request, 403 Forbidden, 500 Internal Server Error
func LeaveGroup(rw http.ResponseWriter, req *http.Request) {
// Appengine
var c appengine.Context = appengine.NewContext(req)
// Result, 0: success, 1: failed
var r int = http.StatusNoContent
// Sender instance ID
var instanceId string
// Group name to leave
var groupName string
// Function to write response header
defer func() {
if r == http.StatusNoContent {
// Return status. WriteHeader() must be called before call to Write
rw.WriteHeader(r)
} else {
http.Error(rw, http.StatusText(r), r)
}
}()
// Get instance ID from header
instanceId = req.Header.Get("Instance-Id")
if instanceId == "" {
c.Warningf("Missing instance ID. Ignore the request.")
r = http.StatusBadRequest
return
}
// Get group name from URL
var tokens []string
tokens = strings.Split(req.URL.Path, "/")
for i, v := range tokens {
if v == "groups" && i+1 < len(tokens) {
groupName = tokens[i+1]
break
}
}
if groupName == "" {
c.Warningf("Missing group name. Ignore the request.")
r = http.StatusBadRequest
return
}
// Vernon debug
c.Debugf("User %s is going to leave group %s", instanceId, groupName)
r = leaveGroup(c, instanceId, groupName)
}
func runFunc(c appengine.Context, w http.ResponseWriter, req *http.Request) {
defer req.Body.Close()
var inv invocation
if err := gob.NewDecoder(req.Body).Decode(&inv); err != nil {
c.Errorf("delay: failed decoding task payload: %v", err)
c.Warningf("delay: dropping task")
return
}
f := funcs[inv.Key]
if f == nil {
c.Errorf("delay: no func with key %q found", inv.Key)
c.Warningf("delay: dropping task")
return
}
ft := f.fv.Type()
in := []reflect.Value{reflect.ValueOf(c)}
for _, arg := range inv.Args {
var v reflect.Value
if arg != nil {
v = reflect.ValueOf(arg)
} else {
// Task was passed a nil argument, so we must construct
// the zero value for the argument here.
n := len(in) // we're constructing the nth argument
var at reflect.Type
if !ft.IsVariadic() || n < ft.NumIn()-1 {
at = ft.In(n)
} else {
at = ft.In(ft.NumIn() - 1).Elem()
}
v = reflect.Zero(at)
}
in = append(in, v)
}
out := f.fv.Call(in)
if n := ft.NumOut(); n > 0 && ft.Out(n-1) == osErrorType {
if errv := out[n-1]; !errv.IsNil() {
c.Errorf("delay: func failed (will retry): %v", errv.Interface())
w.WriteHeader(http.StatusInternalServerError)
return
}
}
}