// Log is a helper function that logs the given message to appenging
// with the given priority. Accepted priorities are "debug", "info",
// "warn", "error", and "crit". Other values default to "error".
func Log(c appengine.Context, r *http.Request, priority string,
message string, params ...interface{}) {
message = fmt.Sprintf("[%s] [%s] [%s]: %s", r.RemoteAddr, r.Method,
r.URL, message)
switch priority {
case "debug":
c.Debugf(message, params...)
case "info":
c.Infof(message, params...)
case "warn":
c.Warningf(message, params...)
case "error":
c.Errorf(message, params...)
case "crit":
c.Criticalf(message, params...)
default:
c.Errorf(message, params...)
}
}
func updateCacheTime(c appengine.Context, seq int64) {
const key = rootMemcacheKey
bseq := []byte(strconv.FormatInt(seq, 10))
for tries := 0; tries < 10; tries++ {
item, err := memcache.Get(c, key)
if err != nil {
c.Infof("memcache.Get %q: %v", key, err)
err = memcache.Add(c, &memcache.Item{Key: key, Value: bseq})
if err == nil {
c.Infof("memcache.Add %q %q ok", key, bseq)
return
}
c.Infof("memcache.Add %q %q: %v", key, bseq, err)
}
v, err := strconv.ParseInt(string(item.Value), 10, 64)
if err != nil {
c.Criticalf("memcache.Get %q = %q (%v)", key, item.Value, err)
return
}
if v >= seq {
return
}
item.Value = bseq
err = memcache.CompareAndSwap(c, item)
if err == nil {
c.Infof("memcache.CAS %q %d->%d ok", key, v, seq)
return
}
c.Infof("memcache.CAS %q %d->%d: %v", key, v, seq, err)
}
c.Criticalf("repeatedly failed to update root key")
}
func mailissue(ctxt appengine.Context, kind, key string) error {
ctxt.Infof("mailissue %s", key)
var cl CL
err := app.ReadData(ctxt, "CL", key, &cl)
if err != nil {
return nil // error already logged
}
if len(cl.NeedMailIssue) == 0 {
return nil
}
var mailed []string
for _, issue := range cl.NeedMailIssue {
err := postIssueComment(ctxt, issue, "CL https://codereview.appspot.com/"+cl.CL+" mentions this issue.")
if err != nil {
ctxt.Criticalf("posting to issue %v: %v", issue, err)
continue
}
mailed = append(mailed, issue)
}
err = app.Transaction(ctxt, func(ctxt appengine.Context) error {
var old CL
if err := app.ReadData(ctxt, "CL", key, &old); err != nil {
return err
}
old.MailedIssue = append(old.MailedIssue, mailed...)
return app.WriteData(ctxt, "CL", key, &old)
})
return err
}
// AppEngineLogHandler sends logs to AppEngine.
// The record must contain the appengine request context.
func AppEngineLogHandler() log15.Handler {
logFormat := log15.JsonFormat()
return log15.FuncHandler(func(r *log15.Record) error {
var c appengine.Context
index := 0
for i, e := range r.Ctx {
if ct, ok := e.(appengine.Context); ok {
c = ct
index = i
break
}
}
if c == nil {
// not in the context of a request
return nil
}
r.Ctx = append(r.Ctx[:index-1], r.Ctx[index+1:]...)
log := string(logFormat.Format(r))
switch r.Lvl {
case log15.LvlCrit:
c.Criticalf(log)
case log15.LvlError:
c.Errorf(log)
case log15.LvlWarn:
c.Warningf(log)
case log15.LvlInfo:
c.Infof(log)
case log15.LvlDebug:
c.Debugf(log)
}
return nil
})
}
// Handler to find all questions answered/being answered by the user in URL
func userHandler(w http.ResponseWriter, r *http.Request, c appengine.Context, user stackongo.User) {
userID, _ := strconv.Atoi(r.FormValue("id"))
query := userData{}
// Create and fill in a new webData struct
tempData := newWebData()
data.CacheLock.Lock()
// range through the question caches golang stackongo and add if the question contains the tag
tempData.Caches["unanswered"] = data.Caches["unanswered"]
if userQuery, ok := data.Users[userID]; ok {
query = userQuery
for cacheType, cache := range data.Users[userID].Caches {
if cacheType != "unanswered" {
tempData.Caches[cacheType] = cache
}
}
tempData.Qns = data.Qns
}
data.CacheLock.Unlock()
page := template.Must(template.ParseFiles("public/template.html"))
var userQuery = []string{
"user",
query.User_info.Display_name,
}
if err := page.Execute(w, writeResponse(user, tempData, c, userQuery)); err != nil {
c.Criticalf("%v", err.Error())
}
}
// Handler to find all questions with specific tags
func tagHandler(w http.ResponseWriter, r *http.Request, c appengine.Context, user stackongo.User) {
// Collect query
tag := r.FormValue("tagSearch")
// Create and fill in a new webData struct
tempData := newWebData()
data.CacheLock.Lock()
// range through the question caches golang stackongoand add if the question contains the tag
for cacheType, cache := range data.Caches {
for _, question := range cache {
if contains(question.Tags, tag) {
tempData.Caches[cacheType] = append(tempData.Caches[cacheType], question)
}
}
}
tempData.Qns = data.Qns
data.CacheLock.Unlock()
page := template.Must(template.ParseFiles("public/template.html"))
var tagQuery = []string{
"tag",
tag,
}
if err := page.Execute(w, writeResponse(user, tempData, c, tagQuery)); err != nil {
c.Criticalf("%v", err.Error())
}
}
//This is the main tags page
//Should display a list of tags that are logged in the database
//User can either click on a tag to view any questions containing that tag or search by a specific tag
func viewTagsHandler(w http.ResponseWriter, r *http.Request, c appengine.Context, user stackongo.User) {
//Read all tags and their counts from the db, and execute the page
query := readTagsFromDb()
//Format array of tags into another array, to be easier formatted on the page into a table
//An array of tagData arrays of size 4
var tagArray [][]tagData
var tempTagArray []tagData
i := 0
for _, t := range query {
tempTagArray = append(tempTagArray, t)
i++
if i == 4 {
tagArray = append(tagArray, tempTagArray)
i = 0
//clear the temp array.
tempTagArray = nil
}
}
tagArray = append(tagArray, tempTagArray)
page := template.Must(template.ParseFiles("public/viewTags.html"))
if err := page.Execute(w, queryReply{user, tagArray}); err != nil {
c.Criticalf("%v", err.Error())
}
}
func userPageHandler(w http.ResponseWriter, r *http.Request, c appengine.Context, user stackongo.User) {
page := template.Must(template.ParseFiles("public/userPage.html"))
usr, _ := strconv.Atoi(r.FormValue("userId"))
currentUser := data.Users[usr]
query := userData{User_info: currentUser.User_info}
var n int
query.Caches = make(map[string][]stackongo.Question)
n = Min(3, len(currentUser.Caches["unanswered"]))
if n > 0 {
query.Caches["answered"] = currentUser.Caches["answered"][0:n]
}
n = Min(3, len(currentUser.Caches["pending"]))
if n > 0 {
query.Caches["pending"] = currentUser.Caches["pending"][0:n]
}
n = Min(3, len(currentUser.Caches["updating"]))
if n > 0 {
query.Caches["updating"] = currentUser.Caches["updating"][0:n]
}
if err := page.Execute(w, queryReply{user, query}); err != nil {
c.Criticalf("%v", err.Error())
}
}
func cachePathTime(c appengine.Context, path string) (t int64, err error) {
t, err = cacheTime(c)
if err != nil {
return 0, err
}
key := fmt.Sprintf("%d,mtime,%s", t, path)
item, err := memcache.Get(c, key)
if err == nil {
v, err := strconv.ParseInt(string(item.Value), 10, 64)
if err == nil {
if chatty {
c.Infof("cachePathTime %q = %v", key, v)
}
return v, nil
}
c.Criticalf("memcache.Get %q = %q (%v) - deleting", key, item.Value, err)
memcache.Delete(c, key)
}
var seq int64
if fi, err := stat(c, path); err == nil {
seq = fi.Seq
}
c.Infof("cachePathTime save %q = %v", key, seq)
item = &memcache.Item{Key: key, Value: []byte(strconv.FormatInt(seq, 10))}
if err := memcache.Set(c, item); err != nil {
c.Criticalf("memcache.Set %q %q: %v", key, item.Value, err)
}
return seq, nil
}
func HandleError(c appengine.Context, w http.ResponseWriter, err error) {
w.WriteHeader(http.StatusInternalServerError)
err2 := Layout.Execute(w, tmplt.Context{"err": err})
if err2 != nil {
c.Criticalf("Got error %v while serving %v", err2, err)
return
}
}
func cacheWrite(c appengine.Context, t int64, kind, name string, data []byte) error {
mkey := fmt.Sprintf("%d,%s,%s", t, kind, name)
if true || chatty {
c.Infof("cacheWrite %s %d bytes", mkey, len(data))
}
err := memcache.Set(c, &memcache.Item{Key: mkey, Value: data})
if err != nil {
c.Criticalf("cacheWrite memcache.Set %q: %v", mkey, err)
}
return err
}
func writeLogMessage(c appengine.Context, level logLevel, msg string) {
const fmt = "%s"
switch level {
case levelDebug:
c.Debugf(fmt, msg)
case levelWarning:
c.Warningf(fmt, msg)
case levelError:
c.Errorf(fmt, msg)
case levelCritical:
c.Criticalf(fmt, msg)
default:
c.Infof(fmt, msg)
}
}
// notify tries to update the CL for the given Commit with a failure message.
// If it doesn't succeed, it sends a failure email to golang-dev.
func notify(c appengine.Context, com *Commit, builder, logHash string) {
var msg bytes.Buffer
err := notifyTmpl.Execute(&msg, struct {
Builder string
LogHash string
Hostname string
}{builder, logHash, domain})
if err != nil {
c.Criticalf("couldn't render template: %v", err)
return
}
if err := postGerritMessage(c, com, msg.String()); err != nil {
c.Errorf("couldn't post to gerrit: %v", err)
}
}
//TODO finish this function with the popper search parameters
func UserHasActiveSubscription(ctx appengine.Context, userKey *datastore.Key) (bool, error) {
count, err := datastore.NewQuery(TXN_KIND).
Ancestor(userKey).
Filter("PaymentActivationDate>=", time.Now().AddDate(0, -6, 0)).
Count(ctx)
if err != nil {
return false, err
}
if count > 1 {
ctx.Criticalf(fmt.Sprintf("User has multiple (%d) active subscriptions, key: %s", count, userKey.String()))
}
return count > 0, nil
}
func doExpire(c appengine.Context, expireTime time.Time, cursorString string) {
query := getExpiredQuery(expireTime).KeysOnly()
if len(cursorString) > 0 {
if cursor, err := datastore.DecodeCursor(cursorString); err != nil {
c.Errorf("Failed to decode cursor: %s", err)
return
} else {
query = query.Start(cursor)
}
}
for {
toDelete := make([]*datastore.Key, 0, 100)
for queryIterator := query.Run(c); ; {
peerKey, err := queryIterator.Next(nil)
if len(toDelete) >= 100 {
break
} else if err == datastore.Done {
c.Infof("Done finding expired peers")
break
} else if err != nil {
c.Criticalf("Failed to get next peer: %#v (%s)", err, err)
return
}
toDelete = append(toDelete, peerKey)
}
deleted := len(toDelete)
c.Infof("Deleting %d expired peers", deleted)
if err := ds.DeleteMulti(c, toDelete); err != nil {
c.Criticalf("Failed to delete peers: %#v (%s)", err, err)
return
}
if deleted <= 0 {
break
}
}
c.Infof("Finished deleting expired peers")
}
// Handler for viewing all users in the database
// Formats the response into an array of userData maps, for easier formatting onto the page.
func viewUsersHandler(w http.ResponseWriter, r *http.Request, c appengine.Context, user stackongo.User) {
query := data.Users
var queryArray []map[int]userData
var tempQueryArray []userData
for i, u := range query {
tempQueryArray = append(tempQueryArray, u)
if i%4 == 0 {
queryArray = append(queryArray, tempQueryArray)
//clear temp array
tempQueryArray = nil
}
}
queryArray = append(queryArray, tempQueryArray)
page := template.Must(template.ParseFiles("public/viewUsers.html"))
if err := page.Execute(w, queryReply{user, queryArray}); err != nil {
c.Criticalf("%v", err.Error())
}
}
func GetAllUsers(ctx appengine.Context) ([]*datastore.Key, []UserDTO, error) {
query := datastore.NewQuery(USER_KIND).
Ancestor(userCollectionParentKey(ctx))
count, err := query.Count(ctx)
if err != nil {
return nil, nil, err
}
if count <= 0 {
return nil, nil, nil
}
users := make([]UserDTO, 0, count)
keys, err := query.GetAll(ctx, &users)
if err != nil {
ctx.Criticalf("error in txn 2")
return nil, nil, err
}
return keys, users, nil
}
/*
SQL Fields: Title, Body, Link, Tags, Users.
*/
func searchHandler(w http.ResponseWriter, r *http.Request, c appengine.Context, user stackongo.User) {
//Collect query
search := r.FormValue("search")
//Convert to string to check ID against questions
i, err := strconv.Atoi(search)
searchType := ""
if err != nil {
i = 0
searchType = "URL"
} else {
searchType = "ID"
}
tempData := newWebData()
data.CacheLock.Lock()
//Range through questions, checking URL and question ID against search term
for cacheType, cache := range data.Caches {
for _, question := range cache {
if question.Question_id == i || question.Link == search || contains(question.Tags, search) ||
strings.Contains(question.Body, search) || strings.Contains(question.Title, search) {
tempData.Caches[cacheType] = append(tempData.Caches[cacheType], question)
}
}
}
tempData.Qns = data.Qns
data.CacheLock.Unlock()
page := template.Must(template.ParseFiles("public/template.html"))
var pageQuery = []string{
searchType,
search,
}
if err := page.Execute(w, writeResponse(user, tempData, c, pageQuery)); err != nil {
c.Criticalf("%v", err.Error())
}
}
func cacheTime(c appengine.Context) (t int64, err error) {
const key = rootMemcacheKey
item, err := memcache.Get(c, key)
if err == nil {
v, err := strconv.ParseInt(string(item.Value), 10, 64)
if err == nil {
if chatty {
c.Infof("cacheTime %q = %v", key, v)
}
return v, nil
}
c.Criticalf("memcache.Get %q = %q (%v) - deleting", key, item.Value, err)
memcache.Delete(c, key)
}
fi, err := stat(c, "/")
if err != nil {
c.Criticalf("stat /: %v", err)
return 0, err
}
updateCacheTime(c, fi.Seq)
return fi.Seq, nil
}
// Inital loading of DB.
func start(c appengine.Context) error {
var err error
loadWait = sync.NewCond(&mu)
c.Infof("Loading db on instance " + appengine.InstanceID())
client := createClient(c, time.Second*30)
resp, err := client.Get(dbUrl)
if err != nil {
c.Criticalf("Error downloading:%s", err.Error())
return err
}
defer resp.Body.Close()
db, err = oui.Open(resp.Body)
if err != nil {
c.Criticalf("Error parsing:%s", err.Error())
return err
}
t := time.Now().Add(time.Hour * 24)
UpdateAt = &t
c.Infof("Loaded, now serving...")
loadWait.Broadcast()
return nil
}
func ParseFeed(c appengine.Context, u string, b []byte) (*Feed, []*Story) {
if strings.TrimSpace(u) == "" {
c.Criticalf("badurl3: %v", u)
return nil, nil
}
f := Feed{Url: u}
var s []*Story
a := atom.Feed{}
var atomerr, rsserr, rdferr error
d := xml.NewDecoder(bytes.NewReader(b))
d.CharsetReader = charset.NewReader
if atomerr = d.Decode(&a); atomerr == nil {
f.Title = a.Title
if t, err := parseDate(c, &f, string(a.Updated)); err == nil {
f.Updated = t
}
for _, l := range a.Link {
if l.Rel != "self" {
f.Link = l.Href
break
}
}
for _, i := range a.Entry {
st := Story{
Id: i.ID,
Title: i.Title,
}
if t, err := parseDate(c, &f, string(i.Updated)); err == nil {
st.Updated = t
}
if t, err := parseDate(c, &f, string(i.Published)); err == nil {
st.Published = t
}
if len(i.Link) > 0 {
st.Link = i.Link[0].Href
}
if i.Author != nil {
st.Author = i.Author.Name
}
if i.Content != nil {
st.content, st.Summary = Sanitize(i.Content.Body)
} else if i.Summary != nil {
st.content, st.Summary = Sanitize(i.Summary.Body)
}
s = append(s, &st)
}
return parseFix(c, &f, s)
}
r := rssgo.Rss{}
d = xml.NewDecoder(bytes.NewReader(b))
d.CharsetReader = charset.NewReader
d.DefaultSpace = "DefaultSpace"
if rsserr = d.Decode(&r); rsserr == nil {
f.Title = r.Title
f.Link = r.Link
if t, err := parseDate(c, &f, r.LastBuildDate, r.PubDate); err == nil {
f.Updated = t
} else {
c.Warningf("no rss feed date: %v", f.Link)
}
for _, i := range r.Items {
st := Story{
Link: i.Link,
Author: i.Author,
}
if i.Title != "" {
st.Title = i.Title
} else if i.Description != "" {
i.Title = i.Description
}
if i.Content != "" {
st.content, st.Summary = Sanitize(i.Content)
} else if i.Title != "" && i.Description != "" {
st.content, st.Summary = Sanitize(i.Description)
}
if i.Guid != nil {
st.Id = i.Guid.Guid
}
if t, err := parseDate(c, &f, i.PubDate, i.Date, i.Published); err == nil {
st.Published = t
st.Updated = t
}
s = append(s, &st)
}
return parseFix(c, &f, s)
}
rdf := RDF{}
d = xml.NewDecoder(bytes.NewReader(b))
d.CharsetReader = charset.NewReader
if rdferr = d.Decode(&rdf); rdferr == nil {
if rdf.Channel != nil {
f.Title = rdf.Channel.Title
f.Link = rdf.Channel.Link
if t, err := parseDate(c, &f, rdf.Channel.Date); err == nil {
f.Updated = t
}
}
for _, i := range rdf.Item {
st := Story{
Id: i.About,
Title: i.Title,
Link: i.Link,
Author: i.Creator,
}
st.content, st.Summary = Sanitize(html.UnescapeString(i.Description))
if t, err := parseDate(c, &f, i.Date); err == nil {
st.Published = t
st.Updated = t
}
s = append(s, &st)
}
return parseFix(c, &f, s)
}
c.Warningf("atom parse error: %s", atomerr.Error())
c.Warningf("xml parse error: %s", rsserr.Error())
c.Warningf("rdf parse error: %s", rdferr.Error())
return nil, nil
}
func Criticalf(context appengine.Context, logMessage string, v ...interface{}) {
context.Criticalf("[ERROR] "+logMessage, v)
}
func cacheRead(c appengine.Context, kind, name, path string) (mtime int64, data []byte, pfi *proto.FileInfo, err error) {
for tries := 0; tries < 10; tries++ {
t, err := cachePathTime(c, path)
if err != nil {
return 0, nil, nil, err
}
key := fmt.Sprintf("%d,%s,%s", t, kind, name)
item, err := memcache.Get(c, key)
var data []byte
if item != nil {
data = item.Value
}
if err != nil {
c.Infof("memcache miss %q %v", key, err)
} else if chatty {
c.Infof("memcache hit %q (%d bytes)", key, len(data))
}
if kind != "data" {
// Not a file; whatever memcache says is all we have.
return t, data, nil, err
}
// Load stat from cache (includes negative entry).
statkey := fmt.Sprintf("%d,stat,%s", t, name)
var st statCacheEntry
_, err = memcache.JSON.Get(c, statkey, &st)
if err == nil {
if st.Error != "" {
if chatty {
c.Infof("memcache hit stat error %q %q", statkey, st.Error)
}
err = errors.New(st.Error)
} else {
if chatty {
c.Infof("memcache hit stat %q", statkey)
}
}
if err != nil || data != nil {
return t, data, st.FileInfo, err
}
}
// Need stat, or maybe stat+data.
var fi *FileInfo
if data != nil {
c.Infof("stat %q", name)
fi, err = stat(c, name)
if err == nil && fi.Seq != t {
c.Criticalf("loaded %s but found stat %d", key, fi.Seq)
continue
}
} else {
c.Infof("read %q", name)
fi, data, err = read(c, name)
if err == nil && fi.Seq != t {
c.Infof("loaded %s but found read %d", key, fi.Seq)
t = fi.Seq
key = fmt.Sprintf("%d,data,%s", t, name)
statkey = fmt.Sprintf("%d,stat,%s", t, name)
}
// Save data to memcache.
if err == nil {
if true || chatty {
c.Infof("save data in memcache %q", key)
}
item := &memcache.Item{Key: key, Value: data}
if err := memcache.Set(c, item); err != nil {
c.Criticalf("failed to cache %s: %v", key, err)
}
}
}
// Cache stat, including error.
st = statCacheEntry{}
if fi != nil {
st.FileInfo = &proto.FileInfo{
Name: fi.Name,
ModTime: fi.ModTime,
Size: fi.Size,
IsDir: fi.IsDir,
}
}
if err != nil {
st.Error = err.Error()
// If this is a deadline exceeded, do not cache.
if strings.Contains(st.Error, "Canceled") || strings.Contains(st.Error, "Deadline") {
return t, data, st.FileInfo, err
}
}
if chatty {
c.Infof("save stat in memcache %q", statkey)
}
if err := memcache.JSON.Set(c, &memcache.Item{Key: statkey, Object: &st}); err != nil {
c.Criticalf("failed to cache %s: %v", statkey, err)
}
// Done!
return t, data, st.FileInfo, err
}
c.Criticalf("failed repeatedly in cacheRead")
return 0, nil, nil, errors.New("cacheRead loop failed")
}
func Criticalf(c appengine.Context, format string, args ...interface{}) {
c.Criticalf(sprintf(format, args...))
}