func DeleteArticlesWithinScope(c appengine.Context, scope ArticleScope) error {
ancestorKey, err := scope.key(c)
if err != nil {
return err
}
batchWriter := NewBatchWriter(c, BatchDelete)
q := datastore.NewQuery("Article").Ancestor(ancestorKey).KeysOnly()
for t := q.Run(c); ; {
articleKey, err := t.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
return err
}
if err := batchWriter.EnqueueKey(articleKey); err != nil {
c.Errorf("Error queueing article for batch delete: %s", err)
return err
}
}
if err := batchWriter.Flush(); err != nil {
c.Errorf("Error flushing batch queue: %s", err)
return err
}
return nil
}
func searchGroup(name string, c appengine.Context) (key *datastore.Key, group *Group, err error) {
var v []Group
// Initial variables
key = nil
group = nil
err = nil
// Query
f := datastore.NewQuery(GroupKind)
f = f.Filter("Name=", name)
k, err := f.GetAll(c, &v)
if err != nil {
c.Errorf("%s in getting data from datastore\n", err)
err = errors.New("Datastore is temporary unavailable")
return
}
if k == nil || len(k) == 0 {
return
}
key = k[0]
group = &v[0]
return
}
func addLogo(profilePtr *image.Image, logo string, context appengine.Context) []byte {
profileImage := *profilePtr
destImage := image.NewRGBA(profileImage.Bounds())
draw.Draw(destImage, destImage.Bounds(), profileImage, image.ZP, draw.Src)
if logoImages, ok := THELOGOIMAGES[logo]; ok {
randi := rand.Intn(len(logoImages))
logoImage := logoImages[randi]
offset := image.Pt(5, 5)
if strings.HasPrefix(logo, "NLD-") {
offset = image.Pt(0, 0)
}
start := profileImage.Bounds().Size()
start = start.Sub(offset)
start = start.Sub(logoImage.Bounds().Size())
bounds := image.Rectangle{start, start.Add(logoImage.Bounds().Size())}
draw.Draw(destImage, bounds, logoImage, image.ZP, draw.Over)
} else {
context.Errorf("Cannot load logoimage for %s", logo)
}
buffer := new(bytes.Buffer)
err := png.Encode(buffer, destImage)
check(err, context)
return buffer.Bytes()
}
func writeEncoded(c appengine.Context, w http.ResponseWriter, data interface{}) {
newEncoder := bencode.NewEncoder(w)
if err := newEncoder.Encode(data); err != nil {
c.Errorf("Failed to encode data: %s", err)
writeInternalError(w)
}
}
func count_uv_pv(c appengine.Context, mins int) (uv int, pv int) {
count := 0
uniq := make(map[string]bool)
query := &log.Query{
AppLogs: true,
StartTime: time.Now().Add(time.Duration(-1*mins) * time.Minute),
Versions: []string{"1"},
}
for results := query.Run(c); ; {
record, err := results.Next()
if err == log.Done {
break
}
if err != nil {
c.Errorf("Failed to retrieve next log: %v", err)
}
if len(record.AppLogs) > 0 && strings.Index(record.Combined, "GET "+config.URL_BEACON) > 0 {
zcookie := record.AppLogs[0].Message
if zcookie != "" {
count++
uniq[zcookie] = true
}
}
}
uv = len(uniq)
pv = count
return
}
func HandleEvent(tags *event.EventHeaderTags, ev event.Event, ctx appengine.Context, sender EventSendService) error {
res, err := handleRecvEvent(tags, ev, ctx)
if nil != err {
ctx.Errorf("Failed to handle event[%d:%d] for reason:%v", ev.GetType(), ev.GetVersion(), err)
return err
}
if nil == res {
var empty bytes.Buffer
sender.Send(&empty)
return nil
}
res.SetHash(ev.GetHash())
compressType := Cfg.CompressType
if httpres, ok := res.(*event.HTTPResponseEvent); ok {
v := httpres.GetHeader("Content-Type")
if len(v) > 0 && Cfg.CompressType != event.COMPRESSOR_NONE {
if isContentTypeInCompressFilter(v) {
compressType = event.COMPRESSOR_NONE
}
}
}
x := new(event.CompressEvent)
x.SetHash(ev.GetHash())
x.CompressType = compressType
x.Ev = res
y := new(event.EncryptEvent)
y.SetHash(ev.GetHash())
y.EncryptType = Cfg.EncryptType
y.Ev = x
var buf bytes.Buffer
tags.Encode(&buf)
event.EncodeEvent(&buf, y)
sender.Send(&buf)
return nil
}
func load(c appengine.Context, id string) (*Group, error) {
group := &Group{}
_, err := memcache.JSON.Get(c, id, group)
if err == nil {
return group, nil
}
if err != memcache.ErrCacheMiss {
c.Errorf("memcache get %q: %v", id, err)
}
group, err = fetch(c, id)
if err != nil {
return nil, err
}
item := &memcache.Item{
Key: id,
Object: group,
Expiration: time.Hour,
}
err = memcache.JSON.Set(c, item)
if err != nil {
c.Errorf("memcache set %q: %v", id, err)
}
return group, nil
}
func GetAllClassesFromProds(c appengine.Context) []T {
c.Infof("GetAllClassesFromProds: Enter")
const cacheKey_product_classes = "product_classes"
var item_list []T
_, err := memcache.JSON.Get(c, cacheKey_product_classes, &item_list)
if err != nil {
//Memcache failed to extract the list.
c.Infof("GetAllClassesFromProds: Unable to retrieve list from Memcache: %v", err)
item_list = datastoreGetAllClassesFromProds(c)
memcache_item := &memcache.Item{
Key: cacheKey_product_classes,
Object: item_list,
Expiration: oneDay,
}
//Store into Memcache
// Add the item to the memcache, if the key does not already exist
if err := memcache.JSON.Set(c, memcache_item); err == memcache.ErrNotStored {
c.Infof("item with key %q already exists", memcache_item.Key)
} else if err != nil {
c.Errorf("error adding item: %v", err)
} else {
c.Infof("got stored as json ")
}
}
return item_list
}
func update(ctxt appengine.Context, kind string, data interface{}) error {
updaters.RLock()
up := updaters.m[kind]
updaters.RUnlock()
t := reflect.TypeOf(data)
dv, _ := strconv.Atoi(t.Elem().Field(0).Tag.Get("dataversion"))
if dv == 0 {
return nil
}
v := reflect.ValueOf(data)
v.Elem().Field(0).SetInt(int64(dv))
for _, fv := range up {
if fv.Type().In(0) != t {
return fmt.Errorf("type mismatch for data kind %q: cannot apply updater %s to %T", kind, fv.Type(), data)
}
rv := fv.Call([]reflect.Value{v})
if len(rv) > 0 {
err := rv[0].Interface().(error)
if err != nil {
ctxt.Errorf("applying updater: %v", err)
return fmt.Errorf("applying updater: %v", err)
}
}
}
return nil
}
func (r *Room) Send(c appengine.Context, message string) error {
var clients []Client
_, err := memcache.JSON.Get(c, r.Name, &clients)
if err != nil && err != memcache.ErrCacheMiss {
return err
}
if err == memcache.ErrCacheMiss {
q := datastore.NewQuery("Client").Ancestor(r.Key(c))
_, err = q.GetAll(c, &clients)
if err != nil {
return err
}
err = memcache.JSON.Set(c, &memcache.Item{
Key: r.Name, Object: clients,
})
if err != nil {
return err
}
}
for _, client := range clients {
err = channel.Send(c, client.ClientID, message)
if err != nil {
c.Errorf("sending %q: %v", message, err)
}
}
return nil
}
func PutInBlobstore(c appengine.Context, toStore interface{}) (appengine.BlobKey, error) {
//TODO: check capabilities
var k appengine.BlobKey
var data bytes.Buffer
enc := gob.NewEncoder(&data)
err := enc.Encode(toStore)
if err != nil {
c.Errorf("Datastore - PutInBlobstore - error 1 - %s", err)
return k, err
}
w, err := blobstore.Create(c, "application/octet-stream")
if err != nil {
c.Errorf("Datastore - PutInBlobstore - error 2 - %s", err)
return k, err
}
_, err = w.Write(data.Bytes())
if err != nil {
c.Errorf("Datastore - PutInBlobstore - error 3 - %s", err)
return k, err
}
err = w.Close()
if err != nil {
c.Errorf("Datastore - PutInBlobstore - error 4 - %s", err)
return k, err
}
k, err = w.Key()
if err != nil {
c.Errorf("Datastore - PutInBlobstore - error 5 - %s", err)
}
return k, err
}
func unregisterUser(addr string, c appengine.Context) {
q := datastore.NewQuery(USER_MODEL).
Filter("Email =", addr).
KeysOnly()
keys, err := q.GetAll(c, nil)
if err != nil {
c.Errorf("Cound not query the model for %s: %v", addr, err)
return
}
if len(keys) == 0 {
c.Infof("No such user to unregister: %s", addr)
return
}
for i := range keys {
datastore.Delete(c, keys[i])
}
c.Infof("Removed user %s", addr)
msg := &gaeMail.Message{
Sender: "*****@*****.**",
To: []string{addr},
Subject: "Email unregistered",
Body: "user " + addr + " has been unregistered",
}
gaeMail.Send(c, msg)
}
func sendSubscription(addr string, c appengine.Context) {
code := getSubCode()
msg := &gaeMail.Message{
Sender: "*****@*****.**",
To: []string{addr},
Subject: "confirm " + code,
Body: "Reply without changing subject",
}
if err := gaeMail.Send(c, msg); err != nil {
c.Errorf("Couldn't send email to %s for %s: %v", addr, code, err)
}
// XXXX if successful, register the code as (email, code, 0 (retry)) tuple.
confirmation := Confirmation{
Email: addr,
Code: code,
Retry: 0,
}
_, err := datastore.Put(c,
datastore.NewIncompleteKey(c, CONFIRM_MODEL, nil),
&confirmation)
if err != nil {
c.Errorf("Couldn't write confirmation code for %s, %s: %v", addr, code, err)
return
}
c.Infof("Wrote confirmation successfully for %s, %s", addr, code)
}
// send uses the Channel API to send the provided message in JSON-encoded form
// to the client identified by clientID.
//
// Channels created with one version of an app (eg, the default frontend)
// cannot be sent on from another version (eg, a backend). This is a limitation
// of the Channel API that should be fixed at some point.
// The send function creates a task that runs on the frontend (where the
// channel was created). The task handler makes the channel.Send API call.
func send(c appengine.Context, clientID string, m Message) {
if clientID == "" {
c.Debugf("no channel; skipping message send")
return
}
switch {
case m.TilesDone:
c.Debugf("tiles done")
case m.ZipDone:
c.Debugf("zip done")
default:
c.Debugf("%d tiles", len(m.IDs))
}
b, err := json.Marshal(m)
if err != nil {
panic(err)
}
task := taskqueue.NewPOSTTask("/send", url.Values{
"clientID": {clientID},
"msg": {string(b)},
})
host := appengine.DefaultVersionHostname(c)
task.Header.Set("Host", host)
if _, err := taskqueue.Add(c, task, sendQueue); err != nil {
c.Errorf("add send task failed: %v", err)
}
}
func TriggerBatchProcessing(c appengine.Context, article ArticleId) error {
// Instead of submitting a task to match incoming bids, resulting in one task per bid,
// we collect bids for up to two seconds and batch-process them afterwards.
semaphoreKey := "semaphore-" + string(article)
if semaphore, err := memcache.Increment(c, semaphoreKey, 1, 0); err != nil {
return err
} else if semaphore >= 2 {
c.Infof("Batch processing already triggered for article %v", article)
memcache.IncrementExisting(c, semaphoreKey, -1)
return nil
} else {
time.Sleep(1 * time.Second)
c.Infof("Starting batch processing...")
memcache.IncrementExisting(c, semaphoreKey, -1)
time_before := time.Now()
matchingErr := MatchIncomingBids(c, article)
time_after := time.Now()
duration := time_after.Sub(time_before)
if duration > 1000*time.Millisecond {
c.Errorf("Batch processing finished after %v. Limit exceeded!", duration)
} else if duration > 500*time.Millisecond {
c.Warningf("Batch processing finished after %v. Limit in danger.", duration)
} else {
c.Infof("Batch processing finished after %v.", duration)
}
return matchingErr
}
}
func fetchNomRoll(c appengine.Context, skey string, actId int) os.Error {
/// req,err := http.NewRequest("GET", fmt.Sprintf("https://cadetone.aafc.org.au/activities/nominalroll.php?ActID=%d",actId), nil)
/// if err != nil {
/// c.Errorf("Fetch Roll - actId:%d - %s", actId, err.String())
/// return err
/// }
/// session := c1SessionCookie(skey)
/// injectSession(req,session)
/// resp, err := GetClient(c).Do(req)
/// if err != nil {
/// c.Errorf("Fetch Roll - actId:%d - %s", actId, err.String())
/// return err
/// }
/// defer resp.Body.Close()
/// c.Infof("Fetch Roll OK - actId:%d", actId)
/// res,err := parseNomRoll(c, &resp.Body)
res, err := parseNomRoll(c)
if err != nil {
c.Errorf("Fetch Roll - actId:%d - %s", actId, err.String())
return err
}
res.ActId = actId
c.Infof(fmt.Sprint(res.ActId))
for _, r := range res.Roll {
c.Infof(fmt.Sprint(*r))
}
return err
}
func createUserKeyID(userID string, c appengine.Context) string {
if userID == "" {
c.Errorf("Creating an appUser entity key with no userID!")
}
return "user:" + userID
}
///func parsePSearch(c appengine.Context, r *io.ReadCloser) (string, os.Error) {
func parsePSearch(c appengine.Context) (string, os.Error) {
tmp := strings.NewReader(samplePSearchHtml)
r := &tmp
n, err := html.Parse(*r)
if err != nil {
c.Errorf("%s", err.String())
return "", err
}
nc := NewCursor(n)
nc = nc.FindById("main1")
if !nc.Valid {
printNode(c, n, 0)
return "", os.NewError("Can't find main1")
}
nc.Prune()
curr := nc.FindChildElement("table").NextSiblingElement("table")
curr = curr.FindChildElement("tr").NextSiblingElement("tr")
curr = curr.FindChildElement("input")
if !curr.Valid {
return "", os.NewError("Person not found")
}
for _, a := range curr.Attr() {
if a.Key == "onclick" {
l := strings.Split(a.Val, "'")
return l[1], nil
}
}
return "", os.NewError("No onclick attribute")
}
func createPostKeyID(postID string, c appengine.Context) string {
if postID == "" {
c.Errorf("Creating a post entity key with no postID!")
}
return "post:" + postID
}
func fooHandler(c appengine.Context, w http.ResponseWriter, r *http.Request) (interface{}, error) {
id := r.URL.Path[len(FOOS_API):]
c.Errorf("Received foo id %v", id)
if len(id) > 0 {
switch r.Method {
case "GET":
id64, err := strconv.ParseInt(id, 10, 64)
if err != nil {
return nil, err
}
foo := new(Foo)
foo.Id = FooId(id64)
return foo.get(c)
default:
return nil, fmt.Errorf(r.Method + " on " + r.URL.Path + " not implemented")
}
} else {
switch r.Method {
case "POST":
foo, err := decodeFoo(r.Body)
if err != nil {
return nil, err
}
return foo.save(c)
case "GET":
return getAllFoos(c)
default:
return nil, fmt.Errorf(r.Method + " on " + r.URL.Path + " not implemented")
}
}
return nil, nil
}
func webuserOK(c appengine.Context, w http.ResponseWriter, r *http.Request) bool {
if !userauthenticated(c) {
url, err := user.LoginURL(c, r.URL.String())
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return false
}
w.Header().Set("Location", url)
w.WriteHeader(http.StatusFound)
return false
}
u := user.Current(c)
authzed, err := userauthorized(c, u.Email)
if err != nil {
c.Errorf("authorization error: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return false
}
if !authzed {
c.Warningf("authorization failure: %v", u.Email)
w.WriteHeader(http.StatusForbidden)
err = templates.ExecuteTemplate(w, "unauthorized.html", nil)
if err != nil {
c.Errorf("unauthorized user and got err on template: %v", err)
}
return false
}
return true
}
func loadFullFeed(domain string, ctx appengine.Context, r *http.Request) (buf *bytes.Buffer) {
newctx := newappengine.NewContext(r)
tok, err := config.Exchange(newctx, r.FormValue("code"))
if err != nil {
ctx.Errorf("exchange error: %v", err)
return
}
ctx.Infof("tok: %v", tok)
client := config.Client(newctx, tok)
expFeedUrl := fmt.Sprintf(feedUrl, domain)
expFeedUrl = expFeedUrl + "&max-results=50000"
res, err := client.Get(expFeedUrl)
if err != nil {
ctx.Errorf("get: %v", err)
return
}
defer res.Body.Close()
buf = new(bytes.Buffer)
io.Copy(buf, res.Body)
return
}
// AppEngineLogHandler sends logs to AppEngine.
// The record must contain the appengine request context.
func AppEngineLogHandler() log15.Handler {
logFormat := log15.JsonFormat()
return log15.FuncHandler(func(r *log15.Record) error {
var c appengine.Context
index := 0
for i, e := range r.Ctx {
if ct, ok := e.(appengine.Context); ok {
c = ct
index = i
break
}
}
if c == nil {
// not in the context of a request
return nil
}
r.Ctx = append(r.Ctx[:index-1], r.Ctx[index+1:]...)
log := string(logFormat.Format(r))
switch r.Lvl {
case log15.LvlCrit:
c.Criticalf(log)
case log15.LvlError:
c.Errorf(log)
case log15.LvlWarn:
c.Warningf(log)
case log15.LvlInfo:
c.Infof(log)
case log15.LvlDebug:
c.Debugf(log)
}
return nil
})
}
// [END func_sign]
func GetAllProds(c appengine.Context) []models.Product {
const cacheKey = "products"
var item_list []models.Product
_, err := memcache.JSON.Get(c, cacheKey, &item_list)
if err != nil {
//Memcache failed to extract the list.
c.Infof("Unable to retrieve list from Memcache: %v", err)
q := datastore.NewQuery("Product").Ancestor(productKey(c)).Limit(1000)
item_list = make([]models.Product, 0, 1000)
if _, err := q.GetAll(c, &item_list); err != nil {
return nil
}
c.Infof("got products")
memcache_item := &memcache.Item{
Key: cacheKey,
Object: item_list,
Expiration: oneDay,
}
//Store into Memcache
// Add the item to the memcache, if the key does not already exist
if err := memcache.JSON.Set(c, memcache_item); err == memcache.ErrNotStored {
c.Infof("item with key %q already exists", memcache_item.Key)
} else if err != nil {
c.Errorf("error adding item: %v", err)
} else {
c.Infof("got stored as json ")
}
}
return item_list
}
// Sends an email to the author of part with a link to continue.
func sendMail(c appengine.Context, story Story) {
if story.Complete {
return
}
var subject, text string
part := story.LastPart()
url := fmt.Sprintf(serverUrl, story.Id, story.NextId)
if part != nil {
subject = "Please write the next part of this story."
text = fmt.Sprintf("%s, %s wrote:\n> %s\n\nPlease visit %s to write the next part.",
capital(fuzzyTime(part.Written)), getFullEmail(c, part.Author), part.Visible, url)
} else {
subject = "Please write the first part of this story."
text = fmt.Sprintf("%s, %s initiated a new story.\n\nPlease visit %s to write the beginning.",
capital(fuzzyTime(story.Created)), getFullEmail(c, story.Creator), url)
}
msg := &mail.Message{
Sender: sender,
To: []string{story.NextAuthor},
Subject: subject,
Body: text,
}
if err := mail.Send(c, msg); err != nil {
c.Errorf("Couldn't send email: %v", err)
panic(err)
}
}
// Log is a helper function that logs the given message to appenging
// with the given priority. Accepted priorities are "debug", "info",
// "warn", "error", and "crit". Other values default to "error".
func Log(c appengine.Context, r *http.Request, priority string,
message string, params ...interface{}) {
message = fmt.Sprintf("[%s] [%s] [%s]: %s", r.RemoteAddr, r.Method,
r.URL, message)
switch priority {
case "debug":
c.Debugf(message, params...)
case "info":
c.Infof(message, params...)
case "warn":
c.Warningf(message, params...)
case "error":
c.Errorf(message, params...)
case "crit":
c.Criticalf(message, params...)
default:
c.Errorf(message, params...)
}
}
func searchUser(instanceId string, c appengine.Context) (key *datastore.Key, user *User, err error) {
var v []User
// Initial variables
key = nil
user = nil
err = nil
// Query
f := datastore.NewQuery(UserKind)
f = f.Filter("InstanceId=", instanceId)
k, err := f.GetAll(c, &v)
if err != nil {
c.Errorf("%s in getting data from datastore\n", err)
err = errors.New("Datastore is temporary unavailable")
return
}
if k == nil || len(k) == 0 {
return
}
key = k[0]
user = &v[0]
return
}
// Object usually too big (1MB limit), so shard.
// http://stackoverflow.com/questions/9127982/
func BytesToShardedMemcache(c appengine.Context, key string, b []byte) {
/*
var buf bytes.Buffer
if err := gob.NewEncoder(&buf).Encode(f); err != nil {
db.C.Errorf("fdb error encoding item: %v", err)
return
}
b := buf.Bytes()
*/
items := []*memcache.Item{}
for i := 0; i < len(b); i += chunksize {
k := fmt.Sprintf("=%d=%s", i, key)
s, e := i, i+chunksize-1
if e >= len(b) {
e = len(b) - 1
}
c.Infof(" #=== [%7d, %7d] (%d) %s", s, e, len(b), k)
items = append(items, &memcache.Item{Key: k, Value: b[s : e+1]}) // slice sytax is [s,e)
}
if err := memcache.SetMulti(c, items); err != nil {
c.Errorf(" #=== cdb sharded store fail: %v", err)
}
c.Infof(" #=== Stored '%s' (len=%d)!", key, len(b))
}
// NewPage returns a new Page initialized embedding the template with the
// given name and data, the current user for the given context, and the
// latest announcement.
func NewPage(ctx appengine.Context, name string, data interface{}) (*Page, error) {
p := &Page{
Content: name,
Data: data,
Topics: topicList,
Cities: cityList,
}
a, err := conf.LatestAnnouncement(ctx)
if err != nil {
ctx.Errorf("latest announcement: %v", err)
}
if a != nil {
p.Announcement = a.Message
}
if u := user.Current(ctx); u != nil {
p.User = u
p.LogoutURL, err = user.LogoutURL(ctx, "/")
} else {
p.LoginURL, err = user.LoginURL(ctx, "/")
}
return p, err
}
func DeleteFolder(c appengine.Context, ref FolderRef) error {
folderKey, err := ref.key(c)
if err != nil {
return err
}
// Get a list of relevant subscriptions
q := datastore.NewQuery("Subscription").Ancestor(folderKey).KeysOnly().Limit(defaultBatchSize)
subscriptionKeys, err := q.GetAll(c, nil)
if err != nil {
return err
}
// Delete folder & subscriptions
if err := datastore.Delete(c, folderKey); err != nil {
c.Errorf("Error deleting folder: %s", err)
return err
}
if subscriptionKeys == nil {
// No subscriptions; nothing more to do
return nil
}
if err := datastore.DeleteMulti(c, subscriptionKeys); err != nil {
c.Errorf("Error deleting subscriptions: %s", err)
return err
}
return nil
}