func updateFeed(c appengine.Context, cl *http.Client, fk *datastore.Key) error {
resp, err := cl.Get(fk.StringID())
if err != nil {
return err
}
defer resp.Body.Close()
decoder := xml.NewDecoder(resp.Body)
var rfeed RSS
err = datastore.Get(c, fk, &rfeed)
if err != nil {
return err
}
if rfeed.IsAtom {
var afeed Atom
err = decoder.Decode(&afeed)
if err != nil {
return err
}
return afeed.update(c, fk)
} else {
err = decoder.Decode(&rfeed)
if err != nil {
return err
}
return rfeed.update(c, fk)
}
panic("unreachable")
}
func (g *gapDatabase) getKeyString(key *datastore.Key) string {
if tmp := key.IntID(); tmp != 0 {
return strconv.FormatInt(tmp, 10)
} else {
return key.StringID()
}
}
func FromGAEWithoutValidate(k *datastore.Key) (result key.Key) {
if k == nil {
return key.Key("")
}
result = key.NewWithoutValidate(k.Kind(), k.StringID(), k.IntID(), FromGAEWithoutValidate(k.Parent()))
return
}
func cacheSetFeed(c appengine.Context, k *datastore.Key, f FeedInfo) error {
id := k.StringID()
if len(id) > mcacheKeyMax { // silently don't set the value
c.Infof("Not caching feed with a big key: %s", id)
return nil
}
return memcache.Gob.Set(c, &memcache.Item{Key: k.StringID(), Object: f})
}
func cacheGetFeed(c appengine.Context, k *datastore.Key) (FeedInfo, error) {
id := k.StringID()
if len(id) > mcacheKeyMax {
return FeedInfo{}, ErrKeyTooBig
}
var f FeedInfo
_, err := memcache.Gob.Get(c, id, &f)
return f, err
}
// Unsubscribe removes a feed from the user's feed list.
func unsubscribe(c appengine.Context, feedKey *datastore.Key) error {
var f FeedInfo
err := datastore.RunInTransaction(c, func(c appengine.Context) error {
u, err := getUser(c)
if err != nil {
return err
}
i := 0
var k *datastore.Key
for i, k = range u.Feeds {
if feedKey.Equal(k) {
break
}
}
if i >= len(u.Feeds) {
return nil
}
err = fixMissingFieldError(datastore.Get(c, feedKey, &f))
if err != nil {
return err
}
f.Refs--
if f.Refs <= 0 {
if err := f.rmArticles(c); err != nil {
return err
}
if err := datastore.Delete(c, feedKey); err != nil {
return err
}
memcache.Delete(c, articlesMcacheKey(feedKey))
} else if _, err := datastore.Put(c, feedKey, &f); err != nil {
return err
}
u.Feeds = append(u.Feeds[:i], u.Feeds[i+1:]...)
return putUser(c, &u)
}, &datastore.TransactionOptions{XG: true})
if err != nil {
return err
}
flushUserPageCache(c)
if f.Refs <= 0 {
memcache.Delete(c, mcacheFeedsKey)
memcache.Delete(c, feedKey.StringID())
return nil
}
return memcache.Gob.Set(c, &memcache.Item{Key: feedKey.StringID(), Object: f})
}
func FromGAE(k *datastore.Key) (result key.Key, err error) {
if k == nil {
return key.Key(""), nil
}
parent, err := FromGAE(k.Parent())
if err != nil {
return
}
return key.New(k.Kind(), k.StringID(), k.IntID(), parent)
}
func refresh(c appengine.Context, k *datastore.Key) error {
f, err := getFeed(c, k)
if err == datastore.ErrNoSuchEntity {
return nil
}
if err != nil {
return errors.New(k.StringID() + " failed to load from the datastore: " + err.Error())
}
if err := f.ensureFresh(c); err != nil {
return errors.New(f.Url + " failed to refresh: " + err.Error())
}
return nil
}
func articlesByFeed(c appengine.Context, key *datastore.Key) (FeedInfo, Articles, []error) {
f, err := getFeed(c, key)
if err != nil {
err = fmt.Errorf("%s: failed to load from the datastore: %s", key.StringID(), err.Error())
return FeedInfo{}, nil, []error{err}
}
as, err := f.articlesSince(c, time.Time{})
var errs []error
if err != nil {
errs = []error{err}
}
return f, as, errs
}
func recursiveJson(key *datastore.Key) Response {
var parentJson Response
if key.Parent() != nil {
parentJson = recursiveJson(key.Parent())
}
return Response{
"stringID": key.StringID(),
"intID": key.IntID(),
"kind": key.Kind(),
"appID": key.AppID(),
"namespace": key.Namespace(),
"parent": parentJson,
}
}
// Can check if the user has permission to perform the action.
func (u *User) Can(c appengine.Context, perm string, key *datastore.Key) bool {
// Users can do anything to their own user object.
if key.Kind() == "User" && u.Key.StringID() == key.StringID() {
return true
}
// Admins can do anything.
if u.HasRole("admin") {
return true
}
// Other permissions must be set.
if ok, _ := acl.Can(c, u.Key.String(), perm, key); ok {
return true
}
return false
}
func translate(sourceKey *datastore.Key, translateAppId bool, targetAppId string, translateNamespace bool, targetNamespace string) (*datastore.Key, error) {
if !translateAppId {
targetAppId = sourceKey.AppID()
}
if !translateNamespace {
targetNamespace = sourceKey.Namespace()
}
var translatedParent *datastore.Key = nil
if sourceKey.Parent() != nil {
var err error
translatedParent, err = translate(sourceKey.Parent(), translateAppId, targetAppId, translateNamespace, targetNamespace)
if err != nil {
return nil, err
}
}
return datastorekey.CreateKey(nil, targetAppId, targetNamespace, sourceKey.Kind(), sourceKey.StringID(), sourceKey.IntID(), translatedParent)
}
func fillFields(key *datastore.Key, data map[string]interface{}) {
data["kind"] = key.Kind()
data["stringid"] = key.StringID()
data["intid"] = key.IntID()
data["appid"] = key.AppID()
data["namespace"] = key.Namespace()
if key.Parent() != nil {
data["kind2"] = key.Parent().Kind()
data["stringid2"] = key.Parent().StringID()
data["intid2"] = key.Parent().IntID()
if key.Parent().Parent() != nil {
data["kind3"] = key.Parent().Parent().Kind()
data["stringid3"] = key.Parent().Parent().StringID()
data["intid3"] = key.Parent().Parent().IntID()
}
}
}
func (key *Key) Save(c appengine.Context) error {
var kk *datastore.Key
if key.Id == "" {
if key.AgentId == 0 || key.PortalId == "" {
return fmt.Errorf("No key id, portal id or agent id")
}
kk = datastore.NewKey(c, "Key", key.PortalId+strconv.FormatInt(key.AgentId, 10), 0, nil)
} else {
kk = datastore.NewKey(c, "Key", key.Id, 0, nil)
}
_, err := datastore.Put(c, kk, key)
if err != nil {
return err
}
key.Id = kk.StringID()
return nil
}
func teacherByKey(c appengine.Context, key *datastore.Key) (*Teacher, error) {
teacher := &Teacher{}
switch err := datastore.Get(c, key, teacher); err {
case nil:
break
case datastore.ErrNoSuchEntity:
return nil, ErrUserIsNotTeacher
default:
if isFieldMismatch(err) {
c.Errorf("Teacher field mismatch: %s", err)
break
}
return nil, err
}
teacher.ID = key.StringID()
return teacher, nil
}
func updateFeedsJob(pfc *PFContext) error {
c := pfc.C
importing := 0
started := time.Now()
fetchTime := time.Now()
doneChannel := make(chan *storage.FeedMeta)
var jobError error
if appengine.IsDevAppServer() {
// On dev server, disregard next update limitations
// (by "forwarding the clock")
fetchTime = fetchTime.Add(time.Duration(24) * time.Hour)
}
q := datastore.NewQuery("FeedMeta").Filter("NextFetch <", fetchTime)
for t := q.Run(c); ; {
feedMeta := new(storage.FeedMeta)
var feedMetaKey *datastore.Key
if key, err := t.Next(feedMeta); err == datastore.Done {
break
} else if err == nil || storage.IsFieldMismatch(err) {
feedMetaKey = key
} else {
c.Errorf("Error fetching feed record: %s", err)
jobError = err
break
}
go updateFeed(pfc.C, doneChannel, feedMetaKey.StringID(), feedMeta)
importing++
}
for i := 0; i < importing; i++ {
<-doneChannel
}
c.Infof("%d feeds completed in %s", importing, time.Since(started))
return jobError
}
func unsubscriber(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
k, err := datastore.DecodeKey(r.URL.RawQuery)
if err != nil {
handleError(w, err)
return
}
err = datastore.Delete(c, k)
if err != nil {
handleError(w, err)
return
}
fk := k.Parent()
uk, err := datastore.DecodeKey(k.StringID())
if err != nil {
handleError(w, err)
return
}
iter := datastore.NewQuery("subscribedItem").Ancestor(uk).KeysOnly().Run(c)
var sik *datastore.Key
for sik, err = iter.Next(nil); err == nil; sik, err = iter.Next(nil) {
ik, err := datastore.DecodeKey(sik.StringID())
if err != nil {
handleError(w, err)
return
}
if ik.Parent().Equal(fk) {
err = datastore.Delete(c, sik)
if err != nil {
handleError(w, err)
return
}
}
}
if err != datastore.Done {
handleError(w, err)
return
}
http.Redirect(w, r, "/feeds/", http.StatusFound)
}
func extract(key *datastore.Key, field string) (string, error) {
switch strings.ToLower(field) {
case "kind":
return key.Kind(), nil
case "appid":
return key.AppID(), nil
case "namespace":
return key.Namespace(), nil
case "name":
return key.StringID(), nil
case "id":
return fmt.Sprintf("%v", key.IntID()), nil
case "parent":
if key.Parent() == nil {
return "", nil
} else {
return key.Parent().Encode(), nil
}
default:
return "", fmt.Errorf("Unsupported field [%v]. Supported fields are kind, appID, namespace, name, id, parentkey.", field)
}
}
func byKey(c appengine.Context, key *datastore.Key) (*Account, error) {
acct := &Account{}
if err := datastore.Get(c, key, acct); err != nil {
switch {
case err == datastore.ErrNoSuchEntity:
return nil, ErrUserNotFound
case isFieldMismatch(err):
c.Warningf("Type mismatch on user %q: %+v", key.StringID(), err)
return acct, nil
default:
c.Errorf("Failed looking up user %q: %s", key.StringID(), err)
return nil, ErrUserNotFound
}
}
acct.ID = key.StringID()
return acct, nil
}
func starID(key *datastore.Key) string {
return fmt.Sprintf("%s|%s", key.Parent().StringID(), key.StringID())
}
func (it *Iterator) Next() bool {
if it.offset+1 < len(it.buffer) {
it.offset++
it.result = &Token{Kind: it.kind, Hash: it.buffer[it.offset]}
return true
}
if it.done {
return false
}
// Reset buffer and offset
it.offset = 0
it.buffer = make([]string, 0, bufferSize)
// Create query
// TODO (panamafrancis) Keys only query?
q := datastore.NewQuery(it.kind).Limit(bufferSize)
if !it.isAll {
// Filter on the direction {subject,objekt...}
q = q.Filter(it.dir.String()+" =", it.name)
}
// Get last cursor position
cursor, err := datastore.DecodeCursor(it.last)
if err == nil {
q = q.Start(cursor)
}
// Buffer the keys of the next 50 matches
t := q.Run(it.qs.context)
for {
// Quirk of the datastore, you cannot pass a nil value to to Next()
// even if you just want the keys
var k *datastore.Key
skip := false
if it.kind == quadKind {
temp := new(QuadEntry)
k, err = t.Next(temp)
// Skip if quad has been deleted
if len(temp.Added) <= len(temp.Deleted) {
skip = true
}
} else {
temp := new(NodeEntry)
k, err = t.Next(temp)
// Skip if node has been deleted
if temp.Size == 0 {
skip = true
}
}
if err == datastore.Done {
it.done = true
break
}
if err != nil {
glog.Errorf("Error fetching next entry %v", err)
it.err = err
return false
}
if !skip {
it.buffer = append(it.buffer, k.StringID())
}
}
// Save cursor position
cursor, err = t.Cursor()
if err == nil {
it.last = cursor.String()
}
// Protect against bad queries
if it.done && len(it.buffer) == 0 {
glog.Warningf("Query did not return any results")
return false
}
// First result
it.result = &Token{Kind: it.kind, Hash: it.buffer[it.offset]}
return true
}
return err
}
return nil
}
var propagate = delay.Func("propagate", func(c appengine.Context, ik *datastore.Key) error {
var it Item
err := datastore.Get(c, ik, &it)
if err != nil {
return err
}
si := subscribedItem{it.PubDate}
iter := datastore.NewQuery("subscription").Ancestor(ik.Parent()).KeysOnly().Run(c)
var sk *datastore.Key
for sk, err = iter.Next(nil); err == nil; sk, err = iter.Next(nil) {
uk, err := datastore.DecodeKey(sk.StringID())
if err != nil {
return err
}
_, err = datastore.Put(c, datastore.NewKey(c, "subscribedItem", ik.Encode(), 0, uk), &si)
if err != nil {
return err
}
}
if err != datastore.Done {
return err
}
return nil
})
func unsubscriber(w http.ResponseWriter, r *http.Request) {
func (g *Goon) setStructKey(src interface{}, key *datastore.Key) error {
v := reflect.ValueOf(src)
t := v.Type()
k := t.Kind()
if k != reflect.Ptr {
return fmt.Errorf("goon: Expected pointer to struct, got instead: %v", k)
}
v = reflect.Indirect(v)
t = v.Type()
k = t.Kind()
if k != reflect.Struct {
return fmt.Errorf(fmt.Sprintf("goon: Expected struct, got instead: %v", k))
}
idSet := false
kindSet := false
parentSet := false
for i := 0; i < v.NumField(); i++ {
tf := t.Field(i)
vf := v.Field(i)
if !vf.CanSet() {
continue
}
tag := tf.Tag.Get("goon")
tagValues := strings.Split(tag, ",")
if len(tagValues) > 0 {
tagValue := tagValues[0]
if tagValue == "id" {
if idSet {
return fmt.Errorf("goon: Only one field may be marked id")
}
switch vf.Kind() {
case reflect.Int64:
vf.SetInt(key.IntID())
idSet = true
case reflect.String:
vf.SetString(key.StringID())
idSet = true
}
} else if tagValue == "kind" {
if kindSet {
return fmt.Errorf("goon: Only one field may be marked kind")
}
if vf.Kind() == reflect.String {
if (len(tagValues) <= 1 || key.Kind() != tagValues[1]) && g.KindNameResolver(src) != key.Kind() {
vf.Set(reflect.ValueOf(key.Kind()))
}
kindSet = true
}
} else if tagValue == "parent" {
if parentSet {
return fmt.Errorf("goon: Only one field may be marked parent")
}
dskeyType := reflect.TypeOf(&datastore.Key{})
vfType := vf.Type()
if vfType.ConvertibleTo(dskeyType) {
vf.Set(reflect.ValueOf(key.Parent()).Convert(vfType))
parentSet = true
}
}
}
}
if !idSet {
return fmt.Errorf("goon: Could not set id field")
}
return nil
}
// cacheKey generates a memcache key for this non-nil *datastore.Key.
func cacheKey(key *datastore.Key) string {
return fmt.Sprintf("%v/%v/sb=>%v", key.Encode(), key.Kind(), key.StringID())
}
func SendEmail(c appengine.Context, key *datastore.Key) error {
// Lookup the publication
var pub Publication
c.Infof("Using key %s", key.StringID())
if err := datastore.Get(c, key, &pub); err != nil {
return err
}
// Update the mod times
pub.LastChecked = time.Now()
// Go and get the content
client := urlfetch.Client(c)
resp, err := client.Get(pub.Url)
if err != nil {
return err
}
if resp.StatusCode != 200 {
return &UrlFetchError{key.IntID(), pub.Url, resp.Status}
}
buf := make([]byte, resp.ContentLength)
_, err = resp.Body.Read(buf)
if err != nil {
return err
}
checksum := strconv.FormatUint(crc64.Checksum(buf, crc64.MakeTable(crc64.ISO)), 36)
if checksum == pub.Checksum {
c.Infof("Checksum has not changed. Not sending mail.")
return nil
}
c.Infof("Checksum for [%s] has changed. Sending mail.", pub.Name)
pub.Length = resp.ContentLength
pub.LastChanged = time.Now()
pub.Checksum = checksum
doc := &mail.Attachment{
Name: "Update from Enbook.me",
Data: buf,
}
message := &mail.Message{
Sender: "Enbook.me <*****@*****.**>",
To: []string{pub.Email},
Subject: "",
Body: "Update from Enbook.me",
Attachments: []mail.Attachment{*doc},
}
if err := mail.Send(c, message); err != nil {
c.Errorf("Failed to send mail [%s].", pub.Name)
return err
}
pub.LastUpdateSent = time.Now()
// Save the changes
if _, err = datastore.Put(c, key, &pub); err != nil {
c.Errorf("Failed to save datastore entity [%s].", pub.Name)
return err
}
return nil
}
func DepositUid(key *datastore.Key) string {
return key.StringID()
}
func updateSubscriptionByKey(c appengine.Context, subscriptionKey *datastore.Key, subscription Subscription) (int, error) {
feedKey := subscription.Feed
largestUpdateIndexWritten := int64(-1)
unreadDelta := 0
batchWriter := NewBatchWriter(c, BatchPut)
q := datastore.NewQuery("EntryMeta").Ancestor(feedKey).Filter("UpdateIndex >", subscription.MaxUpdateIndex)
for t := q.Run(c); ; {
entryMeta := new(EntryMeta)
_, err := t.Next(entryMeta)
if err == datastore.Done {
break
} else if IsFieldMismatch(err) {
// Ignore
} else if err != nil {
c.Errorf("Error reading Entry: %s", err)
return batchWriter.Written(), err
}
articleKey := datastore.NewKey(c, "Article", entryMeta.Entry.StringID(), 0, subscriptionKey)
article := Article{}
if err := datastore.Get(c, articleKey, &article); err == datastore.ErrNoSuchEntity {
// New article
article.Entry = entryMeta.Entry
article.Properties = []string{"unread"}
unreadDelta++
} else if IsFieldMismatch(err) {
// Ignore - migration
} else if err != nil {
c.Warningf("Error reading article %s: %s", entryMeta.Entry.StringID(), err)
continue
}
article.UpdateIndex = entryMeta.UpdateIndex
article.Fetched = entryMeta.Fetched
article.Published = entryMeta.Published
if entryMeta.UpdateIndex > largestUpdateIndexWritten {
largestUpdateIndexWritten = entryMeta.UpdateIndex
}
if err := batchWriter.Enqueue(articleKey, &article); err != nil {
c.Errorf("Error queueing article for batch write: %s", err)
return batchWriter.Written(), err
}
}
if err := batchWriter.Flush(); err != nil {
c.Errorf("Error flushing batch queue: %s", err)
return batchWriter.Written(), err
}
if batchWriter.Written() > 0 {
if appengine.IsDevAppServer() {
c.Debugf("Completed %s: %d records", subscriptionKey.StringID(), batchWriter.Written())
}
// Write the subscription
subscription.Updated = time.Now()
subscription.MaxUpdateIndex = largestUpdateIndexWritten
if subscription.UnreadCount+unreadDelta >= 0 {
subscription.UnreadCount += unreadDelta
}
if _, err := datastore.Put(c, subscriptionKey, &subscription); err != nil {
c.Errorf("Error writing subscription: %s", err)
return batchWriter.Written(), err
}
// Update usage index (rough way to track feed popularity)
// No sharding, no transactions - complete accuracy is unimportant for now
feedUsage := FeedUsage{}
feedUsageKey := datastore.NewKey(c, "FeedUsage", feedKey.StringID(), 0, nil)
if err := datastore.Get(c, feedUsageKey, &feedUsage); err == datastore.ErrNoSuchEntity || err == nil {
if err == datastore.ErrNoSuchEntity {
// Create a new entity
feedUsage.Feed = feedKey
}
feedUsage.UpdateCount++
feedUsage.LastSubscriptionUpdate = time.Now()
if _, err := datastore.Put(c, feedUsageKey, &feedUsage); err != nil {
c.Warningf("Non-critical error updating feed usage (%s): %s", feedKey.StringID(), err)
}
}
}
return batchWriter.Written(), nil
}
func articlesMcacheKey(feedKey *datastore.Key) string {
return "articles:" + feedKey.StringID()
}
func postUnique(c appengine.Context, k *datastore.Key, isNew bool) (err error) {
chu := make(chan uniqueFetch)
go func() {
rv := uniqueFetch{}
rv.u, rv.err = loadUnique(c, k)
chu <- rv
}()
countch := make(chan int)
go func() {
count, err := sharded_counter.Count(c, "unique_"+k.StringID())
if err != nil {
c.Infof("Counter error: %v", err)
count = -1
}
countch <- count
}()
revch := make(chan revRes)
if !isNew {
go loadRev(c, k.StringID(), revch)
} else {
close(revch)
}
// End of async invocations, now we get serious
uf := <-chu
if uf.err != nil {
return uf.err
}
u := uf.u
doc := map[string]interface{}{}
err = json.Unmarshal(u.Data, &doc)
if err != nil {
c.Errorf("Can't process JSON from %s: %v. Skipping", u.Data, err)
return nil
}
count := <-countch
if count >= 0 {
doc["contacts"] = count
}
revres := <-revch
if revres.err == nil && revres.rev != "" {
doc["_rev"] = revres.rev
}
doc["type"] = "unique"
doc["first"] = u.First
doc["latest"] = u.Latest
doc["nodesMax"] = u.NodesMax
doc["bucketsMax"] = u.BucketsMax
doc["mbBucketsMax"] = u.MBBucketsMax
doc["mcBucketsMax"] = u.MCBucketsMax
doc["ramMax"] = u.RAMMax
doc["usedVersions"] = u.UsedVersions
doc["usedOSes"] = u.UsedOSes
doc["usedOSesRaw"] = u.UsedOSesRaw
doc["addr"] = u.Addr
doc["geop"] = geocodeAddr(c, u)
doc["os"] = u.OS
doc["active_version"] = true // XXX
doc["api_ver"] = u.ApiVer
if nodes, ok := doc["nodes"].(map[string]interface{}); ok {
if oses, ok := nodes["os"].([]interface{}); ok {
nodes["os_orig"] = oses
a := []string{}
for _, o := range oses {
if s, ok := o.(string); ok {
a = append(a, recognizeOS(s))
}
}
nodes["os"] = a
}
}
rev, err := storeInCouch(c, doc, "unique_"+k.StringID())
if err != nil {
return err
}
if rev != "" {
err = memcache.Set(c, &memcache.Item{
Key: "rev." + k.StringID(),
Value: []byte(rev),
})
}
return
}
func rebuildKey(c appengine.Context, key *datastore.Key) *datastore.Key {
if key == nil {
return nil
}
return datastore.NewKey(c, key.Kind(), key.StringID(), key.IntID(), rebuildKey(c, key.Parent()))
}