Пример #1
0
func Main(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	if err := templates.ExecuteTemplate(w, "base.html", includes(c, w, r)); err != nil {
		c.Errorf("%v", err)
		serveError(w, err)
		return
	}
}
Пример #2
0
func CFix(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	url := r.FormValue("feed")
	c.Infof("fix feed %s", url)
	f := Feed{Url: url}
	if err := gn.Get(&f); err != nil {
		c.Criticalf("cfix err: %v", err)
		serveError(w, err)
		return
	}

	q := datastore.NewQuery("S").Ancestor(gn.Key(&f))
	var ss []*Story
	keys, err := q.GetAll(c, &ss)
	if err != nil {
		c.Errorf("getall err: %v", err)
		serveError(w, err)
		return
	}
	c.Infof("trying to fix %v stories", len(ss))
	const putLimit = 500
	for i := 0; i <= len(keys)/putLimit; i++ {
		lo := i * putLimit
		hi := (i + 1) * putLimit
		if hi > len(keys) {
			hi = len(keys)
		}
		c.Infof("%v - %v", lo, hi)
		if _, err := datastore.PutMulti(c, keys[lo:hi], ss[lo:hi]); err != nil {
			c.Errorf("err: %v, %v, %v", lo, hi, err)
		}
	}
}
Пример #3
0
func AddSubscription(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	backupOPML(c)
	cu := user.Current(c)
	url := r.FormValue("url")
	o := &OpmlOutline{
		Outline: []*OpmlOutline{
			&OpmlOutline{XmlUrl: url},
		},
	}
	if err := addFeed(c, cu.ID, o); err != nil {
		c.Errorf("add sub error (%s): %s", url, err.Error())
		serveError(w, err)
		return
	}

	gn := goon.FromContext(c)
	ud := UserData{Id: "data", Parent: gn.Key(&User{Id: cu.ID})}
	gn.Get(&ud)
	if err := mergeUserOpml(c, &ud, o); err != nil {
		c.Errorf("add sub error opml (%v): %v", url, err)
		serveError(w, err)
		return
	}
	gn.PutMany(&ud, &Log{
		Parent: ud.Parent,
		Id:     time.Now().UnixNano(),
		Text:   fmt.Sprintf("add sub: %v", url),
	})
	if r.Method == "GET" {
		http.Redirect(w, r, routeUrl("main"), http.StatusFound)
	}
	backupOPML(c)
}
Пример #4
0
func Charge(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	cu := user.Current(c)
	gn := goon.FromContext(c)
	u := User{Id: cu.ID}
	uc := UserCharge{Id: 1, Parent: gn.Key(&u)}
	if err := gn.Get(&u); err != nil {
		serveError(w, err)
		return
	} else if u.Account != AFree {
		serveError(w, fmt.Errorf("You're already subscribed."))
		return
	}
	if err := gn.Get(&uc); err == nil && len(uc.Customer) > 0 {
		serveError(w, fmt.Errorf("You're already subscribed."))
		return
	} else if err != datastore.ErrNoSuchEntity {
		serveError(w, err)
		return
	}
	resp, err := stripe(c, "POST", "customers", url.Values{
		"email":       {u.Email},
		"description": {u.Id},
		"card":        {r.FormValue("stripeToken")},
		"plan":        {STRIPE_PLAN},
	}.Encode())
	if err != nil {
		serveError(w, err)
		return
	} else if resp.StatusCode != http.StatusOK {
		c.Errorf("%s", resp.Body)
		serveError(w, fmt.Errorf("Error"))
		return
	}
	defer resp.Body.Close()
	b, _ := ioutil.ReadAll(resp.Body)
	var sc StripeCustomer
	if err := json.Unmarshal(b, &sc); err != nil {
		serveError(w, err)
		return
	}
	if err := gn.RunInTransaction(func(gn *goon.Goon) error {
		if err := gn.Get(&u); err != nil && err != datastore.ErrNoSuchEntity {
			return err
		}
		if err := gn.Get(&uc); err != nil && err != datastore.ErrNoSuchEntity {
			return err
		}
		u.Account = APaid
		uc.Customer = sc.Id
		uc.Last4 = sc.Card.Last4
		uc.Created = time.Unix(sc.Created, 0)
		_, err := gn.PutMany(&u, &uc)
		return err
	}, nil); err != nil {
		serveError(w, err)
		return
	}
	b, _ = json.Marshal(&uc)
	w.Write(b)
}
Пример #5
0
func doUncheckout(c mpg.Context) (*UserCharge, error) {
	cu := user.Current(c)
	gn := goon.FromContext(c)
	u := User{Id: cu.ID}
	uc := UserCharge{Id: 1, Parent: gn.Key(&u)}
	if err := gn.Get(&u); err != nil {
		return nil, err
	}
	if err := gn.Get(&uc); err != nil || len(uc.Customer) == 0 {
		return nil, err
	}
	resp, err := stripe(c, "DELETE", "customers/"+uc.Customer, "")
	if err != nil {
		return nil, err
	} else if resp.StatusCode != http.StatusOK {
		c.Errorf("%s", resp.Body)
		c.Errorf("stripe delete error, but proceeding")
	}
	if err := gn.RunInTransaction(func(gn *goon.Goon) error {
		if err := gn.Get(&u); err != nil && err != datastore.ErrNoSuchEntity {
			return err
		}
		u.Account = AFree
		u.Until = uc.Next
		if err := gn.Delete(gn.Key(&uc)); err != nil {
			return err
		}
		_, err := gn.Put(&u)
		return err
	}, nil); err != nil {
		return nil, err
	}
	return &uc, nil
}
Пример #6
0
func UploadOpml(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	opml := Opml{}
	if err := json.Unmarshal([]byte(r.FormValue("opml")), &opml.Outline); err != nil {
		serveError(w, err)
		return
	}
	backupOPML(c)
	cu := user.Current(c)
	gn := goon.FromContext(c)
	u := User{Id: cu.ID}
	ud := UserData{Id: "data", Parent: gn.Key(&u)}
	if err := gn.Get(&ud); err != nil {
		serveError(w, err)
		c.Errorf("get err: %v", err)
		return
	}
	if b, err := json.Marshal(&opml); err != nil {
		serveError(w, err)
		c.Errorf("json err: %v", err)
		return
	} else {
		l := Log{
			Parent: ud.Parent,
			Id:     time.Now().UnixNano(),
			Text:   fmt.Sprintf("upload opml: %v -> %v", len(ud.Opml), len(b)),
		}
		ud.Opml = b
		if _, err := gn.PutMany(&ud, &l); err != nil {
			serveError(w, err)
			return
		}
		backupOPML(c)
	}
}
Пример #7
0
func FeedHistory(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	cu := user.Current(c)
	gn := goon.FromContext(c)
	u := User{Id: cu.ID}
	uk := gn.Key(&u)
	if v := r.FormValue("v"); len(v) == 0 {
		q := datastore.NewQuery(gn.Key(&UserOpml{}).Kind()).Ancestor(uk).KeysOnly()
		keys, err := gn.GetAll(q, nil)
		if err != nil {
			serveError(w, err)
			return
		}
		times := make([]string, len(keys))
		for i, k := range keys {
			times[i] = strconv.FormatInt(k.IntID(), 10)
		}
		b, _ := json.Marshal(&times)
		w.Write(b)
	} else {
		a, _ := strconv.ParseInt(v, 10, 64)
		uo := UserOpml{Id: a, Parent: uk}
		c.Errorf("k: %v", uo)
		if err := gn.Get(&uo); err != nil {
			serveError(w, err)
			return
		}
		downloadOpml(w, uo.opml(), cu.Email)
	}
}
Пример #8
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	q := datastore.NewQuery("F").KeysOnly().Filter("n <=", time.Now())
	q = q.Limit(10 * 60 * 2) // 10/s queue, 2 min cron
	it := q.Run(appengine.Timeout(c, time.Minute))
	tc := make(chan *taskqueue.Task)
	done := make(chan bool)
	i := 0
	u := routeUrl("update-feed")
	go taskSender(c, "update-feed", tc, done)
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		}
		tc <- taskqueue.NewPOSTTask(u, url.Values{
			"feed": {k.StringID()},
		})
		i++
	}
	close(tc)
	<-done
	c.Infof("updating %d feeds", i)
}
Пример #9
0
func Donate(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	cu := user.Current(c)
	gn := goon.FromContext(c)
	u := User{Id: cu.ID}
	if err := gn.Get(&u); err != nil {
		serveError(w, err)
		return
	}
	amount, err := strconv.Atoi(r.FormValue("amount"))
	if err != nil || amount < 200 {
		serveError(w, fmt.Errorf("bad amount: %v", r.FormValue("amount")))
		return
	}
	resp, err := stripe(c, "POST", "charges", url.Values{
		"amount":      {r.FormValue("amount")},
		"description": {fmt.Sprintf("%v - %v", u.Id, u.Email)},
		"card":        {r.FormValue("stripeToken")},
		"currency":    {"usd"},
	}.Encode())
	if err != nil {
		serveError(w, err)
		return
	} else if resp.StatusCode != http.StatusOK {
		c.Errorf("%s", resp.Body)
		serveError(w, fmt.Errorf("Error"))
		return
	}
}
Пример #10
0
func UploadOpml(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	opml := Opml{}
	if err := json.Unmarshal([]byte(r.FormValue("opml")), &opml.Outline); err != nil {
		serveError(w, err)
		return
	}
	backupOPML(c)
	cu := user.Current(c)
	gn := goon.FromContext(c)
	u := User{Id: cu.ID}
	ud := UserData{Id: "data", Parent: gn.Key(&u)}
	if err := gn.Get(&ud); err != nil {
		serveError(w, err)
		c.Errorf("get err: %v", err)
		return
	}
	if b, err := json.Marshal(&opml); err != nil {
		saveError(c, fmt.Sprintf("%v", opml), err)
		serveError(w, err)
		c.Errorf("json err: %v", err)
		return
	} else {
		ud.Opml = b
	}
	gn.Put(&ud)
	backupOPML(c)
}
Пример #11
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
	q = q.Filter("n <=", time.Now()).Limit(3000)
	it := gn.Run(q)
	i := 0
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		}
		t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		})
		if _, err := taskqueue.Add(c, t, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
		i++
	}
	c.Infof("updating %d feeds", i)
	fmt.Fprintf(w, "updating %d feeds", i)
}
Пример #12
0
func SubscribeCallback(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	vars := mux.Vars(r)
	b, _ := base64.URLEncoding.DecodeString(vars["feed"])
	f := Feed{Url: string(b)}
	if err := gn.Get(&f); err != nil {
		http.Error(w, "", http.StatusNotFound)
		return
	}
	if r.Method == "GET" {
		if r.FormValue("hub.mode") != "subscribe" || r.FormValue("hub.topic") != f.Url {
			http.Error(w, "", http.StatusNotFound)
			return
		}
		w.Write([]byte(r.FormValue("hub.challenge")))
		i, _ := strconv.Atoi(r.FormValue("hub.lease_seconds"))
		f.Subscribed = time.Now().Add(time.Second * time.Duration(i))
		gn.Put(&f)
		c.Debugf("subscribed: %v - %v", f.Url, f.Subscribed)
		return
	} else {
		c.Infof("push: %v", f.Url)
		defer r.Body.Close()
		b, _ := ioutil.ReadAll(r.Body)
		nf, ss := ParseFeed(c, f.Url, b)
		err := updateFeed(c, f.Url, nf, ss)
		if err != nil {
			c.Errorf("push error: %v", err)
		}
	}
}
Пример #13
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
	q = q.Filter("n <=", time.Now())
	retry, _ := strconv.Atoi(r.FormValue("retry"))
	c.Errorf("retry: %v", retry)

	//* iterator
	it := gn.Run(q)
	i := 0
	done := false
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			done = true
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		}
		t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		})
		if _, err := taskqueue.Add(c, t, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
		i++
	}
	c.Infof("updating %d feeds", i)
	fmt.Fprintf(w, "updating %d feeds", i)
	if !done {
		time.Sleep(time.Second * time.Duration(i) / 50) // sleep about the time it'll take to process them
		t := taskqueue.NewPOSTTask("/tasks/update-feeds", url.Values{
			"retry": {strconv.Itoa(retry + 1)},
		})
		if _, err := taskqueue.Add(c, t, "update-feeds"); err != nil {
			c.Errorf("taskqueue update feeds error: %v", err.Error())
		}
		c.Errorf("ran update again")
		fmt.Fprintf(w, "\nran update again")
	}
	//*/

	/* get all
	q = q.Limit(1000)
	keys, _ := gn.GetAll(q, nil)
	for _, k := range keys {
		t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		})
		if _, err := taskqueue.Add(c, t, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
	}
	c.Infof("updating %d feeds", len(keys))
	fmt.Fprintf(w, "updating %d feeds", len(keys))
	//*/
}
Пример #14
0
func ImportOpml(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	cu := user.Current(c)
	gn := goon.FromContext(c)
	u := User{Id: cu.ID}
	if err := gn.Get(&u); err != nil {
		serveError(w, err)
		return
	}
	backupOPML(c)

	if file, _, err := r.FormFile("file"); err == nil {
		if fdata, err := ioutil.ReadAll(file); err == nil {
			buf := bytes.NewReader(fdata)
			// attempt to extract from google reader takeout zip
			if zb, zerr := zip.NewReader(buf, int64(len(fdata))); zerr == nil {
				for _, f := range zb.File {
					if strings.HasSuffix(f.FileHeader.Name, "Reader/subscriptions.xml") {
						if rc, rerr := f.Open(); rerr == nil {
							if fb, ferr := ioutil.ReadAll(rc); ferr == nil {
								fdata = fb
								break
							}
						}
					}
				}
			}

			// Preflight the OPML, so we can report any errors.
			d := xml.NewDecoder(bytes.NewReader(fdata))
			d.CharsetReader = charset.NewReader
			d.Strict = false
			opml := Opml{}
			if err := d.Decode(&opml); err != nil {
				serveError(w, err)
				c.Errorf("opml error: %v", err.Error())
				return
			}

			var b bytes.Buffer
			enc := gob.NewEncoder(&b)
			err := enc.Encode(&opml)
			if err != nil {
				serveError(w, err)
				return
			}
			bk, err := saveFile(c, b.Bytes())
			if err != nil {
				serveError(w, err)
				return
			}
			task := taskqueue.NewPOSTTask(routeUrl("import-opml-task"), url.Values{
				"key":  {string(bk)},
				"user": {cu.ID},
			})
			taskqueue.Add(c, task, "import-reader")
		}
	}
}
Пример #15
0
func SitemapFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	feed := vars["feed"]
	fk, err := datastore.DecodeKey(feed)
	if err != nil {
		serveError(w, err)
		return
	}
	bf := base64.URLEncoding.EncodeToString([]byte(fk.StringID()))
	q := datastore.NewQuery("S").KeysOnly().Ancestor(fk)
	q = q.Limit(Limit)
	cs := r.FormValue("c")
	if len(cs) > 0 {
		if cur, err := datastore.DecodeCursor(cs); err == nil {
			q = q.Start(cur)
		}
	}
	stories := make(map[string]string)
	it := q.Run(c)
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			break
		} else if err != nil {
			c.Errorf("next error: %v", err)
			break
		}
		stories[k.StringID()] = base64.URLEncoding.EncodeToString([]byte(k.StringID()))
	}
	cs = ""
	if len(stories) == Limit {
		if cur, err := it.Cursor(); err == nil {
			cs = cur.String()
		}
	}
	if err := templates.ExecuteTemplate(w, "sitemap-feed.html", struct {
		Feed, Feed64 string
		Stories      map[string]string
		Cursor       string
	}{
		Feed:    feed,
		Feed64:  bf,
		Stories: stories,
		Cursor:  cs,
	}); err != nil {
		c.Errorf("%v", err)
		serveError(w, err)
		return
	}
}
Пример #16
0
func SubscribeFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	f := Feed{Url: r.FormValue("feed")}
	if err := gn.Get(&f); err != nil {
		c.Errorf("%v: %v", err, f.Url)
		serveError(w, err)
		return
	} else if f.IsSubscribed() {
		return
	}
	u := url.Values{}
	u.Add("hub.callback", f.PubSubURL())
	u.Add("hub.mode", "subscribe")
	u.Add("hub.verify", "sync")
	fu, _ := url.Parse(f.Url)
	fu.Fragment = ""
	u.Add("hub.topic", fu.String())
	req, err := http.NewRequest("POST", PUBSUBHUBBUB_HUB, strings.NewReader(u.Encode()))
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	cl := urlfetch.Client(c)
	resp, err := cl.Do(req)
	if err != nil {
		c.Errorf("req error: %v", err)
	} else if resp.StatusCode != 204 {
		c.Errorf("resp: %v - %v", f.Url, resp.Status)
		c.Errorf("%s", resp.Body)
	} else {
		c.Infof("subscribed: %v", f.Url)
	}
}
Пример #17
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
	q = q.Filter("n <=", time.Now())
	keys, _ := gn.GetAll(q, nil)
	for _, k := range keys {
		t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		})
		if _, err := taskqueue.Add(c, t, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
	}
	c.Infof("updating %d feeds", len(keys))
}
Пример #18
0
func Charge(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	cu := user.Current(c)
	gn := goon.FromContext(c)
	u := User{Id: cu.ID}
	uc := &UserCharge{Id: 1, Parent: gn.Key(&u)}
	if err := gn.Get(&u); err != nil {
		serveError(w, err)
		return
	} else if u.Account != AFree {
		serveError(w, fmt.Errorf("You're already subscribed."))
		return
	}
	if err := gn.Get(uc); err == nil && len(uc.Customer) > 0 {
		serveError(w, fmt.Errorf("You're already subscribed."))
		return
	} else if err != datastore.ErrNoSuchEntity {
		serveError(w, err)
		return
	}
	resp, err := stripe(c, "POST", "customers", url.Values{
		"email":       {u.Email},
		"description": {u.Id},
		"card":        {r.FormValue("token")},
		"plan":        {r.FormValue("plan")},
	}.Encode())
	if err != nil {
		serveError(w, err)
		return
	} else if resp.StatusCode != http.StatusOK {
		var se StripeError
		defer resp.Body.Close()
		b, _ := ioutil.ReadAll(resp.Body)
		if err := json.Unmarshal(b, &se); err == nil {
			serveError(w, fmt.Errorf(se.Error.Message))
		} else {
			serveError(w, fmt.Errorf("Error"))
		}
		c.Errorf("status: %v, %s", resp.StatusCode, b)
		return
	}
	uc, err = setCharge(c, resp)
	if err != nil {
		serveError(w, err)
		return
	}
	b, _ := json.Marshal(&uc)
	w.Write(b)
}
Пример #19
0
func SubscribeCallback(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	furl := r.FormValue("feed")
	oldURL := false
	if len(furl) == 0 {
		vars := mux.Vars(r)
		furl = vars["feed"]
		oldURL = true
	}
	b, _ := base64.URLEncoding.DecodeString(furl)
	f := Feed{Url: string(b)}
	c.Infof("url: %v", f.Url)
	if err := gn.Get(&f); err != nil {
		http.Error(w, "", http.StatusNotFound)
		return
	}
	if r.Method == "GET" {
		if oldURL {
			c.Warningf("old url")
			http.Error(w, "", http.StatusNotFound)
			return
		}
		if f.NotViewed() || r.FormValue("hub.mode") != "subscribe" || r.FormValue("hub.topic") != f.Url {
			http.Error(w, "", http.StatusNotFound)
			return
		}
		w.Write([]byte(r.FormValue("hub.challenge")))
		i, _ := strconv.Atoi(r.FormValue("hub.lease_seconds"))
		f.Subscribed = time.Now().Add(time.Second * time.Duration(i))
		gn.Put(&f)
		c.Debugf("subscribed: %v - %v", f.Url, f.Subscribed)
		return
	} else if !f.NotViewed() {
		c.Infof("push: %v", f.Url)
		defer r.Body.Close()
		b, _ := ioutil.ReadAll(r.Body)
		nf, ss, err := ParseFeed(c, f.Url, b)
		if err != nil {
			c.Errorf("parse error: %v", err)
			return
		}
		if err := updateFeed(c, f.Url, nf, ss, false, true, false); err != nil {
			c.Errorf("push error: %v", err)
		}
	} else {
		c.Infof("not viewed")
	}
}
Пример #20
0
func DeleteAccount(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	if _, err := doUncheckout(c); err != nil {
		c.Errorf("uncheckout err: %v", err)
	}
	cu := user.Current(c)
	gn := goon.FromContext(c)
	u := User{Id: cu.ID}
	ud := UserData{Id: "data", Parent: gn.Key(&u)}
	if err := gn.Get(&u); err != nil {
		serveError(w, err)
		return
	}
	gn.Delete(gn.Key(&ud))
	gn.Delete(ud.Parent)
	http.Redirect(w, r, routeUrl("logout"), http.StatusFound)
}
Пример #21
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
	q = q.Filter("n <=", time.Now())

	q = q.Limit(1000)
	it := gn.Run(q)
	var keys []*datastore.Key
	var del []*datastore.Key
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		} else if len(k.StringID()) == 0 {
			del = append(del, k)
			continue
		}
		keys = append(keys, k)
	}
	tasks := make([]*taskqueue.Task, len(keys))
	for i, k := range keys {
		tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		})
	}
	var ts []*taskqueue.Task
	const taskLimit = 100
	for len(tasks) > 0 {
		if len(tasks) > taskLimit {
			ts = tasks[:taskLimit]
			tasks = tasks[taskLimit:]
		} else {
			ts = tasks
			tasks = tasks[0:0]
		}
		if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
	}
	c.Infof("updating %d feeds", len(keys))

	if len(del) > 0 {
		c.Errorf("attempt to delete %v feeds", len(del))
		if err := gn.DeleteMulti(del); err != nil {
			c.Errorf("delete error: %v", err.Error())
		}
		fmt.Fprintf(w, `<html><head><meta http-equiv="refresh" content="0"></head></html>`)
		fmt.Fprintf(w, "attempt to delete %v feeds", len(del))
		for _, k := range del {
			fmt.Fprintf(w, "\n<br>%v", k)
		}
	}
	fmt.Fprintf(w, "updating %d feeds", len(keys))
}
Пример #22
0
func SubscribeCallback(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	furl := r.FormValue("feed")
	b, _ := base64.URLEncoding.DecodeString(furl)
	f := Feed{Url: string(b)}
	c.Infof("url: %v", f.Url)
	if err := gn.Get(&f); err != nil {
		http.Error(w, "", http.StatusNotFound)
		return
	}
	if r.Method == "GET" {
		if f.NotViewed() || r.FormValue("hub.mode") != "subscribe" || r.FormValue("hub.topic") != f.Url {
			http.Error(w, "", http.StatusNotFound)
			return
		}
		w.Write([]byte(r.FormValue("hub.challenge")))
		i, _ := strconv.Atoi(r.FormValue("hub.lease_seconds"))
		f.Subscribed = time.Now().Add(time.Second * time.Duration(i))
		gn.PutMany(&f, &Log{
			Parent: gn.Key(&f),
			Id:     time.Now().UnixNano(),
			Text:   "SubscribeCallback - subscribed - " + f.Subscribed.String(),
		})
		c.Debugf("subscribed: %v - %v", f.Url, f.Subscribed)
		return
	} else if !f.NotViewed() {
		c.Infof("push: %v", f.Url)
		gn.Put(&Log{
			Parent: gn.Key(&f),
			Id:     time.Now().UnixNano(),
			Text:   "SubscribeCallback - push update",
		})
		defer r.Body.Close()
		b, _ := ioutil.ReadAll(r.Body)
		nf, ss, err := ParseFeed(c, f.Url, f.Url, b)
		if err != nil {
			c.Errorf("parse error: %v", err)
			return
		}
		if err := updateFeed(c, f.Url, nf, ss, false, true, false); err != nil {
			c.Errorf("push error: %v", err)
		}
	} else {
		c.Infof("not viewed")
	}
}
Пример #23
0
func UpdateFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	url := r.FormValue("feed")
	c.Debugf("update feed %s", url)
	f := Feed{Url: url}
	if err := gn.Get(&f); err == datastore.ErrNoSuchEntity {
		c.Errorf("no such entity")
		return
	} else if err != nil {
		c.Errorf("badurl7 error: %v", err.Error())
		return
	} else if time.Now().Before(f.NextUpdate) {
		c.Infof("feed %v already updated", url)
		return
	}
	if f.Url == "" {
		c.Criticalf("badurl7: %v", url)
		return
	}

	feedError := func() {
		f.Errors++
		v := f.Errors + 1
		const max = 24 * 7
		if v > max {
			v = max
		} else if f.Errors == 1 {
			v = 0
		}
		f.NextUpdate = time.Now().Add(time.Hour * time.Duration(v))
		gn.Put(&f)
		c.Warningf("error with %v (%v), bump next update to %v", url, f.Errors, f.NextUpdate)
	}

	c.Infof("fetching")
	if feed, stories := fetchFeed(c, f.Url, f.Url); feed != nil {
		if err := updateFeed(c, f.Url, feed, stories); err != nil {
			feedError()
		}
	} else {
		feedError()
	}
	c.Infof("done")
}
Пример #24
0
// Task used to subscribe a feed to push.
func SubscribeFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	start := time.Now()
	gn := goon.FromContext(c)
	f := Feed{Url: r.FormValue("feed")}
	fk := gn.Key(&f)
	s := ""
	defer func() {
		gn.Put(&Log{
			Parent: fk,
			Id:     time.Now().UnixNano(),
			Text:   "SubscribeFeed - start " + start.String() + " - f.sub " + f.Subscribed.String() + " - " + s,
		})
	}()
	if err := gn.Get(&f); err != nil {
		c.Errorf("%v: %v", err, f.Url)
		serveError(w, err)
		s += "err"
		return
	} else if f.IsSubscribed() {
		s += "is subscribed"
		return
	}
	u := url.Values{}
	u.Add("hub.callback", f.PubSubURL())
	u.Add("hub.mode", "subscribe")
	u.Add("hub.verify", "sync")
	fu, _ := url.Parse(f.Url)
	fu.Fragment = ""
	u.Add("hub.topic", fu.String())
	req, err := http.NewRequest("POST", f.Hub, strings.NewReader(u.Encode()))
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	cl := &http.Client{
		Transport: &urlfetch.Transport{
			Context:  c,
			Deadline: time.Minute,
		},
	}
	resp, err := cl.Do(req)
	if err != nil {
		c.Errorf("req error: %v", err)
	} else if resp.StatusCode != http.StatusNoContent {
		f.Subscribed = time.Now().Add(time.Hour * 48)
		gn.Put(&f)
		if resp.StatusCode != http.StatusConflict {
			c.Errorf("resp: %v - %v", f.Url, resp.Status)
			c.Errorf("%s", resp.Body)
		}
		s += "resp err"
	} else {
		c.Infof("subscribed: %v", f.Url)
		s += "success"
	}
}
Пример #25
0
func ClearRead(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	if !isDevServer {
		return
	}

	cu := user.Current(c)
	gn := goon.FromContext(c)
	u := &User{Id: cu.ID}
	ud := &UserData{Id: "data", Parent: gn.Key(u)}
	if err := gn.Get(u); err != nil {
		c.Errorf("err: %v", err.Error())
		return
	}
	gn.Get(ud)
	u.Read = time.Time{}
	ud.Read = nil
	gn.PutMulti([]interface{}{u, ud})
	http.Redirect(w, r, "/", http.StatusFound)
}
Пример #26
0
func SetStar(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	feed := r.FormValue("feed")
	story := r.FormValue("story")
	if len(feed) == 0 || len(story) == 0 {
		return
	}
	del := r.FormValue("del") != ""
	us := starKey(c, feed, story)
	gn := goon.FromContext(c)
	if del {
		gn.Delete(gn.Key(us))
	} else {
		us.Created = time.Now()
		_, err := gn.Put(us)
		if err != nil {
			c.Errorf("star put err: %v", err)
			serveError(w, err)
		}
	}
}
Пример #27
0
func AddSubscription(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	cu := user.Current(c)
	url := r.FormValue("url")
	o := &OpmlOutline{
		Outline: []*OpmlOutline{
			&OpmlOutline{XmlUrl: url},
		},
	}
	if err := addFeed(c, cu.ID, o); err != nil {
		c.Errorf("add sub error (%s): %s", url, err.Error())
		serveError(w, err)
		return
	}

	gn := goon.FromContext(c)
	ud := UserData{Id: "data", Parent: gn.Key(&User{Id: cu.ID})}
	gn.Get(&ud)
	mergeUserOpml(&ud, o)
	gn.Put(&ud)
}
Пример #28
0
func backupOPML(c mpg.Context) {
	cu := user.Current(c)
	gn := goon.FromContext(c)
	u := User{Id: cu.ID}
	ud := UserData{Id: "data", Parent: gn.Key(&u)}
	if err := gn.Get(&ud); err != nil {
		return
	}
	uo := UserOpml{Id: time.Now().UnixNano(), Parent: gn.Key(&u)}
	buf := &bytes.Buffer{}
	if gz, err := gzip.NewWriterLevel(buf, gzip.BestCompression); err == nil {
		gz.Write([]byte(ud.Opml))
		gz.Close()
		uo.Compressed = buf.Bytes()
	} else {
		c.Errorf("gz err: %v", err)
		uo.Opml = ud.Opml
	}
	gn.Put(&uo)
}
Пример #29
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
	q = q.Filter("n <=", time.Now())

	q = q.Limit(2500)
	it := gn.Run(q)
	var keys []*datastore.Key
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		}
		keys = append(keys, k)
	}
	tasks := make([]*taskqueue.Task, len(keys))
	for i, k := range keys {
		tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		})
	}
	var ts []*taskqueue.Task
	const taskLimit = 100
	for len(tasks) > 0 {
		if len(tasks) > taskLimit {
			ts = tasks[:taskLimit]
			tasks = tasks[taskLimit:]
		} else {
			ts = tasks
			tasks = tasks[0:0]
		}
		if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
	}
	c.Infof("updating %d feeds", len(keys))
	fmt.Fprintf(w, "updating %d feeds", len(keys))
}
Пример #30
0
func UpdateFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	url := r.FormValue("feed")
	c.Debugf("update feed %s", url)
	last := len(r.FormValue("last")) > 0
	f := Feed{Url: url}
	if err := gn.Get(&f); err == datastore.ErrNoSuchEntity {
		c.Errorf("no such entity")
		return
	} else if err != nil {
		return
	} else if last {
		// noop
	} else if time.Now().Before(f.NextUpdate) {
		c.Infof("feed %v already updated: %v", url, f.NextUpdate)
		return
	}
	f.Subscribe(c)

	feedError := func(err error) {
		f.Errors++
		v := f.Errors + 1
		const max = 24 * 7
		if v > max {
			v = max
		} else if f.Errors == 1 {
			v = 0
		}
		f.NextUpdate = time.Now().Add(time.Hour * time.Duration(v))
		gn.Put(&f)
		c.Warningf("error with %v (%v), bump next update to %v, %v", url, f.Errors, f.NextUpdate, err)
	}

	if feed, stories, err := fetchFeed(c, f.Url, f.Url); err == nil {
		if err := updateFeed(c, f.Url, feed, stories, false, false, last); err != nil {
			feedError(err)
		}
	} else {
		feedError(err)
	}
}