Ejemplo n.º 1
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	q := datastore.NewQuery("F").KeysOnly().Filter("n <=", time.Now())
	q = q.Limit(10 * 60 * 2) // 10/s queue, 2 min cron
	it := q.Run(appengine.Timeout(c, time.Minute))
	tc := make(chan *taskqueue.Task)
	done := make(chan bool)
	i := 0
	u := routeUrl("update-feed")
	go taskSender(c, "update-feed", tc, done)
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		}
		tc <- taskqueue.NewPOSTTask(u, url.Values{
			"feed": {k.StringID()},
		})
		i++
	}
	close(tc)
	<-done
	c.Infof("updating %d feeds", i)
}
Ejemplo n.º 2
0
func UpdateFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	url := r.FormValue("feed")
	c.Debugf("update feed %s", url)
	f := Feed{Url: url}
	if err := gn.Get(&f); err == datastore.ErrNoSuchEntity {
		return
	} else if time.Now().Before(f.NextUpdate) {
		c.Infof("feed %v already updated", url)
		return
	}
	if feed, stories := fetchFeed(c, url, url); feed != nil {
		updateFeed(c, url, feed, stories)
	} else {
		f.Errors++
		v := f.Errors + 1
		const max = 24 * 7
		if v > max {
			v = max
		} else if f.Errors == 1 {
			v = 0
		}
		f.NextUpdate = time.Now().Add(time.Hour * time.Duration(v))
		gn.Put(&f)
		c.Warningf("error with %v (%v), bump next update to %v", url, f.Errors, f.NextUpdate)
	}
}
Ejemplo n.º 3
0
func SubscribeCallback(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	vars := mux.Vars(r)
	b, _ := base64.URLEncoding.DecodeString(vars["feed"])
	f := Feed{Url: string(b)}
	if err := gn.Get(&f); err != nil {
		http.Error(w, "", http.StatusNotFound)
		return
	}
	if r.Method == "GET" {
		if r.FormValue("hub.mode") != "subscribe" || r.FormValue("hub.topic") != f.Url {
			http.Error(w, "", http.StatusNotFound)
			return
		}
		w.Write([]byte(r.FormValue("hub.challenge")))
		i, _ := strconv.Atoi(r.FormValue("hub.lease_seconds"))
		f.Subscribed = time.Now().Add(time.Second * time.Duration(i))
		gn.Put(&f)
		c.Debugf("subscribed: %v - %v", f.Url, f.Subscribed)
		return
	} else {
		c.Infof("push: %v", f.Url)
		defer r.Body.Close()
		b, _ := ioutil.ReadAll(r.Body)
		nf, ss := ParseFeed(c, f.Url, b)
		err := updateFeed(c, f.Url, nf, ss)
		if err != nil {
			c.Errorf("push error: %v", err)
		}
	}
}
Ejemplo n.º 4
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
	q = q.Filter("n <=", time.Now()).Limit(3000)
	it := gn.Run(q)
	i := 0
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		}
		t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		})
		if _, err := taskqueue.Add(c, t, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
		i++
	}
	c.Infof("updating %d feeds", i)
	fmt.Fprintf(w, "updating %d feeds", i)
}
Ejemplo n.º 5
0
func DeleteOldFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	g := goon.FromContext(c)
	oldDate := time.Now().Add(-time.Hour * 24 * 90)
	feed := Feed{Url: r.FormValue("f")}
	if err := g.Get(&feed); err != nil {
		c.Criticalf("err: %v", err)
		return
	}
	if feed.LastViewed.After(oldDate) {
		return
	}
	q := datastore.NewQuery(g.Kind(&Story{})).Ancestor(g.Key(&feed)).KeysOnly()
	keys, err := q.GetAll(c, nil)
	if err != nil {
		c.Criticalf("err: %v", err)
		return
	}
	q = datastore.NewQuery(g.Kind(&StoryContent{})).Ancestor(g.Key(&feed)).KeysOnly()
	sckeys, err := q.GetAll(c, nil)
	if err != nil {
		c.Criticalf("err: %v", err)
		return
	}
	keys = append(keys, sckeys...)
	c.Infof("delete: %v - %v", feed.Url, len(keys))
	err = g.DeleteMulti(keys)
	if err != nil {
		c.Criticalf("err: %v", err)
	}
}
Ejemplo n.º 6
0
func SubscribeFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	f := Feed{Url: r.FormValue("feed")}
	if err := gn.Get(&f); err != nil {
		c.Errorf("%v: %v", err, f.Url)
		serveError(w, err)
		return
	} else if f.IsSubscribed() {
		return
	}
	u := url.Values{}
	u.Add("hub.callback", f.PubSubURL())
	u.Add("hub.mode", "subscribe")
	u.Add("hub.verify", "sync")
	fu, _ := url.Parse(f.Url)
	fu.Fragment = ""
	u.Add("hub.topic", fu.String())
	req, err := http.NewRequest("POST", PUBSUBHUBBUB_HUB, strings.NewReader(u.Encode()))
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	cl := urlfetch.Client(c)
	resp, err := cl.Do(req)
	if err != nil {
		c.Errorf("req error: %v", err)
	} else if resp.StatusCode != 204 {
		c.Errorf("resp: %v - %v", f.Url, resp.Status)
		c.Errorf("%s", resp.Body)
	} else {
		c.Infof("subscribed: %v", f.Url)
	}
}
Ejemplo n.º 7
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
	q = q.Filter("n <=", time.Now())
	retry, _ := strconv.Atoi(r.FormValue("retry"))
	c.Errorf("retry: %v", retry)

	//* iterator
	it := gn.Run(q)
	i := 0
	done := false
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			done = true
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		}
		t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		})
		if _, err := taskqueue.Add(c, t, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
		i++
	}
	c.Infof("updating %d feeds", i)
	fmt.Fprintf(w, "updating %d feeds", i)
	if !done {
		time.Sleep(time.Second * time.Duration(i) / 50) // sleep about the time it'll take to process them
		t := taskqueue.NewPOSTTask("/tasks/update-feeds", url.Values{
			"retry": {strconv.Itoa(retry + 1)},
		})
		if _, err := taskqueue.Add(c, t, "update-feeds"); err != nil {
			c.Errorf("taskqueue update feeds error: %v", err.Error())
		}
		c.Errorf("ran update again")
		fmt.Fprintf(w, "\nran update again")
	}
	//*/

	/* get all
	q = q.Limit(1000)
	keys, _ := gn.GetAll(q, nil)
	for _, k := range keys {
		t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		})
		if _, err := taskqueue.Add(c, t, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
	}
	c.Infof("updating %d feeds", len(keys))
	fmt.Fprintf(w, "updating %d feeds", len(keys))
	//*/
}
Ejemplo n.º 8
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
	q = q.Filter("n <=", time.Now())

	q = q.Limit(1000)
	it := gn.Run(q)
	var keys []*datastore.Key
	var del []*datastore.Key
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		} else if len(k.StringID()) == 0 {
			del = append(del, k)
			continue
		}
		keys = append(keys, k)
	}
	tasks := make([]*taskqueue.Task, len(keys))
	for i, k := range keys {
		tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		})
	}
	var ts []*taskqueue.Task
	const taskLimit = 100
	for len(tasks) > 0 {
		if len(tasks) > taskLimit {
			ts = tasks[:taskLimit]
			tasks = tasks[taskLimit:]
		} else {
			ts = tasks
			tasks = tasks[0:0]
		}
		if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
	}
	c.Infof("updating %d feeds", len(keys))

	if len(del) > 0 {
		c.Errorf("attempt to delete %v feeds", len(del))
		if err := gn.DeleteMulti(del); err != nil {
			c.Errorf("delete error: %v", err.Error())
		}
		fmt.Fprintf(w, `<html><head><meta http-equiv="refresh" content="0"></head></html>`)
		fmt.Fprintf(w, "attempt to delete %v feeds", len(del))
		for _, k := range del {
			fmt.Fprintf(w, "\n<br>%v", k)
		}
	}
	fmt.Fprintf(w, "updating %d feeds", len(keys))
}
Ejemplo n.º 9
0
func ClearFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	if !isDevServer {
		return
	}

	cu := user.Current(c)
	gn := goon.FromContext(c)
	done := make(chan bool)
	go func() {
		u := &User{Id: cu.ID}
		defer func() { done <- true }()
		ud := &UserData{Id: "data", Parent: gn.Key(u)}
		if err := gn.Get(u); err != nil {
			c.Errorf("user del err: %v", err.Error())
			return
		}
		gn.Get(ud)
		u.Read = time.Time{}
		ud.Read = nil
		ud.Opml = nil
		gn.PutMany(u, ud)
		c.Infof("%v cleared", u.Email)
	}()
	del := func(kind string) {
		defer func() { done <- true }()
		q := datastore.NewQuery(kind).KeysOnly()
		keys, err := gn.GetAll(q, nil)
		if err != nil {
			c.Errorf("err: %v", err.Error())
			return
		}
		if err := gn.DeleteMulti(keys); err != nil {
			c.Errorf("err: %v", err.Error())
			return
		}
		c.Infof("%v deleted", kind)
	}
	types := []interface{}{
		&Feed{},
		&Story{},
		&StoryContent{},
		&DateFormat{},
		&Log{},
		&UserOpml{},
	}
	for _, i := range types {
		k := gn.Key(i).Kind()
		go del(k)
	}

	for i := 0; i < len(types); i++ {
		<-done
	}

	http.Redirect(w, r, routeUrl("main"), http.StatusFound)
}
Ejemplo n.º 10
0
// Task used to subscribe a feed to push.
func SubscribeFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	start := time.Now()
	gn := goon.FromContext(c)
	f := Feed{Url: r.FormValue("feed")}
	fk := gn.Key(&f)
	s := ""
	defer func() {
		gn.Put(&Log{
			Parent: fk,
			Id:     time.Now().UnixNano(),
			Text:   "SubscribeFeed - start " + start.String() + " - f.sub " + f.Subscribed.String() + " - " + s,
		})
	}()
	if err := gn.Get(&f); err != nil {
		c.Errorf("%v: %v", err, f.Url)
		serveError(w, err)
		s += "err"
		return
	} else if f.IsSubscribed() {
		s += "is subscribed"
		return
	}
	u := url.Values{}
	u.Add("hub.callback", f.PubSubURL())
	u.Add("hub.mode", "subscribe")
	u.Add("hub.verify", "sync")
	fu, _ := url.Parse(f.Url)
	fu.Fragment = ""
	u.Add("hub.topic", fu.String())
	req, err := http.NewRequest("POST", f.Hub, strings.NewReader(u.Encode()))
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	cl := &http.Client{
		Transport: &urlfetch.Transport{
			Context:  c,
			Deadline: time.Minute,
		},
	}
	resp, err := cl.Do(req)
	if err != nil {
		c.Errorf("req error: %v", err)
	} else if resp.StatusCode != http.StatusNoContent {
		f.Subscribed = time.Now().Add(time.Hour * 48)
		gn.Put(&f)
		if resp.StatusCode != http.StatusConflict {
			c.Errorf("resp: %v - %v", f.Url, resp.Status)
			c.Errorf("%s", resp.Body)
		}
		s += "resp err"
	} else {
		c.Infof("subscribed: %v", f.Url)
		s += "success"
	}
}
Ejemplo n.º 11
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
	q = q.Filter("n <=", time.Now())
	keys, _ := gn.GetAll(q, nil)
	for _, k := range keys {
		t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		})
		if _, err := taskqueue.Add(c, t, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
	}
	c.Infof("updating %d feeds", len(keys))
}
Ejemplo n.º 12
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	q := datastore.NewQuery("F").KeysOnly()
	q = q.Filter("n <=", time.Now())
	q = q.Limit(3000)
	var keys []*datastore.Key
	it := q.Run(c)
	retry := 0
	for {
		k, err := it.Next(nil)
		if err != nil {
			c.Errorf("next error: %v, retry: %v", err.Error(), retry)
			if retry == 5 {
				break
			}
			retry++
			it = q.Run(c)
		} else {
			keys = append(keys, k)
		}
	}
	if len(keys) == 0 {
		c.Errorf("giving up")
		return
	}

	tasks := make([]*taskqueue.Task, len(keys))
	for i, k := range keys {
		c.Infof("task: %v, %v", i, k)
		tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		})
	}
	var ts []*taskqueue.Task
	const taskLimit = 100
	for len(tasks) > 0 {
		if len(tasks) > taskLimit {
			ts = tasks[:taskLimit]
			tasks = tasks[taskLimit:]
		} else {
			ts = tasks
			tasks = tasks[0:0]
		}
		if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
	}
	c.Infof("updating %d feeds", len(keys))
}
Ejemplo n.º 13
0
func addFeed(c mpg.Context, userid string, outline *OpmlOutline) error {
	gn := goon.FromContext(c)
	o := outline.Outline[0]
	c.Infof("adding feed %v to user %s", o.XmlUrl, userid)
	fu, ferr := url.Parse(o.XmlUrl)
	if ferr != nil {
		return ferr
	}
	fu.Fragment = ""
	o.XmlUrl = fu.String()

	f := Feed{Url: o.XmlUrl}
	if err := gn.Get(&f); err == datastore.ErrNoSuchEntity {
		if feed, stories, err := fetchFeed(c, o.XmlUrl, o.XmlUrl); err != nil {
			return fmt.Errorf("could not add feed %s: %v", o.XmlUrl, err)
		} else {
			f = *feed
			f.Updated = time.Time{}
			f.Checked = f.Updated
			f.NextUpdate = f.Updated
			f.LastViewed = time.Now()
			gn.Put(&f)
			for _, s := range stories {
				s.Created = s.Published
			}
			if err := updateFeed(c, f.Url, feed, stories, false, false, false); err != nil {
				return err
			}

			o.XmlUrl = feed.Url
			o.HtmlUrl = feed.Link
			if o.Title == "" {
				o.Title = feed.Title
			}
		}
	} else if err != nil {
		return err
	} else {
		o.HtmlUrl = f.Link
		if o.Title == "" {
			o.Title = f.Title
		}
	}
	o.Text = ""

	return nil
}
Ejemplo n.º 14
0
func AdminUser(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	q := datastore.NewQuery(gn.Kind(&User{})).Limit(1)
	q = q.Filter("e =", r.FormValue("u"))
	it := gn.Run(q)
	var u User
	ud := UserData{Id: "data"}
	var h []Log
	k, err := it.Next(&u)
	if err != nil {
		serveError(w, err)
		return
	}
	ud.Parent = gn.Key(&u)
	gn.Get(&ud)
	until := r.FormValue("until")
	if d, err := time.Parse("2006-01-02", until); err == nil {
		u.Until = d
		gn.Put(&u)
	}
	if o := []byte(r.FormValue("opml")); len(o) > 0 {
		opml := Opml{}
		if err := json.Unmarshal(o, &opml); err != nil {
			serveError(w, err)
			return
		}
		ud.Opml = o
		if _, err := gn.Put(&ud); err != nil {
			serveError(w, err)
			return
		}
		c.Infof("opml updated")
	}
	q = datastore.NewQuery(gn.Kind(&Log{})).Ancestor(k)
	_, err = gn.GetAll(q, &h)
	if err := templates.ExecuteTemplate(w, "admin-user.html", struct {
		User User
		Data UserData
		Log  []Log
	}{
		u,
		ud,
		h,
	}); err != nil {
		serveError(w, err)
	}
}
Ejemplo n.º 15
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	q := datastore.NewQuery("F").KeysOnly().Filter("n <=", time.Now())
	q = q.Limit(10 * 60 * 20) // 10/s queue, 20 min cron
	var keys []*datastore.Key
	it := q.Run(appengine.Timeout(c, time.Second*60))
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		}
		keys = append(keys, k)
	}

	if len(keys) == 0 {
		c.Errorf("no results")
		return
	}
	c.Infof("updating %d feeds", len(keys))

	var tasks []*taskqueue.Task
	for _, k := range keys {
		tasks = append(tasks, taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		}))
	}
	var ts []*taskqueue.Task
	const taskLimit = 100
	for len(tasks) > 0 {
		if len(tasks) > taskLimit {
			ts = tasks[:taskLimit]
			tasks = tasks[taskLimit:]
		} else {
			ts = tasks
			tasks = tasks[0:0]
		}
		if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
	}
}
Ejemplo n.º 16
0
func taskSender(c mpg.Context, queue string, tc chan *taskqueue.Task, done chan bool) {
	const taskLimit = 100
	tasks := make([]*taskqueue.Task, 0, taskLimit)
	send := func() {
		taskqueue.AddMulti(c, tasks, queue)
		c.Infof("added %v tasks", len(tasks))
		tasks = tasks[0:0]
	}
	for t := range tc {
		tasks = append(tasks, t)
		if len(tasks) == taskLimit {
			send()
		}
	}
	if len(tasks) > 0 {
		send()
	}
	done <- true
}
Ejemplo n.º 17
0
func addFeed(c mpg.Context, userid string, outline *OpmlOutline) error {
	gn := goon.FromContext(c)
	o := outline.Outline[0]
	c.Infof("adding feed %v to user %s", o.XmlUrl, userid)

	f := Feed{Url: o.XmlUrl}
	if err := gn.Get(&f); err == datastore.ErrNoSuchEntity {
		if feed, stories := fetchFeed(c, o.XmlUrl, o.XmlUrl); feed == nil {
			return errors.New(fmt.Sprintf("could not add feed %s", o.XmlUrl))
		} else {
			f = *feed
			f.Updated = time.Time{}
			f.Checked = f.Updated
			f.NextUpdate = f.Updated
			if strings.TrimSpace(f.Url) == "" {
				c.Criticalf("badurl4: %v, %v", o.XmlUrl, o)
				return errors.New("badurl4")
			}
			gn.PutComplete(&f)
			if err := updateFeed(c, f.Url, feed, stories); err != nil {
				return err
			}

			o.XmlUrl = feed.Url
			o.HtmlUrl = feed.Link
			if o.Title == "" {
				o.Title = feed.Title
			}
		}
	} else if err != nil {
		return err
	} else {
		o.HtmlUrl = f.Link
		if o.Title == "" {
			o.Title = f.Title
		}
	}
	o.Text = ""

	return nil
}
Ejemplo n.º 18
0
func UpdateFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	url := r.FormValue("feed")
	c.Debugf("update feed %s", url)
	last := len(r.FormValue("last")) > 0
	f := Feed{Url: url}
	if err := gn.Get(&f); err == datastore.ErrNoSuchEntity {
		c.Errorf("no such entity")
		return
	} else if err != nil {
		return
	} else if last {
		// noop
	} else if time.Now().Before(f.NextUpdate) {
		c.Infof("feed %v already updated: %v", url, f.NextUpdate)
		return
	}
	f.Subscribe(c)

	feedError := func(err error) {
		f.Errors++
		v := f.Errors + 1
		const max = 24 * 7
		if v > max {
			v = max
		} else if f.Errors == 1 {
			v = 0
		}
		f.NextUpdate = time.Now().Add(time.Hour * time.Duration(v))
		gn.Put(&f)
		c.Warningf("error with %v (%v), bump next update to %v, %v", url, f.Errors, f.NextUpdate, err)
	}

	if feed, stories, err := fetchFeed(c, f.Url, f.Url); err == nil {
		if err := updateFeed(c, f.Url, feed, stories, false, false, last); err != nil {
			feedError(err)
		}
	} else {
		feedError(err)
	}
}
Ejemplo n.º 19
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
	q = q.Filter("n <=", time.Now())

	q = q.Limit(2500)
	it := gn.Run(q)
	var keys []*datastore.Key
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		}
		keys = append(keys, k)
	}
	tasks := make([]*taskqueue.Task, len(keys))
	for i, k := range keys {
		tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		})
	}
	var ts []*taskqueue.Task
	const taskLimit = 100
	for len(tasks) > 0 {
		if len(tasks) > taskLimit {
			ts = tasks[:taskLimit]
			tasks = tasks[taskLimit:]
		} else {
			ts = tasks
			tasks = tasks[0:0]
		}
		if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
	}
	c.Infof("updating %d feeds", len(keys))
	fmt.Fprintf(w, "updating %d feeds", len(keys))
}
Ejemplo n.º 20
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	q := datastore.NewQuery("F").KeysOnly()
	q = q.Filter("n <=", time.Now())
	q = q.Limit(3000)
	var keys []*datastore.Key
	for i := 0; i < 5; i++ {
		if _keys, err := q.GetAll(c, nil); err != nil {
			c.Errorf("get all error: %v, retry %v", err.Error(), i)
		} else {
			c.Errorf("got %v keys", len(_keys))
			keys = _keys
			break
		}
	}
	if len(keys) == 0 {
		c.Errorf("giving up")
		return
	}

	tasks := make([]*taskqueue.Task, len(keys))
	for i, k := range keys {
		tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		})
	}
	var ts []*taskqueue.Task
	const taskLimit = 100
	for len(tasks) > 0 {
		if len(tasks) > taskLimit {
			ts = tasks[:taskLimit]
			tasks = tasks[taskLimit:]
		} else {
			ts = tasks
			tasks = tasks[0:0]
		}
		if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
	}
	c.Infof("updating %d feeds", len(keys))
}
Ejemplo n.º 21
0
func Account(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	cu := user.Current(c)
	gn := goon.FromContext(c)
	u := User{Id: cu.ID}
	uc := &UserCharge{Id: 1, Parent: gn.Key(&u)}
	if err := gn.Get(uc); err == nil {
		if uc.Next.Before(time.Now()) {
			if resp, err := stripe(c, "GET", "customers/"+uc.Customer, ""); err == nil {
				if nuc, err := setCharge(c, resp); err == nil {
					uc = nuc
					c.Infof("updated user charge %v", cu.ID)
				}
			}
		}
		b, _ := json.Marshal(&uc)
		w.Write(b)
	} else if err != datastore.ErrNoSuchEntity {
		serveError(w, err)
		return
	}
}
Ejemplo n.º 22
0
func CFix(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	url := r.FormValue("feed")
	c.Infof("fix feed %s", url)
	f := Feed{Url: url}
	if err := gn.Get(&f); err != nil {
		c.Criticalf("cfix err: %v", err)
		serveError(w, err)
		return
	}

	q := datastore.NewQuery("S").Ancestor(gn.Key(&f))
	var ss []*Story
	keys, err := q.GetAll(c, &ss)
	if err != nil {
		c.Errorf("getall err: %v", err)
		serveError(w, err)
		return
	}
	c.Infof("trying to fix %v stories", len(ss))
	const putLimit = 500
	for i := 0; i <= len(keys)/putLimit; i++ {
		lo := i * putLimit
		hi := (i + 1) * putLimit
		if hi > len(keys) {
			hi = len(keys)
		}
		c.Infof("%v - %v", lo, hi)
		if _, err := datastore.PutMulti(c, keys[lo:hi], ss[lo:hi]); err != nil {
			c.Errorf("err: %v, %v, %v", lo, hi, err)
		}
	}
}
Ejemplo n.º 23
0
func BackendStart(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	const sz = 100
	ic := 0
	gn := goon.FromContext(c)
	fk := gn.Key(&Feed{Url: "a"})
	q := datastore.NewQuery("F").Filter("__key__ <", fk).Order("__key__").KeysOnly().Limit(1)
	keys, _ := q.GetAll(c, nil)
	if len(keys) == 0 {
		return
	}
	c.Errorf("start: %v", keys[0])
	startid := keys[0].IntID() / sz

	var f func(appengine.Context)
	f = func(c appengine.Context) {
		c.Errorf("new request: %d", ic)
		t1 := time.Now()
		wg := sync.WaitGroup{}
		wg.Add(sz)
		var j int64
		for j = 0; j < sz; j++ {
			go func(j int64) {
				k := datastore.NewKey(c, "F", "", startid*sz+j, nil)
				c.Infof("del: %v", k)
				if err := datastore.Delete(c, k); err != nil {
					c.Errorf("delete err: %v", err.Error())
				}
				wg.Done()
			}(j)
		}
		wg.Wait()
		t2 := time.Now()
		c.Infof("%v, %v, %v", t1, t2, t2.Sub(t1))
		ic++
		startid++
		runtime.RunInBackground(c, f)
	}
	runtime.RunInBackground(c, f)
}
Ejemplo n.º 24
0
func Search(c mp.Context, w http.ResponseWriter, r *http.Request) error {
	query := r.URL.Query()
	q := query.Get("q")
	if q == "" {
		return core.Render(w, "songs.search.tmpl.html", &SongSearchResult{Includes: core.NewIncludes(c)})
	}

	searchQuery := m.Wordify(q)
	docList, docIDs, err := data.SearchSongs(c, searchQuery)
	if err != nil {
		return err
	}

	if len(docList) == 0 {
		c.Infof("AND query [%s] return nothing", searchQuery)
		orQuery := strings.Join(strings.Fields(searchQuery), " OR ")
		c.Infof("Searching with OR query [%s]", orQuery)
		docList, docIDs, err = data.SearchSongs(c, orQuery)
		if err != nil {
			return err
		}
	}

	items := make([]*SongSearchResultItem, 0)
	for i, _ := range docList {
		items = append(items, createSongSearchResultItem(q, docIDs[i], docList[i]))
	}

	result := &SongSearchResult{
		Includes: core.NewIncludes(c),
		Query:    q,
		Items:    items,
	}

	return core.Render(w, "songs.search.tmpl.html", result)
}
Ejemplo n.º 25
0
func SubscribeCallback(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	furl := r.FormValue("feed")
	oldURL := false
	if len(furl) == 0 {
		vars := mux.Vars(r)
		furl = vars["feed"]
		oldURL = true
	}
	b, _ := base64.URLEncoding.DecodeString(furl)
	f := Feed{Url: string(b)}
	c.Infof("url: %v", f.Url)
	if err := gn.Get(&f); err != nil {
		http.Error(w, "", http.StatusNotFound)
		return
	}
	if r.Method == "GET" {
		if oldURL {
			c.Warningf("old url")
			http.Error(w, "", http.StatusNotFound)
			return
		}
		if f.NotViewed() || r.FormValue("hub.mode") != "subscribe" || r.FormValue("hub.topic") != f.Url {
			http.Error(w, "", http.StatusNotFound)
			return
		}
		w.Write([]byte(r.FormValue("hub.challenge")))
		i, _ := strconv.Atoi(r.FormValue("hub.lease_seconds"))
		f.Subscribed = time.Now().Add(time.Second * time.Duration(i))
		gn.Put(&f)
		c.Debugf("subscribed: %v - %v", f.Url, f.Subscribed)
		return
	} else if !f.NotViewed() {
		c.Infof("push: %v", f.Url)
		defer r.Body.Close()
		b, _ := ioutil.ReadAll(r.Body)
		nf, ss, err := ParseFeed(c, f.Url, b)
		if err != nil {
			c.Errorf("parse error: %v", err)
			return
		}
		if err := updateFeed(c, f.Url, nf, ss, false, true, false); err != nil {
			c.Errorf("push error: %v", err)
		}
	} else {
		c.Infof("not viewed")
	}
}
Ejemplo n.º 26
0
func SubscribeCallback(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	furl := r.FormValue("feed")
	b, _ := base64.URLEncoding.DecodeString(furl)
	f := Feed{Url: string(b)}
	c.Infof("url: %v", f.Url)
	if err := gn.Get(&f); err != nil {
		http.Error(w, "", http.StatusNotFound)
		return
	}
	if r.Method == "GET" {
		if f.NotViewed() || r.FormValue("hub.mode") != "subscribe" || r.FormValue("hub.topic") != f.Url {
			http.Error(w, "", http.StatusNotFound)
			return
		}
		w.Write([]byte(r.FormValue("hub.challenge")))
		i, _ := strconv.Atoi(r.FormValue("hub.lease_seconds"))
		f.Subscribed = time.Now().Add(time.Second * time.Duration(i))
		gn.PutMany(&f, &Log{
			Parent: gn.Key(&f),
			Id:     time.Now().UnixNano(),
			Text:   "SubscribeCallback - subscribed - " + f.Subscribed.String(),
		})
		c.Debugf("subscribed: %v - %v", f.Url, f.Subscribed)
		return
	} else if !f.NotViewed() {
		c.Infof("push: %v", f.Url)
		gn.Put(&Log{
			Parent: gn.Key(&f),
			Id:     time.Now().UnixNano(),
			Text:   "SubscribeCallback - push update",
		})
		defer r.Body.Close()
		b, _ := ioutil.ReadAll(r.Body)
		nf, ss, err := ParseFeed(c, f.Url, f.Url, b)
		if err != nil {
			c.Errorf("parse error: %v", err)
			return
		}
		if err := updateFeed(c, f.Url, nf, ss, false, true, false); err != nil {
			c.Errorf("push error: %v", err)
		}
	} else {
		c.Infof("not viewed")
	}
}
Ejemplo n.º 27
0
func updateFeed(c mpg.Context, url string, feed *Feed, stories []*Story) error {
	gn := goon.FromContext(c)
	f := Feed{Url: url}
	if err := gn.Get(&f); err != nil {
		return errors.New(fmt.Sprintf("feed not found: %s", url))
	}

	// Compare the feed's listed update to the story's update.
	// Note: these may not be accurate, hence, only compare them to each other,
	// since they should have the same relative error.
	storyDate := f.Updated

	hasUpdated := !feed.Updated.IsZero()
	isFeedUpdated := f.Updated == feed.Updated
	if !hasUpdated {
		feed.Updated = f.Updated
	}
	feed.Date = f.Date
	f = *feed

	if hasUpdated && isFeedUpdated {
		c.Infof("feed %s already updated to %v, putting", url, feed.Updated)
		f.Updated = time.Now()
		if strings.TrimSpace(f.Url) == "" {
			c.Criticalf("badurl5: %v, %v", url, f)
			return errors.New("badurl5")
		}
		gn.PutComplete(&f)
		return nil
	}

	c.Debugf("hasUpdate: %v, isFeedUpdated: %v, storyDate: %v", hasUpdated, isFeedUpdated, storyDate)

	var newStories []*Story
	for _, s := range stories {
		if s.Updated.IsZero() || !s.Updated.Before(storyDate) {
			newStories = append(newStories, s)
		}
	}
	c.Debugf("%v possible stories to update", len(newStories))

	puts := []interface{}{&f}

	// find non existant stories
	fk := gn.Key(&f)
	getStories := make([]*Story, len(newStories))
	for i, s := range newStories {
		getStories[i] = &Story{Id: s.Id, Parent: fk}
	}
	err := gn.GetMulti(getStories)
	if _, ok := err.(appengine.MultiError); err != nil && !ok {
		c.Errorf("get multi error: %v", err.Error())
		return err
	}
	var updateStories []*Story
	for i, s := range getStories {
		if goon.NotFound(err, i) {
			updateStories = append(updateStories, newStories[i])
		} else if !newStories[i].Updated.IsZero() && !newStories[i].Updated.Equal(s.Updated) {
			newStories[i].Created = s.Created
			newStories[i].Published = s.Published
			updateStories = append(updateStories, newStories[i])
		}
	}
	c.Debugf("%v update stories", len(updateStories))

	for _, s := range updateStories {
		puts = append(puts, s)
		gn.Put(&StoryContent{
			Id:      1,
			Parent:  gn.Key(s),
			Content: s.content,
		})
	}

	c.Debugf("putting %v entities", len(puts))
	if len(puts) > 1 {
		f.Date = time.Now()
		if !hasUpdated {
			f.Updated = f.Date
		}
	}
	if f.Url == "" {
		c.Criticalf("badurl6: %v", f)
		return errors.New("badurl6")
	}
	gn.PutMultiComplete(puts)

	return nil
}
Ejemplo n.º 28
0
func CFixer(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	q := datastore.NewQuery("F").KeysOnly()
	q = q.Limit(1000)
	cs := r.FormValue("c")
	if len(cs) > 0 {
		if cur, err := datastore.DecodeCursor(cs); err == nil {
			q = q.Start(cur)
			c.Infof("starting at %v", cur)
		} else {
			c.Errorf("cursor error %v", err.Error())
		}
	}
	var keys []*datastore.Key
	it := q.Run(Timeout(c, time.Second*15))
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		}
		keys = append(keys, k)
	}

	if len(keys) == 0 {
		c.Errorf("no results")
		return
	} else {
		cur, err := it.Cursor()
		if err != nil {
			c.Errorf("to cur error %v", err.Error())
		} else {
			c.Infof("add with cur %v", cur)
			t := taskqueue.NewPOSTTask("/tasks/cfixer", url.Values{
				"c": {cur.String()},
			})
			taskqueue.Add(c, t, "cfixer")
		}
	}
	c.Infof("fixing %d feeds", len(keys))

	var tasks []*taskqueue.Task
	for _, k := range keys {
		c.Infof("f: %v", k.StringID())
		tasks = append(tasks, taskqueue.NewPOSTTask("/tasks/cfix", url.Values{
			"feed": {k.StringID()},
		}))
	}
	var ts []*taskqueue.Task
	const taskLimit = 100
	for len(tasks) > 0 {
		if len(tasks) > taskLimit {
			ts = tasks[:taskLimit]
			tasks = tasks[taskLimit:]
		} else {
			ts = tasks
			tasks = tasks[0:0]
		}
		if _, err := taskqueue.AddMulti(c, ts, "cfixer"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
	}
}
Ejemplo n.º 29
0
func ImportOpmlTask(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	userid := r.FormValue("user")
	bk := r.FormValue("key")
	fr := blobstore.NewReader(c, appengine.BlobKey(bk))

	var skip int
	if s, err := strconv.Atoi(r.FormValue("skip")); err == nil {
		skip = s
	}
	c.Debugf("reader import for %v, skip %v", userid, skip)

	var userOpml []*OpmlOutline
	remaining := skip

	var proc func(label string, outlines []*OpmlOutline)
	proc = func(label string, outlines []*OpmlOutline) {
		for _, o := range outlines {
			if o.Title == "" {
				o.Title = o.Text
			}
			if o.XmlUrl != "" {
				if remaining > 0 {
					remaining--
				} else if len(userOpml) < IMPORT_LIMIT {
					userOpml = append(userOpml, &OpmlOutline{
						Title:   label,
						Outline: []*OpmlOutline{o},
					})
				}
			}

			if o.Title != "" && len(o.Outline) > 0 {
				proc(o.Title, o.Outline)
			}
		}
	}

	opml := Opml{}
	d := xml.NewDecoder(fr)
	d.CharsetReader = charset.NewReader
	d.Strict = false
	if err := d.Decode(&opml); err != nil {
		c.Errorf("opml error: %v", err.Error())
		return
	}
	proc("", opml.Outline)

	// todo: refactor below with similar from ImportReaderTask
	wg := sync.WaitGroup{}
	wg.Add(len(userOpml))
	for i := range userOpml {
		go func(i int) {
			o := userOpml[i].Outline[0]
			if err := addFeed(c, userid, userOpml[i]); err != nil {
				c.Warningf("opml import error: %v", err.Error())
				// todo: do something here?
			}
			c.Debugf("opml import: %s, %s", o.Title, o.XmlUrl)
			wg.Done()
		}(i)
	}
	wg.Wait()

	ud := UserData{Id: "data", Parent: gn.Key(&User{Id: userid})}
	if err := gn.RunInTransaction(func(gn *goon.Goon) error {
		gn.Get(&ud)
		mergeUserOpml(&ud, userOpml...)
		_, err := gn.Put(&ud)
		return err
	}, nil); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		c.Errorf("ude update error: %v", err.Error())
		return
	}

	if len(userOpml) == IMPORT_LIMIT {
		task := taskqueue.NewPOSTTask(routeUrl("import-opml-task"), url.Values{
			"key":  {bk},
			"user": {userid},
			"skip": {strconv.Itoa(skip + IMPORT_LIMIT)},
		})
		taskqueue.Add(c, task, "import-reader")
	} else {
		c.Infof("opml import done: %v", userid)
	}
}
Ejemplo n.º 30
0
func updateFeed(c mpg.Context, url string, feed *Feed, stories []*Story) error {
	gn := goon.FromContext(c)
	f := Feed{Url: url}
	if err := gn.Get(&f); err != nil {
		return fmt.Errorf("feed not found: %s", url)
	}

	// Compare the feed's listed update to the story's update.
	// Note: these may not be accurate, hence, only compare them to each other,
	// since they should have the same relative error.
	storyDate := f.Updated

	hasUpdated := !feed.Updated.IsZero()
	isFeedUpdated := f.Updated == feed.Updated
	if !hasUpdated {
		feed.Updated = f.Updated
	}
	feed.Date = f.Date
	f = *feed

	if hasUpdated && isFeedUpdated {
		c.Infof("feed %s already updated to %v, putting", url, feed.Updated)
		f.Updated = time.Now()
		gn.Put(&f)
		return nil
	}

	c.Debugf("hasUpdate: %v, isFeedUpdated: %v, storyDate: %v", hasUpdated, isFeedUpdated, storyDate)

	var newStories []*Story
	for _, s := range stories {
		if s.Updated.IsZero() || !s.Updated.Before(storyDate) {
			newStories = append(newStories, s)
		}
	}
	c.Debugf("%v possible stories to update", len(newStories))

	puts := []interface{}{&f}

	// find non existant stories
	fk := gn.Key(&f)
	getStories := make([]*Story, len(newStories))
	for i, s := range newStories {
		getStories[i] = &Story{Id: s.Id, Parent: fk}
	}
	err := gn.GetMulti(getStories)
	if _, ok := err.(appengine.MultiError); err != nil && !ok {
		c.Errorf("get multi error: %v", err.Error())
		return err
	}
	var updateStories []*Story
	for i, s := range getStories {
		if goon.NotFound(err, i) {
			updateStories = append(updateStories, newStories[i])
		} else if !newStories[i].Updated.IsZero() && !newStories[i].Updated.Equal(s.Updated) {
			newStories[i].Created = s.Created
			newStories[i].Published = s.Published
			updateStories = append(updateStories, newStories[i])
		}
	}
	c.Debugf("%v update stories", len(updateStories))

	for _, s := range updateStories {
		puts = append(puts, s)
		sc := StoryContent{
			Id:     1,
			Parent: gn.Key(s),
		}
		buf := &bytes.Buffer{}
		if gz, err := gzip.NewWriterLevel(buf, gzip.BestCompression); err == nil {
			gz.Write([]byte(s.content))
			gz.Close()
			sc.Compressed = buf.Bytes()
		}
		if len(sc.Compressed) == 0 {
			sc.Content = s.content
		}
		gn.Put(&sc)
	}

	c.Debugf("putting %v entities", len(puts))
	if len(puts) > 1 {
		f.Date = time.Now()
		if !hasUpdated {
			f.Updated = f.Date
		}
	}
	gn.PutMulti(puts)

	return nil
}