Example #1
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
	q = q.Filter("n <=", time.Now())
	retry, _ := strconv.Atoi(r.FormValue("retry"))
	c.Errorf("retry: %v", retry)

	//* iterator
	it := gn.Run(q)
	i := 0
	done := false
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			done = true
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		}
		t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		})
		if _, err := taskqueue.Add(c, t, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
		i++
	}
	c.Infof("updating %d feeds", i)
	fmt.Fprintf(w, "updating %d feeds", i)
	if !done {
		time.Sleep(time.Second * time.Duration(i) / 50) // sleep about the time it'll take to process them
		t := taskqueue.NewPOSTTask("/tasks/update-feeds", url.Values{
			"retry": {strconv.Itoa(retry + 1)},
		})
		if _, err := taskqueue.Add(c, t, "update-feeds"); err != nil {
			c.Errorf("taskqueue update feeds error: %v", err.Error())
		}
		c.Errorf("ran update again")
		fmt.Fprintf(w, "\nran update again")
	}
	//*/

	/* get all
	q = q.Limit(1000)
	keys, _ := gn.GetAll(q, nil)
	for _, k := range keys {
		t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		})
		if _, err := taskqueue.Add(c, t, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
	}
	c.Infof("updating %d feeds", len(keys))
	fmt.Fprintf(w, "updating %d feeds", len(keys))
	//*/
}
Example #2
0
func TestTasks(t *testing.T) {

	c, err := NewContext(&Options{TaskQueues: []string{"testQueue"}})
	if err != nil {
		t.Fatalf("NewContext: %v", err)
	}
	defer c.Close()

	task := taskqueue.NewPOSTTask("/post", map[string][]string{})
	_, err = taskqueue.Add(c, task, "testQueue")
	if err != nil {
		t.Fatalf("Could not add task to queue")
	}
	stats, err := taskqueue.QueueStats(c, []string{"testQueue"}, 0) // fetch all of them
	if err != nil {
		t.Fatalf("Could not get taskqueue statistics")
	}
	t.Logf("TaskStatistics = %#v", stats)
	if len(stats) == 0 {
		t.Fatalf("Queue statistics are empty")
	} else if stats[0].Tasks != 1 {
		t.Fatalf("Could not find the task we just added")
	}

	err = taskqueue.Purge(c, "testQueue")
	if err != nil {
		t.Fatalf("Could not purge the queue")
	}
	stats, err = taskqueue.QueueStats(c, []string{"testQueue"}, 0) // fetch all of them
	if len(stats) == 0 {
		t.Fatalf("Queue statistics are empty")
	}
	if stats[0].Tasks != 0 {
		t.Fatalf("Purge command not successful")
	}

	tasks := []*taskqueue.Task{
		taskqueue.NewPOSTTask("/post1", map[string][]string{}),
		taskqueue.NewPOSTTask("/post2", map[string][]string{}),
	}
	_, err = taskqueue.AddMulti(c, tasks, "testQueue")
	if err != nil {
		t.Fatalf("Could not add bulk tasklist to queue")
	}
	stats, err = taskqueue.QueueStats(c, []string{"testQueue"}, 0) // fetch all of them
	if err != nil {
		t.Fatalf("Could not get taskqueue statistics")
	}
	if len(stats) == 0 {
		t.Fatalf("Could not find the tasks we just added")
	} else if stats[0].Tasks != 2 {
		t.Fatalf("Could not find the tasks we just added")
	}

}
Example #3
0
// Look for new flights that we should add to our database. Invoked by cron.
func scanHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	if db, err1 := fdb24.NewFlightDBFr24(urlfetch.Client(c)); err1 != nil {
		c.Errorf(" /mdb/scan: newdb: %v", err1)
		http.Error(w, err1.Error(), http.StatusInternalServerError)

	} else {
		if flights, err2 := db.LookupList(sfo.KBoxSFO120K); err2 != nil {
			c.Errorf(" /mdb/scan: lookup: %v", err2)
			http.Error(w, err2.Error(), http.StatusInternalServerError)
		} else {

			set := ftype.FIFOSet{}
			if err3 := loadFIFOSet(c, &set); err3 != nil {
				c.Errorf(" /mdb/scan: loadcache: %v", err3)
				http.Error(w, err3.Error(), http.StatusInternalServerError)
			}
			new := set.FindNew(flights)
			if err4 := saveFIFOSet(c, set); err4 != nil {
				c.Errorf(" /mdb/scan: savecache: %v", err4)
				http.Error(w, err4.Error(), http.StatusInternalServerError)
			}

			// Enqueue the new flights
			n := 1000
			for i, fs := range new {
				if i >= n {
					break
				}
				if fsStr, err5 := fs.Base64Encode(); err5 != nil {
					http.Error(w, err5.Error(), http.StatusInternalServerError)
					return
				} else {
					url := fmt.Sprintf("/fdb/addflight?deb=%s", fs.F.UniqueIdentifier())
					t := taskqueue.NewPOSTTask(url, map[string][]string{
						"flightsnapshot": {fsStr},
					})

					// We could be smarter about this.
					t.Delay = time.Minute * 45

					if _, err6 := taskqueue.Add(c, t, "addflight"); err6 != nil {
						c.Errorf(" /mdb/scan: enqueue: %v", err6)
						http.Error(w, err6.Error(), http.StatusInternalServerError)
						return
					}
				}
			}

			var params = map[string]interface{}{
				"New":     new,
				"Flights": flights,
			}
			if err7 := templates.ExecuteTemplate(w, "fdb-scan", params); err7 != nil {
				http.Error(w, err7.Error(), http.StatusInternalServerError)
			}
		}
	}
}
Example #4
0
// Grab all users, and enqueue them for batch processing
func upgradeHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	cdb := ComplaintDB{C: c, Memcache: false}

	var cps = []types.ComplainerProfile{}
	cps, err := cdb.GetAllProfiles()
	if err != nil {
		c.Errorf("upgradeHandler: getallprofiles: %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	for _, cp := range cps {
		b64 := ""
		if b64, err = cp.Base64Encode(); err != nil {
			c.Errorf("upgradeHandler: profile encode:", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		t := taskqueue.NewPOSTTask("/batch/upgradeuser", map[string][]string{
			"profile": {b64},
		})
		if _, err := taskqueue.Add(c, t, "batch"); err != nil {
			c.Errorf("upgradeHandler: enqueue: %v", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}
	c.Infof("enqueued %d batch", len(cps))
	w.Write([]byte(fmt.Sprintf("OK, enqueued %d", len(cps))))
}
Example #5
0
func del(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	t := taskqueue.NewPOSTTask("/worker", nil)
	if _, err := taskqueue.Add(c, t, ""); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}
Example #6
0
func cron(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	q := datastore.NewQuery("asin")

	for t := q.Run(c); ; {
		var x Asin
		_, err := t.Next(&x)
		if err == datastore.Done {
			break
		}
		if err != nil {
			return
		}

		task := taskqueue.NewPOSTTask("/task/fetching", url.Values{
			"asin": {x.Name},
		})

		if !appengine.IsDevAppServer() {
			host := backendName + "." + appengine.DefaultVersionHostname(c)
			task.Header.Set("Host", host)
		}

		if _, err := taskqueue.Add(c, task, ""); err != nil {
			c.Errorf("add fetching task: %v", err)
			http.Error(w, "Error: couldn't schedule fetching task", 500)
			return
		}

		fmt.Fprintf(w, "OK")
	}
}
Example #7
0
func addCurriculo(c appengine.Context, r render.Render, req *http.Request) {
	curriculo := new(Curriculo)
	decoder := json.NewDecoder(req.Body)
	err := decoder.Decode(&curriculo)
	if err != nil {
		log.Println(err)
		r.JSON(http.StatusInternalServerError, err.Error())
		return
	}

	key := datastore.NewKey(c, "Curriculo", curriculo.Email, 0, nil)
	_, err = datastore.Put(c, key, curriculo)
	if err != nil {
		log.Println(err)
		r.JSON(http.StatusInternalServerError, err.Error())
		return
	}

	t := taskqueue.NewPOSTTask("/matcher/curriculos", map[string][]string{
		"curriculo": {curriculo.Email},
	})
	if _, err := taskqueue.Add(c, t, ""); err != nil {
		log.Println(err)
		r.JSON(http.StatusInternalServerError, err.Error())
		return
	}

	r.JSON(http.StatusOK, "success")
}
Example #8
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
	q = q.Filter("n <=", time.Now()).Limit(3000)
	it := gn.Run(q)
	i := 0
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		}
		t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		})
		if _, err := taskqueue.Add(c, t, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
		i++
	}
	c.Infof("updating %d feeds", i)
	fmt.Fprintf(w, "updating %d feeds", i)
}
Example #9
0
func collectUrls(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	url := "http://catoverflow.com/api/query?offset=0&limit=1000"

	client := urlfetch.Client(c)
	resp, err := client.Get(url)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	//Split the body by new lines to get the url for each image.
	s := string(body)

	urls := strings.Fields(s)
	for _, u := range urls {
		t := taskqueue.NewPOSTTask("/worker", map[string][]string{"url": {u}})
		if _, err := taskqueue.Add(c, t, ""); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}
}
Example #10
0
func batchCouch(c appengine.Context, w http.ResponseWriter, r *http.Request) {
	keys := []string{}
	z, err := gzip.NewReader(r.Body)
	maybePanic(err)
	d := json.NewDecoder(z)
	maybePanic(d.Decode(&keys))
	c.Infof("Got %v keys to couch", len(keys))

	wg := sync.WaitGroup{}
	ch := make(chan string)
	for i := 0; i < 8; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for kstr := range ch {
				err := postToCouchCommon(c, kstr, true)
				if err != nil {
					c.Errorf("Error on %v: %v", kstr, err)
					taskqueue.Add(c, taskqueue.NewPOSTTask("/couchit",
						url.Values{"key": []string{kstr}}), "couchit")
				}
			}
		}()
	}

	for _, kstr := range keys {
		ch <- kstr
	}
	close(ch)
	wg.Wait()
	w.WriteHeader(204)
}
Example #11
0
func backfill(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	c := appengine.NewContext(r)

	index, err := search.Open(xkcdIndex)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	current, _ := GetCurrent(c)
	for i := 1; i <= current.Num; i++ {
		// xcdc returns 404 with issue 404
		if i == 404 {
			continue
		}

		comicNum := strconv.Itoa(i)

		force := r.FormValue("force")
		if force != "yes" {
			var s ComicSearch
			err := index.Get(c, comicNum, &s)
			if err == nil {
				continue
			}
		}

		t := taskqueue.NewPOSTTask("/index", map[string][]string{"id": {comicNum}})
		if _, err := taskqueue.Add(c, t, ""); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}
}
Example #12
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	q := datastore.NewQuery("F").KeysOnly().Filter("n <=", time.Now())
	q = q.Limit(10 * 60 * 2) // 10/s queue, 2 min cron
	it := q.Run(appengine.Timeout(c, time.Minute))
	tc := make(chan *taskqueue.Task)
	done := make(chan bool)
	i := 0
	u := routeUrl("update-feed")
	go taskSender(c, "update-feed", tc, done)
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		}
		tc <- taskqueue.NewPOSTTask(u, url.Values{
			"feed": {k.StringID()},
		})
		i++
	}
	close(tc)
	<-done
	c.Infof("updating %d feeds", i)
}
Example #13
0
// send uses the Channel API to send the provided message in JSON-encoded form
// to the client identified by clientID.
//
// Channels created with one version of an app (eg, the default frontend)
// cannot be sent on from another version (eg, a backend). This is a limitation
// of the Channel API that should be fixed at some point.
// The send function creates a task that runs on the frontend (where the
// channel was created). The task handler makes the channel.Send API call.
func send(c appengine.Context, clientID string, m Message) {
	if clientID == "" {
		c.Debugf("no channel; skipping message send")
		return
	}
	switch {
	case m.TilesDone:
		c.Debugf("tiles done")
	case m.ZipDone:
		c.Debugf("zip done")
	default:
		c.Debugf("%d tiles", len(m.IDs))
	}
	b, err := json.Marshal(m)
	if err != nil {
		panic(err)
	}
	task := taskqueue.NewPOSTTask("/send", url.Values{
		"clientID": {clientID},
		"msg":      {string(b)},
	})
	host := appengine.DefaultVersionHostname(c)
	task.Header.Set("Host", host)
	if _, err := taskqueue.Add(c, task, sendQueue); err != nil {
		c.Errorf("add send task failed: %v", err)
	}
}
func queuePush(w http.ResponseWriter, r *http.Request, mx map[string]interface{}) {

	lg, b := loghttp.BuffLoggerUniversal(w, r)
	_ = b

	c := appengine.NewContext(r)

	m := map[string][]string{"counter_name": []string{nscStringKey}}
	t := taskqueue.NewPOSTTask("/_ah/namespaced-counters/queue-pop", m)

	taskqueue.Add(c, t, "")

	c, err := appengine.Namespace(c, altNamespace)
	lg(err)
	taskqueue.Add(c, t, "")

	io.WriteString(w, "tasks enqueued\n")

	io.WriteString(w, "\ncounter values now: \n")
	readBothNamespaces(w, r, mx)

	io.WriteString(w, "\n\n...sleeping... \n")
	time.Sleep(time.Duration(400) * time.Millisecond)
	readBothNamespaces(w, r, mx)

}
Example #15
0
// addTaskClientGroupPut receives a list of ClientGroups to put into datastore
// and stores it temporarily into memcache. It then submits the key as a
// taskqueue task.
func addTaskClientGroupPut(c appengine.Context, dateStr string, cgs []ClientGroup) {
	// Create unique key for memcache
	key := cgMemcachePutKey()

	// Store CGs into memcache
	item := &memcache.Item{
		Key:    key,
		Object: cgs,
	}
	if err := memcache.Gob.Set(c, item); err != nil {
		c.Errorf("rtt.addTaskClientGroupPut:memcache.Set: %s", err)
		return
	}

	// Submit taskqueue task
	values := make(url.Values)
	values.Add(FormKeyPutKey, key)
	values.Add(FormKeyImportDate, dateStr)
	task := taskqueue.NewPOSTTask(URLTaskImportPut, values)
	_, err := taskqueue.Add(c, task, TaskQueueNameImportPut)
	if err != nil {
		c.Errorf("rtt.addTaskClientGroupPut:taskqueue.Add: %s", err)
		return
	}
}
Example #16
0
// Insert a task to reset the running timer.  Should be done in a transaction.
func resetTimer(c appengine.Context, owner string) error {
	t := taskqueue.NewPOSTTask("/task/reset", url.Values{
		"owner": []string{owner},
	})
	_, err := taskqueue.Add(c, t, "")
	return err
}
Example #17
0
func ImportOpml(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	cu := user.Current(c)
	gn := goon.FromContext(c)
	u := User{Id: cu.ID}
	if err := gn.Get(&u); err != nil {
		serveError(w, err)
		return
	}
	backupOPML(c)

	if file, _, err := r.FormFile("file"); err == nil {
		if fdata, err := ioutil.ReadAll(file); err == nil {
			buf := bytes.NewReader(fdata)
			// attempt to extract from google reader takeout zip
			if zb, zerr := zip.NewReader(buf, int64(len(fdata))); zerr == nil {
				for _, f := range zb.File {
					if strings.HasSuffix(f.FileHeader.Name, "Reader/subscriptions.xml") {
						if rc, rerr := f.Open(); rerr == nil {
							if fb, ferr := ioutil.ReadAll(rc); ferr == nil {
								fdata = fb
								break
							}
						}
					}
				}
			}

			// Preflight the OPML, so we can report any errors.
			d := xml.NewDecoder(bytes.NewReader(fdata))
			d.CharsetReader = charset.NewReader
			d.Strict = false
			opml := Opml{}
			if err := d.Decode(&opml); err != nil {
				serveError(w, err)
				c.Errorf("opml error: %v", err.Error())
				return
			}

			var b bytes.Buffer
			enc := gob.NewEncoder(&b)
			err := enc.Encode(&opml)
			if err != nil {
				serveError(w, err)
				return
			}
			bk, err := saveFile(c, b.Bytes())
			if err != nil {
				serveError(w, err)
				return
			}
			task := taskqueue.NewPOSTTask(routeUrl("import-opml-task"), url.Values{
				"key":  {string(bk)},
				"user": {cu.ID},
			})
			taskqueue.Add(c, task, "import-reader")
		}
	}
}
Example #18
0
File: cl.go Project: timnau/golang
func UpdateCLLater(c appengine.Context, n string, delay time.Duration) {
	t := taskqueue.NewPOSTTask("/update-cl", url.Values{
		"cl": []string{n},
	})
	t.Delay = delay
	if _, err := taskqueue.Add(c, t, "update-cl"); err != nil {
		c.Errorf("Failed adding task: %v", err)
	}
}
Example #19
0
func queueProcessing(filename string, c appengine.Context) error {
	t := taskqueue.NewPOSTTask("/", url.Values{
		"filename": {filename},
	})

	_, err := taskqueue.Add(c, t, "image-processor")

	return err
}
Example #20
0
func taskHandler(w http.ResponseWriter, r *http.Request) {
	max := 12
	var tasks []*taskqueue.Task
	for i := 0; i < max; i++ {
		tasks = append(tasks, taskqueue.NewPOSTTask("/", nil))
	}
	c := appengine.NewContext(r)
	taskqueue.AddMulti(c, tasks, "read")
}
Example #21
0
func doEnqueuePaymentDoneEmail(reqCode, method string, c *Context) error {
	c.Aec().Infof("Enqueuing payment done email for reqCode=%q, method=%q", reqCode, method)
	v := url.Values{}
	v.Set("reqCode", reqCode)
	v.Set("method", method)
	t := taskqueue.NewPOSTTask("/tasks/send-payment-done-email", v)
	_, err := taskqueue.Add(c.Aec(), t, "")
	return err
}
Example #22
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
	q = q.Filter("n <=", time.Now())

	q = q.Limit(1000)
	it := gn.Run(q)
	var keys []*datastore.Key
	var del []*datastore.Key
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		} else if len(k.StringID()) == 0 {
			del = append(del, k)
			continue
		}
		keys = append(keys, k)
	}
	tasks := make([]*taskqueue.Task, len(keys))
	for i, k := range keys {
		tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		})
	}
	var ts []*taskqueue.Task
	const taskLimit = 100
	for len(tasks) > 0 {
		if len(tasks) > taskLimit {
			ts = tasks[:taskLimit]
			tasks = tasks[taskLimit:]
		} else {
			ts = tasks
			tasks = tasks[0:0]
		}
		if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
	}
	c.Infof("updating %d feeds", len(keys))

	if len(del) > 0 {
		c.Errorf("attempt to delete %v feeds", len(del))
		if err := gn.DeleteMulti(del); err != nil {
			c.Errorf("delete error: %v", err.Error())
		}
		fmt.Fprintf(w, `<html><head><meta http-equiv="refresh" content="0"></head></html>`)
		fmt.Fprintf(w, "attempt to delete %v feeds", len(del))
		for _, k := range del {
			fmt.Fprintf(w, "\n<br>%v", k)
		}
	}
	fmt.Fprintf(w, "updating %d feeds", len(keys))
}
Example #23
0
func doEnqueuePayRequestEmails(reqCodes []string, c *Context) error {
	c.Aec().Infof("Enqueuing pay request emails for reqCodes: %v", reqCodes)
	if len(reqCodes) == 0 {
		return nil
	}
	v := url.Values{}
	v.Set("reqCodes", strings.Join(reqCodes, ","))
	t := taskqueue.NewPOSTTask("/tasks/send-pay-request-emails", v)
	_, err := taskqueue.Add(c.Aec(), t, "")
	return err
}
Example #24
0
func couchit(c appengine.Context, k *datastore.Key, v url.Values) error {
	vals := url.Values{
		"key": []string{k.Encode()},
	}
	for k, v := range v {
		vals[k] = v
	}
	_, err := taskqueue.Add(c, taskqueue.NewPOSTTask("/couchit", vals),
		"couchit")
	return err
}
Example #25
0
func (f Feed) Subscribe(c appengine.Context) {
	if !f.IsSubscribed() {
		t := taskqueue.NewPOSTTask(routeUrl("subscribe-feed"), url.Values{
			"feed": {f.Url},
		})
		if _, err := taskqueue.Add(c, t, "update-manual"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		} else {
			c.Warningf("subscribe feed: %v", f.Url)
		}
	}
}
Example #26
0
func UploadHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	u := user.Current(c)
	if u == nil {
		url, _ := user.LoginURL(c, r.URL.String())
		w.Header().Set("Location", url)
		w.WriteHeader(http.StatusFound)
		return
	}

	id := r.FormValue("id")
	if len(id) > 0 {
		w.Header().Set("Location", "/upload2?id=has_key:"+id)
		w.WriteHeader(http.StatusFound)
		//    uploadTemplate.Execute(w, id)
		return
	}

	blobs, other_params, err := blobstore.ParseUpload(r)
	if len(blobs) == 0 {
		//    w.WriteHeader(http.StatusBadRequest)
		//    fmt.Fprintf(w, "No data '%v'", err)
		w.Header().Set("Location", "/upload2?id=Bad+upload:"+err.String())
		w.WriteHeader(http.StatusFound)
		return
	}
	file := blobs["file_data"]
	if len(file) == 0 {
		//    w.WriteHeader(http.StatusBadRequest)
		//    fmt.Fprintf(w, "No data")
		w.Header().Set("Location", "/upload2?id=No_file_data")
		w.WriteHeader(http.StatusFound)
		return
	}

	key := string(file[0].BlobKey)
	if other_params == nil {
		other_params = make(map[string][]string)
	}
	other_params["key"] = append(other_params["key"], key)
	task := taskqueue.NewPOSTTask("/process/gedcom", other_params)
	task.Name = key
	if err := taskqueue.Add(c, task, ""); err != nil {
		//    http.Error(w, err.String(), http.StatusInternalServerError)
		w.Header().Set("Location", "/upload2?id=bad_task:"+err.String())
		w.WriteHeader(http.StatusFound)
		return
	}

	w.Header().Set("Location", "/upload?id="+key)
	w.WriteHeader(http.StatusFound)
	return
}
Example #27
0
func DeleteOldFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	ctx := appengine.Timeout(c, time.Minute)
	gn := goon.FromContext(c)
	q := datastore.NewQuery(gn.Kind(&Feed{})).Filter("n=", timeMax).KeysOnly()
	if cur, err := datastore.DecodeCursor(r.FormValue("c")); err == nil {
		q = q.Start(cur)
	}
	it := q.Run(ctx)
	done := false
	var tasks []*taskqueue.Task
	for i := 0; i < 10000 && len(tasks) < 100; i++ {
		k, err := it.Next(nil)
		if err == datastore.Done {
			c.Criticalf("done")
			done = true
			break
		} else if err != nil {
			c.Errorf("err: %v", err)
			continue
		}
		values := make(url.Values)
		values.Add("f", k.StringID())
		tasks = append(tasks, taskqueue.NewPOSTTask("/tasks/delete-old-feed", values))
	}
	if len(tasks) > 0 {
		c.Errorf("deleting %v feeds", len(tasks))
		if _, err := taskqueue.AddMulti(c, tasks, ""); err != nil {
			c.Errorf("err: %v", err)
		}
	}
	if !done {
		if cur, err := it.Cursor(); err == nil {
			values := make(url.Values)
			values.Add("c", cur.String())
			taskqueue.Add(c, taskqueue.NewPOSTTask("/tasks/delete-old-feeds", values), "")
		} else {
			c.Errorf("err: %v", err)
		}
	}
}
Example #28
0
func saveConfHandler(w io.Writer, r *http.Request, ctx appengine.Context, u *user.User) error {
	c, err := confFromRequest(r)
	if err != nil {
		return fmt.Errorf("conf from request: %v", err)
	}

	err = datastore.RunInTransaction(ctx, func(ctx appengine.Context) error {
		// Save the conference and generate the tickets
		if err := c.Save(ctx); err != nil {
			return fmt.Errorf("save conference: %v", err)
		}
		if err := c.CreateAndSaveTickets(ctx); err != nil {
			return fmt.Errorf("generate tickets: %v", err)
		}

		// Announce the conference
		a := conf.NewAnnouncement(fmt.Sprintf(
			"A new conference has just been scheduled! %s in %s. Don't wait; book now!",
			c.Name, c.City))
		if err := a.Save(ctx); err != nil {
			return fmt.Errorf("announce conference: %v", err)
		}

		// Queue a task to email interested users.
		task := taskqueue.NewPOSTTask(
			"/notifyinterestedusers",
			url.Values{"conf_id": []string{c.ID()}},
		)
		if _, err = taskqueue.Add(ctx, task, ""); err != nil {
			return fmt.Errorf("add task to default queue: %v", err)
		}

		// Queue a task to review the conference.
		task = &taskqueue.Task{
			Method:  "PULL",
			Payload: []byte(c.ID()),
		}
		task, err := taskqueue.Add(ctx, task, "review-conference-queue")
		if err != nil {
			return fmt.Errorf("add task to review queue: %v", err)
		}
		return nil
	}, &datastore.TransactionOptions{XG: true})
	if err != nil {
		return err
	}

	return RedirectTo("/showtickets?conf_id=" + url.QueryEscape(c.ID()))
}
Example #29
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
	q = q.Filter("n <=", time.Now())
	keys, _ := gn.GetAll(q, nil)
	for _, k := range keys {
		t := taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		})
		if _, err := taskqueue.Add(c, t, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
	}
	c.Infof("updating %d feeds", len(keys))
}
Example #30
0
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	q := datastore.NewQuery("F").KeysOnly()
	q = q.Filter("n <=", time.Now())
	q = q.Limit(3000)
	var keys []*datastore.Key
	it := q.Run(c)
	retry := 0
	for {
		k, err := it.Next(nil)
		if err != nil {
			c.Errorf("next error: %v, retry: %v", err.Error(), retry)
			if retry == 5 {
				break
			}
			retry++
			it = q.Run(c)
		} else {
			keys = append(keys, k)
		}
	}
	if len(keys) == 0 {
		c.Errorf("giving up")
		return
	}

	tasks := make([]*taskqueue.Task, len(keys))
	for i, k := range keys {
		c.Infof("task: %v, %v", i, k)
		tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		})
	}
	var ts []*taskqueue.Task
	const taskLimit = 100
	for len(tasks) > 0 {
		if len(tasks) > taskLimit {
			ts = tasks[:taskLimit]
			tasks = tasks[taskLimit:]
		} else {
			ts = tasks
			tasks = tasks[0:0]
		}
		if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
	}
	c.Infof("updating %d feeds", len(keys))
}