Exemple #1
0
func status(ctxt appengine.Context) string {
	w := new(bytes.Buffer)

	var count int64
	app.ReadMeta(ctxt, "issue.count", &count)
	fmt.Fprintln(w, time.Now())

	var t1 string
	app.ReadMeta(ctxt, "issue.mtime", &t1)
	fmt.Fprintf(w, "issue modifications up to %v\n", t1)
	fmt.Fprintln(w, time.Now())

	return "<pre>" + html.EscapeString(w.String()) + "</pre>"
}
Exemple #2
0
func writeCL(ctxt appengine.Context, cl *CL, mtimeKey, modified string) error {
	err := app.Transaction(ctxt, func(ctxt appengine.Context) error {
		var old CL
		if err := app.ReadData(ctxt, "CL", cl.CL, &old); err != nil && err != datastore.ErrNoSuchEntity {
			return err
		}
		if old.CL == "" { // no old data
			var count int64
			app.ReadMeta(ctxt, "codereview.count", &count)
			app.WriteMeta(ctxt, "codereview.count", count+1)
		}

		// Copy CL into original structure.
		// This allows us to maintain other information in the CL structure
		// and not overwrite it when the Rietveld information is updated.
		if cl.Dead {
			old.Dead = true
		} else {
			old.Dead = false
			if old.Modified.After(cl.Modified) {
				return fmt.Errorf("CL %v: have %v but Rietveld sent %v", cl.CL, old.Modified, cl.Modified)
			}
			old.CL = cl.CL
			old.Desc = cl.Desc
			old.Owner = cl.Owner
			old.OwnerEmail = cl.OwnerEmail
			old.Created = cl.Created
			old.Modified = cl.Modified
			old.MessagesLoaded = cl.MessagesLoaded
			if cl.MessagesLoaded {
				old.Messages = cl.Messages
				old.Submitted = cl.Submitted
			}
			old.Reviewers = cl.Reviewers
			old.CC = cl.CC
			old.Closed = cl.Closed
			if !reflect.DeepEqual(old.PatchSets, cl.PatchSets) {
				old.PatchSets = cl.PatchSets
				old.PatchSetsLoaded = false
			}
		}

		if err := app.WriteData(ctxt, "CL", cl.CL, &old); err != nil {
			return err
		}
		if mtimeKey != "" {
			app.WriteMeta(ctxt, mtimeKey, modified)
		}
		return nil
	})
	if err != nil {
		ctxt.Errorf("storing CL %v: %v", cl.CL, err)
	}
	return err
}
Exemple #3
0
func oauthConfig(ctxt appengine.Context) (*oauth.Config, error) {
	var clientID, clientSecret string

	if err := app.ReadMeta(ctxt, "googleapi.clientid", &clientID); err != nil {
		return nil, err
	}
	if err := app.ReadMeta(ctxt, "googleapi.clientsecret", &clientSecret); err != nil {
		return nil, err
	}

	cfg := &oauth.Config{
		ClientId:     clientID,
		ClientSecret: clientSecret,
		Scope:        "https://code.google.com/feeds/issues",
		AuthURL:      "https://accounts.google.com/o/oauth2/auth",
		TokenURL:     "https://accounts.google.com/o/oauth2/token",
		RedirectURL:  "https://go-dev.appspot.com/codetoken",
		AccessType:   "offline",
	}
	return cfg, nil
}
Exemple #4
0
func postIssueComment(ctxt appengine.Context, id, comment string) error {
	cfg, err := oauthConfig(ctxt)
	if err != nil {
		return fmt.Errorf("oauthconfig: %v", err)
	}

	var tok oauth.Token
	if err := app.ReadMeta(ctxt, "codelogin.token", &tok); err != nil {
		return fmt.Errorf("reading token: %v", err)
	}

	tr := &oauth.Transport{
		Config:    cfg,
		Token:     &tok,
		Transport: urlfetch.Client(ctxt).Transport,
	}
	client := tr.Client()

	var buf bytes.Buffer
	buf.WriteString(`<?xml version='1.0' encoding='UTF-8'?>
<entry xmlns='http://www.w3.org/2005/Atom' xmlns:issues='http://schemas.google.com/projecthosting/issues/2009'>
  <content type='html'>`)
	xml.Escape(&buf, []byte(comment))
	buf.WriteString(`</content>
  <author>
    <name>ignored</name>
  </author>
  <issues:updates>
  </issues:updates>
</entry>
`)
	u := "https://code.google.com/feeds/issues/p/go/issues/" + id + "/comments/full"
	req, err := http.NewRequest("POST", u, &buf)
	if err != nil {
		return fmt.Errorf("write: %v", err)
	}
	req.Header.Set("Content-Type", "application/atom+xml")
	resp, err := client.Do(req)
	if err != nil {
		return fmt.Errorf("write: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != 201 {
		buf.Reset()
		io.Copy(&buf, resp.Body)
		return fmt.Errorf("write: %v\n%s", resp.Status, buf.String())
	}
	return nil
}
Exemple #5
0
func writeIssue(ctxt appengine.Context, issue *Issue, stateKey string, state interface{}) error {
	err := app.Transaction(ctxt, func(ctxt appengine.Context) error {
		var old Issue
		if err := app.ReadData(ctxt, "Issue", fmt.Sprint(issue.ID), &old); err != nil && err != datastore.ErrNoSuchEntity {
			return err
		}
		if old.ID == 0 { // no old data
			var count int64
			app.ReadMeta(ctxt, "issue.count", &count)
			app.WriteMeta(ctxt, "issue.count", count+1)
		}

		if old.Modified.After(issue.Modified) {
			return fmt.Errorf("issue %v: have %v but code.google.com sent %v", issue.ID, old.Modified, issue.Modified)
		}

		// Copy Issue into original structure.
		// This allows us to maintain other information in the Issue structure
		// and not overwrite it when the issue information is updated.
		old.ID = issue.ID
		old.Summary = issue.Summary
		old.Status = issue.Status
		old.Duplicate = issue.Duplicate
		old.Owner = issue.Owner
		old.CC = issue.CC
		old.Label = issue.Label
		old.Comment = issue.Comment
		old.State = issue.State
		old.Created = issue.Created
		old.Modified = issue.Modified
		old.Stars = issue.Stars
		old.ClosedDate = issue.ClosedDate
		updateIssue(&old)

		if err := app.WriteData(ctxt, "Issue", fmt.Sprint(issue.ID), &old); err != nil {
			return err
		}
		if stateKey != "" {
			app.WriteMeta(ctxt, stateKey, state)
		}
		return nil
	})
	if err != nil {
		ctxt.Errorf("storing issue %v: %v", issue.ID, err)
	}
	return err
}
Exemple #6
0
func codetoken(ctxt appengine.Context, w http.ResponseWriter, req *http.Request) {
	var randState string
	if err := app.ReadMeta(ctxt, "codelogin.random", &randState); err != nil {
		panic(err)
	}

	cfg, err := oauthConfig(ctxt)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	if req.FormValue("state") != randState {
		http.Error(w, "bad state", 500)
		return
	}
	code := req.FormValue("code")
	if code == "" {
		http.Error(w, "missing code", 500)
		return
	}

	tr := &oauth.Transport{
		Config:    cfg,
		Transport: urlfetch.Client(ctxt).Transport,
	}

	_, err = tr.Exchange(code)
	if err != nil {
		http.Error(w, "exchanging code: "+err.Error(), 500)
		return
	}

	if err := app.WriteMeta(ctxt, "codelogin.token", tr.Token); err != nil {
		http.Error(w, "writing token: "+err.Error(), 500)
		return
	}

	app.DeleteMeta(ctxt, "codelogin.random")

	fmt.Fprintf(w, "have token; expires at %v\n", tr.Token.Expiry)
}
Exemple #7
0
func load(ctxt appengine.Context) error {
	mtime := time.Date(2009, 1, 1, 0, 0, 0, 0, time.UTC)
	if appengine.IsDevAppServer() {
		mtime = time.Now().UTC().Add(-24 * time.Hour)
	}
	app.ReadMeta(ctxt, "issue.mtime", &mtime)

	now := time.Now()

	var issues []*Issue
	var err error
	const maxResults = 500
	var try int
	needMore := false
	for try = 0; ; try++ {
		issues, err = search(ctxt, "go", "all", "", false, mtime, now, maxResults)
		if err != nil {
			ctxt.Errorf("load issues since %v: %v", mtime, err)
			return nil
		}

		if len(issues) == 0 {
			ctxt.Infof("no updates found from %v to %v", mtime, now)
			app.WriteMeta(ctxt, "issue.mtime", now.Add(-1*time.Minute))
			if try > 0 {
				// We shortened the time range; try again now that we've updated mtime.
				return app.ErrMoreCron
			}
			return nil
		}
		if len(issues) < maxResults {
			ctxt.Infof("%d issues from %v to %v", len(issues), mtime, now)
			if try > 0 {
				// Keep exploring once we finish this load.
				needMore = true
			}
			break
		}

		ctxt.Errorf("updater found too many updates from %v to %v", mtime, now)
		if now.Sub(mtime) <= 2*time.Second {
			ctxt.Errorf("cannot shorten update time frame")
			return nil
		}

		if now.Sub(mtime) > 1*time.Hour {
			now = mtime.Add(now.Sub(mtime) / 10)
		} else {
			now = mtime.Add(now.Sub(mtime) / 2)
		}
		ctxt.Infof("shortened to %v to %v", mtime, now)
	}

	issues, err = search(ctxt, "go", "all", "", true, mtime, now, maxResults)
	if err != nil {
		ctxt.Errorf("full load of issues from %v to %v: %v", mtime, now, err)
		return nil
	}

	if len(issues) == 0 {
		ctxt.Errorf("unexpected: no issues from %v to %v", mtime, now)
		return nil
	}

	for _, issue := range issues {
		println("WRITE ISSUE", issue.ID)
		if err := writeIssue(ctxt, issue, "", nil); err != nil {
			return nil
		}
		if mtime.Before(issue.Modified) {
			mtime = issue.Modified
		}
	}

	if try > 0 {
		mtime = now.Add(-1 * time.Second)
	}
	app.WriteMeta(ctxt, "issue.mtime", mtime.UTC())

	if needMore {
		return app.ErrMoreCron
	}
	return nil
}
Exemple #8
0
func status(ctxt appengine.Context) string {
	w := new(bytes.Buffer)
	var count int64
	for _, group := range []string{"golang-dev", "golang-codereviews"} {
		for _, reviewerOrCC := range []string{"reviewer", "cc"} {
			var t string
			mtimeKey := "codereview.mtime." + reviewerOrCC + "." + group
			app.ReadMeta(ctxt, mtimeKey, &t)
			fmt.Fprintf(w, "%v last update for %s\n", t, mtimeKey)
		}
	}
	app.ReadMeta(ctxt, "codereview.count", &count)
	fmt.Fprintf(w, "%d CLs total\n", count)

	var chunk = 20000
	if appengine.IsDevAppServer() {
		chunk = 100
	}
	q := datastore.NewQuery("CL").
		Filter("PatchSetsLoaded <=", false).
		KeysOnly().
		Limit(chunk)

	n := 0
	it := q.Run(ctxt)
	for {
		_, err := it.Next(nil)
		if err != nil {
			break
		}
		n++
	}
	fmt.Fprintf(w, "%d with PatchSetsLoaded = false\n", n)

	q = datastore.NewQuery("CL").
		Filter("MessagesLoaded <=", false).
		KeysOnly().
		Limit(chunk)

	n = 0
	it = q.Run(ctxt)
	for {
		_, err := it.Next(nil)
		if err != nil {
			break
		}
		n++
	}
	fmt.Fprintf(w, "%d with MessagesLoaded = false\n", n)

	fmt.Fprintf(w, "\n")

	q = datastore.NewQuery("RevTodo").
		Limit(10000).
		KeysOnly()

	n = 0
	it = q.Run(ctxt)
	for {
		_, err := it.Next(nil)
		if err != nil {
			break
		}
		n++
	}
	fmt.Fprintf(w, "\n%d hg heads\n", n)

	q = datastore.NewQuery("Meta").
		Filter("__key__ >=", datastore.NewKey(ctxt, "Meta", "commit.count.", 0, nil)).
		Filter("__key__ <=", datastore.NewKey(ctxt, "Meta", "commit.count/", 0, nil)).
		Limit(100)

	type meta struct {
		JSON []byte `datastore:",noindex"`
	}
	it = q.Run(ctxt)
	for {
		var m meta
		key, err := it.Next(&m)
		if err != nil {
			break
		}
		fmt.Fprintf(w, "%s %s\n", key.StringID(), m.JSON)
	}
	fmt.Fprintf(w, "\n")

	q = datastore.NewQuery("CL").
		Filter("Closed =", false).
		Filter("Submitted =", false).
		Filter("HasReviewers =", true).
		Order("Summary").
		KeysOnly().
		Limit(20000)

	n = 0
	it = q.Run(ctxt)
	for {
		_, err := it.Next(nil)
		if err != nil {
			break
		}
		n++
	}
	fmt.Fprintf(w, "\n%d pending CLs.\n", n)

	q = datastore.NewQuery("CL").
		Filter("Active =", true).
		Filter("NeedMailIssue >", "").
		KeysOnly().
		Limit(20000)

	n = 0
	it = q.Run(ctxt)
	for {
		_, err := it.Next(nil)
		if err != nil {
			break
		}
		n++
	}
	fmt.Fprintf(w, "\n%d CLs need issue mails.\n", n)

	return "<pre>" + html.EscapeString(w.String()) + "</pre>\n"
}
Exemple #9
0
func load(ctxt appengine.Context) error {
	// The deadline for task invocation is 10 minutes.
	// Stop when we've run for 5 minutes and ask to be rescheduled.
	deadline := time.Now().Add(5 * time.Minute)

	for _, group := range []string{"golang-dev", "golang-codereviews"} {
		for _, reviewerOrCC := range []string{"reviewer", "cc"} {
			// The stored mtime is the most recent modification time we've seen.
			// We ask for all changes since then.
			mtimeKey := "codereview.mtime." + reviewerOrCC + "." + group
			var mtime string
			if appengine.IsDevAppServer() {
				mtime = "2013-12-01 00:00:00" // limit fetching in empty datastore
			}
			app.ReadMeta(ctxt, mtimeKey, &mtime)
			cursor := ""

			// Rietveld gives us back times with microseconds, but it rejects microseconds
			// in the ModifiedAfter URL parameter. Drop them. We'll see a few of the most
			// recent CLs again. No big deal.
			if i := strings.Index(mtime, "."); i >= 0 {
				mtime = mtime[:i]
			}

			const itemsPerPage = 100
			for n := 0; ; n++ {
				var q struct {
					Cursor  string    `json:"cursor"`
					Results []*jsonCL `json:"results"`
				}
				err := fetchJSON(ctxt, &q, urlWithParams(queryTmpl, map[string]string{
					"ReviewerOrCC":  reviewerOrCC,
					"Group":         group,
					"ModifiedAfter": mtime,
					"Order":         "modified",
					"Cursor":        cursor,
					"Limit":         fmt.Sprint(itemsPerPage),
				}))
				if err != nil {
					ctxt.Errorf("loading codereview by %s: URL <%s>: %v", reviewerOrCC, q, err)
					break
				}
				ctxt.Infof("found %d CLs", len(q.Results))
				if len(q.Results) == 0 {
					break
				}
				cursor = q.Cursor

				for _, jcl := range q.Results {
					cl := jcl.toCL(ctxt)
					if err := writeCL(ctxt, cl, mtimeKey, jcl.Modified); err != nil {
						break // error already logged
					}
				}

				if len(q.Results) < itemsPerPage {
					ctxt.Infof("reached end of results - codereview by %s up to date", reviewerOrCC)
					break
				}

				if time.Now().After(deadline) {
					ctxt.Infof("more to do for codereview by %s - rescheduling", reviewerOrCC)
					return app.ErrMoreCron
				}
			}
		}
	}

	ctxt.Infof("all done")
	return nil
}
Exemple #10
0
func postMovedNote(ctxt appengine.Context, kind, id string) error {
	var old Issue
	if err := app.ReadData(ctxt, "Issue", id, &old); err != nil {
		return err
	}
	updateIssue(&old)
	if !old.NeedGithubNote {
		err := app.Transaction(ctxt, func(ctxt appengine.Context) error {
			var old Issue
			if err := app.ReadData(ctxt, "Issue", id, &old); err != nil {
				return err
			}
			old.NeedGithubNote = false
			return app.WriteData(ctxt, "Issue", id, &old)
		})
		return err
	}

	cfg, err := oauthConfig(ctxt)
	if err != nil {
		return fmt.Errorf("oauthconfig: %v", err)
	}

	var tok oauth.Token
	if err := app.ReadMeta(ctxt, "codelogin.token", &tok); err != nil {
		return fmt.Errorf("reading token: %v", err)
	}

	tr := &oauth.Transport{
		Config:    cfg,
		Token:     &tok,
		Transport: &urlfetch.Transport{Context: ctxt, Deadline: 45 * time.Second},
	}
	client := tr.Client()

	status := ""
	if old.State != "closed" {
		status = "<issues:status>Moved</issues:status>"
	}

	var buf bytes.Buffer
	buf.WriteString(`<?xml version='1.0' encoding='UTF-8'?>
<entry xmlns='http://www.w3.org/2005/Atom' xmlns:issues='http://schemas.google.com/projecthosting/issues/2009'>
  <content type='html'>`)
	xml.Escape(&buf, []byte(fmt.Sprintf("This issue has moved to https://golang.org/issue/%s\n", id)))
	buf.WriteString(`</content>
  <author>
    <name>ignored</name>
  </author>
  <issues:sendEmail>False</issues:sendEmail>
  <issues:updates>
    <issues:label>IssueMoved</issues:label>
    <issues:label>Restrict-AddIssueComment-Commit</issues:label>
    ` + status + `
  </issues:updates>
</entry>
`)
	u := "https://code.google.com/feeds/issues/p/go/issues/" + id + "/comments/full"
	req, err := http.NewRequest("POST", u, &buf)
	if err != nil {
		return fmt.Errorf("write: %v", err)
	}
	req.Header.Set("Content-Type", "application/atom+xml")
	resp, err := client.Do(req)
	if err != nil {
		return fmt.Errorf("write: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != 201 {
		buf.Reset()
		io.Copy(&buf, resp.Body)
		return fmt.Errorf("write: %v\n%s", resp.Status, buf.String())
	}

	err = app.Transaction(ctxt, func(ctxt appengine.Context) error {
		var old Issue
		if err := app.ReadData(ctxt, "Issue", id, &old); err != nil {
			return err
		}
		old.NeedGithubNote = false
		old.Label = append(old.Label, "IssueMoved", "Restrict-AddIssueComment-Commit")
		return app.WriteData(ctxt, "Issue", id, &old)
	})
	return err
}
Exemple #11
0
func loadRevOnce(ctxt appengine.Context, repo, branch, hash string) (nextHash string) {
	ctxt.Infof("load todo %s %s %s", repo, branch, hash)

	// Check that this todo is still valid.
	// If so, extend the expiry time so that no one else tries it while we do.
	// This supercedes the usual use of app.Lock and app.Unlock and also
	// provides a way to rate limit the polling.
	todoKey := fmt.Sprintf("commit.todo.%s.%s", repo, hash)

	err := app.Transaction(ctxt, func(ctxt appengine.Context) error {
		var todo revTodo
		if err := app.ReadData(ctxt, "RevTodo", todoKey, &todo); err != nil {
			return err
		}
		if time.Now().Before(todo.Time) {
			ctxt.Infof("poll %s %s not scheduled until %v", repo, hash, todo.Time)
			return errWait
		}
		dtAll := todo.Time.Sub(todo.Start)
		dtOne := todo.Time.Sub(todo.Last)
		var dtMax time.Duration
		if dtAll < 24*time.Hour {
			dtMax = 5 * time.Minute
		} else if dtAll < 7*24*time.Hour {
			dtMax = 1 * time.Hour
		} else {
			dtMax = 24 * time.Hour
		}
		if dtOne *= 2; dtOne > dtMax {
			dtOne = dtMax
		} else if dtOne == 0 {
			dtOne = 1 * time.Minute
		}
		todo.Last = time.Now()
		todo.Time = todo.Last.Add(dtOne)

		if err := app.WriteData(ctxt, "RevTodo", todoKey, &todo); err != nil {
			return err
		}
		return nil
	})

	if err != nil {
		ctxt.Errorf("skipping poll: %v", err)
		return ""
	}

	r, err := fetchRev(ctxt, repo, hash)
	if err != nil {
		ctxt.Errorf("fetching %v %v: %v", repo, hash, err)
		return ""
	}

	err = app.Transaction(ctxt, func(ctxt appengine.Context) error {
		var old Rev
		if err := app.ReadData(ctxt, "Rev", repo+"."+hash, &old); err != nil && err != datastore.ErrNoSuchEntity {
			return err
		}
		if old.Hash == r.Hash && len(old.Next) == len(r.Next) {
			// up to date
			return nil
		}
		if old.Hash == "" { // no old data
			var count int
			if err := app.ReadMeta(ctxt, "commit.count."+repo, &count); err != nil && err != datastore.ErrNoSuchEntity {
				return err
			}
			count++
			old.Seq = count
			if err := app.WriteMeta(ctxt, "commit.count."+repo, count); err != nil {
				return err
			}
			if r.Branch != branch && len(r.Prev) == 1 {
				ctxt.Infof("detected branch; forcing todo of parent")
				err := writeTodo(ctxt, repo, branch, r.Prev[0], true)
				if err != nil {
					ctxt.Errorf("re-adding todo: %v", err)
				}
				nextHash = r.Prev[0]
			}
		}
		old.Repo = r.Repo
		old.Branch = r.Branch
		// old.Seq already correct; r.Seq is not
		old.Hash = r.Hash
		old.ShortHash = old.Hash[:12]
		old.Prev = r.Prev
		old.Next = r.Next
		old.Author = r.Author
		old.AuthorEmail = r.AuthorEmail
		old.Time = r.Time
		old.Log = r.Log
		old.Files = r.Files

		if err := app.WriteData(ctxt, "Rev", repo+"."+hash, &old); err != nil {
			return err
		}

		return nil
	})

	if err != nil {
		ctxt.Errorf("updating %v %v: %v", repo, hash, err)
		return ""
	}

	if r.Next == nil {
		ctxt.Errorf("leaving todo for %s %s - no next yet", repo, hash)
		return ""
	}

	success := true
	forward := false
	for _, next := range r.Next {
		err := addTodo(ctxt, repo, r.Branch, next)
		if err == errDone {
			forward = true
			continue
		}
		if err == errBranched {
			ctxt.Infof("%v -> %v is a branch", r.Hash[:12], next[:12])
			continue
		}
		if err != nil {
			ctxt.Errorf("storing todo for %s %s: %v %p %p", repo, next, err, err, errDone)
			success = false
		}
		forward = true // innocent until proven guilty
		if nextHash == "" {
			nextHash = next
		} else {
			laterLoadRev.Call(ctxt, repo, r.Branch, next)
		}
	}

	if forward && success {
		ctxt.Infof("delete todo %s\n", todoKey)
		app.DeleteData(ctxt, "RevTodo", todoKey)
	} else {
		ctxt.Errorf("leaving todo for %s %s due to errors or branching", repo, hash)
	}

	return nextHash
}