func status(ctxt appengine.Context) string {
w := new(bytes.Buffer)
var count int64
app.ReadMeta(ctxt, "issue.count", &count)
fmt.Fprintln(w, time.Now())
var t1 string
app.ReadMeta(ctxt, "issue.mtime", &t1)
fmt.Fprintf(w, "issue modifications up to %v\n", t1)
fmt.Fprintln(w, time.Now())
return "<pre>" + html.EscapeString(w.String()) + "</pre>"
}
func writeCL(ctxt appengine.Context, cl *CL, mtimeKey, modified string) error {
err := app.Transaction(ctxt, func(ctxt appengine.Context) error {
var old CL
if err := app.ReadData(ctxt, "CL", cl.CL, &old); err != nil && err != datastore.ErrNoSuchEntity {
return err
}
if old.CL == "" { // no old data
var count int64
app.ReadMeta(ctxt, "codereview.count", &count)
app.WriteMeta(ctxt, "codereview.count", count+1)
}
// Copy CL into original structure.
// This allows us to maintain other information in the CL structure
// and not overwrite it when the Rietveld information is updated.
if cl.Dead {
old.Dead = true
} else {
old.Dead = false
if old.Modified.After(cl.Modified) {
return fmt.Errorf("CL %v: have %v but Rietveld sent %v", cl.CL, old.Modified, cl.Modified)
}
old.CL = cl.CL
old.Desc = cl.Desc
old.Owner = cl.Owner
old.OwnerEmail = cl.OwnerEmail
old.Created = cl.Created
old.Modified = cl.Modified
old.MessagesLoaded = cl.MessagesLoaded
if cl.MessagesLoaded {
old.Messages = cl.Messages
old.Submitted = cl.Submitted
}
old.Reviewers = cl.Reviewers
old.CC = cl.CC
old.Closed = cl.Closed
if !reflect.DeepEqual(old.PatchSets, cl.PatchSets) {
old.PatchSets = cl.PatchSets
old.PatchSetsLoaded = false
}
}
if err := app.WriteData(ctxt, "CL", cl.CL, &old); err != nil {
return err
}
if mtimeKey != "" {
app.WriteMeta(ctxt, mtimeKey, modified)
}
return nil
})
if err != nil {
ctxt.Errorf("storing CL %v: %v", cl.CL, err)
}
return err
}
func oauthConfig(ctxt appengine.Context) (*oauth.Config, error) {
var clientID, clientSecret string
if err := app.ReadMeta(ctxt, "googleapi.clientid", &clientID); err != nil {
return nil, err
}
if err := app.ReadMeta(ctxt, "googleapi.clientsecret", &clientSecret); err != nil {
return nil, err
}
cfg := &oauth.Config{
ClientId: clientID,
ClientSecret: clientSecret,
Scope: "https://code.google.com/feeds/issues",
AuthURL: "https://accounts.google.com/o/oauth2/auth",
TokenURL: "https://accounts.google.com/o/oauth2/token",
RedirectURL: "https://go-dev.appspot.com/codetoken",
AccessType: "offline",
}
return cfg, nil
}
func postIssueComment(ctxt appengine.Context, id, comment string) error {
cfg, err := oauthConfig(ctxt)
if err != nil {
return fmt.Errorf("oauthconfig: %v", err)
}
var tok oauth.Token
if err := app.ReadMeta(ctxt, "codelogin.token", &tok); err != nil {
return fmt.Errorf("reading token: %v", err)
}
tr := &oauth.Transport{
Config: cfg,
Token: &tok,
Transport: urlfetch.Client(ctxt).Transport,
}
client := tr.Client()
var buf bytes.Buffer
buf.WriteString(`<?xml version='1.0' encoding='UTF-8'?>
<entry xmlns='http://www.w3.org/2005/Atom' xmlns:issues='http://schemas.google.com/projecthosting/issues/2009'>
<content type='html'>`)
xml.Escape(&buf, []byte(comment))
buf.WriteString(`</content>
<author>
<name>ignored</name>
</author>
<issues:updates>
</issues:updates>
</entry>
`)
u := "https://code.google.com/feeds/issues/p/go/issues/" + id + "/comments/full"
req, err := http.NewRequest("POST", u, &buf)
if err != nil {
return fmt.Errorf("write: %v", err)
}
req.Header.Set("Content-Type", "application/atom+xml")
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("write: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != 201 {
buf.Reset()
io.Copy(&buf, resp.Body)
return fmt.Errorf("write: %v\n%s", resp.Status, buf.String())
}
return nil
}
func writeIssue(ctxt appengine.Context, issue *Issue, stateKey string, state interface{}) error {
err := app.Transaction(ctxt, func(ctxt appengine.Context) error {
var old Issue
if err := app.ReadData(ctxt, "Issue", fmt.Sprint(issue.ID), &old); err != nil && err != datastore.ErrNoSuchEntity {
return err
}
if old.ID == 0 { // no old data
var count int64
app.ReadMeta(ctxt, "issue.count", &count)
app.WriteMeta(ctxt, "issue.count", count+1)
}
if old.Modified.After(issue.Modified) {
return fmt.Errorf("issue %v: have %v but code.google.com sent %v", issue.ID, old.Modified, issue.Modified)
}
// Copy Issue into original structure.
// This allows us to maintain other information in the Issue structure
// and not overwrite it when the issue information is updated.
old.ID = issue.ID
old.Summary = issue.Summary
old.Status = issue.Status
old.Duplicate = issue.Duplicate
old.Owner = issue.Owner
old.CC = issue.CC
old.Label = issue.Label
old.Comment = issue.Comment
old.State = issue.State
old.Created = issue.Created
old.Modified = issue.Modified
old.Stars = issue.Stars
old.ClosedDate = issue.ClosedDate
updateIssue(&old)
if err := app.WriteData(ctxt, "Issue", fmt.Sprint(issue.ID), &old); err != nil {
return err
}
if stateKey != "" {
app.WriteMeta(ctxt, stateKey, state)
}
return nil
})
if err != nil {
ctxt.Errorf("storing issue %v: %v", issue.ID, err)
}
return err
}
func codetoken(ctxt appengine.Context, w http.ResponseWriter, req *http.Request) {
var randState string
if err := app.ReadMeta(ctxt, "codelogin.random", &randState); err != nil {
panic(err)
}
cfg, err := oauthConfig(ctxt)
if err != nil {
http.Error(w, err.Error(), 500)
return
}
if req.FormValue("state") != randState {
http.Error(w, "bad state", 500)
return
}
code := req.FormValue("code")
if code == "" {
http.Error(w, "missing code", 500)
return
}
tr := &oauth.Transport{
Config: cfg,
Transport: urlfetch.Client(ctxt).Transport,
}
_, err = tr.Exchange(code)
if err != nil {
http.Error(w, "exchanging code: "+err.Error(), 500)
return
}
if err := app.WriteMeta(ctxt, "codelogin.token", tr.Token); err != nil {
http.Error(w, "writing token: "+err.Error(), 500)
return
}
app.DeleteMeta(ctxt, "codelogin.random")
fmt.Fprintf(w, "have token; expires at %v\n", tr.Token.Expiry)
}
func load(ctxt appengine.Context) error {
mtime := time.Date(2009, 1, 1, 0, 0, 0, 0, time.UTC)
if appengine.IsDevAppServer() {
mtime = time.Now().UTC().Add(-24 * time.Hour)
}
app.ReadMeta(ctxt, "issue.mtime", &mtime)
now := time.Now()
var issues []*Issue
var err error
const maxResults = 500
var try int
needMore := false
for try = 0; ; try++ {
issues, err = search(ctxt, "go", "all", "", false, mtime, now, maxResults)
if err != nil {
ctxt.Errorf("load issues since %v: %v", mtime, err)
return nil
}
if len(issues) == 0 {
ctxt.Infof("no updates found from %v to %v", mtime, now)
app.WriteMeta(ctxt, "issue.mtime", now.Add(-1*time.Minute))
if try > 0 {
// We shortened the time range; try again now that we've updated mtime.
return app.ErrMoreCron
}
return nil
}
if len(issues) < maxResults {
ctxt.Infof("%d issues from %v to %v", len(issues), mtime, now)
if try > 0 {
// Keep exploring once we finish this load.
needMore = true
}
break
}
ctxt.Errorf("updater found too many updates from %v to %v", mtime, now)
if now.Sub(mtime) <= 2*time.Second {
ctxt.Errorf("cannot shorten update time frame")
return nil
}
if now.Sub(mtime) > 1*time.Hour {
now = mtime.Add(now.Sub(mtime) / 10)
} else {
now = mtime.Add(now.Sub(mtime) / 2)
}
ctxt.Infof("shortened to %v to %v", mtime, now)
}
issues, err = search(ctxt, "go", "all", "", true, mtime, now, maxResults)
if err != nil {
ctxt.Errorf("full load of issues from %v to %v: %v", mtime, now, err)
return nil
}
if len(issues) == 0 {
ctxt.Errorf("unexpected: no issues from %v to %v", mtime, now)
return nil
}
for _, issue := range issues {
println("WRITE ISSUE", issue.ID)
if err := writeIssue(ctxt, issue, "", nil); err != nil {
return nil
}
if mtime.Before(issue.Modified) {
mtime = issue.Modified
}
}
if try > 0 {
mtime = now.Add(-1 * time.Second)
}
app.WriteMeta(ctxt, "issue.mtime", mtime.UTC())
if needMore {
return app.ErrMoreCron
}
return nil
}
func status(ctxt appengine.Context) string {
w := new(bytes.Buffer)
var count int64
for _, group := range []string{"golang-dev", "golang-codereviews"} {
for _, reviewerOrCC := range []string{"reviewer", "cc"} {
var t string
mtimeKey := "codereview.mtime." + reviewerOrCC + "." + group
app.ReadMeta(ctxt, mtimeKey, &t)
fmt.Fprintf(w, "%v last update for %s\n", t, mtimeKey)
}
}
app.ReadMeta(ctxt, "codereview.count", &count)
fmt.Fprintf(w, "%d CLs total\n", count)
var chunk = 20000
if appengine.IsDevAppServer() {
chunk = 100
}
q := datastore.NewQuery("CL").
Filter("PatchSetsLoaded <=", false).
KeysOnly().
Limit(chunk)
n := 0
it := q.Run(ctxt)
for {
_, err := it.Next(nil)
if err != nil {
break
}
n++
}
fmt.Fprintf(w, "%d with PatchSetsLoaded = false\n", n)
q = datastore.NewQuery("CL").
Filter("MessagesLoaded <=", false).
KeysOnly().
Limit(chunk)
n = 0
it = q.Run(ctxt)
for {
_, err := it.Next(nil)
if err != nil {
break
}
n++
}
fmt.Fprintf(w, "%d with MessagesLoaded = false\n", n)
fmt.Fprintf(w, "\n")
q = datastore.NewQuery("RevTodo").
Limit(10000).
KeysOnly()
n = 0
it = q.Run(ctxt)
for {
_, err := it.Next(nil)
if err != nil {
break
}
n++
}
fmt.Fprintf(w, "\n%d hg heads\n", n)
q = datastore.NewQuery("Meta").
Filter("__key__ >=", datastore.NewKey(ctxt, "Meta", "commit.count.", 0, nil)).
Filter("__key__ <=", datastore.NewKey(ctxt, "Meta", "commit.count/", 0, nil)).
Limit(100)
type meta struct {
JSON []byte `datastore:",noindex"`
}
it = q.Run(ctxt)
for {
var m meta
key, err := it.Next(&m)
if err != nil {
break
}
fmt.Fprintf(w, "%s %s\n", key.StringID(), m.JSON)
}
fmt.Fprintf(w, "\n")
q = datastore.NewQuery("CL").
Filter("Closed =", false).
Filter("Submitted =", false).
Filter("HasReviewers =", true).
Order("Summary").
KeysOnly().
Limit(20000)
n = 0
it = q.Run(ctxt)
for {
_, err := it.Next(nil)
if err != nil {
break
}
n++
}
fmt.Fprintf(w, "\n%d pending CLs.\n", n)
q = datastore.NewQuery("CL").
Filter("Active =", true).
Filter("NeedMailIssue >", "").
KeysOnly().
Limit(20000)
n = 0
it = q.Run(ctxt)
for {
_, err := it.Next(nil)
if err != nil {
break
}
n++
}
fmt.Fprintf(w, "\n%d CLs need issue mails.\n", n)
return "<pre>" + html.EscapeString(w.String()) + "</pre>\n"
}
func load(ctxt appengine.Context) error {
// The deadline for task invocation is 10 minutes.
// Stop when we've run for 5 minutes and ask to be rescheduled.
deadline := time.Now().Add(5 * time.Minute)
for _, group := range []string{"golang-dev", "golang-codereviews"} {
for _, reviewerOrCC := range []string{"reviewer", "cc"} {
// The stored mtime is the most recent modification time we've seen.
// We ask for all changes since then.
mtimeKey := "codereview.mtime." + reviewerOrCC + "." + group
var mtime string
if appengine.IsDevAppServer() {
mtime = "2013-12-01 00:00:00" // limit fetching in empty datastore
}
app.ReadMeta(ctxt, mtimeKey, &mtime)
cursor := ""
// Rietveld gives us back times with microseconds, but it rejects microseconds
// in the ModifiedAfter URL parameter. Drop them. We'll see a few of the most
// recent CLs again. No big deal.
if i := strings.Index(mtime, "."); i >= 0 {
mtime = mtime[:i]
}
const itemsPerPage = 100
for n := 0; ; n++ {
var q struct {
Cursor string `json:"cursor"`
Results []*jsonCL `json:"results"`
}
err := fetchJSON(ctxt, &q, urlWithParams(queryTmpl, map[string]string{
"ReviewerOrCC": reviewerOrCC,
"Group": group,
"ModifiedAfter": mtime,
"Order": "modified",
"Cursor": cursor,
"Limit": fmt.Sprint(itemsPerPage),
}))
if err != nil {
ctxt.Errorf("loading codereview by %s: URL <%s>: %v", reviewerOrCC, q, err)
break
}
ctxt.Infof("found %d CLs", len(q.Results))
if len(q.Results) == 0 {
break
}
cursor = q.Cursor
for _, jcl := range q.Results {
cl := jcl.toCL(ctxt)
if err := writeCL(ctxt, cl, mtimeKey, jcl.Modified); err != nil {
break // error already logged
}
}
if len(q.Results) < itemsPerPage {
ctxt.Infof("reached end of results - codereview by %s up to date", reviewerOrCC)
break
}
if time.Now().After(deadline) {
ctxt.Infof("more to do for codereview by %s - rescheduling", reviewerOrCC)
return app.ErrMoreCron
}
}
}
}
ctxt.Infof("all done")
return nil
}
func postMovedNote(ctxt appengine.Context, kind, id string) error {
var old Issue
if err := app.ReadData(ctxt, "Issue", id, &old); err != nil {
return err
}
updateIssue(&old)
if !old.NeedGithubNote {
err := app.Transaction(ctxt, func(ctxt appengine.Context) error {
var old Issue
if err := app.ReadData(ctxt, "Issue", id, &old); err != nil {
return err
}
old.NeedGithubNote = false
return app.WriteData(ctxt, "Issue", id, &old)
})
return err
}
cfg, err := oauthConfig(ctxt)
if err != nil {
return fmt.Errorf("oauthconfig: %v", err)
}
var tok oauth.Token
if err := app.ReadMeta(ctxt, "codelogin.token", &tok); err != nil {
return fmt.Errorf("reading token: %v", err)
}
tr := &oauth.Transport{
Config: cfg,
Token: &tok,
Transport: &urlfetch.Transport{Context: ctxt, Deadline: 45 * time.Second},
}
client := tr.Client()
status := ""
if old.State != "closed" {
status = "<issues:status>Moved</issues:status>"
}
var buf bytes.Buffer
buf.WriteString(`<?xml version='1.0' encoding='UTF-8'?>
<entry xmlns='http://www.w3.org/2005/Atom' xmlns:issues='http://schemas.google.com/projecthosting/issues/2009'>
<content type='html'>`)
xml.Escape(&buf, []byte(fmt.Sprintf("This issue has moved to https://golang.org/issue/%s\n", id)))
buf.WriteString(`</content>
<author>
<name>ignored</name>
</author>
<issues:sendEmail>False</issues:sendEmail>
<issues:updates>
<issues:label>IssueMoved</issues:label>
<issues:label>Restrict-AddIssueComment-Commit</issues:label>
` + status + `
</issues:updates>
</entry>
`)
u := "https://code.google.com/feeds/issues/p/go/issues/" + id + "/comments/full"
req, err := http.NewRequest("POST", u, &buf)
if err != nil {
return fmt.Errorf("write: %v", err)
}
req.Header.Set("Content-Type", "application/atom+xml")
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("write: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != 201 {
buf.Reset()
io.Copy(&buf, resp.Body)
return fmt.Errorf("write: %v\n%s", resp.Status, buf.String())
}
err = app.Transaction(ctxt, func(ctxt appengine.Context) error {
var old Issue
if err := app.ReadData(ctxt, "Issue", id, &old); err != nil {
return err
}
old.NeedGithubNote = false
old.Label = append(old.Label, "IssueMoved", "Restrict-AddIssueComment-Commit")
return app.WriteData(ctxt, "Issue", id, &old)
})
return err
}
func loadRevOnce(ctxt appengine.Context, repo, branch, hash string) (nextHash string) {
ctxt.Infof("load todo %s %s %s", repo, branch, hash)
// Check that this todo is still valid.
// If so, extend the expiry time so that no one else tries it while we do.
// This supercedes the usual use of app.Lock and app.Unlock and also
// provides a way to rate limit the polling.
todoKey := fmt.Sprintf("commit.todo.%s.%s", repo, hash)
err := app.Transaction(ctxt, func(ctxt appengine.Context) error {
var todo revTodo
if err := app.ReadData(ctxt, "RevTodo", todoKey, &todo); err != nil {
return err
}
if time.Now().Before(todo.Time) {
ctxt.Infof("poll %s %s not scheduled until %v", repo, hash, todo.Time)
return errWait
}
dtAll := todo.Time.Sub(todo.Start)
dtOne := todo.Time.Sub(todo.Last)
var dtMax time.Duration
if dtAll < 24*time.Hour {
dtMax = 5 * time.Minute
} else if dtAll < 7*24*time.Hour {
dtMax = 1 * time.Hour
} else {
dtMax = 24 * time.Hour
}
if dtOne *= 2; dtOne > dtMax {
dtOne = dtMax
} else if dtOne == 0 {
dtOne = 1 * time.Minute
}
todo.Last = time.Now()
todo.Time = todo.Last.Add(dtOne)
if err := app.WriteData(ctxt, "RevTodo", todoKey, &todo); err != nil {
return err
}
return nil
})
if err != nil {
ctxt.Errorf("skipping poll: %v", err)
return ""
}
r, err := fetchRev(ctxt, repo, hash)
if err != nil {
ctxt.Errorf("fetching %v %v: %v", repo, hash, err)
return ""
}
err = app.Transaction(ctxt, func(ctxt appengine.Context) error {
var old Rev
if err := app.ReadData(ctxt, "Rev", repo+"."+hash, &old); err != nil && err != datastore.ErrNoSuchEntity {
return err
}
if old.Hash == r.Hash && len(old.Next) == len(r.Next) {
// up to date
return nil
}
if old.Hash == "" { // no old data
var count int
if err := app.ReadMeta(ctxt, "commit.count."+repo, &count); err != nil && err != datastore.ErrNoSuchEntity {
return err
}
count++
old.Seq = count
if err := app.WriteMeta(ctxt, "commit.count."+repo, count); err != nil {
return err
}
if r.Branch != branch && len(r.Prev) == 1 {
ctxt.Infof("detected branch; forcing todo of parent")
err := writeTodo(ctxt, repo, branch, r.Prev[0], true)
if err != nil {
ctxt.Errorf("re-adding todo: %v", err)
}
nextHash = r.Prev[0]
}
}
old.Repo = r.Repo
old.Branch = r.Branch
// old.Seq already correct; r.Seq is not
old.Hash = r.Hash
old.ShortHash = old.Hash[:12]
old.Prev = r.Prev
old.Next = r.Next
old.Author = r.Author
old.AuthorEmail = r.AuthorEmail
old.Time = r.Time
old.Log = r.Log
old.Files = r.Files
if err := app.WriteData(ctxt, "Rev", repo+"."+hash, &old); err != nil {
return err
}
return nil
})
if err != nil {
ctxt.Errorf("updating %v %v: %v", repo, hash, err)
return ""
}
if r.Next == nil {
ctxt.Errorf("leaving todo for %s %s - no next yet", repo, hash)
return ""
}
success := true
forward := false
for _, next := range r.Next {
err := addTodo(ctxt, repo, r.Branch, next)
if err == errDone {
forward = true
continue
}
if err == errBranched {
ctxt.Infof("%v -> %v is a branch", r.Hash[:12], next[:12])
continue
}
if err != nil {
ctxt.Errorf("storing todo for %s %s: %v %p %p", repo, next, err, err, errDone)
success = false
}
forward = true // innocent until proven guilty
if nextHash == "" {
nextHash = next
} else {
laterLoadRev.Call(ctxt, repo, r.Branch, next)
}
}
if forward && success {
ctxt.Infof("delete todo %s\n", todoKey)
app.DeleteData(ctxt, "RevTodo", todoKey)
} else {
ctxt.Errorf("leaving todo for %s %s due to errors or branching", repo, hash)
}
return nextHash
}
