// commitHandler retrieves commit data or records a new commit.
//
// For GET requests it returns a Commit value for the specified
// packagePath and hash.
//
// For POST requests it reads a JSON-encoded Commit value from the request
// body and creates a new Commit entity. It also updates the "tip" Tag for
// each new commit at tip.
//
// This handler is used by a gobuilder process in -commit mode.
func commitHandler(r *http.Request) (interface{}, error) {
c := appengine.NewContext(r)
com := new(Commit)
if r.Method == "GET" {
com.PackagePath = r.FormValue("packagePath")
com.Hash = r.FormValue("hash")
if err := datastore.Get(c, com.Key(c), com); err != nil {
return nil, fmt.Errorf("getting Commit: %v", err)
}
return com, nil
}
if r.Method != "POST" {
return nil, errBadMethod(r.Method)
}
// POST request
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(com); err != nil {
return nil, fmt.Errorf("decoding Body: %v", err)
}
if len(com.Desc) > maxDatastoreStringLen {
com.Desc = com.Desc[:maxDatastoreStringLen]
}
if err := com.Valid(); err != nil {
return nil, fmt.Errorf("validating Commit: %v", err)
}
defer cache.Tick(c)
tx := func(c appengine.Context) error {
return addCommit(c, com)
}
return nil, datastore.RunInTransaction(c, tx, nil)
}
// perfResultHandler records a becnhmarking result.
func perfResultHandler(r *http.Request) (interface{}, error) {
defer r.Body.Close()
if r.Method != "POST" {
return nil, errBadMethod(r.Method)
}
req := new(PerfRequest)
if err := json.NewDecoder(r.Body).Decode(req); err != nil {
return nil, fmt.Errorf("decoding Body: %v", err)
}
c := contextForRequest(r)
defer cache.Tick(c)
// store the text files if supplied
for i, a := range req.Artifacts {
hash, err := PutLog(c, a.Body)
if err != nil {
return nil, fmt.Errorf("putting Log: %v", err)
}
req.Artifacts[i].Body = hash
}
tx := func(c appengine.Context) error {
return addPerfResult(c, r, req)
}
return nil, datastore.RunInTransaction(c, tx, nil)
}
func initHandler(w http.ResponseWriter, r *http.Request) {
d := dashboardForRequest(r)
c := d.Context(appengine.NewContext(r))
defer cache.Tick(c)
for _, p := range d.Packages {
err := datastore.Get(c, p.Key(c), new(Package))
if _, ok := err.(*datastore.ErrFieldMismatch); ok {
// Some fields have been removed, so it's okay to ignore this error.
err = nil
}
if err == nil {
continue
} else if err != datastore.ErrNoSuchEntity {
logErr(w, r, err)
return
}
if _, err := datastore.Put(c, p.Key(c), p); err != nil {
logErr(w, r, err)
return
}
}
// Create secret key.
key.Secret(c)
fmt.Fprint(w, "OK")
}
// resultHandler records a build result.
// It reads a JSON-encoded Result value from the request body,
// creates a new Result entity, and updates the relevant Commit entity.
// If the Log field is not empty, resultHandler creates a new Log entity
// and updates the LogHash field before putting the Commit entity.
func resultHandler(r *http.Request) (interface{}, error) {
if r.Method != "POST" {
return nil, errBadMethod(r.Method)
}
// For now, the gccgo builders are using the old stuff.
// TODO(adg,cmang): remove this exception when gccgo is updated.
if dashboardForRequest(r) != gccgoDash {
v, _ := strconv.Atoi(r.FormValue("version"))
if v != builderVersion {
return nil, fmt.Errorf("rejecting POST from builder; need version %v", builderVersion)
}
}
c := contextForRequest(r)
res := new(Result)
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(res); err != nil {
return nil, fmt.Errorf("decoding Body: %v", err)
}
if err := res.Valid(); err != nil {
return nil, fmt.Errorf("validating Result: %v", err)
}
defer cache.Tick(c)
// store the Log text if supplied
if len(res.Log) > 0 {
hash, err := PutLog(c, res.Log)
if err != nil {
return nil, fmt.Errorf("putting Log: %v", err)
}
res.LogHash = hash
}
tx := func(c appengine.Context) error {
// check Package exists
if _, err := GetPackage(c, res.PackagePath); err != nil {
return fmt.Errorf("GetPackage: %v", err)
}
// put Result
if _, err := datastore.Put(c, res.Key(c), res); err != nil {
return fmt.Errorf("putting Result: %v", err)
}
// add Result to Commit
com := &Commit{PackagePath: res.PackagePath, Hash: res.Hash}
if err := com.AddResult(c, res); err != nil {
return fmt.Errorf("AddResult: %v", err)
}
// Send build failure notifications, if necessary.
// Note this must run after the call AddResult, which
// populates the Commit's ResultData field.
return notifyOnFailure(c, com, res.Builder)
}
return nil, datastore.RunInTransaction(c, tx, nil)
}
func initHandler(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
defer cache.Tick(c)
for _, p := range defaultPackages {
if err := datastore.Get(c, p.Key(c), new(Package)); err == nil {
continue
} else if err != datastore.ErrNoSuchEntity {
logErr(w, r, err)
return
}
if _, err := datastore.Put(c, p.Key(c), p); err != nil {
logErr(w, r, err)
return
}
}
fmt.Fprint(w, "OK")
}
func initHandler(w http.ResponseWriter, r *http.Request) {
d := dashboardForRequest(r)
c := d.Context(appengine.NewContext(r))
defer cache.Tick(c)
for _, p := range d.Packages {
err := datastore.Get(c, p.Key(c), new(Package))
if _, ok := err.(*datastore.ErrFieldMismatch); ok {
// Some fields have been removed, so it's okay to ignore this error.
err = nil
}
if err == nil {
continue
} else if err != datastore.ErrNoSuchEntity {
logErr(w, r, err)
return
}
p.NextNum = 1 // So we can add the first commit.
if _, err := datastore.Put(c, p.Key(c), p); err != nil {
logErr(w, r, err)
return
}
}
// Create secret key.
key.Secret(c)
// Create dummy config values.
initConfig(c)
// Populate Go 1.4 tag. This is for bootstrapping the new feature of
// building sub-repos against the stable release.
// TODO(adg): remove this after Go 1.5 is released, at which point the
// build system will pick up on the new release tag automatically.
t := &Tag{
Kind: "release",
Name: "release-branch.go1.4",
Hash: "883bc6ed0ea815293fe6309d66f967ea60630e87", // Go 1.4.2
}
if _, err := datastore.Put(c, t.Key(c), t); err != nil {
logErr(w, r, err)
return
}
fmt.Fprint(w, "OK")
}
// commitHandler retrieves commit data or records a new commit.
//
// For GET requests it returns a Commit value for the specified
// packagePath and hash.
//
// For POST requests it reads a JSON-encoded Commit value from the request
// body and creates a new Commit entity. It also updates the "tip" Tag for
// each new commit at tip.
//
// This handler is used by a gobuilder process in -commit mode.
func commitHandler(r *http.Request) (interface{}, error) {
c := contextForRequest(r)
com := new(Commit)
if r.Method == "GET" {
com.PackagePath = r.FormValue("packagePath")
com.Hash = r.FormValue("hash")
if err := datastore.Get(c, com.Key(c), com); err != nil {
return nil, fmt.Errorf("getting Commit: %v", err)
}
// Strip potentially large and unnecessary fields.
com.ResultData = nil
com.PerfResults = nil
return com, nil
}
if r.Method != "POST" {
return nil, errBadMethod(r.Method)
}
if !isMasterKey(c, r.FormValue("key")) {
return nil, errors.New("can only POST commits with master key")
}
// POST request
body, err := ioutil.ReadAll(r.Body)
r.Body.Close()
if err != nil {
return nil, fmt.Errorf("reading Body: %v", err)
}
if !bytes.Contains(body, needsBenchmarkingBytes) {
c.Warningf("old builder detected at %v", r.RemoteAddr)
return nil, fmt.Errorf("rejecting old builder request, body does not contain %s: %q", needsBenchmarkingBytes, body)
}
if err := json.Unmarshal(body, com); err != nil {
return nil, fmt.Errorf("unmarshaling body %q: %v", body, err)
}
com.Desc = limitStringLength(com.Desc, maxDatastoreStringLen)
if err := com.Valid(); err != nil {
return nil, fmt.Errorf("validating Commit: %v", err)
}
defer cache.Tick(c)
tx := func(c appengine.Context) error {
return addCommit(c, com)
}
return nil, datastore.RunInTransaction(c, tx, nil)
}
// tagHandler records a new tag. It reads a JSON-encoded Tag value from the
// request body and updates the Tag entity for the Kind of tag provided.
//
// This handler is used by a gobuilder process in -commit mode.
func tagHandler(r *http.Request) (interface{}, error) {
if r.Method != "POST" {
return nil, errBadMethod(r.Method)
}
t := new(Tag)
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(t); err != nil {
return nil, err
}
if err := t.Valid(); err != nil {
return nil, err
}
c := contextForRequest(r)
defer cache.Tick(c)
_, err := datastore.Put(c, t.Key(c), t)
return nil, err
}
// clearResultsHandler purges the last commitsPerPage results for the given builder.
// It optionally takes a comma-separated list of specific hashes to clear.
func clearResultsHandler(r *http.Request) (interface{}, error) {
if r.Method != "POST" {
return nil, errBadMethod(r.Method)
}
builder := r.FormValue("builder")
if builder == "" {
return nil, errors.New("must specify a builder")
}
clearAll := r.FormValue("hash") == ""
hash := strings.Split(r.FormValue("hash"), ",")
c := contextForRequest(r)
defer cache.Tick(c)
pkg := (&Package{}).Key(c) // TODO(adg): support clearing sub-repos
err := datastore.RunInTransaction(c, func(c appengine.Context) error {
var coms []*Commit
keys, err := datastore.NewQuery("Commit").
Ancestor(pkg).
Order("-Num").
Limit(commitsPerPage).
GetAll(c, &coms)
if err != nil {
return err
}
var rKeys []*datastore.Key
for _, com := range coms {
if !(clearAll || contains(hash, com.Hash)) {
continue
}
r := com.Result(builder, "")
if r == nil {
continue
}
com.RemoveResult(r)
rKeys = append(rKeys, r.Key(c))
}
_, err = datastore.PutMulti(c, keys, coms)
if err != nil {
return err
}
return datastore.DeleteMulti(c, rKeys)
}, nil)
return nil, err
}
// resultHandler records a build result.
// It reads a JSON-encoded Result value from the request body,
// creates a new Result entity, and updates the relevant Commit entity.
// If the Log field is not empty, resultHandler creates a new Log entity
// and updates the LogHash field before putting the Commit entity.
func resultHandler(r *http.Request) (interface{}, error) {
if r.Method != "POST" {
return nil, errBadMethod(r.Method)
}
c := contextForRequest(r)
res := new(Result)
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(res); err != nil {
return nil, fmt.Errorf("decoding Body: %v", err)
}
if err := res.Valid(); err != nil {
return nil, fmt.Errorf("validating Result: %v", err)
}
defer cache.Tick(c)
// store the Log text if supplied
if len(res.Log) > 0 {
hash, err := PutLog(c, res.Log)
if err != nil {
return nil, fmt.Errorf("putting Log: %v", err)
}
res.LogHash = hash
}
tx := func(c appengine.Context) error {
// check Package exists
if _, err := GetPackage(c, res.PackagePath); err != nil {
return fmt.Errorf("GetPackage: %v", err)
}
// put Result
if _, err := datastore.Put(c, res.Key(c), res); err != nil {
return fmt.Errorf("putting Result: %v", err)
}
// add Result to Commit
com := &Commit{PackagePath: res.PackagePath, Hash: res.Hash}
if err := com.AddResult(c, res); err != nil {
return fmt.Errorf("AddResult: %v", err)
}
// Send build failure notifications, if necessary.
// Note this must run after the call AddResult, which
// populates the Commit's ResultData field.
return notifyOnFailure(c, com, res.Builder)
}
return nil, datastore.RunInTransaction(c, tx, nil)
}
// UpdatePerfConfig updates the PerfConfig entity with results of benchmarking.
// Returns whether it's a benchmark that we have not yet seem on the builder.
func UpdatePerfConfig(c appengine.Context, r *http.Request, req *PerfRequest) (newBenchmark bool, err error) {
pc, err := GetPerfConfig(c, r)
if err != nil {
return false, err
}
modified := false
add := func(arr *[]string, str string) {
for _, s := range *arr {
if s == str {
return
}
}
*arr = append(*arr, str)
modified = true
return
}
BenchProcs := strings.Split(req.Benchmark, "-")
benchmark := BenchProcs[0]
procs := "1"
if len(BenchProcs) > 1 {
procs = BenchProcs[1]
}
add(&pc.BuilderBench, req.Builder+"|"+benchmark)
newBenchmark = modified
add(&pc.BuilderProcs, req.Builder+"|"+procs)
for _, m := range req.Metrics {
add(&pc.BenchMetric, benchmark+"|"+m.Type)
}
if modified {
if _, err := datastore.Put(c, PerfConfigKey(c), pc); err != nil {
return false, fmt.Errorf("putting PerfConfig: %v", err)
}
cache.Tick(c)
}
return newBenchmark, nil
}
// commitHandler retrieves commit data or records a new commit.
//
// For GET requests it returns a Commit value for the specified
// packagePath and hash.
//
// For POST requests it reads a JSON-encoded Commit value from the request
// body and creates a new Commit entity. It also updates the "tip" Tag for
// each new commit at tip.
//
// This handler is used by a gobuilder process in -commit mode.
func commitHandler(r *http.Request) (interface{}, error) {
c := contextForRequest(r)
com := new(Commit)
if r.Method == "GET" {
com.PackagePath = r.FormValue("packagePath")
com.Hash = r.FormValue("hash")
if err := datastore.Get(c, com.Key(c), com); err != nil {
return nil, fmt.Errorf("getting Commit: %v", err)
}
// Strip potentially large and unnecessary fields.
com.ResultData = nil
com.PerfResults = nil
return com, nil
}
if r.Method != "POST" {
return nil, errBadMethod(r.Method)
}
if !isMasterKey(c, r.FormValue("key")) {
return nil, errors.New("can only POST commits with master key")
}
// POST request
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(com); err != nil {
return nil, fmt.Errorf("decoding Body: %v", err)
}
com.Desc = limitStringLength(com.Desc, maxDatastoreStringLen)
if err := com.Valid(); err != nil {
return nil, fmt.Errorf("validating Commit: %v", err)
}
defer cache.Tick(c)
tx := func(c appengine.Context) error {
return addCommit(c, com)
}
return nil, datastore.RunInTransaction(c, tx, nil)
}
// commitHandler retrieves commit data or records a new commit.
//
// For GET requests it returns a Commit value for the specified
// packagePath and hash.
//
// For POST requests it reads a JSON-encoded Commit value from the request
// body and creates a new Commit entity. It also updates the "tip" Tag for
// each new commit at tip.
//
// This handler is used by a gobuilder process in -commit mode.
func commitHandler(r *http.Request) (interface{}, error) {
c := contextForRequest(r)
com := new(Commit)
if r.Method == "GET" {
com.PackagePath = r.FormValue("packagePath")
com.Hash = r.FormValue("hash")
err := datastore.Get(c, com.Key(c), com)
if com.Num == 0 && com.Desc == "" {
// Perf builder might have written an incomplete Commit.
// Pretend it doesn't exist, so that we can get complete details.
err = datastore.ErrNoSuchEntity
}
if err != nil {
if err == datastore.ErrNoSuchEntity {
// This error string is special.
// The commit watcher expects it.
// Do not change it.
return nil, errors.New("Commit not found")
}
return nil, fmt.Errorf("getting Commit: %v", err)
}
if com.Num == 0 {
// Corrupt state which shouldn't happen but does.
// Return an error so builders' commit loops will
// be willing to retry submitting this commit.
return nil, errors.New("in datastore with zero Num")
}
if com.Desc == "" || com.User == "" {
// Also shouldn't happen, but at least happened
// once on a single commit when trying to fix data
// in the datastore viewer UI?
return nil, errors.New("missing field")
}
// Strip potentially large and unnecessary fields.
com.ResultData = nil
com.PerfResults = nil
return com, nil
}
if r.Method != "POST" {
return nil, errBadMethod(r.Method)
}
if !isMasterKey(c, r.FormValue("key")) {
return nil, errors.New("can only POST commits with master key")
}
// For now, the commit watcher doesn't support gccgo.
// TODO(adg,cmang): remove this exception when gccgo is supported.
if dashboardForRequest(r) != gccgoDash {
v, _ := strconv.Atoi(r.FormValue("version"))
if v != watcherVersion {
return nil, fmt.Errorf("rejecting POST from commit watcher; need version %v", watcherVersion)
}
}
// POST request
body, err := ioutil.ReadAll(r.Body)
r.Body.Close()
if err != nil {
return nil, fmt.Errorf("reading Body: %v", err)
}
if !bytes.Contains(body, needsBenchmarkingBytes) {
c.Warningf("old builder detected at %v", r.RemoteAddr)
return nil, fmt.Errorf("rejecting old builder request, body does not contain %s: %q", needsBenchmarkingBytes, body)
}
if err := json.Unmarshal(body, com); err != nil {
return nil, fmt.Errorf("unmarshaling body %q: %v", body, err)
}
com.Desc = limitStringLength(com.Desc, maxDatastoreStringLen)
if err := com.Valid(); err != nil {
return nil, fmt.Errorf("validating Commit: %v", err)
}
defer cache.Tick(c)
tx := func(c appengine.Context) error {
return addCommit(c, com)
}
return nil, datastore.RunInTransaction(c, tx, nil)
}
