func sendJabberNotification(ctx httputil.Context, u string, test entities.TestResult) (err error) {
//exit early if we have no url
if u == "" {
return
}
//make sure we have configuration values
if Config.Username == "" || Config.Domain == "" || Config.Password == "" {
ctx.Errorf("Unable to send notification (%s): configuration not specified", test.ImportPath)
return
}
ctx.Infof("Send jabber notification (%s): %s", test.ImportPath, u)
//open a tcp connection to the jabber server
netConn, err := net.Dial("tcp", "talk.google.com:5222")
if err != nil {
return
}
defer netConn.Close()
//use that connection in the xmpp config and dial out
config := &xmpp.Config{Conn: netConn}
conn, err := xmpp.Dial("", Config.Username, Config.Domain, Config.Password, config)
if err != nil {
return
}
//send off the message
message := fmt.Sprintf("%s @ %s status is now %s", test.ImportPath, test.Revision, test.Status)
err = conn.Send(u, message)
return
}
//dispatchWork is the handler that gets called for a queue item. It grabs a builder
//and runner and dispatches the work item to them, recoding when that operation
//started.
func dispatchWork(w http.ResponseWriter, req *http.Request, ctx httputil.Context) (e *httputil.Error) {
//find all the documents that are waiting or (processing and their attempt is
//taking too long)
type L []interface{}
selector := bson.M{
"$or": L{
bson.M{"status": entities.WorkStatusWaiting},
bson.M{
"status": entities.WorkStatusProcessing,
"attemptlog.0.when": bson.M{"$lt": time.Now().Add(-1 * attemptTime)},
},
},
}
iter := ctx.DB.C("Work").Find(selector).Iter()
var work entities.Work
for iter.Next(&work) {
//if its a processing task with too many attempts, store it as a dispatch
//error.
if len(work.AttemptLog) >= maxAttempts {
ctx.Infof("Work item %s had too many attempts", work.ID)
args := &rpc.DispatchResponse{
Key: work.ID.Hex(),
Error: "Unable to complete Work item. Too many failed attempts.",
WorkRev: work.Revision,
}
//send it off to the response rpc
respUrl := httputil.Absolute(router.Lookup("Response"))
cl := client.New(respUrl, http.DefaultClient, client.JsonCodec)
if err := cl.Call("Response.DispatchError", args, new(rpc.None)); err != nil {
ctx.Infof("Couldn't store a dispatch error for work item %s: %s", work.ID, err)
}
continue
}
//attempt to dispatch the work item
err := dispatchWorkItem(ctx, work)
if err != nil {
ctx.Errorf("Error dispatching work: %s", err)
}
}
//check for errors running the iteration
if err := iter.Err(); err != nil {
ctx.Errorf("Error iterating over work items: %s", err)
e = httputil.Errorf(err, "Error iterating over work items")
return
}
return
}
func sendUrlNotification(ctx httputil.Context, u string, test entities.TestResult) (err error) {
//exit early if we have no url
if u == "" {
return
}
ctx.Infof("Send url notification (%s): %s", test.ImportPath, u)
//set up the json payload
var buf bytes.Buffer
if err = json.NewEncoder(&buf).Encode(test); err != nil {
return
}
//send off the request
_, err = http.Post(u, "application/json", &buf)
return
}
//LeasePair returns a pair of Builder and Runners that can be used to run tests.
//It doesn't let you specify the type of runner you want.
func LeasePair(ctx httputil.Context) (b *Builder, r *Runner, err error) {
//grab a runner
r, err = getRunner(ctx, "", "")
if err != nil {
ctx.Infof("couldn't lease runner")
return
}
//update the key we're using
lastSeeds.set("", "", "Runner", r.Seed)
//grab a builder than can make a build for this runner
b, err = getBuilder(ctx, r.GOOS, r.GOARCH)
if err != nil {
ctx.Infof("couldn't lease builder")
return
}
//update the key we're using
lastSeeds.set(r.GOOS, r.GOARCH, "Builder", b.Seed)
return
}
//getService is a helper function that abstracts the logic of grabbing a service
//with a key greater than the one given, and looping back to zero if one wasn't
//found.
func getService(ctx httputil.Context, GOOS, GOARCH, Type string, s interface{}) (err error) {
//grab the most recent run key
seed := lastSeeds.get(GOOS, GOARCH, Type)
again:
ctx.Infof("Finding a %v/%v/%v [%d]", Type, GOOS, GOARCH, seed)
//run the query
query := baseQuery(ctx.DB, GOOS, GOARCH, Type, seed)
err = query.One(s)
//if we didn't find a match
if err == mgo.ErrNotFound {
//try again if we're limiting on the seed
if seed > 0 {
seed = 0
goto again
}
//there just arent any
err = ErrNoneAvailable
}
return
}
func dispatchNotifications(w http.ResponseWriter, req *http.Request, ctx httputil.Context) (e *httputil.Error) {
//find all documents that are waiting or (processing and their attempt is
//taking too long)
type L []interface{}
selector := bson.M{
"$or": L{
bson.M{"status": entities.NotifStatusWaiting},
bson.M{
"status": entities.NotifStatusProcessing,
"attemptlog.0.when": bson.M{"$lt": time.Now().Add(-1 * attemptTime)},
},
},
}
iter := ctx.DB.C("Notification").Find(selector).Iter()
var n entities.Notification
for iter.Next(&n) {
//if it's processing with too may attempts then just give up
if len(n.AttemptLog) >= maxAttempts {
ctx.Infof("Notification %s had too many attempts", n.ID)
ops := []txn.Op{{
C: "Notification",
Id: n.ID,
Assert: bson.M{
"status": entities.NotifStatusProcessing,
"revision": n.Revision,
},
Update: bson.M{
"$set": bson.M{"status": entities.NotifStatusError},
"$inc": bson.M{"revision": 1},
},
}}
//try to update the notification
err := ctx.R.Run(ops, bson.NewObjectId(), nil)
if err == txn.ErrAborted {
ctx.Infof("Lost race updating notification %s", n.ID)
err = nil
}
if err != nil {
ctx.Errorf("Error updating notification %s: %s", n.ID, err)
}
continue
}
err := dispatchNotificationItem(ctx, &n)
if err != nil {
ctx.Errorf("Error processing notification %s: %s", n.ID, err)
continue
}
//update the thing as being done
ops := []txn.Op{{
C: "Notification",
Id: n.ID,
Assert: bson.M{
"revision": n.Revision,
},
Update: bson.M{
"$inc": bson.M{"revision": 1},
"$set": bson.M{"status": entities.NotifStatusCompleted},
},
}}
err = ctx.R.Run(ops, bson.NewObjectId(), nil)
if err == txn.ErrAborted {
ctx.Infof("Lost the race setting the notification %s to complete", n.ID)
err = nil
}
if err != nil {
ctx.Errorf("Error setting notification %s to complete: %s", n.ID, err)
}
}
//check for errors in the iteration
if err := iter.Err(); err != nil {
ctx.Errorf("Error iterating over notifications: %s", err)
e = httputil.Errorf(err, "Error iterating over notifications")
return
}
return
}
func dispatchNotificationItem(ctx httputil.Context, n *entities.Notification) (err error) {
//create an attempt for this notification
a := entities.NotifAttempt{
When: time.Now(),
ID: bson.NewObjectId(),
}
//push it to the start
log := append([]entities.NotifAttempt{a}, n.AttemptLog...)
//transactionally acquire ownership of this notification
ops := []txn.Op{{
C: "Notification",
Id: n.ID,
Assert: bson.M{
"revision": n.Revision,
},
Update: bson.M{
"$inc": bson.M{"revision": 1},
"$set": bson.M{
"attemptlog": log,
"status": entities.NotifStatusProcessing,
},
},
}}
err = ctx.R.Run(ops, bson.NewObjectId(), nil)
if err == txn.ErrAborted {
ctx.Infof("Lost the race dispatching a notification")
err = nil
return
}
if err != nil {
return
}
//inc the revision locally
n.Revision++
//try to load up the last two test results to see if there was a delta.
var test entities.TestResult
err = ctx.DB.C("TestResult").FindId(n.Test).One(&test)
if err != nil {
return
}
//attempt to grab the test previous to it
var oneResult bool
var prev entities.TestResult
query := bson.M{
"revdate": bson.M{"$lt": test.RevDate},
}
err = ctx.DB.C("TestResult").Find(query).Sort("-revdate").One(&prev)
if err == mgo.ErrNotFound {
err = nil
oneResult = true
}
if err != nil {
return
}
//figure out if we meet the conditions to notify
var perform bool
switch strings.ToLower(n.Config.NotifyOn) {
case "pass":
perform = test.Status == "Pass"
case "fail":
perform = test.Status == "Fail"
case "error":
perform = test.Status == "Error"
case "wontbuild":
perform = test.Status == "WontBuild"
case "problem":
perform = false ||
test.Status == "Fail" ||
test.Status == "Error" ||
test.Status == "WontBuild"
case "always":
perform = true
case "change":
perform = !oneResult && test.Status != prev.Status
}
//if we have nothing to perform, we're done
if !perform {
return
}
//do the url and jabber concurrently
errs := make(chan error)
go func() { errs <- sendUrlNotification(ctx, n.Config.NotifyURL, test) }()
go func() { errs <- sendJabberNotification(ctx, n.Config.NotifyJabber, test) }()
//store the errors from it
var me multiError
for i := 0; i < 2; i++ {
me = append(me, <-errs)
}
if !me.isNil() {
err = me
}
return
}
func dispatchWorkItem(ctx httputil.Context, work entities.Work) (err error) {
//lease a builder and runner
builder, runner, err := tracker.LeasePair(ctx)
if err != nil {
return
}
log.Printf("Got:\nBuilder: %+v\nRunner: %+v", builder, runner)
//create an attempt
a := entities.WorkAttempt{
When: time.Now(),
Builder: builder.URL,
Runner: runner.URL,
ID: bson.NewObjectId(),
}
//push the new attempt at the start of the array
log := append([]entities.WorkAttempt{a}, work.AttemptLog...)
//transactionally acquire ownership of the document
ops := []txn.Op{{
C: "Work",
Id: work.ID,
Assert: bson.M{
"revision": work.Revision,
},
Update: bson.M{
"$inc": bson.M{"revision": 1},
"$set": bson.M{
"attemptlog": log,
"status": entities.WorkStatusProcessing,
},
},
}}
err = ctx.R.Run(ops, bson.NewObjectId(), nil)
if err == txn.ErrAborted {
ctx.Infof("Lost the race dispatching a work item")
err = nil
return
}
if err != nil {
return
}
//build the task
task := &rpc.BuilderTask{
Work: work.Work,
Key: work.ID.Hex(),
ID: a.ID.Hex(),
WorkRev: work.Revision + 1,
Runner: runner.URL,
Response: httputil.Absolute(router.Lookup("Response")),
}
//send the task off to the builder queue
cl := client.New(builder.URL, http.DefaultClient, client.JsonCodec)
err = cl.Call("BuilderQueue.Push", task, new(rpc.None))
return
}