//New returns a new Runner ready to be Announced and run tests locally.
func New(runner, tracker, hosted string) *Runner {
n := &Runner{
tcl: client.New(tracker, http.DefaultClient, client.JsonCodec),
base: hosted,
runner: runner,
rpc: gorpc.NewServer(),
rq: rpc.NewRunnerQueue(),
resp: make(chan rpc.Output),
}
//register the run service in the rpc
if err := n.rpc.RegisterService(n.rq, ""); err != nil {
panic(err)
}
//register the pinger
if err := n.rpc.RegisterService(pinger.Pinger{}, ""); err != nil {
panic(err)
}
//register ourselves in the rpc
if err := n.rpc.RegisterService(n, ""); err != nil {
panic(err)
}
//register the codec
n.rpc.RegisterCodec(json.NewCodec(), "application/json")
//start processing
go n.run()
return n
}
func main() {
flag.Parse()
args := flag.Args()
if len(args) != 3 {
check(fmt.Errorf("usage: rpc <url> <method> <args>"), "args")
}
url, method, arg := args[0], args[1], args[2]
hcl := &http.Client{
Transport: rt{},
}
cl := client.New(url, hcl, client.JsonCodec)
var x, y interface{}
var err error
err = json.Unmarshal([]byte(arg), &x)
check(err, "unmar")
err = cl.Call(method, x, &y)
check(err, "call")
var b []byte
b, err = json.MarshalIndent(y, "", "\t")
check(err, "marsh")
fmt.Printf("%s\n", b)
}
//New returns a new Runner ready to be Announced and run tests on the
//heroku dyno grid.
func New(app, api string, tracker, hosted string) *Runner {
n := &Runner{
app: app,
api: api,
tcl: client.New(tracker, http.DefaultClient, client.JsonCodec),
base: hosted,
rpc: gorpc.NewServer(),
rq: rpc.NewRunnerQueue(),
mc: heroku.NewManaged(app, api, 2, 2*time.Minute),
tm: &runnerTaskMap{items: map[string]*runnerTask{}},
}
//register the run service in the rpc
if err := n.rpc.RegisterService(n.rq, ""); err != nil {
panic(err)
}
//register the pinger
if err := n.rpc.RegisterService(pinger.Pinger{}, ""); err != nil {
panic(err)
}
//register ourselves in the rpc
if err := n.rpc.RegisterService(n, ""); err != nil {
panic(err)
}
//register the codec
n.rpc.RegisterCodec(json.NewCodec(), "application/json")
//start processing
go n.run()
return n
}
//New returns a new web Builder ready to Announce to the given tracker. It
//announces that it is available at `hosted` which should be the full url of
//where this builder resides on the internet.
func New(b builder.Builder, tracker, hosted string) *Builder {
//create our new builder
n := &Builder{
b: b,
base: hosted,
rpc: gorpc.NewServer(),
tcl: client.New(tracker, http.DefaultClient, client.JsonCodec),
bq: rpc.NewBuilderQueue(),
mux: http.NewServeMux(),
dler: newDownloader(),
}
//register the build service in the rpc
if err := n.rpc.RegisterService(n.bq, ""); err != nil {
panic(err)
}
//make sure we respond to pings
if err := n.rpc.RegisterService(pinger.Pinger{}, ""); err != nil {
panic(err)
}
//register the codec
n.rpc.RegisterCodec(json.NewCodec(), "application/json")
//add the handlers to our mux
n.mux.Handle("/", n.rpc)
n.mux.Handle("/download/", http.StripPrefix("/download/", n.dler))
//start processing tasks
go n.run()
return n
}
//loadTest loads the test field of the responder, returning any errors.
func (r *responder) loadTest() (err error) {
cl := client.New(r.url, http.DefaultClient, client.JsonCodec)
args := &rpc.TestRequest{
ID: r.id,
Index: r.index,
}
err = cl.Call("Runner.Request", args, &r.test)
log.Printf("Handling request[%s]: %+v", r.url, r.test)
return
}
//dispatchWork is the handler that gets called for a queue item. It grabs a builder
//and runner and dispatches the work item to them, recoding when that operation
//started.
func dispatchWork(w http.ResponseWriter, req *http.Request, ctx httputil.Context) (e *httputil.Error) {
//find all the documents that are waiting or (processing and their attempt is
//taking too long)
type L []interface{}
selector := bson.M{
"$or": L{
bson.M{"status": entities.WorkStatusWaiting},
bson.M{
"status": entities.WorkStatusProcessing,
"attemptlog.0.when": bson.M{"$lt": time.Now().Add(-1 * attemptTime)},
},
},
}
iter := ctx.DB.C("Work").Find(selector).Iter()
var work entities.Work
for iter.Next(&work) {
//if its a processing task with too many attempts, store it as a dispatch
//error.
if len(work.AttemptLog) >= maxAttempts {
ctx.Infof("Work item %s had too many attempts", work.ID)
args := &rpc.DispatchResponse{
Key: work.ID.Hex(),
Error: "Unable to complete Work item. Too many failed attempts.",
WorkRev: work.Revision,
}
//send it off to the response rpc
respUrl := httputil.Absolute(router.Lookup("Response"))
cl := client.New(respUrl, http.DefaultClient, client.JsonCodec)
if err := cl.Call("Response.DispatchError", args, new(rpc.None)); err != nil {
ctx.Infof("Couldn't store a dispatch error for work item %s: %s", work.ID, err)
}
continue
}
//attempt to dispatch the work item
err := dispatchWorkItem(ctx, work)
if err != nil {
ctx.Errorf("Error dispatching work: %s", err)
}
}
//check for errors running the iteration
if err := iter.Err(); err != nil {
ctx.Errorf("Error iterating over work items: %s", err)
e = httputil.Errorf(err, "Error iterating over work items")
return
}
return
}
func (r *Runner) process(task rpc.RunnerTask) {
log.Printf("Incoming task: %+v", task)
//set the task and output slices up
r.task = task
outs := make([]rpc.Output, 0, len(task.Tests)+len(task.WontBuilds))
outs = append(outs, task.WontBuilds...) //copy the wont builds in
//start running all the tests
for i := range task.Tests {
cmd := exec.Command(r.runner, r.base, task.ID, fmt.Sprint(i))
if err := cmd.Start(); err != nil {
panic(err)
}
}
//collect all the outputs
for _ = range task.Tests {
outs = append(outs, <-r.resp)
}
//build a runner response
resp := &rpc.RunnerResponse{
Key: task.Key,
ID: task.ID,
WorkRev: task.WorkRev,
Revision: task.Revision,
RevDate: task.RevDate,
Tests: outs,
}
log.Printf("Pushing response[%s]: %+v", task.Response, resp)
//send it off
cl := client.New(r.task.Response, http.DefaultClient, client.JsonCodec)
if err := cl.Call("Response.Post", resp, new(rpc.None)); err != nil {
log.Printf("Error pushing response: %s", err)
}
}
//run grabs all the items from the channel and sends in a response
func (r *runnerTask) run() {
//grab all of the output
outs := make([]rpc.Output, 0, cap(r.resps)+len(r.task.WontBuilds))
for i := 0; i < cap(r.resps); i++ {
//grab an outout
o := <-r.resps
//signal to the default client that we're finished with this id
idch := r.ids[o.ImportPath]
r.mc.Finished(<-idch)
//append the output
outs = append(outs, o)
}
//we're done grabbing output so delete ourselves from the task map
r.tm.Delete(r.task.ID)
//copy the wontbuilds in to the outputs
outs = append(outs, r.task.WontBuilds...)
//build a RunnerResponse
resp := &rpc.RunnerResponse{
Key: r.task.Key,
ID: r.task.ID,
WorkRev: r.task.WorkRev,
Revision: r.task.Revision,
RevDate: r.task.RevDate,
Tests: outs,
}
log.Printf("Pushing response[%s]: %+v", r.task.Response, resp)
//send if off
cl := client.New(r.task.Response, http.DefaultClient, client.JsonCodec)
if err := cl.Call("Response.Post", resp, new(rpc.None)); err != nil {
log.Printf("Error pushing response: %s", err)
}
}
//Announce adds the given service into the tracker pool.
func (Tracker) Announce(req *http.Request, args *rpc.AnnounceArgs, rep *rpc.AnnounceReply) (err error) {
//wrap our error on the way out
defer rpc.Wrap(&err)
if err = verify(args); err != nil {
return
}
ctx := httputil.NewContext(req)
defer ctx.Close()
ctx.Infof("Got announce request from %s: %+v", req.RemoteAddr, args)
//ping them to make sure we can make valid rpc calls
cl := client.New(args.URL, http.DefaultClient, client.JsonCodec)
err = cl.Call("Pinger.Ping", nil, new(rpc.None))
if err != nil {
ctx.Infof("Failed to Ping the announce.")
return
}
//create the entity
var e interface{}
//make sure we have a nonzero seed
var seed int64
for seed == 0 {
seed = rand.Int63()
}
key := bson.NewObjectId()
switch args.Type {
case "Builder":
e = &Builder{
ID: key,
GOOS: args.GOOS,
GOARCH: args.GOARCH,
URL: args.URL,
Seed: seed,
}
case "Runner":
e = &Runner{
ID: key,
GOOS: args.GOOS,
GOARCH: args.GOARCH,
URL: args.URL,
Seed: seed,
}
default:
panic("unreachable")
}
//TODO(zeebo): check if we have the URL already and grab that key to update
//save the service in the database
if err = ctx.DB.C(args.Type).Insert(e); err != nil {
return
}
//return the hex representation of the key
rep.Key = key.Hex()
return
}
func dispatchWorkItem(ctx httputil.Context, work entities.Work) (err error) {
//lease a builder and runner
builder, runner, err := tracker.LeasePair(ctx)
if err != nil {
return
}
log.Printf("Got:\nBuilder: %+v\nRunner: %+v", builder, runner)
//create an attempt
a := entities.WorkAttempt{
When: time.Now(),
Builder: builder.URL,
Runner: runner.URL,
ID: bson.NewObjectId(),
}
//push the new attempt at the start of the array
log := append([]entities.WorkAttempt{a}, work.AttemptLog...)
//transactionally acquire ownership of the document
ops := []txn.Op{{
C: "Work",
Id: work.ID,
Assert: bson.M{
"revision": work.Revision,
},
Update: bson.M{
"$inc": bson.M{"revision": 1},
"$set": bson.M{
"attemptlog": log,
"status": entities.WorkStatusProcessing,
},
},
}}
err = ctx.R.Run(ops, bson.NewObjectId(), nil)
if err == txn.ErrAborted {
ctx.Infof("Lost the race dispatching a work item")
err = nil
return
}
if err != nil {
return
}
//build the task
task := &rpc.BuilderTask{
Work: work.Work,
Key: work.ID.Hex(),
ID: a.ID.Hex(),
WorkRev: work.Revision + 1,
Runner: runner.URL,
Response: httputil.Absolute(router.Lookup("Response")),
}
//send the task off to the builder queue
cl := client.New(builder.URL, http.DefaultClient, client.JsonCodec)
err = cl.Call("BuilderQueue.Push", task, new(rpc.None))
return
}
//post sends the TestResponse to the TestManager
func (r *responder) post(args *rpc.TestResponse) {
cl := client.New(r.url, http.DefaultClient, client.JsonCodec)
log.Printf("Posting response[%s]: %+v", r.url, args)
cl.Call("Runner.Post", args, new(rpc.None))
}
//process takes a task and builds the result and either responds to the tracker
//with the build failure output, or forwards a request to the Runner given by
//the task to get the info for the build.
func (b *Builder) process(task rpc.BuilderTask) {
log.Printf("Incoming build: %+v", task)
//build the work item
builds, revDate, err := b.b.Build(&task.Work)
//check if we have any errors
if err != nil {
//build our response
resp := &rpc.BuilderResponse{
Key: task.Key,
ID: task.ID,
WorkRev: task.WorkRev,
Error: err.Error(),
//these fields may be zero if we didn't get that far
Revision: task.Work.Revision,
RevDate: revDate,
}
log.Printf("Pushing error[%s]: %+v", task.Response, resp)
//send it off and ignore the error
cl := client.New(task.Response, http.DefaultClient, client.JsonCodec)
if err := cl.Call("Response.Error", resp, new(rpc.None)); err != nil {
//ignored
}
return
}
//build the runner request
req := &rpc.RunnerTask{
Key: task.Key,
ID: task.ID,
WorkRev: task.WorkRev,
Revision: task.Work.Revision,
RevDate: revDate,
Response: task.Response,
}
for _, build := range builds {
//if the build has an error, then add it to the failures and continue
//no need to schedule a download
if build.Error != "" {
req.WontBuilds = append(req.WontBuilds, rpc.Output{
ImportPath: build.ImportPath,
Config: build.Config,
Output: build.Error,
Type: rpc.OutputWontBuild,
})
continue
}
//register the tarball and binary paths with the downloader
binid := b.dler.Register(dl{
path: build.BinaryPath,
clean: func() { build.CleanBinary() },
})
souid := b.dler.Register(dl{
path: build.SourcePath,
clean: func() { build.CleanSource() },
})
//add the task with the urls
req.Tests = append(req.Tests, rpc.RunTest{
BinaryURL: b.urlWithPath("/download/" + binid),
SourceURL: b.urlWithPath("/download/" + souid),
ImportPath: build.ImportPath,
Config: build.Config,
})
}
log.Printf("Pushing request[%s]: %+v", task.Runner, req)
//send off to the runner and ignore the error
cl := client.New(task.Runner, http.DefaultClient, client.JsonCodec)
if err := cl.Call("RunnerQueue.Push", req, new(rpc.None)); err != nil {
}
return
}