Beispiel #1
0
//New returns a new Runner ready to be Announced and run tests locally.
func New(runner, tracker, hosted string) *Runner {
	n := &Runner{
		tcl:    client.New(tracker, http.DefaultClient, client.JsonCodec),
		base:   hosted,
		runner: runner,
		rpc:    gorpc.NewServer(),
		rq:     rpc.NewRunnerQueue(),
		resp:   make(chan rpc.Output),
	}

	//register the run service in the rpc
	if err := n.rpc.RegisterService(n.rq, ""); err != nil {
		panic(err)
	}

	//register the pinger
	if err := n.rpc.RegisterService(pinger.Pinger{}, ""); err != nil {
		panic(err)
	}

	//register ourselves in the rpc
	if err := n.rpc.RegisterService(n, ""); err != nil {
		panic(err)
	}

	//register the codec
	n.rpc.RegisterCodec(json.NewCodec(), "application/json")

	//start processing
	go n.run()

	return n
}
Beispiel #2
0
func main() {
	flag.Parse()
	args := flag.Args()
	if len(args) != 3 {
		check(fmt.Errorf("usage: rpc <url> <method> <args>"), "args")
	}
	url, method, arg := args[0], args[1], args[2]
	hcl := &http.Client{
		Transport: rt{},
	}
	cl := client.New(url, hcl, client.JsonCodec)

	var x, y interface{}
	var err error

	err = json.Unmarshal([]byte(arg), &x)
	check(err, "unmar")

	err = cl.Call(method, x, &y)
	check(err, "call")

	var b []byte
	b, err = json.MarshalIndent(y, "", "\t")
	check(err, "marsh")

	fmt.Printf("%s\n", b)
}
Beispiel #3
0
//New returns a new Runner ready to be Announced and run tests on the
//heroku dyno grid.
func New(app, api string, tracker, hosted string) *Runner {
	n := &Runner{
		app:  app,
		api:  api,
		tcl:  client.New(tracker, http.DefaultClient, client.JsonCodec),
		base: hosted,
		rpc:  gorpc.NewServer(),
		rq:   rpc.NewRunnerQueue(),
		mc:   heroku.NewManaged(app, api, 2, 2*time.Minute),
		tm:   &runnerTaskMap{items: map[string]*runnerTask{}},
	}

	//register the run service in the rpc
	if err := n.rpc.RegisterService(n.rq, ""); err != nil {
		panic(err)
	}

	//register the pinger
	if err := n.rpc.RegisterService(pinger.Pinger{}, ""); err != nil {
		panic(err)
	}

	//register ourselves in the rpc
	if err := n.rpc.RegisterService(n, ""); err != nil {
		panic(err)
	}

	//register the codec
	n.rpc.RegisterCodec(json.NewCodec(), "application/json")

	//start processing
	go n.run()

	return n
}
Beispiel #4
0
//New returns a new web Builder ready to Announce to the given tracker. It
//announces that it is available at `hosted` which should be the full url of
//where this builder resides on the internet.
func New(b builder.Builder, tracker, hosted string) *Builder {
	//create our new builder
	n := &Builder{
		b:    b,
		base: hosted,
		rpc:  gorpc.NewServer(),
		tcl:  client.New(tracker, http.DefaultClient, client.JsonCodec),
		bq:   rpc.NewBuilderQueue(),
		mux:  http.NewServeMux(),
		dler: newDownloader(),
	}

	//register the build service in the rpc
	if err := n.rpc.RegisterService(n.bq, ""); err != nil {
		panic(err)
	}

	//make sure we respond to pings
	if err := n.rpc.RegisterService(pinger.Pinger{}, ""); err != nil {
		panic(err)
	}

	//register the codec
	n.rpc.RegisterCodec(json.NewCodec(), "application/json")

	//add the handlers to our mux
	n.mux.Handle("/", n.rpc)
	n.mux.Handle("/download/", http.StripPrefix("/download/", n.dler))

	//start processing tasks
	go n.run()

	return n
}
Beispiel #5
0
//loadTest loads the test field of the responder, returning any errors.
func (r *responder) loadTest() (err error) {
	cl := client.New(r.url, http.DefaultClient, client.JsonCodec)
	args := &rpc.TestRequest{
		ID:    r.id,
		Index: r.index,
	}
	err = cl.Call("Runner.Request", args, &r.test)
	log.Printf("Handling request[%s]: %+v", r.url, r.test)
	return
}
Beispiel #6
0
//dispatchWork is the handler that gets called for a queue item. It grabs a builder
//and runner and dispatches the work item to them, recoding when that operation
//started.
func dispatchWork(w http.ResponseWriter, req *http.Request, ctx httputil.Context) (e *httputil.Error) {
	//find all the documents that are waiting or (processing and their attempt is
	//taking too long)
	type L []interface{}
	selector := bson.M{
		"$or": L{
			bson.M{"status": entities.WorkStatusWaiting},
			bson.M{
				"status":            entities.WorkStatusProcessing,
				"attemptlog.0.when": bson.M{"$lt": time.Now().Add(-1 * attemptTime)},
			},
		},
	}
	iter := ctx.DB.C("Work").Find(selector).Iter()

	var work entities.Work
	for iter.Next(&work) {
		//if its a processing task with too many attempts, store it as a dispatch
		//error.
		if len(work.AttemptLog) >= maxAttempts {
			ctx.Infof("Work item %s had too many attempts", work.ID)
			args := &rpc.DispatchResponse{
				Key:     work.ID.Hex(),
				Error:   "Unable to complete Work item. Too many failed attempts.",
				WorkRev: work.Revision,
			}

			//send it off to the response rpc
			respUrl := httputil.Absolute(router.Lookup("Response"))
			cl := client.New(respUrl, http.DefaultClient, client.JsonCodec)
			if err := cl.Call("Response.DispatchError", args, new(rpc.None)); err != nil {
				ctx.Infof("Couldn't store a dispatch error for work item %s: %s", work.ID, err)
			}

			continue
		}

		//attempt to dispatch the work item
		err := dispatchWorkItem(ctx, work)
		if err != nil {
			ctx.Errorf("Error dispatching work: %s", err)
		}
	}

	//check for errors running the iteration
	if err := iter.Err(); err != nil {
		ctx.Errorf("Error iterating over work items: %s", err)
		e = httputil.Errorf(err, "Error iterating over work items")
		return
	}

	return
}
Beispiel #7
0
func (r *Runner) process(task rpc.RunnerTask) {
	log.Printf("Incoming task: %+v", task)

	//set the task and output slices up
	r.task = task
	outs := make([]rpc.Output, 0, len(task.Tests)+len(task.WontBuilds))
	outs = append(outs, task.WontBuilds...) //copy the wont builds in

	//start running all the tests
	for i := range task.Tests {
		cmd := exec.Command(r.runner, r.base, task.ID, fmt.Sprint(i))
		if err := cmd.Start(); err != nil {
			panic(err)
		}
	}

	//collect all the outputs
	for _ = range task.Tests {
		outs = append(outs, <-r.resp)
	}

	//build a runner response
	resp := &rpc.RunnerResponse{
		Key:      task.Key,
		ID:       task.ID,
		WorkRev:  task.WorkRev,
		Revision: task.Revision,
		RevDate:  task.RevDate,
		Tests:    outs,
	}

	log.Printf("Pushing response[%s]: %+v", task.Response, resp)

	//send it off
	cl := client.New(r.task.Response, http.DefaultClient, client.JsonCodec)
	if err := cl.Call("Response.Post", resp, new(rpc.None)); err != nil {
		log.Printf("Error pushing response: %s", err)
	}
}
Beispiel #8
0
//run grabs all the items from the channel and sends in a response
func (r *runnerTask) run() {
	//grab all of the output
	outs := make([]rpc.Output, 0, cap(r.resps)+len(r.task.WontBuilds))
	for i := 0; i < cap(r.resps); i++ {
		//grab an outout
		o := <-r.resps

		//signal to the default client that we're finished with this id
		idch := r.ids[o.ImportPath]
		r.mc.Finished(<-idch)

		//append the output
		outs = append(outs, o)
	}

	//we're done grabbing output so delete ourselves from the task map
	r.tm.Delete(r.task.ID)

	//copy the wontbuilds in to the outputs
	outs = append(outs, r.task.WontBuilds...)

	//build a RunnerResponse
	resp := &rpc.RunnerResponse{
		Key:      r.task.Key,
		ID:       r.task.ID,
		WorkRev:  r.task.WorkRev,
		Revision: r.task.Revision,
		RevDate:  r.task.RevDate,
		Tests:    outs,
	}

	log.Printf("Pushing response[%s]: %+v", r.task.Response, resp)

	//send if off
	cl := client.New(r.task.Response, http.DefaultClient, client.JsonCodec)
	if err := cl.Call("Response.Post", resp, new(rpc.None)); err != nil {
		log.Printf("Error pushing response: %s", err)
	}
}
Beispiel #9
0
//Announce adds the given service into the tracker pool.
func (Tracker) Announce(req *http.Request, args *rpc.AnnounceArgs, rep *rpc.AnnounceReply) (err error) {
	//wrap our error on the way out
	defer rpc.Wrap(&err)

	if err = verify(args); err != nil {
		return
	}

	ctx := httputil.NewContext(req)
	defer ctx.Close()
	ctx.Infof("Got announce request from %s: %+v", req.RemoteAddr, args)

	//ping them to make sure we can make valid rpc calls
	cl := client.New(args.URL, http.DefaultClient, client.JsonCodec)
	err = cl.Call("Pinger.Ping", nil, new(rpc.None))
	if err != nil {
		ctx.Infof("Failed to Ping the announce.")
		return
	}

	//create the entity
	var e interface{}

	//make sure we have a nonzero seed
	var seed int64
	for seed == 0 {
		seed = rand.Int63()
	}
	key := bson.NewObjectId()
	switch args.Type {
	case "Builder":
		e = &Builder{
			ID:     key,
			GOOS:   args.GOOS,
			GOARCH: args.GOARCH,
			URL:    args.URL,
			Seed:   seed,
		}
	case "Runner":
		e = &Runner{
			ID:     key,
			GOOS:   args.GOOS,
			GOARCH: args.GOARCH,
			URL:    args.URL,
			Seed:   seed,
		}
	default:
		panic("unreachable")
	}

	//TODO(zeebo): check if we have the URL already and grab that key to update

	//save the service in the database
	if err = ctx.DB.C(args.Type).Insert(e); err != nil {
		return
	}

	//return the hex representation of the key
	rep.Key = key.Hex()
	return
}
Beispiel #10
0
func dispatchWorkItem(ctx httputil.Context, work entities.Work) (err error) {
	//lease a builder and runner
	builder, runner, err := tracker.LeasePair(ctx)
	if err != nil {
		return
	}

	log.Printf("Got:\nBuilder: %+v\nRunner: %+v", builder, runner)

	//create an attempt
	a := entities.WorkAttempt{
		When:    time.Now(),
		Builder: builder.URL,
		Runner:  runner.URL,
		ID:      bson.NewObjectId(),
	}

	//push the new attempt at the start of the array
	log := append([]entities.WorkAttempt{a}, work.AttemptLog...)

	//transactionally acquire ownership of the document
	ops := []txn.Op{{
		C:  "Work",
		Id: work.ID,
		Assert: bson.M{
			"revision": work.Revision,
		},
		Update: bson.M{
			"$inc": bson.M{"revision": 1},
			"$set": bson.M{
				"attemptlog": log,
				"status":     entities.WorkStatusProcessing,
			},
		},
	}}

	err = ctx.R.Run(ops, bson.NewObjectId(), nil)
	if err == txn.ErrAborted {
		ctx.Infof("Lost the race dispatching a work item")
		err = nil
		return
	}
	if err != nil {
		return
	}

	//build the task
	task := &rpc.BuilderTask{
		Work:     work.Work,
		Key:      work.ID.Hex(),
		ID:       a.ID.Hex(),
		WorkRev:  work.Revision + 1,
		Runner:   runner.URL,
		Response: httputil.Absolute(router.Lookup("Response")),
	}

	//send the task off to the builder queue
	cl := client.New(builder.URL, http.DefaultClient, client.JsonCodec)
	err = cl.Call("BuilderQueue.Push", task, new(rpc.None))

	return
}
Beispiel #11
0
//post sends the TestResponse to the TestManager
func (r *responder) post(args *rpc.TestResponse) {
	cl := client.New(r.url, http.DefaultClient, client.JsonCodec)
	log.Printf("Posting response[%s]: %+v", r.url, args)
	cl.Call("Runner.Post", args, new(rpc.None))
}
Beispiel #12
0
//process takes a task and builds the result and either responds to the tracker
//with the build failure output, or forwards a request to the Runner given by
//the task to get the info for the build.
func (b *Builder) process(task rpc.BuilderTask) {
	log.Printf("Incoming build: %+v", task)

	//build the work item
	builds, revDate, err := b.b.Build(&task.Work)

	//check if we have any errors
	if err != nil {

		//build our response
		resp := &rpc.BuilderResponse{
			Key:     task.Key,
			ID:      task.ID,
			WorkRev: task.WorkRev,
			Error:   err.Error(),

			//these fields may be zero if we didn't get that far
			Revision: task.Work.Revision,
			RevDate:  revDate,
		}

		log.Printf("Pushing error[%s]: %+v", task.Response, resp)

		//send it off and ignore the error
		cl := client.New(task.Response, http.DefaultClient, client.JsonCodec)
		if err := cl.Call("Response.Error", resp, new(rpc.None)); err != nil {
			//ignored
		}
		return
	}

	//build the runner request
	req := &rpc.RunnerTask{
		Key:      task.Key,
		ID:       task.ID,
		WorkRev:  task.WorkRev,
		Revision: task.Work.Revision,
		RevDate:  revDate,
		Response: task.Response,
	}
	for _, build := range builds {
		//if the build has an error, then add it to the failures and continue
		//no need to schedule a download
		if build.Error != "" {
			req.WontBuilds = append(req.WontBuilds, rpc.Output{
				ImportPath: build.ImportPath,
				Config:     build.Config,
				Output:     build.Error,
				Type:       rpc.OutputWontBuild,
			})
			continue
		}

		//register the tarball and binary paths with the downloader
		binid := b.dler.Register(dl{
			path:  build.BinaryPath,
			clean: func() { build.CleanBinary() },
		})
		souid := b.dler.Register(dl{
			path:  build.SourcePath,
			clean: func() { build.CleanSource() },
		})

		//add the task with the urls
		req.Tests = append(req.Tests, rpc.RunTest{
			BinaryURL:  b.urlWithPath("/download/" + binid),
			SourceURL:  b.urlWithPath("/download/" + souid),
			ImportPath: build.ImportPath,
			Config:     build.Config,
		})
	}

	log.Printf("Pushing request[%s]: %+v", task.Runner, req)

	//send off to the runner and ignore the error
	cl := client.New(task.Runner, http.DefaultClient, client.JsonCodec)
	if err := cl.Call("RunnerQueue.Push", req, new(rpc.None)); err != nil {

	}
	return
}