コード例 #1
0
ファイル: app.go プロジェクト: kleopatra999/goci
//newDirectRunner returns a service for running tests on the local machine.
func newDirectRunner() Service {
	ru := rudirect.New(
		mustEnv("RUNPATH"),
		httputil.Absolute(router.Lookup("Tracker")),
		httputil.Absolute("/runner/"),
	)
	return ru
}
コード例 #2
0
ファイル: app.go プロジェクト: kleopatra999/goci
//newWebRunner returns a service for running tests on the heroku dyno mesh.
func newWebRunner() Service {
	//create a runner
	ru := ruweb.New(
		mustEnv("APP_NAME"),
		mustEnv("API_KEY"),
		httputil.Absolute(router.Lookup("Tracker")),
		httputil.Absolute("/runner/"),
	)
	return ru
}
コード例 #3
0
ファイル: queue.go プロジェクト: kleopatra999/goci
//dispatchWork is the handler that gets called for a queue item. It grabs a builder
//and runner and dispatches the work item to them, recoding when that operation
//started.
func dispatchWork(w http.ResponseWriter, req *http.Request, ctx httputil.Context) (e *httputil.Error) {
	//find all the documents that are waiting or (processing and their attempt is
	//taking too long)
	type L []interface{}
	selector := bson.M{
		"$or": L{
			bson.M{"status": entities.WorkStatusWaiting},
			bson.M{
				"status":            entities.WorkStatusProcessing,
				"attemptlog.0.when": bson.M{"$lt": time.Now().Add(-1 * attemptTime)},
			},
		},
	}
	iter := ctx.DB.C("Work").Find(selector).Iter()

	var work entities.Work
	for iter.Next(&work) {
		//if its a processing task with too many attempts, store it as a dispatch
		//error.
		if len(work.AttemptLog) >= maxAttempts {
			ctx.Infof("Work item %s had too many attempts", work.ID)
			args := &rpc.DispatchResponse{
				Key:     work.ID.Hex(),
				Error:   "Unable to complete Work item. Too many failed attempts.",
				WorkRev: work.Revision,
			}

			//send it off to the response rpc
			respUrl := httputil.Absolute(router.Lookup("Response"))
			cl := client.New(respUrl, http.DefaultClient, client.JsonCodec)
			if err := cl.Call("Response.DispatchError", args, new(rpc.None)); err != nil {
				ctx.Infof("Couldn't store a dispatch error for work item %s: %s", work.ID, err)
			}

			continue
		}

		//attempt to dispatch the work item
		err := dispatchWorkItem(ctx, work)
		if err != nil {
			ctx.Errorf("Error dispatching work: %s", err)
		}
	}

	//check for errors running the iteration
	if err := iter.Err(); err != nil {
		ctx.Errorf("Error iterating over work items: %s", err)
		e = httputil.Errorf(err, "Error iterating over work items")
		return
	}

	return
}
コード例 #4
0
ファイル: queue.go プロジェクト: kleopatra999/goci
//QueueWork takes a Distiller and adds it into the work queue.
func QueueWork(ctx httputil.Context, d Distiller) (err error) {
	//distill and create our work item
	work, data := d.Distill()
	q := &entities.Work{
		ID:      bson.NewObjectId(),
		Work:    work,
		Data:    data,
		Status:  entities.WorkStatusWaiting,
		Created: time.Now(),
	}

	//store it in the datastore
	if err = ctx.DB.C("Work").Insert(q); err != nil {
		return
	}

	//send a request to dispatch the queue
	go http.Get(httputil.Absolute(handleUrl))

	return
}
コード例 #5
0
ファイル: queue.go プロジェクト: kleopatra999/goci
func dispatchWorkItem(ctx httputil.Context, work entities.Work) (err error) {
	//lease a builder and runner
	builder, runner, err := tracker.LeasePair(ctx)
	if err != nil {
		return
	}

	log.Printf("Got:\nBuilder: %+v\nRunner: %+v", builder, runner)

	//create an attempt
	a := entities.WorkAttempt{
		When:    time.Now(),
		Builder: builder.URL,
		Runner:  runner.URL,
		ID:      bson.NewObjectId(),
	}

	//push the new attempt at the start of the array
	log := append([]entities.WorkAttempt{a}, work.AttemptLog...)

	//transactionally acquire ownership of the document
	ops := []txn.Op{{
		C:  "Work",
		Id: work.ID,
		Assert: bson.M{
			"revision": work.Revision,
		},
		Update: bson.M{
			"$inc": bson.M{"revision": 1},
			"$set": bson.M{
				"attemptlog": log,
				"status":     entities.WorkStatusProcessing,
			},
		},
	}}

	err = ctx.R.Run(ops, bson.NewObjectId(), nil)
	if err == txn.ErrAborted {
		ctx.Infof("Lost the race dispatching a work item")
		err = nil
		return
	}
	if err != nil {
		return
	}

	//build the task
	task := &rpc.BuilderTask{
		Work:     work.Work,
		Key:      work.ID.Hex(),
		ID:       a.ID.Hex(),
		WorkRev:  work.Revision + 1,
		Runner:   runner.URL,
		Response: httputil.Absolute(router.Lookup("Response")),
	}

	//send the task off to the builder queue
	cl := client.New(builder.URL, http.DefaultClient, client.JsonCodec)
	err = cl.Call("BuilderQueue.Push", task, new(rpc.None))

	return
}
コード例 #6
0
ファイル: app.go プロジェクト: kleopatra999/goci
func main() {
	//load up the environment if its specified
	if config.env != "" {
		if err := loader.Load(config.env); err != nil {
			panic(err)
		}
	}

	//configure the frontend
	frontend.Config.Templates = env("TEMPLATES", "./templates")
	frontend.Config.Static = env("STATIC", "./static")
	frontend.Config.Debug = env("DEBUG", "") != ""

	//configure the notifications
	notifications.Config.Username = mustEnv("XMPPUSER")
	notifications.Config.Password = mustEnv("XMPPPASS")
	notifications.Config.Domain = mustEnv("XMPPDOMAIN")

	//connect to the mongo database.
	sess, err := mgo.Dial(env("DATABASE", "mongodb://localhost/gocitest"))
	if err != nil {
		panic(err)
	}
	//empty implies whatever was specified in dial.
	httputil.Config.DB = sess.DB("")

	//set up the httputil domain so we can build absolute urls
	httputil.Config.Domain = mustEnv("DOMAIN")

	//start the server.
	//we can't use listenandserve because the scheduler might not give it the
	//opportunity to set up the listen socket before we attempt to announce.
	l, err := net.Listen("tcp", "0.0.0.0:"+mustEnv("PORT"))
	if err != nil {
		panic(err)
	}
	defer l.Close()
	go http.Serve(l, nil)

	//set up some vars for our target os and arch and the runner
	var GOOS, GOARCH string
	var runner Service

	//check if we're running direct or not
	if env("DIRECTRUN", "") == "" {
		//we're running on heroku so build for that target
		GOOS, GOARCH = "linux", "amd64"
		runner = newWebRunner()
	} else {
		//we're running things directly
		GOOS, GOARCH = env("GOOS", runtime.GOOS), env("GOARCH", runtime.GOARCH)
		runner = newDirectRunner()
	}

	//add the runner to our system
	http.Handle("/runner/", http.StripPrefix("/runner", runner))

	//announce the runner
	if err := runner.Announce(); err != nil {
		panic(err)
	}
	defer runner.Remove()

	var goroot string
	if err := checkTools(); err != nil {
		//create a temporary directory to house the go tool and hg+bzr.
		tmpdir, err := ioutil.TempDir("", "tools")
		if err != nil {
			panic(err)
		}
		defer os.RemoveAll(tmpdir)

		//execute our setup script
		cmd := exec.Command("bash", "heroku_setup.sh", "heroku/dist", tmpdir)
		if err := cmd.Run(); err != nil {
			panic(err)
		}

		//store our goroot
		goroot = filepath.Join(tmpdir, "go1.0.2.linux-amd64", "go")

		//add goroot/bin and venv/bin to path
		path := os.Getenv("PATH")
		path += string(filepath.ListSeparator) + filepath.Join(goroot, "bin")
		path += string(filepath.ListSeparator) + filepath.Join(tmpdir, "venv", "bin")
		os.Setenv("PATH", path)

		//check for the tools again as they should installed
		if err := checkTools(); err != nil {
			panic("couldn't find tools after installing: " + err.Error())
		}
	}

	//if we don't have goroot set yet set it to where we find the go command
	if goroot == "" {
		path, err := exec.LookPath("go")
		if err != nil {
			panic("unable to find go tool")
		}
		goroot = filepath.Dir(filepath.Dir(path))
	}

	//create the builder and announce it
	bu := buweb.New(
		builder.New(GOOS, GOARCH, goroot),
		httputil.Absolute(router.Lookup("Tracker")),
		httputil.Absolute("/builder/"),
	)
	http.Handle("/builder/", http.StripPrefix("/builder", bu))

	if err := bu.Announce(); err != nil {
		panic(err)
	}
	defer bu.Remove()

	//wait for a signal
	signals := []os.Signal{
		syscall.SIGQUIT,
		syscall.SIGKILL,
		syscall.SIGINT,
		syscall.SIGTERM,
	}
	ch := make(chan os.Signal, 1)
	signal.Notify(ch, signals...)
	sig := <-ch
	log.Printf("Captured a %v\n", sig)
}
コード例 #7
0
ファイル: response.go プロジェクト: kleopatra999/goci
//Post is the rpc method that the Runner uses to give a response about an item.
func (Response) Post(req *http.Request, args *rpc.RunnerResponse, resp *rpc.None) (err error) {
	//wrap our error on the way out
	defer rpc.Wrap(&err)

	//create our context
	ctx := httputil.NewContext(req)
	defer ctx.Close()

	//build the keys we need to reference
	key := bson.ObjectIdHex(args.Key)
	wkey := bson.NewObjectId()

	ops := []txn.Op{{ //make sure we have the given work item
		C:  "Work",
		Id: key,
		Assert: bson.M{
			"status":          entities.WorkStatusProcessing,
			"attemptlog.0.id": bson.ObjectIdHex(args.ID),
			"revision":        args.WorkRev,
		},
		Update: bson.M{
			"$set": bson.M{"status": entities.WorkStatusCompleted},
			"$inc": bson.M{"revision": 1},
		},
	}, { //insert the work result
		C:  "WorkResult",
		Id: wkey,
		Insert: entities.WorkResult{
			WorkID:   key,
			Success:  true,
			Revision: args.Revision,
			RevDate:  args.RevDate,
			When:     time.Now(),
		},
	}}

	//operations for notifications
	var nots []txn.Op

	//store the test results
	for _, out := range args.Tests {

		//get the status from the output type and output
		var status string
		switch out.Type {
		case rpc.OutputSuccess:
			if strings.HasSuffix(out.Output, "\nPASS\n") {
				status = entities.TestStatusPass
			} else {
				status = entities.TestStatusFail
			}
		case rpc.OutputWontBuild:
			status = entities.TestStatusWontBuild
		case rpc.OutputError:
			status = entities.TestStatusError
		default:
			err = fmt.Errorf("unknown output type: %s", out.Type)
			return
		}

		//add the test result to the operation
		tid := bson.NewObjectId()
		ops = append(ops, txn.Op{
			C:  "TestResult",
			Id: tid,
			Insert: entities.TestResult{
				WorkResultID: wkey,
				ImportPath:   out.ImportPath,
				Revision:     args.Revision,
				RevDate:      args.RevDate,
				When:         time.Now(),
				Output:       out.Output,
				Status:       status,
			},
		})

		//skip if we don't have a notification
		if out.Config.NotifyOn == "" {
			continue
		}

		//add in the notification
		nots = append(nots, txn.Op{
			C:  "Notification",
			Id: bson.NewObjectId(),
			Insert: entities.Notification{
				Test:   tid,
				Config: out.Config,
				Status: entities.NotifStatusWaiting,
			},
		})

	}

	//append the notification operations
	ops = append(ops, nots...)

	//run the transaction
	err = ctx.R.Run(ops, bson.NewObjectId(), nil)
	if err == txn.ErrAborted {
		ctx.Infof("Lost the race inserting result.")
		err = nil
	}

	//tell it to dispatch notifications
	if len(nots) > 0 {
		go http.Get(httputil.Absolute("/notifications/dispatch"))
	}

	return
}
コード例 #8
0
ファイル: httputil_test.go プロジェクト: kleopatra999/goci
// http://goci.me/some/path
func ExampleAbsolute() {
	httputil.Config.Domain = "goci.me"
	fmt.Println(httputil.Absolute("/some/path"))
}