Ejemplo n.º 1
0
func main() {
	log.Init(&log.GlogRecorder{})

	opts, err := config.LoadOpts(os.Args[1:])

	switch err {
	case nil:
	case flag.ErrHelp:
		os.Exit(0)
	default:
		os.Exit(2)
	}

	if opts.PrintVersion {
		fmt.Printf("Walter version %s\n", version.Version)
		os.Exit(0)
	}

	walter, err := walter.New(opts)
	if err != nil {
		log.Error(err.Error())
		log.Error("failed to create Walter")
		return
	}
	log.Info("running Walter")
	result := walter.Run()
	if result == false {
		log.Info("more than one failures were detected running Walter")
		log.Flush()
		os.Exit(1)
	}
	log.Info("succeded to finish Walter")
	log.Flush()
}
Ejemplo n.º 2
0
func (e *Walter) runService() bool {
	// load .walter-update
	log.Infof("Loading update file... \"%s\"", e.Engine.Resources.RepoService.GetUpdateFilePath())
	update, err := services.LoadLastUpdate(e.Engine.Resources.RepoService.GetUpdateFilePath())
	log.Infof("Succeeded loading update file")

	log.Info("Updating status...")
	update.Status = "inprogress"
	result := services.SaveLastUpdate(e.Engine.Resources.RepoService.GetUpdateFilePath(), update)
	if result == false {
		log.Error("Failed to save status update")
		return false
	}
	log.Info("Succeeded updating status")

	// get latest commit and pull requests
	log.Info("downloading commits and pull requests...")
	commits, err := e.Engine.Resources.RepoService.GetCommits(update)
	if err != nil {
		log.Errorf("Failed getting commits: %s", err)
		return false
	}

	log.Info("Succeeded getting commits")
	log.Info("Size of commits: " + strconv.Itoa(commits.Len()))
	hasFailedProcess := false
	for commit := commits.Front(); commit != nil; commit = commit.Next() {
		commitType := reflect.TypeOf(commit.Value)
		if commitType.Name() == "RepositoryCommit" {
			log.Info("Found new repository commit")
			trunkCommit := commit.Value.(github.RepositoryCommit)
			if result := e.processTrunkCommit(trunkCommit); result == false {
				hasFailedProcess = true
			}
		} else if commitType.Name() == "PullRequest" {
			log.Info("Found new pull request commit")
			pullreq := commit.Value.(github.PullRequest)
			if result := e.processPullRequest(pullreq); result == false {
				hasFailedProcess = true
			}
		} else {
			log.Errorf("Nothing commit type: %s", commitType)
			hasFailedProcess = true
		}
	}

	// save .walter-update
	log.Info("Saving update file...")
	update.Status = "finished"
	update.Time = time.Now()
	result = services.SaveLastUpdate(e.Engine.Resources.RepoService.GetUpdateFilePath(), update)
	if result == false {
		log.Error("Failed to save update")
		return false
	}
	return !hasFailedProcess
}
Ejemplo n.º 3
0
func (e *Walter) processPullRequest(pullrequest github.PullRequest) bool {
	// checkout pullrequest
	num := *pullrequest.Number
	_, err := exec.Command("git", "fetch", "origin", "refs/pull/"+strconv.Itoa(num)+"/head:pr_"+strconv.Itoa(num)).Output()

	defer exec.Command("git", "checkout", "master", "-f").Output() // TODO: make trunk branch configurable
	defer log.Info("returning master branch...")

	if err != nil {
		log.Errorf("Failed to fetch pull request: %s", err)
		return false
	}

	_, err = exec.Command("git", "checkout", "pr_"+strconv.Itoa(num)).Output()
	if err != nil {
		log.Errorf("Failed to checkout pullrequest branch (\"%s\") : %s", "pr_"+strconv.Itoa(num), err)
		log.Error("Skip execution...")
		return false
	}

	// run pipeline
	log.Info("Running pipeline...")
	w, err := New(e.Opts)
	if err != nil {
		log.Errorf("Failed to create Walter object...: %s", err)
		log.Error("Skip execution...")
		return false
	}

	result := w.Engine.RunOnce()

	// register the result to hosting service
	if result.IsSucceeded() {
		log.Info("succeeded.")
		e.Engine.Resources.RepoService.RegisterResult(
			services.Result{
				State:   "success",
				Message: "Succeeded running pipeline...",
				SHA:     *pullrequest.Head.SHA})
		return true
	}
	log.Error("Error reported...")
	e.Engine.Resources.RepoService.RegisterResult(
		services.Result{
			State:   "failure",
			Message: "Failed running pipleline ...",
			SHA:     *pullrequest.Head.SHA})
	return false

}
Ejemplo n.º 4
0
func main() {
	var host, dbDir string

	flags := flag.NewFlagSet("walter", flag.ExitOnError)
	flags.StringVar(&host, "host", "0.0.0.0:8080", "The host of the application.")
	flags.StringVar(&dbDir, "db_dir", "/var/lib/walter", "The directory of the sqlite3 db file.")

	if err := flags.Parse(os.Args[1:]); err != nil {
		panic(err)
	}

	if err := os.MkdirAll(dbDir, 0755); err != nil {
		panic(err)
	}

	db.Init(dbDir)

	r := route.GetRegexpHandler()
	http.Handle("/", r)

	log.Info(fmt.Sprintf("walter-server is listening on %s", host))

	if err := http.ListenAndServe(host, nil); err != nil {
		log.Error(fmt.Sprintf("ListenAndServe: %s", err))
	}
}
Ejemplo n.º 5
0
func (parser *Parser) setFieldVal(fieldVal reflect.Value, stageOptVal interface{}, isReplace string) {
	if fieldVal.Type() != reflect.ValueOf("string").Type() {
		log.Error("found non string field value type...")
		return
	}
	if isReplace == "true" {
		fieldVal.SetString(parser.EnvVariables.Replace(stageOptVal.(string)))
	} else {
		fieldVal.SetString(parser.EnvVariables.ReplaceSpecialVariableToEnvVariable(stageOptVal.(string)))
	}
}
Ejemplo n.º 6
0
func (e *Walter) processTrunkCommit(commit github.RepositoryCommit) bool {
	log.Infof("Checkout master branch")
	_, err := exec.Command("git", "checkout", "master", "-f").Output()
	if err != nil {
		log.Errorf("Failed to checkout master branch: %s", err)
		return false
	}
	log.Infof("Downloading new commit from master")
	_, err = exec.Command("git", "pull", "origin", "master").Output()
	if err != nil {
		log.Errorf("Failed to download new commit from master: %s", err)
		return false
	}
	log.Infof("Running the latest commit in master")
	w, err := New(e.Opts)
	if err != nil {
		log.Errorf("Failed to create Walter object...: %s", err)
		log.Error("Skip execution...")
		return false
	}
	result := w.Engine.RunOnce()

	// register the result to hosting service
	if result.IsSucceeded() {
		log.Info("Succeeded.")
		e.Engine.Resources.RepoService.RegisterResult(
			services.Result{
				State:   "success",
				Message: "Succeeded running pipeline...",
				SHA:     *commit.SHA})
		return true
	}
	log.Error("Error reported...")
	e.Engine.Resources.RepoService.RegisterResult(
		services.Result{
			State:   "failure",
			Message: "Failed running pipleline ...",
			SHA:     *commit.SHA})
	return false

}
Ejemplo n.º 7
0
//GetCommits get a list of all the commits for the current update
func (githubClient *GitHubClient) GetCommits(update Update) (*list.List, error) {
	log.Info("getting commits\n")
	commits := list.New()
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: githubClient.Token},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)
	client := github.NewClient(tc)

	// get a list of pull requests with Pull Request API
	pullreqs, _, err := client.PullRequests.List(
		githubClient.From, githubClient.Repo,
		&github.PullRequestListOptions{})
	if err != nil {
		log.Errorf("Failed to get pull requests")
		return list.New(), err
	}

	re, err := regexp.Compile(githubClient.TargetBranch)
	if err != nil {
		log.Error("Failed to compile branch pattern...")
		return list.New(), err
	}

	log.Infof("Size of pull reqests: %d", len(pullreqs))
	for _, pullreq := range pullreqs {
		log.Infof("Branch name is \"%s\"", *pullreq.Head.Ref)

		if githubClient.TargetBranch != "" {
			matched := re.Match([]byte(*pullreq.Head.Ref))
			if matched != true {
				log.Infof("Not add a branch, \"%s\" since this branch name is not match the filtering pattern", *pullreq.Head.Ref)
				continue
			}
		}

		if *pullreq.State == "open" && pullreq.UpdatedAt.After(update.Time) {
			log.Infof("Adding pullrequest %d", *pullreq.Number)
			commits.PushBack(pullreq)
		}
	}

	// get the latest commit with Commit API if the commit is newer than last update
	masterCommits, _, _ := client.Repositories.ListCommits(
		githubClient.From, githubClient.Repo, &github.CommitsListOptions{})
	if masterCommits[0].Commit.Author.Date.After(update.Time) {
		commits.PushBack(masterCommits[0])
	}
	return commits, nil
}
Ejemplo n.º 8
0
func printOutput(r io.Reader, prefix string, name string, out chan string, done chan bool) {
	reader := bufio.NewReaderSize(r, 1048576)
	for {
		line, _, err := reader.ReadLine()

		if err == io.EOF {
			break
		} else if err != nil {
			log.Error(err.Error())
			os.Exit(1)
		}

		log.Infof("[%s][command] %s output: %s", name, prefix, string(line))
		out <- fmt.Sprintf("%s\n", string(line))
	}
	done <- true
}
Ejemplo n.º 9
0
func (j *Jobs) handlePullRequestEvent(w http.ResponseWriter, body string) {
	var data payloadPullRequestEvent
	err := json.Unmarshal([]byte(body), &data)
	if err != nil {
		log.Error(err.Error())
		returnError(w, err.Error())
		return
	}

	if data.Action != "opened" && data.Action != "synchronize" {
		return
	}

	if data.PullRequest.Head.Repo.FullName == data.PullRequest.Base.Repo.FullName {
		return
	}

	job := &Job{}

	job.Project = data.PullRequest.Base.Repo.FullName
	job.Revision = data.PullRequest.Head.Sha
	job.HtmlUrl = data.PullRequest.Base.Repo.HtmlUrl
	job.CloneUrl = data.PullRequest.Base.Repo.CloneUrl
	job.Branch = data.PullRequest.Head.Ref
	job.PullRequestUrl = data.PullRequest.HtmlUrl
	job.CompareUrl = job.PullRequestUrl
	job.PullRequestNumber = data.Number
	job.StatusesUrl = data.PullRequest.StatusesUrl

	job.TriggeredBy = &triggeredBy{
		Name:      data.Sender.Login,
		Url:       data.Sender.HtmlUrl,
		AvatarUrl: data.Sender.AvatarUrl,
	}

	c := &commit{
		Revision: data.PullRequest.Head.Sha,
		Author:   data.PullRequest.User.Login,
		Message:  fmt.Sprintf("Pull request #%d \"%s\"", data.Number, data.PullRequest.Title),
	}

	job.Commits = append(job.Commits, c)

	j.jobs = append(j.jobs, job)
}
Ejemplo n.º 10
0
func validateWaitForCondition(wait *WaitFor) bool {
	// check duplicate targets
	if wait.Port != 0 && wait.File != "" {
		log.Error("[command] Port and File are not able to be specified at the same time.")
		return false
	} else if wait.Port != 0 && wait.Delay > 0.0 {
		log.Error("[command] Port and Delay are not able to be specified at the same time.")
		return false
	} else if wait.File != "" && wait.Delay > 0.0 {
		log.Error("[command] File and Delay are not able to be specified at the same time.")
		return false
	}

	// check illegal conditions
	if wait.Delay < 0 {
		log.Error("[command] Delay must be positive.")
		return false
	} else if wait.Port < 0 {
		log.Error("[command] Port must be positive.")
		return false
	} else if wait.Port > 0 && wait.Host == "" {
		log.Error("[command] Host must be specified when port number is specified.")
		return false
	}

	// check illegal states
	if wait.State != "present" && wait.State != "ready" && wait.State != "absent" && wait.State != "unready" {
		log.Errorf("[command] \"%s\" is an unsupported state", wait.State)
		return false
	}

	// misc checks
	if wait.Port > 0 && wait.State == "" {
		log.Error("[command] State must be specified for port.")
		return false
	} else if wait.File != "" && wait.State == "" {
		log.Error("[command] State must be specified for file.")
		return false
	}

	return true
}
Ejemplo n.º 11
0
func copyStream(reader io.Reader, prefix string, name string) string {
	var err error
	var n int
	var buffer bytes.Buffer
	tmpBuf := make([]byte, 1024)
	for {
		if n, err = reader.Read(tmpBuf); err != nil {
			break
		}
		buffer.Write(tmpBuf[0:n])
		log.Infof("[%s][command] %s output: %s", name, prefix, tmpBuf[0:n])
	}
	if err == io.EOF {
		err = nil
	} else {
		log.Error("ERROR: " + err.Error())
	}
	return buffer.String()
}
Ejemplo n.º 12
0
func (j *Jobs) handlePushEvent(w http.ResponseWriter, body string) {
	var data payloadPushEvent
	err := json.Unmarshal([]byte(body), &data)
	if err != nil {
		log.Error(err.Error())
		returnError(w, err.Error())
		return
	}

	job := &Job{}

	job.Project = data.Repository.FullName
	job.Revision = data.After
	job.HtmlUrl = data.Repository.HtmlUrl
	job.CloneUrl = data.Repository.CloneUrl
	job.CompareUrl = data.Compare
	job.StatusesUrl = strings.Replace(data.Repository.StatusesUrl, "{sha}", job.Revision, 1)

	ref := strings.Split(data.Ref, "/")
	job.Branch = ref[len(ref)-1]

	for _, c := range data.Commits {
		job.Commits = append(job.Commits, &commit{
			Revision: c.Id,
			Author:   c.Author.Username,
			Message:  c.Message,
			Url:      c.Url,
		})
	}

	job.TriggeredBy = &triggeredBy{
		Name:      data.Sender.Login,
		Url:       data.Sender.HtmlUrl,
		AvatarUrl: data.Sender.AvatarUrl,
	}

	j.jobs = append(j.jobs, job)
}
Ejemplo n.º 13
0
// Parse reads the specified configuration and create the pipeline.Resource.
func (parser *Parser) Parse() (*pipelines.Resources, error) {
	// parse require block
	requireFiles, ok := (*parser.ConfigData)["require"].([]interface{})
	var required map[string]map[interface{}]interface{}
	var err error
	if ok == true {
		log.Info("found \"require\" block")
		required, err = parser.mapRequires(requireFiles)
		if err != nil {
			log.Error("failed to load requires...")
			return nil, err
		}
		log.Info("number of registered stages: " + strconv.Itoa(len(required)))
	} else {
		log.Info("not found \"require\" block")
	}

	// parse service block
	serviceOps, ok := (*parser.ConfigData)["service"].(map[interface{}]interface{})
	var repoService services.Service
	if ok == true {
		log.Info("found \"service\" block")
		repoService, err = parser.mapService(serviceOps)
		if err != nil {
			log.Error("failed to load service settings...")
			return nil, err
		}
	} else {
		log.Info("not found \"service\" block")
		repoService, err = services.InitService("local")
		if err != nil {
			log.Error("failed to init local mode...")
			return nil, err
		}
	}

	// parse messenger block
	messengerOps, ok := (*parser.ConfigData)["messenger"].(map[interface{}]interface{})
	var messenger messengers.Messenger
	if ok == true {
		log.Info("found messenger block")
		messenger, err = parser.mapMessenger(messengerOps)
		if err != nil {
			log.Error("failed to init messenger...")
			return nil, err
		}
	} else {
		log.Info("not found messenger block")
		messenger, err = messengers.InitMessenger("fake")
		if err != nil {
			return nil, err
		}
	}

	// parse cleanup block
	var cleanup = &pipelines.Pipeline{}
	cleanupData, ok := (*parser.ConfigData)["cleanup"].([]interface{})
	if ok == true {
		log.Info("found cleanup block")
		cleanupList, err := parser.convertYamlMapToStages(cleanupData, required)
		if err != nil {
			log.Error("failed to create a stage in cleanup...")
			return nil, err
		}
		for stageItem := cleanupList.Front(); stageItem != nil; stageItem = stageItem.Next() {
			cleanup.AddStage(stageItem.Value.(stages.Stage))
		}
	} else {
		log.Info("not found cleanup block in the input file")
	}

	// parse pipeline block
	var pipeline = &pipelines.Pipeline{}

	pipelineData, ok := (*parser.ConfigData)["pipeline"].([]interface{})
	if ok == false {
		return nil, fmt.Errorf("no pipeline block in the input file")
	}
	stageList, err := parser.convertYamlMapToStages(pipelineData, required)
	if err != nil {
		log.Error("failed to create a stage in pipeline...")
		return nil, err
	}
	for stageItem := stageList.Front(); stageItem != nil; stageItem = stageItem.Next() {
		pipeline.AddStage(stageItem.Value.(stages.Stage))
	}
	var resources = &pipelines.Resources{Pipeline: pipeline, Cleanup: cleanup, Reporter: messenger, RepoService: repoService}

	return resources, nil
}
Ejemplo n.º 14
0
func createReport(w http.ResponseWriter, r *http.Request) {
	rb := bufio.NewReader(r.Body)
	var body string
	for {
		s, err := rb.ReadString('\n')
		body = body + s
		if err == io.EOF {
			break
		}
	}
	var data Report
	err := json.Unmarshal([]byte(body), &data)
	if err != nil {
		log.Error(err.Error())
		returnError(w, err.Error())
		return
	}

	dh := db.GetHandler()
	defer dh.Close()

	var projects []db.Project
	if err := dh.Select(&projects, dh.Where("repo", "=", data.Project.Repo)); err != nil {
		log.Error(err.Error())
		returnError(w, err.Error())
		return
	}

	var projectId int64
	if len(projects) == 0 {
		project := &db.Project{Name: data.Project.Name, Repo: data.Project.Repo}
		dh.Insert(project)
		projectId, _ = dh.LastInsertId()
	} else {
		projectId = projects[0].Id
	}

	var users []db.User
	if err := dh.Select(&users, dh.Where("url", "=", data.TriggeredBy.Url)); err != nil {
		log.Error(err.Error())
		returnError(w, err.Error())
		return
	}

	var userId int64
	if len(users) == 0 {
		user := &db.User{
			Name:      data.TriggeredBy.Name,
			Url:       data.TriggeredBy.Url,
			AvatarUrl: data.TriggeredBy.AvatarUrl,
		}
		dh.Insert(user)
		userId, _ = dh.LastInsertId()
	} else {
		userId = users[0].Id
	}

	report := &db.Report{
		Status:      data.Status,
		ProjectId:   projectId,
		Branch:      data.Branch,
		CompareUrl:  data.CompareUrl,
		Start:       time.Unix(data.Start, 0),
		End:         time.Unix(data.End, 0),
		TriggeredBy: userId,
	}

	dh.Insert(report)

	reportId, _ := dh.LastInsertId()

	for _, commit := range data.Commits {
		c := &db.Commit{
			ReportId: reportId,
			Revision: commit.Revision,
			Author:   commit.Author,
			Message:  commit.Message,
			Url:      commit.Url,
		}
		dh.Insert(c)
	}

	for _, stage := range data.Stages {
		s := &db.Stage{
			Id:       stage.Id,
			ReportId: reportId,
			Name:     stage.Name,
			Status:   stage.Status,
			Log:      stage.Log,
			Start:    time.Unix(stage.Start, 0),
			End:      time.Unix(stage.End, 0),
		}
		dh.Insert(s)
		stageId, _ := dh.LastInsertId()

		for _, childStage := range stage.Stages {
			s := &db.Stage{
				Id:            stage.Id,
				ReportId:      reportId,
				ParentStageId: stageId,
				Name:          childStage.Name,
				Status:        childStage.Status,
				Log:           childStage.Log,
				Start:         time.Unix(childStage.Start, 0),
				End:           time.Unix(childStage.End, 0),
			}
			dh.Insert(s)
		}
	}

	data.Id = reportId
	b, _ := json.Marshal(data)
	fmt.Fprint(w, string(b))
}