func main() {
log.Init(&log.GlogRecorder{})
opts, err := config.LoadOpts(os.Args[1:])
switch err {
case nil:
case flag.ErrHelp:
os.Exit(0)
default:
os.Exit(2)
}
if opts.PrintVersion {
fmt.Printf("Walter version %s\n", version.Version)
os.Exit(0)
}
walter, err := walter.New(opts)
if err != nil {
log.Error(err.Error())
log.Error("failed to create Walter")
return
}
log.Info("running Walter")
result := walter.Run()
if result == false {
log.Info("more than one failures were detected running Walter")
log.Flush()
os.Exit(1)
}
log.Info("succeded to finish Walter")
log.Flush()
}
func (e *Walter) runService() bool {
// load .walter-update
log.Infof("Loading update file... \"%s\"", e.Engine.Resources.RepoService.GetUpdateFilePath())
update, err := services.LoadLastUpdate(e.Engine.Resources.RepoService.GetUpdateFilePath())
log.Infof("Succeeded loading update file")
log.Info("Updating status...")
update.Status = "inprogress"
result := services.SaveLastUpdate(e.Engine.Resources.RepoService.GetUpdateFilePath(), update)
if result == false {
log.Error("Failed to save status update")
return false
}
log.Info("Succeeded updating status")
// get latest commit and pull requests
log.Info("downloading commits and pull requests...")
commits, err := e.Engine.Resources.RepoService.GetCommits(update)
if err != nil {
log.Errorf("Failed getting commits: %s", err)
return false
}
log.Info("Succeeded getting commits")
log.Info("Size of commits: " + strconv.Itoa(commits.Len()))
hasFailedProcess := false
for commit := commits.Front(); commit != nil; commit = commit.Next() {
commitType := reflect.TypeOf(commit.Value)
if commitType.Name() == "RepositoryCommit" {
log.Info("Found new repository commit")
trunkCommit := commit.Value.(github.RepositoryCommit)
if result := e.processTrunkCommit(trunkCommit); result == false {
hasFailedProcess = true
}
} else if commitType.Name() == "PullRequest" {
log.Info("Found new pull request commit")
pullreq := commit.Value.(github.PullRequest)
if result := e.processPullRequest(pullreq); result == false {
hasFailedProcess = true
}
} else {
log.Errorf("Nothing commit type: %s", commitType)
hasFailedProcess = true
}
}
// save .walter-update
log.Info("Saving update file...")
update.Status = "finished"
update.Time = time.Now()
result = services.SaveLastUpdate(e.Engine.Resources.RepoService.GetUpdateFilePath(), update)
if result == false {
log.Error("Failed to save update")
return false
}
return !hasFailedProcess
}
func (e *Walter) processPullRequest(pullrequest github.PullRequest) bool {
// checkout pullrequest
num := *pullrequest.Number
_, err := exec.Command("git", "fetch", "origin", "refs/pull/"+strconv.Itoa(num)+"/head:pr_"+strconv.Itoa(num)).Output()
defer exec.Command("git", "checkout", "master", "-f").Output() // TODO: make trunk branch configurable
defer log.Info("returning master branch...")
if err != nil {
log.Errorf("Failed to fetch pull request: %s", err)
return false
}
_, err = exec.Command("git", "checkout", "pr_"+strconv.Itoa(num)).Output()
if err != nil {
log.Errorf("Failed to checkout pullrequest branch (\"%s\") : %s", "pr_"+strconv.Itoa(num), err)
log.Error("Skip execution...")
return false
}
// run pipeline
log.Info("Running pipeline...")
w, err := New(e.Opts)
if err != nil {
log.Errorf("Failed to create Walter object...: %s", err)
log.Error("Skip execution...")
return false
}
result := w.Engine.RunOnce()
// register the result to hosting service
if result.IsSucceeded() {
log.Info("succeeded.")
e.Engine.Resources.RepoService.RegisterResult(
services.Result{
State: "success",
Message: "Succeeded running pipeline...",
SHA: *pullrequest.Head.SHA})
return true
}
log.Error("Error reported...")
e.Engine.Resources.RepoService.RegisterResult(
services.Result{
State: "failure",
Message: "Failed running pipleline ...",
SHA: *pullrequest.Head.SHA})
return false
}
func main() {
var host, dbDir string
flags := flag.NewFlagSet("walter", flag.ExitOnError)
flags.StringVar(&host, "host", "0.0.0.0:8080", "The host of the application.")
flags.StringVar(&dbDir, "db_dir", "/var/lib/walter", "The directory of the sqlite3 db file.")
if err := flags.Parse(os.Args[1:]); err != nil {
panic(err)
}
if err := os.MkdirAll(dbDir, 0755); err != nil {
panic(err)
}
db.Init(dbDir)
r := route.GetRegexpHandler()
http.Handle("/", r)
log.Info(fmt.Sprintf("walter-server is listening on %s", host))
if err := http.ListenAndServe(host, nil); err != nil {
log.Error(fmt.Sprintf("ListenAndServe: %s", err))
}
}
func (parser *Parser) setFieldVal(fieldVal reflect.Value, stageOptVal interface{}, isReplace string) {
if fieldVal.Type() != reflect.ValueOf("string").Type() {
log.Error("found non string field value type...")
return
}
if isReplace == "true" {
fieldVal.SetString(parser.EnvVariables.Replace(stageOptVal.(string)))
} else {
fieldVal.SetString(parser.EnvVariables.ReplaceSpecialVariableToEnvVariable(stageOptVal.(string)))
}
}
func (e *Walter) processTrunkCommit(commit github.RepositoryCommit) bool {
log.Infof("Checkout master branch")
_, err := exec.Command("git", "checkout", "master", "-f").Output()
if err != nil {
log.Errorf("Failed to checkout master branch: %s", err)
return false
}
log.Infof("Downloading new commit from master")
_, err = exec.Command("git", "pull", "origin", "master").Output()
if err != nil {
log.Errorf("Failed to download new commit from master: %s", err)
return false
}
log.Infof("Running the latest commit in master")
w, err := New(e.Opts)
if err != nil {
log.Errorf("Failed to create Walter object...: %s", err)
log.Error("Skip execution...")
return false
}
result := w.Engine.RunOnce()
// register the result to hosting service
if result.IsSucceeded() {
log.Info("Succeeded.")
e.Engine.Resources.RepoService.RegisterResult(
services.Result{
State: "success",
Message: "Succeeded running pipeline...",
SHA: *commit.SHA})
return true
}
log.Error("Error reported...")
e.Engine.Resources.RepoService.RegisterResult(
services.Result{
State: "failure",
Message: "Failed running pipleline ...",
SHA: *commit.SHA})
return false
}
//GetCommits get a list of all the commits for the current update
func (githubClient *GitHubClient) GetCommits(update Update) (*list.List, error) {
log.Info("getting commits\n")
commits := list.New()
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: githubClient.Token},
)
tc := oauth2.NewClient(oauth2.NoContext, ts)
client := github.NewClient(tc)
// get a list of pull requests with Pull Request API
pullreqs, _, err := client.PullRequests.List(
githubClient.From, githubClient.Repo,
&github.PullRequestListOptions{})
if err != nil {
log.Errorf("Failed to get pull requests")
return list.New(), err
}
re, err := regexp.Compile(githubClient.TargetBranch)
if err != nil {
log.Error("Failed to compile branch pattern...")
return list.New(), err
}
log.Infof("Size of pull reqests: %d", len(pullreqs))
for _, pullreq := range pullreqs {
log.Infof("Branch name is \"%s\"", *pullreq.Head.Ref)
if githubClient.TargetBranch != "" {
matched := re.Match([]byte(*pullreq.Head.Ref))
if matched != true {
log.Infof("Not add a branch, \"%s\" since this branch name is not match the filtering pattern", *pullreq.Head.Ref)
continue
}
}
if *pullreq.State == "open" && pullreq.UpdatedAt.After(update.Time) {
log.Infof("Adding pullrequest %d", *pullreq.Number)
commits.PushBack(pullreq)
}
}
// get the latest commit with Commit API if the commit is newer than last update
masterCommits, _, _ := client.Repositories.ListCommits(
githubClient.From, githubClient.Repo, &github.CommitsListOptions{})
if masterCommits[0].Commit.Author.Date.After(update.Time) {
commits.PushBack(masterCommits[0])
}
return commits, nil
}
func printOutput(r io.Reader, prefix string, name string, out chan string, done chan bool) {
reader := bufio.NewReaderSize(r, 1048576)
for {
line, _, err := reader.ReadLine()
if err == io.EOF {
break
} else if err != nil {
log.Error(err.Error())
os.Exit(1)
}
log.Infof("[%s][command] %s output: %s", name, prefix, string(line))
out <- fmt.Sprintf("%s\n", string(line))
}
done <- true
}
func (j *Jobs) handlePullRequestEvent(w http.ResponseWriter, body string) {
var data payloadPullRequestEvent
err := json.Unmarshal([]byte(body), &data)
if err != nil {
log.Error(err.Error())
returnError(w, err.Error())
return
}
if data.Action != "opened" && data.Action != "synchronize" {
return
}
if data.PullRequest.Head.Repo.FullName == data.PullRequest.Base.Repo.FullName {
return
}
job := &Job{}
job.Project = data.PullRequest.Base.Repo.FullName
job.Revision = data.PullRequest.Head.Sha
job.HtmlUrl = data.PullRequest.Base.Repo.HtmlUrl
job.CloneUrl = data.PullRequest.Base.Repo.CloneUrl
job.Branch = data.PullRequest.Head.Ref
job.PullRequestUrl = data.PullRequest.HtmlUrl
job.CompareUrl = job.PullRequestUrl
job.PullRequestNumber = data.Number
job.StatusesUrl = data.PullRequest.StatusesUrl
job.TriggeredBy = &triggeredBy{
Name: data.Sender.Login,
Url: data.Sender.HtmlUrl,
AvatarUrl: data.Sender.AvatarUrl,
}
c := &commit{
Revision: data.PullRequest.Head.Sha,
Author: data.PullRequest.User.Login,
Message: fmt.Sprintf("Pull request #%d \"%s\"", data.Number, data.PullRequest.Title),
}
job.Commits = append(job.Commits, c)
j.jobs = append(j.jobs, job)
}
func validateWaitForCondition(wait *WaitFor) bool {
// check duplicate targets
if wait.Port != 0 && wait.File != "" {
log.Error("[command] Port and File are not able to be specified at the same time.")
return false
} else if wait.Port != 0 && wait.Delay > 0.0 {
log.Error("[command] Port and Delay are not able to be specified at the same time.")
return false
} else if wait.File != "" && wait.Delay > 0.0 {
log.Error("[command] File and Delay are not able to be specified at the same time.")
return false
}
// check illegal conditions
if wait.Delay < 0 {
log.Error("[command] Delay must be positive.")
return false
} else if wait.Port < 0 {
log.Error("[command] Port must be positive.")
return false
} else if wait.Port > 0 && wait.Host == "" {
log.Error("[command] Host must be specified when port number is specified.")
return false
}
// check illegal states
if wait.State != "present" && wait.State != "ready" && wait.State != "absent" && wait.State != "unready" {
log.Errorf("[command] \"%s\" is an unsupported state", wait.State)
return false
}
// misc checks
if wait.Port > 0 && wait.State == "" {
log.Error("[command] State must be specified for port.")
return false
} else if wait.File != "" && wait.State == "" {
log.Error("[command] State must be specified for file.")
return false
}
return true
}
func copyStream(reader io.Reader, prefix string, name string) string {
var err error
var n int
var buffer bytes.Buffer
tmpBuf := make([]byte, 1024)
for {
if n, err = reader.Read(tmpBuf); err != nil {
break
}
buffer.Write(tmpBuf[0:n])
log.Infof("[%s][command] %s output: %s", name, prefix, tmpBuf[0:n])
}
if err == io.EOF {
err = nil
} else {
log.Error("ERROR: " + err.Error())
}
return buffer.String()
}
func (j *Jobs) handlePushEvent(w http.ResponseWriter, body string) {
var data payloadPushEvent
err := json.Unmarshal([]byte(body), &data)
if err != nil {
log.Error(err.Error())
returnError(w, err.Error())
return
}
job := &Job{}
job.Project = data.Repository.FullName
job.Revision = data.After
job.HtmlUrl = data.Repository.HtmlUrl
job.CloneUrl = data.Repository.CloneUrl
job.CompareUrl = data.Compare
job.StatusesUrl = strings.Replace(data.Repository.StatusesUrl, "{sha}", job.Revision, 1)
ref := strings.Split(data.Ref, "/")
job.Branch = ref[len(ref)-1]
for _, c := range data.Commits {
job.Commits = append(job.Commits, &commit{
Revision: c.Id,
Author: c.Author.Username,
Message: c.Message,
Url: c.Url,
})
}
job.TriggeredBy = &triggeredBy{
Name: data.Sender.Login,
Url: data.Sender.HtmlUrl,
AvatarUrl: data.Sender.AvatarUrl,
}
j.jobs = append(j.jobs, job)
}
// Parse reads the specified configuration and create the pipeline.Resource.
func (parser *Parser) Parse() (*pipelines.Resources, error) {
// parse require block
requireFiles, ok := (*parser.ConfigData)["require"].([]interface{})
var required map[string]map[interface{}]interface{}
var err error
if ok == true {
log.Info("found \"require\" block")
required, err = parser.mapRequires(requireFiles)
if err != nil {
log.Error("failed to load requires...")
return nil, err
}
log.Info("number of registered stages: " + strconv.Itoa(len(required)))
} else {
log.Info("not found \"require\" block")
}
// parse service block
serviceOps, ok := (*parser.ConfigData)["service"].(map[interface{}]interface{})
var repoService services.Service
if ok == true {
log.Info("found \"service\" block")
repoService, err = parser.mapService(serviceOps)
if err != nil {
log.Error("failed to load service settings...")
return nil, err
}
} else {
log.Info("not found \"service\" block")
repoService, err = services.InitService("local")
if err != nil {
log.Error("failed to init local mode...")
return nil, err
}
}
// parse messenger block
messengerOps, ok := (*parser.ConfigData)["messenger"].(map[interface{}]interface{})
var messenger messengers.Messenger
if ok == true {
log.Info("found messenger block")
messenger, err = parser.mapMessenger(messengerOps)
if err != nil {
log.Error("failed to init messenger...")
return nil, err
}
} else {
log.Info("not found messenger block")
messenger, err = messengers.InitMessenger("fake")
if err != nil {
return nil, err
}
}
// parse cleanup block
var cleanup = &pipelines.Pipeline{}
cleanupData, ok := (*parser.ConfigData)["cleanup"].([]interface{})
if ok == true {
log.Info("found cleanup block")
cleanupList, err := parser.convertYamlMapToStages(cleanupData, required)
if err != nil {
log.Error("failed to create a stage in cleanup...")
return nil, err
}
for stageItem := cleanupList.Front(); stageItem != nil; stageItem = stageItem.Next() {
cleanup.AddStage(stageItem.Value.(stages.Stage))
}
} else {
log.Info("not found cleanup block in the input file")
}
// parse pipeline block
var pipeline = &pipelines.Pipeline{}
pipelineData, ok := (*parser.ConfigData)["pipeline"].([]interface{})
if ok == false {
return nil, fmt.Errorf("no pipeline block in the input file")
}
stageList, err := parser.convertYamlMapToStages(pipelineData, required)
if err != nil {
log.Error("failed to create a stage in pipeline...")
return nil, err
}
for stageItem := stageList.Front(); stageItem != nil; stageItem = stageItem.Next() {
pipeline.AddStage(stageItem.Value.(stages.Stage))
}
var resources = &pipelines.Resources{Pipeline: pipeline, Cleanup: cleanup, Reporter: messenger, RepoService: repoService}
return resources, nil
}
func createReport(w http.ResponseWriter, r *http.Request) {
rb := bufio.NewReader(r.Body)
var body string
for {
s, err := rb.ReadString('\n')
body = body + s
if err == io.EOF {
break
}
}
var data Report
err := json.Unmarshal([]byte(body), &data)
if err != nil {
log.Error(err.Error())
returnError(w, err.Error())
return
}
dh := db.GetHandler()
defer dh.Close()
var projects []db.Project
if err := dh.Select(&projects, dh.Where("repo", "=", data.Project.Repo)); err != nil {
log.Error(err.Error())
returnError(w, err.Error())
return
}
var projectId int64
if len(projects) == 0 {
project := &db.Project{Name: data.Project.Name, Repo: data.Project.Repo}
dh.Insert(project)
projectId, _ = dh.LastInsertId()
} else {
projectId = projects[0].Id
}
var users []db.User
if err := dh.Select(&users, dh.Where("url", "=", data.TriggeredBy.Url)); err != nil {
log.Error(err.Error())
returnError(w, err.Error())
return
}
var userId int64
if len(users) == 0 {
user := &db.User{
Name: data.TriggeredBy.Name,
Url: data.TriggeredBy.Url,
AvatarUrl: data.TriggeredBy.AvatarUrl,
}
dh.Insert(user)
userId, _ = dh.LastInsertId()
} else {
userId = users[0].Id
}
report := &db.Report{
Status: data.Status,
ProjectId: projectId,
Branch: data.Branch,
CompareUrl: data.CompareUrl,
Start: time.Unix(data.Start, 0),
End: time.Unix(data.End, 0),
TriggeredBy: userId,
}
dh.Insert(report)
reportId, _ := dh.LastInsertId()
for _, commit := range data.Commits {
c := &db.Commit{
ReportId: reportId,
Revision: commit.Revision,
Author: commit.Author,
Message: commit.Message,
Url: commit.Url,
}
dh.Insert(c)
}
for _, stage := range data.Stages {
s := &db.Stage{
Id: stage.Id,
ReportId: reportId,
Name: stage.Name,
Status: stage.Status,
Log: stage.Log,
Start: time.Unix(stage.Start, 0),
End: time.Unix(stage.End, 0),
}
dh.Insert(s)
stageId, _ := dh.LastInsertId()
for _, childStage := range stage.Stages {
s := &db.Stage{
Id: stage.Id,
ReportId: reportId,
ParentStageId: stageId,
Name: childStage.Name,
Status: childStage.Status,
Log: childStage.Log,
Start: time.Unix(childStage.Start, 0),
End: time.Unix(childStage.End, 0),
}
dh.Insert(s)
}
}
data.Id = reportId
b, _ := json.Marshal(data)
fmt.Fprint(w, string(b))
}