func main() {
log.Init(&log.GlogRecorder{})
opts, err := config.LoadOpts(os.Args[1:])
switch err {
case nil:
case flag.ErrHelp:
os.Exit(0)
default:
os.Exit(2)
}
if opts.PrintVersion {
fmt.Printf("Walter version %s\n", version.Version)
os.Exit(0)
}
walter, err := walter.New(opts)
if err != nil {
log.Error(err.Error())
log.Error("failed to create Walter")
return
}
log.Info("running Walter")
result := walter.Run()
if result == false {
log.Info("more than one failures were detected running Walter")
log.Flush()
os.Exit(1)
}
log.Info("succeded to finish Walter")
log.Flush()
}
// Run executes registered Pipeline.
func (e *Walter) Run() bool {
repoServiceValue := reflect.ValueOf(e.Engine.Resources.RepoService)
if e.Engine.Opts.Mode == "local" ||
repoServiceValue.Type().String() == "*services.LocalClient" {
log.Info("Starting Walter in local mode")
result := e.Engine.RunOnce()
return result.IsSucceeded()
}
log.Info("Starting Walter in repository service mode")
return e.runService()
}
func (e *Walter) processPullRequest(pullrequest github.PullRequest) bool {
// checkout pullrequest
num := *pullrequest.Number
_, err := exec.Command("git", "fetch", "origin", "refs/pull/"+strconv.Itoa(num)+"/head:pr_"+strconv.Itoa(num)).Output()
defer exec.Command("git", "checkout", "master", "-f").Output() // TODO: make trunk branch configurable
defer log.Info("returning master branch...")
if err != nil {
log.Errorf("Failed to fetch pull request: %s", err)
return false
}
_, err = exec.Command("git", "checkout", "pr_"+strconv.Itoa(num)).Output()
if err != nil {
log.Errorf("Failed to checkout pullrequest branch (\"%s\") : %s", "pr_"+strconv.Itoa(num), err)
log.Error("Skip execution...")
return false
}
// run pipeline
log.Info("Running pipeline...")
w, err := New(e.Opts)
if err != nil {
log.Errorf("Failed to create Walter object...: %s", err)
log.Error("Skip execution...")
return false
}
result := w.Engine.RunOnce()
// register the result to hosting service
if result.IsSucceeded() {
log.Info("succeeded.")
e.Engine.Resources.RepoService.RegisterResult(
services.Result{
State: "success",
Message: "Succeeded running pipeline...",
SHA: *pullrequest.Head.SHA})
return true
}
log.Error("Error reported...")
e.Engine.Resources.RepoService.RegisterResult(
services.Result{
State: "failure",
Message: "Failed running pipleline ...",
SHA: *pullrequest.Head.SHA})
return false
}
//InitService initializes the service
func InitService(stype string) (Service, error) {
var service Service
switch stype {
case "github":
log.Info("GitHub client was created")
service = new(GitHubClient)
case "local":
log.Info("local client was created")
service = new(LocalClient)
default:
err := fmt.Errorf("no messenger type: %s", stype)
return nil, err
}
return service, nil
}
//RegisterResult registers the supplied result
func (githubClient *GitHubClient) RegisterResult(result Result) error {
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: githubClient.Token},
)
tc := oauth2.NewClient(oauth2.NoContext, ts)
client := github.NewClient(tc)
if githubClient.BaseUrl != nil {
client.BaseURL = githubClient.BaseUrl
}
log.Info("Submitting result")
repositories := client.Repositories
status, _, err := repositories.CreateStatus(
githubClient.From,
githubClient.Repo,
result.SHA,
&github.RepoStatus{
State: github.String(result.State),
TargetURL: github.String(result.Url),
Description: github.String(result.Message),
Context: github.String("continuous-integraion/walter"),
})
log.Infof("Submit status: %s", status)
if err != nil {
log.Errorf("Failed to register result: %s", err)
}
return err
}
func main() {
var host, dbDir string
flags := flag.NewFlagSet("walter", flag.ExitOnError)
flags.StringVar(&host, "host", "0.0.0.0:8080", "The host of the application.")
flags.StringVar(&dbDir, "db_dir", "/var/lib/walter", "The directory of the sqlite3 db file.")
if err := flags.Parse(os.Args[1:]); err != nil {
panic(err)
}
if err := os.MkdirAll(dbDir, 0755); err != nil {
panic(err)
}
db.Init(dbDir)
r := route.GetRegexpHandler()
http.Handle("/", r)
log.Info(fmt.Sprintf("walter-server is listening on %s", host))
if err := http.ListenAndServe(host, nil); err != nil {
log.Error(fmt.Sprintf("ListenAndServe: %s", err))
}
}
func (parser *Parser) mapMessenger(messengerMap map[interface{}]interface{}) (messengers.Messenger, error) {
messengerType := messengerMap["type"].(string)
log.Info("type of reporter is " + messengerType)
messenger, err := messengers.InitMessenger(messengerType)
if err != nil {
return nil, err
}
newMessengerValue := reflect.ValueOf(messenger).Elem()
newMessengerType := reflect.TypeOf(messenger).Elem()
for i := 0; i < newMessengerType.NumField(); i++ {
tagName := newMessengerType.Field(i).Tag.Get("config")
for messengerOptKey, messengerOptVal := range messengerMap {
if tagName != messengerOptKey {
continue
}
fieldVal := newMessengerValue.Field(i)
if fieldVal.Type() == reflect.ValueOf("string").Type() {
fieldVal.SetString(parser.EnvVariables.Replace(messengerOptVal.(string)))
} else if fieldVal.Type().String() == "messengers.BaseMessenger" {
elements := messengerOptVal.([]interface{})
suppressor := fieldVal.Interface().(messengers.BaseMessenger)
for _, element := range elements {
suppressor.SuppressFields = append(suppressor.SuppressFields, element.(string))
}
fieldVal.Set(reflect.ValueOf(suppressor))
}
}
}
return messenger, nil
}
//LoadLastUpdate loads the last update
func LoadLastUpdate(fname string) (Update, error) {
file, err := ioutil.ReadFile(fname)
log.Infof("opening file: \"%s\"...", fname)
if err != nil {
log.Warnf("error occured opening file: \"%s\" ...", fname)
log.Warnf(err.Error())
log.Warnf("continue the process with the new settings")
update := Update{
Time: time.Date(1970, time.November, 10, 15, 0, 0, 0, time.Local),
Succeeded: true,
Status: "inprogress"}
return update, nil
}
log.Infof("loading last update form \"%s\"\n", string(file))
var update Update
if err := json.Unmarshal(file, &update); err != nil {
log.Warnf("failed to load \"%s\" ...", fname)
log.Warnf(err.Error())
log.Warnf("continue the process with the new settings")
update := Update{
Time: time.Now(),
Succeeded: true,
Status: "inprogress"}
return update, nil
}
if update.Status == "inprogress" {
return Update{}, errors.New("update is currently run in another process")
}
log.Info("setting update status into \"inprogress\"...")
return update, nil
}
func (e *Walter) runService() bool {
// load .walter-update
log.Infof("Loading update file... \"%s\"", e.Engine.Resources.RepoService.GetUpdateFilePath())
update, err := services.LoadLastUpdate(e.Engine.Resources.RepoService.GetUpdateFilePath())
log.Infof("Succeeded loading update file")
log.Info("Updating status...")
update.Status = "inprogress"
result := services.SaveLastUpdate(e.Engine.Resources.RepoService.GetUpdateFilePath(), update)
if result == false {
log.Error("Failed to save status update")
return false
}
log.Info("Succeeded updating status")
// get latest commit and pull requests
log.Info("downloading commits and pull requests...")
commits, err := e.Engine.Resources.RepoService.GetCommits(update)
if err != nil {
log.Errorf("Failed getting commits: %s", err)
return false
}
log.Info("Succeeded getting commits")
log.Info("Size of commits: " + strconv.Itoa(commits.Len()))
hasFailedProcess := false
for commit := commits.Front(); commit != nil; commit = commit.Next() {
commitType := reflect.TypeOf(commit.Value)
if commitType.Name() == "RepositoryCommit" {
log.Info("Found new repository commit")
trunkCommit := commit.Value.(github.RepositoryCommit)
if result := e.processTrunkCommit(trunkCommit); result == false {
hasFailedProcess = true
}
} else if commitType.Name() == "PullRequest" {
log.Info("Found new pull request commit")
pullreq := commit.Value.(github.PullRequest)
if result := e.processPullRequest(pullreq); result == false {
hasFailedProcess = true
}
} else {
log.Errorf("Nothing commit type: %s", commitType)
hasFailedProcess = true
}
}
// save .walter-update
log.Info("Saving update file...")
update.Status = "finished"
update.Time = time.Now()
result = services.SaveLastUpdate(e.Engine.Resources.RepoService.GetUpdateFilePath(), update)
if result == false {
log.Error("Failed to save update")
return false
}
return !hasFailedProcess
}
func (parser *Parser) mapRequire(requireData map[interface{}]interface{},
requires *map[string]map[interface{}]interface{}) {
namespace := requireData["namespace"].(string)
log.Info("detect namespace: " + namespace)
stages := requireData["stages"].([]interface{})
log.Info("number of detected stages: " + strconv.Itoa(len(stages)))
for _, stageDetail := range stages {
stageMap := stageDetail.(map[interface{}]interface{})
for _, values := range stageMap {
valueMap := values.(map[interface{}]interface{})
stageKey := namespace + "::" + valueMap["name"].(string)
log.Info("register stage: " + stageKey)
(*requires)[stageKey] = valueMap
}
}
}
// WaitFor wait until the condtions are satisfied
func (waitFor *WaitFor) Wait() {
// delay
if waitFor.Delay > 0.0 {
log.Info("Wait specified time: " + strconv.FormatFloat(waitFor.Delay, 'f', 6, 64))
time.Sleep(time.Duration(waitFor.Delay) * time.Second)
return
}
// file created
if waitFor.File != "" && (waitFor.State == "present" || waitFor.State == "ready") {
log.Info("Wait for file: " + waitFor.File + " is created...")
for {
if isFileExist(waitFor.File) {
log.Info("File: " + waitFor.File + " found.")
return
} else {
time.Sleep(10 * time.Millisecond)
}
}
}
// file removed
if waitFor.File != "" && (waitFor.State == "absent" || waitFor.State == "unready") {
log.Info("Wait for file: " + waitFor.File + " is removed...")
for {
if !isFileExist(waitFor.File) {
log.Info("File: " + waitFor.File + " removed.")
return
} else {
time.Sleep(10 * time.Millisecond)
}
}
}
// port open
if waitFor.Host != "" && waitFor.Port > 0 && (waitFor.State == "present" || waitFor.State == "ready") {
log.Info("Wait for port: " + waitFor.Host + ":" + strconv.Itoa(waitFor.Port) + " is opened...")
for {
if isConnect(waitFor.Host, waitFor.Port) {
return
} else {
time.Sleep(10 * time.Millisecond)
}
}
}
// port close
if waitFor.Host != "" && waitFor.Port > 0 && (waitFor.State == "absent" || waitFor.State == "unready") {
log.Info("Wait for: " + waitFor.Host + ":" + strconv.Itoa(waitFor.Port) + " is closed...")
for {
if !isConnect(waitFor.Host, waitFor.Port) {
return
} else {
time.Sleep(10 * time.Millisecond)
}
}
}
}
func (parser *Parser) mapRequires(requireList []interface{}) (map[string]map[interface{}]interface{}, error) {
requires := make(map[string]map[interface{}]interface{})
for _, requireFile := range requireList {
replacedFilePath := parser.EnvVariables.Replace(requireFile.(string))
log.Info("register require file: " + replacedFilePath)
requireData, err := ReadConfig(replacedFilePath)
if err != nil {
return nil, err
}
parser.mapRequire(*requireData, &requires)
}
return requires, nil
}
//GetCommits get a list of all the commits for the current update
func (githubClient *GitHubClient) GetCommits(update Update) (*list.List, error) {
log.Info("getting commits\n")
commits := list.New()
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: githubClient.Token},
)
tc := oauth2.NewClient(oauth2.NoContext, ts)
client := github.NewClient(tc)
// get a list of pull requests with Pull Request API
pullreqs, _, err := client.PullRequests.List(
githubClient.From, githubClient.Repo,
&github.PullRequestListOptions{})
if err != nil {
log.Errorf("Failed to get pull requests")
return list.New(), err
}
re, err := regexp.Compile(githubClient.TargetBranch)
if err != nil {
log.Error("Failed to compile branch pattern...")
return list.New(), err
}
log.Infof("Size of pull reqests: %d", len(pullreqs))
for _, pullreq := range pullreqs {
log.Infof("Branch name is \"%s\"", *pullreq.Head.Ref)
if githubClient.TargetBranch != "" {
matched := re.Match([]byte(*pullreq.Head.Ref))
if matched != true {
log.Infof("Not add a branch, \"%s\" since this branch name is not match the filtering pattern", *pullreq.Head.Ref)
continue
}
}
if *pullreq.State == "open" && pullreq.UpdatedAt.After(update.Time) {
log.Infof("Adding pullrequest %d", *pullreq.Number)
commits.PushBack(pullreq)
}
}
// get the latest commit with Commit API if the commit is newer than last update
masterCommits, _, _ := client.Repositories.ListCommits(
githubClient.From, githubClient.Repo, &github.CommitsListOptions{})
if masterCommits[0].Commit.Author.Date.After(update.Time) {
commits.PushBack(masterCommits[0])
}
return commits, nil
}
func (parser *Parser) extractStage(stageMap map[interface{}]interface{},
requiredStages map[string]map[interface{}]interface{}) (map[interface{}]interface{}, error) {
if stageMap["call"] == nil {
return stageMap, nil
}
// when "call" is applied
log.Info("detect call")
stageName := stageMap["call"].(string)
calledMap := requiredStages[stageName]
if calledMap == nil {
return nil, errors.New(stageName + " is not registerd")
}
for fieldName, fieldValue := range calledMap {
log.Info("fieldName: " + fieldName.(string))
if _, ok := stageMap[fieldName]; ok {
return nil, errors.New("overriding required stage is forbidden")
}
stageMap[fieldName] = fieldValue
}
log.Info("stage name: " + stageName)
stageMap["name"] = stageName
return stageMap, nil
}
func (e *Walter) processTrunkCommit(commit github.RepositoryCommit) bool {
log.Infof("Checkout master branch")
_, err := exec.Command("git", "checkout", "master", "-f").Output()
if err != nil {
log.Errorf("Failed to checkout master branch: %s", err)
return false
}
log.Infof("Downloading new commit from master")
_, err = exec.Command("git", "pull", "origin", "master").Output()
if err != nil {
log.Errorf("Failed to download new commit from master: %s", err)
return false
}
log.Infof("Running the latest commit in master")
w, err := New(e.Opts)
if err != nil {
log.Errorf("Failed to create Walter object...: %s", err)
log.Error("Skip execution...")
return false
}
result := w.Engine.RunOnce()
// register the result to hosting service
if result.IsSucceeded() {
log.Info("Succeeded.")
e.Engine.Resources.RepoService.RegisterResult(
services.Result{
State: "success",
Message: "Succeeded running pipeline...",
SHA: *commit.SHA})
return true
}
log.Error("Error reported...")
e.Engine.Resources.RepoService.RegisterResult(
services.Result{
State: "failure",
Message: "Failed running pipleline ...",
SHA: *commit.SHA})
return false
}
//RegisterResult registers the supplied result
func (githubClient *GitHubClient) RegisterResult(result Result) error {
t := &oauth.Transport{
Token: &oauth.Token{AccessToken: githubClient.Token},
}
client := github.NewClient(t.Client())
log.Info("Submitting result")
repositories := client.Repositories
status, _, err := repositories.CreateStatus(
githubClient.From,
githubClient.Repo,
result.SHA,
&github.RepoStatus{
State: github.String(result.State),
TargetURL: github.String(result.Url),
Description: github.String(result.Message),
Context: github.String("continuous-integraion/walter"),
})
log.Infof("Submit status: %s", status)
if err != nil {
log.Errorf("Failed to register result: %s", err)
}
return err
}
func (h *RegexpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
line := fmt.Sprintf("%s %s", r.Method, r.URL.Path)
if q := r.URL.RawQuery; q != "" {
line = fmt.Sprintf("%s?%s", line, q)
}
log.Info(line)
for _, route := range h.routes {
if route.pattern.MatchString(r.URL.Path) {
route.handler.ServeHTTP(w, r)
return
}
}
path := r.URL.Path
if path == "/" {
path = "/index.html"
}
b, err := assets.Asset("web" + path)
if err != nil {
w.WriteHeader(http.StatusNotFound)
fmt.Fprint(w, err.Error())
return
}
ext := filepath.Ext(path)
switch ext {
case ".css":
w.Header().Set("Content-Type", "text/css")
case ".js":
w.Header().Set("Content-Type", "application/javascript")
}
w.Write(b)
}
func (parser *Parser) mapService(serviceMap map[interface{}]interface{}) (services.Service, error) {
serviceType := serviceMap["type"].(string)
log.Info("type of service is " + serviceType)
service, err := services.InitService(serviceType)
if err != nil {
return nil, err
}
newServiceValue := reflect.ValueOf(service).Elem()
newServiceType := reflect.TypeOf(service).Elem()
for i := 0; i < newServiceType.NumField(); i++ {
tagName := newServiceType.Field(i).Tag.Get("config")
for serviceOptKey, serviceOptVal := range serviceMap {
if tagName != serviceOptKey {
continue
}
fieldVal := newServiceValue.Field(i)
if fieldVal.Type() == reflect.ValueOf("string").Type() {
fieldVal.SetString(parser.EnvVariables.Replace(serviceOptVal.(string)))
}
}
}
return service, nil
}
// Parse reads the specified configuration and create the pipeline.Resource.
func (parser *Parser) Parse() (*pipelines.Resources, error) {
// parse require block
requireFiles, ok := (*parser.ConfigData)["require"].([]interface{})
var required map[string]map[interface{}]interface{}
var err error
if ok == true {
log.Info("found \"require\" block")
required, err = parser.mapRequires(requireFiles)
if err != nil {
log.Error("failed to load requires...")
return nil, err
}
log.Info("number of registered stages: " + strconv.Itoa(len(required)))
} else {
log.Info("not found \"require\" block")
}
// parse service block
serviceOps, ok := (*parser.ConfigData)["service"].(map[interface{}]interface{})
var repoService services.Service
if ok == true {
log.Info("found \"service\" block")
repoService, err = parser.mapService(serviceOps)
if err != nil {
log.Error("failed to load service settings...")
return nil, err
}
} else {
log.Info("not found \"service\" block")
repoService, err = services.InitService("local")
if err != nil {
log.Error("failed to init local mode...")
return nil, err
}
}
// parse messenger block
messengerOps, ok := (*parser.ConfigData)["messenger"].(map[interface{}]interface{})
var messenger messengers.Messenger
if ok == true {
log.Info("found messenger block")
messenger, err = parser.mapMessenger(messengerOps)
if err != nil {
log.Error("failed to init messenger...")
return nil, err
}
} else {
log.Info("not found messenger block")
messenger, err = messengers.InitMessenger("fake")
if err != nil {
return nil, err
}
}
// parse cleanup block
var cleanup = &pipelines.Pipeline{}
cleanupData, ok := (*parser.ConfigData)["cleanup"].([]interface{})
if ok == true {
log.Info("found cleanup block")
cleanupList, err := parser.convertYamlMapToStages(cleanupData, required)
if err != nil {
log.Error("failed to create a stage in cleanup...")
return nil, err
}
for stageItem := cleanupList.Front(); stageItem != nil; stageItem = stageItem.Next() {
cleanup.AddStage(stageItem.Value.(stages.Stage))
}
} else {
log.Info("not found cleanup block in the input file")
}
// parse pipeline block
var pipeline = &pipelines.Pipeline{}
pipelineData, ok := (*parser.ConfigData)["pipeline"].([]interface{})
if ok == false {
return nil, fmt.Errorf("no pipeline block in the input file")
}
stageList, err := parser.convertYamlMapToStages(pipelineData, required)
if err != nil {
log.Error("failed to create a stage in pipeline...")
return nil, err
}
for stageItem := stageList.Front(); stageItem != nil; stageItem = stageItem.Next() {
pipeline.AddStage(stageItem.Value.(stages.Stage))
}
var resources = &pipelines.Resources{Pipeline: pipeline, Cleanup: cleanup, Reporter: messenger, RepoService: repoService}
return resources, nil
}