Пример #1
0
func main() {
	log.Init(&log.GlogRecorder{})

	opts, err := config.LoadOpts(os.Args[1:])

	switch err {
	case nil:
	case flag.ErrHelp:
		os.Exit(0)
	default:
		os.Exit(2)
	}

	if opts.PrintVersion {
		fmt.Printf("Walter version %s\n", version.Version)
		os.Exit(0)
	}

	walter, err := walter.New(opts)
	if err != nil {
		log.Error(err.Error())
		log.Error("failed to create Walter")
		return
	}
	log.Info("running Walter")
	result := walter.Run()
	if result == false {
		log.Info("more than one failures were detected running Walter")
		log.Flush()
		os.Exit(1)
	}
	log.Info("succeded to finish Walter")
	log.Flush()
}
Пример #2
0
// Run executes registered Pipeline.
func (e *Walter) Run() bool {
	repoServiceValue := reflect.ValueOf(e.Engine.Resources.RepoService)
	if e.Engine.Opts.Mode == "local" ||
		repoServiceValue.Type().String() == "*services.LocalClient" {
		log.Info("Starting Walter in local mode")
		result := e.Engine.RunOnce()
		return result.IsSucceeded()
	} else {
		log.Info("Starting Walter in repository service mode")
		return e.runService()
	}
}
Пример #3
0
func (e *Walter) processPullRequest(pullrequest github.PullRequest) bool {
	// checkout pullrequest
	num := *pullrequest.Number
	_, err := exec.Command("git", "fetch", "origin", "refs/pull/"+strconv.Itoa(num)+"/head:pr_"+strconv.Itoa(num)).Output()

	defer exec.Command("git", "checkout", "master", "-f").Output() // TODO: make trunk branch configurable
	defer log.Info("returning master branch...")

	if err != nil {
		log.Errorf("Failed to fetch pull request: %s", err)
		return false
	}

	_, err = exec.Command("git", "checkout", "pr_"+strconv.Itoa(num)).Output()
	if err != nil {
		log.Errorf("Failed to checkout pullrequest branch (\"%s\") : %s", "pr_"+strconv.Itoa(num), err)
		log.Error("Skip execution...")
		return false
	}

	// run pipeline
	log.Info("Running pipeline...")
	w, err := New(e.Opts)
	if err != nil {
		log.Errorf("Failed to create Walter object...: %s", err)
		log.Error("Skip execution...")
		return false
	}

	result := w.Engine.RunOnce()

	// register the result to hosting service
	if result.IsSucceeded() {
		log.Info("succeeded.")
		e.Engine.Resources.RepoService.RegisterResult(
			services.Result{
				State:   "success",
				Message: "Succeeded running pipeline...",
				SHA:     *pullrequest.Head.SHA})
		return true
	} else {
		log.Error("Error reported...")
		e.Engine.Resources.RepoService.RegisterResult(
			services.Result{
				State:   "failure",
				Message: "Failed running pipleline ...",
				SHA:     *pullrequest.Head.SHA})
		return false
	}
}
Пример #4
0
func InitService(stype string) (Service, error) {
	var service Service
	switch stype {
	case "github":
		log.Info("GitHub client was created")
		service = new(GitHubClient)
	case "local":
		log.Info("local client was created")
		service = new(LocalClient)
	default:
		err := fmt.Errorf("no messenger type: %s", stype)
		return nil, err
	}
	return service, nil
}
Пример #5
0
func (parser *Parser) mapMessenger(messengerMap map[interface{}]interface{}) (messengers.Messenger, error) {
	messengerType := messengerMap["type"].(string)
	log.Info("type of reporter is " + messengerType)
	messenger, err := messengers.InitMessenger(messengerType)
	if err != nil {
		return nil, err
	}
	newMessengerValue := reflect.ValueOf(messenger).Elem()
	newMessengerType := reflect.TypeOf(messenger).Elem()
	for i := 0; i < newMessengerType.NumField(); i++ {
		tagName := newMessengerType.Field(i).Tag.Get("config")
		for messengerOptKey, messengerOptVal := range messengerMap {
			if tagName == messengerOptKey {
				fieldVal := newMessengerValue.Field(i)
				if fieldVal.Type() == reflect.ValueOf("string").Type() {
					fieldVal.SetString(parser.EnvVariables.Replace(messengerOptVal.(string)))
				} else if fieldVal.Type().String() == "messengers.BaseMessenger" {
					elements := messengerOptVal.([]interface{})
					suppressor := fieldVal.Interface().(messengers.BaseMessenger)
					for _, element := range elements {
						suppressor.SuppressFields = append(suppressor.SuppressFields, element.(string))
					}
					fieldVal.Set(reflect.ValueOf(suppressor))
				}
			}
		}
	}
	return messenger, nil
}
Пример #6
0
func LoadLastUpdate(fname string) (Update, error) {
	file, err := ioutil.ReadFile(fname)
	log.Infof("opening file: \"%s\"...", fname)
	if err != nil {
		log.Warnf("error occured opening file: \"%s\" ...", fname)
		log.Warnf(err.Error())
		log.Warnf("continue the process with the new settings")
		update := Update{
			Time:      time.Date(1970, time.November, 10, 15, 0, 0, 0, time.Local),
			Succeeded: true,
			Status:    "inprogress"}
		return update, nil
	}

	log.Infof("loading last update form \"%s\"\n", string(file))
	var update Update
	if err := json.Unmarshal(file, &update); err != nil {
		log.Warnf("failed to load \"%s\" ...", fname)
		log.Warnf(err.Error())
		log.Warnf("continue the process with the new settings")
		update := Update{
			Time:      time.Now(),
			Succeeded: true,
			Status:    "inprogress"}
		return update, nil
	}

	if update.Status == "inprogress" {
		return Update{}, errors.New("update is currently run in another process")
	}

	log.Info("setting update status into \"inprogress\"...")
	return update, nil
}
Пример #7
0
func (e *Walter) runService() bool {
	// load .walter-update
	log.Infof("Loading update file... \"%s\"", e.Engine.Resources.RepoService.GetUpdateFilePath())
	update, err := services.LoadLastUpdate(e.Engine.Resources.RepoService.GetUpdateFilePath())
	log.Infof("Succeeded loading update file")

	log.Info("Updating status...")
	update.Status = "inprogress"
	result := services.SaveLastUpdate(e.Engine.Resources.RepoService.GetUpdateFilePath(), update)
	if result == false {
		log.Error("Failed to save status update")
		return false
	}
	log.Info("Succeeded updating status")

	// get latest commit and pull requests
	log.Info("downloading commits and pull requests...")
	commits, err := e.Engine.Resources.RepoService.GetCommits(update)
	if err != nil {
		log.Errorf("Failed getting commits: %s", err)
		return false
	}

	log.Info("Succeeded getting commits")
	log.Info("Size of commits: " + strconv.Itoa(commits.Len()))
	has_failed_process := false
	for commit := commits.Front(); commit != nil; commit = commit.Next() {
		commitType := reflect.TypeOf(commit.Value)
		if commitType.Name() == "RepositoryCommit" {
			log.Info("Found new repository commit")
			trunkCommit := commit.Value.(github.RepositoryCommit)
			if result := e.processTrunkCommit(trunkCommit); result == false {
				has_failed_process = true
			}
		} else if commitType.Name() == "PullRequest" {
			log.Info("Found new pull request commit")
			pullreq := commit.Value.(github.PullRequest)
			if result := e.processPullRequest(pullreq); result == false {
				has_failed_process = true
			}
		} else {
			log.Errorf("Nothing commit type: %s", commitType)
			has_failed_process = true
		}
	}

	// save .walter-update
	log.Info("Saving update file...")
	update.Status = "finished"
	update.Time = time.Now()
	result = services.SaveLastUpdate(e.Engine.Resources.RepoService.GetUpdateFilePath(), update)
	if result == false {
		log.Error("Failed to save update")
		return false
	}
	return !has_failed_process
}
Пример #8
0
func (parser *Parser) mapRequire(requireData map[interface{}]interface{},
	requires *map[string]map[interface{}]interface{}) {
	namespace := requireData["namespace"].(string)
	log.Info("detect namespace: " + namespace)

	stages := requireData["stages"].([]interface{})
	log.Info("number of detected stages: " + strconv.Itoa(len(stages)))

	for _, stageDetail := range stages {
		stageMap := stageDetail.(map[interface{}]interface{})
		for _, values := range stageMap {
			valueMap := values.(map[interface{}]interface{})
			stageKey := namespace + "::" + valueMap["name"].(string)
			log.Info("register stage: " + stageKey)
			(*requires)[stageKey] = valueMap
		}
	}
}
Пример #9
0
func (parser *Parser) mapRequires(requireList []interface{}) (map[string]map[interface{}]interface{}, error) {
	requires := make(map[string]map[interface{}]interface{})
	for _, requireFile := range requireList {
		replacedFilePath := parser.EnvVariables.Replace(requireFile.(string))
		log.Info("register require file: " + replacedFilePath)
		requireData, err := ReadConfig(replacedFilePath)
		if err != nil {
			return nil, err
		}
		parser.mapRequire(*requireData, &requires)
	}
	return requires, nil
}
Пример #10
0
func (self *Parser) extractStage(stageMap map[interface{}]interface{},
	requiredStages map[string]map[interface{}]interface{}) (map[interface{}]interface{}, error) {
	if stageMap["call"] != nil {
		log.Info("detect call")
		stageName := stageMap["call"].(string)
		calledMap := requiredStages[stageName]
		if calledMap == nil {
			return nil, errors.New(stageName + " is not registerd")
		}
		for fieldName, fieldValue := range calledMap {
			log.Info("fieldName: " + fieldName.(string))
			if _, ok := stageMap[fieldName]; ok {
				return nil, errors.New("overriding required stage is forbidden")
			} else {
				stageMap[fieldName] = fieldValue
			}
		}
		log.Info("stageName: " + stageName)
		stageMap["name"] = stageName
	}
	return stageMap, nil
}
Пример #11
0
//GetCommits get a list of all the commits for the current update
func (githubClient *GitHubClient) GetCommits(update Update) (*list.List, error) {
	log.Info("getting commits\n")
	commits := list.New()
	t := &oauth.Transport{
		Token: &oauth.Token{AccessToken: githubClient.Token},
	}
	client := github.NewClient(t.Client())

	// get a list of pull requests with Pull Request API
	pullreqs, _, err := client.PullRequests.List(
		githubClient.From, githubClient.Repo,
		&github.PullRequestListOptions{})
	if err != nil {
		log.Errorf("Failed to get pull requests")
		return list.New(), err
	}

	re, err := regexp.Compile(githubClient.TargetBranch)
	if err != nil {
		log.Error("Failed to compile branch pattern...")
		return list.New(), err
	}

	log.Infof("Size of pull reqests: %d", len(pullreqs))
	for _, pullreq := range pullreqs {
		log.Infof("Branch name is \"%s\"", *pullreq.Head.Ref)

		if githubClient.TargetBranch != "" {
			matched := re.Match([]byte(*pullreq.Head.Ref))
			if matched != true {
				log.Infof("Not add a branch, \"%s\" since this branch name is not match the filtering pattern", *pullreq.Head.Ref)
				continue
			}
		}

		if *pullreq.State == "open" && pullreq.UpdatedAt.After(update.Time) {
			log.Infof("Adding pullrequest %d", *pullreq.Number)
			commits.PushBack(pullreq)
		}
	}

	// get the latest commit with Commit API if the commit is newer than last update
	masterCommits, _, _ := client.Repositories.ListCommits(
		githubClient.From, githubClient.Repo, &github.CommitsListOptions{})
	if masterCommits[0].Commit.Author.Date.After(update.Time) {
		commits.PushBack(masterCommits[0])
	}
	return commits, nil
}
Пример #12
0
func (e *Walter) processTrunkCommit(commit github.RepositoryCommit) bool {
	log.Infof("Checkout master branch")
	_, err := exec.Command("git", "checkout", "master", "-f").Output()
	if err != nil {
		log.Errorf("Failed to checkout master branch: %s", err)
		return false
	}
	log.Infof("Downloading new commit from master")
	_, err = exec.Command("git", "pull", "origin", "master").Output()
	if err != nil {
		log.Errorf("Failed to download new commit from master: %s", err)
		return false
	}
	log.Infof("Running the latest commit in master")
	w, err := New(e.Opts)
	if err != nil {
		log.Errorf("Failed to create Walter object...: %s", err)
		log.Error("Skip execution...")
		return false
	}
	result := w.Engine.RunOnce()

	// register the result to hosting service
	if result.IsSucceeded() {
		log.Info("Succeeded.")
		e.Engine.Resources.RepoService.RegisterResult(
			services.Result{
				State:   "success",
				Message: "Succeeded running pipeline...",
				SHA:     *commit.SHA})
		return true
	} else {
		log.Error("Error reported...")
		e.Engine.Resources.RepoService.RegisterResult(
			services.Result{
				State:   "failure",
				Message: "Failed running pipleline ...",
				SHA:     *commit.SHA})
		return false
	}
}
Пример #13
0
func (parser *Parser) mapService(serviceMap map[interface{}]interface{}) (services.Service, error) {
	serviceType := serviceMap["type"].(string)
	log.Info("type of service is " + serviceType)
	service, err := services.InitService(serviceType)
	if err != nil {
		return nil, err
	}

	newServiceValue := reflect.ValueOf(service).Elem()
	newServiceType := reflect.TypeOf(service).Elem()
	for i := 0; i < newServiceType.NumField(); i++ {
		tagName := newServiceType.Field(i).Tag.Get("config")
		for serviceOptKey, serviceOptVal := range serviceMap {
			if tagName == serviceOptKey {
				fieldVal := newServiceValue.Field(i)
				if fieldVal.Type() == reflect.ValueOf("string").Type() {
					fieldVal.SetString(parser.EnvVariables.Replace(serviceOptVal.(string)))
				}
			}
		}
	}
	return service, nil
}
Пример #14
0
//RegisterResult registers the supplied result
func (githubClient *GitHubClient) RegisterResult(result Result) error {
	t := &oauth.Transport{
		Token: &oauth.Token{AccessToken: githubClient.Token},
	}
	client := github.NewClient(t.Client())

	log.Info("Submitting result")
	repositories := client.Repositories
	status, _, err := repositories.CreateStatus(
		githubClient.From,
		githubClient.Repo,
		result.SHA,
		&github.RepoStatus{
			State:       github.String(result.State),
			TargetURL:   github.String(""),
			Description: github.String(result.Message),
			Context:     github.String("continuous-integraion/walter"),
		})
	log.Infof("Submit status: %s", status)
	if err != nil {
		log.Errorf("Failed to register result: %s", err)
	}
	return err
}
Пример #15
0
// Parse reads the specified configuration and create the pipeline.Resource.
func (parser *Parser) Parse() (*pipelines.Resources, error) {
	// parse require block
	requireFiles, ok := (*parser.ConfigData)["require"].([]interface{})
	var required map[string]map[interface{}]interface{}
	var err error
	if ok == true {
		log.Info("found \"require\" block")
		required, err = parser.mapRequires(requireFiles)
		if err != nil {
			log.Error("failed to load requires...")
			return nil, err
		}
		log.Info("number of registered stages: " + strconv.Itoa(len(required)))
	} else {
		log.Info("not found \"require\" block")
	}

	// parse service block
	serviceOps, ok := (*parser.ConfigData)["service"].(map[interface{}]interface{})
	var repoService services.Service
	if ok == true {
		log.Info("found \"service\" block")
		repoService, err = parser.mapService(serviceOps)
		if err != nil {
			log.Error("failed to load service settings...")
			return nil, err
		}
	} else {
		log.Info("not found \"service\" block")
		repoService, err = services.InitService("local")
		if err != nil {
			log.Error("failed to init local mode...")
			return nil, err
		}
	}

	// parse messenger block
	messengerOps, ok := (*parser.ConfigData)["messenger"].(map[interface{}]interface{})
	var messenger messengers.Messenger
	if ok == true {
		log.Info("found messenger block")
		messenger, err = parser.mapMessenger(messengerOps)
		if err != nil {
			log.Error("failed to init messenger...")
			return nil, err
		}
	} else {
		log.Info("not found messenger block")
		messenger, err = messengers.InitMessenger("fake")
		if err != nil {
			return nil, err
		}
	}

	// parse cleanup block
	var cleanup = &pipelines.Pipeline{}
	cleanupData, ok := (*parser.ConfigData)["cleanup"].([]interface{})
	if ok == true {
		log.Info("found cleanup block")
		cleanupList, err := parser.convertYamlMapToStages(cleanupData, required)
		if err != nil {
			log.Error("failed to create a stage in cleanup...")
			return nil, err
		}
		for stageItem := cleanupList.Front(); stageItem != nil; stageItem = stageItem.Next() {
			cleanup.AddStage(stageItem.Value.(stages.Stage))
		}
	} else {
		log.Info("not found cleanup block in the input file")
	}

	// parse pipeline block
	var pipeline = &pipelines.Pipeline{}

	pipelineData, ok := (*parser.ConfigData)["pipeline"].([]interface{})
	if ok == false {
		return nil, fmt.Errorf("no pipeline block in the input file")
	}
	stageList, err := parser.convertYamlMapToStages(pipelineData, required)
	if err != nil {
		log.Error("failed to create a stage in pipeline...")
		return nil, err
	}
	for stageItem := stageList.Front(); stageItem != nil; stageItem = stageItem.Next() {
		pipeline.AddStage(stageItem.Value.(stages.Stage))
	}
	var resources = &pipelines.Resources{Pipeline: pipeline, Cleanup: cleanup, Reporter: messenger, RepoService: repoService}

	return resources, nil
}